diff --git a/.gitignore b/.gitignore index 7772403297c4..114956a07d99 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ target esvm .htpasswd installedPlugins +webpackstats.json +config/kibana.dev.yml diff --git a/Gruntfile.js b/Gruntfile.js index 521cc60586df..9e837033d02c 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -1,4 +1,4 @@ -require('babel/register'); +require('babel/register')(require('./src/optimize/babelOptions')); module.exports = function (grunt) { // set the config once before calling load-grunt-config diff --git a/installedPlugins/.empty b/installedPlugins/.empty new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/karma.conf.js b/karma.conf.js index 8047db42cd63..1e9f14240caf 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -16,8 +16,10 @@ module.exports = function (config) { // list of files / patterns to load in the browser files: [ + 'http://localhost:5601/bundles/commons.bundle.js', 'http://localhost:5601/bundles/tests.bundle.js', - 'http://localhost:5601/bundles/tests.bundle.style.css' + 'http://localhost:5601/bundles/commons.style.css', + 'http://localhost:5601/bundles/tests.style.css' ], proxies: { @@ -57,6 +59,14 @@ module.exports = function (config) { // Continuous Integration mode // if true, Karma captures browsers, runs the tests and exits - singleRun: false + singleRun: false, + + client: { + mocha: { + reporter: 'html', // change Karma's debug.html to the mocha web reporter + timeout: 10000, + slow: 5000 + } + } }); }; diff --git a/package.json b/package.json index 41e44eca7b56..d0559f6aedc0 100644 --- a/package.json +++ b/package.json @@ -58,10 +58,9 @@ "auto-preload-rjscommon-deps-loader": "^1.0.4", "autoprefixer": "^5.2.0", "autoprefixer-loader": "^2.0.0", - "babel": "^5.8.19", - "babel-core": "^5.8.19", + "babel": "^5.8.21", + "babel-core": "^5.8.21", "babel-loader": "^5.3.2", - "babel-runtime": "^5.8.19", "bluebird": "^2.9.27", "boom": "^2.8.0", "bootstrap": "^3.3.5", diff --git a/src/cli/Command.js b/src/cli/Command.js index f8ffbbb8cd07..d079263a3b77 100644 --- a/src/cli/Command.js +++ b/src/cli/Command.js @@ -80,4 +80,16 @@ Command.prototype.parseOptions = _.wrap(Command.prototype.parseOptions, function return opts; }); +Command.prototype.action = _.wrap(Command.prototype.action, function (action, fn) { + return action.call(this, function (...args) { + var ret = fn.apply(this, args); + if (ret && typeof ret.then === 'function') { + ret.then(null, function (e) { + console.log('FATAL CLI ERROR', e.stack); + process.exit(1); + }); + } + }); +}); + module.exports = Command; diff --git a/src/cli/cluster/ClusterManager.js b/src/cli/cluster/ClusterManager.js new file mode 100644 index 000000000000..95e29eae5e7d --- /dev/null +++ b/src/cli/cluster/ClusterManager.js @@ -0,0 +1,94 @@ +let cluster = require('cluster'); +let { join } = require('path'); +let { compact, invoke, bindAll, once, get } = require('lodash'); + +let Log = require('../Log'); +let Worker = require('./Worker'); + +module.exports = class ClusterManager { + constructor(opts) { + this.log = new Log(opts.quiet, opts.silent); + this.addedCount = 0; + + this.workers = [ + new Worker({ + type: 'optmzr', + title: 'optimizer', + log: this.log, + argv: compact([ + '--plugins.initialize=false', + '--server.autoListen=false' + ]), + watch: false + }), + + new Worker({ + type: 'server', + log: this.log + }) + ]; + + // broker messages between workers + this.workers.forEach((worker) => { + worker.on('broadcast', (msg) => { + this.workers.forEach((to) => { + if (to !== worker && to.online) { + to.fork.send(msg); + } + }); + }); + }); + + bindAll(this, 'onWatcherAdd', 'onWatcherError', 'onWatcherChange'); + + if (opts.watch) this.setupWatching(); + else this.startCluster(); + } + + startCluster() { + invoke(this.workers, 'start'); + } + + setupWatching() { + var chokidar = require('chokidar'); + let utils = require('requirefrom')('src/utils'); + let fromRoot = utils('fromRoot'); + + this.watcher = chokidar.watch([ + 'src/plugins', + 'src/server', + 'src/ui', + 'src/utils', + 'config', + 'installedPlugins' + ], { + cwd: fromRoot('.'), + ignored: /[\\\/](node_modules|bower_components|public)[\\\/]/, + }); + + this.watcher.on('add', this.onWatcherAdd); + this.watcher.on('error', this.onWatcherError); + + this.watcher.on('ready', once(() => { + // start sending changes to workers + this.watcher.removeListener('add', this.onWatcherAdd); + this.watcher.on('all', this.onWatcherChange); + + this.log.good('Watching for changes', `(${this.addedCount} files)`); + this.startCluster(); + })); + } + + onWatcherAdd() { + this.addedCount += 1; + } + + onWatcherChange(e, path) { + invoke(this.workers, 'onChange', path); + } + + onWatcherError(err) { + this.log.bad('Failed to watch files!\n', err.stack); + process.exit(1); // eslint-disable-line no-process-exit + } +}; diff --git a/src/cli/watch/Worker.js b/src/cli/cluster/Worker.js similarity index 82% rename from src/cli/watch/Worker.js rename to src/cli/cluster/Worker.js index db1f79ddc7f6..e0c2c2294011 100644 --- a/src/cli/watch/Worker.js +++ b/src/cli/cluster/Worker.js @@ -22,7 +22,8 @@ module.exports = class Worker extends EventEmitter { this.log = opts.log; this.type = opts.type; this.title = opts.title || opts.type; - this.filters = opts.filters; + this.watch = (opts.watch !== false); + this.online = false; this.changes = []; let argv = _.union(baseArgv, opts.argv || []); @@ -31,7 +32,7 @@ module.exports = class Worker extends EventEmitter { kbnWorkerArgv: JSON.stringify(argv) }; - _.bindAll(this, ['onExit', 'onMessage', 'shutdown', 'start']); + _.bindAll(this, ['onExit', 'onMessage', 'onOnline', 'onDisconnect', 'shutdown', 'start']); this.start = _.debounce(this.start, 25); cluster.on('exit', this.onExit); @@ -53,15 +54,7 @@ module.exports = class Worker extends EventEmitter { } onChange(path) { - var valid = true; - - if (this.filters) { - valid = _.any(this.filters, function (filter) { - return filter.test(path); - }); - } - - if (!valid) return; + if (!this.watch) return; this.changes.push(path); this.start(); } @@ -70,6 +63,8 @@ module.exports = class Worker extends EventEmitter { if (this.fork && !this.fork.isDead()) { this.fork.kill(); this.fork.removeListener('message', this.onMessage); + this.fork.removeListener('online', this.onOnline); + this.fork.removeListener('disconnect', this.onDisconnect); } } @@ -78,6 +73,14 @@ module.exports = class Worker extends EventEmitter { this.emit('broadcast', msg[1]); } + onOnline() { + this.online = true; + } + + onDisconnect() { + this.online = false; + } + flushChangeBuffer() { let files = _.unique(this.changes.splice(0)); let prefix = files.length > 1 ? '\n - ' : ''; @@ -100,5 +103,7 @@ module.exports = class Worker extends EventEmitter { this.fork = cluster.fork(this.env); this.fork.on('message', this.onMessage); + this.fork.on('online', this.onOnline); + this.fork.on('disconnect', this.onDisconnect); } }; diff --git a/src/cli/plugin/pluginInstaller.js b/src/cli/plugin/pluginInstaller.js index 7a674de03671..aedfc50dfb75 100644 --- a/src/cli/plugin/pluginInstaller.js +++ b/src/cli/plugin/pluginInstaller.js @@ -13,7 +13,7 @@ function install(settings, logger) { try { fs.statSync(settings.pluginPath); - logger.error(`Plugin ${settings.package} already exists. Please remove before installing a new version.`); + logger.error(`Plugin ${settings.package} already exists, please remove before installing a new version`); process.exit(70); // eslint-disable-line no-process-exit } catch (e) { if (e.code !== 'ENOENT') throw e; @@ -31,7 +31,7 @@ function install(settings, logger) { }) .then(function (curious) { fs.renameSync(settings.workingPath, settings.pluginPath); - logger.log('Plugin installation complete!'); + logger.log('Plugin installation complete'); }) .catch(function (e) { logger.error(`Plugin installation was unsuccessful due to error "${e.message}"`); diff --git a/src/cli/plugin/pluginRemover.js b/src/cli/plugin/pluginRemover.js index 57d62b7a291f..2beb128819f8 100644 --- a/src/cli/plugin/pluginRemover.js +++ b/src/cli/plugin/pluginRemover.js @@ -10,7 +10,7 @@ function remove(settings, logger) { try { fs.statSync(settings.pluginPath); } catch (e) { - logger.log(`Plugin ${settings.package} does not exist.`); + logger.log(`Plugin ${settings.package} does not exist`); return; } diff --git a/src/cli/plugin/progressReporter.js b/src/cli/plugin/progressReporter.js index f35361018e31..16f02f19d4a5 100644 --- a/src/cli/plugin/progressReporter.js +++ b/src/cli/plugin/progressReporter.js @@ -56,7 +56,7 @@ module.exports = function (logger, request) { function handleEnd() { if (hasError) return; - logger.log('Download Complete.'); + logger.log('Download Complete'); _resolve(); } diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js index 51a1fe1cf696..0af3b370a8a6 100644 --- a/src/cli/serve/serve.js +++ b/src/cli/serve/serve.js @@ -3,10 +3,8 @@ let { isWorker } = require('cluster'); let { resolve } = require('path'); let cwd = process.cwd(); -let readYamlConfig = require('./readYamlConfig'); let src = require('requirefrom')('src'); let fromRoot = src('utils/fromRoot'); -let KbnServer = src('server/KbnServer'); let pathCollector = function () { let paths = []; @@ -51,22 +49,29 @@ module.exports = function (program) { ) .option('--plugins ', 'an alias for --plugin-dir', pluginDirCollector) .option('--dev', 'Run the server with development mode defaults') - .option('--no-watch', 'Prevent watching, use with --dev to prevent server restarts') - .action(function (opts) { - if (opts.dev && opts.watch && !isWorker) { - // stop processing the action and handoff to watch cluster manager - return require('../watch/watch')(opts); + .option('--no-watch', 'Prevents automatic restarts of the server in --dev mode') + .action(async function (opts) { + if (opts.dev && !isWorker) { + // stop processing the action and handoff to cluster manager + let ClusterManager = require('../cluster/ClusterManager'); + new ClusterManager(opts); + return; } + let readYamlConfig = require('./readYamlConfig'); + let KbnServer = src('server/KbnServer'); + let settings = readYamlConfig(opts.config || fromRoot('config/kibana.yml')); + + if (opts.dev) { + try { _.merge(settings, readYamlConfig(fromRoot('config/kibana.dev.yml'))); } + catch (e) { null; } + } + let set = _.partial(_.set, settings); let get = _.partial(_.get, settings); - if (opts.dev) { - set('env', 'development'); - set('optimize.watch', opts.watch); - } - + if (opts.dev) set('env', 'development'); if (opts.elasticsearch) set('elasticsearch.url', opts.elasticsearch); if (opts.port) set('server.port', opts.port); if (opts.host) set('server.host', opts.host); @@ -82,13 +87,22 @@ module.exports = function (program) { set('plugins.paths', [].concat(opts.pluginPath || [])); - let server = new KbnServer(_.merge(settings, this.getUnknownOptions())); + let kbnServer = {}; - server.ready().catch(function (err) { - console.error(err.stack); + try { + kbnServer = new KbnServer(_.merge(settings, this.getUnknownOptions())); + await kbnServer.ready(); + } + catch (err) { + let { server } = kbnServer; + + if (server) server.log(['fatal'], err); + else console.error('FATAL', err); + + kbnServer.close(); process.exit(1); // eslint-disable-line no-process-exit - }); + } - return server; + return kbnServer; }); }; diff --git a/src/cli/watch/watch.js b/src/cli/watch/watch.js deleted file mode 100644 index 4f5f77a9fc17..000000000000 --- a/src/cli/watch/watch.js +++ /dev/null @@ -1,90 +0,0 @@ -let cluster = require('cluster'); -let { join } = require('path'); -let _ = require('lodash'); -var chokidar = require('chokidar'); - -let utils = require('requirefrom')('src/utils'); -let fromRoot = utils('fromRoot'); -let Log = require('../Log'); -let Worker = require('./Worker'); - -module.exports = function (opts) { - - let watcher = chokidar.watch([ - 'src/cli', - 'src/fixtures', - 'src/server', - 'src/utils', - 'src/plugins', - 'config', - ], { - cwd: fromRoot('.'), - ignore: 'src/plugins/*/public/**/*' - }); - - let log = new Log(opts.quiet, opts.silent); - let customLogging = opts.quiet || opts.silent || opts.verbose; - - let workers = [ - new Worker({ - type: 'optmzr', - title: 'optimizer', - log: log, - argv: _.compact([ - customLogging ? null : '--quiet', - '--plugins.initialize=false', - '--server.autoListen=false', - '--optimize._workerRole=send' - ]), - filters: [ - /src\/server\/(optimize|ui|plugins)\//, - /src\/plugins\/[^\/]+\/[^\/]+\.js$/, - /src\/cli\// - ] - }), - - new Worker({ - type: 'server', - log: log, - argv: [ - '--optimize._workerRole=receive' - ] - }) - ]; - - workers.forEach(function (worker) { - worker.on('broadcast', function (msg) { - workers.forEach(function (to) { - if (to !== worker && to.fork) to.fork.send(msg); - }); - }); - }); - - - var addedCount = 0; - function onAddBeforeReady() { - addedCount += 1; - } - - function onReady() { - // start sending changes to workers - watcher.removeListener('add', onAddBeforeReady); - watcher.on('all', onAnyChangeAfterReady); - - log.good('Watching for changes', `(${addedCount} files)`); - _.invoke(workers, 'start'); - } - - function onAnyChangeAfterReady(e, path) { - _.invoke(workers, 'onChange', path); - } - - function onError(err) { - log.bad('Failed to watch files!\n', err.stack); - process.exit(1); // eslint-disable-line no-process-exit - } - - watcher.on('add', onAddBeforeReady); - watcher.on('ready', onReady); - watcher.on('error', onError); -}; diff --git a/src/optimize/BaseOptimizer.js b/src/optimize/BaseOptimizer.js index f93ad6fc901e..170a892f59b3 100644 --- a/src/optimize/BaseOptimizer.js +++ b/src/optimize/BaseOptimizer.js @@ -1,32 +1,62 @@ -let { EventEmitter } = require('events'); let { inherits } = require('util'); -let _ = require('lodash'); -let { join } = require('path'); -let write = require('fs').writeFileSync; +let { defaults } = require('lodash'); +let { resolve } = require('path'); +let { writeFile } = require('fs'); let webpack = require('webpack'); +var Boom = require('boom'); let DirectoryNameAsMain = require('webpack-directory-name-as-main'); let ExtractTextPlugin = require('extract-text-webpack-plugin'); +var CommonsChunkPlugin = require('webpack/lib/optimize/CommonsChunkPlugin'); let utils = require('requirefrom')('src/utils'); let fromRoot = utils('fromRoot'); -let OptmzBundles = require('./OptmzBundles'); -let OptmzUiModules = require('./OptmzUiModules'); let babelOptions = require('./babelOptions'); -let kbnTag = `Kibana ${ utils('packageJson').version }`; - -class BaseOptimizer extends EventEmitter { +class BaseOptimizer { constructor(opts) { - super(); + this.env = opts.env; + this.bundles = opts.bundles; + this.profile = opts.profile || false; - this.sourceMaps = opts.sourceMaps || false; - this.modules = new OptmzUiModules(opts.plugins); - this.bundles = new OptmzBundles( - opts, - `${kbnTag} ${this.constructor.name} ${ this.sourceMaps ? ' (with source maps)' : ''}` - ); + switch (opts.sourceMaps) { + case true: + this.sourceMaps = 'source-map'; + break; - _.bindAll(this, 'getConfig'); + case 'fast': + this.sourceMaps = 'cheap-module-eval-source-map'; + break; + + default: + this.sourceMaps = opts.sourceMaps || false; + break; + } + + this.unsafeCache = opts.unsafeCache || false; + if (typeof this.unsafeCache === 'string') { + this.unsafeCache = [ + new RegExp(this.unsafeCache.slice(1, -1)) + ]; + } + } + + async initCompiler() { + if (this.compiler) return this.compiler; + + let compilerConfig = this.getConfig(); + this.compiler = webpack(compilerConfig); + + this.compiler.plugin('done', stats => { + if (!this.profile) return; + + let path = resolve(this.env.workingDir, 'stats.json'); + let content = JSON.stringify(stats.toJson()); + writeFile(path, content, function (err) { + if (err) throw err; + }); + }); + + return this.compiler; } getConfig() { @@ -34,18 +64,21 @@ class BaseOptimizer extends EventEmitter { return { context: fromRoot('.'), - entry: this.bundles.getEntriesConfig(), + entry: this.bundles.toWebpackEntries(), - devtool: this.sourceMaps ? '#source-map' : false, + devtool: this.sourceMaps, + profile: this.profile || false, output: { - path: this.bundles.dir, + path: this.env.workingDir, filename: '[name].bundle.js', sourceMapFilename: '[file].map', publicPath: '/bundles/', devtoolModuleFilenameTemplate: '[absolute-resource-path]' }, + recordsPath: resolve(this.env.workingDir, 'webpack.records'), + plugins: [ new webpack.ResolverPlugin([ new DirectoryNameAsMain() @@ -54,7 +87,11 @@ class BaseOptimizer extends EventEmitter { new webpack.optimize.DedupePlugin(), new ExtractTextPlugin('[name].style.css', { allChunks: true - }) + }), + new CommonsChunkPlugin({ + name: 'commons', + filename: 'commons.bundle.js' + }), ], module: { @@ -74,33 +111,62 @@ class BaseOptimizer extends EventEmitter { { test: /[\/\\]src[\/\\](plugins|ui)[\/\\].+\.js$/, loader: `auto-preload-rjscommon-deps${mapQ}` }, { test: /\.js$/, - exclude: /(node_modules|bower_components)/, + exclude: /[\/\\](node_modules|bower_components)[\/\\]/, loader: 'babel', query: babelOptions }, { - // explicitly require .jsx extension to support jsx test: /\.jsx$/, - exclude: /(node_modules|bower_components)/, + exclude: /[\/\\](node_modules|bower_components)[\/\\]/, loader: 'babel', - query: _.defaults({ + query: defaults({ nonStandard: true }, babelOptions) } - ].concat(this.modules.loaders), - noParse: this.modules.noParse, + ].concat(this.env.loaders), + noParse: this.env.noParse, }, resolve: { - extensions: ['.js', '.less', ''], + extensions: ['.babel.js', '.js', '.less', ''], postfixes: [''], modulesDirectories: ['node_modules'], loaderPostfixes: ['-loader', ''], root: fromRoot('.'), - alias: this.modules.aliases - } + alias: this.env.aliases, + unsafeCache: this.unsafeCache, + }, }; } + + failedStatsToError(stats) { + let statFormatOpts = { + hash: false, // add the hash of the compilation + version: false, // add webpack version information + timings: false, // add timing information + assets: false, // add assets information + chunks: false, // add chunk information + chunkModules: false, // add built modules information to chunk information + modules: false, // add built modules information + cached: false, // add also information about cached (not built) modules + reasons: false, // add information about the reasons why modules are included + source: false, // add the source code of modules + errorDetails: false, // add details to errors (like resolving log) + chunkOrigins: false, // add the origins of chunks and chunk merging info + modulesSort: false, // (string) sort the modules by that field + chunksSort: false, // (string) sort the chunks by that field + assetsSort: false, // (string) sort the assets by that field + children: false, + }; + + let details = stats.toString(defaults({ colors: true }, statFormatOpts)); + + return Boom.create( + 500, + `Optimizations failure.\n${details.split('\n').join('\n ')}\n`, + stats.toJson(statFormatOpts) + ); + } } module.exports = BaseOptimizer; diff --git a/src/optimize/CachedOptimizer.js b/src/optimize/CachedOptimizer.js deleted file mode 100644 index 4ba2d52a68f1..000000000000 --- a/src/optimize/CachedOptimizer.js +++ /dev/null @@ -1,47 +0,0 @@ -let _ = require('lodash'); -let webpack = require('webpack'); - -let BaseOptimizer = require('./BaseOptimizer'); - -module.exports = class CachedOptimizer extends BaseOptimizer { - constructor(opts) { - super(opts); - _.bindAll(this, 'init', 'setupCompiler', 'run'); - } - - init(autoRun) { - return this.bundles.ensureAllEntriesExist().then(autoRun ? this.run : this.setupCompiler); - } - - setupCompiler(autoRun) { - this.entries = this.bundles.getMissingEntries(); - if (!this.entries.length) return; - - this.compilerConfig = this.getConfig(); - this.compiler = webpack(this.compilerConfig); - - if (autoRun) this.run(); - } - - run() { - if (!this.compiler) { - return this.setupCompiler(true); - } - - var self = this; - let entries = self.entries; - - self.emit('build-start', entries); - self.compiler.run(function (err, stats) { - if (err) { - self.emit('error', entries, stats, err); - } - else if (stats.hasErrors() || stats.hasWarnings()) { - self.emit('error', entries, stats, new Error('Optimization must not produce errors or warnings')); - } - else { - self.emit('done', entries, stats); - } - }); - } -}; diff --git a/src/optimize/FsOptimizer.js b/src/optimize/FsOptimizer.js new file mode 100644 index 000000000000..0d35439a5a64 --- /dev/null +++ b/src/optimize/FsOptimizer.js @@ -0,0 +1,28 @@ +let { fromNode } = require('bluebird'); +let { writeFile } = require('fs'); + +let BaseOptimizer = require('./BaseOptimizer'); +let fromRoot = require('../utils/fromRoot'); + +module.exports = class FsOptimizer extends BaseOptimizer { + async init() { + await this.initCompiler(); + } + + async run() { + if (!this.compiler) await this.init(); + + let stats = await fromNode(cb => { + this.compiler.run((err, stats) => { + if (err || !stats) return cb(err); + + if (stats.hasErrors() || stats.hasWarnings()) { + return cb(this.failedStatsToError(stats)); + } + else { + cb(null, stats); + } + }); + }); + } +}; diff --git a/src/optimize/LiveOptimizer.js b/src/optimize/LiveOptimizer.js deleted file mode 100644 index b17785a02da3..000000000000 --- a/src/optimize/LiveOptimizer.js +++ /dev/null @@ -1,53 +0,0 @@ -let _ = require('lodash'); -let { join } = require('path'); -let { promisify } = require('bluebird'); -let webpack = require('webpack'); -let MemoryFileSystem = require('memory-fs'); -let BaseOptimizer = require('./BaseOptimizer'); - -module.exports = class LiveOptimizer extends BaseOptimizer { - constructor(opts) { - super(opts); - - this.compilerConfig = this.getConfig(); - // this.compilerConfig.profile = true; - - this.compiler = webpack(this.compilerConfig); - this.outFs = this.compiler.outputFileSystem = new MemoryFileSystem(); - - _.bindAll(this, 'get', 'init'); - - this.compile = promisify(this.compiler.run, this.compiler); - } - - init() { - return this.bundles.ensureAllEntriesExist(true); - } - - get(id) { - let self = this; - let fs = self.outFs; - let filename = join(self.compiler.outputPath, `${id}.bundle.js`); - let mapFilename = join(self.compiler.outputPath, `${id}.bundle.js.map`); - let styleFilename = join(self.compiler.outputPath, `${id}.style.css`); - - if (!self.active) { - self.active = self.compile().finally(function () { - self.active = null; - }); - } - - return self.active.then(function (stats) { - if (stats.hasErrors() || stats.hasWarnings()) { - console.log(stats.toString({ colors: true })); - return null; - } - - return { - bundle: fs.readFileSync(filename), - sourceMap: self.sourceMaps ? fs.readFileSync(mapFilename) : false, - style: fs.readFileSync(styleFilename) - }; - }); - } -}; diff --git a/src/optimize/OptmzBundles.js b/src/optimize/OptmzBundles.js deleted file mode 100644 index c53b2fdb9974..000000000000 --- a/src/optimize/OptmzBundles.js +++ /dev/null @@ -1,111 +0,0 @@ -let _ = require('lodash'); -let { join } = require('path'); -let { resolve, promisify } = require('bluebird'); -let rimraf = promisify(require('rimraf')); -let mkdirp = promisify(require('mkdirp')); -let stat = promisify(require('fs').stat); -let read = promisify(require('fs').readFile); -let write = promisify(require('fs').writeFile); -let unlink = promisify(require('fs').unlink); -let readdir = promisify(require('fs').readdir); -let readSync = require('fs').readFileSync; - -let entryFileTemplate = _.template(readSync(join(__dirname, 'entry.js.tmpl'))); - -class OptmzBundles { - constructor(opts, optimizerTagline) { - this.dir = _.get(opts, 'bundleDir'); - if (!_.isString(this.dir)) { - throw new TypeError('Optimizer requires a working directory'); - } - - this.sourceMaps = _.get(opts, 'sourceMaps'); - this.entries = _.get(opts, 'entries', []).map(function (spec) { - let entry = { - id: spec.id, - deps: _.get(spec, 'deps', []), - modules: _.get(spec, 'modules', []), - - path: join(this.dir, spec.id + '.entry.js'), - bundlePath: join(this.dir, spec.id + '.bundle.js'), - - exists: undefined, - content: undefined - }; - - entry.content = _.get(spec, 'template', entryFileTemplate)({ - entry: entry, - optimizerTagline: optimizerTagline - }); - - return entry; - }, this); - - - _.bindAll(this, [ - 'ensureDir', - 'ensureAllEntriesExist', - 'checkIfEntryExists', - 'writeEntryFile', - 'clean', - 'getMissingEntries', - 'getEntriesConfig' - ]); - } - - ensureDir() { - return mkdirp(this.dir); - } - - ensureAllEntriesExist(overwrite) { - return this.ensureDir() - .return(this.entries) - .map(overwrite ? this.checkIfEntryExists : _.noop) - .return(this.entries) - .map(this.writeEntryFile) - .return(undefined); - } - - checkIfEntryExists(entry) { - return resolve([ - read(entry.path), - stat(entry.bundlePath) - ]) - .settle() - .spread(function (readEntry, statBundle) { - let existingEntry = readEntry.isFulfilled() && readEntry.value().toString('utf8'); - let bundleExists = statBundle.isFulfilled() && !statBundle.value().isDirectory(); - entry.exists = existingEntry && bundleExists && (existingEntry === entry.content); - }); - } - - writeEntryFile(entry) { - return this.clean([entry.path, entry.bundlePath]).then(function () { - entry.exists = false; - return write(entry.path, entry.content, { encoding: 'utf8' }); - }); - } - - // unlinks files, swallows missing file errors - clean(paths) { - return resolve( - _.flatten([paths]).map(function (path) { - return rimraf(path); - }) - ) - .settle() - .return(undefined); - } - - getMissingEntries() { - return _.reject(this.entries, 'exists'); - } - - getEntriesConfig() { - return _.transform(this.getMissingEntries(), function (map, entry) { - map[entry.id] = entry.path; - }, {}); - } -} - -module.exports = OptmzBundles; diff --git a/src/optimize/OptmzUiModules.js b/src/optimize/OptmzUiModules.js deleted file mode 100644 index 619f747ca9ce..000000000000 --- a/src/optimize/OptmzUiModules.js +++ /dev/null @@ -1,98 +0,0 @@ -var _ = require('lodash'); -var fromRoot = require('../utils/fromRoot'); - -var asRegExp = _.flow( - _.escapeRegExp, - function (path) { - return path + '(?:\\.js)?$'; - }, - RegExp -); - -function OptmzUiExports(plugins) { - // regular expressions which will prevent webpack from parsing the file - var noParse = this.noParse = []; - - // webpack aliases, like require paths, mapping a prefix to a directory - var aliases = this.aliases = { - ui: fromRoot('src/ui/public'), - testHarness: fromRoot('src/testHarness/public') - }; - - // webpack loaders map loader configuration to regexps - var loaders = this.loaders = []; - - var claimedModuleIds = {}; - _.each(plugins, function (plugin) { - var exports = plugin.uiExportsSpecs; - - // add an alias for this plugins public directory - if (plugin.publicDir) { - aliases[`plugins/${plugin.id}`] = plugin.publicDir; - } - - // consume the plugin's "modules" exports - _.forOwn(exports.modules, function (spec, id) { - if (claimedModuleIds[id]) { - throw new TypeError(`Plugin ${plugin.id} attempted to override export "${id}" from ${claimedModuleIds[id]}`); - } else { - claimedModuleIds[id] = plugin.id; - } - - // configurable via spec - var path; - var parse = true; - var imports = null; - var exports = null; - var expose = null; - - // basic style, just a path - if (_.isString(spec)) path = spec; - - if (_.isArray(spec)) { - path = spec[0]; - imports = spec[1]; - exports = spec[2]; - } - - if (_.isPlainObject(spec)) { - path = spec.path; - parse = _.get(spec, 'parse', parse); - imports = _.get(spec, 'imports', imports); - exports = _.get(spec, 'exports', exports); - expose = _.get(spec, 'expose', expose); - } - - if (!path) { - throw new TypeError('Invalid spec definition, unable to identify path'); - } - - aliases[id] = path; - - var loader = []; - if (imports) { - loader.push(`imports?${imports}`); - } - - if (exports) loader.push(`exports?${exports}`); - if (expose) loader.push(`expose?${expose}`); - if (loader.length) loaders.push({ test: asRegExp(path), loader: loader.join('!') }); - - if (!parse) noParse.push(asRegExp(path)); - }); - - // consume the plugin's "loaders" exports - _.each(exports.loaders, function (loader) { - loaders.push(loader); - }); - - - // consume the plugin's "noParse" exports - _.each(exports.noParse, function (regExp) { - noParse.push(regExp); - }); - - }); -} - -module.exports = OptmzUiExports; diff --git a/src/optimize/WatchingOptimizer.js b/src/optimize/WatchingOptimizer.js deleted file mode 100644 index 8d31d3197674..000000000000 --- a/src/optimize/WatchingOptimizer.js +++ /dev/null @@ -1,118 +0,0 @@ -let _ = require('lodash'); -let webpack = require('webpack'); - -let BaseOptimizer = require('./BaseOptimizer'); - -const STATUS_BUNDLE_INVALID = 'bundle invalid'; -const STATUS_BUNDLING = 'optimizing'; -const STATUS_REBUNDLING = 'bundle invalid during optimizing'; -const STATUS_ERROR = 'error'; -const STATUS_DONE = 'done'; - -class WatchingOptimizer extends BaseOptimizer { - constructor(opts) { - super(opts); - - this.bundleStatus = null; - _.bindAll(this, 'init', 'setupCompiler', 'onBundlesInvalid', 'setStatus', 'enable', 'disable'); - - this.run = this.enable; // enable makes a bit more sense here, but alias for consistency with CachedOptimizer - } - - init(autoEnable) { - return this.bundles.ensureAllEntriesExist(true).then(autoEnable ? this.enable : this.setupCompiler); - } - - setupCompiler(autoEnable) { - if (!_.size(this.bundles.entries)) return; - - this.compilerConfig = this.getConfig(); - this.compiler = webpack(this.compilerConfig); - this.compiler.plugin('watch-run', _.partial(this.setStatus, STATUS_BUNDLING)); - this.compiler.plugin('invalid', this.onBundlesInvalid); - this.compiler.plugin('failed', _.partial(this.setStatus, STATUS_ERROR)); - this.compiler.plugin('done', _.partial(this.setStatus, STATUS_DONE)); - - if (autoEnable) this.enable(); - } - - onBundlesInvalid() { - switch (this.bundleStatus || null) { - case STATUS_BUNDLING: - case STATUS_REBUNDLING: - // if the source changed during building, we immediately rebuild - return this.setStatus(STATUS_REBUNDLING); - case null: - // the bundle has to be something before that something can be invalid - return; - default: - return this.setStatus(STATUS_BUNDLE_INVALID); - } - } - - setStatus(status) { - let self = this; - let entries = self.bundles.entries; - let stats; - let error; - let shouldBeFinal = false; - - switch (status) { - case 'done': - stats = self.watcher.stats; - error = null; - shouldBeFinal = true; - - if (stats.hasErrors()) { - error = new Error('Optimization must not produce errors or warnings'); - status = 'error'; - } - break; - - case 'error': - stats = self.watcher.stats; - error = self.watcher.error; - } - - let apply = function () { - clearTimeout(self.tentativeStatusChange); - self.tentativeStatusChange = null; - self.emit(self.bundleStatus = status, entries, stats, error); - }; - - if (shouldBeFinal) { - // this looks race-y, but it's how webpack does it: https://goo.gl/ShVo2o - self.tentativeStatusChange = setTimeout(apply, 0); - } else { - apply(); - } - - // webpack allows some plugins to be async, we don't want to hold up webpack, - // so just always callback if we get a cb(); - let cb = _.last(arguments); - if (typeof cb === 'function') cb(); - } - - enable() { - if (!this.compiler) { - return this.setupCompiler(true); - } - - if (this.watcher) { - throw new Error('WatchingOptimizer already watching!'); - } - - this.watcher = this.compiler.watch({}, _.noop); - } - - disable() { - if (!this.compiler) return; - if (!this.watcher) return; - - this.watcher.close(); - this.watcher = null; - this.compiler = null; - } -} - -module.exports = WatchingOptimizer; diff --git a/src/optimize/babelOptions.js b/src/optimize/babelOptions.js index 48698b7cbb8f..69b7c81ef901 100644 --- a/src/optimize/babelOptions.js +++ b/src/optimize/babelOptions.js @@ -1,5 +1,4 @@ module.exports = { - optional: ['runtime'], stage: 1, nonStandard: false }; diff --git a/src/optimize/browserTests/TestBundler.js b/src/optimize/browserTests/TestBundler.js deleted file mode 100644 index 77a27937a5ec..000000000000 --- a/src/optimize/browserTests/TestBundler.js +++ /dev/null @@ -1,78 +0,0 @@ -let _ = require('lodash'); -let { resolve } = require('path'); -let { readFileSync } = require('fs'); - -let src = require('requirefrom')('src'); -let fromRoot = src('utils/fromRoot'); -let pathContains = src('utils/pathContains'); -let LiveOptimizer = src('optimize/LiveOptimizer'); - -let id = 'tests'; -let globAll = require('./globAll'); -let testEntryFileTemplate = _.template(readFileSync(resolve(__dirname, './testBundleEntry.js.tmpl'))); - -class TestBundler { - constructor(kbnServer) { - this.kbnServer = kbnServer; - this.init = _.once(this.init); - _.bindAll(this, ['init', 'findTestFiles', 'setupOptimizer', 'render']); - } - - init() { - return this.findTestFiles().then(this.setupOptimizer); - } - - findTestFiles() { - return globAll(fromRoot('src'), [ - '**/public/**/__tests__/**/*.js' - ]); - } - - setupOptimizer(testFiles) { - let plugins = this.kbnServer.plugins; - let bundleDir = this.kbnServer.config.get('optimize.bundleDir'); - - let deps = []; - let modules = []; - - if (testFiles) { - modules = modules.concat(testFiles); - } - - plugins.forEach(function (plugin) { - if (!plugin.app) return; - modules = modules.concat(plugin.app.getModules()); - deps = deps.concat(plugin.app.getRelatedPlugins()); - }); - - this.optimizer = new LiveOptimizer({ - sourceMaps: true, - bundleDir: bundleDir, - entries: [ - { - id: id, - deps: deps, - modules: modules, - template: testEntryFileTemplate - } - ], - plugins: plugins - }); - - return this.optimizer.init(); - } - - render() { - let self = this; - let first = !this.optimizer; - let server = this.kbnServer.server; - - return self.init() - .then(function () { - server.log(['optimize', 'testHarness', first ? 'info' : 'debug'], 'Test harness built, compiling test bundle'); - return self.optimizer.get(id); - }); - } -} - -module.exports = TestBundler; diff --git a/src/optimize/browserTests/globAll.js b/src/optimize/browserTests/globAll.js deleted file mode 100644 index 1e45503df118..000000000000 --- a/src/optimize/browserTests/globAll.js +++ /dev/null @@ -1,17 +0,0 @@ -let _ = require('lodash'); -let { resolve } = require('path'); -let { promisify } = require('bluebird'); -let { all } = require('bluebird'); -let glob = promisify(require('glob')); - -module.exports = function (path, patterns) { - return all([].concat(patterns || [])) - .map(function (pattern) { - return glob(pattern, { cwd: path, ignore: '**/_*.js' }); - }) - .then(_.flatten) - .then(_.uniq) - .map(function (match) { - return resolve(path, match); - }); -}; diff --git a/src/optimize/browserTests/index.js b/src/optimize/browserTests/index.js deleted file mode 100644 index b7207d431350..000000000000 --- a/src/optimize/browserTests/index.js +++ /dev/null @@ -1,52 +0,0 @@ -module.exports = function (kbnServer, server, config) { - if (!config.get('env.dev')) return; - - let Boom = require('boom'); - let src = require('requirefrom')('src'); - let fromRoot = src('utils/fromRoot'); - let TestBundler = require('./TestBundler'); - - let bundler = new TestBundler(kbnServer, fromRoot('src')); - let renderPromise = false; - let renderComplete = false; - - function send(reply, part, mimeType) { - if (!renderPromise || (part === 'bundle' && renderComplete)) { - renderPromise = bundler.render(); - renderComplete = false; - renderPromise.then(function () { renderComplete = true; }); - } - - renderPromise.then(function (output) { - if (!output || !output.bundle) { - return reply(Boom.create(500, 'failed to build test bundle')); - } - - return reply(output[part]).type(mimeType); - }, reply); - } - - server.route({ - path: '/bundles/tests.bundle.js', - method: 'GET', - handler: function (req, reply) { - send(reply, 'bundle', 'application/javascript'); - } - }); - - server.route({ - path: '/bundles/tests.bundle.js.map', - method: 'GET', - handler: function (req, reply) { - send(reply, 'sourceMap', 'text/plain'); - } - }); - - server.route({ - path: '/bundles/tests.bundle.style.css', - method: 'GET', - handler: function (req, reply) { - send(reply, 'style', 'text/css'); - } - }); -}; diff --git a/src/optimize/entry.js.tmpl b/src/optimize/entry.js.tmpl deleted file mode 100644 index 7febb3985fe3..000000000000 --- a/src/optimize/entry.js.tmpl +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Optimized application entry file - * - * This is programatically created and updated, do not modify - * - * built using: <%= optimizerTagline %> - * includes code from: -<% - entry.deps.sort().forEach(function (plugin) { - print(` * - ${plugin}\n`); - }) -%> * - */ - -require('ui/chrome'); -<% -entry.modules.forEach(function (id) { - if (id !== 'ui/chrome') print(`require('${id}');\n`); -}); -%>require('ui/chrome').bootstrap(/* xoxo */); diff --git a/src/optimize/index.js b/src/optimize/index.js index fadc8c52d1ad..c55eef3fb47f 100644 --- a/src/optimize/index.js +++ b/src/optimize/index.js @@ -1,115 +1,53 @@ -module.exports = function (kbnServer, server, config) { +module.exports = async (kbnServer, server, config) => { if (!config.get('optimize.enabled')) return; - var _ = require('lodash'); - var { resolve } = require('path'); - var fromRoot = require('../utils/fromRoot'); - - var CachedOptimizer = require('./CachedOptimizer'); - var WatchingOptimizer = require('./WatchingOptimizer'); - - var bundleDir = resolve(config.get('optimize.bundleDir')); - var status = kbnServer.status.create('optimize'); - - server.exposeStaticDir('/bundles/{path*}', bundleDir); - - function logStats(tag, stats) { - if (config.get('logging.json')) { - server.log(['optimize', tag], _.pick(stats.toJson(), 'errors', 'warnings')); - } else { - server.log(['optimize', tag], `\n${ stats.toString({ colors: true }) }`); - } + // the lazy optimizer sets up two threads, one is the server listening + // on 5601 and the other is a server listening on 5602 that builds the + // bundles in a "middleware" style. + // + // the server listening on 5601 may be restarted a number of times, depending + // on the watch setup managed by the cli. It proxies all bundles/* requests to + // the other server. The server on 5602 is long running, in order to prevent + // complete rebuilds of the optimize content. + let lazy = config.get('optimize.lazy'); + if (lazy) { + return await kbnServer.mixin(require('./lazy/lazy')); } - function describeEntries(entries) { - let ids = _.pluck(entries, 'id').join('", "'); - return `application${ entries.length === 1 ? '' : 's'} "${ids}"`; + let bundles = kbnServer.bundles; + server.exposeStaticDir('/bundles/{path*}', bundles.env.workingDir); + await bundles.writeEntryFiles(); + + // in prod, only bundle what looks invalid or missing + if (config.get('env.prod')) bundles = await kbnServer.bundles.getInvalidBundles(); + + // we might not have any work to do + if (!bundles.getIds().length) { + server.log( + ['debug', 'optimize'], + `All bundles are cached and ready to go!` + ); + return; } - function onMessage(handle, filter) { - filter = filter || _.constant(true); - process.on('message', function (msg) { - var optimizeMsg = msg && msg.optimizeMsg; - if (!optimizeMsg || !filter(optimizeMsg)) return; - handle(optimizeMsg); - }); - } - - var role = config.get('optimize._workerRole'); - if (role === 'receive') { - // query for initial status - process.send(['WORKER_BROADCAST', { optimizeMsg: '?' }]); - onMessage(function (wrkrStatus) { - status[wrkrStatus.state](wrkrStatus.message); - }); - } - - if (role === 'send') { - let send = function () { - process.send(['WORKER_BROADCAST', { optimizeMsg: status }]); - }; - - status.on('change', send); - onMessage(send, _.partial(_.eq, '?')); - send(); - } - - let watching = config.get('optimize.watch'); - let Optimizer = watching ? WatchingOptimizer : CachedOptimizer; - let optmzr = kbnServer.optimizer = new Optimizer({ + // only require the FsOptimizer when we need to + let FsOptimizer = require('./FsOptimizer'); + let optimizer = new FsOptimizer({ + env: bundles.env, + bundles: bundles, + profile: config.get('optimize.profile'), sourceMaps: config.get('optimize.sourceMaps'), - bundleDir: bundleDir, - entries: _.map(kbnServer.uiExports.allApps(), function (app) { - return { - id: app.id, - deps: app.getRelatedPlugins(), - modules: app.getModules() - }; - }), - plugins: kbnServer.plugins + unsafeCache: config.get('optimize.unsafeCache'), }); - server.on('close', _.bindKey(optmzr.disable || _.noop, optmzr)); + server.log( + ['info', 'optimize'], + `Optimizing and caching ${bundles.desc()}. This may take a few minutes` + ); - kbnServer.mixin(require('./browserTests')) - .then(function () { + let start = Date.now(); + await optimizer.run(); + let seconds = ((Date.now() - start) / 1000).toFixed(2); - if (role === 'receive') return; - - optmzr.on('bundle invalid', function () { - status.yellow('Source file change detected, reoptimizing source files'); - }); - - optmzr.on('done', function (entries, stats) { - logStats('debug', stats); - status.green(`Optimization of ${describeEntries(entries)} complete`); - }); - - optmzr.on('error', function (entries, stats, err) { - if (stats) logStats('fatal', stats); - status.red('Optimization failure! ' + err.message); - }); - - return optmzr.init() - .then(function () { - let entries = optmzr.bundles.getMissingEntries(); - if (!entries.length) { - if (watching) { - status.red('No optimizable applications found'); - } else { - status.green('Reusing previously cached application source files'); - } - return; - } - - if (watching) { - status.yellow(`Optimizing and watching all application source files`); - } else { - status.yellow(`Optimizing and caching ${describeEntries(entries)}`); - } - - optmzr.run(); - return null; - }); - }); + server.log(['info', 'optimize'], `Optimization of ${bundles.desc()} complete in ${seconds} seconds`); }; diff --git a/src/optimize/lazy/LazyOptimizer.js b/src/optimize/lazy/LazyOptimizer.js new file mode 100644 index 000000000000..a221e2d1ecce --- /dev/null +++ b/src/optimize/lazy/LazyOptimizer.js @@ -0,0 +1,117 @@ +let { once, pick, size } = require('lodash'); +let { join } = require('path'); +let Boom = require('boom'); + +let BaseOptimizer = require('../BaseOptimizer'); +let WeirdControlFlow = require('./WeirdControlFlow'); + +module.exports = class LazyOptimizer extends BaseOptimizer { + constructor(opts) { + super(opts); + this.log = opts.log || (() => null); + this.prebuild = opts.prebuild || false; + + this.timer = { + ms: null, + start: () => this.timer.ms = Date.now(), + end: () => this.timer.ms = ((Date.now() - this.timer.ms) / 1000).toFixed(2) + }; + + this.build = new WeirdControlFlow(); + } + + async init() { + this.initializing = true; + + await this.bundles.writeEntryFiles(); + await this.initCompiler(); + + this.compiler.plugin('watch-run', (w, webpackCb) => { + this.build.work(once(() => { + this.timer.start(); + this.logRunStart(); + webpackCb(); + })); + }); + + this.compiler.plugin('done', stats => { + if (!stats.hasErrors() && !stats.hasWarnings()) { + this.logRunSuccess(); + this.build.success(); + return; + } + + let err = this.failedStatsToError(stats); + this.logRunFailure(err); + this.build.failure(err); + this.watching.invalidate(); + }); + + this.watching = this.compiler.watch({ aggregateTimeout: 200 }, err => { + if (err) { + this.log('fatal', err); + process.exit(1); + } + }); + + let buildPromise = this.build.get(); + if (this.prebuild) await buildPromise; + + this.initializing = false; + this.log(['info', 'optimize'], { + tmpl: `Lazy optimization of ${this.bundles.desc()} ready`, + bundles: this.bundles.getIds() + }); + } + + async getPath(relativePath) { + await this.build.get(); + return join(this.compiler.outputPath, relativePath); + } + + bindToServer(server) { + server.route({ + path: '/bundles/{asset*}', + method: 'GET', + handler: async (request, reply) => { + try { + let path = await this.getPath(request.params.asset); + return reply.file(path); + } catch (error) { + console.log(error.stack); + return reply(error); + } + } + }); + } + + logRunStart() { + this.log(['info', 'optimize'], { + tmpl: `Lazy optimization started`, + bundles: this.bundles.getIds() + }); + } + + logRunSuccess() { + this.log(['info', 'optimize'], { + tmpl: 'Lazy optimization <%= status %> in <%= seconds %> seconds', + bundles: this.bundles.getIds(), + status: 'success', + seconds: this.timer.end() + }); + } + + logRunFailure(err) { + // errors during initialization to the server, unlike the rest of the + // errors produced here. Lets not muddy the console with extra errors + if (this.initializing) return; + + this.log(['fatal', 'optimize'], { + tmpl: 'Lazy optimization <%= status %> in <%= seconds %> seconds<%= err %>', + bundles: this.bundles.getIds(), + status: 'failed', + seconds: this.timer.end(), + err: err + }); + } +}; diff --git a/src/optimize/lazy/LazyServer.js b/src/optimize/lazy/LazyServer.js new file mode 100644 index 000000000000..7527cf1505d5 --- /dev/null +++ b/src/optimize/lazy/LazyServer.js @@ -0,0 +1,22 @@ + +let { Server } = require('hapi'); +let { fromNode } = require('bluebird'); +let Boom = require('boom'); + + +module.exports = class LazyServer { + constructor(host, port, optimizer) { + this.optimizer = optimizer; + this.server = new Server(); + this.server.connection({ + host: host, + port: port + }); + } + + async init() { + await this.optimizer.init(); + this.optimizer.bindToServer(this.server); + await fromNode(cb => this.server.start(cb)); + } +}; diff --git a/src/optimize/lazy/WeirdControlFlow.js b/src/optimize/lazy/WeirdControlFlow.js new file mode 100644 index 000000000000..5825c12c9346 --- /dev/null +++ b/src/optimize/lazy/WeirdControlFlow.js @@ -0,0 +1,58 @@ + +let { fromNode } = require('bluebird'); + +module.exports = class WeirdControlFlow { + constructor(work) { + this.handlers = []; + } + + get() { + return fromNode(cb => { + if (this.ready) return cb(); + this.handlers.push(cb); + this.start(); + }); + } + + work(work) { + this._work = work; + this.stop(); + + if (this.handlers.length) { + this.start(); + } + } + + start() { + if (this.running) return; + this.stop(); + if (this._work) { + this.running = true; + this._work(); + } + } + + stop() { + this.ready = false; + this.error = false; + this.running = false; + } + + success(...args) { + this.stop(); + this.ready = true; + this._flush(args); + } + + failure(err) { + this.stop(); + this.error = err; + this._flush([err]); + } + + _flush(args) { + for (let fn of this.handlers.splice(0)) { + fn.apply(null, args); + } + } +}; diff --git a/src/optimize/lazy/lazy.js b/src/optimize/lazy/lazy.js new file mode 100644 index 000000000000..f7160b9a319b --- /dev/null +++ b/src/optimize/lazy/lazy.js @@ -0,0 +1,35 @@ +module.exports = async (kbnServer, server, config) => { + + let { isWorker } = require('cluster'); + + if (!isWorker) { + throw new Error(`lazy optimization is only available in "watch" mode`); + } + + /** + * When running in lazy mode two workers/threads run in one + * of the modes: 'optmzr' or 'server' + * + * optmzr: this thread runs the LiveOptimizer and the LazyServer + * which serves the LiveOptimizer's output and blocks requests + * while the optimizer is running + * + * server: this thread runs the entire kibana server and proxies + * all requests for /bundles/* to the optmzr + * + * @param {string} process.env.kbnWorkerType + */ + switch (process.env.kbnWorkerType) { + case 'optmzr': + await kbnServer.mixin(require('./optmzrRole')); + break; + + case 'server': + await kbnServer.mixin(require('./proxyRole')); + break; + + default: + throw new Error(`unkown kbnWorkerType "${process.env.kbnWorkerType}"`); + } + +}; diff --git a/src/optimize/lazy/optmzrRole.js b/src/optimize/lazy/optmzrRole.js new file mode 100644 index 000000000000..4f7dca6be5bb --- /dev/null +++ b/src/optimize/lazy/optmzrRole.js @@ -0,0 +1,32 @@ +module.exports = async (kbnServer, kibanaHapiServer, config) => { + + let src = require('requirefrom')('src'); + let fromRoot = src('utils/fromRoot'); + let LazyServer = require('./LazyServer'); + let LazyOptimizer = require('./LazyOptimizer'); + + let server = new LazyServer( + config.get('optimize.lazyHost'), + config.get('optimize.lazyPort'), + new LazyOptimizer({ + log: (tags, data) => kibanaHapiServer.log(tags, data), + env: kbnServer.bundles.env, + bundles: kbnServer.bundles, + profile: config.get('optimize.profile'), + sourceMaps: config.get('optimize.sourceMaps'), + prebuild: config.get('optimize.lazyPrebuild'), + unsafeCache: config.get('optimize.unsafeCache'), + }) + ); + + await server.init(); + + let sendReady = () => { + process.send(['WORKER_BROADCAST', { optimizeReady: true }]); + }; + + sendReady(); + process.on('message', (msg) => { + if (msg && msg.optimizeReady === '?') sendReady(); + }); +}; diff --git a/src/optimize/lazy/proxyRole.js b/src/optimize/lazy/proxyRole.js new file mode 100644 index 000000000000..2d088136dc01 --- /dev/null +++ b/src/optimize/lazy/proxyRole.js @@ -0,0 +1,33 @@ +let { fromNode } = require('bluebird'); +let { get } = require('lodash'); + +module.exports = (kbnServer, server, config) => { + + server.route({ + path: '/bundles/{path*}', + method: 'GET', + handler: { + proxy: { + host: config.get('optimize.lazyHost'), + port: config.get('optimize.lazyPort'), + passThrough: true, + xforward: true + } + } + }); + + return fromNode(cb => { + let timeout = setTimeout(() => { + cb(new Error('Server timedout waiting for the optimizer to become ready')); + }, config.get('optimize.lazyProxyTimeout')); + + process.send(['WORKER_BROADCAST', { optimizeReady: '?' }]); + process.on('message', (msg) => { + if (get(msg, 'optimizeReady')) { + clearTimeout(timeout); + cb(); + } + }); + }); + +}; diff --git a/src/plugins/appSwitcher/index.js b/src/plugins/appSwitcher/index.js index c64887fb1467..d20235503173 100644 --- a/src/plugins/appSwitcher/index.js +++ b/src/plugins/appSwitcher/index.js @@ -2,7 +2,7 @@ module.exports = function (kibana) { return new kibana.Plugin({ uiExports: { app: { - id: 'switcher', + id: 'appSwitcher', main: 'plugins/appSwitcher/appSwitcher', hidden: true, autoload: kibana.autoload.styles diff --git a/src/plugins/bundledLibs/index.js b/src/plugins/bundledLibs/index.js index fc6eb8cd0afe..6f62e6de17cc 100644 --- a/src/plugins/bundledLibs/index.js +++ b/src/plugins/bundledLibs/index.js @@ -4,7 +4,10 @@ module.exports = function (kibana) { let { readdirSync } = require('fs'); let { resolve, basename } = require('path'); - let modules = {}; + let modules = { + moment$: fromRoot('node_modules/moment/min/moment.min.js') + }; + let metaLibs = resolve(__dirname, 'metaLibs'); readdirSync(metaLibs).forEach(function (file) { if (file[0] === '.') return; @@ -17,7 +20,8 @@ module.exports = function (kibana) { uiExports: { modules: modules, noParse: [ - /node_modules[\/\\](angular|elasticsearch-browser|mocha)[\/\\]/ + /node_modules[\/\\](angular|elasticsearch-browser)[\/\\]/, + /node_modules[\/\\](angular-nvd3|mocha|moment)[\/\\]/ ] } }); diff --git a/src/plugins/bundledLibs/metaLibs/angular-nvd3.js b/src/plugins/bundledLibs/metaLibs/angular-nvd3.js new file mode 100644 index 000000000000..1323b7cb7c6e --- /dev/null +++ b/src/plugins/bundledLibs/metaLibs/angular-nvd3.js @@ -0,0 +1,5 @@ +require('d3'); +require('nvd3/build/nv.d3.css'); +require('nvd3/build/nv.d3.js'); +require('angular-nvd3/dist/angular-nvd3.min.js'); +module.exports = window.nv; diff --git a/src/plugins/devMode/index.js b/src/plugins/devMode/index.js index eae23ec2fa12..256667bb2174 100644 --- a/src/plugins/devMode/index.js +++ b/src/plugins/devMode/index.js @@ -1,24 +1,10 @@ -module.exports = function (kibana) { +module.exports = (kibana) => { if (!kibana.config.get('env.dev')) return; - - let utils = require('requirefrom')('src/utils'); - let fromRoot = utils('fromRoot'); - return new kibana.Plugin({ uiExports: { spyModes: [ 'plugins/devMode/visDebugSpyPanel' - ], - - modules: { - ngMock$: fromRoot('src/plugins/devMode/public/ngMock'), - fixtures: fromRoot('src/fixtures'), - testUtils: fromRoot('src/testUtils'), - 'angular-mocks': { - path: require.resolve('angular-mocks'), - imports: 'angular' - }, - } + ] } }); }; diff --git a/src/plugins/elasticsearch/lib/__tests__/health_check.js b/src/plugins/elasticsearch/lib/__tests__/health_check.js index fe0ca4bb0429..4394ebda2da1 100644 --- a/src/plugins/elasticsearch/lib/__tests__/health_check.js +++ b/src/plugins/elasticsearch/lib/__tests__/health_check.js @@ -9,14 +9,11 @@ describe('plugins/elasticsearch', function () { describe('lib/health_check', function () { var health; - var plugin; - var server; - var get; - var client; + beforeEach(function () { // setup the plugin stub plugin = { @@ -75,7 +72,7 @@ describe('plugins/elasticsearch', function () { }); it('should set the cluster red if the ping fails, then to green', function () { - this.timeout(3000); + get.withArgs('elasticsearch.url').returns('http://localhost:9200'); get.withArgs('elasticsearch.minimumVerison').returns('1.4.4'); get.withArgs('kibana.index').returns('.my-kibana'); @@ -100,7 +97,6 @@ describe('plugins/elasticsearch', function () { }); it('should set the cluster red if the health check status is red, then to green', function () { - this.timeout(3000); get.withArgs('elasticsearch.url').returns('http://localhost:9200'); get.withArgs('elasticsearch.minimumVerison').returns('1.4.4'); get.withArgs('kibana.index').returns('.my-kibana'); @@ -124,7 +120,6 @@ describe('plugins/elasticsearch', function () { }); it('should set the cluster yellow if the health check timed_out and create index', function () { - this.timeout(3000); get.withArgs('elasticsearch.url').returns('http://localhost:9200'); get.withArgs('elasticsearch.minimumVerison').returns('1.4.4'); get.withArgs('kibana.index').returns('.my-kibana'); diff --git a/src/plugins/elasticsearch/lib/__tests__/routes.js b/src/plugins/elasticsearch/lib/__tests__/routes.js index 93c34bf2ecd0..177f89e1cd59 100644 --- a/src/plugins/elasticsearch/lib/__tests__/routes.js +++ b/src/plugins/elasticsearch/lib/__tests__/routes.js @@ -13,8 +13,6 @@ describe('plugins/elasticsearch', function () { var kbnServer; before(function () { - this.timeout(10000); - kbnServer = new KbnServer({ server: { autoListen: false }, logging: { quiet: true }, diff --git a/src/plugins/elasticsearch/lib/__tests__/validate.js b/src/plugins/elasticsearch/lib/__tests__/validate.js index 56cc81771bc9..e2cb6e2b6ca0 100644 --- a/src/plugins/elasticsearch/lib/__tests__/validate.js +++ b/src/plugins/elasticsearch/lib/__tests__/validate.js @@ -28,7 +28,7 @@ describe('plugins/elasticsearch', function () { }); after(function () { - kbnServer.close(); + return kbnServer.close(); }); describe('lib/validate', function () { diff --git a/src/plugins/kibana/public/dashboard/index.html b/src/plugins/kibana/public/dashboard/index.html index ab8b5ec64f2d..fe946db78e49 100644 --- a/src/plugins/kibana/public/dashboard/index.html +++ b/src/plugins/kibana/public/dashboard/index.html @@ -2,7 +2,7 @@ -
diff --git a/src/plugins/kibana/public/discover/__tests__/hit_sort_fn.js b/src/plugins/kibana/public/discover/__tests__/hit_sort_fn.js index ef4825a7dde7..3380732b02b5 100644 --- a/src/plugins/kibana/public/discover/__tests__/hit_sort_fn.js +++ b/src/plugins/kibana/public/discover/__tests__/hit_sort_fn.js @@ -16,19 +16,18 @@ describe('hit sort function', function () { var groupSize = _.random(10, 30); var total = sortOpts.length * groupSize; - var hits = new Array(total); sortOpts = sortOpts.map(function (opt) { if (_.isArray(opt)) return opt; else return [opt]; }); var sortOptLength = sortOpts.length; - for (let i = 0; i < hits.length; i++) { - hits[i] = { + var hits = _.times(total, function (i) { + return { _source: {}, sort: sortOpts[i % sortOptLength] }; - } + }); hits.sort(createHitSortFn(dir)) .forEach(function (hit, i) { diff --git a/src/plugins/kibana/public/discover/index.js b/src/plugins/kibana/public/discover/index.js index 603fdc7bb6b9..4b776cacf88a 100644 --- a/src/plugins/kibana/public/discover/index.js +++ b/src/plugins/kibana/public/discover/index.js @@ -5,4 +5,7 @@ define(function (require, module, exports) { require('plugins/kibana/discover/components/field_chooser/field_chooser'); require('plugins/kibana/discover/controllers/discover'); require('plugins/kibana/discover/styles/main.less'); + + // preload + require('ui/doc_table/components/table_row'); }); diff --git a/src/plugins/kibana/public/settings/index.js b/src/plugins/kibana/public/settings/index.js index 98a4e85976b3..3fe76bcde2d9 100644 --- a/src/plugins/kibana/public/settings/index.js +++ b/src/plugins/kibana/public/settings/index.js @@ -36,4 +36,9 @@ define(function (require, module, exports) { } }; }); + + // preload + require('ui/field_editor'); + require('plugins/kibana/settings/sections/indices/_indexed_fields'); + require('plugins/kibana/settings/sections/indices/_scripted_fields'); }); diff --git a/src/plugins/kibana/public/visualize/index.js b/src/plugins/kibana/public/visualize/index.js index 4fd8a507b9c5..84703656a400 100644 --- a/src/plugins/kibana/public/visualize/index.js +++ b/src/plugins/kibana/public/visualize/index.js @@ -8,4 +8,19 @@ define(function (require) { .when('/visualize', { redirectTo: '/visualize/step/1' }); + + // preloading + require('plugins/kibana/visualize/editor/add_bucket_agg'); + require('plugins/kibana/visualize/editor/agg'); + require('plugins/kibana/visualize/editor/agg_add'); + require('plugins/kibana/visualize/editor/agg_filter'); + require('plugins/kibana/visualize/editor/agg_group'); + require('plugins/kibana/visualize/editor/agg_param'); + require('plugins/kibana/visualize/editor/agg_params'); + require('plugins/kibana/visualize/editor/editor'); + require('plugins/kibana/visualize/editor/nesting_indicator'); + require('plugins/kibana/visualize/editor/sidebar'); + require('plugins/kibana/visualize/editor/vis_options'); + require('plugins/kibana/visualize/saved_visualizations/_saved_vis'); + require('plugins/kibana/visualize/saved_visualizations/saved_visualizations'); }); diff --git a/src/plugins/statusPage/index.js b/src/plugins/statusPage/index.js index ca95adecb2b6..66930183ba9d 100644 --- a/src/plugins/statusPage/index.js +++ b/src/plugins/statusPage/index.js @@ -11,22 +11,7 @@ module.exports = function (kibana) { 'ui/chrome', 'angular' ) - }, - - modules: { - nvd3$: { - path: 'nvd3/build/nv.d3.js', - exports: 'window.nv', - imports: 'd3,nvd3Styles' - }, - nvd3Styles$: { - path: 'nvd3/build/nv.d3.css' - } - }, - - loaders: [ - { test: /\/angular-nvd3\//, loader: 'imports?angular,nv=nvd3,d3' } - ] + } } }); }; diff --git a/src/plugins/testsBundle/index.js b/src/plugins/testsBundle/index.js new file mode 100644 index 000000000000..e1bc0a7c907d --- /dev/null +++ b/src/plugins/testsBundle/index.js @@ -0,0 +1,47 @@ +module.exports = (kibana) => { + if (!kibana.config.get('optimize.tests')) return; + + let { union } = require('lodash'); + + let utils = require('requirefrom')('src/utils'); + let fromRoot = utils('fromRoot'); + let findSourceFiles = utils('findSourceFiles'); + + return new kibana.Plugin({ + uiExports: { + bundle: async (UiBundle, env, apps) => { + + let modules = []; + + // add the modules from all of the apps + for (let app of apps) { + modules = union(modules, app.getModules()); + } + + let testFiles = await findSourceFiles([ + 'src/**/public/**/__tests__/**/*.js', + 'installedPlugins/*/public/**/__tests__/**/*.js' + ]); + + for (let f of testFiles) modules.push(f); + + return new UiBundle({ + id: 'tests', + modules: modules, + template: require('./testsEntryTemplate'), + env: env + }); + }, + + modules: { + ngMock$: fromRoot('src/plugins/devMode/public/ngMock'), + fixtures: fromRoot('src/fixtures'), + testUtils: fromRoot('src/testUtils'), + 'angular-mocks': { + path: require.resolve('angular-mocks'), + imports: 'angular' + }, + } + } + }); +}; diff --git a/src/plugins/testsBundle/package.json b/src/plugins/testsBundle/package.json new file mode 100644 index 000000000000..dd351b09560c --- /dev/null +++ b/src/plugins/testsBundle/package.json @@ -0,0 +1,4 @@ +{ + "name": "tests_bundle", + "version": "0.0.0" +} diff --git a/src/plugins/testsBundle/testsEntryTemplate.js b/src/plugins/testsBundle/testsEntryTemplate.js new file mode 100644 index 000000000000..b1b695cd2447 --- /dev/null +++ b/src/plugins/testsBundle/testsEntryTemplate.js @@ -0,0 +1,34 @@ + +module.exports = require('lodash').template( +` +/** + * Optimized application entry file + * + * This is programatically created and updated, do not modify + * + * context: <%= JSON.stringify(env.context) %> + * includes code from: +<% + +env.pluginInfo.sort().forEach(function (plugin, i) { + if (i > 0) print('\\n'); + print(' * - ' + plugin); +}); + +%> + * + */ + +require('ui/testHarness'); +<% + +bundle.modules.forEach(function (id, i) { + if (i > 0) print('\\n'); + print(\`require('\${id.replace(/\\\\/g, '\\\\\\\\')}');\`); +}); + +%> +require('ui/testHarness').bootstrap(/* go! */); + +` +); diff --git a/src/server/KbnServer.js b/src/server/KbnServer.js index f3f181cf9ec6..525c92473bcf 100644 --- a/src/server/KbnServer.js +++ b/src/server/KbnServer.js @@ -1,5 +1,4 @@ -let _ = require('lodash'); -let { EventEmitter } = require('events'); +let { constant, once, compact, flatten } = require('lodash'); let { promisify, resolve, fromNode } = require('bluebird'); let Hapi = require('hapi'); @@ -7,36 +6,45 @@ let utils = require('requirefrom')('src/utils'); let rootDir = utils('fromRoot')('.'); let pkg = utils('packageJson'); -module.exports = class KbnServer extends EventEmitter { +module.exports = class KbnServer { constructor(settings) { - super(); - this.name = pkg.name; this.version = pkg.version; this.build = pkg.build || false; this.rootDir = rootDir; - this.server = new Hapi.Server(); this.settings = settings || {}; - this.ready = _.constant(this.mixin( - require('./config/setup'), - require('./http'), + this.ready = constant(this.mixin( + require('./config/setup'), // sets this.config, reads this.settings + require('./http'), // sets this.server require('./logging'), - require('./status'), // sets this.status - require('./plugins'), // sets this.plugins + require('./status'), + + // find plugins and set this.plugins + require('./plugins/scan'), + + // tell the config we are done loading plugins require('./config/complete'), - require('../ui'), // sets this.uiExports + // setup this.uiExports and this.bundles + require('../ui'), + + // ensure that all bundles are built, or that the + // lazy bundle server is running require('../optimize'), - function () { + // finally, initialize the plugins + require('./plugins/initialize'), + + () => { if (this.config.get('server.autoListen')) { - this.listen(); + this.ready = constant(resolve()); + return this.listen(); } } )); - this.listen = _.once(this.listen); + this.listen = once(this.listen); } /** @@ -49,55 +57,30 @@ module.exports = class KbnServer extends EventEmitter { * and can return a promise to delay execution of the next mixin * @return {Promise} - promise that is resolved when the final mixin completes. */ - mixin(/* ...fns */) { - let self = this; - return resolve(_.toArray(arguments)) - .then(_.compact) - .each(function (fn) { - return fn.call(self, self, self.server, self.config); - }) - .catch(function (err) { - self.server.log('fatal', err); - self.emit('error', err); - - return self.close() - .then(function () { - // retrow once server is closed - throw err; - }); - }) - .return(undefined); + async mixin(...fns) { + for (let fn of compact(flatten(fns))) { + await fn.call(this, this, this.server, this.config); + } } /** * Tell the server to listen for incoming requests, or get * a promise that will be resolved once the server is listening. * - * Calling this function has no effect, unless the "server.autoListen" - * is set to false. - * * @return undefined */ - listen() { - let self = this; + async listen() { + let { server, config } = this; - return self.ready() - .then(function () { - return self.mixin( - function () { - return fromNode(_.bindKey(self.server, 'start')); - }, - require('./pid') - ); - }) - .then(function () { - self.server.log(['listening', 'info'], 'Server running at ' + self.server.info.uri); - self.emit('listening'); - return self.server; - }); + await this.ready(); + await fromNode(cb => server.start(cb)); + await require('./pid')(this, server, config); + + server.log(['listening', 'info'], 'Server running at ' + server.info.uri); + return server; } - close() { - return fromNode(_.bindKey(this.server, 'stop')); + async close() { + await fromNode(cb => this.server.stop(cb)); } }; diff --git a/src/server/config/complete.js b/src/server/config/complete.js index f4a0da751cdb..0fd408114305 100644 --- a/src/server/config/complete.js +++ b/src/server/config/complete.js @@ -1,6 +1,10 @@ module.exports = function (kbnServer, server, config) { let _ = require('lodash'); + server.decorate('server', 'config', function () { + return kbnServer.config; + }); + _.forOwn(config.unappliedDefaults, function (val, key) { if (val === null) return; server.log(['warning', 'config'], { diff --git a/src/server/config/schema.js b/src/server/config/schema.js index 40eb8ef16a05..0b373398185e 100644 --- a/src/server/config/schema.js +++ b/src/server/config/schema.js @@ -15,8 +15,7 @@ module.exports = Joi.object({ env: Joi.object({ name: Joi.string().default(Joi.ref('$env')), dev: Joi.boolean().default(Joi.ref('$dev')), - prod: Joi.boolean().default(Joi.ref('$prod')), - test: Joi.boolean().default(Joi.ref('$test')), + prod: Joi.boolean().default(Joi.ref('$prod')) }).default(), pid: Joi.object({ @@ -32,7 +31,14 @@ module.exports = Joi.object({ ssl: Joi.object({ cert: Joi.string(), key: Joi.string() - }).default() + }).default(), + cors: Joi.when('$dev', { + is: true, + then: Joi.object().default({ + origin: ['*://localhost:9876'] // karma test server + }), + otherwise: Joi.boolean().default(false) + }) }).default(), logging: Joi.object().keys({ @@ -72,11 +78,38 @@ module.exports = Joi.object({ optimize: Joi.object({ enabled: Joi.boolean().default(true), + bundleFilter: Joi.string().when('tests', { + is: true, + then: Joi.default('tests'), + otherwise: Joi.default('*') + }), bundleDir: Joi.string().default(fromRoot('optimize/bundles')), viewCaching: Joi.boolean().default(Joi.ref('$prod')), - watch: Joi.boolean().default(Joi.ref('$dev')), - sourceMaps: Joi.boolean().default(Joi.ref('$dev')), - _workerRole: Joi.valid('send', 'receive', null).default(null) + lazy: Joi.boolean().when('$dev', { + is: true, + then: Joi.default(true), + otherwise: Joi.default(false) + }), + lazyPort: Joi.number().default(5602), + lazyHost: Joi.string().hostname().default('0.0.0.0'), + lazyPrebuild: Joi.boolean().default(false), + lazyProxyTimeout: Joi.number().default(5 * 60000), + unsafeCache: Joi + .alternatives() + .try( + Joi.boolean(), + Joi.string().regex(/^\/.+\/$/) + ) + .default('/[\\/\\\\](node_modules|bower_components)[\\/\\\\]/'), + sourceMaps: Joi + .alternatives() + .try( + Joi.string().required(), + Joi.boolean() + ) + .default(Joi.ref('$dev')), + profile: Joi.boolean().default(false), + tests: Joi.boolean().default(false), }).default() }).default(); diff --git a/src/server/config/setup.js b/src/server/config/setup.js index 7a85df0bfd30..d1d9f8d69b2e 100644 --- a/src/server/config/setup.js +++ b/src/server/config/setup.js @@ -1,9 +1,6 @@ -module.exports = function (kbnServer, server) { +module.exports = function (kbnServer) { let Config = require('./Config'); let schema = require('./schema'); kbnServer.config = new Config(schema, kbnServer.settings || {}); - server.decorate('server', 'config', function () { - return kbnServer.config; - }); }; diff --git a/src/server/http/getDefaultRoute.js b/src/server/http/getDefaultRoute.js index d2f6b4c7a95b..91546528e944 100644 --- a/src/server/http/getDefaultRoute.js +++ b/src/server/http/getDefaultRoute.js @@ -6,9 +6,6 @@ module.exports = _.once(function (kbnServer) { if (defaultConfig) return defaultConfig; // redirect to the single app - if (kbnServer.uiExports.apps.length === 1) { - return `/app/${kbnServer.uiExports.apps[0].id}`; - } - - return '/apps'; + let apps = kbnServer.uiExports.apps.toArray(); + return apps.length === 1 ? `/app/${apps[0].id}` : '/apps'; }); diff --git a/src/server/http/index.js b/src/server/http/index.js index a5366cf986bf..5b4c2adfc394 100644 --- a/src/server/http/index.js +++ b/src/server/http/index.js @@ -1,14 +1,21 @@ module.exports = function (kbnServer, server, config) { let _ = require('lodash'); let Boom = require('boom'); + let Hapi = require('hapi'); let parse = require('url').parse; let format = require('url').format; + let getDefaultRoute = require('./getDefaultRoute'); + server = kbnServer.server = new Hapi.Server(); + // Create a new connection server.connection({ host: config.get('server.host'), - port: config.get('server.port') + port: config.get('server.port'), + routes: { + cors: config.get('server.cors') + } }); // provide a simple way to expose static directories diff --git a/src/server/logging/LogFormatString.js b/src/server/logging/LogFormatString.js index 1b0c54c4dd5a..0360e1914cae 100644 --- a/src/server/logging/LogFormatString.js +++ b/src/server/logging/LogFormatString.js @@ -4,6 +4,16 @@ let moment = require('moment'); let LogFormat = require('./LogFormat'); +let statuses = [ + 'err', + 'info', + 'error', + 'warning', + 'fatal', + 'status', + 'debug' +]; + let typeColors = { log: 'blue', req: 'green', @@ -39,7 +49,9 @@ module.exports = class KbnLoggerJsonFormat extends LogFormat { let tags = _(data.tags) .sortBy(function (tag) { - return color(tag) === _.identity ? `1${tag}` : `0${tag}`; + if (color(tag) === _.identity) return `2${tag}`; + if (_.includes(statuses, tag)) return `0${tag}`; + return `1${tag}`; }) .reduce(function (s, t) { return s + `[${ color(t)(t) }]`; diff --git a/src/server/pid/index.js b/src/server/pid/index.js index 3e95936b4e79..b93a464d2c46 100644 --- a/src/server/pid/index.js +++ b/src/server/pid/index.js @@ -1,4 +1,5 @@ var _ = require('lodash'); +var Boom = require('boom'); var Promise = require('bluebird'); var writeFile = Promise.promisify(require('fs').writeFile); var unlink = require('fs').unlinkSync; @@ -20,8 +21,7 @@ module.exports = Promise.method(function (kbnServer, server, config) { }; if (config.get('pid.exclusive')) { - server.log(['pid', 'fatal'], log); - process.exit(1); // eslint-disable-line no-process-exit + throw Boom.create(500, _.template(log.tmpl)(log), log); } else { server.log(['pid', 'warning'], log); } diff --git a/src/server/plugins/Plugin.js b/src/server/plugins/Plugin.js index 64e6a6986424..702e4275a1ca 100644 --- a/src/server/plugins/Plugin.js +++ b/src/server/plugins/Plugin.js @@ -1,6 +1,6 @@ let _ = require('lodash'); let Joi = require('joi'); -let Promise = require('bluebird'); +let { attempt, fromNode } = require('bluebird'); let { resolve } = require('path'); let { inherits } = require('util'); @@ -33,63 +33,49 @@ module.exports = class Plugin { }; } - init() { - let self = this; + async setupConfig() { + let { config } = this.kbnServer; + let schema = await this.getConfigSchema(Joi); + this.kbnServer.config.extendSchema(this.id, schema || defaultConfigSchema); + } - let id = self.id; - let version = self.version; - let kbnStatus = self.kbnServer.status; - let server = self.kbnServer.server; - let config = self.kbnServer.config; + async init() { + let { id, version, kbnServer } = this; + let { config } = kbnServer; - server.log(['plugins', 'debug'], { - tmpl: 'Initializing plugin <%= plugin.id %>', - plugin: self - }); + // setup the hapi register function and get on with it + let register = (server, options, next) => { + this.server = server; - self.status = kbnStatus.create(`plugin:${self.id}`); - return Promise.try(function () { - return self.getConfigSchema(Joi); - }) - .then(function (schema) { - if (schema) config.extendSchema(id, schema); - else config.extendSchema(id, defaultConfigSchema); - }) - .then(function () { - if (config.get([id, 'enabled'])) { - return self.externalCondition(config); - } - }) - .then(function (enabled) { - if (!enabled) { - // Only change the plugin status if it wasn't set by the externalCondition - if (self.status.state === 'uninitialized') { - self.status.disabled(); - } - return; - } - - let register = function (server, options, next) { - server.expose('status', self.status); - Promise.try(self.externalInit, [server, options], self).nodeify(next); - }; - - register.attributes = { name: id, version: version }; - - return Promise.fromNode(function (cb) { - server.register({ - register: register, - options: config.has(id) ? config.get(id) : null - }, cb); - }) - .then(function () { - // Only change the plugin status to green if the - // intial status has not been updated - if (self.status.state === 'uninitialized') { - self.status.green('Ready'); - } + server.log(['plugins', 'debug'], { + tmpl: 'Initializing plugin <%= plugin.id %>', + plugin: this }); + + if (this.publicDir) { + server.exposeStaticDir(`/plugins/${id}/{path*}`, this.publicDir); + } + + this.status = kbnServer.status.create(`plugin:${this.id}`); + server.expose('status', this.status); + + attempt(this.externalInit, [server, options], this).nodeify(next); + }; + + register.attributes = { name: id, version: version }; + + await fromNode(cb => { + kbnServer.server.register({ + register: register, + options: config.has(id) ? config.get(id) : null + }, cb); }); + + // Only change the plugin status to green if the + // intial status has not been changed + if (this.status.state === 'uninitialized') { + this.status.green('Ready'); + } } toJSON() { diff --git a/src/server/plugins/PluginApi.js b/src/server/plugins/PluginApi.js index c04bd4a3fcf2..85e2168b169f 100644 --- a/src/server/plugins/PluginApi.js +++ b/src/server/plugins/PluginApi.js @@ -4,7 +4,7 @@ let { basename, join } = require('path'); module.exports = class PluginApi { constructor(kibana, pluginPath) { - this.config = kibana.server.config(); + this.config = kibana.config; this.rootDir = kibana.rootDir; this.package = require(join(pluginPath, 'package.json')); this.autoload = require('../../ui/autoload'); diff --git a/src/server/plugins/PluginCollection.js b/src/server/plugins/PluginCollection.js new file mode 100644 index 000000000000..089a211a23af --- /dev/null +++ b/src/server/plugins/PluginCollection.js @@ -0,0 +1,34 @@ +let _ = require('lodash'); +let inspect = require('util').inspect; + +let PluginApi = require('./PluginApi'); +let Collection = require('requirefrom')('src')('utils/Collection'); + +let byIdCache = Symbol('byIdCache'); + +module.exports = class Plugins extends Collection { + + constructor(kbnServer) { + super(); + this.kbnServer = kbnServer; + } + + new(path) { + var api = new PluginApi(this.kbnServer, path); + let output = [].concat(require(path)(api) || []); + + for (let product of output) { + if (product instanceof api.Plugin) { + this[byIdCache] = null; + this.add(product); + } else { + throw new TypeError('unexpected plugin export ' + inspect(product)); + } + } + } + + get byId() { + return this[byIdCache] || (this[byIdCache] = _.indexBy([...this], 'id')); + } + +}; diff --git a/src/server/plugins/Plugins.js b/src/server/plugins/Plugins.js deleted file mode 100644 index 3252d11b7427..000000000000 --- a/src/server/plugins/Plugins.js +++ /dev/null @@ -1,35 +0,0 @@ -let _ = require('lodash'); -let inspect = require('util').inspect; -let PluginApi = require('./PluginApi'); - -module.exports = class Plugins extends Array { - - constructor(kbnServer) { - super(); - this.kbnServer = kbnServer; - } - - new(path) { - var self = this; - var api = new PluginApi(this.kbnServer, path); - - [].concat(require(path)(api) || []) - .forEach(function (out) { - if (out instanceof api.Plugin) { - self._byId = null; - self.push(out); - } else { - throw new TypeError('unexpected plugin export ' + inspect(out)); - } - }); - } - - get byId() { - return this._byId || (this._byId = _.indexBy(this, 'id')); - } - - toJSON() { - return this.slice(0); - } - -}; diff --git a/src/server/plugins/index.js b/src/server/plugins/index.js deleted file mode 100644 index 26f04eccda5d..000000000000 --- a/src/server/plugins/index.js +++ /dev/null @@ -1,25 +0,0 @@ -module.exports = function (kbnServer, server, config) { - var _ = require('lodash'); - var Promise = require('bluebird'); - var Boom = require('boom'); - var { join } = require('path'); - - server.exposeStaticDir('/plugins/{id}/{path*}', function (req) { - var id = req.params.id; - var plugin = kbnServer.plugins.byId[id]; - return (plugin && plugin.publicDir) ? plugin.publicDir : Boom.notFound(); - }); - - server.method('kbnPluginById', function (id, next) { - if (kbnServer.plugins.byId[id]) { - next(null, kbnServer.plugins.byId[id]); - } else { - next(Boom.notFound(`no plugin with the id "${id}"`)); - } - }); - - return kbnServer.mixin( - require('./scan'), - require('./load') - ); -}; diff --git a/src/server/plugins/initialize.js b/src/server/plugins/initialize.js new file mode 100644 index 000000000000..0d01ce94915c --- /dev/null +++ b/src/server/plugins/initialize.js @@ -0,0 +1,49 @@ +module.exports = async function (kbnServer, server, config) { + let { includes, keys } = require('lodash'); + + if (!config.get('plugins.initialize')) { + server.log(['info'], 'Plugin initialization disabled.'); + return []; + } + + let { plugins } = kbnServer; + let enabledPlugins = {}; + + // setup config and filter out disabled plugins + for (let plugin of plugins) { + await plugin.setupConfig(); + if (config.get([plugin.id, 'enabled'])) { + enabledPlugins[plugin.id] = plugin; + } + } + + + let path = []; + let initialize = async id => { + let plugin = enabledPlugins[id]; + + if (includes(path, id)) { + throw new Error(`circular dependencies found: "${path.concat(id).join(' -> ')}"`); + } + + path.push(id); + + + for (let reqId of plugin.requiredIds) { + if (!enabledPlugins[reqId]) { + if (plugins.byId[reqId]) { + throw new Error(`Requirement "${reqId}" for plugin "${plugin.id}" is disabled.`); + } else { + throw new Error(`Unmet requirement "${reqId}" for plugin "${plugin.id}"`); + } + } + await initialize(reqId); + } + + await plugin.init(); + + path.pop(); + }; + + for (let id of keys(enabledPlugins)) await initialize(id); +}; diff --git a/src/server/plugins/load.js b/src/server/plugins/load.js deleted file mode 100644 index 34a7e2752376..000000000000 --- a/src/server/plugins/load.js +++ /dev/null @@ -1,52 +0,0 @@ -module.exports = function (kbnServer, server, config) { - let _ = require('lodash'); - let resolve = require('bluebird').resolve; - let all = require('bluebird').all; - let attempt = require('bluebird').attempt; - - var Plugins = require('./Plugins'); - var plugins = kbnServer.plugins = new Plugins(kbnServer); - - let path = []; - let step = function (id, block) { - return resolve(id) - .then(function (id) { - if (_.includes(path, id)) { - throw new Error(`circular dependencies found: "${path.concat(id).join(' -> ')}"`); - } - - path.push(id); - return block(); - }) - .then(function () { - path.pop(id); - }); - }; - - return resolve(kbnServer.pluginPaths) - .map(function (path) { - return plugins.new(path); - }) - .then(function () { - if (!config.get('plugins.initialize')) { - server.log(['info'], 'Plugin initialization disabled.'); - return []; - } - - return _.toArray(plugins); - }) - .each(function loadReqsAndInit(plugin) { - return step(plugin.id, function () { - return resolve(plugin.requiredIds).map(function (reqId) { - if (!plugins.byId[reqId]) { - throw new Error(`Unmet requirement "${reqId}" for plugin "${plugin.id}"`); - } - - return loadReqsAndInit(plugins.byId[reqId]); - }) - .then(function () { - return plugin.init(); - }); - }); - }); -}; diff --git a/src/server/plugins/scan.js b/src/server/plugins/scan.js index 76bbd1cda910..90719e449f01 100644 --- a/src/server/plugins/scan.js +++ b/src/server/plugins/scan.js @@ -1,63 +1,60 @@ -module.exports = function (kbnServer, server, config) { +module.exports = async (kbnServer, server, config) => { let _ = require('lodash'); - let Promise = require('bluebird'); - let readdir = Promise.promisify(require('fs').readdir); - let stat = Promise.promisify(require('fs').stat); - let resolve = require('path').resolve; + let { fromNode } = require('bluebird'); + let { readdir, stat } = require('fs'); + let { resolve } = require('path'); + let { each } = require('bluebird'); + + var PluginCollection = require('./PluginCollection'); + var plugins = kbnServer.plugins = new PluginCollection(kbnServer); let scanDirs = [].concat(config.get('plugins.scanDirs') || []); - let absolutePaths = [].concat(config.get('plugins.paths') || []); + let pluginPaths = [].concat(config.get('plugins.paths') || []); + let debug = _.bindKey(server, 'log', ['plugins', 'debug']); let warning = _.bindKey(server, 'log', ['plugins', 'warning']); - return Promise.map(scanDirs, function (dir) { + // scan all scanDirs to find pluginPaths + await each(scanDirs, async dir => { debug({ tmpl: 'Scanning `<%= dir %>` for plugins', dir: dir }); - return readdir(dir) - .catch(function (err) { - if (err.code !== 'ENOENT') { - throw err; - } + let filenames = null; + try { + filenames = await fromNode(cb => readdir(dir, cb)); + } catch (err) { + if (err.code !== 'ENOENT') throw err; + + filenames = []; warning({ tmpl: '<%= err.code %>: Unable to scan non-existent directory for plugins "<%= dir %>"', err: err, dir: dir }); - - return []; - }) - .map(function (file) { - if (file === '.' || file === '..') return false; - let path = resolve(dir, file); - - return stat(path).then(function (stat) { - return stat.isDirectory() ? path : false; - }); - }); - }) - .then(function (dirs) { - return _([dirs, absolutePaths]) - .flattenDeep() - .compact() - .uniq() - .value(); - }) - .filter(function (dir) { - let path; - try { path = require.resolve(dir); } - catch (e) { path = false; } - - if (!path) { - warning({ tmpl: 'Skipping non-plugin directory at <%= dir %>', dir: dir }); - return false; - } else { - require(path); - debug({ tmpl: 'Found plugin at <%= dir %>', dir: dir }); - return true; } - }) - .then(function (pluginPaths) { - kbnServer.pluginPaths = pluginPaths; + + await each(filenames, async name => { + if (name[0] === '.') return; + + let path = resolve(dir, name); + let stats = await fromNode(cb => stat(path, cb)); + if (stats.isDirectory()) { + pluginPaths.push(path); + } + }); }); + + for (let path of pluginPaths) { + let modulePath; + try { + modulePath = require.resolve(path); + } catch (e) { + warning({ tmpl: 'Skipping non-plugin directory at <%= path %>', path: path }); + continue; + } + + require(modulePath); + plugins.new(path); + debug({ tmpl: 'Found plugin at <%= path %>', path: modulePath }); + } }; diff --git a/src/server/status/index.js b/src/server/status/index.js index 8adf557096b2..557383e064a8 100644 --- a/src/server/status/index.js +++ b/src/server/status/index.js @@ -1,12 +1,9 @@ -module.exports = function (kbnServer) { +module.exports = function (kbnServer, server, config) { var _ = require('lodash'); var Samples = require('./Samples'); var ServerStatus = require('./ServerStatus'); var { join } = require('path'); - var server = kbnServer.server; - var config = server.config(); - kbnServer.status = new ServerStatus(kbnServer.server); kbnServer.metrics = new Samples(60); @@ -39,7 +36,7 @@ module.exports = function (kbnServer) { }); server.decorate('reply', 'renderStatusPage', function () { - var app = _.get(kbnServer, 'uiExports.apps.hidden.byId.statusPage'); + var app = kbnServer.uiExports.getHiddenApp('statusPage'); var resp = app ? this.renderApp(app) : this(kbnServer.status.toString()); resp.code(kbnServer.status.isGreen() ? 200 : 503); return resp; diff --git a/src/ui/UiApp.js b/src/ui/UiApp.js index d0c83a876c05..686c8cb314b4 100644 --- a/src/ui/UiApp.js +++ b/src/ui/UiApp.js @@ -19,7 +19,6 @@ class UiApp { this.hidden = this.spec.hidden; this.autoloadOverrides = this.spec.autoload; this.templateName = this.spec.templateName || 'uiApp'; - this.requireOptimizeGreen = this.spec.requireOptimizeGreen !== false; this.getModules = _.once(this.getModules); } @@ -34,18 +33,6 @@ class UiApp { .value(); } - getRelatedPlugins() { - var pluginsById = this.uiExports.kbnServer.plugins.byId; - return _.transform(this.getModules(), function (plugins, id) { - var matches = id.match(/^plugins\/([^\/]+)(?:\/|$)/); - if (!matches) return; - - var plugin = pluginsById[matches[1]]; - if (_.includes(plugins, plugin)) return; - plugins.push(plugin); - }, []); - } - toJSON() { return _.pick(this, ['id', 'title', 'description', 'icon', 'main']); } diff --git a/src/ui/UiApps.js b/src/ui/UiAppCollection.js similarity index 64% rename from src/ui/UiApps.js rename to src/ui/UiAppCollection.js index a94ff7c61469..e4eda77fb0ed 100644 --- a/src/ui/UiApps.js +++ b/src/ui/UiAppCollection.js @@ -1,7 +1,10 @@ let _ = require('lodash'); let UiApp = require('./UiApp'); +let Collection = require('requirefrom')('src')('utils/Collection'); -module.exports = class UiApps extends Array { +let byIdCache = Symbol('byId'); + +module.exports = class UiAppCollection extends Collection { constructor(uiExports, parent) { super(); @@ -10,7 +13,7 @@ module.exports = class UiApps extends Array { if (!parent) { this.claimedIds = []; - this.hidden = new UiApps(uiExports, this); + this.hidden = new UiAppCollection(uiExports, this); } else { this.claimedIds = parent.claimedIds; } @@ -30,17 +33,13 @@ module.exports = class UiApps extends Array { this.claimedIds.push(app.id); } - this._byId = null; - this.push(app); + this[byIdCache] = null; + this.add(app); return app; } get byId() { - return this._byId || (this._byId = _.indexBy(this, 'id')); - } - - toJSON() { - return this.slice(0); + return this[byIdCache] || (this[byIdCache] = _.indexBy([...this], 'id')); } }; diff --git a/src/ui/UiBundle.js b/src/ui/UiBundle.js new file mode 100644 index 000000000000..b610c24b7b4d --- /dev/null +++ b/src/ui/UiBundle.js @@ -0,0 +1,68 @@ + +let { join } = require('path'); +let { promisify } = require('bluebird'); +let read = promisify(require('fs').readFile); +let write = promisify(require('fs').writeFile); +let unlink = promisify(require('fs').unlink); +let stat = promisify(require('fs').stat); + +module.exports = class UiBundle { + constructor(opts) { + + opts = opts || {}; + this.id = opts.id; + this.modules = opts.modules; + this.template = opts.template; + this.env = opts.env; + + let pathBase = join(this.env.workingDir, this.id); + this.entryPath = `${pathBase}.entry.js`; + this.outputPath = `${pathBase}.bundle.js`; + + } + + renderContent() { + return this.template({ + env: this.env, + bundle: this + }); + } + + async readEntryFile() { + try { + let content = await read(this.entryPath); + return content.toString('utf8'); + } + catch (e) { + return null; + } + } + + async writeEntryFile() { + return await write(this.entryPath, this.renderContent(), { encoding: 'utf8' }); + } + + async clearBundleFile() { + try { await unlink(this.outputPath); } + catch (e) { return null; } + } + + async checkForExistingOutput() { + try { + await stat(this.outputPath); + return true; + } + catch (e) { + return false; + } + } + + toJSON() { + return { + id: this.id, + modules: this.modules, + entryPath: this.entryPath, + outputPath: this.outputPath + }; + } +}; diff --git a/src/ui/UiBundleCollection.js b/src/ui/UiBundleCollection.js new file mode 100644 index 000000000000..328c03dad156 --- /dev/null +++ b/src/ui/UiBundleCollection.js @@ -0,0 +1,104 @@ +let { pull, transform, pluck } = require('lodash'); +let { join } = require('path'); +let { resolve, promisify } = require('bluebird'); +let { makeRe } = require('minimatch'); +let rimraf = promisify(require('rimraf')); +let mkdirp = promisify(require('mkdirp')); +let unlink = promisify(require('fs').unlink); +let readdir = promisify(require('fs').readdir); +let readSync = require('fs').readFileSync; + +let UiBundle = require('./UiBundle'); +let appEntryTemplate = require('./appEntryTemplate'); + +class UiBundleCollection { + constructor(bundlerEnv, filter) { + this.each = []; + this.env = bundlerEnv; + this.filter = makeRe(filter || '*', { + noglobstar: true, + noext: true, + matchBase: true + }); + } + + add(bundle) { + if (!(bundle instanceof UiBundle)) { + throw new TypeError('expected bundle to be an instance of UiBundle'); + } + + if (this.filter.test(bundle.id)) { + this.each.push(bundle); + } + } + + addApp(app) { + this.add(new UiBundle({ + id: app.id, + modules: app.getModules(), + template: appEntryTemplate, + env: this.env + })); + } + + desc() { + switch (this.each.length) { + case 0: + return '0 bundles'; + case 1: + return `bundle for ${this.each[0].id}`; + default: + var ids = this.getIds(); + var last = ids.pop(); + var commas = ids.join(', '); + return `bundles for ${commas} and ${last}`; + } + } + + async ensureDir() { + await mkdirp(this.env.workingDir); + } + + async writeEntryFiles() { + await this.ensureDir(); + + for (let bundle of this.each) { + let existing = await bundle.readEntryFile(); + let expected = bundle.renderContent(); + + if (existing !== expected) { + await bundle.writeEntryFile(); + await bundle.clearBundleFile(); + } + } + } + + async getInvalidBundles() { + let invalids = new UiBundleCollection(this.env); + + for (let bundle of this.each) { + let exists = await bundle.checkForExistingOutput(); + if (!exists) { + invalids.add(bundle); + } + } + + return invalids; + } + + toWebpackEntries() { + return transform(this.each, function (entries, bundle) { + entries[bundle.id] = bundle.entryPath; + }, {}); + } + + getIds() { + return pluck(this.each, 'id'); + } + + toJSON() { + return this.each; + } +} + +module.exports = UiBundleCollection; diff --git a/src/ui/UiBundlerEnv.js b/src/ui/UiBundlerEnv.js new file mode 100644 index 000000000000..b97909f0fd65 --- /dev/null +++ b/src/ui/UiBundlerEnv.js @@ -0,0 +1,147 @@ +let { includes, flow, escapeRegExp } = require('lodash'); +let { isString, isArray, isPlainObject, get } = require('lodash'); +let { keys } = require('lodash'); +let fromRoot = require('../utils/fromRoot'); + +let asRegExp = flow( + escapeRegExp, + function (path) { + return path + '(?:\\.js)?$'; + }, + RegExp +); + +let arr = v => [].concat(v || []); + +module.exports = class UiBundlerEnv { + constructor(workingDir) { + + // the location that bundle entry files and all compiles files will + // be written + this.workingDir = workingDir; + + // the context that the bundler is running in, this is not officially + // used for anything but it is serialized into the entry file to ensure + // that they are invalidated when the context changes + this.context = {}; + + // the plugins that are used to build this environment + // are tracked and embedded into the entry file so that when the + // environment changes we can rebuild the bundles + this.pluginInfo = []; + + // regular expressions which will prevent webpack from parsing the file + this.noParse = []; + + // webpack aliases, like require paths, mapping a prefix to a directory + this.aliases = { + ui: fromRoot('src/ui/public'), + testHarness: fromRoot('src/testHarness/public') + }; + + // map of which plugins created which aliases + this.aliasOwners = {}; + + // webpack loaders map loader configuration to regexps + this.loaders = []; + } + + consumePlugin(plugin) { + let tag = `${plugin.id}@${plugin.version}`; + if (includes(this.pluginInfo, tag)) return; + + if (plugin.publicDir) { + this.aliases[`plugins/${plugin.id}`] = plugin.publicDir; + } + + this.pluginInfo.push(tag); + } + + exportConsumer(type) { + switch (type) { + case 'loaders': + return (plugin, spec) => { + for (let loader of arr(spec)) this.addLoader(loader); + }; + + case 'noParse': + return (plugin, spec) => { + for (let re of arr(spec)) this.addNoParse(re); + }; + + case 'modules': + return (plugin, spec) => { + for (let id of keys(spec)) this.addModule(id, spec[id], plugin.id); + }; + } + } + + addContext(key, val) { + this.context[key] = val; + } + + addLoader(loader) { + this.loaders.push(loader); + } + + addNoParse(regExp) { + this.noParse.push(regExp); + } + + addModule(id, spec, pluginId) { + this.claim(id, pluginId); + + // configurable via spec + let path; + let parse = true; + let imports = null; + let exports = null; + let expose = null; + + // basic style, just a path + if (isString(spec)) path = spec; + + if (isArray(spec)) { + path = spec[0]; + imports = spec[1]; + exports = spec[2]; + } + + if (isPlainObject(spec)) { + path = spec.path; + parse = get(spec, 'parse', parse); + imports = get(spec, 'imports', imports); + exports = get(spec, 'exports', exports); + expose = get(spec, 'expose', expose); + } + + if (!path) { + throw new TypeError('Invalid spec definition, unable to identify path'); + } + + this.aliases[id] = path; + + let loader = []; + if (imports) { + loader.push(`imports?${imports}`); + } + + if (exports) loader.push(`exports?${exports}`); + if (expose) loader.push(`expose?${expose}`); + if (loader.length) this.loaders.push({ test: asRegExp(path), loader: loader.join('!') }); + if (!parse) this.noParse.push(asRegExp(path)); + } + + claim(id, pluginId) { + let owner = pluginId ? `Plugin ${pluginId}` : 'Kibana Server'; + + // TODO(spalger): we could do a lot more to detect colliding module defs + var existingOwner = this.aliasOwners[id] || this.aliasOwners[`${id}$`]; + + if (existingOwner) { + throw new TypeError(`${owner} attempted to override export "${id}" from ${existingOwner}`); + } + + this.aliasOwners[id] = owner; + } +}; diff --git a/src/ui/UiExports.js b/src/ui/UiExports.js index 0c3c0af72a50..4c1ea9812961 100644 --- a/src/ui/UiExports.js +++ b/src/ui/UiExports.js @@ -1,61 +1,71 @@ var _ = require('lodash'); var minimatch = require('minimatch'); -var UiApps = require('./UiApps'); +var UiAppCollection = require('./UiAppCollection'); class UiExports { constructor(kbnServer) { - this.kbnServer = kbnServer; - this.apps = new UiApps(this); + this.apps = new UiAppCollection(this); this.aliases = {}; this.exportConsumer = _.memoize(this.exportConsumer); - - kbnServer.plugins.forEach(_.bindKey(this, 'consumePlugin')); + this.consumers = []; + this.bundleProviders = []; } consumePlugin(plugin) { - var self = this; var types = _.keys(plugin.uiExportsSpecs); - if (!types) return false; - var unkown = _.reject(types, self.exportConsumer, self); + var unkown = _.reject(types, this.exportConsumer, this); if (unkown.length) { throw new Error('unknown export types ' + unkown.join(', ') + ' in plugin ' + plugin.id); } - types.forEach(function (type) { - self.exportConsumer(type)(plugin, plugin.uiExportsSpecs[type]); + for (let consumer of this.consumers) { + consumer.consumePlugin && consumer.consumePlugin(plugin); + } + + types.forEach((type) => { + this.exportConsumer(type)(plugin, plugin.uiExportsSpecs[type]); }); } + addConsumer(consumer) { + this.consumers.push(consumer); + } + exportConsumer(type) { - var self = this; + for (let consumer of this.consumers) { + if (!consumer.exportConsumer) continue; + let fn = consumer.exportConsumer(type); + if (fn) return fn; + } + switch (type) { case 'app': - return function (plugin, spec) { - spec = _.defaults({}, spec, { id: plugin.id }); - plugin.app = self.apps.new(spec); + case 'apps': + return (plugin, specs) => { + for (let spec of [].concat(specs || [])) { + this.apps.new(_.defaults({}, spec, { id: plugin.id })); + } }; case 'visTypes': case 'fieldFormats': case 'spyModes': - return function (plugin, spec) { - self.aliases[type] = _.union(self.aliases[type] || [], spec); + return (plugin, spec) => { + this.aliases[type] = _.union(this.aliases[type] || [], spec); }; - case 'modules': - case 'loaders': - case 'noParse': - return function (plugin, spec) { - plugin.uiExportsSpecs[type] = spec; + case 'bundle': + return (plugin, spec) => { + this.bundleProviders.push(spec); }; case 'aliases': - return function (plugin, specs) { - _.forOwn(specs, function (spec, adhocType) { - self.aliases[adhocType] = _.union(self.aliases[adhocType] || [], spec); + return (plugin, specs) => { + _.forOwn(specs, (spec, adhocType) => { + this.aliases[adhocType] = _.union(this.aliases[adhocType] || [], spec); }); }; } @@ -81,8 +91,21 @@ class UiExports { .value(); } - allApps() { - return _.union(this.apps, this.apps.hidden); + getAllApps() { + let { apps } = this; + return [...apps].concat(...apps.hidden); + } + + getApp(id) { + return this.apps.byId[id]; + } + + getHiddenApp(id) { + return this.apps.hidden.byId[id]; + } + + getBundleProviders() { + return this.bundleProviders; } } diff --git a/src/ui/appEntryTemplate.js b/src/ui/appEntryTemplate.js new file mode 100644 index 000000000000..9163b491fbf3 --- /dev/null +++ b/src/ui/appEntryTemplate.js @@ -0,0 +1,37 @@ + +module.exports = require('lodash').template( +` +/** + * Optimized application entry file + * + * This is programatically created and updated, do not modify + * + * context: <%= JSON.stringify(env.context) %> + * includes code from: +<% + + env.pluginInfo.sort().forEach(function (plugin) { + print(\` * - \${plugin}\n\`); + }); + +%> * + */ + +require('ui/chrome'); +<% + +bundle.modules +.filter(function (id) { + return id !== 'ui/chrome'; +}) +.forEach(function (id, i) { + + if (i > 0) print('\\n'); + print(\`require('\${id}');\`); + +}); + +%> +require('ui/chrome').bootstrap(/* xoxo */); +` +); diff --git a/src/ui/index.js b/src/ui/index.js index 93ab35b3be56..5d03cd13091b 100644 --- a/src/ui/index.js +++ b/src/ui/index.js @@ -1,23 +1,51 @@ -module.exports = function (kbnServer, server, config) { +module.exports = async (kbnServer, server, config) => { let _ = require('lodash'); let Boom = require('boom'); let formatUrl = require('url').format; - let { join, resolve } = require('path'); + let { resolve } = require('path'); + let readFile = require('fs').readFileSync; + + let fromRoot = require('../utils/fromRoot'); let UiExports = require('./UiExports'); + let UiBundle = require('./UiBundle'); + let UiBundleCollection = require('./UiBundleCollection'); + let UiBundlerEnv = require('./UiBundlerEnv'); + let loadingGif = readFile(fromRoot('src/ui/public/loading.gif'), { encoding: 'base64'}); let uiExports = kbnServer.uiExports = new UiExports(kbnServer); - let apps = uiExports.apps; - let hiddenApps = uiExports.apps.hidden; + + let bundlerEnv = new UiBundlerEnv(config.get('optimize.bundleDir')); + bundlerEnv.addContext('env', config.get('env.name')); + bundlerEnv.addContext('sourceMaps', config.get('optimize.sourceMaps')); + bundlerEnv.addContext('kbnVersion', config.get('pkg.version')); + bundlerEnv.addContext('buildNum', config.get('pkg.buildNum')); + uiExports.addConsumer(bundlerEnv); + + for (let plugin of kbnServer.plugins) { + uiExports.consumePlugin(plugin); + } + + let bundles = kbnServer.bundles = new UiBundleCollection(bundlerEnv, config.get('optimize.bundleFilter')); + + for (let app of uiExports.getAllApps()) { + bundles.addApp(app); + } + + for (let gen of uiExports.getBundleProviders()) { + let bundle = await gen(UiBundle, bundlerEnv, uiExports.getAllApps()); + if (bundle) bundles.add(bundle); + } // render all views from the ui/views directory server.setupViews(resolve(__dirname, 'views')); + server.exposeStaticFile('/loading.gif', resolve(__dirname, 'public/loading.gif')); // serve the app switcher server.route({ path: '/apps', method: 'GET', handler: function (req, reply) { - let switcher = hiddenApps.byId.switcher; + let switcher = uiExports.getHiddenApp('appSwitcher'); if (!switcher) return reply(Boom.notFound('app switcher not installed')); return reply.renderApp(switcher); } @@ -28,7 +56,7 @@ module.exports = function (kbnServer, server, config) { path: '/api/apps', method: 'GET', handler: function (req, reply) { - return reply(apps); + return reply(uiExports.apps); } }); @@ -37,7 +65,7 @@ module.exports = function (kbnServer, server, config) { method: 'GET', handler: function (req, reply) { let id = req.params.id; - let app = apps.byId[id]; + let app = uiExports.apps.byId[id]; if (!app) return reply(Boom.notFound('Unkown app ' + id)); if (kbnServer.status.isGreen()) { @@ -49,39 +77,22 @@ module.exports = function (kbnServer, server, config) { }); server.decorate('reply', 'renderApp', function (app) { - if (app.requireOptimizeGreen) { - let optimizeStatus = kbnServer.status.get('optimize'); - switch (optimizeStatus && optimizeStatus.state) { - case 'yellow': - return this(` - - - ${optimizeStatus.message} - - `).code(503); - - case 'red': - return this(` - ${optimizeStatus.message} - `).code(500); - } - } - let payload = { app: app, - appCount: apps.length, + appCount: uiExports.apps.size, version: kbnServer.version, buildSha: _.get(kbnServer, 'build.sha', '@@buildSha'), buildNumber: _.get(kbnServer, 'build.number', '@@buildNum'), cacheBust: _.get(kbnServer, 'build.number', ''), kbnIndex: config.get('kibana.index'), - esShardTimeout: config.get('elasticsearch.shardTimeout') + esShardTimeout: config.get('elasticsearch.shardTimeout'), }; return this.view(app.templateName, { app: app, cacheBust: payload.cacheBust, - kibanaPayload: payload + kibanaPayload: payload, + loadingGif: loadingGif, }); }); }; diff --git a/src/ui/public/agg_types/index.js b/src/ui/public/agg_types/index.js index f0b0aa43c681..16395c10f8aa 100644 --- a/src/ui/public/agg_types/index.js +++ b/src/ui/public/agg_types/index.js @@ -58,4 +58,7 @@ define(function (require) { initialSet: aggs.metrics.concat(aggs.buckets) }); }; + + // preload + require('ui/agg_types/AggParams'); }); diff --git a/src/ui/public/chrome/api/angular.js b/src/ui/public/chrome/api/angular.js index bbfa941d94a0..ac30527a3b1d 100644 --- a/src/ui/public/chrome/api/angular.js +++ b/src/ui/public/chrome/api/angular.js @@ -37,7 +37,7 @@ module.exports = function (chrome, internals) { $app.html(internals.rootTemplate); } - $el.append($content); + $el.html($content); }, controllerAs: 'chrome', controller: function ($scope, $rootScope, $location, $http) { diff --git a/src/ui/public/clipboard/__tests__/clipboard.js b/src/ui/public/clipboard/__tests__/clipboard.js index 36a1d23f8cfe..57e30bf68093 100644 --- a/src/ui/public/clipboard/__tests__/clipboard.js +++ b/src/ui/public/clipboard/__tests__/clipboard.js @@ -35,7 +35,6 @@ describe('Clipboard directive', function () { describe.skip('With flash disabled', function () { beforeEach(function () { - this.timeout(5000); sinon.stub(window.ZeroClipboard, 'isFlashUnusable', _.constant(true)); init(); }); @@ -57,7 +56,6 @@ describe('Clipboard directive', function () { describe.skip('With flash enabled', function () { beforeEach(function () { - this.timeout(5000); sinon.stub(window.ZeroClipboard, 'isFlashUnusable', _.constant(false)); init(); }); diff --git a/src/ui/public/loading.gif b/src/ui/public/loading.gif new file mode 100644 index 000000000000..2bde128cc5db Binary files /dev/null and b/src/ui/public/loading.gif differ diff --git a/src/ui/public/styles/base.less b/src/ui/public/styles/base.less index 5e08690140b6..64716f80d2f1 100644 --- a/src/ui/public/styles/base.less +++ b/src/ui/public/styles/base.less @@ -395,10 +395,6 @@ textarea { resize: vertical; } -.initial-load { - margin-top: 60px; -} - .field-collapse-toggle { color: #999; margin-left: 10px !important; diff --git a/src/ui/public/timefilter/__tests__/diff_interval.js b/src/ui/public/timefilter/__tests__/diff_interval.js index 523fe5d0fffb..19bfad1bde47 100644 --- a/src/ui/public/timefilter/__tests__/diff_interval.js +++ b/src/ui/public/timefilter/__tests__/diff_interval.js @@ -6,12 +6,10 @@ describe('Timefilter service', function () { describe('Refresh interval diff watcher', function () { var fn; - var update; - var fetch; - var timefilter; + beforeEach(ngMock.module('kibana')); beforeEach(ngMock.inject(function (Private) { diff --git a/src/ui/public/utils/__tests__/ObjDefine.js b/src/ui/public/utils/__tests__/ObjDefine.js index a0b4def5d349..8b84f51cd7a3 100644 --- a/src/ui/public/utils/__tests__/ObjDefine.js +++ b/src/ui/public/utils/__tests__/ObjDefine.js @@ -90,6 +90,8 @@ describe('ObjDefine Utility', function () { var obj = def.create(); expect(function () { + 'use strict'; // eslint-disable-line strict + obj.name = notval; }).to.throwException(); }); diff --git a/src/ui/public/vislib/__tests__/visualizations/tile_maps/map.js b/src/ui/public/vislib/__tests__/visualizations/tile_maps/map.js index 75e9d4b63e92..3a276f0d0414 100644 --- a/src/ui/public/vislib/__tests__/visualizations/tile_maps/map.js +++ b/src/ui/public/vislib/__tests__/visualizations/tile_maps/map.js @@ -24,7 +24,6 @@ var geoJsonData = require('fixtures/vislib/mock_data/geohash/_geo_json'); // ]; describe('TileMap Map Tests', function () { - this.timeout(0); var $mockMapEl = $('
'); var TileMapMap; var leafletStubs = {}; diff --git a/src/ui/views/uiApp.jade b/src/ui/views/uiApp.jade index 72b90e4e09ca..96e3c445f35e 100644 --- a/src/ui/views/uiApp.jade +++ b/src/ui/views/uiApp.jade @@ -1,7 +1,67 @@ extends ./chrome.jade -block head - link(rel='stylesheet', href='/bundles/#{app.id}.style.css') - block content - script(src='/bundles/#{app.id}.bundle.js' src-map='/bundles/#{app.id}.bundle.js.map') + style. + .ui-app-loading { + width: 33.3%; + margin: 60px auto; + padding: 0 15px; + text-align: center; + font-family: "Lato", "Helvetica Neue", Helvetica, Arial, sans-serif; + color: #444444; + padding-top: 45px; + background-size: 128px; + background-position: top center; + background-repeat: no-repeat; + background-image: url('data:image/gif;base64,#{loadingGif}'); + } + + .ui-app-loading small { + font-size: 65%; + font-weight: 400; + color: #b4bcc2; + } + + + div.ui-app-loading + h1 + strong Kibana + small. +  is loading. Give me a moment here. I'm loading a whole bunch of code. Don't worry, all this good stuff will be cached up for next time! + + script. + window.onload = function () { + + var hideLoadingMessage = /#.*[?&]embed(&|$)/.test(window.location.href); + if (hideLoadingMessage) { + var loading = document.querySelector('.ui-app-loading h1'); + loading.removeChild(loading.lastChild); + } + + var files = [ + '/bundles/commons.style.css', + '/bundles/#{app.id}.style.css', + '/bundles/commons.bundle.js', + '/bundles/#{app.id}.bundle.js' + ]; + + (function next() { + var file = files.shift(); + if (!file) return; + + var type = /\.js$/.test(file) ? 'script' : 'link'; + var dom = document.createElement(type); + dom.setAttribute('async', ''); + + if (type === 'script') { + dom.setAttribute('src', file); + dom.addEventListener('load', next); + document.head.appendChild(dom); + } else { + dom.setAttribute('rel', 'stylesheet'); + dom.setAttribute('href', file); + document.head.appendChild(dom); + next(); + } + }()); + }; diff --git a/src/utils/Collection.js b/src/utils/Collection.js new file mode 100644 index 000000000000..7a18c2913ee7 --- /dev/null +++ b/src/utils/Collection.js @@ -0,0 +1,68 @@ + +let set = Symbol('internal set'); + +module.exports = class Collection { + constructor() { // Set's have a length of 0, mimic that + this[set] = new Set(arguments[0] || []); + } + + /****** + ** Collection API + ******/ + + toArray() { + return [...this.values()]; + } + + toJSON() { + return this.toArray(); + } + + /****** + ** ES Set Api + ******/ + + static get [Symbol.species]() { + return Collection; + } + + get size() { + return this[set].size; + } + + add(value) { + return this[set].add(value); + } + + clear() { + return this[set].clear(); + } + + delete(value) { + return this[set].delete(value); + } + + entries() { + return this[set].entries(); + } + + forEach(callbackFn, thisArg) { + return this[set].forEach(callbackFn, thisArg); + } + + has(value) { + return this[set].has(value); + } + + keys() { + return this[set].keys(); + } + + values() { + return this[set].values(); + } + + [Symbol.iterator]() { + return this[set][Symbol.iterator](); + } +}; diff --git a/src/utils/findSourceFiles.js b/src/utils/findSourceFiles.js new file mode 100644 index 000000000000..4c7de2065727 --- /dev/null +++ b/src/utils/findSourceFiles.js @@ -0,0 +1,41 @@ +let { chain, memoize } = require('lodash'); +let { resolve } = require('path'); +let { map, fromNode } = require('bluebird'); + +let fromRoot = require('./fromRoot'); +let { Glob } = require('glob'); + + +let findSourceFiles = async (patterns, cwd = fromRoot('.')) => { + patterns = [].concat(patterns || []); + + let matcheses = await map(patterns, async pattern => { + return await fromNode(cb => { + let g = new Glob(pattern, { + cwd: cwd, + ignore: [ + 'node_modules/**/*', + 'bower_components/**/*', + '**/_*.js' + ], + symlinks: findSourceFiles.symlinks, + statCache: findSourceFiles.statCache, + realpathCache: findSourceFiles.realpathCache, + cache: findSourceFiles.cache + }, cb); + }); + }); + + return chain(matcheses) + .flatten() + .uniq() + .map(match => resolve(cwd, match)) + .value(); +}; + +findSourceFiles.symlinks = {}; +findSourceFiles.statCache = {}; +findSourceFiles.realpathCache = {}; +findSourceFiles.cache = {}; + +module.exports = findSourceFiles; diff --git a/tasks/config/run.js b/tasks/config/run.js index d0496dbf9796..56ae1a7da48c 100644 --- a/tasks/config/run.js +++ b/tasks/config/run.js @@ -3,15 +3,20 @@ module.exports = function (grunt) { let root = p => resolve(__dirname, '../../', p); return { - devServer: { + testServer: { options: { wait: false, - ready: /\[optimize\]\[status\] Status changed from [a-zA-Z]+ to green/, + ready: /Server running/, quiet: false, failOnError: false }, cmd: './bin/kibana', - args: ['--dev', '--no-watch', '--logging.json=false'] + args: [ + '--env.name=development', + '--logging.json=false', + '--optimize.tests=true', + '--optimize.lazy=false' + ] } }; diff --git a/tasks/config/simplemocha.js b/tasks/config/simplemocha.js index 0678b66c9d93..563dab4fddf8 100644 --- a/tasks/config/simplemocha.js +++ b/tasks/config/simplemocha.js @@ -1,6 +1,7 @@ module.exports = { options: { - timeout: 2000, + timeout: 10000, + slow: 5000, ignoreLeaks: false, reporter: 'dot' }, diff --git a/tasks/lintStagedFiles.js b/tasks/lintStagedFiles.js index 166048f0e6de..3396abcc62c9 100644 --- a/tasks/lintStagedFiles.js +++ b/tasks/lintStagedFiles.js @@ -14,7 +14,7 @@ module.exports = function (grunt) { diff(['--name-only', '--cached']) .then(function (files) { // match these patterns - var patterns = grunt.config.get('lintThese'); + var patterns = grunt.config.get('eslint.files.src'); files = files.split('\n').filter(Boolean).map(function (file) { return resolve(root, file); }); diff --git a/tasks/maybeStartKibana.js b/tasks/maybeStartTestServer.js similarity index 94% rename from tasks/maybeStartKibana.js rename to tasks/maybeStartTestServer.js index 24717f44ff12..343a91a1f240 100644 --- a/tasks/maybeStartKibana.js +++ b/tasks/maybeStartTestServer.js @@ -50,9 +50,9 @@ module.exports = function (grunt) { }; }; - grunt.registerTask('maybeStartKibana', maybeStartServer({ + grunt.registerTask('maybeStartTestServer', maybeStartServer({ name: 'kibana-server', port: grunt.option('port') || 5601, - tasks: ['run:devServer'] + tasks: ['run:testServer'] })); }; diff --git a/tasks/test.js b/tasks/test.js index 6aa1f121d7ad..9d09312b98d4 100644 --- a/tasks/test.js +++ b/tasks/test.js @@ -8,7 +8,7 @@ module.exports = function (grunt) { grunt.task.run(_.compact([ 'eslint:source', - 'maybeStartKibana', + 'maybeStartTestServer', 'simplemocha:all', 'karma:unit' ])); @@ -16,14 +16,14 @@ module.exports = function (grunt) { grunt.registerTask('quick-test', function () { grunt.task.run([ - 'maybeStartKibana', + 'maybeStartTestServer', 'simplemocha:all', 'karma:unit' ]); }); grunt.registerTask('test:watch', [ - 'maybeStartKibana', + 'maybeStartTestServer', 'watch:test' ]); };