[pack installer] implemented changes from PR review

This commit is contained in:
Jim Unger 2016-03-11 16:14:48 -06:00
parent c86b13edc2
commit 416c9c5111
19 changed files with 279 additions and 256 deletions

View file

@ -3,8 +3,8 @@ import pkg from '../utils/package_json';
import Command from './command';
import serveCommand from './serve/serve';
let argv = process.env.kbnWorkerArgv ? JSON.parse(process.env.kbnWorkerArgv) : process.argv.slice();
let program = new Command('bin/kibana');
const argv = process.env.kbnWorkerArgv ? JSON.parse(process.env.kbnWorkerArgv) : process.argv.slice();
const program = new Command('bin/kibana');
program
.version(pkg.version)
@ -20,7 +20,7 @@ program
.command('help <command>')
.description('Get the help for a specific command')
.action(function (cmdName) {
var cmd = _.find(program.commands, { _name: cmdName });
const cmd = _.find(program.commands, { _name: cmdName });
if (!cmd) return program.error(`unknown command ${cmdName}`);
cmd.help();
});
@ -32,7 +32,7 @@ program
});
// check for no command name
var subCommand = argv[2] && !String(argv[2][0]).match(/^-|^\.|\//);
const subCommand = argv[2] && !String(argv[2][0]).match(/^-|^\.|\//);
if (!subCommand) {
if (_.intersection(argv.slice(2), ['-h', '--help']).length) {

View file

@ -11,8 +11,8 @@ let program = new Command('bin/kibana-plugin');
program
.version(pkg.version)
.description(
'Kibana is an open source (Apache Licensed), browser ' +
'based analytics and search dashboard for Elasticsearch.'
'The Kibana plugin manager enables you to install and remove plugins that ' +
'provide additional functionality to Kibana'
);
listCommand(program);

View file

@ -3,7 +3,7 @@ import sinon from 'sinon';
import fs from 'fs';
import rimraf from 'rimraf';
import { cleanPrevious, cleanError } from '../cleanup';
import { cleanPrevious, cleanArtifacts } from '../cleanup';
import Logger from '../../lib/logger';
describe('kibana cli', function () {
@ -108,7 +108,7 @@ describe('kibana cli', function () {
});
describe('cleanError', function () {
describe('cleanArtifacts', function () {
let logger;
beforeEach(function () {
@ -122,7 +122,7 @@ describe('kibana cli', function () {
it('should attempt to delete the working directory', function () {
sinon.stub(rimraf, 'sync');
cleanError(settings);
cleanArtifacts(settings);
expect(rimraf.sync.calledWith(settings.workingPath)).to.be(true);
});
@ -131,7 +131,7 @@ describe('kibana cli', function () {
throw new Error('Something bad happened.');
});
expect(cleanError).withArgs(settings).to.not.throwError();
expect(cleanArtifacts).withArgs(settings).to.not.throwError();
});
});

View file

@ -61,11 +61,11 @@ describe('kibana cli', function () {
describe('http downloader', function () {
it('should throw an ENOTFOUND error for a http ulr that returns 404', function () {
const couchdb = nock('http://www.files.com')
const couchdb = nock('http://example.com')
.get('/plugin.tar.gz')
.reply(404);
const sourceUrl = 'http://www.files.com/plugin.tar.gz';
const sourceUrl = 'http://example.com/plugin.tar.gz';
return _downloadSingle(settings, logger, sourceUrl)
.then(shouldReject, function (err) {
@ -87,7 +87,7 @@ describe('kibana cli', function () {
it('should download a file from a valid http url', function () {
const filePath = join(__dirname, 'replies/banana.jpg');
const couchdb = nock('http://www.files.com')
const couchdb = nock('http://example.com')
.defaultReplyHeaders({
'content-length': '341965',
'content-type': 'application/zip'
@ -95,7 +95,7 @@ describe('kibana cli', function () {
.get('/plugin.zip')
.replyWithFile(200, filePath);
const sourceUrl = 'http://www.files.com/plugin.zip';
const sourceUrl = 'http://example.com/plugin.zip';
return _downloadSingle(settings, logger, sourceUrl)
.then(function () {
@ -136,13 +136,13 @@ describe('kibana cli', function () {
it('should loop through bad urls until it finds a good one.', function () {
const filePath = join(__dirname, 'replies/test_plugin.zip');
settings.urls = [
'http://www.files.com/badfile1.tar.gz',
'http://www.files.com/badfile2.tar.gz',
'http://example.com/badfile1.tar.gz',
'http://example.com/badfile2.tar.gz',
'I am a bad uri',
'http://www.files.com/goodfile.tar.gz'
'http://example.com/goodfile.tar.gz'
];
const couchdb = nock('http://www.files.com')
const couchdb = nock('http://example.com')
.defaultReplyHeaders({
'content-length': '10'
})
@ -166,13 +166,13 @@ describe('kibana cli', function () {
it('should stop looping through urls when it finds a good one.', function () {
const filePath = join(__dirname, 'replies/test_plugin.zip');
settings.urls = [
'http://www.files.com/badfile1.tar.gz',
'http://www.files.com/badfile2.tar.gz',
'http://www.files.com/goodfile.tar.gz',
'http://www.files.com/badfile3.tar.gz'
'http://example.com/badfile1.tar.gz',
'http://example.com/badfile2.tar.gz',
'http://example.com/goodfile.tar.gz',
'http://example.com/badfile3.tar.gz'
];
const couchdb = nock('http://www.files.com')
const couchdb = nock('http://example.com')
.defaultReplyHeaders({
'content-length': '10'
})
@ -196,12 +196,12 @@ describe('kibana cli', function () {
it('should throw an error when it doesn\'t find a good url.', function () {
settings.urls = [
'http://www.files.com/badfile1.tar.gz',
'http://www.files.com/badfile2.tar.gz',
'http://www.files.com/badfile3.tar.gz'
'http://example.com/badfile1.tar.gz',
'http://example.com/badfile2.tar.gz',
'http://example.com/badfile3.tar.gz'
];
const couchdb = nock('http://www.files.com')
const couchdb = nock('http://example.com')
.defaultReplyHeaders({
'content-length': '10'
})

View file

@ -21,7 +21,7 @@ export function cleanPrevious(settings, logger) {
});
};
export function cleanError(settings) {
export function cleanArtifacts(settings) {
// delete the working directory.
// At this point we're bailing, so swallow any errors on delete.
try {

View file

@ -1,4 +1,3 @@
import _ from 'lodash';
import downloadHttpFile from './downloaders/http';
import downloadLocalFile from './downloaders/file';
import { parse } from 'url';

View file

@ -4,21 +4,21 @@ import Logger from '../lib/logger';
import pkg from '../../utils/packageJson';
import { parse, parseMilliseconds } from './settings';
export default function pluginInstall(program) {
function processCommand(command, options) {
let settings;
try {
settings = parse(command, options, pkg);
} catch (ex) {
//The logger has not yet been initialized.
console.error(ex.message);
process.exit(64); // eslint-disable-line no-process-exit
}
const logger = new Logger(settings);
install(settings, logger);
function processCommand(command, options) {
let settings;
try {
settings = parse(command, options, pkg);
} catch (ex) {
//The logger has not yet been initialized.
console.error(ex.message);
process.exit(64); // eslint-disable-line no-process-exit
}
const logger = new Logger(settings);
install(settings, logger);
}
export default function pluginInstall(program) {
program
.command('install <plugin/url>')
.option('-q, --quiet', 'Disable all process messaging except errors')

View file

@ -1,25 +1,25 @@
import _ from 'lodash';
import { download } from './download';
import Promise from 'bluebird';
import { cleanPrevious, cleanError } from './cleanup';
import { cleanPrevious, cleanArtifacts } from './cleanup';
import { extract, getPackData } from './pack';
import { sync as rimrafSync } from 'rimraf';
import { statSync, renameSync } from 'fs';
import { renameSync } from 'fs';
import { existingInstall, rebuildCache, checkVersion } from './kibana';
import mkdirp from 'mkdirp';
const mkdirp = Promise.promisify(require('mkdirp'));
const mkdir = Promise.promisify(mkdirp);
export default async function install(settings, logger) {
try {
await cleanPrevious(settings, logger);
await mkdirp(settings.workingPath);
await mkdir(settings.workingPath);
await download(settings, logger);
await getPackData(settings, logger);
await extract (settings, logger);
await extract(settings, logger);
rimrafSync(settings.tempArchiveFile);
@ -34,7 +34,7 @@ export default async function install(settings, logger) {
logger.log('Plugin installation complete');
} catch (err) {
logger.error(`Plugin installation was unsuccessful due to error "${err.message}"`);
cleanError(settings);
cleanArtifacts(settings);
process.exit(70); // eslint-disable-line no-process-exit
}
}

View file

@ -4,32 +4,29 @@ import { resolve } from 'path';
import { sync as rimrafSync } from 'rimraf';
import validate from 'validate-npm-package-name';
//*****************************************
//Return a list of package.json files in the archive
//*****************************************
/**
* Returns an array of package objects. There will be one for each of
* package.json files in the archive
* @param {object} settings - a plugin installer settings object
*/
async function listPackages(settings) {
const regExp = new RegExp('(kibana/([^/]+))/package.json', 'i');
const archiveFiles = await listFiles(settings.tempArchiveFile);
let packages = archiveFiles.map((file) => {
file = file.replace(/\\/g, '/');
const matches = file.match(regExp);
if (matches) {
return {
file: matches[0],
folder: matches[2]
};
}
});
packages = _.chain(packages).compact().uniq().value();
return packages;
return _(archiveFiles)
.map(file => file.replace(/\\/g, '/'))
.map(file => file.match(regExp))
.compact()
.map(([ file, _, folder ]) => ({ file, folder }))
.uniq()
.value();
}
//*****************************************
//Extract the package.json files into the workingPath
//*****************************************
/**
* Extracts the package.json files into the workingPath
* @param {object} settings - a plugin installer settings object
* @param {array} packages - array of package objects from listPackages()
*/
async function extractPackageFiles(settings, packages) {
const filter = {
files: packages.map((pkg) => pkg.file)
@ -37,31 +34,35 @@ async function extractPackageFiles(settings, packages) {
await extractFiles(settings.tempArchiveFile, settings.workingPath, 0, filter);
}
//*****************************************
//Extract the package.json files into the workingPath
//*****************************************
function deletePackageFiles(settings, packages) {
packages.forEach((pkg) => {
const fullPath = resolve(settings.workingPath, 'kibana');
rimrafSync(fullPath);
});
/**
* Deletes the package.json files created by extractPackageFiles()
* @param {object} settings - a plugin installer settings object
*/
function deletePackageFiles(settings) {
const fullPath = resolve(settings.workingPath, 'kibana');
rimrafSync(fullPath);
}
//*****************************************
//Check the plugin name
//*****************************************
function validatePackageName(plugin) {
/**
* Checks the plugin name. Will throw an exception if it does not meet
* npm package naming conventions
* @param {object} plugin - a package object from listPackages()
*/
function assertValidPackageName(plugin) {
const validation = validate(plugin.name);
if (!validation.validForNewPackages) {
throw new Error(`Invalid plugin name [${plugin.name}] in package.json`);
}
}
//*****************************************
//Examine each package.json file to determine the plugin name,
//version, and platform.
//*****************************************
async function readPackageData(settings, packages) {
/**
* Examine each package.json file to determine the plugin name,
* version, and platform. Mutates the package objects in the packages array
* @param {object} settings - a plugin installer settings object
* @param {array} packages - array of package objects from listPackages()
*/
async function mergePackageData(settings, packages) {
return packages.map((pkg) => {
const fullPath = resolve(settings.workingPath, pkg.file);
const packageInfo = require(fullPath);
@ -78,11 +79,12 @@ async function readPackageData(settings, packages) {
});
}
//*****************************************
//Extracts the first plugin in the archive.
//This will need to be changed in later versions of the pack installer
//that allow for the installation of more than one plugin at once.
//*****************************************
/**
* Extracts the first plugin in the archive.
* NOTE: This will need to be changed in later versions of the pack installer
* that allow for the installation of more than one plugin at once.
* @param {object} settings - a plugin installer settings object
*/
async function extractArchive(settings) {
const filter = {
paths: [ settings.plugins[0].folder ]
@ -90,20 +92,23 @@ async function extractArchive(settings) {
await extractFiles(settings.tempArchiveFile, settings.workingPath, 2, filter);
}
//*****************************************
//Returns the detailed information about each kibana plugin in the
//pack.
//TODO: If there are platform specific folders, determine which one to use.
//*****************************************
/**
* Returns the detailed information about each kibana plugin in the pack.
* TODO: If there are platform specific folders, determine which one to use.
* @param {object} settings - a plugin installer settings object
* @param {object} logger - a plugin installer logger object
*/
export async function getPackData(settings, logger) {
let packages;
try {
logger.log('Retrieving metadata from plugin archive');
packages = await listPackages(settings);
await extractPackageFiles(settings, packages);
await readPackageData(settings, packages);
await deletePackageFiles(settings, packages);
await mergePackageData(settings, packages);
await deletePackageFiles(settings);
} catch (err) {
logger.error(err);
throw new Error('Error retrieving metadata from plugin archive');
@ -112,7 +117,7 @@ export async function getPackData(settings, logger) {
if (packages.length === 0) {
throw new Error('No kibana plugins found in archive');
}
packages.forEach(validatePackageName);
packages.forEach(assertValidPackageName);
settings.plugins = packages;
}

View file

@ -1,39 +1,38 @@
/*
Generates file transfer progress messages
*/
export default function Progress(logger) {
const self = this;
/**
* Generates file transfer progress messages
*/
export default class Progress {
self.dotCount = 0;
self.runningTotal = 0;
self.totalSize = 0;
self.logger = logger;
}
constructor(logger) {
const self = this;
Progress.prototype.init = function (size) {
const self = this;
self.totalSize = size;
const totalDesc = self.totalSize || 'unknown number of';
self.logger.log(`Transferring ${totalDesc} bytes`, true);
};
Progress.prototype.progress = function (size) {
const self = this;
if (!self.totalSize) return;
self.runningTotal += size;
let newDotCount = Math.round(self.runningTotal / self.totalSize * 100 / 5);
if (newDotCount > 20) newDotCount = 20;
for (let i = 0; i < (newDotCount - self.dotCount); i++) {
self.logger.log('.', true);
self.dotCount = 0;
self.runningTotal = 0;
self.totalSize = 0;
self.logger = logger;
}
self.dotCount = newDotCount;
};
Progress.prototype.complete = function () {
const self = this;
self.logger.log(`Transfer complete`, false);
};
init(size) {
this.totalSize = size;
const totalDesc = this.totalSize || 'unknown number of';
this.logger.log(`Transferring ${totalDesc} bytes`, true);
}
progress(size) {
if (!this.totalSize) return;
this.runningTotal += size;
let newDotCount = Math.round(this.runningTotal / this.totalSize * 100 / 5);
if (newDotCount > 20) newDotCount = 20;
for (let i = 0; i < (newDotCount - this.dotCount); i++) {
this.logger.log('.', true);
}
this.dotCount = newDotCount;
}
complete() {
this.logger.log(`Transfer complete`, false);
}
}

View file

@ -3,8 +3,7 @@ import { intersection } from 'lodash';
import { resolve } from 'path';
import { arch, platform } from 'os';
function generateUrls(settings) {
const { version, plugin } = settings;
function generateUrls({ version, plugin }) {
return [
plugin,
`https://download.elastic.co/packs/${plugin}/${plugin}-${version}.zip`
@ -15,7 +14,7 @@ export function parseMilliseconds(val) {
let result;
try {
let timeVal = expiry(val);
const timeVal = expiry(val);
result = timeVal.asMilliseconds();
} catch (ex) {
result = 0;
@ -26,13 +25,13 @@ export function parseMilliseconds(val) {
export function parse(command, options, kbnPackage) {
const settings = {
timeout: options.timeout ? options.timeout : 0,
quiet: options.quiet ? options.quiet : false,
silent: options.silent ? options.silent : false,
config: options.config ? options.config : '',
timeout: options.timeout || 0,
quiet: options.quiet || false,
silent: options.silent || false,
config: options.config || '',
plugin: command,
version: kbnPackage.version,
pluginDir: options.pluginDir ? options.pluginDir : ''
pluginDir: options.pluginDir || ''
};
settings.urls = generateUrls(settings);

View file

@ -1,45 +1,62 @@
import _ from 'lodash';
import DecompressZip from '@bigfunger/decompress-zip';
//***********************************************
//Creates a filter function to be consumed by extractFiles
//
//filter: an object with either a files or paths property.
//filter.files: an array of full file paths to extract. Should match
// exactly a value from listFiles
//filter.paths: an array of root paths from the archive. All files and
// folders will be extracted recursively using these paths as roots.
//***********************************************
const SYMBOLIC_LINK = 'SymbolicLink';
/**
* Creates a filter function to be consumed by extractFiles that filters by
* an array of files
* @param {array} files - an array of full file paths to extract. Should match
* exactly a value from listFiles
*/
function extractFilterFromFiles(files) {
const filterFiles = files.map((file) => file.replace(/\\/g, '/'));
return function filterByFiles(file) {
if (file.type === SYMBOLIC_LINK) return false;
const path = file.path.replace(/\\/g, '/');
return _.includes(filterFiles, path);
};
}
/**
* Creates a filter function to be consumed by extractFiles that filters by
* an array of root paths
* @param {array} paths - an array of root paths from the archive. All files and
* folders will be extracted recursively using these paths as roots.
*/
function extractFilterFromPaths(paths) {
return function filterByRootPath(file) {
if (file.type === SYMBOLIC_LINK) return false;
return paths.some(path => {
const regex = new RegExp(`${path}($|/)`, 'i');
return file.parent.match(regex);
});
};
}
/**
* Creates a filter function to be consumed by extractFiles
* @param {object} filter - an object with either a files or paths property.
*/
function extractFilter(filter) {
if (filter.files) {
const filterFiles = filter.files.map((file) => file.replace(/\\/g, '/'));
return function filterByFiles(file) {
if (file.type === 'SymbolicLink') return false;
const path = file.path.replace(/\\/g, '/');
return !!(_.indexOf(filterFiles, path) !== -1);
};
}
if (filter.paths) {
return function filterByRootPath(file) {
if (file.type === 'SymbolicLink') return false;
let include = false;
filter.paths.forEach((path) => {
const regex = new RegExp(`${path}($|/)`, 'i');
if ((file.parent.match(regex)) && file.type !== 'SymbolicLink') {
include = true;
}
});
return include;
};
}
if (filter.files) return extractFilterFromFiles(filter.files);
if (filter.paths) return extractFilterFromPaths(filter.paths);
return _.noop;
}
/**
* Extracts files from a zip archive to a file path using a filter function
* @param {string} zipPath - file path to a zip archive
* @param {string} targetPath - directory path to where the files should
* extracted
* @param {integer} strip - Number of nested directories within the archive
* that should be ignored when determining the target path of an archived
* file.
* @param {function} filter - A function that accepts a single parameter 'file'
* and returns true if the file should be extracted from the archive
*/
export async function extractFiles(zipPath, targetPath, strip, filter) {
await new Promise((resolve, reject) => {
const unzipper = new DecompressZip(zipPath);
@ -56,6 +73,11 @@ export async function extractFiles(zipPath, targetPath, strip, filter) {
});
}
/**
* Returns all files within an archive
* @param {string} zipPath - file path to a zip archive
* @returns {array} all files within an archive with their relative paths
*/
export async function listFiles(zipPath) {
return await new Promise((resolve, reject) => {
const unzipper = new DecompressZip(zipPath);

View file

@ -1,45 +1,46 @@
export default function Logger(settings) {
const self = this;
/**
* Logs messages and errors
*/
export default class Logger {
constructor(settings) {
this.previousLineEnded = true;
this.silent = !!settings.silent;
this.quiet = !!settings.quiet;
}
log(data, sameLine) {
if (this.silent || this.quiet) return;
if (!sameLine && !this.previousLineEnded) {
process.stdout.write('\n');
}
//if data is a stream, pipe it.
if (data.readable) {
data.pipe(process.stdout);
return;
}
process.stdout.write(data);
if (!sameLine) process.stdout.write('\n');
this.previousLineEnded = !sameLine;
}
error(data) {
if (this.silent) return;
if (!this.previousLineEnded) {
process.stderr.write('\n');
}
//if data is a stream, pipe it.
if (data.readable) {
data.pipe(process.stderr);
return;
}
process.stderr.write(`${data}\n`);
this.previousLineEnded = true;
};
self.previousLineEnded = true;
self.silent = !!settings.silent;
self.quiet = !!settings.quiet;
}
Logger.prototype.log = function (data, sameLine) {
const self = this;
if (self.silent || self.quiet) return;
if (!sameLine && !self.previousLineEnded) {
process.stdout.write('\n');
}
//if data is a stream, pipe it.
if (data.readable) {
data.pipe(process.stdout);
return;
}
process.stdout.write(data);
if (!sameLine) process.stdout.write('\n');
self.previousLineEnded = !sameLine;
};
Logger.prototype.error = function (data) {
const self = this;
if (self.silent) return;
if (!self.previousLineEnded) {
process.stderr.write('\n');
}
//if data is a stream, pipe it.
if (data.readable) {
data.pipe(process.stderr);
return;
}
process.stderr.write(`${data}\n`);
self.previousLineEnded = true;
};

View file

@ -3,21 +3,21 @@ import list from './list';
import Logger from '../lib/logger';
import { parse } from './settings';
export default function pluginList(program) {
function processCommand(command, options) {
let settings;
try {
settings = parse(command, options);
} catch (ex) {
//The logger has not yet been initialized.
console.error(ex.message);
process.exit(64); // eslint-disable-line no-process-exit
}
const logger = new Logger(settings);
list(settings, logger);
function processCommand(command, options) {
let settings;
try {
settings = parse(command, options);
} catch (ex) {
//The logger has not yet been initialized.
console.error(ex.message);
process.exit(64); // eslint-disable-line no-process-exit
}
const logger = new Logger(settings);
list(settings, logger);
}
export default function pluginList(program) {
program
.command('list')
.option(

View file

@ -10,5 +10,5 @@ export default function list(settings, logger) {
logger.log(filename);
}
});
logger.log('');
logger.log(''); //intentional blank line for aesthetics
}

View file

@ -2,7 +2,7 @@ import { resolve } from 'path';
export function parse(command, options) {
const settings = {
pluginDir: command.pluginDir ? command.pluginDir : ''
pluginDir: command.pluginDir || ''
};
return settings;

View file

@ -10,15 +10,13 @@ import { writeFileSync } from 'fs';
describe('kibana cli', function () {
describe('plugin lister', function () {
describe('plugin remover', function () {
const pluginDir = join(__dirname, '.test.data');
let processExitStub;
let logger;
const settings = {
pluginDir: pluginDir
};
const settings = { pluginDir };
beforeEach(function () {
processExitStub = sinon.stub(process, 'exit');

View file

@ -3,21 +3,21 @@ import remove from './remove';
import Logger from '../lib/logger';
import { parse } from './settings';
export default function pluginList(program) {
function processCommand(command, options) {
let settings;
try {
settings = parse(command, options);
} catch (ex) {
//The logger has not yet been initialized.
console.error(ex.message);
process.exit(64); // eslint-disable-line no-process-exit
}
const logger = new Logger(settings);
remove(settings, logger);
function processCommand(command, options) {
let settings;
try {
settings = parse(command, options);
} catch (ex) {
//The logger has not yet been initialized.
console.error(ex.message);
process.exit(64); // eslint-disable-line no-process-exit
}
const logger = new Logger(settings);
remove(settings, logger);
}
export default function pluginRemove(program) {
program
.command('remove <plugin>')
.option('-q, --quiet', 'Disable all process messaging except errors')

View file

@ -2,10 +2,10 @@ import { resolve } from 'path';
export function parse(command, options) {
const settings = {
quiet: options.quiet ? options.quiet : false,
silent: options.silent ? options.silent : false,
config: options.config ? options.config : '',
pluginDir: options.pluginDir ? options.pluginDir : '',
quiet: options.quiet || false,
silent: options.silent || false,
config: options.config || '',
pluginDir: options.pluginDir || '',
plugin: command
};