maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[kbn/pm] Allow to include/exclude projects in kbn watch. (#17421)

Browse source
This commit is contained in:
Aleh Zasypkin 2018-04-12 17:40:11 +02:00 committed by GitHub
parent b6758083fd
commit 8b5330ccec
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 474 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
packages/kbn-pm/dist/index.js vendored
View file

@ -6459,7 +6459,7 @@ Object.defineProperty(exports, "__esModule", {
exports.getProjects = undefined;
let getProjects = exports.getProjects = (() => {
var _ref = _asyncToGenerator(function* (rootPath, projectsPathsPatterns) {
var _ref = _asyncToGenerator(function* (rootPath, projectsPathsPatterns, { include = [], exclude = [] } = {}) {
const projects = new Map();
for (const pattern of projectsPathsPatterns) {
const pathsToProcess = yield packagesFromGlobPattern({ pattern, rootPath });
@ -6467,6 +6467,10 @@ let getProjects = exports.getProjects = (() => {
const projectConfigPath = normalize(filePath);
const projectDir = _path2.default.dirname(projectConfigPath);
const project = yield _project.Project.fromPath(projectDir);
const excludeProject = exclude.includes(project.name) || include.length > 0 && !include.includes(project.name);
if (excludeProject) {
continue;
}
if (projects.has(project.name)) {
throw new _errors.CliError(`There are multiple projects with the same name [${project.name}]`, {
name: project.name,
@ -6544,36 +6548,30 @@ function buildProjectGraph(projects) {
return projectGraph;
}
function topologicallyBatchProjects(projectsToBatch, projectGraph) {
// We're going to be chopping stuff out of this array, so copy it.
const projects = [...projectsToBatch.values()];
// This maps project names to the number of projects that depend on them.
// As projects are completed their names will be removed from this object.
const refCounts = {};
projects.forEach(pkg => projectGraph.get(pkg.name).forEach(dep => {
if (!refCounts[dep.name]) refCounts[dep.name] = 0;
refCounts[dep.name]++;
}));
// We're going to be chopping stuff out of this list, so copy it.
const projectToBatchNames = new Set(projectsToBatch.keys());
const batches = [];
while (projects.length > 0) {
while (projectToBatchNames.size > 0) {
// Get all projects that have no remaining dependencies within the repo
// that haven't yet been picked.
const batch = projects.filter(pkg => {
const projectDeps = projectGraph.get(pkg.name);
return projectDeps.filter(dep => refCounts[dep.name] > 0).length === 0;
});
const batch = [];
for (const projectName of projectToBatchNames) {
const projectDeps = projectGraph.get(projectName);
const hasNotBatchedDependencies = projectDeps.some(dep => projectToBatchNames.has(dep.name));
if (!hasNotBatchedDependencies) {
batch.push(projectsToBatch.get(projectName));
}
}
// If we weren't able to find a project with no remaining dependencies,
// then we've encountered a cycle in the dependency graph.
const hasCycles = projects.length > 0 && batch.length === 0;
const hasCycles = batch.length === 0;
if (hasCycles) {
const cycleProjectNames = projects.map(p => p.name);
const cycleProjectNames = [...projectToBatchNames];
const message = 'Encountered a cycle in the dependency graph. Projects in cycle are:\n' + cycleProjectNames.join(', ');
throw new _errors.CliError(message);
}
batches.push(batch);
batch.forEach(pkg => {
delete refCounts[pkg.name];
projects.splice(projects.indexOf(pkg), 1);
});
batch.forEach(project => projectToBatchNames.delete(project.name));
}
return batches;
}
@ -36264,7 +36262,9 @@ let run = exports.run = (() => {
}
const options = (0, _getopts2.default)(argv, {
alias: {
h: 'help'
h: 'help',
i: 'include',
e: 'exclude'
}
});
const args = options._;
@ -36327,6 +36327,8 @@ function help() {
Global options:
-e, --exclude Exclude specified project. Can be specified multiple times to exclude multiple projects, e.g. '-e kibana -e @kbn/pm'.
-i, --include Include only specified projects. If left unspecified, it defaults to including all projects.
--skip-kibana Do not include the root Kibana project when running command.
--skip-kibana-extra Filter all plugins in ../kibana-extra when running command.
`);
@ -48765,19 +48767,24 @@ const WatchCommand = exports.WatchCommand = {
description: 'Runs `kbn:watch` script for every project.',
run(projects, projectGraph) {
return _asyncToGenerator(function* () {
const projectsWithWatchScript = new Map();
const projectsToWatch = new Map();
for (const project of projects.values()) {
// We can't watch project that doesn't have `kbn:watch` script.
if (project.hasScript(watchScriptName)) {
projectsWithWatchScript.set(project.name, project);
projectsToWatch.set(project.name, project);
}
}
const projectNames = Array.from(projectsWithWatchScript.keys());
if (projectsToWatch.size === 0) {
console.log(_chalk2.default.red(`\nThere are no projects to watch found. Make sure that projects define 'kbn:watch' script in 'package.json'.\n`));
return;
}
const projectNames = Array.from(projectsToWatch.keys());
console.log(_chalk2.default.bold(_chalk2.default.green(`Running ${watchScriptName} scripts for [${projectNames.join(', ')}].`)));
// Kibana should always be run the last, so we don't rely on automatic
// topological batching and push it to the last one-entry batch manually.
projectsWithWatchScript.delete(kibanaProjectName);
const batchedProjects = (0, _projects.topologicallyBatchProjects)(projectsWithWatchScript, projectGraph);
if (projects.has(kibanaProjectName)) {
const shouldWatchKibanaProject = projectsToWatch.delete(kibanaProjectName);
const batchedProjects = (0, _projects.topologicallyBatchProjects)(projectsToWatch, projectGraph);
if (shouldWatchKibanaProject) {
batchedProjects.push([projects.get(kibanaProjectName)]);
}
yield (0, _parallelize.parallelizeBatches)(batchedProjects, (() => {
@ -59802,7 +59809,14 @@ let runCommand = exports.runCommand = (() => {
try {
console.log(_chalk2.default.bold(`Running [${_chalk2.default.green(command.name)}] command from [${_chalk2.default.yellow(config.rootPath)}]:\n`));
const projectPaths = (0, _config.getProjectPaths)(config.rootPath, config.options);
const projects = yield (0, _projects.getProjects)(config.rootPath, projectPaths);
const projects = yield (0, _projects.getProjects)(config.rootPath, projectPaths, {
exclude: toArray(config.options.exclude),
include: toArray(config.options.include)
});
if (projects.size === 0) {
console.log(_chalk2.default.red(`There are no projects found. Double check project name(s) in '-i/--include' and '-e/--exclude' filters.\n`));
return process.exit(1);
}
const projectGraph = (0, _projects.buildProjectGraph)(projects);
console.log(_chalk2.default.bold(`Found [${_chalk2.default.green(projects.size.toString())}] projects:\n`));
console.log((0, _projects_tree.renderProjectsTree)(config.rootPath, projects));
@ -59857,6 +59871,13 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
function toArray(value) {
if (value == null) {
return [];
}
return Array.isArray(value) ? value : [value];
}
/***/ }),
/* 735 */
/***/ (function(module, exports, __webpack_require__) {

113
packages/kbn-pm/src/__snapshots__/run.test.ts.snap Normal file
View file

@ -0,0 +1,113 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`excludes project if single \`exclude\` filter is specified 1`] = `
Object {
"graph": Object {
"bar": Array [],
"baz": Array [
"bar",
],
"kibana": Array [],
"quux": Array [
"bar",
"baz",
],
"with-additional-projects": Array [],
},
"projects": Array [
"bar",
"kibana",
"with-additional-projects",
"baz",
"quux",
],
}
`;
exports[`excludes projects if multiple \`exclude\` filter are specified 1`] = `
Object {
"graph": Object {
"kibana": Array [],
"quux": Array [],
"with-additional-projects": Array [],
},
"projects": Array [
"kibana",
"with-additional-projects",
"quux",
],
}
`;
exports[`includes only projects specified in multiple \`include\` filters 1`] = `
Object {
"graph": Object {
"bar": Array [
"foo",
],
"baz": Array [
"bar",
],
"foo": Array [],
},
"projects": Array [
"bar",
"foo",
"baz",
],
}
`;
exports[`includes single project if single \`include\` filter is specified 1`] = `
Object {
"graph": Object {
"foo": Array [],
},
"projects": Array [
"foo",
],
}
`;
exports[`passes all found projects to the command if no filter is specified 1`] = `
Object {
"graph": Object {
"bar": Array [
"foo",
],
"baz": Array [
"bar",
],
"foo": Array [],
"kibana": Array [
"foo",
],
"quux": Array [
"bar",
"baz",
],
"with-additional-projects": Array [],
},
"projects": Array [
"bar",
"foo",
"kibana",
"with-additional-projects",
"baz",
"quux",
],
}
`;
exports[`respects both \`include\` and \`exclude\` filters if specified at the same time 1`] = `
Object {
"graph": Object {
"baz": Array [],
"foo": Array [],
},
"projects": Array [
"foo",
"baz",
],
}
`;

4
packages/kbn-pm/src/cli.ts
View file

@ -23,6 +23,8 @@ function help() {
Global options:
-e, --exclude Exclude specified project. Can be specified multiple times to exclude multiple projects, e.g. '-e kibana -e @kbn/pm'.
-i, --include Include only specified projects. If left unspecified, it defaults to including all projects.
--skip-kibana Do not include the root Kibana project when running command.
--skip-kibana-extra Filter all plugins in ../kibana-extra when running command.
`);
@ -44,6 +46,8 @@ export async function run(argv: string[]) {
const options = getopts(argv, {
alias: {
h: 'help',
i: 'include',
e: 'exclude',
},
});

22
packages/kbn-pm/src/commands/watch.ts
View file

@ -29,14 +29,24 @@ export const WatchCommand: Command = {
description: 'Runs `kbn:watch` script for every project.',
async run(projects, projectGraph) {
const projectsWithWatchScript: ProjectMap = new Map();
const projectsToWatch: ProjectMap = new Map();
for (const project of projects.values()) {
// We can't watch project that doesn't have `kbn:watch` script.
if (project.hasScript(watchScriptName)) {
projectsWithWatchScript.set(project.name, project);
projectsToWatch.set(project.name, project);
}
}
const projectNames = Array.from(projectsWithWatchScript.keys());
if (projectsToWatch.size === 0) {
console.log(
chalk.red(
`\nThere are no projects to watch found. Make sure that projects define 'kbn:watch' script in 'package.json'.\n`
)
);
return;
}
const projectNames = Array.from(projectsToWatch.keys());
console.log(
chalk.bold(
chalk.green(
@ -47,14 +57,14 @@ export const WatchCommand: Command = {
// Kibana should always be run the last, so we don't rely on automatic
// topological batching and push it to the last one-entry batch manually.
projectsWithWatchScript.delete(kibanaProjectName);
const shouldWatchKibanaProject = projectsToWatch.delete(kibanaProjectName);
const batchedProjects = topologicallyBatchProjects(
projectsWithWatchScript,
projectsToWatch,
projectGraph
);
if (projects.has(kibanaProjectName)) {
if (shouldWatchKibanaProject) {
batchedProjects.push([projects.get(kibanaProjectName)!]);
}

117
packages/kbn-pm/src/run.test.ts Normal file
View file

@ -0,0 +1,117 @@
import { resolve } from 'path';
import { runCommand } from './run';
import { Project } from './utils/project';
import { Command, CommandConfig } from './commands';
const rootPath = resolve(`${__dirname}/utils/__fixtures__/kibana`);
function getExpectedProjectsAndGraph(runMock: any) {
const [fullProjects, fullProjectGraph] = (runMock as jest.Mock<
any
>).mock.calls[0];
const projects = [...fullProjects.keys()];
const graph = [...fullProjectGraph.entries()].reduce(
(expected, [projectName, dependencies]) => {
expected[projectName] = dependencies.map(
(project: Project) => project.name
);
return expected;
},
{}
);
return { projects, graph };
}
let command: Command;
let config: CommandConfig;
beforeEach(() => {
command = {
name: 'test name',
description: 'test description',
run: jest.fn(),
};
config = {
extraArgs: [],
options: {},
rootPath,
};
// Reduce the noise that comes from the run command.
jest.spyOn(console, 'log').mockImplementation(() => {});
});
test('passes all found projects to the command if no filter is specified', async () => {
await runCommand(command, config);
expect(command.run).toHaveBeenCalledTimes(1);
expect(getExpectedProjectsAndGraph(command.run)).toMatchSnapshot();
});
test('excludes project if single `exclude` filter is specified', async () => {
await runCommand(command, {
...config,
options: { exclude: 'foo' },
});
expect(command.run).toHaveBeenCalledTimes(1);
expect(getExpectedProjectsAndGraph(command.run)).toMatchSnapshot();
});
test('excludes projects if multiple `exclude` filter are specified', async () => {
await runCommand(command, {
...config,
options: { exclude: ['foo', 'bar', 'baz'] },
});
expect(command.run).toHaveBeenCalledTimes(1);
expect(getExpectedProjectsAndGraph(command.run)).toMatchSnapshot();
});
test('includes single project if single `include` filter is specified', async () => {
await runCommand(command, {
...config,
options: { include: 'foo' },
});
expect(command.run).toHaveBeenCalledTimes(1);
expect(getExpectedProjectsAndGraph(command.run)).toMatchSnapshot();
});
test('includes only projects specified in multiple `include` filters', async () => {
await runCommand(command, {
...config,
options: { include: ['foo', 'bar', 'baz'] },
});
expect(command.run).toHaveBeenCalledTimes(1);
expect(getExpectedProjectsAndGraph(command.run)).toMatchSnapshot();
});
test('respects both `include` and `exclude` filters if specified at the same time', async () => {
await runCommand(command, {
...config,
options: { include: ['foo', 'bar', 'baz'], exclude: 'bar' },
});
expect(command.run).toHaveBeenCalledTimes(1);
expect(getExpectedProjectsAndGraph(command.run)).toMatchSnapshot();
});
test('does not run command if all projects are filtered out', async () => {
let mockProcessExit = jest
.spyOn(process, 'exit')
.mockImplementation(() => {});
await runCommand(command, {
...config,
// Including and excluding the same project will result in 0 projects selected.
options: { include: ['foo'], exclude: ['foo'] },
});
expect(command.run).not.toHaveBeenCalled();
expect(mockProcessExit).toHaveBeenCalledWith(1);
});

23
packages/kbn-pm/src/run.ts
View file

@ -23,7 +23,20 @@ export async function runCommand(command: Command, config: CommandConfig) {
config.options as ProjectPathOptions
);
const projects = await getProjects(config.rootPath, projectPaths);
const projects = await getProjects(config.rootPath, projectPaths, {
exclude: toArray(config.options.exclude),
include: toArray(config.options.include),
});
if (projects.size === 0) {
console.log(
chalk.red(
`There are no projects found. Double check project name(s) in '-i/--include' and '-e/--exclude' filters.\n`
)
);
return process.exit(1);
}
const projectGraph = buildProjectGraph(projects);
console.log(
@ -56,3 +69,11 @@ export async function runCommand(command: Command, config: CommandConfig) {
process.exit(1);
}
}
function toArray<T>(value?: T | T[]) {
if (value == null) {
return [];
}
return Array.isArray(value) ? value : [value];
}

13
packages/kbn-pm/src/utils/__snapshots__/projects.test.ts.snap
View file

@ -32,3 +32,16 @@ Array [
],
]
`;
exports[`#topologicallyBatchProjects batches projects topologically even if graph contains projects not presented in the project map 1`] = `
Array [
Array [
"kibana",
"bar",
"baz",
],
Array [
"quux",
],
]
`;

117
packages/kbn-pm/src/utils/projects.test.ts
View file

@ -5,6 +5,8 @@ import {
buildProjectGraph,
topologicallyBatchProjects,
includeTransitiveProjects,
ProjectMap,
ProjectGraph,
} from './projects';
import { Project } from './project';
import { getProjectPaths } from '../config';
@ -68,6 +70,96 @@ describe('#getProjects', () => {
);
expect(projects.size).toBe(expectedProjects.length);
});
describe('with exclude/include filters', () => {
let projectPaths: string[];
beforeEach(() => {
projectPaths = getProjectPaths(rootPath, {});
});
test('excludes projects specified in `exclude` filter', async () => {
const projects = await getProjects(rootPath, projectPaths, {
exclude: ['foo', 'bar', 'baz'],
});
expect([...projects.keys()].sort()).toEqual([
'kibana',
'quux',
'with-additional-projects',
]);
});
test('ignores unknown projects specified in `exclude` filter', async () => {
const projects = await getProjects(rootPath, projectPaths, {
exclude: ['unknown-foo', 'bar', 'unknown-baz'],
});
expect([...projects.keys()].sort()).toEqual([
'baz',
'foo',
'kibana',
'quux',
'with-additional-projects',
]);
});
test('includes only projects specified in `include` filter', async () => {
const projects = await getProjects(rootPath, projectPaths, {
include: ['foo', 'bar'],
});
expect([...projects.keys()].sort()).toEqual(['bar', 'foo']);
});
test('ignores unknown projects specified in `include` filter', async () => {
const projects = await getProjects(rootPath, projectPaths, {
include: ['unknown-foo', 'bar', 'unknown-baz'],
});
expect([...projects.keys()].sort()).toEqual(['bar']);
});
test('respects both `include` and `exclude` filters if specified at the same time', async () => {
const projects = await getProjects(rootPath, projectPaths, {
exclude: ['bar'],
include: ['foo', 'bar', 'baz'],
});
expect([...projects.keys()].sort()).toEqual(['baz', 'foo']);
});
test('does not return any project if wrong `include` filter is specified', async () => {
const projects = await getProjects(rootPath, projectPaths, {
include: ['unknown-foo', 'unknown-bar'],
});
expect(projects.size).toBe(0);
});
test('does not return any project if `exclude` filter is specified for all projects', async () => {
const projects = await getProjects(rootPath, projectPaths, {
exclude: [
'kibana',
'bar',
'foo',
'with-additional-projects',
'quux',
'baz',
],
});
expect(projects.size).toBe(0);
});
test('does not return any project if `exclude` and `include` filters are mutually exclusive', async () => {
const projects = await getProjects(rootPath, projectPaths, {
exclude: ['foo', 'bar'],
include: ['foo', 'bar'],
});
expect(projects.size).toBe(0);
});
});
});
describe('#buildProjectGraph', () => {
@ -89,13 +181,26 @@ describe('#buildProjectGraph', () => {
});
describe('#topologicallyBatchProjects', () => {
let projects: ProjectMap;
let graph: ProjectGraph;
beforeEach(async () => {
projects = await getProjects(rootPath, ['.', 'packages/*', '../plugins/*']);
graph = buildProjectGraph(projects);
});
test('batches projects topologically based on their project dependencies', async () => {
const projects = await getProjects(rootPath, [
'.',
'packages/*',
'../plugins/*',
]);
const graph = buildProjectGraph(projects);
const batches = topologicallyBatchProjects(projects, graph);
const expectedBatches = batches.map(batch =>
batch.map(project => project.name)
);
expect(expectedBatches).toMatchSnapshot();
});
test('batches projects topologically even if graph contains projects not presented in the project map', async () => {
// Make sure that the project we remove really existed in the projects map.
expect(projects.delete('foo')).toBe(true);
const batches = topologicallyBatchProjects(projects, graph);

52
packages/kbn-pm/src/utils/projects.ts
View file

@ -9,10 +9,12 @@ const glob = promisify(globSync);
export type ProjectMap = Map<string, Project>;
export type ProjectGraph = Map<string, Project[]>;
export type ProjectsOptions = { include?: string[]; exclude?: string[] };
export async function getProjects(
rootPath: string,
projectsPathsPatterns: string[]
projectsPathsPatterns: string[],
{ include = [], exclude = [] }: ProjectsOptions = {}
) {
const projects: ProjectMap = new Map();
@ -24,6 +26,14 @@ export async function getProjects(
const projectDir = path.dirname(projectConfigPath);
const project = await Project.fromPath(projectDir);
const excludeProject =
exclude.includes(project.name) ||
(include.length > 0 && !include.includes(project.name));
if (excludeProject) {
continue;
}
if (projects.has(project.name)) {
throw new CliError(
`There are multiple projects with the same name [${project.name}]`,
@ -99,33 +109,30 @@ export function topologicallyBatchProjects(
projectsToBatch: ProjectMap,
projectGraph: ProjectGraph
) {
// We're going to be chopping stuff out of this array, so copy it.
const projects = [...projectsToBatch.values()];
// This maps project names to the number of projects that depend on them.
// As projects are completed their names will be removed from this object.
const refCounts: { [k: string]: number } = {};
projects.forEach(pkg =>
projectGraph.get(pkg.name)!.forEach(dep => {
if (!refCounts[dep.name]) refCounts[dep.name] = 0;
refCounts[dep.name]++;
})
);
// We're going to be chopping stuff out of this list, so copy it.
const projectToBatchNames = new Set(projectsToBatch.keys());
const batches = [];
while (projects.length > 0) {
while (projectToBatchNames.size > 0) {
// Get all projects that have no remaining dependencies within the repo
// that haven't yet been picked.
const batch = projects.filter(pkg => {
const projectDeps = projectGraph.get(pkg.name)!;
return projectDeps.filter(dep => refCounts[dep.name] > 0).length === 0;
});
const batch = [];
for (const projectName of projectToBatchNames) {
const projectDeps = projectGraph.get(projectName)!;
const hasNotBatchedDependencies = projectDeps.some(dep =>
projectToBatchNames.has(dep.name)
);
if (!hasNotBatchedDependencies) {
batch.push(projectsToBatch.get(projectName)!);
}
}
// If we weren't able to find a project with no remaining dependencies,
// then we've encountered a cycle in the dependency graph.
const hasCycles = projects.length > 0 && batch.length === 0;
const hasCycles = batch.length === 0;
if (hasCycles) {
const cycleProjectNames = projects.map(p => p.name);
const cycleProjectNames = [...projectToBatchNames];
const message =
'Encountered a cycle in the dependency graph. Projects in cycle are:\n' +
cycleProjectNames.join(', ');
@ -135,10 +142,7 @@ export function topologicallyBatchProjects(
batches.push(batch);
batch.forEach(pkg => {
delete refCounts[pkg.name];
projects.splice(projects.indexOf(pkg), 1);
});
batch.forEach(project => projectToBatchNames.delete(project.name));
}
return batches;