Merge pull request #11021 from Microsoft/throttle-in-runAsync

move throttling to runAsync
This commit is contained in:
jramsay 2016-09-20 16:10:46 -07:00 committed by GitHub
commit ec7c7a4a36
5 changed files with 268 additions and 298 deletions

View file

@ -3,6 +3,10 @@
/// <reference path="../../server/typingsInstaller/typingsInstaller.ts" />
namespace ts.projectSystem {
function createTestTypingsInstaller(host: server.ServerHost) {
return new TestTypingsInstaller("/a/data/", /*throttleLimit*/5, host);
}
describe("CompileOnSave affected list", () => {
function sendAffectedFileRequestAndCheckResult(session: server.Session, request: server.protocol.Request, expectedFileList: { projectFileName: string, files: FileOrFolder[] }[]) {
const response: server.protocol.CompileOnSaveAffectedFileListSingleProject[] = session.executeCommand(request).response;
@ -105,7 +109,7 @@ namespace ts.projectSystem {
it("should contains only itself if a module file's shape didn't change, and all files referencing it if its shape changed", () => {
const host = createServerHost([moduleFile1, file1Consumer1, file1Consumer2, globalFile3, moduleFile2, configFile, libFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([moduleFile1, file1Consumer1], session);
@ -130,7 +134,7 @@ namespace ts.projectSystem {
it("should be up-to-date with the reference map changes", () => {
const host = createServerHost([moduleFile1, file1Consumer1, file1Consumer2, globalFile3, moduleFile2, configFile, libFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([moduleFile1, file1Consumer1], session);
@ -177,7 +181,7 @@ namespace ts.projectSystem {
it("should be up-to-date with changes made in non-open files", () => {
const host = createServerHost([moduleFile1, file1Consumer1, file1Consumer2, globalFile3, moduleFile2, configFile, libFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([moduleFile1], session);
@ -195,7 +199,7 @@ namespace ts.projectSystem {
it("should be up-to-date with deleted files", () => {
const host = createServerHost([moduleFile1, file1Consumer1, file1Consumer2, globalFile3, moduleFile2, configFile, libFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([moduleFile1], session);
@ -210,7 +214,7 @@ namespace ts.projectSystem {
it("should be up-to-date with newly created files", () => {
const host = createServerHost([moduleFile1, file1Consumer1, file1Consumer2, globalFile3, moduleFile2, configFile, libFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([moduleFile1], session);
@ -247,7 +251,7 @@ namespace ts.projectSystem {
};
const host = createServerHost([moduleFile1, file1Consumer1, configFile, libFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([moduleFile1, file1Consumer1], session);
@ -264,7 +268,7 @@ namespace ts.projectSystem {
it("should return all files if a global file changed shape", () => {
const host = createServerHost([moduleFile1, file1Consumer1, file1Consumer2, globalFile3, moduleFile2, configFile, libFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([globalFile3], session);
@ -290,7 +294,7 @@ namespace ts.projectSystem {
};
const host = createServerHost([moduleFile1, file1Consumer1, file1Consumer2, configFile, libFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([moduleFile1], session);
sendAffectedFileRequestAndCheckResult(session, moduleFile1FileListRequest, []);
@ -308,7 +312,7 @@ namespace ts.projectSystem {
};
const host = createServerHost([moduleFile1, file1Consumer1, configFile, libFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([moduleFile1], session);
@ -337,7 +341,7 @@ namespace ts.projectSystem {
};
const host = createServerHost([moduleFile1, file1Consumer1, configFile, libFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([moduleFile1], session);
@ -359,7 +363,7 @@ namespace ts.projectSystem {
content: `import {y} from "./file1Consumer1";`
};
const host = createServerHost([moduleFile1, file1Consumer1, file1Consumer1Consumer1, globalFile3, configFile, libFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([moduleFile1, file1Consumer1], session);
@ -392,7 +396,7 @@ namespace ts.projectSystem {
export var t2 = 10;`
};
const host = createServerHost([file1, file2, configFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([file1, file2], session);
@ -475,7 +479,7 @@ namespace ts.projectSystem {
content: `{}`
};
const host = createServerHost([file1, file2, configFile, libFile]);
const typingsInstaller = new TestTypingsInstaller("/a/data/", host);
const typingsInstaller = createTestTypingsInstaller(host);
const session = new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
openFilesForSession([file1, file2], session);

View file

@ -2,6 +2,8 @@
/// <reference path="../../server/typingsInstaller/typingsInstaller.ts" />
namespace ts.projectSystem {
import TI = server.typingsInstaller;
const safeList = {
path: <Path>"/safeList.json",
content: JSON.stringify({
@ -14,6 +16,7 @@ namespace ts.projectSystem {
};
export interface PostExecAction {
readonly requestKind: TI.RequestKind;
readonly error: Error;
readonly stdout: string;
readonly stderr: string;
@ -46,21 +49,27 @@ namespace ts.projectSystem {
content: libFileContent
};
export class TestTypingsInstaller extends server.typingsInstaller.TypingsInstaller implements server.ITypingsInstaller {
export class TestTypingsInstaller extends TI.TypingsInstaller implements server.ITypingsInstaller {
protected projectService: server.ProjectService;
constructor(readonly globalTypingsCacheLocation: string, readonly installTypingHost: server.ServerHost) {
super(globalTypingsCacheLocation, safeList.path);
constructor(readonly globalTypingsCacheLocation: string, throttleLimit: number, readonly installTypingHost: server.ServerHost) {
super(globalTypingsCacheLocation, "npm", safeList.path, throttleLimit);
this.init();
}
safeFileList = safeList.path;
protected postExecActions: PostExecAction[] = [];
runPostExecActions() {
for (const action of this.postExecActions) {
executePendingCommands() {
const actionsToRun = this.postExecActions;
this.postExecActions = [];
for (const action of actionsToRun) {
action.callback(action.error, action.stdout, action.stderr);
}
this.postExecActions = [];
}
checkPendingCommands(expected: TI.RequestKind[]) {
assert.equal(this.postExecActions.length, expected.length, `Expected ${expected.length} post install actions`);
this.postExecActions.forEach((act, i) => assert.equal(act.requestKind, expected[i], "Unexpected post install action"));
}
onProjectClosed(p: server.Project) {
@ -74,16 +83,15 @@ namespace ts.projectSystem {
return this.installTypingHost;
}
execAsync(prefix: string, command: string, cwd: string, requestId: number, cb: (err: Error, stdout: string, stderr: string) => void): void {
switch (prefix) {
case "npm view":
case "npm install":
case "npm ls":
runCommand(requestKind: TI.RequestKind, requestId: number, command: string, cwd: string, cb: (err: Error, stdout: string, stderr: string) => void): void {
switch (requestKind) {
case TI.NpmViewRequest:
case TI.NpmInstallRequest:
break;
default:
throw new Error("TypingsInstaller: execAsync command not yet implemented");
assert.isTrue(false, `request ${requestKind} is not supported`);
}
this.addPostExecAction("success", cb);
this.addPostExecAction(requestKind, "success", cb);
}
sendResponse(response: server.SetTypings | server.InvalidateCachedTypings) {
@ -95,13 +103,14 @@ namespace ts.projectSystem {
this.install(request);
}
addPostExecAction(stdout: string | string[], cb: (err: Error, stdout: string, stderr: string) => void) {
addPostExecAction(requestKind: TI.RequestKind, stdout: string | string[], cb: TI.RequestCompletedAction) {
const out = typeof stdout === "string" ? stdout : createNpmPackageJsonString(stdout);
const action: PostExecAction = {
error: undefined,
stdout: out,
stderr: "",
callback: cb
callback: cb,
requestKind
};
this.postExecActions.push(action);
}
@ -152,7 +161,7 @@ namespace ts.projectSystem {
export function createSession(host: server.ServerHost, typingsInstaller?: server.ITypingsInstaller) {
if (typingsInstaller === undefined) {
typingsInstaller = new TestTypingsInstaller("/a/data/", host);
typingsInstaller = new TestTypingsInstaller("/a/data/", /*throttleLimit*/5, host);
}
return new server.Session(host, nullCancellationToken, /*useSingleInferredProject*/ false, typingsInstaller, Utils.byteLength, process.hrtime, nullLogger, /*canUseEvents*/ false);
}
@ -1643,7 +1652,7 @@ namespace ts.projectSystem {
const host: TestServerHost & ModuleResolutionHost = createServerHost([file1, lib]);
const resolutionTrace: string[] = [];
host.trace = resolutionTrace.push.bind(resolutionTrace);
const projectService = createProjectService(host, { typingsInstaller: new TestTypingsInstaller("/a/cache", host) });
const projectService = createProjectService(host, { typingsInstaller: new TestTypingsInstaller("/a/cache", /*throttleLimit*/5, host) });
projectService.setCompilerOptionsForInferredProjects({ traceResolution: true, allowJs: true });
projectService.openClientFile(file1.path);

View file

@ -3,23 +3,47 @@
/// <reference path="../../server/typingsInstaller/typingsInstaller.ts" />
namespace ts.projectSystem {
import TI = server.typingsInstaller;
interface InstallerParams {
globalTypingsCacheLocation?: string;
throttleLimit?: number;
}
class Installer extends TestTypingsInstaller {
constructor(host: server.ServerHost, p?: InstallerParams) {
super(
(p && p.globalTypingsCacheLocation) || "/a/data",
(p && p.throttleLimit) || 5,
host);
}
installAll(expectedView: typeof TI.NpmViewRequest[], expectedInstall: typeof TI.NpmInstallRequest[]) {
this.checkPendingCommands(expectedView);
this.executePendingCommands();
this.checkPendingCommands(expectedInstall);
this.executePendingCommands();
}
}
describe("typingsInstaller", () => {
function execHelper(self: TestTypingsInstaller, host: TestServerHost, installedTypings: string[], typingFiles: FileOrFolder[], prefix: string, cb: (err: Error, stdout: string, stderr: string) => void): boolean {
let execSuper = true;
switch (prefix) {
case "npm install":
for (const file of typingFiles) {
host.createFileOrFolder(file, /*createParentDirectory*/ true);
}
function executeCommand(self: Installer, host: TestServerHost, installedTypings: string[], typingFiles: FileOrFolder[], requestKind: TI.RequestKind, cb: TI.RequestCompletedAction): void {
switch (requestKind) {
case TI.NpmInstallRequest:
self.addPostExecAction(requestKind, installedTypings, (err, stdout, stderr) => {
for (const file of typingFiles) {
host.createFileOrFolder(file, /*createParentDirectory*/ true);
}
cb(err, stdout, stderr);
});
break;
case "npm ls":
self.addPostExecAction(installedTypings, cb);
execSuper = false;
case TI.NpmViewRequest:
self.addPostExecAction(requestKind, installedTypings, cb);
break;
default:
assert.isTrue(false, `unexpected request kind ${requestKind}`);
break;
}
return execSuper;
}
it("configured projects (typings installed) 1", () => {
const file1 = {
@ -52,16 +76,14 @@ namespace ts.projectSystem {
content: "declare const $: { x: number }"
};
const host = createServerHost([file1, tsconfig, packageJson]);
const installer = new (class extends TestTypingsInstaller {
const installer = new (class extends Installer {
constructor() {
super("/a/data/", host);
super(host);
}
execAsync(prefix: string, command: string, cwd: string, requestId: number, cb: (err: Error, stdout: string, stderr: string) => void): void {
runCommand(requestKind: TI.RequestKind, requestId: number, command: string, cwd: string, cb: server.typingsInstaller.RequestCompletedAction) {
const installedTypings = ["@types/jquery"];
const typingFiles = [jquery];
if (execHelper(this, host, installedTypings, typingFiles, prefix, cb)) {
super.execAsync(prefix, command, cwd, requestId, cb);
}
executeCommand(this, host, installedTypings, typingFiles, requestKind, cb);
}
})();
@ -72,9 +94,7 @@ namespace ts.projectSystem {
const p = projectService.configuredProjects[0];
checkProjectActualFiles(p, [file1.path]);
assert(host.fileExists(combinePaths(installer.globalTypingsCacheLocation, "package.json")));
installer.runPostExecActions();
installer.installAll([TI.NpmViewRequest], [TI.NpmInstallRequest]);
checkNumberOfProjects(projectService, { configuredProjects: 1 });
checkProjectActualFiles(p, [file1.path, jquery.path]);
@ -100,16 +120,14 @@ namespace ts.projectSystem {
content: "declare const $: { x: number }"
};
const host = createServerHost([file1, packageJson]);
const installer = new (class extends TestTypingsInstaller {
const installer = new (class extends Installer {
constructor() {
super("/a/data/", host);
super(host);
}
execAsync(prefix: string, command: string, cwd: string, requestId: number, cb: (err: Error, stdout: string, stderr: string) => void): void {
runCommand(requestKind: TI.RequestKind, requestId: number, command: string, cwd: string, cb: server.typingsInstaller.RequestCompletedAction) {
const installedTypings = ["@types/jquery"];
const typingFiles = [jquery];
if (execHelper(this, host, installedTypings, typingFiles, prefix, cb)) {
super.execAsync(prefix, command, cwd, requestId, cb);
}
executeCommand(this, host, installedTypings, typingFiles, requestKind, cb);
}
})();
@ -120,9 +138,7 @@ namespace ts.projectSystem {
const p = projectService.inferredProjects[0];
checkProjectActualFiles(p, [file1.path]);
assert(host.fileExists(combinePaths(installer.globalTypingsCacheLocation, "package.json")));
installer.runPostExecActions();
installer.installAll([TI.NpmViewRequest], [TI.NpmInstallRequest]);
checkNumberOfProjects(projectService, { inferredProjects: 1 });
checkProjectActualFiles(p, [file1.path, jquery.path]);
@ -134,9 +150,9 @@ namespace ts.projectSystem {
content: ""
};
const host = createServerHost([file1]);
const installer = new (class extends TestTypingsInstaller {
const installer = new (class extends Installer {
constructor() {
super("", host);
super(host);
}
enqueueInstallTypingsRequest() {
assert(false, "auto discovery should not be enabled");
@ -150,6 +166,7 @@ namespace ts.projectSystem {
options: {},
rootFiles: [toExternalFile(file1.path)]
});
installer.checkPendingCommands([]);
// by default auto discovery will kick in if project contain only .js/.d.ts files
// in this case project contain only ts files - no auto discovery
projectService.checkNumberOfProjects({ externalProjects: 1 });
@ -161,9 +178,9 @@ namespace ts.projectSystem {
content: ""
};
const host = createServerHost([file1]);
const installer = new (class extends TestTypingsInstaller {
const installer = new (class extends Installer {
constructor() {
super("", host);
super(host);
}
enqueueInstallTypingsRequest() {
assert(false, "auto discovery should not be enabled");
@ -178,6 +195,7 @@ namespace ts.projectSystem {
rootFiles: [toExternalFile(file1.path)],
typingOptions: { include: ["jquery"] }
});
installer.checkPendingCommands([]);
// by default auto discovery will kick in if project contain only .js/.d.ts files
// in this case project contain only ts files - no auto discovery even if typing options is set
projectService.checkNumberOfProjects({ externalProjects: 1 });
@ -194,24 +212,18 @@ namespace ts.projectSystem {
};
const host = createServerHost([file1]);
let enqueueIsCalled = false;
let runInstallIsCalled = false;
const installer = new (class extends TestTypingsInstaller {
const installer = new (class extends Installer {
constructor() {
super("", host);
super(host);
}
enqueueInstallTypingsRequest(project: server.Project, typingOptions: TypingOptions) {
enqueueIsCalled = true;
super.enqueueInstallTypingsRequest(project, typingOptions);
}
execAsync(prefix: string, command: string, cwd: string, requestId: number, cb: (err: Error, stdout: string, stderr: string) => void): void {
runCommand(requestKind: TI.RequestKind, requestId: number, command: string, cwd: string, cb: TI.RequestCompletedAction): void {
const installedTypings = ["@types/jquery"];
const typingFiles = [jquery];
if (execHelper(this, host, installedTypings, typingFiles, prefix, cb)) {
super.execAsync(prefix, command, cwd, requestId, cb);
}
if (prefix === "npm install") {
runInstallIsCalled = true;
}
executeCommand(this, host, installedTypings, typingFiles, requestKind, cb);
}
})();
@ -224,12 +236,11 @@ namespace ts.projectSystem {
typingOptions: { enableAutoDiscovery: true, include: ["node"] }
});
installer.runPostExecActions();
assert.isTrue(enqueueIsCalled, "expected enqueueIsCalled to be true");
installer.installAll([TI.NpmViewRequest], [TI.NpmInstallRequest]);
// autoDiscovery is set in typing options - use it even if project contains only .ts files
projectService.checkNumberOfProjects({ externalProjects: 1 });
assert.isTrue(enqueueIsCalled, "expected 'enqueueIsCalled' to be true");
assert.isTrue(runInstallIsCalled, "expected 'runInstallIsCalled' to be true");
});
it("external project - no typing options, with only js, jsx, d.ts files", () => {
@ -259,16 +270,14 @@ namespace ts.projectSystem {
};
const host = createServerHost([file1, file2, file3]);
const installer = new (class extends TestTypingsInstaller {
const installer = new (class extends Installer {
constructor() {
super("/a/data/", host);
super(host);
}
execAsync(prefix: string, command: string, cwd: string, requestId: number, cb: (err: Error, stdout: string, stderr: string) => void): void {
runCommand(requestKind: TI.RequestKind, requestId: number, command: string, cwd: string, cb: TI.RequestCompletedAction): void {
const installedTypings = ["@types/lodash", "@types/react"];
const typingFiles = [lodash, react];
if (execHelper(this, host, installedTypings, typingFiles, prefix, cb)) {
super.execAsync(prefix, command, cwd, requestId, cb);
}
executeCommand(this, host, installedTypings, typingFiles, requestKind, cb);
}
})();
@ -285,7 +294,7 @@ namespace ts.projectSystem {
projectService.checkNumberOfProjects({ externalProjects: 1 });
checkProjectActualFiles(p, [file1.path, file2.path, file3.path]);
installer.runPostExecActions();
installer.installAll([TI.NpmViewRequest, TI.NpmViewRequest], [TI.NpmInstallRequest], );
checkNumberOfProjects(projectService, { externalProjects: 1 });
checkProjectActualFiles(p, [file1.path, file2.path, file3.path, lodash.path, react.path]);
@ -305,24 +314,18 @@ namespace ts.projectSystem {
const host = createServerHost([file1, file2]);
let enqueueIsCalled = false;
let runInstallIsCalled = false;
const installer = new (class extends TestTypingsInstaller {
const installer = new (class extends Installer {
constructor() {
super("/a/data/", host);
super(host);
}
enqueueInstallTypingsRequest(project: server.Project, typingOptions: TypingOptions) {
enqueueIsCalled = true;
super.enqueueInstallTypingsRequest(project, typingOptions);
}
execAsync(prefix: string, command: string, cwd: string, requestId: number, cb: (err: Error, stdout: string, stderr: string) => void): void {
runCommand(requestKind: TI.RequestKind, requestId: number, command: string, cwd: string, cb: TI.RequestCompletedAction): void {
const installedTypings: string[] = [];
const typingFiles: FileOrFolder[] = [];
if (execHelper(this, host, installedTypings, typingFiles, prefix, cb)) {
super.execAsync(prefix, command, cwd, requestId, cb);
}
if (prefix === "npm install") {
runInstallIsCalled = true;
}
executeCommand(this, host, installedTypings, typingFiles, requestKind, cb);
}
})();
@ -339,12 +342,10 @@ namespace ts.projectSystem {
projectService.checkNumberOfProjects({ externalProjects: 1 });
checkProjectActualFiles(p, [file1.path, file2.path]);
installer.runPostExecActions();
installer.checkPendingCommands([]);
checkNumberOfProjects(projectService, { externalProjects: 1 });
checkProjectActualFiles(p, [file1.path, file2.path]);
assert.isFalse(enqueueIsCalled, "expected 'enqueueIsCalled' to be false");
assert.isFalse(runInstallIsCalled, "expected 'runInstallIsCalled' to be false");
});
it("external project - with typing options, with only js, d.ts files", () => {
@ -392,16 +393,14 @@ namespace ts.projectSystem {
};
const host = createServerHost([file1, file2, file3, packageJson]);
const installer = new (class extends TestTypingsInstaller {
const installer = new (class extends Installer {
constructor() {
super("/a/data/", host);
super(host);
}
execAsync(prefix: string, command: string, cwd: string, requestId: number, cb: (err: Error, stdout: string, stderr: string) => void): void {
runCommand(requestKind: TI.RequestKind, requestId: number, command: string, cwd: string, cb: TI.RequestCompletedAction): void {
const installedTypings = ["@types/commander", "@types/express", "@types/jquery", "@types/moment"];
const typingFiles = [commander, express, jquery, moment];
if (execHelper(this, host, installedTypings, typingFiles, prefix, cb)) {
super.execAsync(prefix, command, cwd, requestId, cb);
}
executeCommand(this, host, installedTypings, typingFiles, requestKind, cb);
}
})();
@ -418,18 +417,21 @@ namespace ts.projectSystem {
projectService.checkNumberOfProjects({ externalProjects: 1 });
checkProjectActualFiles(p, [file1.path, file2.path, file3.path]);
installer.runPostExecActions();
installer.installAll(
[TI.NpmViewRequest, TI.NpmViewRequest, TI.NpmViewRequest, TI.NpmViewRequest],
[TI.NpmInstallRequest]
);
checkNumberOfProjects(projectService, { externalProjects: 1 });
checkProjectActualFiles(p, [file1.path, file2.path, file3.path, commander.path, express.path, jquery.path, moment.path]);
});
it("Throttle - delayed typings to install", () => {
const file1 = {
const lodashJs = {
path: "/a/b/lodash.js",
content: ""
};
const file2 = {
const commanderJs = {
path: "/a/b/commander.js",
content: ""
};
@ -442,11 +444,7 @@ namespace ts.projectSystem {
content: JSON.stringify({
name: "test",
dependencies: {
express: "^3.1.0",
cordova: "1.0.0",
grunt: "1.0.0",
gulp: "1.0.0",
forever: "1.0.0",
express: "^3.1.0"
}
})
};
@ -471,42 +469,16 @@ namespace ts.projectSystem {
path: "/a/data/node_modules/@types/lodash/index.d.ts",
content: "declare const lodash: { x: number }"
};
const cordova = {
path: "/a/data/node_modules/@types/cordova/index.d.ts",
content: "declare const cordova: { x: number }"
};
const grunt = {
path: "/a/data/node_modules/@types/grunt/index.d.ts",
content: "declare const grunt: { x: number }"
};
const gulp = {
path: "/a/data/node_modules/@types/gulp/index.d.ts",
content: "declare const gulp: { x: number }"
};
const forever = {
path: "/a/data/node_modules/@types/forever/index.d.ts",
content: "declare const forever: { x: number }"
};
let npmViewCount = 0;
let npmInstallCount = 0;
const host = createServerHost([file1, file2, file3, packageJson]);
const installer = new (class extends TestTypingsInstaller {
const typingFiles = [commander, express, jquery, moment, lodash];
const host = createServerHost([lodashJs, commanderJs, file3, packageJson]);
const installer = new (class extends Installer {
constructor() {
super("/a/data/", host);
super(host, { throttleLimit: 3 });
}
execAsync(prefix: string, command: string, cwd: string, requestId: number, cb: (err: Error, stdout: string, stderr: string) => void): void {
if (prefix === "npm view") {
npmViewCount++;
}
if (prefix === "npm install") {
npmInstallCount++;
}
const installedTypings = ["@types/commander", "@types/express", "@types/jquery", "@types/moment", "@types/lodash", "@types/cordova", "@types/grunt", "@types/gulp", "@types/forever"];
const typingFiles = [commander, express, jquery, moment, lodash, cordova, grunt, gulp, forever];
if (execHelper(this, host, installedTypings, typingFiles, prefix, cb)) {
super.execAsync(prefix, command, cwd, requestId, cb);
}
runCommand(requestKind: TI.RequestKind, requestId: number, command: string, cwd: string, cb: TI.RequestCompletedAction): void {
const installedTypings = ["@types/commander", "@types/express", "@types/jquery", "@types/moment", "@types/lodash"];
executeCommand(this, host, installedTypings, typingFiles, requestKind, cb);
}
})();
@ -515,33 +487,41 @@ namespace ts.projectSystem {
projectService.openExternalProject({
projectFileName,
options: { allowJS: true, moduleResolution: ModuleResolutionKind.NodeJs },
rootFiles: [toExternalFile(file1.path), toExternalFile(file2.path), toExternalFile(file3.path)],
rootFiles: [toExternalFile(lodashJs.path), toExternalFile(commanderJs.path), toExternalFile(file3.path)],
typingOptions: { include: ["jquery", "moment"] }
});
const p = projectService.externalProjects[0];
projectService.checkNumberOfProjects({ externalProjects: 1 });
checkProjectActualFiles(p, [file1.path, file2.path, file3.path]);
// The npm view count should be 5 even though there are 9 typings to acquire.
// The throttle limit has been reached so no more execAsync requests will be
// queued until these have been processed.
assert.isTrue(npmViewCount === 5);
assert.isTrue(npmInstallCount === 0);
checkProjectActualFiles(p, [lodashJs.path, commanderJs.path, file3.path]);
// expected 3 view requests in the queue
installer.checkPendingCommands([TI.NpmViewRequest, TI.NpmViewRequest, TI.NpmViewRequest]);
assert.equal(installer.pendingRunRequests.length, 2, "expected 2 pending requests");
installer.runPostExecActions();
// push view requests
installer.executePendingCommands();
// expected 2 remaining view requests in the queue
installer.checkPendingCommands([TI.NpmViewRequest, TI.NpmViewRequest]);
// push view requests
installer.executePendingCommands();
// expected one install request
installer.checkPendingCommands([TI.NpmInstallRequest]);
installer.executePendingCommands();
// expected all typings file to exist
for (const f of typingFiles) {
assert.isTrue(host.fileExists(f.path), `expected file ${f.path} to exist`);
}
assert.isTrue(npmViewCount === 9);
assert.isTrue(npmInstallCount === 1);
checkNumberOfProjects(projectService, { externalProjects: 1 });
checkProjectActualFiles(p, [file1.path, file2.path, file3.path, commander.path, express.path, jquery.path, moment.path, lodash.path, cordova.path, grunt.path, gulp.path, forever.path]);
checkProjectActualFiles(p, [lodashJs.path, commanderJs.path, file3.path, commander.path, express.path, jquery.path, moment.path, lodash.path]);
});
it("Throttle - delayed run install requests", () => {
const file1 = {
const lodashJs = {
path: "/a/b/lodash.js",
content: ""
};
const file2 = {
const commanderJs = {
path: "/a/b/commander.js",
content: ""
};
@ -552,119 +532,100 @@ namespace ts.projectSystem {
const commander = {
path: "/a/data/node_modules/@types/commander/index.d.ts",
content: "declare const commander: { x: number }"
};
const express = {
path: "/a/data/node_modules/@types/express/index.d.ts",
content: "declare const express: { x: number }"
content: "declare const commander: { x: number }",
typings: "@types/commander"
};
const jquery = {
path: "/a/data/node_modules/@types/jquery/index.d.ts",
content: "declare const jquery: { x: number }"
};
const moment = {
path: "/a/data/node_modules/@types/moment/index.d.ts",
content: "declare const moment: { x: number }"
content: "declare const jquery: { x: number }",
typings: "@types/jquery"
};
const lodash = {
path: "/a/data/node_modules/@types/lodash/index.d.ts",
content: "declare const lodash: { x: number }"
content: "declare const lodash: { x: number }",
typings: "@types/lodash"
};
const cordova = {
path: "/a/data/node_modules/@types/cordova/index.d.ts",
content: "declare const cordova: { x: number }"
content: "declare const cordova: { x: number }",
typings: "@types/cordova"
};
const grunt = {
path: "/a/data/node_modules/@types/grunt/index.d.ts",
content: "declare const grunt: { x: number }"
content: "declare const grunt: { x: number }",
typings: "@types/grunt"
};
const gulp = {
path: "/a/data/node_modules/@types/gulp/index.d.ts",
content: "declare const gulp: { x: number }"
};
const forever = {
path: "/a/data/node_modules/@types/forever/index.d.ts",
content: "declare const forever: { x: number }"
content: "declare const gulp: { x: number }",
typings: "@types/gulp"
};
let npmViewCount = 0;
let npmInstallCount = 0;
let hasRunInstall2Typings = false;
const host = createServerHost([file1, file2, file3]);
const installer = new (class extends TestTypingsInstaller {
const host = createServerHost([lodashJs, commanderJs, file3]);
const installer = new (class extends Installer {
constructor() {
super("/a/data/", host);
super(host, { throttleLimit: 3 });
}
execAsync(prefix: string, command: string, cwd: string, requestId: number, cb: (err: Error, stdout: string, stderr: string) => void): void {
switch (prefix) {
case "npm view":
if (command.match("grunt|gulp|forever")) {
hasRunInstall2Typings = true;
}
npmViewCount++;
break;
case "npm install":
npmInstallCount++;
break;
}
let installedTypings: string[];
let typingFiles: FileOrFolder[];
if (npmInstallCount <= 1) {
installedTypings = ["@types/commander", "@types/express", "@types/jquery", "@types/moment", "@types/lodash", "@types/cordova"];
typingFiles = [commander, express, jquery, moment, lodash, cordova];
runCommand(requestKind: TI.RequestKind, requestId: number, command: string, cwd: string, cb: TI.RequestCompletedAction): void {
if (requestKind === TI.NpmInstallRequest) {
let typingFiles: (FileOrFolder & { typings: string}) [] = [];
if (command.indexOf("commander") >= 0) {
typingFiles = [commander, jquery, lodash, cordova];
}
else {
typingFiles = [grunt, gulp];
}
executeCommand(this, host, typingFiles.map(f => f.typings), typingFiles, requestKind, cb);
}
else {
installedTypings = ["@types/grunt", "@types/gulp", "@types/forever"];
typingFiles = [grunt, gulp, forever];
}
if (execHelper(this, host, installedTypings, typingFiles, prefix, cb)) {
super.execAsync(prefix, command, cwd, requestId, cb);
executeCommand(this, host, [], [], requestKind, cb);
}
}
})();
// Create project #1 with 6 typings
// Create project #1 with 4 typings
const projectService = createProjectService(host, { typingsInstaller: installer });
const projectFileName1 = "/a/app/test1.csproj";
projectService.openExternalProject({
projectFileName: projectFileName1,
options: { allowJS: true, moduleResolution: ModuleResolutionKind.NodeJs },
rootFiles: [toExternalFile(file1.path), toExternalFile(file2.path), toExternalFile(file3.path)],
typingOptions: { include: ["jquery", "moment", "express", "cordova"] }
rootFiles: [toExternalFile(lodashJs.path), toExternalFile(commanderJs.path), toExternalFile(file3.path)],
typingOptions: { include: ["jquery", "cordova" ] }
});
// Create project #2 with 3 typings
installer.checkPendingCommands([TI.NpmViewRequest, TI.NpmViewRequest, TI.NpmViewRequest]);
assert.equal(installer.pendingRunRequests.length, 1, "expect one throttled request");
// Create project #2 with 2 typings
const projectFileName2 = "/a/app/test2.csproj";
projectService.openExternalProject({
projectFileName: projectFileName2,
options: { allowJS: true, moduleResolution: ModuleResolutionKind.NodeJs },
rootFiles: [toExternalFile(file3.path)],
typingOptions: { include: ["grunt", "gulp", "forever"] }
typingOptions: { include: ["grunt", "gulp"] }
});
assert.equal(installer.pendingRunRequests.length, 3, "expect three throttled request");
const p1 = projectService.externalProjects[0];
const p2 = projectService.externalProjects[1];
projectService.checkNumberOfProjects({ externalProjects: 2 });
checkProjectActualFiles(p1, [file1.path, file2.path, file3.path]);
checkProjectActualFiles(p1, [lodashJs.path, commanderJs.path, file3.path]);
checkProjectActualFiles(p2, [file3.path]);
// The npm view count should be 5 even though there are 9 typings to acquire.
// The throttle limit has been reached so no more run execAsync requests will be
// queued until these have been processed. Assert that typings from the second
// run install request have not been queued.
assert.isTrue(npmViewCount === 5);
assert.isTrue(npmInstallCount === 0);
assert.isFalse(hasRunInstall2Typings);
installer.runPostExecActions();
installer.executePendingCommands();
// expected one view request from the first project and two - from the second one
installer.checkPendingCommands([TI.NpmViewRequest, TI.NpmViewRequest, TI.NpmViewRequest]);
assert.equal(installer.pendingRunRequests.length, 0, "expected no throttled requests");
assert.isTrue(npmViewCount === 9);
assert.isTrue(npmInstallCount === 2);
assert.isTrue(hasRunInstall2Typings);
checkProjectActualFiles(p1, [file1.path, file2.path, file3.path, commander.path, express.path, jquery.path, moment.path, lodash.path, cordova.path]);
checkProjectActualFiles(p2, [file3.path, grunt.path, gulp.path, forever.path]);
installer.executePendingCommands();
// should be two install requests from both projects
installer.checkPendingCommands([TI.NpmInstallRequest, TI.NpmInstallRequest]);
installer.executePendingCommands();
checkProjectActualFiles(p1, [lodashJs.path, commanderJs.path, file3.path, commander.path, jquery.path, lodash.path, cordova.path]);
checkProjectActualFiles(p2, [file3.path, grunt.path, gulp.path ]);
});
});
}

View file

@ -6,6 +6,11 @@ namespace ts.server.typingsInstaller {
appendFileSync(file: string, content: string): void
} = require("fs");
const path: {
join(...parts: string[]): string;
dirname(path: string): string;
} = require("path");
class FileLog implements Log {
constructor(private readonly logFile?: string) {
}
@ -19,12 +24,17 @@ namespace ts.server.typingsInstaller {
}
export class NodeTypingsInstaller extends TypingsInstaller {
private exec: { (command: string, options: { cwd: string }, callback?: (error: Error, stdout: string, stderr: string) => void): any };
private readonly exec: { (command: string, options: { cwd: string }, callback?: (error: Error, stdout: string, stderr: string) => void): any };
readonly installTypingHost: InstallTypingHost = sys;
constructor(globalTypingsCacheLocation: string, log: Log) {
super(globalTypingsCacheLocation, toPath("typingSafeList.json", __dirname, createGetCanonicalFileName(sys.useCaseSensitiveFileNames)), log);
constructor(globalTypingsCacheLocation: string, throttleLimit: number, log: Log) {
super(
globalTypingsCacheLocation,
/*npmPath*/ `"${path.join(path.dirname(process.argv[0]), "npm")}"`,
toPath("typingSafeList.json", __dirname, createGetCanonicalFileName(sys.useCaseSensitiveFileNames)),
throttleLimit,
log);
if (this.log.isEnabled()) {
this.log.writeLine(`Process id: ${process.pid}`);
}
@ -55,16 +65,16 @@ namespace ts.server.typingsInstaller {
}
}
protected execAsync(prefix: string, command: string, cwd: string, requestId: number, cb: (err: Error, stdout: string, stderr: string) => void) {
protected runCommand(requestKind: RequestKind, requestId: number, command: string, cwd: string, onRequestCompleted: RequestCompletedAction): void {
if (this.log.isEnabled()) {
this.log.writeLine(`#${requestId} running command '${command}'.`);
}
this.exec(command, { cwd }, (err, stdout, stderr) => {
if (this.log.isEnabled()) {
this.log.writeLine(`${prefix} #${requestId} stdout: ${stdout}`);
this.log.writeLine(`${prefix} #${requestId} stderr: ${stderr}`);
this.log.writeLine(`${requestKind} #${requestId} stdout: ${stdout}`);
this.log.writeLine(`${requestKind} #${requestId} stderr: ${stderr}`);
}
cb(err, stdout, stderr);
onRequestCompleted(err, stdout, stderr);
});
}
}
@ -90,6 +100,6 @@ namespace ts.server.typingsInstaller {
}
process.exit(0);
});
const installer = new NodeTypingsInstaller(globalTypingsCacheLocation, log);
const installer = new NodeTypingsInstaller(globalTypingsCacheLocation, /*throttleLimit*/5, log);
installer.init();
}

View file

@ -8,19 +8,11 @@ namespace ts.server.typingsInstaller {
devDependencies: MapLike<any>;
}
interface RunInstallRequest {
readonly cachePath: string;
readonly typingsToInstall: string[];
readonly postInstallAction: (installedTypings: string[]) => void;
}
export interface Log {
isEnabled(): boolean;
writeLine(text: string): void;
}
const throttleLimit = 5;
const nullLog: Log = {
isEnabled: () => false,
writeLine: () => {}
@ -31,27 +23,44 @@ namespace ts.server.typingsInstaller {
return result.resolvedModule && result.resolvedModule.resolvedFileName;
}
export const NpmViewRequest: "npm view" = "npm view";
export const NpmInstallRequest: "npm install" = "npm install";
export type RequestKind = typeof NpmViewRequest | typeof NpmInstallRequest;
export type RequestCompletedAction = (err: Error, stdout: string, stderr: string) => void;
type PendingRequest = {
requestKind: RequestKind;
requestId: number;
command: string;
cwd: string;
onRequestCompleted: RequestCompletedAction
};
export abstract class TypingsInstaller {
private readonly packageNameToTypingLocation: Map<string> = createMap<string>();
private readonly missingTypingsSet: Map<true> = createMap<true>();
private readonly knownCachesSet: Map<true> = createMap<true>();
private readonly projectWatchers: Map<FileWatcher[]> = createMap<FileWatcher[]>();
readonly pendingRunRequests: PendingRequest[] = [];
private installRunCount = 1;
private throttleCount = 0;
private packageNameToTypingLocation: Map<string> = createMap<string>();
private missingTypingsSet: Map<true> = createMap<true>();
private knownCachesSet: Map<true> = createMap<true>();
private projectWatchers: Map<FileWatcher[]> = createMap<FileWatcher[]>();
private delayedRunInstallRequests: RunInstallRequest[] = [];
private npmPath: string;
private inFlightRequestCount = 0;
abstract readonly installTypingHost: InstallTypingHost;
constructor(readonly globalCachePath: string, readonly safeListPath: Path, protected readonly log = nullLog) {
constructor(
readonly globalCachePath: string,
readonly npmPath: string,
readonly safeListPath: Path,
readonly throttleLimit: number,
protected readonly log = nullLog) {
if (this.log.isEnabled()) {
this.log.writeLine(`Global cache location '${globalCachePath}', safe file path '${safeListPath}'`);
}
}
init() {
const path = require("path");
this.npmPath = `"${path.join(path.dirname(process.argv[0]), "npm")}"`;
this.processCacheLocation(this.globalCachePath);
}
@ -239,77 +248,37 @@ namespace ts.server.typingsInstaller {
}
private runInstall(cachePath: string, typingsToInstall: string[], postInstallAction: (installedTypings: string[]) => void): void {
if (this.throttleCount === throttleLimit) {
const request = {
cachePath: cachePath,
typingsToInstall: typingsToInstall,
postInstallAction: postInstallAction
};
this.delayedRunInstallRequests.push(request);
return;
}
const id = this.installRunCount;
const requestId = this.installRunCount;
this.installRunCount++;
let execInstallCmdCount = 0;
const filteredTypings: string[] = [];
const delayedTypingsToInstall: string[] = [];
for (const typing of typingsToInstall) {
if (this.throttleCount === throttleLimit) {
delayedTypingsToInstall.push(typing);
continue;
}
execNpmViewTyping(this, typing);
}
function execNpmViewTyping(self: TypingsInstaller, typing: string) {
self.throttleCount++;
const command = `${self.npmPath} view @types/${typing} --silent name`;
self.execAsync("npm view", command, cachePath, id, (err, stdout, stderr) => {
self.execAsync(NpmViewRequest, requestId, command, cachePath, (err, stdout, stderr) => {
if (stdout) {
filteredTypings.push(typing);
}
execInstallCmdCount++;
self.throttleCount--;
if (delayedTypingsToInstall.length > 0) {
return execNpmViewTyping(self, delayedTypingsToInstall.pop());
}
if (execInstallCmdCount === typingsToInstall.length) {
installFilteredTypings(self, filteredTypings);
if (self.delayedRunInstallRequests.length > 0) {
const request = self.delayedRunInstallRequests.pop();
return self.runInstall(request.cachePath, request.typingsToInstall, request.postInstallAction);
}
}
});
}
function installFilteredTypings(self: TypingsInstaller, filteredTypings: string[]) {
if (filteredTypings.length === 0) {
reportInstalledTypings(self);
postInstallAction([]);
return;
}
const command = `${self.npmPath} install ${filteredTypings.map(t => "@types/" + t).join(" ")} --save-dev`;
self.execAsync("npm install", command, cachePath, id, (err, stdout, stderr) => {
if (stdout) {
reportInstalledTypings(self);
}
});
}
function reportInstalledTypings(self: TypingsInstaller) {
const command = `${self.npmPath} ls -json`;
self.execAsync("npm ls", command, cachePath, id, (err, stdout, stderr) => {
let installedTypings: string[];
try {
const response = JSON.parse(stdout);
if (response.dependencies) {
installedTypings = getOwnKeys(response.dependencies);
}
}
catch (e) {
self.log.writeLine(`Error parsing installed @types dependencies. Error details: ${e.message}`);
}
postInstallAction(installedTypings || []);
const scopedTypings = filteredTypings.map(t => "@types/" + t);
const command = `${self.npmPath} install ${scopedTypings.join(" ")} --save-dev`;
self.execAsync(NpmInstallRequest, requestId, command, cachePath, (err, stdout, stderr) => {
postInstallAction(stdout ? scopedTypings : []);
});
}
}
@ -357,7 +326,24 @@ namespace ts.server.typingsInstaller {
};
}
protected abstract execAsync(prefix: string, command: string, cwd: string, requestId: number, cb: (err: Error, stdout: string, stderr: string) => void): void;
private execAsync(requestKind: RequestKind, requestId: number, command: string, cwd: string, onRequestCompleted: RequestCompletedAction): void {
this.pendingRunRequests.unshift({ requestKind, requestId, command, cwd, onRequestCompleted });
this.executeWithThrottling();
}
private executeWithThrottling() {
while (this.inFlightRequestCount < this.throttleLimit && this.pendingRunRequests.length) {
this.inFlightRequestCount++;
const request = this.pendingRunRequests.pop();
this.runCommand(request.requestKind, request.requestId, request.command, request.cwd, (err, stdout, stderr) => {
this.inFlightRequestCount--;
request.onRequestCompleted(err, stdout, stderr);
this.executeWithThrottling();
});
}
}
protected abstract runCommand(requestKind: RequestKind, requestId: number, command: string, cwd: string, onRequestCompleted: RequestCompletedAction): void;
protected abstract sendResponse(response: SetTypings | InvalidateCachedTypings): void;
}
}