maxmustermann/TypeScript
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Invalidation + separated downstream builds

Browse source
This commit is contained in:
Ryan Cavanaugh 2018-05-31 14:38:26 -07:00
parent cd7a844a48
commit d21c03ab9d
3 changed files with 227 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/compiler/diagnosticMessages.json
View file

@ -3685,6 +3685,10 @@
"category": "Error",
"code": 6369
},
"Skipping clean because not all projects could be located": {
"category": "Error",
"code": 6340
},
"Variable '{0}' implicitly has an '{1}' type.": {
"category": "Error",

194
src/compiler/tsbuild.ts
View file

@ -38,6 +38,10 @@ namespace ts {
* Issue a verbose diagnostic message. No-ops when options.verbose is false.
*/
verbose(diag: DiagnosticMessage, ...args: any[]): void;
invalidatedProjects: FileMap<true>;
queuedProjects: FileMap<true>;
missingRoots: Map<true>;
}
type Mapper = ReturnType<typeof createDependencyMapper>;
@ -73,7 +77,7 @@ namespace ts {
AnyErrors = ConfigFileErrors | SyntaxErrors | TypeErrors | DeclarationEmitErrors
}
enum UpToDateStatusType {
export enum UpToDateStatusType {
Unbuildable,
UpToDate,
/**
@ -89,7 +93,7 @@ namespace ts {
UpstreamBlocked
}
type UpToDateStatus =
export type UpToDateStatus =
| Status.Unbuildable
| Status.UpToDate
| Status.OutputMissing
@ -98,7 +102,7 @@ namespace ts {
| Status.UpstreamOutOfDate
| Status.UpstreamBlocked;
namespace Status {
export namespace Status {
/**
* The project can't be built at all in its current state. For example,
* its config file cannot be parsed, or it has a syntax error or missing file
@ -170,6 +174,9 @@ namespace ts {
setValue(fileName: string, value: T): void;
getValue(fileName: string): T | never;
getValueOrUndefined(fileName: string): T | undefined;
hasKey(fileName: string): boolean;
removeKey(fileName: string): void;
getKeys(): string[];
}
/**
@ -183,8 +190,23 @@ namespace ts {
setValue,
getValue,
getValueOrUndefined,
removeKey,
getKeys,
hasKey
};
function getKeys(): string[] {
return Object.keys(lookup);
}
function hasKey(fileName: string) {
return normalizePath(fileName) in lookup;
}
function removeKey(fileName: string) {
delete lookup[fileName];
}
function setValue(fileName: string, value: T) {
lookup[normalizePath(fileName)] = value;
}
@ -211,30 +233,30 @@ namespace ts {
}
export function createDependencyMapper() {
const childToParents: { [key: string]: string[] } = {};
const parentToChildren: { [key: string]: string[] } = {};
const allKeys: string[] = [];
const childToParents: { [key: string]: ResolvedConfigFileName[] } = {};
const parentToChildren: { [key: string]: ResolvedConfigFileName[] } = {};
const allKeys: ResolvedConfigFileName[] = [];
function addReference(childConfigFileName: string, parentConfigFileName: string): void {
function addReference(childConfigFileName: ResolvedConfigFileName, parentConfigFileName: ResolvedConfigFileName): void {
addEntry(childToParents, childConfigFileName, parentConfigFileName);
addEntry(parentToChildren, parentConfigFileName, childConfigFileName);
}
function getReferencesTo(parentConfigFileName: string): string[] {
function getReferencesTo(parentConfigFileName: ResolvedConfigFileName): ResolvedConfigFileName[] {
return parentToChildren[normalizePath(parentConfigFileName)] || [];
}
function getReferencesOf(childConfigFileName: string): string[] {
function getReferencesOf(childConfigFileName: ResolvedConfigFileName): ResolvedConfigFileName[] {
return childToParents[normalizePath(childConfigFileName)] || [];
}
function getKeys(): ReadonlyArray<string> {
function getKeys(): ReadonlyArray<ResolvedConfigFileName> {
return allKeys;
}
function addEntry(mapToAddTo: typeof childToParents | typeof parentToChildren, key: string, element: string) {
key = normalizePath(key);
element = normalizePath(element);
function addEntry(mapToAddTo: typeof childToParents | typeof parentToChildren, key: ResolvedConfigFileName, element: ResolvedConfigFileName) {
key = normalizePath(key) as ResolvedConfigFileName;
element = normalizePath(element) as ResolvedConfigFileName;
const arr = (mapToAddTo[key] = mapToAddTo[key] || []);
if (arr.indexOf(element) < 0) {
arr.push(element);
@ -316,8 +338,13 @@ namespace ts {
return parsed;
}
function removeKey(configFilePath: ResolvedConfigFileName) {
cache.removeKey(configFilePath);
}
return {
parseConfigFile
parseConfigFile,
removeKey
};
}
@ -331,11 +358,19 @@ namespace ts {
export function createBuildContext(options: BuildOptions, reportDiagnostic: DiagnosticReporter): BuildContext {
const verboseDiag = options.verbose && reportDiagnostic;
const invalidatedProjects = createFileMap<true>();
const queuedProjects = createFileMap<true>();
const missingRoots = createMap<true>();
return {
options,
projectStatus: createFileMap(),
unchangedOutputs: createFileMap(),
verbose: verboseDiag ? (diag, ...args) => verboseDiag(createCompilerDiagnostic(diag, ...args)) : () => undefined
verbose: verboseDiag ? (diag, ...args) => verboseDiag(createCompilerDiagnostic(diag, ...args)) : () => undefined,
invalidatedProjects,
missingRoots,
queuedProjects
};
}
@ -437,12 +472,12 @@ namespace ts {
addProject(".");
}
const builder = createSolutionBuilder(host, reportDiagnostic, { verbose, dry, force });
const builder = createSolutionBuilder(host, projects, reportDiagnostic, { verbose, dry, force });
if (clean) {
builder.cleanProjects(projects);
builder.cleanAllProjects();
}
else {
builder.buildProjects(projects);
builder.buildAllProjects();
}
function addProject(projectSpecification: string) {
@ -461,7 +496,11 @@ namespace ts {
}
}
export function createSolutionBuilder(host: CompilerHost, reportDiagnostic: DiagnosticReporter, defaultOptions: BuildOptions) {
/**
* A SolutionBuilder has an immutable set of rootNames that are the "entry point" projects, but
* can dynamically add/remove other projects based on changes on the rootNames' references
*/
export function createSolutionBuilder(host: CompilerHost, rootNames: ReadonlyArray<string>, reportDiagnostic: DiagnosticReporter, defaultOptions: BuildOptions) {
if (!host.getModifiedTime || !host.setModifiedTime) {
throw new Error("Host must support timestamp APIs");
}
@ -470,12 +509,18 @@ namespace ts {
let context = createBuildContext(defaultOptions, reportDiagnostic);
return {
buildAllProjects,
getUpToDateStatus,
getUpToDateStatusOfFile,
buildProjects,
cleanProjects,
cleanAllProjects,
resetBuildContext,
getBuildGraph
getBuildGraph,
invalidateProject,
buildInvalidatedProjects,
buildDependentInvalidatedProjects,
resolveProjectName
};
function resetBuildContext(opts = defaultOptions) {
@ -486,13 +531,17 @@ namespace ts {
return getUpToDateStatus(configFileCache.parseConfigFile(configFileName));
}
function getBuildGraph(configFileNames: string[]) {
function getBuildGraph(configFileNames: ReadonlyArray<string>) {
const resolvedNames: ResolvedConfigFileName[] | undefined = resolveProjectNames(configFileNames);
if (resolvedNames === undefined) return;
if (resolvedNames === undefined) return undefined;
return createDependencyGraph(resolvedNames);
}
function getGlobalDependencyGraph() {
return getBuildGraph(rootNames);
}
function getUpToDateStatus(project: ParsedCommandLine | undefined): UpToDateStatus {
if (project === undefined) {
return { type: UpToDateStatusType.Unbuildable, reason: "File deleted mid-build" };
@ -507,6 +556,73 @@ namespace ts {
return actual;
}
function invalidateProject(configFileName: string) {
const resolved = resolveProjectName(configFileName);
if (resolved === undefined) {
// If this was a rootName, we need to track it as missing.
// Otherwise we can just ignore it and have it possibly surface as an error in any downstream projects,
// if they exist
// TODO: do those things
return;
}
configFileCache.removeKey(resolved);
context.invalidatedProjects.setValue(resolved, true);
context.projectStatus.removeKey(resolved);
const graph = getGlobalDependencyGraph()!;
if (graph) {
queueBuildForDownstreamReferences(resolved);
}
// Mark all downstream projects of this one needing to be built "later"
function queueBuildForDownstreamReferences(root: ResolvedConfigFileName) {
debugger;
const deps = graph.dependencyMap.getReferencesTo(root);
for (const ref of deps) {
// Can skip circular references
if (!context.queuedProjects.hasKey(ref)) {
context.queuedProjects.setValue(ref, true);
queueBuildForDownstreamReferences(ref);
}
}
}
}
function buildInvalidatedProjects() {
buildSomeProjects(p => context.invalidatedProjects.hasKey(p));
}
function buildDependentInvalidatedProjects() {
buildSomeProjects(p => context.queuedProjects.hasKey(p));
}
function buildSomeProjects(predicate: (projName: ResolvedConfigFileName) => boolean) {
const resolvedNames: ResolvedConfigFileName[] | undefined = resolveProjectNames(rootNames);
if (resolvedNames === undefined) return;
const graph = createDependencyGraph(resolvedNames)!;
for (const next of graph.buildQueue) {
if (!predicate(next)) continue;
const resolved = resolveProjectName(next);
if (!resolved) continue; // ??
const proj = configFileCache.parseConfigFile(resolved);
if (!proj) continue; // ?
const status = getUpToDateStatus(proj);
reportProjectStatus(next, status);
if (status.type === UpToDateStatusType.UpstreamBlocked) {
context.verbose(Diagnostics.Skipping_build_of_project_0_because_its_upstream_project_1_has_errors, resolved, status.upstreamProjectName);
continue;
}
buildSingleProject(next);
}
}
function getAllProjectOutputs(project: ParsedCommandLine): ReadonlyArray<string> {
if (project.options.outFile) {
return getOutFileOutputs(project);
@ -824,7 +940,7 @@ namespace ts {
context.projectStatus.setValue(proj.options.configFilePath!, { type: UpToDateStatusType.UpToDate, newestDeclarationFileContentChangedTime: priorNewestUpdateTime } as UpToDateStatus);
}
function getFilesToClean(configFileNames: ResolvedConfigFileName[]): string[] | undefined {
function getFilesToClean(configFileNames: ReadonlyArray<ResolvedConfigFileName>): string[] | undefined {
const resolvedNames: ResolvedConfigFileName[] | undefined = resolveProjectNames(configFileNames);
if (resolvedNames === undefined) return undefined;
@ -849,12 +965,24 @@ namespace ts {
return filesToDelete;
}
function cleanProjects(configFileNames: string[]) {
const resolvedNames: ResolvedConfigFileName[] | undefined = resolveProjectNames(configFileNames);
if (resolvedNames === undefined) return;
function getAllProjectsInScope(): ReadonlyArray<ResolvedConfigFileName> | undefined {
const resolvedNames = resolveProjectNames(rootNames);
if (resolvedNames === undefined) return undefined;
const graph = createDependencyGraph(resolvedNames);
if (graph === undefined) return undefined;
return graph.buildQueue;
}
function cleanAllProjects() {
const resolvedNames: ReadonlyArray<ResolvedConfigFileName> | undefined = getAllProjectsInScope();
if (resolvedNames === undefined) {
reportDiagnostic(createCompilerDiagnostic(Diagnostics.Skipping_clean_because_not_all_projects_could_be_located));
return;
}
const filesToDelete = getFilesToClean(resolvedNames);
if (filesToDelete === undefined) {
reportDiagnostic(createCompilerDiagnostic(Diagnostics.Skipping_clean_because_not_all_projects_could_be_located));
return;
}
@ -885,7 +1013,7 @@ namespace ts {
return undefined;
}
function resolveProjectNames(configFileNames: string[]): ResolvedConfigFileName[] | undefined {
function resolveProjectNames(configFileNames: ReadonlyArray<string>): ResolvedConfigFileName[] | undefined {
const resolvedNames: ResolvedConfigFileName[] = [];
for (const name of configFileNames) {
const resolved = resolveProjectName(name);
@ -897,12 +1025,8 @@ namespace ts {
return resolvedNames;
}
function buildProjects(configFileNames: string[]) {
const resolvedNames: ResolvedConfigFileName[] | undefined = resolveProjectNames(configFileNames);
if (resolvedNames === undefined) return;
// Establish what needs to be built
const graph = createDependencyGraph(resolvedNames);
function buildAllProjects() {
const graph = getGlobalDependencyGraph();
if (graph === undefined) return;
const queue = graph.buildQueue;

96
src/harness/unittests/tsbuild.ts
View file

@ -19,11 +19,10 @@ namespace ts {
it("can build the sample project 'sample1' without error", () => {
const fs = bfs.shadow();
const host = new fakes.CompilerHost(fs);
const builder = createSolutionBuilder(host, reportDiagnostic, { dry: false, force: false, verbose: false });
const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: false, verbose: false });
clearDiagnostics();
fs.chdir("/src/tests");
builder.buildProjects(["."]);
builder.buildAllProjects();
assertDiagnosticMessages(/*empty*/);
// Check for outputs. Not an exhaustive list
@ -38,9 +37,8 @@ namespace ts {
clearDiagnostics();
const fs = bfs.shadow();
const host = new fakes.CompilerHost(fs);
const builder = createSolutionBuilder(host, reportDiagnostic, { dry: true, force: false, verbose: false });
fs.chdir("/src/tests");
builder.buildProjects(["."]);
const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: true, force: false, verbose: false });
builder.buildAllProjects();
assertDiagnosticMessages(Diagnostics.Would_build_project_0, Diagnostics.Would_build_project_0, Diagnostics.Would_build_project_0);
// Check for outputs to not be written. Not an exhaustive list
@ -54,14 +52,13 @@ namespace ts {
const fs = bfs.shadow();
const host = new fakes.CompilerHost(fs);
let builder = createSolutionBuilder(host, reportDiagnostic, { dry: false, force: false, verbose: false });
fs.chdir("/src/tests");
builder.buildProjects(["."]);
let builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: false, verbose: false });
builder.buildAllProjects();
tick();
clearDiagnostics();
builder = createSolutionBuilder(host, reportDiagnostic, { dry: true, force: false, verbose: false });
builder.buildProjects(["."]);
builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: true, force: false, verbose: false });
builder.buildAllProjects();
assertDiagnosticMessages(Diagnostics.Project_0_is_up_to_date, Diagnostics.Project_0_is_up_to_date, Diagnostics.Project_0_is_up_to_date);
});
});
@ -72,20 +69,19 @@ namespace ts {
const fs = bfs.shadow();
const host = new fakes.CompilerHost(fs);
const builder = createSolutionBuilder(host, reportDiagnostic, { dry: false, force: false, verbose: false });
fs.chdir("/src/tests");
builder.buildProjects(["."]);
const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: false, verbose: false });
builder.buildAllProjects();
// Verify they exist
for (const output of allExpectedOutputs) {
assert(fs.existsSync(output), `Expect file ${output} to exist`);
}
builder.cleanProjects(["."]);
builder.cleanAllProjects();
// Verify they are gone
for (const output of allExpectedOutputs) {
assert(!fs.existsSync(output), `Expect file ${output} to not exist`);
}
// Subsequent clean shouldn't throw / etc
builder.cleanProjects(["."]);
builder.cleanAllProjects();
});
});
@ -94,16 +90,15 @@ namespace ts {
const fs = bfs.shadow();
const host = new fakes.CompilerHost(fs);
const builder = createSolutionBuilder(host, reportDiagnostic, { dry: false, force: true, verbose: false });
fs.chdir("/src/tests");
builder.buildProjects(["."]);
const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: true, verbose: false });
builder.buildAllProjects();
let currentTime = time();
checkOutputTimestamps(currentTime);
tick();
Debug.assert(time() !== currentTime, "Time moves on");
currentTime = time();
builder.buildProjects(["."]);
builder.buildAllProjects();
checkOutputTimestamps(currentTime);
function checkOutputTimestamps(expected: number) {
@ -119,14 +114,12 @@ namespace ts {
describe("tsbuild - can detect when and what to rebuild", () => {
const fs = bfs.shadow();
const host = new fakes.CompilerHost(fs);
const builder = createSolutionBuilder(host, reportDiagnostic, { dry: false, force: false, verbose: true });
fs.chdir("/src/tests");
const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: false, verbose: true });
it("Builds the project", () => {
clearDiagnostics();
builder.resetBuildContext();
builder.buildProjects(["."]);
builder.buildAllProjects();
assertDiagnosticMessages(Diagnostics.Sorted_list_of_input_projects_Colon_0,
Diagnostics.Project_0_is_out_of_date_because_output_file_1_does_not_exist,
Diagnostics.Building_project_0,
@ -141,7 +134,7 @@ namespace ts {
it("Detects that all projects are up to date", () => {
clearDiagnostics();
builder.resetBuildContext();
builder.buildProjects(["."]);
builder.buildAllProjects();
assertDiagnosticMessages(Diagnostics.Sorted_list_of_input_projects_Colon_0,
Diagnostics.Project_0_is_up_to_date_because_newest_input_1_is_older_than_oldest_output_2,
Diagnostics.Project_0_is_up_to_date_because_newest_input_1_is_older_than_oldest_output_2,
@ -154,7 +147,7 @@ namespace ts {
clearDiagnostics();
fs.writeFileSync("/src/tests/index.ts", "const m = 10;");
builder.resetBuildContext();
builder.buildProjects(["."]);
builder.buildAllProjects();
assertDiagnosticMessages(Diagnostics.Sorted_list_of_input_projects_Colon_0,
Diagnostics.Project_0_is_up_to_date_because_newest_input_1_is_older_than_oldest_output_2,
@ -169,7 +162,7 @@ namespace ts {
clearDiagnostics();
replaceText(fs, "/src/core/index.ts", "HELLO WORLD", "WELCOME PLANET");
builder.resetBuildContext();
builder.buildProjects(["."]);
builder.buildAllProjects();
assertDiagnosticMessages(Diagnostics.Sorted_list_of_input_projects_Colon_0,
Diagnostics.Project_0_is_out_of_date_because_oldest_output_1_is_older_than_newest_input_2,
@ -185,14 +178,13 @@ namespace ts {
it("won't build downstream projects if upstream projects have errors", () => {
const fs = bfs.shadow();
const host = new fakes.CompilerHost(fs);
const builder = createSolutionBuilder(host, reportDiagnostic, { dry: false, force: false, verbose: true });
const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: false, verbose: true });
clearDiagnostics();
// Induce an error in the middle project
replaceText(fs, "/src/logic/index.ts", "c.multiply(10, 15)", `c.muitply()`);
fs.chdir("/src/tests");
builder.buildProjects(["."]);
builder.buildAllProjects();
assertDiagnosticMessages(
Diagnostics.Sorted_list_of_input_projects_Colon_0,
Diagnostics.Project_0_is_out_of_date_because_output_file_1_does_not_exist,
@ -221,8 +213,6 @@ namespace ts {
writeProjects(fs, ["A", "B", "C", "D", "E", "F", "G"], deps);
const builder = createSolutionBuilder(host, reportDiagnostic, { dry: true, force: false, verbose: false });
it("orders the graph correctly - specify two roots", () => {
checkGraphOrdering(["A", "G"], ["A", "B", "C", "D", "E", "G"]);
});
@ -240,6 +230,8 @@ namespace ts {
});
function checkGraphOrdering(rootNames: string[], expectedBuildSet: string[]) {
const builder = createSolutionBuilder(host, rootNames, reportDiagnostic, { dry: true, force: false, verbose: false });
const projFileNames = rootNames.map(getProjectFileName);
const graph = builder.getBuildGraph(projFileNames);
if (graph === undefined) throw new Error("Graph shouldn't be undefined");
@ -280,6 +272,39 @@ namespace ts {
}
});
describe("tsbuild - project invalidation", () => {
it ("invalidates projects correctly", () => {
const fs = bfs.shadow();
const host = new fakes.CompilerHost(fs);
const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: false, verbose: false });
clearDiagnostics();
builder.buildAllProjects();
assertDiagnosticMessages(/*empty*/);
// Update a timestamp in the middle project
tick();
touch(fs, "/src/logic/index.ts");
// Because we haven't reset the build context, the builder should assume there's nothing to do right now
const status = builder.getUpToDateStatusOfFile(builder.resolveProjectName("/src/logic")!);
assert.equal(status.type, UpToDateStatusType.UpToDate, "Project should be assumed to be up-to-date");
// Rebuild this project
tick();
builder.invalidateProject("/src/logic");
builder.buildInvalidatedProjects();
// The file should be updated
assert.equal(fs.statSync("/src/logic/index.js").mtimeMs, time(), "JS file should have been rebuilt");
assert.isBelow(fs.statSync("/src/tests/index.js").mtimeMs, time(), "Downstream JS file should *not* have been rebuilt");
// Build downstream projects should update 'tests', but not 'core'
tick();
builder.buildDependentInvalidatedProjects();
assert.equal(fs.statSync("/src/tests/index.js").mtimeMs, time(), "Downstream JS file should have been rebuilt");
assert.isBelow(fs.statSync("/src/core/index.js").mtimeMs, time(), "Upstream JS file should not have been rebuilt");
});
});
function replaceText(fs: vfs.FileSystem, path: string, oldText: string, newText: string) {
if (!fs.statSync(path).isFile()) {
throw new Error(`File ${path} does not exist`);
@ -324,6 +349,13 @@ namespace ts {
return currentTime;
}
function touch(fs: vfs.FileSystem, path: string) {
if (!fs.statSync(path).isFile()) {
throw new Error(`File ${path} does not exist`);
}
fs.utimesSync(path, new Date(time()), new Date(time()));
}
function loadFsMirror(vfs: vfs.FileSystem, localRoot: string, virtualRoot: string) {
vfs.mkdirpSync(virtualRoot);
for (const path of Harness.IO.readDirectory(localRoot)) {