diff --git a/src/compiler/diagnosticMessages.json b/src/compiler/diagnosticMessages.json index f76b0a39fb..e9022bff0d 100644 --- a/src/compiler/diagnosticMessages.json +++ b/src/compiler/diagnosticMessages.json @@ -3685,6 +3685,10 @@ "category": "Error", "code": 6369 }, + "Skipping clean because not all projects could be located": { + "category": "Error", + "code": 6340 + }, "Variable '{0}' implicitly has an '{1}' type.": { "category": "Error", diff --git a/src/compiler/tsbuild.ts b/src/compiler/tsbuild.ts index 2a509a8c74..1f36f09393 100644 --- a/src/compiler/tsbuild.ts +++ b/src/compiler/tsbuild.ts @@ -38,6 +38,10 @@ namespace ts { * Issue a verbose diagnostic message. No-ops when options.verbose is false. */ verbose(diag: DiagnosticMessage, ...args: any[]): void; + + invalidatedProjects: FileMap; + queuedProjects: FileMap; + missingRoots: Map; } type Mapper = ReturnType; @@ -73,7 +77,7 @@ namespace ts { AnyErrors = ConfigFileErrors | SyntaxErrors | TypeErrors | DeclarationEmitErrors } - enum UpToDateStatusType { + export enum UpToDateStatusType { Unbuildable, UpToDate, /** @@ -89,7 +93,7 @@ namespace ts { UpstreamBlocked } - type UpToDateStatus = + export type UpToDateStatus = | Status.Unbuildable | Status.UpToDate | Status.OutputMissing @@ -98,7 +102,7 @@ namespace ts { | Status.UpstreamOutOfDate | Status.UpstreamBlocked; - namespace Status { + export namespace Status { /** * The project can't be built at all in its current state. For example, * its config file cannot be parsed, or it has a syntax error or missing file @@ -170,6 +174,9 @@ namespace ts { setValue(fileName: string, value: T): void; getValue(fileName: string): T | never; getValueOrUndefined(fileName: string): T | undefined; + hasKey(fileName: string): boolean; + removeKey(fileName: string): void; + getKeys(): string[]; } /** @@ -183,8 +190,23 @@ namespace ts { setValue, getValue, getValueOrUndefined, + removeKey, + getKeys, + hasKey }; + function getKeys(): string[] { + return Object.keys(lookup); + } + + function hasKey(fileName: string) { + return normalizePath(fileName) in lookup; + } + + function removeKey(fileName: string) { + delete lookup[fileName]; + } + function setValue(fileName: string, value: T) { lookup[normalizePath(fileName)] = value; } @@ -211,30 +233,30 @@ namespace ts { } export function createDependencyMapper() { - const childToParents: { [key: string]: string[] } = {}; - const parentToChildren: { [key: string]: string[] } = {}; - const allKeys: string[] = []; + const childToParents: { [key: string]: ResolvedConfigFileName[] } = {}; + const parentToChildren: { [key: string]: ResolvedConfigFileName[] } = {}; + const allKeys: ResolvedConfigFileName[] = []; - function addReference(childConfigFileName: string, parentConfigFileName: string): void { + function addReference(childConfigFileName: ResolvedConfigFileName, parentConfigFileName: ResolvedConfigFileName): void { addEntry(childToParents, childConfigFileName, parentConfigFileName); addEntry(parentToChildren, parentConfigFileName, childConfigFileName); } - function getReferencesTo(parentConfigFileName: string): string[] { + function getReferencesTo(parentConfigFileName: ResolvedConfigFileName): ResolvedConfigFileName[] { return parentToChildren[normalizePath(parentConfigFileName)] || []; } - function getReferencesOf(childConfigFileName: string): string[] { + function getReferencesOf(childConfigFileName: ResolvedConfigFileName): ResolvedConfigFileName[] { return childToParents[normalizePath(childConfigFileName)] || []; } - function getKeys(): ReadonlyArray { + function getKeys(): ReadonlyArray { return allKeys; } - function addEntry(mapToAddTo: typeof childToParents | typeof parentToChildren, key: string, element: string) { - key = normalizePath(key); - element = normalizePath(element); + function addEntry(mapToAddTo: typeof childToParents | typeof parentToChildren, key: ResolvedConfigFileName, element: ResolvedConfigFileName) { + key = normalizePath(key) as ResolvedConfigFileName; + element = normalizePath(element) as ResolvedConfigFileName; const arr = (mapToAddTo[key] = mapToAddTo[key] || []); if (arr.indexOf(element) < 0) { arr.push(element); @@ -316,8 +338,13 @@ namespace ts { return parsed; } + function removeKey(configFilePath: ResolvedConfigFileName) { + cache.removeKey(configFilePath); + } + return { - parseConfigFile + parseConfigFile, + removeKey }; } @@ -331,11 +358,19 @@ namespace ts { export function createBuildContext(options: BuildOptions, reportDiagnostic: DiagnosticReporter): BuildContext { const verboseDiag = options.verbose && reportDiagnostic; + + const invalidatedProjects = createFileMap(); + const queuedProjects = createFileMap(); + const missingRoots = createMap(); + return { options, projectStatus: createFileMap(), unchangedOutputs: createFileMap(), - verbose: verboseDiag ? (diag, ...args) => verboseDiag(createCompilerDiagnostic(diag, ...args)) : () => undefined + verbose: verboseDiag ? (diag, ...args) => verboseDiag(createCompilerDiagnostic(diag, ...args)) : () => undefined, + invalidatedProjects, + missingRoots, + queuedProjects }; } @@ -437,12 +472,12 @@ namespace ts { addProject("."); } - const builder = createSolutionBuilder(host, reportDiagnostic, { verbose, dry, force }); + const builder = createSolutionBuilder(host, projects, reportDiagnostic, { verbose, dry, force }); if (clean) { - builder.cleanProjects(projects); + builder.cleanAllProjects(); } else { - builder.buildProjects(projects); + builder.buildAllProjects(); } function addProject(projectSpecification: string) { @@ -461,7 +496,11 @@ namespace ts { } } - export function createSolutionBuilder(host: CompilerHost, reportDiagnostic: DiagnosticReporter, defaultOptions: BuildOptions) { + /** + * A SolutionBuilder has an immutable set of rootNames that are the "entry point" projects, but + * can dynamically add/remove other projects based on changes on the rootNames' references + */ + export function createSolutionBuilder(host: CompilerHost, rootNames: ReadonlyArray, reportDiagnostic: DiagnosticReporter, defaultOptions: BuildOptions) { if (!host.getModifiedTime || !host.setModifiedTime) { throw new Error("Host must support timestamp APIs"); } @@ -470,12 +509,18 @@ namespace ts { let context = createBuildContext(defaultOptions, reportDiagnostic); return { + buildAllProjects, getUpToDateStatus, getUpToDateStatusOfFile, - buildProjects, - cleanProjects, + cleanAllProjects, resetBuildContext, - getBuildGraph + getBuildGraph, + + invalidateProject, + buildInvalidatedProjects, + buildDependentInvalidatedProjects, + + resolveProjectName }; function resetBuildContext(opts = defaultOptions) { @@ -486,13 +531,17 @@ namespace ts { return getUpToDateStatus(configFileCache.parseConfigFile(configFileName)); } - function getBuildGraph(configFileNames: string[]) { + function getBuildGraph(configFileNames: ReadonlyArray) { const resolvedNames: ResolvedConfigFileName[] | undefined = resolveProjectNames(configFileNames); - if (resolvedNames === undefined) return; + if (resolvedNames === undefined) return undefined; return createDependencyGraph(resolvedNames); } + function getGlobalDependencyGraph() { + return getBuildGraph(rootNames); + } + function getUpToDateStatus(project: ParsedCommandLine | undefined): UpToDateStatus { if (project === undefined) { return { type: UpToDateStatusType.Unbuildable, reason: "File deleted mid-build" }; @@ -507,6 +556,73 @@ namespace ts { return actual; } + function invalidateProject(configFileName: string) { + const resolved = resolveProjectName(configFileName); + if (resolved === undefined) { + // If this was a rootName, we need to track it as missing. + // Otherwise we can just ignore it and have it possibly surface as an error in any downstream projects, + // if they exist + + // TODO: do those things + return; + } + + configFileCache.removeKey(resolved); + context.invalidatedProjects.setValue(resolved, true); + context.projectStatus.removeKey(resolved); + + const graph = getGlobalDependencyGraph()!; + if (graph) { + queueBuildForDownstreamReferences(resolved); + } + + // Mark all downstream projects of this one needing to be built "later" + function queueBuildForDownstreamReferences(root: ResolvedConfigFileName) { + debugger; + const deps = graph.dependencyMap.getReferencesTo(root); + for (const ref of deps) { + // Can skip circular references + if (!context.queuedProjects.hasKey(ref)) { + context.queuedProjects.setValue(ref, true); + queueBuildForDownstreamReferences(ref); + } + } + } + } + + function buildInvalidatedProjects() { + buildSomeProjects(p => context.invalidatedProjects.hasKey(p)); + } + + function buildDependentInvalidatedProjects() { + buildSomeProjects(p => context.queuedProjects.hasKey(p)); + } + + function buildSomeProjects(predicate: (projName: ResolvedConfigFileName) => boolean) { + const resolvedNames: ResolvedConfigFileName[] | undefined = resolveProjectNames(rootNames); + if (resolvedNames === undefined) return; + + const graph = createDependencyGraph(resolvedNames)!; + for (const next of graph.buildQueue) { + if (!predicate(next)) continue; + + const resolved = resolveProjectName(next); + if (!resolved) continue; // ?? + const proj = configFileCache.parseConfigFile(resolved); + if (!proj) continue; // ? + + const status = getUpToDateStatus(proj); + reportProjectStatus(next, status); + + if (status.type === UpToDateStatusType.UpstreamBlocked) { + context.verbose(Diagnostics.Skipping_build_of_project_0_because_its_upstream_project_1_has_errors, resolved, status.upstreamProjectName); + continue; + } + + buildSingleProject(next); + } + } + function getAllProjectOutputs(project: ParsedCommandLine): ReadonlyArray { if (project.options.outFile) { return getOutFileOutputs(project); @@ -824,7 +940,7 @@ namespace ts { context.projectStatus.setValue(proj.options.configFilePath!, { type: UpToDateStatusType.UpToDate, newestDeclarationFileContentChangedTime: priorNewestUpdateTime } as UpToDateStatus); } - function getFilesToClean(configFileNames: ResolvedConfigFileName[]): string[] | undefined { + function getFilesToClean(configFileNames: ReadonlyArray): string[] | undefined { const resolvedNames: ResolvedConfigFileName[] | undefined = resolveProjectNames(configFileNames); if (resolvedNames === undefined) return undefined; @@ -849,12 +965,24 @@ namespace ts { return filesToDelete; } - function cleanProjects(configFileNames: string[]) { - const resolvedNames: ResolvedConfigFileName[] | undefined = resolveProjectNames(configFileNames); - if (resolvedNames === undefined) return; + function getAllProjectsInScope(): ReadonlyArray | undefined { + const resolvedNames = resolveProjectNames(rootNames); + if (resolvedNames === undefined) return undefined; + const graph = createDependencyGraph(resolvedNames); + if (graph === undefined) return undefined; + return graph.buildQueue; + } + + function cleanAllProjects() { + const resolvedNames: ReadonlyArray | undefined = getAllProjectsInScope(); + if (resolvedNames === undefined) { + reportDiagnostic(createCompilerDiagnostic(Diagnostics.Skipping_clean_because_not_all_projects_could_be_located)); + return; + } const filesToDelete = getFilesToClean(resolvedNames); if (filesToDelete === undefined) { + reportDiagnostic(createCompilerDiagnostic(Diagnostics.Skipping_clean_because_not_all_projects_could_be_located)); return; } @@ -885,7 +1013,7 @@ namespace ts { return undefined; } - function resolveProjectNames(configFileNames: string[]): ResolvedConfigFileName[] | undefined { + function resolveProjectNames(configFileNames: ReadonlyArray): ResolvedConfigFileName[] | undefined { const resolvedNames: ResolvedConfigFileName[] = []; for (const name of configFileNames) { const resolved = resolveProjectName(name); @@ -897,12 +1025,8 @@ namespace ts { return resolvedNames; } - function buildProjects(configFileNames: string[]) { - const resolvedNames: ResolvedConfigFileName[] | undefined = resolveProjectNames(configFileNames); - if (resolvedNames === undefined) return; - - // Establish what needs to be built - const graph = createDependencyGraph(resolvedNames); + function buildAllProjects() { + const graph = getGlobalDependencyGraph(); if (graph === undefined) return; const queue = graph.buildQueue; diff --git a/src/harness/unittests/tsbuild.ts b/src/harness/unittests/tsbuild.ts index ed1fd96795..46fee63a7c 100644 --- a/src/harness/unittests/tsbuild.ts +++ b/src/harness/unittests/tsbuild.ts @@ -19,11 +19,10 @@ namespace ts { it("can build the sample project 'sample1' without error", () => { const fs = bfs.shadow(); const host = new fakes.CompilerHost(fs); - const builder = createSolutionBuilder(host, reportDiagnostic, { dry: false, force: false, verbose: false }); + const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: false, verbose: false }); clearDiagnostics(); - fs.chdir("/src/tests"); - builder.buildProjects(["."]); + builder.buildAllProjects(); assertDiagnosticMessages(/*empty*/); // Check for outputs. Not an exhaustive list @@ -38,9 +37,8 @@ namespace ts { clearDiagnostics(); const fs = bfs.shadow(); const host = new fakes.CompilerHost(fs); - const builder = createSolutionBuilder(host, reportDiagnostic, { dry: true, force: false, verbose: false }); - fs.chdir("/src/tests"); - builder.buildProjects(["."]); + const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: true, force: false, verbose: false }); + builder.buildAllProjects(); assertDiagnosticMessages(Diagnostics.Would_build_project_0, Diagnostics.Would_build_project_0, Diagnostics.Would_build_project_0); // Check for outputs to not be written. Not an exhaustive list @@ -54,14 +52,13 @@ namespace ts { const fs = bfs.shadow(); const host = new fakes.CompilerHost(fs); - let builder = createSolutionBuilder(host, reportDiagnostic, { dry: false, force: false, verbose: false }); - fs.chdir("/src/tests"); - builder.buildProjects(["."]); + let builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: false, verbose: false }); + builder.buildAllProjects(); tick(); clearDiagnostics(); - builder = createSolutionBuilder(host, reportDiagnostic, { dry: true, force: false, verbose: false }); - builder.buildProjects(["."]); + builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: true, force: false, verbose: false }); + builder.buildAllProjects(); assertDiagnosticMessages(Diagnostics.Project_0_is_up_to_date, Diagnostics.Project_0_is_up_to_date, Diagnostics.Project_0_is_up_to_date); }); }); @@ -72,20 +69,19 @@ namespace ts { const fs = bfs.shadow(); const host = new fakes.CompilerHost(fs); - const builder = createSolutionBuilder(host, reportDiagnostic, { dry: false, force: false, verbose: false }); - fs.chdir("/src/tests"); - builder.buildProjects(["."]); + const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: false, verbose: false }); + builder.buildAllProjects(); // Verify they exist for (const output of allExpectedOutputs) { assert(fs.existsSync(output), `Expect file ${output} to exist`); } - builder.cleanProjects(["."]); + builder.cleanAllProjects(); // Verify they are gone for (const output of allExpectedOutputs) { assert(!fs.existsSync(output), `Expect file ${output} to not exist`); } // Subsequent clean shouldn't throw / etc - builder.cleanProjects(["."]); + builder.cleanAllProjects(); }); }); @@ -94,16 +90,15 @@ namespace ts { const fs = bfs.shadow(); const host = new fakes.CompilerHost(fs); - const builder = createSolutionBuilder(host, reportDiagnostic, { dry: false, force: true, verbose: false }); - fs.chdir("/src/tests"); - builder.buildProjects(["."]); + const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: true, verbose: false }); + builder.buildAllProjects(); let currentTime = time(); checkOutputTimestamps(currentTime); tick(); Debug.assert(time() !== currentTime, "Time moves on"); currentTime = time(); - builder.buildProjects(["."]); + builder.buildAllProjects(); checkOutputTimestamps(currentTime); function checkOutputTimestamps(expected: number) { @@ -119,14 +114,12 @@ namespace ts { describe("tsbuild - can detect when and what to rebuild", () => { const fs = bfs.shadow(); const host = new fakes.CompilerHost(fs); - const builder = createSolutionBuilder(host, reportDiagnostic, { dry: false, force: false, verbose: true }); - - fs.chdir("/src/tests"); + const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: false, verbose: true }); it("Builds the project", () => { clearDiagnostics(); builder.resetBuildContext(); - builder.buildProjects(["."]); + builder.buildAllProjects(); assertDiagnosticMessages(Diagnostics.Sorted_list_of_input_projects_Colon_0, Diagnostics.Project_0_is_out_of_date_because_output_file_1_does_not_exist, Diagnostics.Building_project_0, @@ -141,7 +134,7 @@ namespace ts { it("Detects that all projects are up to date", () => { clearDiagnostics(); builder.resetBuildContext(); - builder.buildProjects(["."]); + builder.buildAllProjects(); assertDiagnosticMessages(Diagnostics.Sorted_list_of_input_projects_Colon_0, Diagnostics.Project_0_is_up_to_date_because_newest_input_1_is_older_than_oldest_output_2, Diagnostics.Project_0_is_up_to_date_because_newest_input_1_is_older_than_oldest_output_2, @@ -154,7 +147,7 @@ namespace ts { clearDiagnostics(); fs.writeFileSync("/src/tests/index.ts", "const m = 10;"); builder.resetBuildContext(); - builder.buildProjects(["."]); + builder.buildAllProjects(); assertDiagnosticMessages(Diagnostics.Sorted_list_of_input_projects_Colon_0, Diagnostics.Project_0_is_up_to_date_because_newest_input_1_is_older_than_oldest_output_2, @@ -169,7 +162,7 @@ namespace ts { clearDiagnostics(); replaceText(fs, "/src/core/index.ts", "HELLO WORLD", "WELCOME PLANET"); builder.resetBuildContext(); - builder.buildProjects(["."]); + builder.buildAllProjects(); assertDiagnosticMessages(Diagnostics.Sorted_list_of_input_projects_Colon_0, Diagnostics.Project_0_is_out_of_date_because_oldest_output_1_is_older_than_newest_input_2, @@ -185,14 +178,13 @@ namespace ts { it("won't build downstream projects if upstream projects have errors", () => { const fs = bfs.shadow(); const host = new fakes.CompilerHost(fs); - const builder = createSolutionBuilder(host, reportDiagnostic, { dry: false, force: false, verbose: true }); + const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: false, verbose: true }); clearDiagnostics(); // Induce an error in the middle project replaceText(fs, "/src/logic/index.ts", "c.multiply(10, 15)", `c.muitply()`); - fs.chdir("/src/tests"); - builder.buildProjects(["."]); + builder.buildAllProjects(); assertDiagnosticMessages( Diagnostics.Sorted_list_of_input_projects_Colon_0, Diagnostics.Project_0_is_out_of_date_because_output_file_1_does_not_exist, @@ -221,8 +213,6 @@ namespace ts { writeProjects(fs, ["A", "B", "C", "D", "E", "F", "G"], deps); - const builder = createSolutionBuilder(host, reportDiagnostic, { dry: true, force: false, verbose: false }); - it("orders the graph correctly - specify two roots", () => { checkGraphOrdering(["A", "G"], ["A", "B", "C", "D", "E", "G"]); }); @@ -240,6 +230,8 @@ namespace ts { }); function checkGraphOrdering(rootNames: string[], expectedBuildSet: string[]) { + const builder = createSolutionBuilder(host, rootNames, reportDiagnostic, { dry: true, force: false, verbose: false }); + const projFileNames = rootNames.map(getProjectFileName); const graph = builder.getBuildGraph(projFileNames); if (graph === undefined) throw new Error("Graph shouldn't be undefined"); @@ -280,6 +272,39 @@ namespace ts { } }); + describe("tsbuild - project invalidation", () => { + it ("invalidates projects correctly", () => { + const fs = bfs.shadow(); + const host = new fakes.CompilerHost(fs); + const builder = createSolutionBuilder(host, ["/src/tests"], reportDiagnostic, { dry: false, force: false, verbose: false }); + + clearDiagnostics(); + builder.buildAllProjects(); + assertDiagnosticMessages(/*empty*/); + + // Update a timestamp in the middle project + tick(); + touch(fs, "/src/logic/index.ts"); + // Because we haven't reset the build context, the builder should assume there's nothing to do right now + const status = builder.getUpToDateStatusOfFile(builder.resolveProjectName("/src/logic")!); + assert.equal(status.type, UpToDateStatusType.UpToDate, "Project should be assumed to be up-to-date"); + + // Rebuild this project + tick(); + builder.invalidateProject("/src/logic"); + builder.buildInvalidatedProjects(); + // The file should be updated + assert.equal(fs.statSync("/src/logic/index.js").mtimeMs, time(), "JS file should have been rebuilt"); + assert.isBelow(fs.statSync("/src/tests/index.js").mtimeMs, time(), "Downstream JS file should *not* have been rebuilt"); + + // Build downstream projects should update 'tests', but not 'core' + tick(); + builder.buildDependentInvalidatedProjects(); + assert.equal(fs.statSync("/src/tests/index.js").mtimeMs, time(), "Downstream JS file should have been rebuilt"); + assert.isBelow(fs.statSync("/src/core/index.js").mtimeMs, time(), "Upstream JS file should not have been rebuilt"); + }); + }); + function replaceText(fs: vfs.FileSystem, path: string, oldText: string, newText: string) { if (!fs.statSync(path).isFile()) { throw new Error(`File ${path} does not exist`); @@ -324,6 +349,13 @@ namespace ts { return currentTime; } + function touch(fs: vfs.FileSystem, path: string) { + if (!fs.statSync(path).isFile()) { + throw new Error(`File ${path} does not exist`); + } + fs.utimesSync(path, new Date(time()), new Date(time())); + } + function loadFsMirror(vfs: vfs.FileSystem, localRoot: string, virtualRoot: string) { vfs.mkdirpSync(virtualRoot); for (const path of Harness.IO.readDirectory(localRoot)) {