Attempt to rewrite the jakefile and fail

Jakefile.js

@@ -6,6 +6,7 @@ const fs = require("fs");
 const os = require("os");
 const path = require("path");
 const child_process = require("child_process");
+const removeInternal = require("remove-internal");
 const fold = require("travis-fold");
 const ts = require("./lib/typescript");
 const del = require("del");
@@ -13,60 +14,290 @@ const getDirSize = require("./scripts/build/getDirSize");
 
 const host = process.env.TYPESCRIPT_HOST || process.env.host || "node";
 
+const locales = ["cs", "de", "es", "fr", "it", "ja", "ko", "pl", "pt-br", "ru", "tr", "zh-cn", "zh-tw"];
+
 const defaultTestTimeout = 40000;
 
+let useDebugMode = true;
+
 const TaskNames = {
+    local: "local",
     runtests: "runtests",
     runtestsParallel: "runtests-parallel",
     buildRules: "build-rules",
-    clean: "clean"
-}
+    clean: "clean",
+    lib: "lib",
+    buildFoldStart: "build-fold-start",
+    buildFoldEnd: "build-fold-end",
+    generateDiagnostics: "generate-diagnostics",
+    coreBuild: "core-build",
+    lkg: "LKG",
+    release: "release",
+    lssl: "lssl",
+    lint: "lint"
+};
 
 const Paths = {
+    lkg: "lib",
     lkgCompiler: "lib/tsc.js",
+
     built: "built",
     builtLocal: "built/local",
     builtLocalCompiler: "built/local/tsc.js",
     builtLocalRun: "built/local/run.js",
+    locLcg: "built/local/enu/diagnosticMessages.generated.json.lcg",
+    typesMapOutput: "built/local/typesMap.json",
+    servicesFile: "built/local/typescriptServices.js",
+    servicesDefinitionFile: "built/local/typescriptServices.d.ts",
+    tsserverLibraryFile: "built/local/tsserverlibrary.js",
+    tsserverLibraryDefinitionFile: "built/local/tsserverlibrary.d.ts",
+
     baselines: {
         local: "tests/baselines/local",
         localTest262: "tests/baselines/test262/local",
         localRwc: "tests/baselines/rwc/local",
         reference: "tests/baselines/reference",
         referenceTest262: "tests/baselines/test262/reference",
-        referenceTwc: "tests/baselines/rwc/reference"
-    }
-}
+        referenceRwc: "tests/baselines/rwc/reference"
+    },
+    copyright: "CopyrightNotice.txt",
+    thirdParty: "ThirdPartyNoticeText.txt",
+    library: "src/lib",
+
+    processDiagnosticMessagesJs: "scripts/processDiagnosticMessages.js",
+    diagnosticInformationMap: "src/parser/diagnosticInformationMap.generated.ts",
+    diagnosticMessagesJson: "src/parser/diagnosticMessages.json",
+
+    srcServer: "src/server",
+
+};
 
 const ConfigFileFor = {
     tsc: "src/tsc",
     tsserver: "src/tsserver",
     runjs: "src/testRunner",
-    lint: "scripts/tslint"
+    lint: "scripts/tslint",
+    all: "src"
 };
 
-desc("Runs all the tests in parallel using the built run.js file. Optional arguments are: t[ests]=category1|category2|... d[ebug]=true.");
-task(TaskNames.runtestsParallel, [TaskNames.buildRules], function() {
-    tsbuild(ConfigFileFor.runjs);
-    runConsoleTests("min", /*parallel*/ true);
-}, { async: true});
+const ExpectedLKGFiles = [
+    "tsc.js",
+    "tsserver.js",
+    "typescriptServices.js",
+    "typescriptServices.d.ts",
+    "typescript.js",
+    "typescript.d.ts",
+    "tsserverlibrary.js",
+    "tsserverlibrary.d.js",
+    "cancellationToken.js",
+    "typingsInstaller.js",
+    "protocol.d.ts",
+    "watchGuard.js"
+];
+
+/** @type {{ libs: string[], paths?: Record<string, string>, sources?: Record<string, string[]> }} */
+const libraries = readJson("./src/lib/libs.json");
+
+directory(Paths.builtLocal);
+
+// Local target to build the compiler and services
+desc("Builds the full compiler and services");
+task(TaskNames.local, [
+    TaskNames.buildFoldStart,
+    TaskNames.generateDiagnostics,
+    TaskNames.lib,
+    TaskNames.coreBuild,
+    // buildProtocolDts,
+    // builtGeneratedDiagnosticMessagesJSON,
+    TaskNames.lssl,
+    // "localize",
+    TaskNames.buildFoldEnd
+]);
 
 desc("Runs all the tests in parallel using the built run.js file. Optional arguments are: t[ests]=category1|category2|... d[ebug]=true.");
-task(TaskNames.runtests, [TaskNames.buildRules], function() {
-    tsbuild(ConfigFileFor.runjs);
+task(TaskNames.runtestsParallel, [TaskNames.lib], function () {
+    tsbuild([ConfigFileFor.runjs, ConfigFileFor.lint]);
+    runConsoleTests("min", /*parallel*/ true);
+}, { async: true });
+
+desc("Runs all the tests in parallel using the built run.js file. Optional arguments are: t[ests]=category1|category2|... d[ebug]=true.");
+task(TaskNames.runtests, [TaskNames.lib], function () {
+    tsbuild([ConfigFileFor.runjs, ConfigFileFor.lint]);
     runConsoleTests('mocha-fivemat-progress-reporter', /*runInParallel*/ false);;
-}, { async: true});
+}, { async: true });
+
+const libraryTargets = getLibraryTargets();
+desc("Builds the library targets");
+task(TaskNames.lib, libraryTargets);
+
+desc("Builds language service server library");
+task("lssl", [Paths.tsserverLibraryFile, Paths.tsserverLibraryDefinitionFile, Paths.typesMapOutput]);
+
+// Makes a new LKG. This target does not build anything, but errors if not all the outputs are present in the built/local directory
+desc("Makes a new LKG out of the built js files");
+task(TaskNames.lkg, [
+    TaskNames.clean,
+    TaskNames.release,
+    TaskNames.local,
+    ...libraryTargets
+], () => {
+    const sizeBefore = getDirSize(Paths.lkg);
+    const localizationTargets = locales.map(f => path.join(Paths.builtLocal, f)).concat(path.dirname(Paths.locLcg));
+
+    const copyrightContent = fs.readFileSync(Paths.copyright, { encoding: 'utf-8' });
+
+    const expectedFiles = [...libraryTargets, ...ExpectedLKGFiles, ...localizationTargets];
+    const missingFiles = expectedFiles.filter(f => !fs.existsSync(f));
+    if (missingFiles.length > 0) {
+        fail(new Error("Cannot replace the LKG unless all built targets are present in directory " + Paths.builtLocal +
+            ". The following files are missing:\n" + missingFiles.join("\n")));
+    }
+    // Copy all the targets into the LKG directory
+    jake.mkdirP(Paths.lkg);
+    expectedFiles.forEach(f => {
+        let content = fs.readFileSync(f, { encoding: 'utf-8' });
+
+        // If this is a .d.ts file, run remove-internal on it
+        if (f.endsWith(".d.ts")) {
+            content = removeInternal.elide(content).result;
+        }
+
+        if (f.endsWith(".d.ts") || f.endsWith(".js")) {
+            // Prepend the copyright header to it
+            content = copyrightContent + content;
+        }
+
+        fs.writeFile(path.join(Paths.lkg, path.basename(f)), content, { encoding: 'utf-8' }, (err) => {
+            if (err) throw err;
+        });
+    });
+
+    const sizeAfter = getDirSize(Paths.lkg);
+    if (sizeAfter > (sizeBefore * 1.10)) {
+        throw new Error("The lib folder increased by 10% or more. This likely indicates a bug.");
+    }
+});
+
+desc("Runs tslint on the compiler sources. Optional arguments are: f[iles]=regex");
+task(TaskNames.lint, [TaskNames.buildRules], () => {
+    if (fold.isTravis()) console.log(fold.start("lint"));
+    function lint(project, cb) {
+        const fix = process.env.fix || process.env.f;
+        const cmd = `node node_modules/tslint/bin/tslint --project ${project} --formatters-dir ./built/local/tslint/formatters --format autolinkableStylish${fix ? " --fix" : ""}`;
+        console.log("Linting: " + cmd);
+        jake.exec([cmd], cb, /** @type {jake.ExecOptions} */({ interactive: true, windowsVerbatimArguments: true }));
+    }
+    lint("scripts/tslint/tsconfig.json", () => lint("src/tsconfig-base.json", () => {
+        if (fold.isTravis()) console.log(fold.end("lint"));
+        complete();
+    }));
+});
+
+desc("Diffs the compiler baselines using the diff tool specified by the 'DIFF' environment variable");
+task('diff', function () {
+    var cmd = `"${getDiffTool()} ${Paths.baselines.reference} ${Paths.baselines.local}`;
+    console.log(cmd);
+    exec(cmd);
+}, { async: true });
+
+desc("Diffs the RWC baselines using the diff tool specified by the 'DIFF' environment variable");
+task('diff-rwc', function () {
+    var cmd = `"${getDiffTool()} ${Paths.baselines.referenceRwc} ${Paths.baselines.localRwc}`;
+    console.log(cmd);
+    exec(cmd);
+}, { async: true });
+
+desc("Sets the release mode flag");
+task("release", function () {
+    useDebugMode = false;
+});
+
+desc("Clears the release mode flag");
+task("setDebugMode", function () {
+    useDebugMode = true;
+});
+
+desc("Emit the start of the build fold");
+task(TaskNames.buildFoldStart, [], function () {
+    if (fold.isTravis()) console.log(fold.start("build"));
+});
+
+desc("Emit the end of the build fold");
+task(TaskNames.buildFoldEnd, [], function () {
+    if (fold.isTravis()) console.log(fold.end("build"));
+});
 
 desc("Compiles tslint rules to js");
-task(TaskNames.buildRules, [], function() {
+task(TaskNames.buildRules, [], function () {
     tsbuild(ConfigFileFor.lint);
 }, { async: true });
 
 desc("Cleans the compiler output, declare files, and tests");
-task("clean", function () {
-    jake.rmRf(Paths.built);;
+task(TaskNames.clean, function () {
+    jake.rmRf(Paths.built);
 });
 
+desc("Generates a diagnostic file in TypeScript based on an input JSON file");
+task(TaskNames.generateDiagnostics, [Paths.diagnosticInformationMap]);
+
+task(TaskNames.coreBuild, function () {
+    tsbuild(ConfigFileFor.all);
+});
+
+file(Paths.diagnosticMessagesJson);
+file(Paths.diagnosticInformationMap, [Paths.diagnosticMessagesJson], function (complete) {
+    tsbuild("scripts/processDiagnosticMessages.tsconfig.json", /*lkg*/ true, function () {
+        const cmd = `${host} scripts/processDiagnosticMessages.js ${Paths.diagnosticMessagesJson}`;
+        console.log(cmd);
+        var ex = jake.createExec([cmd]);
+        // Add listeners for output and error
+        ex.addListener("stdout", function (output) {
+            process.stdout.write(output);
+        });
+        ex.addListener("stderr", function (error) {
+            process.stderr.write(error);
+        });
+        ex.addListener("cmdEnd", function () {
+            complete();
+        });
+        ex.run();
+    });
+}, { async: true });
+
+file(Paths.typesMapOutput, /** @type {*} */(function () {
+    var content = fs.readFileSync(path.join(Paths.srcServer, 'typesMap.json'));
+    // Validate that it's valid JSON
+    try {
+        JSON.parse(content.toString());
+    } catch (e) {
+        console.log("Parse error in typesMap.json: " + e);
+    }
+    fs.writeFileSync(Paths.typesMapOutput, content);
+}));
+
+file(Paths.tsserverLibraryDefinitionFile, [TaskNames.coreBuild, Paths.copyright, ...libraryTargets], function () {
+    const content = fs.readFileSync(Paths.tsserverLibraryDefinitionFile, { encoding: 'utf-8' });
+    const newContent =
+        removeConstModifierFromEnumDeclarations(content) +
+        `\nexport = ts` +
+        `\nexport as namespace ts;`;
+
+    fs.writeFileSync(Paths.tsserverLibraryDefinitionFile, newContent, { encoding: 'utf-8' });
+});
+
+function getLibraryTargets() {
+    return libraries.libs.map(function (lib) {
+        var relativeSources = ["header.d.ts"].concat(libraries.sources && libraries.sources[lib] || [lib + ".d.ts"]);
+        var relativeTarget = libraries.paths && libraries.paths[lib] || ("lib." + lib + ".d.ts");
+        var sources = [Paths.copyright].concat(relativeSources.map(s => path.join(Paths.library, s)));
+        var target = path.join(Paths.builtLocal, relativeTarget);
+        file(target, [Paths.builtLocal].concat(sources), function () {
+            concatenateFiles(target, sources);
+        });
+        return target;
+    });
+}
+
 function runConsoleTests(defaultReporter, runInParallel) {
     var dirty = process.env.dirty;
     if (!dirty) {
@@ -236,11 +467,11 @@ function cleanTestDirs() {
     jake.mkdirP(Paths.baselines.localTest262);
 }
 
-function tsbuild(tsconfigPath, useLkg = false) {
+function tsbuild(tsconfigPath, useLkg = true, done = undefined) {
     const startCompileTime = Travis.mark();
 
     const compilerPath = useLkg ? Paths.lkgCompiler : Paths.builtLocalCompiler;
-    const cmd = `${host} ${compilerPath} -b -v ${tsconfigPath}`;
+    const cmd = `${host} ${compilerPath} -b -v ${Array.isArray(tsconfigPath) ? tsconfigPath.join(" ") : tsconfigPath}`;
 
     var ex = jake.createExec([cmd]);
     // Add listeners for output and error
@@ -264,7 +495,7 @@ function tsbuild(tsconfigPath, useLkg = false) {
         */
 
         Travis.measure(startCompileTime);
-        complete();
+        done && done();
     });
     ex.addListener("error", function () {
         fail(`Compilation of ${tsconfigPath} unsuccessful`);
@@ -321,4 +552,63 @@ function exec(cmd, completeHandler, errorHandler) {
     });
 
     ex.run();
-}
+}
+
+/** @param jsonPath {string} */
+function readJson(jsonPath) {
+    const jsonText = fs.readFileSync(jsonPath, "utf8");
+    const result = ts.parseConfigFileTextToJson(jsonPath, jsonText);
+    if (result.error) {
+        reportDiagnostics([result.error]);
+        throw new Error("An error occurred during parse.");
+    }
+    return result.config;
+}
+
+/** @param diagnostics {ts.Diagnostic[]} */
+function reportDiagnostics(diagnostics) {
+    console.log(diagnosticsToString(diagnostics, process.stdout.isTTY));
+}
+
+/**
+ * @param diagnostics {ts.Diagnostic[]}
+ * @param [pretty] {boolean}
+ */
+function diagnosticsToString(diagnostics, pretty) {
+    const host = {
+        getCurrentDirectory() { return process.cwd(); },
+        getCanonicalFileName(fileName) { return fileName; },
+        getNewLine() { return os.EOL; }
+    };
+    return pretty ? ts.formatDiagnosticsWithColorAndContext(diagnostics, host) :
+        ts.formatDiagnostics(diagnostics, host);
+}
+
+// concatenate a list of sourceFiles to a destinationFile
+function concatenateFiles(destinationFile, sourceFiles) {
+    var temp = "temptemp";
+    // append all files in sequence
+    var text = "";
+    for (var i = 0; i < sourceFiles.length; i++) {
+        if (!fs.existsSync(sourceFiles[i])) {
+            fail(sourceFiles[i] + " does not exist!");
+        }
+        if (i > 0) { text += "\n\n"; }
+        text += fs.readFileSync(sourceFiles[i]).toString().replace(/\r?\n/g, "\n");
+    }
+    fs.writeFileSync(temp, text);
+    // Move the file to the final destination
+    fs.renameSync(temp, destinationFile);
+}
+
+function getDiffTool() {
+    var program = process.env['DIFF'];
+    if (!program) {
+        fail("Add the 'DIFF' environment variable to the path of the program you want to use.");
+    }
+    return program;
+}
+
+function removeConstModifierFromEnumDeclarations(text) {
+    return text.replace(/^(\s*)(export )?const enum (\S+) {(\s*)$/gm, '$1$2enum $3 {$4');
+}

package.json

@@ -75,6 +75,7 @@
         "mocha": "latest",
         "mocha-fivemat-progress-reporter": "latest",
         "q": "latest",
+        "remove-internal": "^2.9.2",
         "run-sequence": "latest",
         "sorcery": "latest",
         "source-map-support": "latest",

scripts/processDiagnosticMessages.ts

@@ -1,3 +1,6 @@
+import path = require("path");
+import fs = require("fs");
+
 interface DiagnosticDetails {
     category: string;
     code: number;
@@ -5,32 +8,36 @@ interface DiagnosticDetails {
     isEarly?: boolean;
 }
 
-type InputDiagnosticMessageTable = ts.Map<DiagnosticDetails>;
+type InputDiagnosticMessageTable = Map<string, DiagnosticDetails>;
 
 function main(): void {
-    const sys = ts.sys;
-    if (sys.args.length < 1) {
-        sys.write("Usage:" + sys.newLine);
-        sys.write("\tnode processDiagnosticMessages.js <diagnostic-json-input-file>" + sys.newLine);
+    if (process.argv.length < 3) {
+        console.log("Usage:");
+        console.log("\tnode processDiagnosticMessages.js <diagnostic-json-input-file>");
         return;
     }
 
     function writeFile(fileName: string, contents: string) {
-        const inputDirectory = ts.getDirectoryPath(inputFilePath);
-        const fileOutputPath = ts.combinePaths(inputDirectory, fileName);
-        sys.writeFile(fileOutputPath, contents);
+        fs.writeFile(path.join(path.dirname(inputFilePath), fileName), contents, { encoding: "utf-8" }, err => {
+            if (err) throw err;
+        })
     }
 
-    const inputFilePath = sys.args[0].replace(/\\/g, "/");
-    const inputStr = sys.readFile(inputFilePath)!;
+    const inputFilePath = process.argv[2].replace(/\\/g, "/");
+    console.log(`Reading diagnostics from ${inputFilePath}`);
+    const inputStr = fs.readFileSync(inputFilePath, { encoding: "utf-8" });
 
     const diagnosticMessagesJson: { [key: string]: DiagnosticDetails } = JSON.parse(inputStr);
 
-    const diagnosticMessages: InputDiagnosticMessageTable = ts.createMapFromTemplate(diagnosticMessagesJson);
+    const diagnosticMessages: InputDiagnosticMessageTable = new Map();
+    for (const key in diagnosticMessagesJson) {
+        if (Object.hasOwnProperty.call(diagnosticMessagesJson, key)) {
+            diagnosticMessages.set(key, diagnosticMessagesJson[key]);
+        }
+    }
 
-    const outputFilesDir = ts.getDirectoryPath(inputFilePath);
-    const thisFilePathRel = ts.getRelativePathToDirectoryOrUrl(outputFilesDir, sys.getExecutingFilePath(),
-        sys.getCurrentDirectory(), ts.createGetCanonicalFileName(sys.useCaseSensitiveFileNames), /* isAbsolutePathAnUrl */ false);
+    const outputFilesDir = path.dirname(inputFilePath);
+    const thisFilePathRel = path.relative(process.cwd(), outputFilesDir);
 
     const infoFileOutput = buildInfoFileOutput(diagnosticMessages, "./diagnosticInformationMap.generated.ts", thisFilePathRel);
     checkForUniqueCodes(diagnosticMessages);

scripts/processDiagnosticMessages.tsconfig.json

@@ -1,20 +1,15 @@
 {
     "compilerOptions": {
         "removeComments": false,
-        "outFile": "processDiagnosticMessages.js",
         "target": "es5",
         "declaration": false,
         "lib": [
             "es6",
             "scripthost"
-        ]
+        ],
+        "types": ["node"]
     },
     "files": [
-        "../src/compiler/types.ts",
-        "../src/compiler/performance.ts",
-        "../src/compiler/core.ts",
-        "../src/compiler/sys.ts",
-
         "processDiagnosticMessages.ts"
     ]
 }

scripts/tslint/tsconfig.json

@@ -10,6 +10,10 @@
 		"noUnusedParameters": true,
 		"strictNullChecks": true,
 		"module": "commonjs",
-		"outDir": "../../built/local/tslint"
+		"outDir": "../../built/local/tslint",
+		"baseUrl": "../..",
+		"paths": {
+			"typescript": ["lib/typescript.d.ts"]
+		}
 	}
 }

src/cancellationToken/tsconfig.json

@@ -1,7 +1,7 @@
 {
     "extends": "../tsconfig-base",
     "compilerOptions": {
-        "outDir": "../built/local/",
+        "outDir": "../../built/local/",
         "composite": false,
         "declaration": false,
         "removeComments": true,

src/harness/tsconfig.json

@@ -1,7 +1,7 @@
 {
     "extends": "../tsconfig-base",
     "compilerOptions": {
-        "outFile": "../built/local/harness.js",
+        "outFile": "../../built/local/harness.js",
         "types": [
             "node", "mocha", "chai"
         ],
@@ -16,7 +16,7 @@
         { "path": "../compiler" },
         { "path": "../services" },
         { "path": "../server" },
-        { "path": "../typingsInstallerCore" },
+        { "path": "../typingsInstallerCore" }
     ],
 
     "files": [

src/parser/diagnosticInformationMap.generated.ts

@@ -1,5 +1,5 @@
 // <auto-generated />
-// generated from './diagnosticInformationMap.generated.ts' by '../../scripts/processDiagnosticMessages.js'
+// generated from './diagnosticInformationMap.generated.ts' by 'src\parser'
 /* @internal */
 namespace ts {
     function diag(code: number, category: DiagnosticCategory, key: string, message: string, reportsUnnecessary?: {}): DiagnosticMessage {

src/tsc/tsconfig.json

@@ -8,6 +8,7 @@
     ],
     "references": [
         { "path": "../core", "prepend": true },
+        { "path": "../parser", "prepend": true },
         { "path": "../compiler", "prepend": true }
     ]
 }

src/tsconfig.json

@@ -2,6 +2,11 @@
     "files": [],
     "include": [],
     "references": [
-        { "path": "./tsc" }
+        { "path": "./tsc" },
+        { "path": "./tsserver" },
+        { "path": "./typingsInstaller" },
+        { "path": "./watchGuard" },
+        { "path": "./cancellationToken" },
+        { "path": "./testRunner" }
     ]
 }

src/tsserver/tsconfig.json

@@ -12,6 +12,7 @@
     ],
     "references": [
         { "path": "../core", "prepend": true },
+        { "path": "../parser", "prepend": true },
         { "path": "../compiler", "prepend": true },
         { "path": "../services", "prepend": true },
         { "path": "../server", "prepend": true }

src/typingsInstaller/tsconfig.json

@@ -2,7 +2,7 @@
     "extends": "../tsconfig-base",
     "compilerOptions": {
         "removeComments": true,
-        "outFile": "../built/local/typingsInstaller.js",
+        "outFile": "../../built/local/typingsInstaller.js",
         "types": [
             "node"
         ],

src/watchGuard/tsconfig.json

@@ -2,7 +2,7 @@
     "extends": "../tsconfig-base",
     "compilerOptions": {
         "removeComments": true,
-        "outFile": "../built/local/watchGuard.js",
+        "outFile": "../../built/local/watchGuard.js",
         "types": [
             "node"
         ],