From 8b1a814d17ff4f1680b50bebe9cf7d91cd2ffde4 Mon Sep 17 00:00:00 2001 From: joeduffy Date: Fri, 13 Jan 2017 07:08:49 -0800 Subject: [PATCH] Bind all module export clauses This completes support for module imports and exports. Many forms of exports are supported, and are all now tested, for example: // Export individual declarations as we go: export class C {} export interface I {} export let x = 42; // Export individual declarations explicitly: class D {} interface J {} let w = 42; export {D, J, w}; // Rename exports: export {D as E, J as K, w as x}; // Re-export members from other modules: export {A, B, c} from "other"; // Rename re-exports: export {A as M, B as N, c as o} from "other"; // Re-export entire module as a submodule: import * as other from "other"; export {other}; --- tools/mujs/lib/compiler/transform.ts | 279 +++++++++----- tools/mujs/tests/output/index.ts | 1 + .../mujs/tests/output/modules/export/Mu.json | 4 + .../tests/output/modules/export/Mu.out.json | 359 ++++++++++++++++++ .../mujs/tests/output/modules/export/index.ts | 10 + .../mujs/tests/output/modules/export/other.ts | 4 + .../tests/output/modules/export/tsconfig.json | 7 + 7 files changed, 576 insertions(+), 88 deletions(-) create mode 100644 tools/mujs/tests/output/modules/export/Mu.json create mode 100644 tools/mujs/tests/output/modules/export/Mu.out.json create mode 100644 tools/mujs/tests/output/modules/export/index.ts create mode 100644 tools/mujs/tests/output/modules/export/other.ts create mode 100644 tools/mujs/tests/output/modules/export/tsconfig.json diff --git a/tools/mujs/lib/compiler/transform.ts b/tools/mujs/lib/compiler/transform.ts index 493e79227..ceb153a75 100644 --- a/tools/mujs/lib/compiler/transform.ts +++ b/tools/mujs/lib/compiler/transform.ts @@ -155,10 +155,16 @@ function notYetImplemented(node: ts.Node | undefined, label?: string): never { // A transpiler is responsible for transforming TypeScript program artifacts into MuPack/MuIL AST forms. export class Transformer { - private meta: pack.Metadata; // the package's metadata. - private script: Script; // the package's compiled TypeScript tree and context. - private dctx: diag.Context; // the diagnostics context. - private diagnostics: diag.Diagnostic[]; // any diagnostics encountered during translation. + // Immutable elements of the transformer that exist throughout an entire pass: + private readonly meta: pack.Metadata; // the package's metadata. + private readonly script: Script; // the package's compiled TypeScript tree and context. + private readonly dctx: diag.Context; // the diagnostics context. + private readonly diagnostics: diag.Diagnostic[]; // any diagnostics encountered during translation. + + // Mutable elements of the transformer that are pushed/popped as we perform visitations: + private currentSourceFile: ts.SourceFile | undefined; + private currentModuleMembers: ast.ModuleMembers | undefined; + private currentModuleImports: Map; constructor(meta: pack.Metadata, script: Script) { contract.requires(!!script.tree, "script", "A valid MuJS AST is required to lower to MuPack/MuIL"); @@ -252,8 +258,8 @@ export class Transformer { return moduleTok; } - // createModuleDefinitionToken binds a string-based export name to the associated token that references it. - private createModuleDefinitionToken(mod: ModuleReference, name: string): symbols.Token { + // createModuleMemberToken binds a string-based exported member name to the associated token that references it. + private createModuleMemberToken(mod: ModuleReference, name: string): symbols.Token { // The concatenated name of the module plus identifier will resolve correctly to an exported definition. let modtok: symbols.ModuleToken = this.createModuleToken(mod); return `${modtok}${symbols.tokenSep}${name}`; @@ -330,7 +336,7 @@ export class Transformer { `Expected discovered module '${this.createModuleReference(moduleSymbol.name)}' to equal '${mod}'`, ); for (let expsym of this.checker().getExportsOfModule(moduleSymbol)) { - exports.push(this.createModuleDefinitionToken(mod, expsym.name)); + exports.push(this.createModuleMemberToken(mod, expsym.name)); } return exports; @@ -352,59 +358,71 @@ export class Transformer { // MuPack/MuIL. As such, the appropriate top-level definitions (variables, functions, and classes) are returned as // definitions, while any loose code (including variable initializers) is bundled into module inits and entrypoints. private transformSourceFile(node: ts.SourceFile): ast.Module { - // All definitions will go into a map keyed by their identifier. - let members: ast.ModuleMembers = {}; + // Each source file is a separate module, and we maintain some amount of context about it. Push some state. + let priorSourceFile: ts.SourceFile | undefined = this.currentSourceFile; + let priorModuleMembers: ast.ModuleMembers | undefined = this.currentModuleMembers; + let priorModuleImports: Map | undefined = this.currentModuleImports; + try { + this.currentSourceFile = node; + this.currentModuleMembers = {}; + this.currentModuleImports = new Map(); - // Any top-level non-definition statements will pile up into the module initializer. - let statements: ast.Statement[] = []; + // Any top-level non-definition statements will pile up into the module initializer. + let statements: ast.Statement[] = []; - // Enumerate the module's statements and put them in the respective places. - for (let statement of node.statements) { - let elements: ModuleElement[] = this.transformSourceFileStatement(statement); - for (let element of elements) { - if (isVariableDeclaration(element)) { - // This is a module property with a possible initializer. The property should get registered as a - // member in this module's member map, and the initializer must happen in the module initializer. - // TODO(joe): respect legacyVar to emulate "var"-like scoping. - let decl = >element; - if (decl.initializer) { - statements.push(this.makeVariableInitializer(decl)); + // Enumerate the module's statements and put them in the respective places. + for (let statement of node.statements) { + let elements: ModuleElement[] = this.transformSourceFileStatement(statement); + for (let element of elements) { + if (isVariableDeclaration(element)) { + // This is a module property with a possible initializer. The property must be registered as a + // member in this module's member map, and the initializer must go into the module initializer. + // TODO(joe): respect legacyVar to emulate "var"-like scoping. + let decl = >element; + if (decl.initializer) { + statements.push(this.makeVariableInitializer(decl)); + } + this.currentModuleMembers[decl.variable.name.ident] = decl.variable; + } + else if (ast.isDefinition(element)) { + // This is a module member; simply add it to the list. + let member = element; + this.currentModuleMembers[member.name.ident] = member; + } + else { + // This is a top-level module statement; place it into the module initializer. + statements.push(element); } - members[decl.variable.name.ident] = decl.variable; - } - else if (ast.isDefinition(element)) { - // This is a module member; simply add it to the list. - let member = element; - members[member.name.ident] = member; - } - else { - // This is a top-level module statement; place it into the module initializer. - statements.push(element); } + } + + // If the initialization statements are non-empty, add an initializer method. + if (statements.length > 0) { + let initializer: ast.ModuleMethod = { + kind: ast.moduleMethodKind, + name: ident(symbols.specialFunctionInitializer), + access: symbols.publicAccessibility, + body: { + kind: ast.blockKind, + statements: statements, + }, + }; + this.currentModuleMembers[initializer.name.ident] = initializer; } - } - // If the initialization statements are non-empty, add an initializer method. - if (statements.length > 0) { - let initializer: ast.ModuleMethod = { - kind: ast.moduleMethodKind, - name: ident(symbols.specialFunctionInitializer), - access: symbols.publicAccessibility, - body: { - kind: ast.blockKind, - statements: statements, - }, - }; - members[initializer.name.ident] = initializer; + let modref: ModuleReference = this.createModuleReference(node.fileName); + let modtok: symbols.ModuleToken = this.createModuleToken(modref); + return this.withLocation(node, { + kind: ast.moduleKind, + name: ident(modtok), + members: this.currentModuleMembers, + }); + } + finally { + this.currentSourceFile = priorSourceFile; + this.currentModuleMembers = priorModuleMembers; + this.currentModuleImports = priorModuleImports; } - - let modref: ModuleReference = this.createModuleReference(node.fileName); - let modtok: symbols.ModuleToken = this.createModuleToken(modref); - return this.withLocation(node, { - kind: ast.moduleKind, - name: ident(modtok), - members: members, - }); } // This transforms a top-level TypeScript module statement. It might return multiple elements in the rare @@ -459,54 +477,83 @@ export class Transformer { } private transformExportDeclaration(node: ts.ExportDeclaration): ast.ModuleMember[] { - // In the case of a module specifier, we are re-exporting elements from another module. - if (node.moduleSpecifier) { - return this.transformReExportDeclaration(node); - } + let exports: ast.Export[] = []; // Otherwise, we are exporting already-imported names from the current module. - // TODO(joe): support this, by enumerating all exports, and flipping any privates to publics. As we proceed, we - // will need to keep an eye out for exporting whole sub-modules. - return notYetImplemented(node, "manual exports"); - } + // TODO: in ECMAScript, this is order independent, so we can actually export before declaring something. + // To simplify things, we are only allowing you to export things declared lexically before the export. - private transformReExportDeclaration(node: ts.ExportDeclaration): ast.ModuleMember[] { - contract.assert(!!node.moduleSpecifier); - contract.assert(node.moduleSpecifier!.kind === ts.SyntaxKind.StringLiteral); + // In the case of a module specifier, we are re-exporting elements from another module. + let sourceModule: ModuleReference | undefined; + if (node.moduleSpecifier) { + // The module specifier will be a string literal; fetch and resolve it to a module. + contract.assert(node.moduleSpecifier.kind === ts.SyntaxKind.StringLiteral); + let spec: ts.StringLiteral = node.moduleSpecifier; + let source: string = this.transformStringLiteral(spec).value; + sourceModule = this.resolveModuleReferenceByName(node, source); + } - let exports: ast.Export[] = []; - let spec: ts.StringLiteral = node.moduleSpecifier!; - - // The module specifier will be a string literal; fetch that so we can resolve to a symbol token. - let source: string = this.transformStringLiteral(spec).value; - let sourceModule: ModuleReference = this.resolveModuleReferenceByName(node, source); if (node.exportClause) { // This is an export declaration of the form // - // export { a, b, c } from "module"; + // export { a, b, c }[ from "module"]; // - // in which a, b, and c are elements from another module that shall be exported. Each re-export may + // in which a, b, and c are elements that shall be exported, possibly from another module "module". If not + // another module, then these are expected to be definitions within the current module. Each re-export may // optionally rename the symbol being exported. For example: // - // export { a as x, b as y, c as z } from "module"; + // export { a as x, b as y, c as z }[ from "module"]; // // For every export clause, we will issue a top-level MuIL re-export AST node. for (let exportClause of node.exportClause.elements) { let name: ast.Identifier = this.transformIdentifier(exportClause.name); - let propertyName: ast.Identifier; if (exportClause.propertyName) { - // The export is being renamed (` as as `). This yields an export node, even for + // elements exported from the current module. + let propertyName: ast.Identifier = this.transformIdentifier(exportClause.propertyName); + let token: symbols.Token = propertyName.ident; + if (sourceModule) { + token = this.createModuleMemberToken(sourceModule, token); + } + exports.push({ + kind: ast.exportKind, + name: name, + token: token, + }); } else { - propertyName = name; + // If this is an export from another module, create an export definition. Otherwise, for exports + // from within the same module, just look up the definition and change its accessibility to public. + if (sourceModule) { + exports.push({ + kind: ast.exportKind, + name: name, + token: this.createModuleMemberToken(sourceModule, name.ident), + }); + } + else { + contract.assert(!!this.currentModuleMembers); + contract.assert(!!this.currentModuleImports); + // First look for a module member, for reexporting classes, interfaces, and variables. + let member: ast.ModuleMember = this.currentModuleMembers![name.ident]; + if (member) { + contract.assert(member.access !== symbols.publicAccessibility); + member.access = symbols.publicAccessibility; + } + else { + // If that failed, look for a known import. This enables reexporting whole modules, e.g.: + // import * as other from "other"; + // export {other}; + let otherModule: ModuleReference | undefined = this.currentModuleImports!.get(name.ident); + contract.assert(!!otherModule, "Expected either a member or import match for export name"); + exports.push({ + kind: ast.exportKind, + name: name, + token: this.createModuleToken(otherModule!), + }); + } + } } - - exports.push({ - kind: ast.exportKind, - name: name, - token: this.createModuleDefinitionToken(sourceModule, propertyName.ident), - }); } } else { @@ -515,7 +562,8 @@ export class Transformer { // export * from "module"; // // For this to work, we simply enumerate all known exports from "module". - for (let name of this.resolveModuleExportNames(node, sourceModule)) { + contract.assert(!!sourceModule); + for (let name of this.resolveModuleExportNames(node, sourceModule!)) { exports.push({ kind: ast.exportKind, name: { @@ -526,14 +574,69 @@ export class Transformer { }); } } + return exports; } private transformImportDeclaration(node: ts.ImportDeclaration): ModuleElement { - // TODO[marapongo/mu#46]: we are ignoring import declarations for the time being. Eventually we need to - // transform all dependency symbols into real MuIL references. (Today, bound node information is - // discarded.) When that day comes (soon), import declarations will most likely still be ignored, however, - // I am leaving this comment in here so that we can make an explicit decision about this. + // An import declaration is erased in the output AST, however, we must keep track of the set of known import + // names so that we can easily look them up by name later on (e.g., in the case of reexporting whole modules). + if (node.importClause) { + // First turn the module path into a reference. The module path may be relative, so we need to consult the + // current file's module table in order to find its fully resolved path. + contract.assert(node.moduleSpecifier.kind === ts.SyntaxKind.StringLiteral); + let importModule: ModuleReference = + this.resolveModuleReferenceByName(node, (node.moduleSpecifier).text); + + // Figure out what kind of import statement this is (there are many, see below). + let name: ts.Identifier | undefined; + let namedImports: ts.NamedImports | undefined; + if (node.importClause.name) { + name = name; + } + else if (node.importClause.namedBindings) { + if (node.importClause.namedBindings.kind === ts.SyntaxKind.NamespaceImport) { + name = (node.importClause.namedBindings).name; + } + else { + contract.assert(node.importClause.namedBindings.kind === ts.SyntaxKind.NamedImports); + namedImports = node.importClause.namedBindings; + } + } + + // Now associate the import names with the module and/or members within it. + if (name) { + // This is an import of the form + // import * as from ""; + // Just bind the name to an identifier and module to its module reference, and remember the association. + let importName: ast.Identifier = this.transformIdentifier(name); + log.out(5).info(`Detected bulk import ${importName.ident}=${importModule}`); + this.currentModuleImports.set(importName.ident, importModule); + } + else if (namedImports) { + // This is an import of the form + // import {a, b, c} from ""; + // In which case we will need to bind each name and associate it with a fully qualified token. + for (let importSpec of namedImports.elements) { + let member: ast.Identifier = this.transformIdentifier(importSpec.name); + let memberToken: symbols.Token = this.createModuleMemberToken(importModule, member.ident); + let memberName: string; + if (importSpec.propertyName) { + // This is of the form + // import {a as x} from ""; + // in other words, the member is renamed for purposes of this source file. But we still need to + // be able to trace it back to the actual fully qualified exported token name later on. + memberName = this.transformIdentifier(importSpec.propertyName).ident; + } + else { + // Otherwise, simply associate the raw member name with the fully qualified member token. + memberName = member.ident; + } + this.currentModuleImports.set(memberName, memberToken); + log.out(5).info(`Detected named import ${memberToken} as ${memberName} from ${importModule}`); + } + } + } return { kind: ast.emptyStatementKind }; } diff --git a/tools/mujs/tests/output/index.ts b/tools/mujs/tests/output/index.ts index 9e0d3ec75..822b85a0c 100644 --- a/tools/mujs/tests/output/index.ts +++ b/tools/mujs/tests/output/index.ts @@ -26,6 +26,7 @@ let testCases: string[] = [ "modules/reexport", "modules/reexport_all", "modules/reexport_rename", + "modules/export", // These are not quite real-world-code, but they are more complex "integration" style tests. "scenarios/point", diff --git a/tools/mujs/tests/output/modules/export/Mu.json b/tools/mujs/tests/output/modules/export/Mu.json new file mode 100644 index 000000000..65625cabd --- /dev/null +++ b/tools/mujs/tests/output/modules/export/Mu.json @@ -0,0 +1,4 @@ +{ + "name": "export" +} + diff --git a/tools/mujs/tests/output/modules/export/Mu.out.json b/tools/mujs/tests/output/modules/export/Mu.out.json new file mode 100644 index 000000000..71d91529c --- /dev/null +++ b/tools/mujs/tests/output/modules/export/Mu.out.json @@ -0,0 +1,359 @@ +{ + "name": "export", + "modules": { + "other": { + "kind": "Module", + "name": { + "kind": "Identifier", + "ident": "other" + }, + "members": { + "D": { + "kind": "Class", + "name": { + "kind": "Identifier", + "ident": "D", + "loc": { + "file": "other.ts", + "start": { + "line": 1, + "column": 13 + }, + "end": { + "line": 1, + "column": 14 + } + } + }, + "access": "public", + "members": {}, + "abstract": false, + "loc": { + "file": "other.ts", + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 17 + } + } + }, + "J": { + "kind": "Class", + "name": { + "kind": "Identifier", + "ident": "J", + "loc": { + "file": "other.ts", + "start": { + "line": 2, + "column": 17 + }, + "end": { + "line": 2, + "column": 18 + } + } + }, + "access": "public", + "members": {}, + "interface": true, + "loc": { + "file": "other.ts", + "start": { + "line": 2, + "column": 0 + }, + "end": { + "line": 2, + "column": 21 + } + } + }, + "w": { + "kind": "ModuleProperty", + "name": { + "kind": "Identifier", + "ident": "w", + "loc": { + "file": "other.ts", + "start": { + "line": 3, + "column": 11 + }, + "end": { + "line": 3, + "column": 12 + } + } + }, + "access": "public", + "type": "any" + }, + ".init": { + "kind": "ModuleMethod", + "name": { + "kind": "Identifier", + "ident": ".init" + }, + "access": "public", + "body": { + "kind": "Block", + "statements": [ + { + "kind": "BinaryOperatorExpression", + "left": { + "kind": "LoadLocationExpression", + "name": { + "kind": "Identifier", + "ident": "w", + "loc": { + "file": "other.ts", + "start": { + "line": 3, + "column": 11 + }, + "end": { + "line": 3, + "column": 12 + } + } + } + }, + "operator": "=", + "right": { + "kind": "NumberLiteral", + "raw": "42", + "value": 42, + "loc": { + "file": "other.ts", + "start": { + "line": 3, + "column": 15 + }, + "end": { + "line": 3, + "column": 17 + } + } + }, + "loc": { + "file": "other.ts", + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 18 + } + } + } + ] + } + } + }, + "loc": { + "file": "other.ts", + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 5, + "column": 0 + } + } + }, + "index": { + "kind": "Module", + "name": { + "kind": "Identifier", + "ident": "index" + }, + "members": { + "other": { + "kind": "Export", + "name": { + "kind": "Identifier", + "ident": "other", + "loc": { + "file": "index.ts", + "start": { + "line": 3, + "column": 8 + }, + "end": { + "line": 3, + "column": 13 + } + } + }, + "token": "other" + }, + "C": { + "kind": "Class", + "name": { + "kind": "Identifier", + "ident": "C", + "loc": { + "file": "index.ts", + "start": { + "line": 6, + "column": 6 + }, + "end": { + "line": 6, + "column": 7 + } + } + }, + "access": "public", + "members": {}, + "abstract": false, + "loc": { + "file": "index.ts", + "start": { + "line": 6, + "column": 0 + }, + "end": { + "line": 6, + "column": 10 + } + } + }, + "I": { + "kind": "Class", + "name": { + "kind": "Identifier", + "ident": "I", + "loc": { + "file": "index.ts", + "start": { + "line": 7, + "column": 10 + }, + "end": { + "line": 7, + "column": 11 + } + } + }, + "access": "public", + "members": {}, + "interface": true, + "loc": { + "file": "index.ts", + "start": { + "line": 7, + "column": 0 + }, + "end": { + "line": 7, + "column": 14 + } + } + }, + "v": { + "kind": "ModuleProperty", + "name": { + "kind": "Identifier", + "ident": "v", + "loc": { + "file": "index.ts", + "start": { + "line": 8, + "column": 4 + }, + "end": { + "line": 8, + "column": 5 + } + } + }, + "access": "public", + "type": "any" + }, + ".init": { + "kind": "ModuleMethod", + "name": { + "kind": "Identifier", + "ident": ".init" + }, + "access": "public", + "body": { + "kind": "Block", + "statements": [ + { + "kind": "EmptyStatement" + }, + { + "kind": "BinaryOperatorExpression", + "left": { + "kind": "LoadLocationExpression", + "name": { + "kind": "Identifier", + "ident": "v", + "loc": { + "file": "index.ts", + "start": { + "line": 8, + "column": 4 + }, + "end": { + "line": 8, + "column": 5 + } + } + } + }, + "operator": "=", + "right": { + "kind": "NumberLiteral", + "raw": "42", + "value": 42, + "loc": { + "file": "index.ts", + "start": { + "line": 8, + "column": 8 + }, + "end": { + "line": 8, + "column": 10 + } + } + }, + "loc": { + "file": "index.ts", + "start": { + "line": 8, + "column": 0 + }, + "end": { + "line": 8, + "column": 11 + } + } + } + ] + } + } + }, + "loc": { + "file": "index.ts", + "start": { + "line": 2, + "column": 0 + }, + "end": { + "line": 11, + "column": 0 + } + } + } + } +} diff --git a/tools/mujs/tests/output/modules/export/index.ts b/tools/mujs/tests/output/modules/export/index.ts new file mode 100644 index 000000000..9b53574ed --- /dev/null +++ b/tools/mujs/tests/output/modules/export/index.ts @@ -0,0 +1,10 @@ +// Export a whole submodule: +import * as other from "./other"; +export {other}; + +// Manually export C, I, and v without using export declarations: +class C {} +interface I {} +let v = 42; +export {C, I, v}; + diff --git a/tools/mujs/tests/output/modules/export/other.ts b/tools/mujs/tests/output/modules/export/other.ts new file mode 100644 index 000000000..07bb96b9c --- /dev/null +++ b/tools/mujs/tests/output/modules/export/other.ts @@ -0,0 +1,4 @@ +export class D {} +export interface J {} +export let w = 42; + diff --git a/tools/mujs/tests/output/modules/export/tsconfig.json b/tools/mujs/tests/output/modules/export/tsconfig.json new file mode 100644 index 000000000..8816de4b0 --- /dev/null +++ b/tools/mujs/tests/output/modules/export/tsconfig.json @@ -0,0 +1,7 @@ +{ + "files": [ + "index.ts", + "other.ts" + ] +} +