From 4a77c97ae7e2917ca15e8e3c2d4d6524e947395d Mon Sep 17 00:00:00 2001 From: Sheetal Nandi Date: Fri, 17 Jun 2016 14:59:40 -0700 Subject: [PATCH 1/6] Create tokens using different constructor --- src/compiler/core.ts | 2 + src/compiler/parser.ts | 10 ++++- src/compiler/types.ts | 4 +- src/services/services.ts | 84 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 92 insertions(+), 8 deletions(-) diff --git a/src/compiler/core.ts b/src/compiler/core.ts index fe4731a1c9..07100a982e 100644 --- a/src/compiler/core.ts +++ b/src/compiler/core.ts @@ -1231,6 +1231,7 @@ namespace ts { export interface ObjectAllocator { getNodeConstructor(): new (kind: SyntaxKind, pos?: number, end?: number) => Node; + getTokenConstructor(): new (kind: SyntaxKind, pos?: number, end?: number) => Token; getSourceFileConstructor(): new (kind: SyntaxKind, pos?: number, end?: number) => SourceFile; getSymbolConstructor(): new (flags: SymbolFlags, name: string) => Symbol; getTypeConstructor(): new (checker: TypeChecker, flags: TypeFlags) => Type; @@ -1260,6 +1261,7 @@ namespace ts { export let objectAllocator: ObjectAllocator = { getNodeConstructor: () => Node, + getTokenConstructor: () => Node, getSourceFileConstructor: () => Node, getSymbolConstructor: () => Symbol, getTypeConstructor: () => Type, diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts index d720cd0648..9fc06dcba6 100644 --- a/src/compiler/parser.ts +++ b/src/compiler/parser.ts @@ -5,12 +5,16 @@ namespace ts { /* @internal */ export let parseTime = 0; let NodeConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; + let TokenConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let SourceFileConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; export function createNode(kind: SyntaxKind, pos?: number, end?: number): Node { if (kind === SyntaxKind.SourceFile) { return new (SourceFileConstructor || (SourceFileConstructor = objectAllocator.getSourceFileConstructor()))(kind, pos, end); } + else if (kind < SyntaxKind.FirstNode) { + return new (TokenConstructor || (TokenConstructor = objectAllocator.getTokenConstructor()))(kind, pos, end); + } else { return new (NodeConstructor || (NodeConstructor = objectAllocator.getNodeConstructor()))(kind, pos, end); } @@ -466,6 +470,7 @@ namespace ts { // capture constructors in 'initializeState' to avoid null checks let NodeConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; + let TokenConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let SourceFileConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let sourceFile: SourceFile; @@ -576,6 +581,7 @@ namespace ts { function initializeState(fileName: string, _sourceText: string, languageVersion: ScriptTarget, _syntaxCursor: IncrementalParser.SyntaxCursor, scriptKind: ScriptKind) { NodeConstructor = objectAllocator.getNodeConstructor(); + TokenConstructor = objectAllocator.getTokenConstructor(); SourceFileConstructor = objectAllocator.getSourceFileConstructor(); sourceText = _sourceText; @@ -1024,7 +1030,9 @@ namespace ts { pos = scanner.getStartPos(); } - return new NodeConstructor(kind, pos, pos); + return kind >= SyntaxKind.FirstNode ? + new NodeConstructor(kind, pos, pos) : + new TokenConstructor(kind, pos, pos); } function finishNode(node: T, end?: number): T { diff --git a/src/compiler/types.ts b/src/compiler/types.ts index 3a3a937ade..cc10d9348f 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -472,6 +472,8 @@ namespace ts { flags: NodeFlags; } + export interface Token extends Node { } + // @kind(SyntaxKind.AbstractKeyword) // @kind(SyntaxKind.AsyncKeyword) // @kind(SyntaxKind.ConstKeyword) @@ -482,7 +484,7 @@ namespace ts { // @kind(SyntaxKind.PrivateKeyword) // @kind(SyntaxKind.ProtectedKeyword) // @kind(SyntaxKind.StaticKeyword) - export interface Modifier extends Node { } + export interface Modifier extends Token { } // @kind(SyntaxKind.Identifier) export interface Identifier extends PrimaryExpression { diff --git a/src/services/services.ts b/src/services/services.ts index 529ce8a7bf..b734bb7f06 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -180,9 +180,8 @@ namespace ts { ]; let jsDocCompletionEntries: CompletionEntry[]; - function createNode(kind: SyntaxKind, pos: number, end: number, flags: NodeFlags, parent?: Node): NodeObject { - const node = new NodeObject(kind, pos, end); - node.flags = flags; + function createNode(kind: SyntaxKind, pos: number, end: number, parent?: Node): NodeObject | TokenObject { + const node = kind >= SyntaxKind.FirstNode ? new NodeObject(kind, pos, end) : new TokenObject(kind, pos, end); node.parent = parent; return node; } @@ -200,7 +199,6 @@ namespace ts { this.kind = kind; this.pos = pos; this.end = end; - this.flags = NodeFlags.None; this.parent = undefined; } @@ -246,7 +244,7 @@ namespace ts { const token = useJSDocScanner ? scanner.scanJSDocToken() : scanner.scan(); const textPos = scanner.getTextPos(); if (textPos <= end) { - nodes.push(createNode(token, pos, textPos, 0, this)); + nodes.push(createNode(token, pos, textPos, this)); } pos = textPos; } @@ -254,7 +252,7 @@ namespace ts { } private createSyntaxList(nodes: NodeArray): Node { - const list = createNode(SyntaxKind.SyntaxList, nodes.pos, nodes.end, 0, this); + const list = createNode(SyntaxKind.SyntaxList, nodes.pos, nodes.end, this); list._children = []; let pos = nodes.pos; @@ -345,6 +343,79 @@ namespace ts { } } + class TokenObject implements Token { + public kind: SyntaxKind; + public pos: number; + public end: number; + public flags: NodeFlags; + public parent: Node; + public jsDocComments: JSDocComment[]; + + constructor(kind: SyntaxKind, pos: number, end: number) { + this.kind = kind; + this.pos = pos; + this.end = end; + this.flags = NodeFlags.None; + this.parent = undefined; + } + + public getSourceFile(): SourceFile { + return getSourceFileOfNode(this); + } + + public getStart(sourceFile?: SourceFile, includeJsDocComment?: boolean): number { + return getTokenPosOfNode(this, sourceFile, includeJsDocComment); + } + + public getFullStart(): number { + return this.pos; + } + + public getEnd(): number { + return this.end; + } + + public getWidth(sourceFile?: SourceFile): number { + return this.getEnd() - this.getStart(sourceFile); + } + + public getFullWidth(): number { + return this.end - this.pos; + } + + public getLeadingTriviaWidth(sourceFile?: SourceFile): number { + return this.getStart(sourceFile) - this.pos; + } + + public getFullText(sourceFile?: SourceFile): string { + return (sourceFile || this.getSourceFile()).text.substring(this.pos, this.end); + } + + public getText(sourceFile?: SourceFile): string { + return (sourceFile || this.getSourceFile()).text.substring(this.getStart(), this.getEnd()); + } + + public getChildCount(sourceFile?: SourceFile): number { + return 0; + } + + public getChildAt(index: number, sourceFile?: SourceFile): Node { + return undefined; + } + + public getChildren(sourceFile?: SourceFile): Node[] { + return emptyArray; + } + + public getFirstToken(sourceFile?: SourceFile): Node { + return undefined; + } + + public getLastToken(sourceFile?: SourceFile): Node { + return undefined; + } + } + class SymbolObject implements Symbol { flags: SymbolFlags; name: string; @@ -8696,6 +8767,7 @@ namespace ts { function initializeServices() { objectAllocator = { getNodeConstructor: () => NodeObject, + getTokenConstructor: () => TokenObject, getSourceFileConstructor: () => SourceFileObject, getSymbolConstructor: () => SymbolObject, getTypeConstructor: () => TypeObject, From a2ff4ea0f8ea189e0763b09094cbf875eada6ea8 Mon Sep 17 00:00:00 2001 From: Sheetal Nandi Date: Thu, 9 Jun 2016 16:18:24 -0700 Subject: [PATCH 2/6] Identifier constructor --- src/compiler/core.ts | 2 ++ src/compiler/parser.ts | 12 +++++++++--- src/services/services.ts | 28 ++++++++++++++++++++++------ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/compiler/core.ts b/src/compiler/core.ts index 07100a982e..9aa5741cc7 100644 --- a/src/compiler/core.ts +++ b/src/compiler/core.ts @@ -1232,6 +1232,7 @@ namespace ts { export interface ObjectAllocator { getNodeConstructor(): new (kind: SyntaxKind, pos?: number, end?: number) => Node; getTokenConstructor(): new (kind: SyntaxKind, pos?: number, end?: number) => Token; + getIdentifierConstructor(): new (kind: SyntaxKind, pos?: number, end?: number) => Token; getSourceFileConstructor(): new (kind: SyntaxKind, pos?: number, end?: number) => SourceFile; getSymbolConstructor(): new (flags: SymbolFlags, name: string) => Symbol; getTypeConstructor(): new (checker: TypeChecker, flags: TypeFlags) => Type; @@ -1262,6 +1263,7 @@ namespace ts { export let objectAllocator: ObjectAllocator = { getNodeConstructor: () => Node, getTokenConstructor: () => Node, + getIdentifierConstructor: () => Node, getSourceFileConstructor: () => Node, getSymbolConstructor: () => Symbol, getTypeConstructor: () => Type, diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts index 9fc06dcba6..9109734858 100644 --- a/src/compiler/parser.ts +++ b/src/compiler/parser.ts @@ -6,12 +6,16 @@ namespace ts { let NodeConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let TokenConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; + let IdentifierConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let SourceFileConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; export function createNode(kind: SyntaxKind, pos?: number, end?: number): Node { if (kind === SyntaxKind.SourceFile) { return new (SourceFileConstructor || (SourceFileConstructor = objectAllocator.getSourceFileConstructor()))(kind, pos, end); } + else if (kind === SyntaxKind.Identifier) { + return new (IdentifierConstructor || (IdentifierConstructor = objectAllocator.getIdentifierConstructor()))(kind, pos, end); + } else if (kind < SyntaxKind.FirstNode) { return new (TokenConstructor || (TokenConstructor = objectAllocator.getTokenConstructor()))(kind, pos, end); } @@ -471,6 +475,7 @@ namespace ts { // capture constructors in 'initializeState' to avoid null checks let NodeConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let TokenConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; + let IdentifierConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let SourceFileConstructor: new (kind: SyntaxKind, pos: number, end: number) => Node; let sourceFile: SourceFile; @@ -582,6 +587,7 @@ namespace ts { function initializeState(fileName: string, _sourceText: string, languageVersion: ScriptTarget, _syntaxCursor: IncrementalParser.SyntaxCursor, scriptKind: ScriptKind) { NodeConstructor = objectAllocator.getNodeConstructor(); TokenConstructor = objectAllocator.getTokenConstructor(); + IdentifierConstructor = objectAllocator.getIdentifierConstructor(); SourceFileConstructor = objectAllocator.getSourceFileConstructor(); sourceText = _sourceText; @@ -1030,9 +1036,9 @@ namespace ts { pos = scanner.getStartPos(); } - return kind >= SyntaxKind.FirstNode ? - new NodeConstructor(kind, pos, pos) : - new TokenConstructor(kind, pos, pos); + return kind >= SyntaxKind.FirstNode ? new NodeConstructor(kind, pos, pos) : + kind === SyntaxKind.Identifier ? new IdentifierConstructor(kind, pos, pos) : + new TokenConstructor(kind, pos, pos); } function finishNode(node: T, end?: number): T { diff --git a/src/services/services.ts b/src/services/services.ts index b734bb7f06..d519520540 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -180,8 +180,10 @@ namespace ts { ]; let jsDocCompletionEntries: CompletionEntry[]; - function createNode(kind: SyntaxKind, pos: number, end: number, parent?: Node): NodeObject | TokenObject { - const node = kind >= SyntaxKind.FirstNode ? new NodeObject(kind, pos, end) : new TokenObject(kind, pos, end); + function createNode(kind: SyntaxKind, pos: number, end: number, parent?: Node): NodeObject | TokenObject | IdentifierObject { + const node = kind >= SyntaxKind.FirstNode ? new NodeObject(kind, pos, end) : + kind === SyntaxKind.Identifier ? new IdentifierObject(kind, pos, end) : + new TokenObject(kind, pos, end); node.parent = parent; return node; } @@ -343,7 +345,7 @@ namespace ts { } } - class TokenObject implements Token { + class TokenOrIdentifierObject implements Token { public kind: SyntaxKind; public pos: number; public end: number; @@ -351,11 +353,9 @@ namespace ts { public parent: Node; public jsDocComments: JSDocComment[]; - constructor(kind: SyntaxKind, pos: number, end: number) { - this.kind = kind; + constructor(pos: number, end: number) { this.pos = pos; this.end = end; - this.flags = NodeFlags.None; this.parent = undefined; } @@ -416,6 +416,21 @@ namespace ts { } } + class TokenObject extends TokenOrIdentifierObject { + constructor(kind: SyntaxKind, pos: number, end: number) { + super(pos, end); + this.kind = kind; + this.flags = NodeFlags.None; + } + } + + class IdentifierObject extends TokenOrIdentifierObject { + constructor(kind: SyntaxKind, pos: number, end: number) { + super(pos, end); + } + } + IdentifierObject.prototype.kind = SyntaxKind.Identifier; + class SymbolObject implements Symbol { flags: SymbolFlags; name: string; @@ -8768,6 +8783,7 @@ namespace ts { objectAllocator = { getNodeConstructor: () => NodeObject, getTokenConstructor: () => TokenObject, + getIdentifierConstructor: () => IdentifierObject, getSourceFileConstructor: () => SourceFileObject, getSymbolConstructor: () => SymbolObject, getTypeConstructor: () => TypeObject, From 08a0137bbe2b0d76e977d1ffd4cde94c178a48d5 Mon Sep 17 00:00:00 2001 From: Sheetal Nandi Date: Wed, 6 Jul 2016 12:17:27 -0700 Subject: [PATCH 3/6] Set default flags on all nodes --- src/services/services.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/services.ts b/src/services/services.ts index d519520540..cd3e4a4341 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -201,6 +201,7 @@ namespace ts { this.kind = kind; this.pos = pos; this.end = end; + this.flags = NodeFlags.None; this.parent = undefined; } @@ -356,6 +357,7 @@ namespace ts { constructor(pos: number, end: number) { this.pos = pos; this.end = end; + this.flags = NodeFlags.None; this.parent = undefined; } @@ -420,7 +422,6 @@ namespace ts { constructor(kind: SyntaxKind, pos: number, end: number) { super(pos, end); this.kind = kind; - this.flags = NodeFlags.None; } } From 076459069177e8b64692c8a4f3dcdb7086af9f96 Mon Sep 17 00:00:00 2001 From: Sheetal Nandi Date: Mon, 11 Jul 2016 11:03:38 -0700 Subject: [PATCH 4/6] Make sure the order of setting pos, end, flags, parent, kind is consistent among nodes, tokens and identifiers --- src/services/services.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index cd3e4a4341..bffb44d9fb 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -198,11 +198,11 @@ namespace ts { private _children: Node[]; constructor(kind: SyntaxKind, pos: number, end: number) { - this.kind = kind; this.pos = pos; this.end = end; this.flags = NodeFlags.None; this.parent = undefined; + this.kind = kind; } public getSourceFile(): SourceFile { @@ -348,15 +348,11 @@ namespace ts { class TokenOrIdentifierObject implements Token { public kind: SyntaxKind; - public pos: number; - public end: number; public flags: NodeFlags; public parent: Node; public jsDocComments: JSDocComment[]; - constructor(pos: number, end: number) { - this.pos = pos; - this.end = end; + constructor(public pos: number, public end: number) { this.flags = NodeFlags.None; this.parent = undefined; } @@ -419,9 +415,8 @@ namespace ts { } class TokenObject extends TokenOrIdentifierObject { - constructor(kind: SyntaxKind, pos: number, end: number) { + constructor(public kind: SyntaxKind, pos: number, end: number) { super(pos, end); - this.kind = kind; } } From 767da71cc13422c65a6369e26c3a6cfe894a05ef Mon Sep 17 00:00:00 2001 From: Sheetal Nandi Date: Mon, 11 Jul 2016 11:28:14 -0700 Subject: [PATCH 5/6] Add __tokentag to Token --- src/compiler/parser.ts | 6 +++--- src/compiler/types.ts | 4 +++- src/services/services.ts | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts index 9109734858..2cb1b35b79 100644 --- a/src/compiler/parser.ts +++ b/src/compiler/parser.ts @@ -1030,7 +1030,7 @@ namespace ts { } // note: this function creates only node - function createNode(kind: SyntaxKind, pos?: number): Node { + function createNode(kind: SyntaxKind, pos?: number): Node | Token | Identifier { nodeCount++; if (!(pos >= 0)) { pos = scanner.getStartPos(); @@ -5109,7 +5109,7 @@ namespace ts { } flags |= modifierToFlag(modifierKind); - modifiers.push(finishNode(createNode(modifierKind, modifierStart))); + modifiers.push(finishNode(createNode(modifierKind, modifierStart))); } if (modifiers) { modifiers.flags = flags; @@ -5128,7 +5128,7 @@ namespace ts { modifiers = []; modifiers.pos = modifierStart; flags |= modifierToFlag(modifierKind); - modifiers.push(finishNode(createNode(modifierKind, modifierStart))); + modifiers.push(finishNode(createNode(modifierKind, modifierStart))); modifiers.flags = flags; modifiers.end = scanner.getStartPos(); } diff --git a/src/compiler/types.ts b/src/compiler/types.ts index cc10d9348f..de4f6bc526 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -472,7 +472,9 @@ namespace ts { flags: NodeFlags; } - export interface Token extends Node { } + export interface Token extends Node { + __tokenTag: any; + } // @kind(SyntaxKind.AbstractKeyword) // @kind(SyntaxKind.AsyncKeyword) diff --git a/src/services/services.ts b/src/services/services.ts index bffb44d9fb..5fb4251e60 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -351,6 +351,7 @@ namespace ts { public flags: NodeFlags; public parent: Node; public jsDocComments: JSDocComment[]; + public __tokenTag: any; constructor(public pos: number, public end: number) { this.flags = NodeFlags.None; From 78a1ca7d5009b3d361df7822971bf546827e7718 Mon Sep 17 00:00:00 2001 From: Sheetal Nandi Date: Mon, 11 Jul 2016 14:02:23 -0700 Subject: [PATCH 6/6] Make setting properties explicit instead of using parameter properties --- src/services/services.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 5fb4251e60..56f2a87509 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -348,12 +348,17 @@ namespace ts { class TokenOrIdentifierObject implements Token { public kind: SyntaxKind; + public pos: number; + public end: number; public flags: NodeFlags; public parent: Node; public jsDocComments: JSDocComment[]; public __tokenTag: any; - constructor(public pos: number, public end: number) { + constructor(pos: number, end: number) { + // Set properties in same order as NodeObject + this.pos = pos; + this.end = end; this.flags = NodeFlags.None; this.parent = undefined; } @@ -416,8 +421,10 @@ namespace ts { } class TokenObject extends TokenOrIdentifierObject { - constructor(public kind: SyntaxKind, pos: number, end: number) { + public kind: SyntaxKind; + constructor(kind: SyntaxKind, pos: number, end: number) { super(pos, end); + this.kind = kind; } }