From ea30c68128499f649be64711ae94b7946d16acff Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 19 Jan 2015 18:07:54 -0800 Subject: [PATCH 01/10] Rudimentary, but imperfect, lexical classification for templates. --- src/services/services.ts | 95 +++++++++++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 11 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index def895880e..d585722e7e 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1143,6 +1143,9 @@ module ts { InMultiLineCommentTrivia, InSingleQuoteStringLiteral, InDoubleQuoteStringLiteral, + InTemplateHeadLiteral, // this could also be a NoSubstitutionTemplateLiteral + InTemplateMiddleLiteral, //this could also be a TemplateTail + InTemplateSubstitutionPosition, } export enum TokenClass { @@ -5650,12 +5653,12 @@ module ts { // if there are more cases we want the classifier to be better at. return true; } - - // 'classifyKeywordsInGenerics' should be 'true' when a syntactic classifier is not present. - function getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult { + + function getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult { var offset = 0; var token = SyntaxKind.Unknown; var lastNonTriviaToken = SyntaxKind.Unknown; + var templateStack: SyntaxKind[]; // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. @@ -5675,6 +5678,21 @@ module ts { text = "/*\n" + text; offset = 3; break; + case EndOfLineState.InTemplateHeadLiteral: + if (syntacticClassifierAbsent) { + text = "`\n" + text; + offset = 2; + } + break; + case EndOfLineState.InTemplateMiddleLiteral: + if (syntacticClassifierAbsent) { + text = "${\n" + text; + offset = 3; + } + // fallthrough + case EndOfLineState.InTemplateSubstitutionPosition: + templateStack = [SyntaxKind.TemplateHead]; + break; } scanner.setText(text); @@ -5739,12 +5757,50 @@ module ts { token === SyntaxKind.StringKeyword || token === SyntaxKind.NumberKeyword || token === SyntaxKind.BooleanKeyword) { - if (angleBracketStack > 0 && !classifyKeywordsInGenerics) { - // If it looks like we're could be in something generic, don't classify this - // as a keyword. We may just get overwritten by the syntactic classifier, - // causing a noisy experience for the user. - token = SyntaxKind.Identifier; - } + if (angleBracketStack > 0 && !syntacticClassifierAbsent) { + // If it looks like we're could be in something generic, don't classify this + // as a keyword. We may just get overwritten by the syntactic classifier, + // causing a noisy experience for the user. + token = SyntaxKind.Identifier; + } + } + else if (token === SyntaxKind.TemplateHead && syntacticClassifierAbsent) { + if (!templateStack) { + templateStack = [token]; + } + else { + templateStack.push(token); + } + } + else if (token === SyntaxKind.OpenBraceToken && syntacticClassifierAbsent) { + // If we don't have anything on the template stack, + // then we aren't trying to keep track of a previously scanned template head. + if (templateStack && templateStack.length > 0) { + templateStack.push(token); + } + } + else if (token === SyntaxKind.CloseBraceToken && syntacticClassifierAbsent) { + // If we don't have anything on the template stack, + // then we aren't trying to keep track of a previously scanned template head. + if (templateStack && templateStack.length > 0) { + var lastTemplateStackToken = lastOrUndefined(templateStack); + + if (lastTemplateStackToken === SyntaxKind.TemplateHead) { + token = scanner.reScanTemplateToken(); + + // Only pop on a TemplateTail; a TemplateMiddle indicates there is more for us. + if (token === SyntaxKind.TemplateTail) { + templateStack.pop(); + } + else { + Debug.assert(token === SyntaxKind.TemplateMiddle, "Should have been a template middle. Was " + token); + } + } + else { + Debug.assert(token === SyntaxKind.CloseBraceToken, "Should have been an open brace. Was: " + token); + templateStack.pop(); + } + } } lastNonTriviaToken = token; @@ -5760,7 +5816,7 @@ module ts { var start = scanner.getTokenPos(); var end = scanner.getTextPos(); - addResult(end - start, classFromKind(token)); + addResult(end - start, classFromKind(token, syntacticClassifierAbsent)); if (end >= text.length) { if (token === SyntaxKind.StringLiteral) { @@ -5789,6 +5845,19 @@ module ts { result.finalLexState = EndOfLineState.InMultiLineCommentTrivia; } } + else if (isTemplateLiteralKind(token) && syntacticClassifierAbsent) { + if (scanner.isUnterminated()) { + if (token === SyntaxKind.TemplateMiddle) { + result.finalLexState = EndOfLineState.InTemplateMiddleLiteral; + } + else { + result.finalLexState = EndOfLineState.InTemplateHeadLiteral; + } + } + } + else if (templateStack && templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) { + result.finalLexState = EndOfLineState.InTemplateSubstitutionPosition; + } } } @@ -5866,7 +5935,7 @@ module ts { return token >= SyntaxKind.FirstKeyword && token <= SyntaxKind.LastKeyword; } - function classFromKind(token: SyntaxKind) { + function classFromKind(token: SyntaxKind, syntacticClassifierAbsent?: boolean) { if (isKeyword(token)) { return TokenClass.Keyword; } @@ -5892,6 +5961,10 @@ module ts { return TokenClass.Whitespace; case SyntaxKind.Identifier: default: + // Only give a classification if nothing will more accurately classify. + if (syntacticClassifierAbsent && isTemplateLiteralKind(token)) { + return TokenClass.StringLiteral; // should make a TemplateLiteral + } return TokenClass.Identifier; } } From 3429fab6fbdd5fb37975c088b88079a4cd22fb30 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 20 Jan 2015 12:42:03 -0800 Subject: [PATCH 02/10] Fixed issue with the kinds we check. --- src/services/services.ts | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index d585722e7e..50b852c241 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1143,8 +1143,8 @@ module ts { InMultiLineCommentTrivia, InSingleQuoteStringLiteral, InDoubleQuoteStringLiteral, - InTemplateHeadLiteral, // this could also be a NoSubstitutionTemplateLiteral - InTemplateMiddleLiteral, //this could also be a TemplateTail + InTemplateHeadOrNoSubstitutionTemplate, + InTemplateMiddleOrTail, InTemplateSubstitutionPosition, } @@ -5678,20 +5678,22 @@ module ts { text = "/*\n" + text; offset = 3; break; - case EndOfLineState.InTemplateHeadLiteral: + case EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate: if (syntacticClassifierAbsent) { text = "`\n" + text; offset = 2; } break; - case EndOfLineState.InTemplateMiddleLiteral: + case EndOfLineState.InTemplateMiddleOrTail: if (syntacticClassifierAbsent) { - text = "${\n" + text; - offset = 3; + text = "}\n" + text; + offset = 2; } // fallthrough case EndOfLineState.InTemplateSubstitutionPosition: - templateStack = [SyntaxKind.TemplateHead]; + if (syntacticClassifierAbsent) { + templateStack = [SyntaxKind.TemplateHead]; + } break; } @@ -5847,11 +5849,14 @@ module ts { } else if (isTemplateLiteralKind(token) && syntacticClassifierAbsent) { if (scanner.isUnterminated()) { - if (token === SyntaxKind.TemplateMiddle) { - result.finalLexState = EndOfLineState.InTemplateMiddleLiteral; + if (token === SyntaxKind.TemplateTail) { + result.finalLexState = EndOfLineState.InTemplateMiddleOrTail; + } + else if (token === SyntaxKind.NoSubstitutionTemplateLiteral) { + result.finalLexState = EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate; } else { - result.finalLexState = EndOfLineState.InTemplateHeadLiteral; + Debug.fail("Only 'NoSubstitutionTemplateLiteral's and 'TemplateTail's can be unterminated; got SyntaxKind #" + token); } } } From f1f085eda65876cb3d8ae2a57b3ec8a03f86ac7b Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 12:06:01 -0800 Subject: [PATCH 03/10] Addressed CR feedback. --- src/services/services.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 50b852c241..858a1ac175 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5620,6 +5620,10 @@ module ts { noRegexTable[SyntaxKind.TrueKeyword] = true; noRegexTable[SyntaxKind.FalseKeyword] = true; + // Just a stack of TemplateHeads and OpenCurlyBraces, used + // to perform rudimentary classification on templates. + var templateStack: SyntaxKind[] = []; + function isAccessibilityModifier(kind: SyntaxKind) { switch (kind) { case SyntaxKind.PublicKeyword: @@ -5658,7 +5662,6 @@ module ts { var offset = 0; var token = SyntaxKind.Unknown; var lastNonTriviaToken = SyntaxKind.Unknown; - var templateStack: SyntaxKind[]; // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. @@ -5725,6 +5728,11 @@ module ts { // work well enough in practice. var angleBracketStack = 0; + // Empty out the template stack for reuse. + while (templateStack.length > 0) { + templateStack.pop(); + } + do { token = scanner.scan(); @@ -5767,24 +5775,19 @@ module ts { } } else if (token === SyntaxKind.TemplateHead && syntacticClassifierAbsent) { - if (!templateStack) { - templateStack = [token]; - } - else { - templateStack.push(token); - } + templateStack.push(token); } else if (token === SyntaxKind.OpenBraceToken && syntacticClassifierAbsent) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. - if (templateStack && templateStack.length > 0) { + if (templateStack.length > 0) { templateStack.push(token); } } else if (token === SyntaxKind.CloseBraceToken && syntacticClassifierAbsent) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. - if (templateStack && templateStack.length > 0) { + if (templateStack.length > 0) { var lastTemplateStackToken = lastOrUndefined(templateStack); if (lastTemplateStackToken === SyntaxKind.TemplateHead) { @@ -5860,7 +5863,7 @@ module ts { } } } - else if (templateStack && templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) { + else if (templateStack.length > 0 && lastOrUndefined(templateStack) === SyntaxKind.TemplateHead) { result.finalLexState = EndOfLineState.InTemplateSubstitutionPosition; } } From 3fea0aefbef3ff808de8799bd1a622cd41c08885 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 12:37:03 -0800 Subject: [PATCH 04/10] classifyKeywordsInGenerics -> syntacticClassifierAbsent --- src/services/services.ts | 2 +- src/services/shims.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 858a1ac175..172fb4ab00 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1171,7 +1171,7 @@ module ts { } export interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; } /** diff --git a/src/services/shims.ts b/src/services/shims.ts index 0f9f1a12cd..a0eb1cb1ac 100644 --- a/src/services/shims.ts +++ b/src/services/shims.ts @@ -165,7 +165,7 @@ module ts { } export interface ClassifierShim extends Shim { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): string; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): string; } export interface CoreServicesShim extends Shim { From ab79faef85568a9944709c0940c6f484aaa5187c Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 15:41:01 -0800 Subject: [PATCH 05/10] Added tests, fixed order of emptying templateStack, unconditionally perform template classification. --- src/services/services.ts | 43 +++--- .../cases/unittests/services/colorization.ts | 130 ++++++++++++++++-- 2 files changed, 137 insertions(+), 36 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 172fb4ab00..673748d696 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5663,6 +5663,11 @@ module ts { var token = SyntaxKind.Unknown; var lastNonTriviaToken = SyntaxKind.Unknown; + // Empty out the template stack for reuse. + while (templateStack.length > 0) { + templateStack.pop(); + } + // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. // @@ -5682,21 +5687,15 @@ module ts { offset = 3; break; case EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate: - if (syntacticClassifierAbsent) { - text = "`\n" + text; - offset = 2; - } + text = "`\n" + text; + offset = 2; break; case EndOfLineState.InTemplateMiddleOrTail: - if (syntacticClassifierAbsent) { - text = "}\n" + text; - offset = 2; - } + text = "}\n" + text; + offset = 2; // fallthrough case EndOfLineState.InTemplateSubstitutionPosition: - if (syntacticClassifierAbsent) { - templateStack = [SyntaxKind.TemplateHead]; - } + templateStack.push(SyntaxKind.TemplateHead); break; } @@ -5728,11 +5727,6 @@ module ts { // work well enough in practice. var angleBracketStack = 0; - // Empty out the template stack for reuse. - while (templateStack.length > 0) { - templateStack.pop(); - } - do { token = scanner.scan(); @@ -5774,17 +5768,17 @@ module ts { token = SyntaxKind.Identifier; } } - else if (token === SyntaxKind.TemplateHead && syntacticClassifierAbsent) { + else if (token === SyntaxKind.TemplateHead) { templateStack.push(token); } - else if (token === SyntaxKind.OpenBraceToken && syntacticClassifierAbsent) { + else if (token === SyntaxKind.OpenBraceToken) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. if (templateStack.length > 0) { templateStack.push(token); } } - else if (token === SyntaxKind.CloseBraceToken && syntacticClassifierAbsent) { + else if (token === SyntaxKind.CloseBraceToken) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. if (templateStack.length > 0) { @@ -5821,7 +5815,7 @@ module ts { var start = scanner.getTokenPos(); var end = scanner.getTextPos(); - addResult(end - start, classFromKind(token, syntacticClassifierAbsent)); + addResult(end - start, classFromKind(token)); if (end >= text.length) { if (token === SyntaxKind.StringLiteral) { @@ -5850,7 +5844,7 @@ module ts { result.finalLexState = EndOfLineState.InMultiLineCommentTrivia; } } - else if (isTemplateLiteralKind(token) && syntacticClassifierAbsent) { + else if (isTemplateLiteralKind(token)) { if (scanner.isUnterminated()) { if (token === SyntaxKind.TemplateTail) { result.finalLexState = EndOfLineState.InTemplateMiddleOrTail; @@ -5943,7 +5937,7 @@ module ts { return token >= SyntaxKind.FirstKeyword && token <= SyntaxKind.LastKeyword; } - function classFromKind(token: SyntaxKind, syntacticClassifierAbsent?: boolean) { + function classFromKind(token: SyntaxKind) { if (isKeyword(token)) { return TokenClass.Keyword; } @@ -5969,9 +5963,8 @@ module ts { return TokenClass.Whitespace; case SyntaxKind.Identifier: default: - // Only give a classification if nothing will more accurately classify. - if (syntacticClassifierAbsent && isTemplateLiteralKind(token)) { - return TokenClass.StringLiteral; // should make a TemplateLiteral + if (isTemplateLiteralKind(token)) { + return TokenClass.StringLiteral; // maybe make a TemplateLiteral } return TokenClass.Identifier; } diff --git a/tests/cases/unittests/services/colorization.ts b/tests/cases/unittests/services/colorization.ts index 0114976430..c834f85806 100644 --- a/tests/cases/unittests/services/colorization.ts +++ b/tests/cases/unittests/services/colorization.ts @@ -4,6 +4,7 @@ interface ClassificationEntry { value: any; classification: ts.TokenClass; + position?: number; } describe('Colorization', function () { @@ -23,16 +24,23 @@ describe('Colorization', function () { return undefined; } - function punctuation(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Punctuation }; } - function keyword(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Keyword }; } - function operator(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Operator }; } - function comment(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Comment }; } - function whitespace(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Whitespace }; } - function identifier(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Identifier }; } - function numberLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.NumberLiteral }; } - function stringLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.StringLiteral }; } - function regExpLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.RegExpLiteral }; } - function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, classification: undefined }; } + function punctuation(text: string, position?: number) { return createClassification(text, ts.TokenClass.Punctuation, position); } + function keyword(text: string, position?: number) { return createClassification(text, ts.TokenClass.Keyword, position); } + function operator(text: string, position?: number) { return createClassification(text, ts.TokenClass.Operator, position); } + function comment(text: string, position?: number) { return createClassification(text, ts.TokenClass.Comment, position); } + function whitespace(text: string, position?: number) { return createClassification(text, ts.TokenClass.Whitespace, position); } + function identifier(text: string, position?: number) { return createClassification(text, ts.TokenClass.Identifier, position); } + function numberLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.NumberLiteral, position); } + function stringLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.StringLiteral, position); } + function regExpLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.RegExpLiteral, position); } + function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, classification: undefined, position: 0 }; } + function createClassification(text: string, tokenClass: ts.TokenClass, position?: number): ClassificationEntry { + return { + value: text, + classification: tokenClass, + position: position, + }; + } function testLexicalClassification(text: string, initialEndOfLineState: ts.EndOfLineState, ...expectedEntries: ClassificationEntry[]): void { var result = classifier.getClassificationsForLine(text, initialEndOfLineState); @@ -44,7 +52,7 @@ describe('Colorization', function () { assert.equal(result.finalLexState, expectedEntry.value, "final endOfLineState does not match expected."); } else { - var actualEntryPosition = text.indexOf(expectedEntry.value); + var actualEntryPosition = expectedEntry.position !== undefined ? expectedEntry.position : text.indexOf(expectedEntry.value); assert(actualEntryPosition >= 0, "token: '" + expectedEntry.value + "' does not exit in text: '" + text + "'."); var actualEntry = getEntryAtPosistion(result, actualEntryPosition); @@ -254,6 +262,106 @@ describe('Colorization', function () { finalEndOfLineState(ts.EndOfLineState.Start)); }); + it("classifies a single line no substitution template string correctly", () => { + testLexicalClassification("`number number public string`", + ts.EndOfLineState.Start, + stringLiteral("`number number public string`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies substitution parts of a template string correctly", () => { + testLexicalClassification("`number '${ 1 + 1 }' string '${ 'hello' }'`", + ts.EndOfLineState.Start, + stringLiteral("`number '${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}' string '${"), + stringLiteral("'hello'"), + stringLiteral("}'`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies an unterminated no substitution template string correctly", () => { + testLexicalClassification("`hello world", + ts.EndOfLineState.Start, + stringLiteral("`hello world"), + finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); + }); + it("classifies the entire line of an unterminated multiline no-substitution/head template", () => { + testLexicalClassification("...", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("..."), + finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); + }); + it("classifies the entire line of an unterminated multiline template middle/end",() => { + testLexicalClassification("...", + ts.EndOfLineState.InTemplateMiddleOrTail, + stringLiteral("..."), + finalEndOfLineState(ts.EndOfLineState.InTemplateMiddleOrTail)); + }); + it("classifies a termination of a multiline template head", () => { + testLexicalClassification("...${", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("...${"), + finalEndOfLineState(ts.EndOfLineState.InTemplateSubstitutionPosition)); + }); + it("classifies the termination of a multiline no substitution template", () => { + testLexicalClassification("...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies the substitution parts and middle/tail of a multiline template string", () => { + testLexicalClassification("${ 1 + 1 }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies a template middle and propagates the end of line state",() => { + testLexicalClassification("${ 1 + 1 }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies substitution expressions with curly braces appropriately", () => { + var pos = 0; + var lastLength = 0; + + testLexicalClassification("...${ () => { } } ${ { x: `1` } }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral(track("...${"), pos), + punctuation(track(" ", "("), pos), + punctuation(track(")"), pos), + punctuation(track(" ", "=>"), pos), + punctuation(track(" ", "{"), pos), + punctuation(track(" ", "}"), pos), + stringLiteral(track(" ", "} ${"), pos), + punctuation(track(" ", "{"), pos), + identifier(track(" ", "x"), pos), + punctuation(track(":"), pos), + stringLiteral(track(" ", "`1`"), pos), + punctuation(track(" ", "}"), pos), + stringLiteral(track(" ", "}...`"), pos), + finalEndOfLineState(ts.EndOfLineState.Start)); + + // Adjusts 'pos' by accounting for the length of each portion of the string, + // but only return the last given string + function track(...vals: string[]): string { + for (var i = 0, n = vals.length; i < n; i++) { + pos += lastLength; + lastLength = vals[i].length; + } + return ts.lastOrUndefined(vals); + } + }); + it("classifies partially written generics correctly.", function () { testLexicalClassification("Foo Date: Mon, 26 Jan 2015 15:43:26 -0800 Subject: [PATCH 06/10] Fixed assertion --- src/services/services.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/services.ts b/src/services/services.ts index 673748d696..90227eeb1f 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5796,7 +5796,7 @@ module ts { } } else { - Debug.assert(token === SyntaxKind.CloseBraceToken, "Should have been an open brace. Was: " + token); + Debug.assert(lastTemplateStackToken === SyntaxKind.OpenBraceToken, "Should have been an open brace. Was: " + token); templateStack.pop(); } } From 15b333e10281fcb3d136b57d80597233154d1742 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 16:06:56 -0800 Subject: [PATCH 07/10] Added comment for 'syntacticClassifierAbsent'. --- src/services/services.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/services/services.ts b/src/services/services.ts index 90227eeb1f..39eab2f7e1 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5658,6 +5658,8 @@ module ts { return true; } + // If there is a syntactic classifier ('syntacticClassifierAbsent' is false), + // we will be more conservative in order to avoid conflicting with the syntactic classifier. function getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult { var offset = 0; var token = SyntaxKind.Unknown; From 65e23b1e60699ed22add237a75ada0e3589f4aa2 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Thu, 12 Feb 2015 16:46:25 -0800 Subject: [PATCH 08/10] Added API baseline changes. --- tests/baselines/reference/APISample_compile.js | 5 ++++- tests/baselines/reference/APISample_compile.types | 15 ++++++++++++--- tests/baselines/reference/APISample_linter.js | 5 ++++- tests/baselines/reference/APISample_linter.types | 15 ++++++++++++--- tests/baselines/reference/APISample_transform.js | 5 ++++- .../baselines/reference/APISample_transform.types | 15 ++++++++++++--- tests/baselines/reference/APISample_watcher.js | 5 ++++- tests/baselines/reference/APISample_watcher.types | 15 ++++++++++++--- 8 files changed, 64 insertions(+), 16 deletions(-) diff --git a/tests/baselines/reference/APISample_compile.js b/tests/baselines/reference/APISample_compile.js index 9a44ae12df..98b80ebc86 100644 --- a/tests/baselines/reference/APISample_compile.js +++ b/tests/baselines/reference/APISample_compile.js @@ -1741,6 +1741,9 @@ declare module "typescript" { InMultiLineCommentTrivia = 1, InSingleQuoteStringLiteral = 2, InDoubleQuoteStringLiteral = 3, + InTemplateHeadOrNoSubstitutionTemplate = 4, + InTemplateMiddleOrTail = 5, + InTemplateSubstitutionPosition = 6, } enum TokenClass { Punctuation = 0, @@ -1762,7 +1765,7 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_compile.types b/tests/baselines/reference/APISample_compile.types index 6945832023..77958b6b16 100644 --- a/tests/baselines/reference/APISample_compile.types +++ b/tests/baselines/reference/APISample_compile.types @@ -5539,6 +5539,15 @@ declare module "typescript" { InDoubleQuoteStringLiteral = 3, >InDoubleQuoteStringLiteral : EndOfLineState + + InTemplateHeadOrNoSubstitutionTemplate = 4, +>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState + + InTemplateMiddleOrTail = 5, +>InTemplateMiddleOrTail : EndOfLineState + + InTemplateSubstitutionPosition = 6, +>InTemplateSubstitutionPosition : EndOfLineState } enum TokenClass { >TokenClass : TokenClass @@ -5594,12 +5603,12 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState ->classifyKeywordsInGenerics : boolean +>syntacticClassifierAbsent : boolean >ClassificationResult : ClassificationResult } /** diff --git a/tests/baselines/reference/APISample_linter.js b/tests/baselines/reference/APISample_linter.js index 5413cdb8a7..f22532d353 100644 --- a/tests/baselines/reference/APISample_linter.js +++ b/tests/baselines/reference/APISample_linter.js @@ -1772,6 +1772,9 @@ declare module "typescript" { InMultiLineCommentTrivia = 1, InSingleQuoteStringLiteral = 2, InDoubleQuoteStringLiteral = 3, + InTemplateHeadOrNoSubstitutionTemplate = 4, + InTemplateMiddleOrTail = 5, + InTemplateSubstitutionPosition = 6, } enum TokenClass { Punctuation = 0, @@ -1793,7 +1796,7 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_linter.types b/tests/baselines/reference/APISample_linter.types index 55613f8d3e..09b242ac04 100644 --- a/tests/baselines/reference/APISample_linter.types +++ b/tests/baselines/reference/APISample_linter.types @@ -5683,6 +5683,15 @@ declare module "typescript" { InDoubleQuoteStringLiteral = 3, >InDoubleQuoteStringLiteral : EndOfLineState + + InTemplateHeadOrNoSubstitutionTemplate = 4, +>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState + + InTemplateMiddleOrTail = 5, +>InTemplateMiddleOrTail : EndOfLineState + + InTemplateSubstitutionPosition = 6, +>InTemplateSubstitutionPosition : EndOfLineState } enum TokenClass { >TokenClass : TokenClass @@ -5738,12 +5747,12 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState ->classifyKeywordsInGenerics : boolean +>syntacticClassifierAbsent : boolean >ClassificationResult : ClassificationResult } /** diff --git a/tests/baselines/reference/APISample_transform.js b/tests/baselines/reference/APISample_transform.js index 13ec31cd7d..56d7e7db5b 100644 --- a/tests/baselines/reference/APISample_transform.js +++ b/tests/baselines/reference/APISample_transform.js @@ -1773,6 +1773,9 @@ declare module "typescript" { InMultiLineCommentTrivia = 1, InSingleQuoteStringLiteral = 2, InDoubleQuoteStringLiteral = 3, + InTemplateHeadOrNoSubstitutionTemplate = 4, + InTemplateMiddleOrTail = 5, + InTemplateSubstitutionPosition = 6, } enum TokenClass { Punctuation = 0, @@ -1794,7 +1797,7 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_transform.types b/tests/baselines/reference/APISample_transform.types index a8ba6e9f52..ad9623c570 100644 --- a/tests/baselines/reference/APISample_transform.types +++ b/tests/baselines/reference/APISample_transform.types @@ -5635,6 +5635,15 @@ declare module "typescript" { InDoubleQuoteStringLiteral = 3, >InDoubleQuoteStringLiteral : EndOfLineState + + InTemplateHeadOrNoSubstitutionTemplate = 4, +>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState + + InTemplateMiddleOrTail = 5, +>InTemplateMiddleOrTail : EndOfLineState + + InTemplateSubstitutionPosition = 6, +>InTemplateSubstitutionPosition : EndOfLineState } enum TokenClass { >TokenClass : TokenClass @@ -5690,12 +5699,12 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState ->classifyKeywordsInGenerics : boolean +>syntacticClassifierAbsent : boolean >ClassificationResult : ClassificationResult } /** diff --git a/tests/baselines/reference/APISample_watcher.js b/tests/baselines/reference/APISample_watcher.js index 5c3f944faf..f03a36cfc2 100644 --- a/tests/baselines/reference/APISample_watcher.js +++ b/tests/baselines/reference/APISample_watcher.js @@ -1810,6 +1810,9 @@ declare module "typescript" { InMultiLineCommentTrivia = 1, InSingleQuoteStringLiteral = 2, InDoubleQuoteStringLiteral = 3, + InTemplateHeadOrNoSubstitutionTemplate = 4, + InTemplateMiddleOrTail = 5, + InTemplateSubstitutionPosition = 6, } enum TokenClass { Punctuation = 0, @@ -1831,7 +1834,7 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_watcher.types b/tests/baselines/reference/APISample_watcher.types index 54304c2e1b..41f6fd1ced 100644 --- a/tests/baselines/reference/APISample_watcher.types +++ b/tests/baselines/reference/APISample_watcher.types @@ -5808,6 +5808,15 @@ declare module "typescript" { InDoubleQuoteStringLiteral = 3, >InDoubleQuoteStringLiteral : EndOfLineState + + InTemplateHeadOrNoSubstitutionTemplate = 4, +>InTemplateHeadOrNoSubstitutionTemplate : EndOfLineState + + InTemplateMiddleOrTail = 5, +>InTemplateMiddleOrTail : EndOfLineState + + InTemplateSubstitutionPosition = 6, +>InTemplateSubstitutionPosition : EndOfLineState } enum TokenClass { >TokenClass : TokenClass @@ -5863,12 +5872,12 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, classifyKeywordsInGenerics?: boolean) => ClassificationResult + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState ->classifyKeywordsInGenerics : boolean +>syntacticClassifierAbsent : boolean >ClassificationResult : ClassificationResult } /** From 1dca5620f52d13af066187cd39357cdfcc9a6123 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Fri, 13 Feb 2015 12:56:21 -0800 Subject: [PATCH 09/10] Addressed CR feedback. --- src/services/services.ts | 45 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 39eab2f7e1..0a901f0df0 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1171,7 +1171,26 @@ module ts { } export interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** @@ -5620,8 +5639,26 @@ module ts { noRegexTable[SyntaxKind.TrueKeyword] = true; noRegexTable[SyntaxKind.FalseKeyword] = true; - // Just a stack of TemplateHeads and OpenCurlyBraces, used - // to perform rudimentary classification on templates. + // Just a stack of TemplateHeads and OpenCurlyBraces, used to perform rudimentary (inexact) + // classification on template strings. Because of the context free nature of templates, + // the only precise way to classify a template portion would be by propagating the stack across + // lines, just as we do with the end-of-line state. However, this is a burden for implementers, + // and the behavior is entirely subsumed by the syntactic classifier anyway, so we instead + // flatten any nesting when the template stack is non-empty and encode it in the end-of-line state. + // Situations in which this fails are + // 1) When template strings are nested across different lines: + // `hello ${ `world + // ` }` + // + // Where on the second line, you will get the closing of a template, + // a closing curly, and a new template. + // + // 2) When substitution expressions have curly braces and the curly brace falls on the next line: + // `hello ${ () => { + // return "world" } } ` + // + // Where on the second line, you will get the 'return' keyword, + // a string literal, and a template end consisting of '} } `'. var templateStack: SyntaxKind[] = []; function isAccessibilityModifier(kind: SyntaxKind) { @@ -5966,7 +6003,7 @@ module ts { case SyntaxKind.Identifier: default: if (isTemplateLiteralKind(token)) { - return TokenClass.StringLiteral; // maybe make a TemplateLiteral + return TokenClass.StringLiteral; } return TokenClass.Identifier; } From a1b90f0e323a93de4e28f2bdcab3b4e15cb0593d Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Fri, 13 Feb 2015 13:33:12 -0800 Subject: [PATCH 10/10] Added baselines. --- .../baselines/reference/APISample_compile.js | 21 ++++++++++++++++- .../reference/APISample_compile.types | 23 +++++++++++++++++-- tests/baselines/reference/APISample_linter.js | 21 ++++++++++++++++- .../reference/APISample_linter.types | 23 +++++++++++++++++-- .../reference/APISample_transform.js | 21 ++++++++++++++++- .../reference/APISample_transform.types | 23 +++++++++++++++++-- .../baselines/reference/APISample_watcher.js | 21 ++++++++++++++++- .../reference/APISample_watcher.types | 23 +++++++++++++++++-- 8 files changed, 164 insertions(+), 12 deletions(-) diff --git a/tests/baselines/reference/APISample_compile.js b/tests/baselines/reference/APISample_compile.js index 98b80ebc86..b91a706aa4 100644 --- a/tests/baselines/reference/APISample_compile.js +++ b/tests/baselines/reference/APISample_compile.js @@ -1765,7 +1765,26 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_compile.types b/tests/baselines/reference/APISample_compile.types index 77958b6b16..ae53c66680 100644 --- a/tests/baselines/reference/APISample_compile.types +++ b/tests/baselines/reference/APISample_compile.types @@ -5603,8 +5603,27 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState diff --git a/tests/baselines/reference/APISample_linter.js b/tests/baselines/reference/APISample_linter.js index f22532d353..68c13edb7d 100644 --- a/tests/baselines/reference/APISample_linter.js +++ b/tests/baselines/reference/APISample_linter.js @@ -1796,7 +1796,26 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_linter.types b/tests/baselines/reference/APISample_linter.types index 09b242ac04..0c153e7b1d 100644 --- a/tests/baselines/reference/APISample_linter.types +++ b/tests/baselines/reference/APISample_linter.types @@ -5747,8 +5747,27 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState diff --git a/tests/baselines/reference/APISample_transform.js b/tests/baselines/reference/APISample_transform.js index 56d7e7db5b..0e80f6b55b 100644 --- a/tests/baselines/reference/APISample_transform.js +++ b/tests/baselines/reference/APISample_transform.js @@ -1797,7 +1797,26 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_transform.types b/tests/baselines/reference/APISample_transform.types index ad9623c570..0c201b0a5f 100644 --- a/tests/baselines/reference/APISample_transform.types +++ b/tests/baselines/reference/APISample_transform.types @@ -5699,8 +5699,27 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState diff --git a/tests/baselines/reference/APISample_watcher.js b/tests/baselines/reference/APISample_watcher.js index f03a36cfc2..27a2ce5e41 100644 --- a/tests/baselines/reference/APISample_watcher.js +++ b/tests/baselines/reference/APISample_watcher.js @@ -1834,7 +1834,26 @@ declare module "typescript" { classification: TokenClass; } interface Classifier { - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; } /** * The document registry represents a store of SourceFile objects that can be shared between diff --git a/tests/baselines/reference/APISample_watcher.types b/tests/baselines/reference/APISample_watcher.types index 41f6fd1ced..4e137cae59 100644 --- a/tests/baselines/reference/APISample_watcher.types +++ b/tests/baselines/reference/APISample_watcher.types @@ -5872,8 +5872,27 @@ declare module "typescript" { interface Classifier { >Classifier : Classifier - getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean): ClassificationResult; ->getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent?: boolean) => ClassificationResult + /** + * Gives lexical classifications of tokens on a line without any syntactic context. + * For instance, a token consisting of the text 'string' can be either an identifier + * named 'string' or the keyword 'string', however, because this classifier is not aware, + * it relies on certain heuristics to give acceptable results. For classifications where + * speed trumps accuracy, this function is preferable; however, for true accuracy, the + * syntactic classifier is ideal. In fact, in certain editing scenarios, combining the + * lexical, syntactic, and semantic classifiers may issue the best user experience. + * + * @param text The text of a line to classify. + * @param lexState The state of the lexical classifier at the end of the previous line. + * @param syntacticClassifierAbsent Whether the client is *not* using a syntactic classifier. + * If there is no syntactic classifier (syntacticClassifierAbsent=true), + * certain heuristics may be used in its place; however, if there is a + * syntactic classifier (syntacticClassifierAbsent=false), certain + * classifications which may be incorrectly categorized will be given + * back as Identifiers in order to allow the syntactic classifier to + * subsume the classification. + */ + getClassificationsForLine(text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean): ClassificationResult; +>getClassificationsForLine : (text: string, lexState: EndOfLineState, syntacticClassifierAbsent: boolean) => ClassificationResult >text : string >lexState : EndOfLineState >EndOfLineState : EndOfLineState