diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts index 5fce6df3bd..85f2b9ebce 100644 --- a/src/compiler/parser.ts +++ b/src/compiler/parser.ts @@ -5416,4 +5416,4 @@ module ts { Value = -1 } } -} \ No newline at end of file +} diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index 53d213f5bd..c2b693d677 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -24,7 +24,9 @@ module ts { reScanSlashToken(): SyntaxKind; reScanTemplateToken(): SyntaxKind; scan(): SyntaxKind; - setText(text: string): void; + // Sets the text for the scanner to scan. An optional subrange starting point and length + // can be provided to have the scanner only scan a portion of the text. + setText(text: string, start?: number, length?: number): void; setOnError(onError: ErrorCallback): void; setScriptTarget(scriptTarget: ScriptTarget): void; setTextPos(textPos: number): void; @@ -597,11 +599,11 @@ module ts { ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierPart(ch, languageVersion); } - /* @internal */ - export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback): Scanner { + // Creates a scanner over a (possibly unspecified) range of a piece of text. + /* @internal */ + export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback, start?: number, length?: number): Scanner { let pos: number; // Current position (end position of text of current token) - let len: number; // Length of text - + let end: number; // end of text let startPos: number; // Start position of whitespace before current token let tokenPos: number; // Start position of text of current token let token: SyntaxKind; @@ -610,7 +612,7 @@ module ts { let hasExtendedUnicodeEscape: boolean; let tokenIsUnterminated: boolean; - setText(text); + setText(text, start, length); return { getStartPos: () => startPos, @@ -732,7 +734,7 @@ module ts { let result = ""; let start = pos; while (true) { - if (pos >= len) { + if (pos >= end) { result += text.substring(start, pos); tokenIsUnterminated = true; error(Diagnostics.Unterminated_string_literal); @@ -774,7 +776,7 @@ module ts { let resultingToken: SyntaxKind; while (true) { - if (pos >= len) { + if (pos >= end) { contents += text.substring(start, pos); tokenIsUnterminated = true; error(Diagnostics.Unterminated_template_literal); @@ -793,7 +795,7 @@ module ts { } // '${' - if (currChar === CharacterCodes.$ && pos + 1 < len && text.charCodeAt(pos + 1) === CharacterCodes.openBrace) { + if (currChar === CharacterCodes.$ && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.openBrace) { contents += text.substring(start, pos); pos += 2; resultingToken = startedWithBacktick ? SyntaxKind.TemplateHead : SyntaxKind.TemplateMiddle; @@ -814,7 +816,7 @@ module ts { contents += text.substring(start, pos); pos++; - if (pos < len && text.charCodeAt(pos) === CharacterCodes.lineFeed) { + if (pos < end && text.charCodeAt(pos) === CharacterCodes.lineFeed) { pos++; } @@ -834,7 +836,7 @@ module ts { function scanEscapeSequence(): string { pos++; - if (pos >= len) { + if (pos >= end) { error(Diagnostics.Unexpected_end_of_text); return ""; } @@ -860,7 +862,7 @@ module ts { return "\""; case CharacterCodes.u: // '\u{DDDDDDDD}' - if (pos < len && text.charCodeAt(pos) === CharacterCodes.openBrace) { + if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) { hasExtendedUnicodeEscape = true; pos++; return scanExtendedUnicodeEscape(); @@ -876,7 +878,7 @@ module ts { // when encountering a LineContinuation (i.e. a backslash and a line terminator sequence), // the line terminator is interpreted to be "the empty code unit sequence". case CharacterCodes.carriageReturn: - if (pos < len && text.charCodeAt(pos) === CharacterCodes.lineFeed) { + if (pos < end && text.charCodeAt(pos) === CharacterCodes.lineFeed) { pos++; } // fall through @@ -915,7 +917,7 @@ module ts { isInvalidExtendedEscape = true; } - if (pos >= len) { + if (pos >= end) { error(Diagnostics.Unexpected_end_of_text); isInvalidExtendedEscape = true; } @@ -952,7 +954,7 @@ module ts { // Current character is known to be a backslash. Check for Unicode escape of the form '\uXXXX' // and return code point value if valid Unicode escape is found. Otherwise return -1. function peekUnicodeEscape(): number { - if (pos + 5 < len && text.charCodeAt(pos + 1) === CharacterCodes.u) { + if (pos + 5 < end && text.charCodeAt(pos + 1) === CharacterCodes.u) { let start = pos; pos += 2; let value = scanExactNumberOfHexDigits(4); @@ -965,7 +967,7 @@ module ts { function scanIdentifierParts(): string { let result = ""; let start = pos; - while (pos < len) { + while (pos < end) { let ch = text.charCodeAt(pos); if (isIdentifierPart(ch)) { pos++; @@ -1032,7 +1034,7 @@ module ts { tokenIsUnterminated = false; while (true) { tokenPos = pos; - if (pos >= len) { + if (pos >= end) { return token = SyntaxKind.EndOfFileToken; } let ch = text.charCodeAt(pos); @@ -1045,7 +1047,7 @@ module ts { continue; } else { - if (ch === CharacterCodes.carriageReturn && pos + 1 < len && text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) { + if (ch === CharacterCodes.carriageReturn && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) { // consume both CR and LF pos += 2; } @@ -1063,7 +1065,7 @@ module ts { continue; } else { - while (pos < len && isWhiteSpace(text.charCodeAt(pos))) { + while (pos < end && isWhiteSpace(text.charCodeAt(pos))) { pos++; } return token = SyntaxKind.WhitespaceTrivia; @@ -1136,7 +1138,7 @@ module ts { if (text.charCodeAt(pos + 1) === CharacterCodes.slash) { pos += 2; - while (pos < len) { + while (pos < end) { if (isLineBreak(text.charCodeAt(pos))) { break; } @@ -1156,7 +1158,7 @@ module ts { pos += 2; let commentClosed = false; - while (pos < len) { + while (pos < end) { let ch = text.charCodeAt(pos); if (ch === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) { @@ -1191,7 +1193,7 @@ module ts { return pos++, token = SyntaxKind.SlashToken; case CharacterCodes._0: - if (pos + 2 < len && (text.charCodeAt(pos + 1) === CharacterCodes.X || text.charCodeAt(pos + 1) === CharacterCodes.x)) { + if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.X || text.charCodeAt(pos + 1) === CharacterCodes.x)) { pos += 2; let value = scanMinimumNumberOfHexDigits(1); if (value < 0) { @@ -1201,7 +1203,7 @@ module ts { tokenValue = "" + value; return token = SyntaxKind.NumericLiteral; } - else if (pos + 2 < len && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) { + else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) { pos += 2; let value = scanBinaryOrOctalDigits(/* base */ 2); if (value < 0) { @@ -1211,7 +1213,7 @@ module ts { tokenValue = "" + value; return token = SyntaxKind.NumericLiteral; } - else if (pos + 2 < len && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) { + else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) { pos += 2; let value = scanBinaryOrOctalDigits(/* base */ 8); if (value < 0) { @@ -1222,7 +1224,7 @@ module ts { return token = SyntaxKind.NumericLiteral; } // Try to parse as an octal - if (pos + 1 < len && isOctalDigit(text.charCodeAt(pos + 1))) { + if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) { tokenValue = "" + scanOctalDigits(); return token = SyntaxKind.NumericLiteral; } @@ -1337,7 +1339,7 @@ module ts { default: if (isIdentifierStart(ch)) { pos++; - while (pos < len && isIdentifierPart(ch = text.charCodeAt(pos))) pos++; + while (pos < end && isIdentifierPart(ch = text.charCodeAt(pos))) pos++; tokenValue = text.substring(tokenPos, pos); if (ch === CharacterCodes.backslash) { tokenValue += scanIdentifierParts(); @@ -1388,7 +1390,7 @@ module ts { while (true) { // If we reach the end of a file, or hit a newline, then this is an unterminated // regex. Report error and return what we have so far. - if (p >= len) { + if (p >= end) { tokenIsUnterminated = true; error(Diagnostics.Unterminated_regular_expression_literal) break; @@ -1424,7 +1426,7 @@ module ts { p++; } - while (p < len && isIdentifierPart(text.charCodeAt(p))) { + while (p < end && isIdentifierPart(text.charCodeAt(p))) { p++; } pos = p; @@ -1473,10 +1475,10 @@ module ts { return speculationHelper(callback, /*isLookahead:*/ false); } - function setText(newText: string) { + function setText(newText: string, start: number, length: number) { text = newText || ""; - len = text.length; - setTextPos(0); + end = length === undefined ? text.length : start + length; + setTextPos(start || 0); } function setOnError(errorCallback: ErrorCallback) { @@ -1488,6 +1490,7 @@ module ts { } function setTextPos(textPos: number) { + Debug.assert(textPos >= 0); pos = textPos; startPos = textPos; tokenPos = textPos; diff --git a/src/compiler/utilities.ts b/src/compiler/utilities.ts index 0f2bd5daba..b1aa59be32 100644 --- a/src/compiler/utilities.ts +++ b/src/compiler/utilities.ts @@ -274,8 +274,7 @@ module ts { } export function getSpanOfTokenAtPosition(sourceFile: SourceFile, pos: number): TextSpan { - let scanner = createScanner(sourceFile.languageVersion, /*skipTrivia*/ true, sourceFile.text); - scanner.setTextPos(pos); + let scanner = createScanner(sourceFile.languageVersion, /*skipTrivia*/ true, sourceFile.text, /*onError:*/ undefined, pos); scanner.scan(); let start = scanner.getTokenPos(); return createTextSpanFromBounds(start, scanner.getTextPos());