maxmustermann/TypeScript
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Improve JSON parser error recovery (#42657)

Browse source 
* Improve JSON parser error recovery

* Add error baselines

* Move tsconfig root checking out of common JSON checking function

* Use new function in parseConfigFileTextToJson

* Fix test

* Replace non-null assertion with explicit debug assertion
This commit is contained in:
Andrew Branch 2021-02-23 09:31:09 -08:00 committed by GitHub
parent 9de8dbbfb8
commit 2a01f923ca
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 286 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
src/compiler/commandLineParser.ts
View file

@ -1603,7 +1603,7 @@ namespace ts {
export function parseConfigFileTextToJson(fileName: string, jsonText: string): { config?: any; error?: Diagnostic } {
const jsonSourceFile = parseJsonText(fileName, jsonText);
return {
config: convertToObject(jsonSourceFile, jsonSourceFile.parseDiagnostics),
config: convertConfigFileToObject(jsonSourceFile, jsonSourceFile.parseDiagnostics, /*reportOptionsErrors*/ false, /*optionsIterator*/ undefined),
error: jsonSourceFile.parseDiagnostics.length ? jsonSourceFile.parseDiagnostics[0] : undefined
};
}
@ -1767,11 +1767,35 @@ namespace ts {
onSetUnknownOptionKeyValueInRoot(key: string, keyNode: PropertyName, value: CompilerOptionsValue, valueNode: Expression): void;
}
function convertConfigFileToObject(sourceFile: JsonSourceFile, errors: Push<Diagnostic>, reportOptionsErrors: boolean, optionsIterator: JsonConversionNotifier | undefined): any {
const rootExpression: Expression | undefined = sourceFile.statements[0]?.expression;
const knownRootOptions = reportOptionsErrors ? getTsconfigRootOptionsMap() : undefined;
if (rootExpression && rootExpression.kind !== SyntaxKind.ObjectLiteralExpression) {
errors.push(createDiagnosticForNodeInSourceFile(
sourceFile,
rootExpression,
Diagnostics.The_root_value_of_a_0_file_must_be_an_object,
getBaseFileName(sourceFile.fileName) === "jsconfig.json" ? "jsconfig.json" : "tsconfig.json"
));
// Last-ditch error recovery. Somewhat useful because the JSON parser will recover from some parse errors by
// synthesizing a top-level array literal expression. There's a reasonable chance the first element of that
// array is a well-formed configuration object, made into an array element by stray characters.
if (isArrayLiteralExpression(rootExpression)) {
const firstObject = find(rootExpression.elements, isObjectLiteralExpression);
if (firstObject) {
return convertToObjectWorker(sourceFile, firstObject, errors, /*returnValue*/ true, knownRootOptions, optionsIterator);
}
}
return {};
}
return convertToObjectWorker(sourceFile, rootExpression, errors, /*returnValue*/ true, knownRootOptions, optionsIterator);
}
/**
* Convert the json syntax tree into the json value
*/
export function convertToObject(sourceFile: JsonSourceFile, errors: Push<Diagnostic>): any {
return convertToObjectWorker(sourceFile, errors, /*returnValue*/ true, /*knownRootOptions*/ undefined, /*jsonConversionNotifier*/ undefined);
return convertToObjectWorker(sourceFile, sourceFile.statements[0]?.expression, errors, /*returnValue*/ true, /*knownRootOptions*/ undefined, /*jsonConversionNotifier*/ undefined);
}
/**
@ -1782,15 +1806,16 @@ namespace ts {
/*@internal*/
export function convertToObjectWorker(
sourceFile: JsonSourceFile,
rootExpression: Expression | undefined,
errors: Push<Diagnostic>,
returnValue: boolean,
knownRootOptions: CommandLineOption | undefined,
jsonConversionNotifier: JsonConversionNotifier | undefined): any {
if (!sourceFile.statements.length) {
if (!rootExpression) {
return returnValue ? {} : undefined;
}
return convertPropertyValueToJson(sourceFile.statements[0].expression, knownRootOptions);
return convertPropertyValueToJson(rootExpression, knownRootOptions);
function isRootOptionMap(knownOptions: ESMap<string, CommandLineOption> | undefined) {
return knownRootOptions && (knownRootOptions as TsConfigOnlyOption).elementOptions === knownOptions;
@ -2733,7 +2758,7 @@ namespace ts {
}
}
};
const json = convertToObjectWorker(sourceFile, errors, /*returnValue*/ true, getTsconfigRootOptionsMap(), optionsIterator);
const json = convertConfigFileToObject(sourceFile, errors, /*reportOptionsErrors*/ true, optionsIterator);
if (!typeAcquisition) {
if (typingOptionstypeAcquisition) {

4
src/compiler/diagnosticMessages.json
View file

@ -3850,6 +3850,10 @@
"category": "Error",
"code": 5091
},
"The root value of a '{0}' file must be an object.": {
"category": "Error",
"code": 5092
},
"Generates a sourcemap for each corresponding '.d.ts' file.": {
"category": "Message",

74
src/compiler/parser.ts
View file

@ -819,7 +819,7 @@ namespace ts {
scriptKind = ensureScriptKind(fileName, scriptKind);
if (scriptKind === ScriptKind.JSON) {
const result = parseJsonText(fileName, sourceText, languageVersion, syntaxCursor, setParentNodes);
convertToObjectWorker(result, result.parseDiagnostics, /*returnValue*/ false, /*knownRootOptions*/ undefined, /*jsonConversionNotifier*/ undefined);
convertToObjectWorker(result, result.statements[0]?.expression, result.parseDiagnostics, /*returnValue*/ false, /*knownRootOptions*/ undefined, /*jsonConversionNotifier*/ undefined);
result.referencedFiles = emptyArray;
result.typeReferenceDirectives = emptyArray;
result.libReferenceDirectives = emptyArray;
@ -862,36 +862,56 @@ namespace ts {
endOfFileToken = parseTokenNode<EndOfFileToken>();
}
else {
let expression;
switch (token()) {
case SyntaxKind.OpenBracketToken:
expression = parseArrayLiteralExpression();
break;
case SyntaxKind.TrueKeyword:
case SyntaxKind.FalseKeyword:
case SyntaxKind.NullKeyword:
expression = parseTokenNode<BooleanLiteral | NullLiteral>();
break;
case SyntaxKind.MinusToken:
if (lookAhead(() => nextToken() === SyntaxKind.NumericLiteral && nextToken() !== SyntaxKind.ColonToken)) {
expression = parsePrefixUnaryExpression() as JsonMinusNumericLiteral;
}
else {
expression = parseObjectLiteralExpression();
}
break;
case SyntaxKind.NumericLiteral:
case SyntaxKind.StringLiteral:
if (lookAhead(() => nextToken() !== SyntaxKind.ColonToken)) {
expression = parseLiteralNode() as StringLiteral | NumericLiteral;
// Loop and synthesize an ArrayLiteralExpression if there are more than
// one top-level expressions to ensure all input text is consumed.
let expressions: Expression[] | Expression | undefined;
while (token() !== SyntaxKind.EndOfFileToken) {
let expression;
switch (token()) {
case SyntaxKind.OpenBracketToken:
expression = parseArrayLiteralExpression();
break;
case SyntaxKind.TrueKeyword:
case SyntaxKind.FalseKeyword:
case SyntaxKind.NullKeyword:
expression = parseTokenNode<BooleanLiteral | NullLiteral>();
break;
case SyntaxKind.MinusToken:
if (lookAhead(() => nextToken() === SyntaxKind.NumericLiteral && nextToken() !== SyntaxKind.ColonToken)) {
expression = parsePrefixUnaryExpression() as JsonMinusNumericLiteral;
}
else {
expression = parseObjectLiteralExpression();
}
break;
case SyntaxKind.NumericLiteral:
case SyntaxKind.StringLiteral:
if (lookAhead(() => nextToken() !== SyntaxKind.ColonToken)) {
expression = parseLiteralNode() as StringLiteral | NumericLiteral;
break;
}
// falls through
default:
expression = parseObjectLiteralExpression();
break;
}
// Error recovery: collect multiple top-level expressions
if (expressions && isArray(expressions)) {
expressions.push(expression);
}
else if (expressions) {
expressions = [expressions, expression];
}
else {
expressions = expression;
if (token() !== SyntaxKind.EndOfFileToken) {
parseErrorAtCurrentToken(Diagnostics.Unexpected_token);
}
// falls through
default:
expression = parseObjectLiteralExpression();
break;
}
}
const expression = isArray(expressions) ? finishNode(factory.createArrayLiteralExpression(expressions), pos) : Debug.checkDefined(expressions);
const statement = factory.createExpressionStatement(expression) as JsonObjectExpressionStatement;
finishNode(statement, pos);
statements = createNodeArray([statement], pos);

1
src/testRunner/tsconfig.json
View file

@ -66,6 +66,7 @@
"unittests/factory.ts",
"unittests/incrementalParser.ts",
"unittests/jsDocParsing.ts",
"unittests/jsonParserRecovery.ts",
"unittests/moduleResolution.ts",
"unittests/parsePseudoBigInt.ts",
"unittests/paths.ts",

81
src/testRunner/unittests/config/convertCompilerOptionsFromJson.ts
View file

@ -69,7 +69,7 @@ namespace ts {
assert.equal(actualError.category, expectedError.category, `Expected error-category: ${JSON.stringify(expectedError.category)}. Actual error-category: ${JSON.stringify(actualError.category)}.`);
if (!ignoreLocation) {
assert(actualError.file);
assert(actualError.start);
assert.isDefined(actualError.start);
assert(actualError.length);
}
}
@ -603,5 +603,84 @@ namespace ts {
hasParseErrors: true
});
});
it("Convert a tsconfig file with stray trailing characters", () => {
assertCompilerOptionsWithJsonText(`{
"compilerOptions": {
"target": "esnext"
}
} blah`, "tsconfig.json", {
compilerOptions: {
target: ScriptTarget.ESNext
},
hasParseErrors: true,
errors: [{
...Diagnostics.The_root_value_of_a_0_file_must_be_an_object,
messageText: "The root value of a 'tsconfig.json' file must be an object.",
file: undefined,
start: 0,
length: 0
}]
});
});
it("Convert a tsconfig file with stray leading characters", () => {
assertCompilerOptionsWithJsonText(`blah {
"compilerOptions": {
"target": "esnext"
}
}`, "tsconfig.json", {
compilerOptions: {
target: ScriptTarget.ESNext
},
hasParseErrors: true,
errors: [{
...Diagnostics.The_root_value_of_a_0_file_must_be_an_object,
messageText: "The root value of a 'tsconfig.json' file must be an object.",
file: undefined,
start: 0,
length: 0
}]
});
});
it("Convert a tsconfig file as an array", () => {
assertCompilerOptionsWithJsonText(`[{
"compilerOptions": {
"target": "esnext"
}
}]`, "tsconfig.json", {
compilerOptions: {
target: ScriptTarget.ESNext
},
errors: [{
...Diagnostics.The_root_value_of_a_0_file_must_be_an_object,
messageText: "The root value of a 'tsconfig.json' file must be an object.",
file: undefined,
start: 0,
length: 0
}]
});
});
it("Don't crash when root expression is not objecty at all", () => {
assertCompilerOptionsWithJsonText(`42`, "tsconfig.json", {
compilerOptions: {},
errors: [{
...Diagnostics.The_root_value_of_a_0_file_must_be_an_object,
messageText: "The root value of a 'tsconfig.json' file must be an object.",
file: undefined,
start: 0,
length: 0
}]
});
});
it("Allow trailing comments", () => {
assertCompilerOptionsWithJsonText(`{} // no options`, "tsconfig.json", {
compilerOptions: {},
errors: []
});
});
});
}

39
src/testRunner/unittests/jsonParserRecovery.ts Normal file
View file

@ -0,0 +1,39 @@
namespace ts {
describe("unittests:: jsonParserRecovery", () => {
function parsesToValidSourceFileWithErrors(name: string, text: string) {
it(name, () => {
const file = parseJsonText(name, text);
assert(file.parseDiagnostics.length, "Should have parse errors");
Harness.Baseline.runBaseline(
`jsonParserRecovery/${name.replace(/[^a-z0-9_-]/ig, "_")}.errors.txt`,
Harness.Compiler.getErrorBaseline([{
content: text,
unitName: name
}], file.parseDiagnostics));
// Will throw if parse tree does not cover full input text
file.getChildren();
});
}
parsesToValidSourceFileWithErrors("trailing identifier", "{} blah");
parsesToValidSourceFileWithErrors("TypeScript code", "interface Foo {} blah");
parsesToValidSourceFileWithErrors("Two comma-separated objects", "{}, {}");
parsesToValidSourceFileWithErrors("Two objects", "{} {}");
parsesToValidSourceFileWithErrors("JSX", `
interface Test {}
const Header = () => (
<div>
<h1>Header</h1>
<style jsx>
{\`
h1 {
color: red;
}
\`}
</style>
</div>
)`);
});
}

37
tests/baselines/reference/jsonParserRecovery/JSX.errors.txt Normal file
View file

@ -0,0 +1,37 @@
JSX(2,9): error TS1005: '{' expected.
JSX(2,19): error TS1005: ',' expected.
JSX(2,24): error TS1005: ',' expected.
JSX(4,9): error TS1012: Unexpected token.
JSX(4,15): error TS1005: ':' expected.
JSX(15,10): error TS1005: '}' expected.
==== JSX (6 errors) ====
interface Test {}
~~~~~~~~~
!!! error TS1005: '{' expected.
~~~~
!!! error TS1005: ',' expected.
~
!!! error TS1005: ',' expected.
const Header = () => (
~~~~~
!!! error TS1012: Unexpected token.
~~~~~~
!!! error TS1005: ':' expected.
<div>
<h1>Header</h1>
<style jsx>
{`
h1 {
color: red;
}
`}
</style>
</div>
)
!!! error TS1005: '}' expected.
!!! related TS1007 JSX:4:9: The parser expected to find a '}' to match the '{' token here.

10
tests/baselines/reference/jsonParserRecovery/Two_comma-separated_objects.errors.txt Normal file
View file

@ -0,0 +1,10 @@
Two comma-separated objects(1,3): error TS1012: Unexpected token.
Two comma-separated objects(1,5): error TS1136: Property assignment expected.
==== Two comma-separated objects (2 errors) ====
{}, {}
~
!!! error TS1012: Unexpected token.
~
!!! error TS1136: Property assignment expected.

7
tests/baselines/reference/jsonParserRecovery/Two_objects.errors.txt Normal file
View file

@ -0,0 +1,7 @@
Two objects(1,4): error TS1012: Unexpected token.
==== Two objects (1 errors) ====
{} {}
~
!!! error TS1012: Unexpected token.

20
tests/baselines/reference/jsonParserRecovery/TypeScript_code.errors.txt Normal file
View file

@ -0,0 +1,20 @@
TypeScript code(1,1): error TS1005: '{' expected.
TypeScript code(1,11): error TS1005: ',' expected.
TypeScript code(1,15): error TS1005: ',' expected.
TypeScript code(1,18): error TS1012: Unexpected token.
TypeScript code(1,22): error TS1005: '}' expected.
==== TypeScript code (5 errors) ====
interface Foo {} blah
~~~~~~~~~
!!! error TS1005: '{' expected.
~~~
!!! error TS1005: ',' expected.
~
!!! error TS1005: ',' expected.
~~~~
!!! error TS1012: Unexpected token.
!!! error TS1005: '}' expected.
!!! related TS1007 TypeScript code:1:18: The parser expected to find a '}' to match the '{' token here.

11
tests/baselines/reference/jsonParserRecovery/trailing_identifier.errors.txt Normal file
View file

@ -0,0 +1,11 @@
trailing identifier(1,4): error TS1012: Unexpected token.
trailing identifier(1,8): error TS1005: '}' expected.
==== trailing identifier (2 errors) ====
{} blah
~~~~
!!! error TS1012: Unexpected token.
!!! error TS1005: '}' expected.
!!! related TS1007 trailing identifier:1:4: The parser expected to find a '}' to match the '{' token here.