maxmustermann/vscode
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add test for microsoft/monaco-editor#1235

Browse source
This commit is contained in:
Alexandru Dima 2020-12-30 10:43:55 +01:00
parent 92d4b14e29
commit 4a121608e8
No known key found for this signature in database
GPG key ID: 6E58D7B045760DA0
2 changed files with 62 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/vs/editor/standalone/common/monarch/monarchLexer.ts
View file

@ -563,7 +563,7 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
}
let groupMatching: GroupMatching | null = null;
// See https://github.com/microsoft/monaco-editor/issues/1235:
// See https://github.com/microsoft/monaco-editor/issues/1235
// Evaluate rules at least once for an empty line
let forceEvaluation = true;

61
src/vs/editor/standalone/test/monarch/monarch.test.ts
View file

@ -103,4 +103,65 @@ suite('Monarch', () => {
innerModeRegistration.dispose();
});
test('microsoft/monaco-editor#1235: Empty Line Handling', () => {
const modeService = new ModeServiceImpl();
const tokenizer = createMonarchTokenizer(modeService, 'test', {
tokenizer: {
root: [
{ include: '@comments' },
],
comments: [
[/\/\/$/, 'comment'], // empty single-line comment
[/\/\//, 'comment', '@comment_cpp'],
],
comment_cpp: [
[/(?:[^\\]|(?:\\.))+$/, 'comment', '@pop'],
[/.+$/, 'comment'],
[/$/, 'comment', '@pop']
// No possible rule to detect an empty line and @pop?
],
},
});
const lines = [
`// This comment \\`,
` continues on the following line`,
``,
`// This comment does NOT continue \\\\`,
` because the escape char was itself escaped`,
``,
`// This comment DOES continue because \\\\\\`,
` the 1st '\\' escapes the 2nd; the 3rd escapes EOL`,
``,
`// This comment continues to the following line \\`,
``,
`But the line was empty. This line should not be commented.`,
];
const actualTokens: Token[][] = [];
let state = tokenizer.getInitialState();
for (const line of lines) {
const result = tokenizer.tokenize(line, state, 0);
actualTokens.push(result.tokens);
state = result.endState;
}
assert.deepEqual(actualTokens, [
[{ 'offset': 0, 'type': 'comment.test', 'language': 'test' }],
[{ 'offset': 0, 'type': 'comment.test', 'language': 'test' }],
[],
[{ 'offset': 0, 'type': 'comment.test', 'language': 'test' }],
[{ 'offset': 0, 'type': 'source.test', 'language': 'test' }],
[],
[{ 'offset': 0, 'type': 'comment.test', 'language': 'test' }],
[{ 'offset': 0, 'type': 'comment.test', 'language': 'test' }],
[],
[{ 'offset': 0, 'type': 'comment.test', 'language': 'test' }],
[],
[{ 'offset': 0, 'type': 'source.test', 'language': 'test' }]
]);
});
});