maxmustermann/vscode
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Fixes #117264: Abandon tokenizing a line after 500ms if possible.

Browse source
This commit is contained in:
Alex Dima 2021-11-19 14:49:29 +01:00
parent 4970694100
commit a3f6aadf28
No known key found for this signature in database
GPG key ID: 39563C1504FDD0C9
8 changed files with 32 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -83,7 +83,7 @@
"vscode-proxy-agent": "^0.11.0",
"vscode-regexpp": "^3.1.0",
"vscode-ripgrep": "^1.12.1",
"vscode-textmate": "5.4.1",
"vscode-textmate": "5.5.0",
"xterm": "4.16.0-beta.2",
"xterm-addon-search": "0.9.0-beta.6",
"xterm-addon-serialize": "0.7.0-beta.3",

2
remote/package.json
View file

@ -23,7 +23,7 @@
"vscode-proxy-agent": "^0.11.0",
"vscode-regexpp": "^3.1.0",
"vscode-ripgrep": "^1.12.1",
"vscode-textmate": "5.4.1",
"vscode-textmate": "5.5.0",
"xterm": "4.16.0-beta.2",
"xterm-addon-search": "0.9.0-beta.6",
"xterm-addon-serialize": "0.7.0-beta.3",

2
remote/web/package.json
View file

@ -9,7 +9,7 @@
"jschardet": "3.0.0",
"tas-client-umd": "0.1.4",
"vscode-oniguruma": "1.6.1",
"vscode-textmate": "5.4.1",
"vscode-textmate": "5.5.0",
"xterm": "4.16.0-beta.2",
"xterm-addon-search": "0.9.0-beta.6",
"xterm-addon-unicode11": "0.4.0-beta.1",

8
remote/web/yarn.lock
View file

@ -108,10 +108,10 @@ vscode-oniguruma@1.6.1:
resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.6.1.tgz#2bf4dfcfe3dd2e56eb549a3068c8ee39e6c30ce5"
integrity sha512-vc4WhSIaVpgJ0jJIejjYxPvURJavX6QG41vu0mGhqywMkQqulezEqEQ3cO3gc8GvcOpX6ycmKGqRoROEMBNXTQ==
vscode-textmate@5.4.1:
version "5.4.1"
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.4.1.tgz#09d566724fc76b60b3ad9791eebf1f0b50f29e5a"
integrity sha512-4CvPHmfuZQaXrcCpathdh6jo7myuR+MU8BvscgQADuponpbqfmu2rwTOtCXhGwwEgStvJF8V4s9FwMKRVLNmKQ==
vscode-textmate@5.5.0:
version "5.5.0"
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.5.0.tgz#d83776562c07d1e3181c2c7f1b3d5f20afcab483"
integrity sha512-jToQkPGMNKn0eyKyitYeINJF0NoD240aYyKPIWJv5W2jfPt++jIRg0OSergubtGhbw6SoefkvBYEpX7TsfoSUQ==
xterm-addon-search@0.9.0-beta.6:
version "0.9.0-beta.6"

8
remote/yarn.lock
View file

@ -512,10 +512,10 @@ vscode-ripgrep@^1.12.1:
https-proxy-agent "^4.0.0"
proxy-from-env "^1.1.0"
vscode-textmate@5.4.1:
version "5.4.1"
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.4.1.tgz#09d566724fc76b60b3ad9791eebf1f0b50f29e5a"
integrity sha512-4CvPHmfuZQaXrcCpathdh6jo7myuR+MU8BvscgQADuponpbqfmu2rwTOtCXhGwwEgStvJF8V4s9FwMKRVLNmKQ==
vscode-textmate@5.5.0:
version "5.5.0"
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.5.0.tgz#d83776562c07d1e3181c2c7f1b3d5f20afcab483"
integrity sha512-jToQkPGMNKn0eyKyitYeINJF0NoD240aYyKPIWJv5W2jfPt++jIRg0OSergubtGhbw6SoefkvBYEpX7TsfoSUQ==
vscode-windows-ca-certs@^0.3.0:
version "0.3.0"

8
src/vs/workbench/services/textMate/browser/abstractTextMateService.ts
View file

@ -484,7 +484,13 @@ class TMTokenization extends Disposable {
}
public tokenize2(line: string, state: StackElement): TokenizationResult2 {
let textMateResult = this._grammar.tokenizeLine2(line, state);
const textMateResult = this._grammar.tokenizeLine2(line, state, 500);
if (textMateResult.stoppedEarly) {
console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`);
// return the state at the beginning of the line
return new TokenizationResult2(textMateResult.tokens, state);
}
if (this._containsEmbeddedLanguages) {
let seenLanguages = this._seenLanguages;

12
src/vs/workbench/services/textMate/common/textMateService.ts
View file

@ -33,7 +33,7 @@ export interface IGrammar {
/**
* Tokenize `lineText` using previous line state `prevState`.
*/
tokenizeLine(lineText: string, prevState: StackElement | null): ITokenizeLineResult;
tokenizeLine(lineText: string, prevState: StackElement | null, timeLimit?: number): ITokenizeLineResult;
/**
* Tokenize `lineText` using previous line state `prevState`.
* The result contains the tokens in binary format, resolved with the following information:
@ -44,7 +44,7 @@ export interface IGrammar {
* - background color
* e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET`
*/
tokenizeLine2(lineText: string, prevState: StackElement | null): ITokenizeLineResult2;
tokenizeLine2(lineText: string, prevState: StackElement | null, timeLimit?: number): ITokenizeLineResult2;
}
export interface ITokenizeLineResult {
readonly tokens: IToken[];
@ -52,6 +52,10 @@ export interface ITokenizeLineResult {
* The `prevState` to be passed on to the next line tokenization.
*/
readonly ruleStack: StackElement;
/**
* Did tokenization stop early due to reaching the time limit.
*/
readonly stoppedEarly: boolean;
}
/**
* Helpers to manage the "collapsed" metadata of an entire StackElement stack.
@ -97,6 +101,10 @@ export interface ITokenizeLineResult2 {
* The `prevState` to be passed on to the next line tokenization.
*/
readonly ruleStack: StackElement;
/**
* Did tokenization stop early due to reaching the time limit.
*/
readonly stoppedEarly: boolean;
}
export interface IToken {
startIndex: number;

8
yarn.lock
View file

@ -10476,10 +10476,10 @@ vscode-telemetry-extractor@^1.9.5:
ts-morph "^12.2.0"
vscode-ripgrep "^1.12.1"
vscode-textmate@5.4.1:
version "5.4.1"
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.4.1.tgz#09d566724fc76b60b3ad9791eebf1f0b50f29e5a"
integrity sha512-4CvPHmfuZQaXrcCpathdh6jo7myuR+MU8BvscgQADuponpbqfmu2rwTOtCXhGwwEgStvJF8V4s9FwMKRVLNmKQ==
vscode-textmate@5.5.0:
version "5.5.0"
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.5.0.tgz#d83776562c07d1e3181c2c7f1b3d5f20afcab483"
integrity sha512-jToQkPGMNKn0eyKyitYeINJF0NoD240aYyKPIWJv5W2jfPt++jIRg0OSergubtGhbw6SoefkvBYEpX7TsfoSUQ==
vscode-windows-ca-certs@^0.3.0:
version "0.3.0"