maxmustermann/vscode
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Extract IModelLinesTokens

Browse source
This commit is contained in:
Alex Dima 2019-07-03 16:41:05 +02:00
parent 9f20ed4d6e
commit 8a5f705e96
4 changed files with 109 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
src/vs/editor/common/model/textModel.ts
View file

@ -23,7 +23,7 @@ import { IntervalNode, IntervalTree, getNodeIsInOverviewRuler, recomputeMaxEnd }
import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder';
import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, InternalModelContentChangeEvent, ModelRawChange, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/model/textModelEvents';
import { SearchData, SearchParams, TextModelSearch } from 'vs/editor/common/model/textModelSearch';
import { ModelLinesTokens, ModelTokensChangedEventBuilder } from 'vs/editor/common/model/textModelTokens';
import { ModelLinesTokens, ModelTokensChangedEventBuilder, IModelLinesTokens, safeTokenize } from 'vs/editor/common/model/textModelTokens';
import { getWordAtText } from 'vs/editor/common/model/wordHelper';
import { IState, LanguageId, LanguageIdentifier, TokenizationRegistry, FormattingOptions } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
@ -287,7 +287,7 @@ export class TextModel extends Disposable implements model.ITextModel {
private readonly _tokenizationListener: IDisposable;
private readonly _languageRegistryListener: IDisposable;
private _revalidateTokensTimeout: any;
/*private*/_tokens: ModelLinesTokens;
/*private*/_tokens: IModelLinesTokens;
//#endregion
constructor(source: string | model.ITextBufferFactory, creationOptions: model.ITextModelCreationOptions, languageIdentifier: LanguageIdentifier | null, associatedResource: URI | null = null) {
@ -1345,7 +1345,7 @@ export class TextModel extends Disposable implements model.ITextModel {
this._tokens.applyEdits(change.range, eolCount, firstLineLength);
} catch (err) {
// emergency recovery => reset tokens
this._tokens = new ModelLinesTokens(this._tokens.languageIdentifier, this._tokens.tokenizationSupport);
this._tokens = new ModelLinesTokens(this._languageIdentifier, this._tokens.tokenizationSupport);
}
this._onDidChangeDecorations.fire();
this._decorationsTree.acceptReplace(change.rangeOffset, change.rangeLength, change.text.length, change.forceMoveMarkers);
@ -1806,7 +1806,7 @@ export class TextModel extends Disposable implements model.ITextModel {
}
if (newNonWhitespaceIndex < nonWhitespaceColumn) {
initialState = this._tokens._getState(i - 1);
initialState = this._tokens.getState(i - 1);
if (initialState) {
break;
}
@ -1819,33 +1819,18 @@ export class TextModel extends Disposable implements model.ITextModel {
initialState = this._tokens.tokenizationSupport.getInitialState();
}
let state = initialState.clone();
let state = initialState;
for (let i = fakeLines.length - 1; i >= 0; i--) {
let r = this._tokens._tokenizeText(this._buffer, fakeLines[i], state);
let r = safeTokenize(this._languageIdentifier, this._tokens.tokenizationSupport, fakeLines[i], state);
if (r) {
state = r.endState.clone();
state = r.endState;
} else {
state = initialState.clone();
state = initialState;
}
}
const eventBuilder = new ModelTokensChangedEventBuilder();
for (let i = startLineNumber; i <= endLineNumber; i++) {
let text = this.getLineContent(i);
let r = this._tokens._tokenizeText(this._buffer, text, state);
if (r) {
this._tokens._setTokens(this._tokens.languageIdentifier.id, i - 1, text.length, r.tokens);
// We cannot trust these states/tokens to be valid!
// (see https://github.com/Microsoft/vscode/issues/67607)
this._tokens._invalidateLine(i - 1);
this._tokens._setState(i - 1, state);
state = r.endState.clone();
eventBuilder.registerChangedTokens(i);
} else {
state = initialState.clone();
}
}
this._tokens.fakeTokenizeLines(this._buffer, eventBuilder, state, startLineNumber, endLineNumber);
const e = eventBuilder.build();
if (e) {
@ -1871,7 +1856,7 @@ export class TextModel extends Disposable implements model.ITextModel {
const eventBuilder = new ModelTokensChangedEventBuilder();
this._tokens._updateTokensUntilLine(this._buffer, eventBuilder, lineNumber);
this._tokens.updateTokensUntilLine(this._buffer, eventBuilder, lineNumber);
const e = eventBuilder.build();
if (e) {
@ -1989,7 +1974,7 @@ export class TextModel extends Disposable implements model.ITextModel {
break;
}
const tokenizedLineNumber = this._tokens._tokenizeOneLine(this._buffer, eventBuilder);
const tokenizedLineNumber = this._tokens.tokenizeOneInvalidLine(this._buffer, eventBuilder);
if (tokenizedLineNumber >= toLineNumber) {
break;

133
src/vs/editor/common/model/textModelTokens.ts
View file

@ -388,7 +388,27 @@ class TokensStore {
//#endregion
}
export class ModelLinesTokens {
export interface IModelLinesTokens {
readonly tokenizationSupport: ITokenizationSupport | null;
readonly invalidLineStartIndex: number;
setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void;
getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens;
isCheapToTokenize(lineNumber: number): boolean;
hasLinesToTokenize(buffer: ITextBuffer): boolean;
getState(lineIndex: number): IState | null;
applyEdits(range: Range, eolCount: number, firstLineLength: number): void;
tokenizeOneInvalidLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number;
updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void;
fakeTokenizeLines(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, initialState: IState, startLineNumber: number, endLineNumber: number): void;
_getAllStates(linesLength: number): (IState | null)[];
_getAllInvalid(linesLength: number): number[];
}
export class ModelLinesTokens implements IModelLinesTokens {
public readonly languageIdentifier: LanguageIdentifier;
public readonly tokenizationSupport: ITokenizationSupport | null;
@ -427,19 +447,15 @@ export class ModelLinesTokens {
return (this.store.invalidLineStartIndex < buffer.getLineCount());
}
_isInvalid(lineIndex: number): boolean {
return this.store.isInvalid(lineIndex);
}
_getState(lineIndex: number): IState | null {
public getState(lineIndex: number): IState | null {
return this.store.getState(lineIndex);
}
_setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void {
public setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void {
this.store.setTokens(topLevelLanguageId, lineIndex, lineTextLength, tokens);
}
_setState(lineIndex: number, state: IState): void {
private _setState(lineIndex: number, state: IState): void {
this.store.setState(lineIndex, state);
}
@ -449,7 +465,7 @@ export class ModelLinesTokens {
this.store.applyEdits(range, eolCount, firstLineLength);
}
_invalidateLine(lineIndex: number): void {
private _invalidateLine(lineIndex: number): void {
this.store.invalidateLine(lineIndex);
}
@ -457,33 +473,16 @@ export class ModelLinesTokens {
//#region Tokenization
public _tokenizeOneLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number {
public tokenizeOneInvalidLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number {
if (!this.hasLinesToTokenize(buffer)) {
return buffer.getLineCount() + 1;
}
const lineNumber = this.store.invalidLineStartIndex + 1;
this._updateTokensUntilLine(buffer, eventBuilder, lineNumber);
this.updateTokensUntilLine(buffer, eventBuilder, lineNumber);
return lineNumber;
}
public _tokenizeText(buffer: ITextBuffer, text: string, state: IState): TokenizationResult2 {
let r: TokenizationResult2 | null = null;
if (this.tokenizationSupport) {
try {
r = this.tokenizationSupport.tokenize2(text, state, 0);
} catch (e) {
onUnexpectedError(e);
}
}
if (!r) {
r = nullTokenize2(this.languageIdentifier.id, text, state, 0);
}
return r;
}
public _updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
public updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
if (!this.tokenizationSupport) {
this.store._invalidLineStartIndex = buffer.getLineCount();
return;
@ -495,28 +494,76 @@ export class ModelLinesTokens {
// Validate all states up to and including endLineIndex
for (let lineIndex = this.store.invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) {
const text = buffer.getLineContent(lineIndex + 1);
const lineStartState = this._getState(lineIndex);
const lineStartState = this.getState(lineIndex);
let r: TokenizationResult2 | null = null;
try {
// Tokenize only the first X characters
let freshState = lineStartState!.clone();
r = this.tokenizationSupport.tokenize2(text, freshState, 0);
} catch (e) {
onUnexpectedError(e);
}
if (!r) {
r = nullTokenize2(this.languageIdentifier.id, text, lineStartState, 0);
}
const r = safeTokenize(this.languageIdentifier, this.tokenizationSupport, text, lineStartState!);
this.store.setGoodTokens(this.languageIdentifier.id, linesLength, lineIndex, text, r);
eventBuilder.registerChangedTokens(lineIndex + 1);
lineIndex = this.store.invalidLineStartIndex - 1; // -1 because the outer loop increments it
}
}
public fakeTokenizeLines(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, initialState: IState, startLineNumber: number, endLineNumber: number): void {
if (!this.tokenizationSupport) {
return;
}
let state = initialState;
for (let i = startLineNumber; i <= endLineNumber; i++) {
let text = buffer.getLineContent(i);
let r = safeTokenize(this.languageIdentifier, this.tokenizationSupport, text, state);
if (r) {
this.setTokens(this.languageIdentifier.id, i - 1, text.length, r.tokens);
// We cannot trust these states/tokens to be valid!
// (see https://github.com/Microsoft/vscode/issues/67607)
this._invalidateLine(i - 1);
this._setState(i - 1, state);
state = r.endState;
eventBuilder.registerChangedTokens(i);
} else {
state = initialState;
}
}
}
// #endregion
_getAllStates(linesLength: number): (IState | null)[] {
const r: (IState | null)[] = [];
for (let i = 0; i < linesLength; i++) {
r[i] = this.getState(i);
}
r[linesLength] = this.store._lastState;
return r;
}
_getAllInvalid(linesLength: number): number[] {
const r: number[] = [];
for (let i = 0; i < linesLength; i++) {
if (this.store.isInvalid(i)) {
r.push(i);
}
}
return r;
}
}
export function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null, text: string, state: IState): TokenizationResult2 {
let r: TokenizationResult2 | null = null;
if (tokenizationSupport) {
try {
r = tokenizationSupport.tokenize2(text, state.clone(), 0);
} catch (e) {
onUnexpectedError(e);
}
}
if (!r) {
r = nullTokenize2(languageIdentifier.id, text, state, 0);
}
return r;
}
export class ModelTokensChangedEventBuilder {

6
src/vs/editor/test/common/model/model.line.test.ts
View file

@ -110,7 +110,7 @@ suite('ModelLinesTokens', () => {
for (let lineIndex = 0; lineIndex < initial.length; lineIndex++) {
const lineTokens = initial[lineIndex].tokens;
const lineTextLength = model.getLineMaxColumn(lineIndex + 1) - 1;
model._tokens._setTokens(0, lineIndex, lineTextLength, TestToken.toTokens(lineTokens));
model._tokens.setTokens(0, lineIndex, lineTextLength, TestToken.toTokens(lineTokens));
}
model.applyEdits(edits.map((ed) => ({
@ -441,14 +441,14 @@ suite('ModelLinesTokens', () => {
test('insertion on empty line', () => {
const model = new TextModel('some text', TextModel.DEFAULT_CREATION_OPTIONS, new LanguageIdentifier('test', 0));
model._tokens._setTokens(0, 0, model.getLineMaxColumn(1) - 1, TestToken.toTokens([new TestToken(0, 1)]));
model._tokens.setTokens(0, 0, model.getLineMaxColumn(1) - 1, TestToken.toTokens([new TestToken(0, 1)]));
model.applyEdits([{
range: new Range(1, 1, 1, 10),
text: ''
}]);
model._tokens._setTokens(0, 0, model.getLineMaxColumn(1) - 1, new Uint32Array(0));
model._tokens.setTokens(0, 0, model.getLineMaxColumn(1) - 1, new Uint32Array(0));
model.applyEdits([{
range: new Range(1, 1, 1, 1),

17
src/vs/editor/test/common/model/model.modes.test.ts
View file

@ -180,25 +180,18 @@ suite('Editor Model - Model Modes 2', () => {
};
function invalidEqual(model: TextModel, expected: number[]): void {
let actual: number[] = [];
for (let i = 0, len = model.getLineCount(); i < len; i++) {
if (model._tokens._isInvalid(i)) {
actual.push(i);
}
}
assert.deepEqual(actual, expected);
assert.deepEqual(model._tokens._getAllInvalid(model.getLineCount()), expected);
}
function stateEqual(state: modes.IState, content: string): void {
assert.equal((<ModelState2>state).prevLineContent, content);
}
function statesEqual(model: TextModel, states: string[]): void {
let i, len = states.length - 1;
for (i = 0; i < len; i++) {
stateEqual(model._tokens._getState(i)!, states[i]);
function statesEqual(model: TextModel, expectedStates: string[]): void {
const actualStates = model._tokens._getAllStates(model.getLineCount());
for (let i = 0, len = expectedStates.length; i < len; i++) {
stateEqual(actualStates[i]!, expectedStates[i]);
}
stateEqual(model._tokens.store._lastState!, states[len]);
}
let thisModel: TextModel;