From d1c16cff65c6aed727787d27d67cff8ad8ceb209 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Wed, 3 Jul 2019 17:23:24 +0200 Subject: [PATCH] Reduce IModelLinesTokens shape --- src/vs/editor/common/model/textModel.ts | 60 +-------- src/vs/editor/common/model/textModelTokens.ts | 122 +++++++++++------- .../test/common/model/model.line.test.ts | 6 +- 3 files changed, 80 insertions(+), 108 deletions(-) diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index 5f404d807e8..4273d95645b 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -23,9 +23,9 @@ import { IntervalNode, IntervalTree, getNodeIsInOverviewRuler, recomputeMaxEnd } import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder'; import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, InternalModelContentChangeEvent, ModelRawChange, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/model/textModelEvents'; import { SearchData, SearchParams, TextModelSearch } from 'vs/editor/common/model/textModelSearch'; -import { ModelLinesTokens, ModelTokensChangedEventBuilder, IModelLinesTokens, safeTokenize } from 'vs/editor/common/model/textModelTokens'; +import { ModelLinesTokens, ModelTokensChangedEventBuilder, IModelLinesTokens } from 'vs/editor/common/model/textModelTokens'; import { getWordAtText } from 'vs/editor/common/model/wordHelper'; -import { IState, LanguageId, LanguageIdentifier, TokenizationRegistry, FormattingOptions } from 'vs/editor/common/modes'; +import { LanguageId, LanguageIdentifier, TokenizationRegistry, FormattingOptions } from 'vs/editor/common/modes'; import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; import { NULL_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/nullMode'; import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports'; @@ -1342,7 +1342,7 @@ export class TextModel extends Disposable implements model.ITextModel { const change = contentChanges[i]; const [eolCount, firstLineLength] = TextModel._eolCount(change.text); try { - this._tokens.applyEdits(change.range, eolCount, firstLineLength); + this._tokens.store.applyEdits(change.range, eolCount, firstLineLength); } catch (err) { // emergency recovery => reset tokens this._tokens = new ModelLinesTokens(this._languageIdentifier, this._tokens.tokenizationSupport); @@ -1776,61 +1776,11 @@ export class TextModel extends Disposable implements model.ITextModel { //#region Tokenization public tokenizeViewport(startLineNumber: number, endLineNumber: number): void { - if (!this._tokens.tokenizationSupport) { - // nothing to do - return; - } - startLineNumber = Math.max(1, startLineNumber); endLineNumber = Math.min(this.getLineCount(), endLineNumber); - if (endLineNumber <= this._tokens.invalidLineStartIndex) { - // nothing to do - return; - } - - if (startLineNumber <= this._tokens.invalidLineStartIndex) { - // tokenization has reached the viewport start... - this.forceTokenization(endLineNumber); - return; - } - - let nonWhitespaceColumn = this.getLineFirstNonWhitespaceColumn(startLineNumber); - let fakeLines: string[] = []; - let initialState: IState | null = null; - for (let i = startLineNumber - 1; nonWhitespaceColumn > 0 && i >= 1; i--) { - let newNonWhitespaceIndex = this.getLineFirstNonWhitespaceColumn(i); - - if (newNonWhitespaceIndex === 0) { - continue; - } - - if (newNonWhitespaceIndex < nonWhitespaceColumn) { - initialState = this._tokens.getState(i - 1); - if (initialState) { - break; - } - fakeLines.push(this.getLineContent(i)); - nonWhitespaceColumn = newNonWhitespaceIndex; - } - } - - if (!initialState) { - initialState = this._tokens.tokenizationSupport.getInitialState(); - } - - let state = initialState; - for (let i = fakeLines.length - 1; i >= 0; i--) { - let r = safeTokenize(this._languageIdentifier, this._tokens.tokenizationSupport, fakeLines[i], state); - if (r) { - state = r.endState; - } else { - state = initialState; - } - } - const eventBuilder = new ModelTokensChangedEventBuilder(); - this._tokens.fakeTokenizeLines(this._buffer, eventBuilder, state, startLineNumber, endLineNumber); + this._tokens.tokenizeViewport(this._buffer, eventBuilder, startLineNumber, endLineNumber); const e = eventBuilder.build(); if (e) { @@ -1896,7 +1846,7 @@ export class TextModel extends Disposable implements model.ITextModel { private _getLineTokens(lineNumber: number): LineTokens { const lineText = this._buffer.getLineContent(lineNumber); - return this._tokens.getTokens(this._languageIdentifier.id, lineNumber - 1, lineText); + return this._tokens.store.getTokens(this._languageIdentifier.id, lineNumber - 1, lineText); } public getLanguageIdentifier(): LanguageIdentifier { diff --git a/src/vs/editor/common/model/textModelTokens.ts b/src/vs/editor/common/model/textModelTokens.ts index 35fac7434f5..eaa45a33d4b 100644 --- a/src/vs/editor/common/model/textModelTokens.ts +++ b/src/vs/editor/common/model/textModelTokens.ts @@ -390,19 +390,15 @@ class TokensStore { export interface IModelLinesTokens { readonly tokenizationSupport: ITokenizationSupport | null; + readonly store: TokensStore; readonly invalidLineStartIndex: number; - setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void; - getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens; - isCheapToTokenize(lineNumber: number): boolean; hasLinesToTokenize(buffer: ITextBuffer): boolean; - getState(lineIndex: number): IState | null; - applyEdits(range: Range, eolCount: number, firstLineLength: number): void; tokenizeOneInvalidLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number; updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void; - fakeTokenizeLines(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, initialState: IState, startLineNumber: number, endLineNumber: number): void; + tokenizeViewport(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, startLineNumber: number, endLineNumber: number): void; _getAllStates(linesLength: number): (IState | null)[]; _getAllInvalid(linesLength: number): number[]; @@ -410,12 +406,16 @@ export interface IModelLinesTokens { export class ModelLinesTokens implements IModelLinesTokens { - public readonly languageIdentifier: LanguageIdentifier; + private readonly _languageIdentifier: LanguageIdentifier; public readonly tokenizationSupport: ITokenizationSupport | null; public readonly store: TokensStore; + public get invalidLineStartIndex() { + return this.store.invalidLineStartIndex; + } + constructor(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null) { - this.languageIdentifier = languageIdentifier; + this._languageIdentifier = languageIdentifier; this.tokenizationSupport = tokenizationSupport; let initialState: IState | null = null; @@ -430,14 +430,6 @@ export class ModelLinesTokens implements IModelLinesTokens { this.store = new TokensStore(initialState); } - public get invalidLineStartIndex() { - return this.store.invalidLineStartIndex; - } - - public getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens { - return this.store.getTokens(topLevelLanguageId, lineIndex, lineText); - } - public isCheapToTokenize(lineNumber: number): boolean { const firstInvalidLineNumber = this.store.invalidLineStartIndex + 1; return (firstInvalidLineNumber >= lineNumber); @@ -447,30 +439,6 @@ export class ModelLinesTokens implements IModelLinesTokens { return (this.store.invalidLineStartIndex < buffer.getLineCount()); } - public getState(lineIndex: number): IState | null { - return this.store.getState(lineIndex); - } - - public setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void { - this.store.setTokens(topLevelLanguageId, lineIndex, lineTextLength, tokens); - } - - private _setState(lineIndex: number, state: IState): void { - this.store.setState(lineIndex, state); - } - - //#region Editing - - public applyEdits(range: Range, eolCount: number, firstLineLength: number): void { - this.store.applyEdits(range, eolCount, firstLineLength); - } - - private _invalidateLine(lineIndex: number): void { - this.store.invalidateLine(lineIndex); - } - - //#endregion - //#region Tokenization public tokenizeOneInvalidLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number { @@ -494,16 +462,70 @@ export class ModelLinesTokens implements IModelLinesTokens { // Validate all states up to and including endLineIndex for (let lineIndex = this.store.invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) { const text = buffer.getLineContent(lineIndex + 1); - const lineStartState = this.getState(lineIndex); + const lineStartState = this.store.getState(lineIndex); - const r = safeTokenize(this.languageIdentifier, this.tokenizationSupport, text, lineStartState!); - this.store.setGoodTokens(this.languageIdentifier.id, linesLength, lineIndex, text, r); + const r = safeTokenize(this._languageIdentifier, this.tokenizationSupport, text, lineStartState!); + this.store.setGoodTokens(this._languageIdentifier.id, linesLength, lineIndex, text, r); eventBuilder.registerChangedTokens(lineIndex + 1); lineIndex = this.store.invalidLineStartIndex - 1; // -1 because the outer loop increments it } } - public fakeTokenizeLines(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, initialState: IState, startLineNumber: number, endLineNumber: number): void { + public tokenizeViewport(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, startLineNumber: number, endLineNumber: number): void { + if (!this.tokenizationSupport) { + // nothing to do + return; + } + + if (endLineNumber <= this.invalidLineStartIndex) { + // nothing to do + return; + } + + if (startLineNumber <= this.invalidLineStartIndex) { + // tokenization has reached the viewport start... + this.updateTokensUntilLine(buffer, eventBuilder, endLineNumber); + return; + } + + let nonWhitespaceColumn = buffer.getLineFirstNonWhitespaceColumn(startLineNumber); + let fakeLines: string[] = []; + let initialState: IState | null = null; + for (let i = startLineNumber - 1; nonWhitespaceColumn > 0 && i >= 1; i--) { + let newNonWhitespaceIndex = buffer.getLineFirstNonWhitespaceColumn(i); + + if (newNonWhitespaceIndex === 0) { + continue; + } + + if (newNonWhitespaceIndex < nonWhitespaceColumn) { + initialState = this.store.getState(i - 1); + if (initialState) { + break; + } + fakeLines.push(buffer.getLineContent(i)); + nonWhitespaceColumn = newNonWhitespaceIndex; + } + } + + if (!initialState) { + initialState = this.tokenizationSupport.getInitialState(); + } + + let state = initialState; + for (let i = fakeLines.length - 1; i >= 0; i--) { + let r = safeTokenize(this._languageIdentifier, this.tokenizationSupport, fakeLines[i], state); + if (r) { + state = r.endState; + } else { + state = initialState; + } + } + + this._fakeTokenizeLines(buffer, eventBuilder, state, startLineNumber, endLineNumber); + } + + private _fakeTokenizeLines(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, initialState: IState, startLineNumber: number, endLineNumber: number): void { if (!this.tokenizationSupport) { return; } @@ -511,14 +533,14 @@ export class ModelLinesTokens implements IModelLinesTokens { let state = initialState; for (let i = startLineNumber; i <= endLineNumber; i++) { let text = buffer.getLineContent(i); - let r = safeTokenize(this.languageIdentifier, this.tokenizationSupport, text, state); + let r = safeTokenize(this._languageIdentifier, this.tokenizationSupport, text, state); if (r) { - this.setTokens(this.languageIdentifier.id, i - 1, text.length, r.tokens); + this.store.setTokens(this._languageIdentifier.id, i - 1, text.length, r.tokens); // We cannot trust these states/tokens to be valid! // (see https://github.com/Microsoft/vscode/issues/67607) - this._invalidateLine(i - 1); - this._setState(i - 1, state); + this.store.invalidateLine(i - 1); + this.store.setState(i - 1, state); state = r.endState; eventBuilder.registerChangedTokens(i); } else { @@ -532,7 +554,7 @@ export class ModelLinesTokens implements IModelLinesTokens { _getAllStates(linesLength: number): (IState | null)[] { const r: (IState | null)[] = []; for (let i = 0; i < linesLength; i++) { - r[i] = this.getState(i); + r[i] = this.store.getState(i); } r[linesLength] = this.store._lastState; return r; @@ -549,7 +571,7 @@ export class ModelLinesTokens implements IModelLinesTokens { } } -export function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null, text: string, state: IState): TokenizationResult2 { +function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null, text: string, state: IState): TokenizationResult2 { let r: TokenizationResult2 | null = null; if (tokenizationSupport) { diff --git a/src/vs/editor/test/common/model/model.line.test.ts b/src/vs/editor/test/common/model/model.line.test.ts index 3c00d5a72b3..4c65dc30d48 100644 --- a/src/vs/editor/test/common/model/model.line.test.ts +++ b/src/vs/editor/test/common/model/model.line.test.ts @@ -110,7 +110,7 @@ suite('ModelLinesTokens', () => { for (let lineIndex = 0; lineIndex < initial.length; lineIndex++) { const lineTokens = initial[lineIndex].tokens; const lineTextLength = model.getLineMaxColumn(lineIndex + 1) - 1; - model._tokens.setTokens(0, lineIndex, lineTextLength, TestToken.toTokens(lineTokens)); + model._tokens.store.setTokens(0, lineIndex, lineTextLength, TestToken.toTokens(lineTokens)); } model.applyEdits(edits.map((ed) => ({ @@ -441,14 +441,14 @@ suite('ModelLinesTokens', () => { test('insertion on empty line', () => { const model = new TextModel('some text', TextModel.DEFAULT_CREATION_OPTIONS, new LanguageIdentifier('test', 0)); - model._tokens.setTokens(0, 0, model.getLineMaxColumn(1) - 1, TestToken.toTokens([new TestToken(0, 1)])); + model._tokens.store.setTokens(0, 0, model.getLineMaxColumn(1) - 1, TestToken.toTokens([new TestToken(0, 1)])); model.applyEdits([{ range: new Range(1, 1, 1, 10), text: '' }]); - model._tokens.setTokens(0, 0, model.getLineMaxColumn(1) - 1, new Uint32Array(0)); + model._tokens.store.setTokens(0, 0, model.getLineMaxColumn(1) - 1, new Uint32Array(0)); model.applyEdits([{ range: new Range(1, 1, 1, 1),