From 9e1d730cf2950297d7a85fc66be89411d7b8bcd9 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Thu, 16 Jan 2020 17:53:24 +0100 Subject: [PATCH] SemanticTokens - implement feedback received in API call: - extract a separate DocumentRangeSemanticTokensProvider that deals with a document range - extract a separate provideDocumentSemanticTokensEdits that deals with updating via SemanticTokensEdits a previous result --- .../client/src/htmlMain.ts | 18 ++- .../src/features/semanticTokens.ts | 106 +++++++++--------- .../src/colorizerTestMain.ts | 6 +- src/vs/editor/common/modes.ts | 18 ++- .../common/services/modelServiceImpl.ts | 26 ++--- src/vs/monaco.d.ts | 11 +- src/vs/vscode.proposed.d.ts | 100 ++++++++--------- .../api/browser/mainThreadLanguageFeatures.ts | 50 +++++++-- .../workbench/api/common/extHost.api.impl.ts | 8 +- .../workbench/api/common/extHost.protocol.ts | 8 +- .../api/common/extHostLanguageFeatures.ts | 90 ++++++++++----- .../inspectEditorTokens.ts | 7 +- 12 files changed, 272 insertions(+), 176 deletions(-) diff --git a/extensions/html-language-features/client/src/htmlMain.ts b/extensions/html-language-features/client/src/htmlMain.ts index f460c850ce1..c54a97f9b07 100644 --- a/extensions/html-language-features/client/src/htmlMain.ts +++ b/extensions/html-language-features/client/src/htmlMain.ts @@ -11,7 +11,7 @@ const localize = nls.loadMessageBundle(); import { languages, ExtensionContext, IndentAction, Position, TextDocument, Range, CompletionItem, CompletionItemKind, SnippetString, workspace, Disposable, FormattingOptions, CancellationToken, ProviderResult, TextEdit, CompletionContext, CompletionList, SemanticTokensLegend, - SemanticTokensProvider, SemanticTokens + DocumentSemanticTokensProvider, DocumentRangeSemanticTokensProvider, SemanticTokens } from 'vscode'; import { LanguageClient, LanguageClientOptions, ServerOptions, TransportKind, RequestType, TextDocumentPositionParams, DocumentRangeFormattingParams, @@ -153,18 +153,26 @@ export function activate(context: ExtensionContext) { client.sendRequest(SemanticTokenLegendRequest.type).then(legend => { if (legend) { - const provider: SemanticTokensProvider = { - provideSemanticTokens(doc, opts) { + const provider: DocumentSemanticTokensProvider & DocumentRangeSemanticTokensProvider = { + provideDocumentSemanticTokens(doc) { const params: SemanticTokenParams = { textDocument: client.code2ProtocolConverter.asTextDocumentIdentifier(doc), - ranges: opts.ranges?.map(r => client.code2ProtocolConverter.asRange(r)) + }; + return client.sendRequest(SemanticTokenRequest.type, params).then(data => { + return data && new SemanticTokens(new Uint32Array(data)); + }); + }, + provideDocumentRangeSemanticTokens(doc, range) { + const params: SemanticTokenParams = { + textDocument: client.code2ProtocolConverter.asTextDocumentIdentifier(doc), + ranges: [client.code2ProtocolConverter.asRange(range)] }; return client.sendRequest(SemanticTokenRequest.type, params).then(data => { return data && new SemanticTokens(new Uint32Array(data)); }); } }; - toDispose.push(languages.registerSemanticTokensProvider(documentSelector, provider, new SemanticTokensLegend(legend.types, legend.modifiers))); + toDispose.push(languages.registerDocumentSemanticTokensProvider(documentSelector, provider, new SemanticTokensLegend(legend.types, legend.modifiers))); } }); diff --git a/extensions/typescript-language-features/src/features/semanticTokens.ts b/extensions/typescript-language-features/src/features/semanticTokens.ts index e34dc09de9a..fc6c770a971 100644 --- a/extensions/typescript-language-features/src/features/semanticTokens.ts +++ b/extensions/typescript-language-features/src/features/semanticTokens.ts @@ -16,17 +16,20 @@ const minTypeScriptVersion = API.fromVersionString(`${VersionRequirement.major}. export function register(selector: vscode.DocumentSelector, client: ITypeScriptServiceClient) { return new VersionDependentRegistration(client, minTypeScriptVersion, () => { - const provider = new SemanticTokensProvider(client); - return vscode.languages.registerSemanticTokensProvider(selector, provider, provider.getLegend()); + const provider = new DocumentSemanticTokensProvider(client); + return vscode.Disposable.from( + vscode.languages.registerDocumentSemanticTokensProvider(selector, provider, provider.getLegend()), + vscode.languages.registerDocumentRangeSemanticTokensProvider(selector, provider, provider.getLegend()), + ); }); } /** - * Prototype of a SemanticTokensProvider, relying on the experimental `encodedSemanticClassifications-full` request from the TypeScript server. + * Prototype of a DocumentSemanticTokensProvider, relying on the experimental `encodedSemanticClassifications-full` request from the TypeScript server. * As the results retured by the TypeScript server are limited, we also add a Typescript plugin (typescript-vscode-sh-plugin) to enrich the returned token. * See https://github.com/aeschli/typescript-vscode-sh-plugin. */ -class SemanticTokensProvider implements vscode.SemanticTokensProvider { +class DocumentSemanticTokensProvider implements vscode.DocumentSemanticTokensProvider, vscode.DocumentRangeSemanticTokensProvider { constructor(private readonly client: ITypeScriptServiceClient) { } @@ -41,68 +44,65 @@ class SemanticTokensProvider implements vscode.SemanticTokensProvider { return new vscode.SemanticTokensLegend(tokenTypes, tokenModifiers); } - async provideSemanticTokens(document: vscode.TextDocument, options: vscode.SemanticTokensRequestOptions, token: vscode.CancellationToken): Promise { + async provideDocumentSemanticTokens(document: vscode.TextDocument, token: vscode.CancellationToken): Promise { + const file = this.client.toOpenedFilePath(document); + if (!file) { + return null; + } + return this._provideSemanticTokens(document, { file, start: 0, length: document.getText().length }, token); + } + + async provideDocumentRangeSemanticTokens(document: vscode.TextDocument, range: vscode.Range, token: vscode.CancellationToken): Promise { + const file = this.client.toOpenedFilePath(document); + if (!file) { + return null; + } + const start = document.offsetAt(range.start); + const length = document.offsetAt(range.end) - start; + return this._provideSemanticTokens(document, { file, start, length }, token); + } + + async _provideSemanticTokens(document: vscode.TextDocument, requestArg: ExperimentalProtocol.EncodedSemanticClassificationsRequestArgs, token: vscode.CancellationToken): Promise { const file = this.client.toOpenedFilePath(document); if (!file) { return null; } - const versionBeforeRequest = document.version; - - const allTokenSpans: number[][] = []; - - let requestArgs: ExperimentalProtocol.EncodedSemanticClassificationsRequestArgs[] = []; - if (options.ranges) { - requestArgs = options.ranges.map(r => { const start = document.offsetAt(r.start); const length = document.offsetAt(r.end) - start; return { file, start, length }; }); - requestArgs = requestArgs.sort((a1, a2) => a1.start - a2.start); - } else { - requestArgs = [{ file, start: 0, length: document.getText().length }]; // full document - } - for (const requestArg of requestArgs) { - const response = await (this.client as ExperimentalProtocol.IExtendedTypeScriptServiceClient).execute('encodedSemanticClassifications-full', requestArg, token); - if (response.type === 'response' && response.body) { - allTokenSpans.push(response.body.spans); - } else { - return null; - } - } - - const versionAfterRequest = document.version; - if (versionBeforeRequest !== versionAfterRequest) { - // A new request will come in soon... + const response = await (this.client as ExperimentalProtocol.IExtendedTypeScriptServiceClient).execute('encodedSemanticClassifications-full', requestArg, token); + if (response.type !== 'response' || !response.body) { return null; } + const tokenSpan = response.body.spans; + const builder = new vscode.SemanticTokensBuilder(); - for (const tokenSpan of allTokenSpans) { - let i = 0; - while (i < tokenSpan.length) { - const offset = tokenSpan[i++]; - const length = tokenSpan[i++]; - const tsClassification = tokenSpan[i++]; + let i = 0; + while (i < tokenSpan.length) { + const offset = tokenSpan[i++]; + const length = tokenSpan[i++]; + const tsClassification = tokenSpan[i++]; - let tokenModifiers = 0; - let tokenType = getTokenTypeFromClassification(tsClassification); - if (tokenType !== undefined) { - // it's a classification as returned by the typescript-vscode-sh-plugin - tokenModifiers = getTokenModifierFromClassification(tsClassification); - } else { - // typescript-vscode-sh-plugin is not present - tokenType = tokenTypeMap[tsClassification]; - if (tokenType === undefined) { - continue; - } + let tokenModifiers = 0; + let tokenType = getTokenTypeFromClassification(tsClassification); + if (tokenType !== undefined) { + // it's a classification as returned by the typescript-vscode-sh-plugin + tokenModifiers = getTokenModifierFromClassification(tsClassification); + } else { + // typescript-vscode-sh-plugin is not present + tokenType = tokenTypeMap[tsClassification]; + if (tokenType === undefined) { + continue; } + } - // we can use the document's range conversion methods because the result is at the same version as the document - const startPos = document.positionAt(offset); - const endPos = document.positionAt(offset + length); + // we can use the document's range conversion methods because the result is at the same version as the document + const startPos = document.positionAt(offset); + const endPos = document.positionAt(offset + length); - for (let line = startPos.line; line <= endPos.line; line++) { - const startCharacter = (line === startPos.line ? startPos.character : 0); - const endCharacter = (line === endPos.line ? endPos.character : document.lineAt(line).text.length); - builder.push(line, startCharacter, endCharacter - startCharacter, tokenType, tokenModifiers); - } + for (let line = startPos.line; line <= endPos.line; line++) { + const startCharacter = (line === startPos.line ? startPos.character : 0); + const endCharacter = (line === endPos.line ? endPos.character : document.lineAt(line).text.length); + builder.push(line, startCharacter, endCharacter - startCharacter, tokenType, tokenModifiers); } } return new vscode.SemanticTokens(builder.build()); diff --git a/extensions/vscode-colorize-tests/src/colorizerTestMain.ts b/extensions/vscode-colorize-tests/src/colorizerTestMain.ts index 9586fc0bb97..a014275a7e6 100644 --- a/extensions/vscode-colorize-tests/src/colorizerTestMain.ts +++ b/extensions/vscode-colorize-tests/src/colorizerTestMain.ts @@ -15,8 +15,8 @@ export function activate(context: vscode.ExtensionContext): any { const outputChannel = vscode.window.createOutputChannel('Semantic Tokens Test'); - const semanticHighlightProvider: vscode.SemanticTokensProvider = { - provideSemanticTokens(document: vscode.TextDocument): vscode.ProviderResult { + const documentSemanticHighlightProvider: vscode.DocumentSemanticTokensProvider = { + provideDocumentSemanticTokens(document: vscode.TextDocument): vscode.ProviderResult { const builder = new vscode.SemanticTokensBuilder(); function addToken(value: string, startLine: number, startCharacter: number, length: number) { @@ -61,6 +61,6 @@ export function activate(context: vscode.ExtensionContext): any { }; - context.subscriptions.push(vscode.languages.registerSemanticTokensProvider({ pattern: '**/*semantic-test.json' }, semanticHighlightProvider, legend)); + context.subscriptions.push(vscode.languages.registerDocumentSemanticTokensProvider({ pattern: '**/*semantic-test.json' }, documentSemanticHighlightProvider, legend)); } diff --git a/src/vs/editor/common/modes.ts b/src/vs/editor/common/modes.ts index 56fba6eb6b7..7ba8aa14c8b 100644 --- a/src/vs/editor/common/modes.ts +++ b/src/vs/editor/common/modes.ts @@ -1500,10 +1500,15 @@ export interface SemanticTokensEdits { readonly edits: SemanticTokensEdit[]; } -export interface SemanticTokensProvider { +export interface DocumentSemanticTokensProvider { getLegend(): SemanticTokensLegend; - provideSemanticTokens(model: model.ITextModel, lastResultId: string | null, ranges: Range[] | null, token: CancellationToken): ProviderResult; - releaseSemanticTokens(resultId: string | undefined): void; + provideDocumentSemanticTokens(model: model.ITextModel, lastResultId: string | null, token: CancellationToken): ProviderResult; + releaseDocumentSemanticTokens(resultId: string | undefined): void; +} + +export interface DocumentRangeSemanticTokensProvider { + getLegend(): SemanticTokensLegend; + provideDocumentRangeSemanticTokens(model: model.ITextModel, range: Range, token: CancellationToken): ProviderResult; } // --- feature registries ------ @@ -1611,7 +1616,12 @@ export const FoldingRangeProviderRegistry = new LanguageFeatureRegistry(); +export const DocumentSemanticTokensProviderRegistry = new LanguageFeatureRegistry(); + +/** + * @internal + */ +export const DocumentRangeSemanticTokensProviderRegistry = new LanguageFeatureRegistry(); /** * @internal diff --git a/src/vs/editor/common/services/modelServiceImpl.ts b/src/vs/editor/common/services/modelServiceImpl.ts index 61e87ec9ff9..26222549279 100644 --- a/src/vs/editor/common/services/modelServiceImpl.ts +++ b/src/vs/editor/common/services/modelServiceImpl.ts @@ -14,7 +14,7 @@ import { Range } from 'vs/editor/common/core/range'; import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model'; import { TextModel, createTextBuffer } from 'vs/editor/common/model/textModel'; import { IModelLanguageChangedEvent, IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents'; -import { LanguageIdentifier, SemanticTokensProviderRegistry, SemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits, TokenMetadata } from 'vs/editor/common/modes'; +import { LanguageIdentifier, DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits, TokenMetadata } from 'vs/editor/common/modes'; import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry'; import { ILanguageSelection } from 'vs/editor/common/services/modeService'; import { IModelService } from 'vs/editor/common/services/modelService'; @@ -498,23 +498,23 @@ class SemanticColoringFeature extends Disposable { class SemanticStyling extends Disposable { - private _caches: WeakMap; + private _caches: WeakMap; constructor( private readonly _themeService: IThemeService, private readonly _logService: ILogService ) { super(); - this._caches = new WeakMap(); + this._caches = new WeakMap(); if (this._themeService) { // workaround for tests which use undefined... :/ this._register(this._themeService.onThemeChange(() => { - this._caches = new WeakMap(); + this._caches = new WeakMap(); })); } } - public get(provider: SemanticTokensProvider): SemanticColoringProviderStyling { + public get(provider: DocumentSemanticTokensProvider): SemanticColoringProviderStyling { if (!this._caches.has(provider)) { this._caches.set(provider, new SemanticColoringProviderStyling(provider.getLegend(), this._themeService, this._logService)); } @@ -676,13 +676,13 @@ const enum SemanticColoringConstants { class SemanticTokensResponse { constructor( - private readonly _provider: SemanticTokensProvider, + private readonly _provider: DocumentSemanticTokensProvider, public readonly resultId: string | undefined, public readonly data: Uint32Array ) { } public dispose(): void { - this._provider.releaseSemanticTokens(this.resultId); + this._provider.releaseDocumentSemanticTokens(this.resultId); } } @@ -710,7 +710,7 @@ class ModelSemanticColoring extends Disposable { this._fetchSemanticTokens.schedule(); } })); - this._register(SemanticTokensProviderRegistry.onDidChange(e => this._fetchSemanticTokens.schedule())); + this._register(DocumentSemanticTokensProviderRegistry.onDidChange(e => this._fetchSemanticTokens.schedule())); if (themeService) { // workaround for tests which use undefined... :/ this._register(themeService.onThemeChange(_ => { @@ -756,7 +756,7 @@ class ModelSemanticColoring extends Disposable { const styling = this._semanticStyling.get(provider); const lastResultId = this._currentResponse ? this._currentResponse.resultId || null : null; - const request = Promise.resolve(provider.provideSemanticTokens(this._model, lastResultId, null, this._currentRequestCancellationTokenSource.token)); + const request = Promise.resolve(provider.provideDocumentSemanticTokens(this._model, lastResultId, this._currentRequestCancellationTokenSource.token)); request.then((res) => { this._currentRequestCancellationTokenSource = null; @@ -784,7 +784,7 @@ class ModelSemanticColoring extends Disposable { } } - private _setSemanticTokens(provider: SemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticColoringProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void { + private _setSemanticTokens(provider: DocumentSemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticColoringProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void { const currentResponse = this._currentResponse; if (this._currentResponse) { this._currentResponse.dispose(); @@ -793,7 +793,7 @@ class ModelSemanticColoring extends Disposable { if (this._isDisposed) { // disposed! if (provider && tokens) { - provider.releaseSemanticTokens(tokens.resultId); + provider.releaseDocumentSemanticTokens(tokens.resultId); } return; } @@ -954,8 +954,8 @@ class ModelSemanticColoring extends Disposable { this._model.setSemanticTokens(null); } - private _getSemanticColoringProvider(): SemanticTokensProvider | null { - const result = SemanticTokensProviderRegistry.ordered(this._model); + private _getSemanticColoringProvider(): DocumentSemanticTokensProvider | null { + const result = DocumentSemanticTokensProviderRegistry.ordered(this._model); return (result.length > 0 ? result[0] : null); } } diff --git a/src/vs/monaco.d.ts b/src/vs/monaco.d.ts index ced17ac0b31..b1f6cf15b9a 100644 --- a/src/vs/monaco.d.ts +++ b/src/vs/monaco.d.ts @@ -5948,10 +5948,15 @@ declare namespace monaco.languages { readonly edits: SemanticTokensEdit[]; } - export interface SemanticTokensProvider { + export interface DocumentSemanticTokensProvider { getLegend(): SemanticTokensLegend; - provideSemanticTokens(model: editor.ITextModel, lastResultId: string | null, ranges: Range[] | null, token: CancellationToken): ProviderResult; - releaseSemanticTokens(resultId: string | undefined): void; + provideDocumentSemanticTokens(model: editor.ITextModel, lastResultId: string | null, token: CancellationToken): ProviderResult; + releaseDocumentSemanticTokens(resultId: string | undefined): void; + } + + export interface DocumentRangeSemanticTokensProvider { + getLegend(): SemanticTokensLegend; + provideDocumentRangeSemanticTokens(model: editor.ITextModel, range: Range, token: CancellationToken): ProviderResult; } export interface ILanguageExtensionPoint { diff --git a/src/vs/vscode.proposed.d.ts b/src/vs/vscode.proposed.d.ts index 0de7f957f3f..25eb939f4eb 100644 --- a/src/vs/vscode.proposed.d.ts +++ b/src/vs/vscode.proposed.d.ts @@ -180,8 +180,7 @@ declare module 'vscode' { /** * The result id of the tokens. * - * On a next call to `provideSemanticTokens`, if VS Code still holds in memory this result, - * the result id will be passed in as `SemanticTokensRequestOptions.previousResultId`. + * This is the id that will be passed to `DocumentSemanticTokensProvider.provideDocumentSemanticTokensEdits` (if implemented). */ readonly resultId?: string; readonly data: Uint32Array; @@ -193,8 +192,7 @@ declare module 'vscode' { /** * The result id of the tokens. * - * On a next call to `provideSemanticTokens`, if VS Code still holds in memory this result, - * the result id will be passed in as `SemanticTokensRequestOptions.previousResultId`. + * This is the id that will be passed to `DocumentSemanticTokensProvider.provideDocumentSemanticTokensEdits` (if implemented). */ readonly resultId?: string; readonly edits: SemanticTokensEdit[]; @@ -210,21 +208,11 @@ declare module 'vscode' { constructor(start: number, deleteCount: number, data?: Uint32Array); } - export interface SemanticTokensRequestOptions { - readonly ranges?: readonly Range[]; - /** - * The previous result id that the editor still holds in memory. - * - * Only when this is set it is safe for a `SemanticTokensProvider` to return `SemanticTokensEdits`. - */ - readonly previousResultId?: string; - } - /** - * The semantic tokens provider interface defines the contract between extensions and + * The document semantic tokens provider interface defines the contract between extensions and * semantic tokens. */ - export interface SemanticTokensProvider { + export interface DocumentSemanticTokensProvider { /** * A file can contain many tokens, perhaps even hundreds of thousands of tokens. Therefore, to improve * the memory consumption around describing semantic tokens, we have decided to avoid allocating an object @@ -232,21 +220,18 @@ declare module 'vscode' { * of each token is expressed relative to the token before it because most tokens remain stable relative to * each other when edits are made in a file. * - * * --- - * In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following fields: + * In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following array indices: * - at index `5*i` - `deltaLine`: token line number, relative to the previous token * - at index `5*i+1` - `deltaStart`: token start character, relative to the previous token (relative to 0 or the previous token's start if they are on the same line) * - at index `5*i+2` - `length`: the length of the token. A token cannot be multiline. * - at index `5*i+3` - `tokenType`: will be looked up in `SemanticTokensLegend.tokenTypes` * - at index `5*i+4` - `tokenModifiers`: each set bit will be looked up in `SemanticTokensLegend.tokenModifiers` * - * - * * --- * ### How to encode tokens * - * Here is an example for encoding a file with 3 tokens: + * Here is an example for encoding a file with 3 tokens in a uint32 array: * ``` * { line: 2, startChar: 5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] }, * { line: 2, startChar: 10, length: 4, tokenType: "types", tokenModifiers: [] }, @@ -285,8 +270,12 @@ declare module 'vscode' { * // 1st token, 2nd token, 3rd token * [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] * ``` - * - * + */ + provideDocumentSemanticTokens(document: TextDocument, token: CancellationToken): ProviderResult; + + /** + * Instead of always returning all the tokens in a file, it is possible for a `DocumentSemanticTokensProvider` to implement + * this method (`updateSemanticTokens`) and then return incremental updates to the previously provided semantic tokens. * * --- * ### How tokens change when the document changes @@ -307,8 +296,8 @@ declare module 'vscode' { * ``` * It is possible to express these new tokens in terms of an edit applied to the previous tokens: * ``` - * [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] - * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] + * [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] // old tokens + * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] // new tokens * * edit: { start: 0, deleteCount: 1, data: [3] } // replace integer at offset 0 with 3 * ``` @@ -327,51 +316,56 @@ declare module 'vscode' { * ``` * Again, it is possible to express these new tokens in terms of an edit applied to the previous tokens: * ``` - * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] - * [ 3,5,3,0,3, 0,5,4,1,0, 1,3,5,0,2, 2,2,7,2,0, ] + * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] // old tokens + * [ 3,5,3,0,3, 0,5,4,1,0, 1,3,5,0,2, 2,2,7,2,0, ] // new tokens * * edit: { start: 10, deleteCount: 1, data: [1,3,5,0,2,2] } // replace integer at offset 10 with [1,3,5,0,2,2] * ``` * - * - * - * --- - * ### When to return `SemanticTokensEdits` - * - * When doing edits, it is possible that multiple edits occur until VS Code decides to invoke the semantic tokens provider. - * In principle, each call to `provideSemanticTokens` can return a full representations of the semantic tokens, and that would - * be a perfectly reasonable semantic tokens provider implementation. - * - * However, when having a language server running in a separate process, transferring all the tokens between processes - * might be slow, so VS Code allows to return the new tokens expressed in terms of multiple edits applied to the previous - * tokens. - * - * To clearly define what "previous tokens" means, it is possible to return a `resultId` with the semantic tokens. If the - * editor still has in memory the previous result, the editor will pass in options the previous `resultId` at - * `SemanticTokensRequestOptions.previousResultId`. Only when the editor passes in the previous `resultId`, it is allowed - * that a semantic tokens provider returns the new tokens expressed as edits to be applied to the previous result. Even in this - * case, the semantic tokens provider needs to return a new `resultId` that will identify these new tokens as a basis - * for the next request. - * - * *NOTE 1*: It is illegal to return `SemanticTokensEdits` if `options.previousResultId` is not set. - * *NOTE 2*: All edits in `SemanticTokensEdits` contain indices in the old integers array, so they all refer to the previous result state. + * *NOTE*: When doing edits, it is possible that multiple edits occur until VS Code decides to invoke the semantic tokens provider. + * *NOTE*: If the provider cannot compute `SemanticTokensEdits`, it can "give up" and return all the tokens in the document again. + * *NOTE*: All edits in `SemanticTokensEdits` contain indices in the old integers array, so they all refer to the previous result state. */ - provideSemanticTokens(document: TextDocument, options: SemanticTokensRequestOptions, token: CancellationToken): ProviderResult; + provideDocumentSemanticTokensEdits?(document: TextDocument, previousResultId: string, token: CancellationToken): ProviderResult; + } + + /** + * The document range semantic tokens provider interface defines the contract between extensions and + * semantic tokens. + */ + export interface DocumentRangeSemanticTokensProvider { + /** + * See [provideDocumentSemanticTokens](#DocumentSemanticTokensProvider.provideDocumentSemanticTokens). + */ + provideDocumentRangeSemanticTokens(document: TextDocument, range: Range, token: CancellationToken): ProviderResult; } export namespace languages { /** - * Register a semantic tokens provider. + * Register a semantic tokens provider for a whole document. * * Multiple providers can be registered for a language. In that case providers are sorted * by their [score](#languages.match) and the best-matching provider is used. Failure * of the selected provider will cause a failure of the whole operation. * * @param selector A selector that defines the documents this provider is applicable to. - * @param provider A semantic tokens provider. + * @param provider A document semantic tokens provider. * @return A [disposable](#Disposable) that unregisters this provider when being disposed. */ - export function registerSemanticTokensProvider(selector: DocumentSelector, provider: SemanticTokensProvider, legend: SemanticTokensLegend): Disposable; + export function registerDocumentSemanticTokensProvider(selector: DocumentSelector, provider: DocumentSemanticTokensProvider, legend: SemanticTokensLegend): Disposable; + + /** + * Register a semantic tokens provider for a document range. + * + * Multiple providers can be registered for a language. In that case providers are sorted + * by their [score](#languages.match) and the best-matching provider is used. Failure + * of the selected provider will cause a failure of the whole operation. + * + * @param selector A selector that defines the documents this provider is applicable to. + * @param provider A document range semantic tokens provider. + * @return A [disposable](#Disposable) that unregisters this provider when being disposed. + */ + export function registerDocumentRangeSemanticTokensProvider(selector: DocumentSelector, provider: DocumentRangeSemanticTokensProvider, legend: SemanticTokensLegend): Disposable; } //#endregion diff --git a/src/vs/workbench/api/browser/mainThreadLanguageFeatures.ts b/src/vs/workbench/api/browser/mainThreadLanguageFeatures.ts index da77e770f4e..99c8d442d61 100644 --- a/src/vs/workbench/api/browser/mainThreadLanguageFeatures.ts +++ b/src/vs/workbench/api/browser/mainThreadLanguageFeatures.ts @@ -327,8 +327,12 @@ export class MainThreadLanguageFeatures implements MainThreadLanguageFeaturesSha // --- semantic tokens - $registerSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void { - this._registrations.set(handle, modes.SemanticTokensProviderRegistry.register(selector, new MainThreadSemanticTokensProvider(this._proxy, handle, legend))); + $registerDocumentSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void { + this._registrations.set(handle, modes.DocumentSemanticTokensProviderRegistry.register(selector, new MainThreadDocumentSemanticTokensProvider(this._proxy, handle, legend))); + } + + $registerDocumentRangeSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void { + this._registrations.set(handle, modes.DocumentRangeSemanticTokensProviderRegistry.register(selector, new MainThreadDocumentRangeSemanticTokensProvider(this._proxy, handle, legend))); } // --- suggest @@ -607,7 +611,7 @@ export class MainThreadLanguageFeatures implements MainThreadLanguageFeaturesSha } -export class MainThreadSemanticTokensProvider implements modes.SemanticTokensProvider { +export class MainThreadDocumentSemanticTokensProvider implements modes.DocumentSemanticTokensProvider { constructor( private readonly _proxy: ExtHostLanguageFeaturesShape, @@ -616,9 +620,9 @@ export class MainThreadSemanticTokensProvider implements modes.SemanticTokensPro ) { } - public releaseSemanticTokens(resultId: string | undefined): void { + public releaseDocumentSemanticTokens(resultId: string | undefined): void { if (resultId) { - this._proxy.$releaseSemanticTokens(this._handle, parseInt(resultId, 10)); + this._proxy.$releaseDocumentSemanticTokens(this._handle, parseInt(resultId, 10)); } } @@ -626,9 +630,9 @@ export class MainThreadSemanticTokensProvider implements modes.SemanticTokensPro return this._legend; } - async provideSemanticTokens(model: ITextModel, lastResultId: string | null, ranges: EditorRange[] | null, token: CancellationToken): Promise { + async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise { const nLastResultId = lastResultId ? parseInt(lastResultId, 10) : 0; - const encodedDto = await this._proxy.$provideSemanticTokens(this._handle, model.uri, ranges, nLastResultId, token); + const encodedDto = await this._proxy.$provideDocumentSemanticTokens(this._handle, model.uri, nLastResultId, token); if (!encodedDto) { return null; } @@ -648,3 +652,35 @@ export class MainThreadSemanticTokensProvider implements modes.SemanticTokensPro }; } } + +export class MainThreadDocumentRangeSemanticTokensProvider implements modes.DocumentRangeSemanticTokensProvider { + + constructor( + private readonly _proxy: ExtHostLanguageFeaturesShape, + private readonly _handle: number, + private readonly _legend: modes.SemanticTokensLegend, + ) { + } + + public getLegend(): modes.SemanticTokensLegend { + return this._legend; + } + + async provideDocumentRangeSemanticTokens(model: ITextModel, range: EditorRange, token: CancellationToken): Promise { + const encodedDto = await this._proxy.$provideDocumentRangeSemanticTokens(this._handle, model.uri, range, token); + if (!encodedDto) { + return null; + } + if (token.isCancellationRequested) { + return null; + } + const dto = decodeSemanticTokensDto(encodedDto); + if (dto.type === 'full') { + return { + resultId: String(dto.id), + data: dto.data + }; + } + throw new Error(`Unexpected`); + } +} diff --git a/src/vs/workbench/api/common/extHost.api.impl.ts b/src/vs/workbench/api/common/extHost.api.impl.ts index 8cd44e1bc53..6a8dc3eaa66 100644 --- a/src/vs/workbench/api/common/extHost.api.impl.ts +++ b/src/vs/workbench/api/common/extHost.api.impl.ts @@ -379,9 +379,13 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I registerOnTypeFormattingEditProvider(selector: vscode.DocumentSelector, provider: vscode.OnTypeFormattingEditProvider, firstTriggerCharacter: string, ...moreTriggerCharacters: string[]): vscode.Disposable { return extHostLanguageFeatures.registerOnTypeFormattingEditProvider(extension, checkSelector(selector), provider, [firstTriggerCharacter].concat(moreTriggerCharacters)); }, - registerSemanticTokensProvider(selector: vscode.DocumentSelector, provider: vscode.SemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable { + registerDocumentSemanticTokensProvider(selector: vscode.DocumentSelector, provider: vscode.DocumentSemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable { checkProposedApiEnabled(extension); - return extHostLanguageFeatures.registerSemanticTokensProvider(extension, checkSelector(selector), provider, legend); + return extHostLanguageFeatures.registerDocumentSemanticTokensProvider(extension, checkSelector(selector), provider, legend); + }, + registerDocumentRangeSemanticTokensProvider(selector: vscode.DocumentSelector, provider: vscode.DocumentRangeSemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable { + checkProposedApiEnabled(extension); + return extHostLanguageFeatures.registerDocumentRangeSemanticTokensProvider(extension, checkSelector(selector), provider, legend); }, registerSignatureHelpProvider(selector: vscode.DocumentSelector, provider: vscode.SignatureHelpProvider, firstItem?: string | vscode.SignatureHelpProviderMetadata, ...remaining: string[]): vscode.Disposable { if (typeof firstItem === 'object') { diff --git a/src/vs/workbench/api/common/extHost.protocol.ts b/src/vs/workbench/api/common/extHost.protocol.ts index 2d08ee74397..dcbcdc1c5ae 100644 --- a/src/vs/workbench/api/common/extHost.protocol.ts +++ b/src/vs/workbench/api/common/extHost.protocol.ts @@ -358,7 +358,8 @@ export interface MainThreadLanguageFeaturesShape extends IDisposable { $registerOnTypeFormattingSupport(handle: number, selector: IDocumentFilterDto[], autoFormatTriggerCharacters: string[], extensionId: ExtensionIdentifier): void; $registerNavigateTypeSupport(handle: number): void; $registerRenameSupport(handle: number, selector: IDocumentFilterDto[], supportsResolveInitialValues: boolean): void; - $registerSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void; + $registerDocumentSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void; + $registerDocumentRangeSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void; $registerSuggestSupport(handle: number, selector: IDocumentFilterDto[], triggerCharacters: string[], supportsResolveDetails: boolean, extensionId: ExtensionIdentifier): void; $registerSignatureHelpProvider(handle: number, selector: IDocumentFilterDto[], metadata: ISignatureHelpProviderMetadataDto): void; $registerDocumentLinkProvider(handle: number, selector: IDocumentFilterDto[], supportsResolve: boolean): void; @@ -1188,8 +1189,9 @@ export interface ExtHostLanguageFeaturesShape { $releaseWorkspaceSymbols(handle: number, id: number): void; $provideRenameEdits(handle: number, resource: UriComponents, position: IPosition, newName: string, token: CancellationToken): Promise; $resolveRenameLocation(handle: number, resource: UriComponents, position: IPosition, token: CancellationToken): Promise; - $provideSemanticTokens(handle: number, resource: UriComponents, ranges: IRange[] | null, previousResultId: number, token: CancellationToken): Promise; - $releaseSemanticTokens(handle: number, semanticColoringResultId: number): void; + $provideDocumentSemanticTokens(handle: number, resource: UriComponents, previousResultId: number, token: CancellationToken): Promise; + $releaseDocumentSemanticTokens(handle: number, semanticColoringResultId: number): void; + $provideDocumentRangeSemanticTokens(handle: number, resource: UriComponents, range: IRange, token: CancellationToken): Promise; $provideCompletionItems(handle: number, resource: UriComponents, position: IPosition, context: modes.CompletionContext, token: CancellationToken): Promise; $resolveCompletionItem(handle: number, resource: UriComponents, position: IPosition, id: ChainedCacheId, token: CancellationToken): Promise; $releaseCompletionItems(handle: number, id: number): void; diff --git a/src/vs/workbench/api/common/extHostLanguageFeatures.ts b/src/vs/workbench/api/common/extHostLanguageFeatures.ts index f3cb6fade93..9c97223adcb 100644 --- a/src/vs/workbench/api/common/extHostLanguageFeatures.ts +++ b/src/vs/workbench/api/common/extHostLanguageFeatures.ts @@ -629,37 +629,38 @@ class SemanticTokensPreviousResult { ) { } } -export class SemanticTokensAdapter { +export class DocumentSemanticTokensAdapter { private readonly _previousResults: Map; private _nextResultId = 1; constructor( private readonly _documents: ExtHostDocuments, - private readonly _provider: vscode.SemanticTokensProvider, + private readonly _provider: vscode.DocumentSemanticTokensProvider, ) { this._previousResults = new Map(); } - provideSemanticTokens(resource: URI, ranges: IRange[] | null, previousResultId: number, token: CancellationToken): Promise { + provideDocumentSemanticTokens(resource: URI, previousResultId: number, token: CancellationToken): Promise { const doc = this._documents.getDocument(resource); const previousResult = (previousResultId !== 0 ? this._previousResults.get(previousResultId) : null); - const opts: vscode.SemanticTokensRequestOptions = { - ranges: (Array.isArray(ranges) && ranges.length > 0 ? ranges.map(typeConvert.Range.to) : undefined), - previousResultId: (previousResult ? previousResult.resultId : undefined) - }; - return asPromise(() => this._provider.provideSemanticTokens(doc, opts, token)).then(value => { - if (!value) { - return null; + return asPromise(() => { + if (previousResult && typeof previousResult.resultId === 'string' && typeof this._provider.provideDocumentSemanticTokensEdits === 'function') { + return this._provider.provideDocumentSemanticTokensEdits(doc, previousResult.resultId, token); } + return this._provider.provideDocumentSemanticTokens(doc, token); + }).then(value => { if (previousResult) { this._previousResults.delete(previousResultId); } - return this._send(SemanticTokensAdapter._convertToEdits(previousResult, value), value); + if (!value) { + return null; + } + return this._send(DocumentSemanticTokensAdapter._convertToEdits(previousResult, value), value); }); } - async releaseSemanticColoring(semanticColoringResultId: number): Promise { + async releaseDocumentSemanticColoring(semanticColoringResultId: number): Promise { this._previousResults.delete(semanticColoringResultId); } @@ -672,7 +673,7 @@ export class SemanticTokensAdapter { } private static _convertToEdits(previousResult: SemanticTokensPreviousResult | null | undefined, newResult: vscode.SemanticTokens | vscode.SemanticTokensEdits): vscode.SemanticTokens | vscode.SemanticTokensEdits { - if (!SemanticTokensAdapter._isSemanticTokens(newResult)) { + if (!DocumentSemanticTokensAdapter._isSemanticTokens(newResult)) { return newResult; } if (!previousResult || !previousResult.tokens) { @@ -708,7 +709,7 @@ export class SemanticTokensAdapter { } private _send(value: vscode.SemanticTokens | vscode.SemanticTokensEdits, original: vscode.SemanticTokens | vscode.SemanticTokensEdits): VSBuffer | null { - if (SemanticTokensAdapter._isSemanticTokens(value)) { + if (DocumentSemanticTokensAdapter._isSemanticTokens(value)) { const myId = this._nextResultId++; this._previousResults.set(myId, new SemanticTokensPreviousResult(value.resultId, value.data)); return encodeSemanticTokensDto({ @@ -718,9 +719,9 @@ export class SemanticTokensAdapter { }); } - if (SemanticTokensAdapter._isSemanticTokensEdits(value)) { + if (DocumentSemanticTokensAdapter._isSemanticTokensEdits(value)) { const myId = this._nextResultId++; - if (SemanticTokensAdapter._isSemanticTokens(original)) { + if (DocumentSemanticTokensAdapter._isSemanticTokens(original)) { // store the original this._previousResults.set(myId, new SemanticTokensPreviousResult(original.resultId, original.data)); } else { @@ -737,6 +738,33 @@ export class SemanticTokensAdapter { } } +export class DocumentRangeSemanticTokensAdapter { + + constructor( + private readonly _documents: ExtHostDocuments, + private readonly _provider: vscode.DocumentRangeSemanticTokensProvider, + ) { + } + + provideDocumentRangeSemanticTokens(resource: URI, range: IRange, token: CancellationToken): Promise { + const doc = this._documents.getDocument(resource); + return asPromise(() => this._provider.provideDocumentRangeSemanticTokens(doc, typeConvert.Range.to(range), token)).then(value => { + if (!value) { + return null; + } + return this._send(value); + }); + } + + private _send(value: vscode.SemanticTokens): VSBuffer | null { + return encodeSemanticTokensDto({ + id: 0, + type: 'full', + data: value.data + }); + } +} + class SuggestAdapter { static supportsResolving(provider: vscode.CompletionItemProvider): boolean { @@ -1324,9 +1352,9 @@ class CallHierarchyAdapter { type Adapter = DocumentSymbolAdapter | CodeLensAdapter | DefinitionAdapter | HoverAdapter | DocumentHighlightAdapter | ReferenceAdapter | CodeActionAdapter | DocumentFormattingAdapter | RangeFormattingAdapter | OnTypeFormattingAdapter | NavigateTypeAdapter | RenameAdapter - | SemanticTokensAdapter | SuggestAdapter | SignatureHelpAdapter | LinkProviderAdapter - | ImplementationAdapter | TypeDefinitionAdapter | ColorProviderAdapter | FoldingProviderAdapter - | DeclarationAdapter | SelectionRangeAdapter | CallHierarchyAdapter; + | SuggestAdapter | SignatureHelpAdapter | LinkProviderAdapter | ImplementationAdapter + | TypeDefinitionAdapter | ColorProviderAdapter | FoldingProviderAdapter | DeclarationAdapter + | SelectionRangeAdapter | CallHierarchyAdapter | DocumentSemanticTokensAdapter | DocumentRangeSemanticTokensAdapter; class AdapterData { constructor( @@ -1657,18 +1685,28 @@ export class ExtHostLanguageFeatures implements extHostProtocol.ExtHostLanguageF //#region semantic coloring - registerSemanticTokensProvider(extension: IExtensionDescription, selector: vscode.DocumentSelector, provider: vscode.SemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable { - const handle = this._addNewAdapter(new SemanticTokensAdapter(this._documents, provider), extension); - this._proxy.$registerSemanticTokensProvider(handle, this._transformDocumentSelector(selector), legend); + registerDocumentSemanticTokensProvider(extension: IExtensionDescription, selector: vscode.DocumentSelector, provider: vscode.DocumentSemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable { + const handle = this._addNewAdapter(new DocumentSemanticTokensAdapter(this._documents, provider), extension); + this._proxy.$registerDocumentSemanticTokensProvider(handle, this._transformDocumentSelector(selector), legend); return this._createDisposable(handle); } - $provideSemanticTokens(handle: number, resource: UriComponents, ranges: IRange[] | null, previousResultId: number, token: CancellationToken): Promise { - return this._withAdapter(handle, SemanticTokensAdapter, adapter => adapter.provideSemanticTokens(URI.revive(resource), ranges, previousResultId, token), null); + $provideDocumentSemanticTokens(handle: number, resource: UriComponents, previousResultId: number, token: CancellationToken): Promise { + return this._withAdapter(handle, DocumentSemanticTokensAdapter, adapter => adapter.provideDocumentSemanticTokens(URI.revive(resource), previousResultId, token), null); } - $releaseSemanticTokens(handle: number, semanticColoringResultId: number): void { - this._withAdapter(handle, SemanticTokensAdapter, adapter => adapter.releaseSemanticColoring(semanticColoringResultId), undefined); + $releaseDocumentSemanticTokens(handle: number, semanticColoringResultId: number): void { + this._withAdapter(handle, DocumentSemanticTokensAdapter, adapter => adapter.releaseDocumentSemanticColoring(semanticColoringResultId), undefined); + } + + registerDocumentRangeSemanticTokensProvider(extension: IExtensionDescription, selector: vscode.DocumentSelector, provider: vscode.DocumentRangeSemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable { + const handle = this._addNewAdapter(new DocumentRangeSemanticTokensAdapter(this._documents, provider), extension); + this._proxy.$registerDocumentRangeSemanticTokensProvider(handle, this._transformDocumentSelector(selector), legend); + return this._createDisposable(handle); + } + + $provideDocumentRangeSemanticTokens(handle: number, resource: UriComponents, range: IRange, token: CancellationToken): Promise { + return this._withAdapter(handle, DocumentRangeSemanticTokensAdapter, adapter => adapter.provideDocumentRangeSemanticTokens(URI.revive(resource), range, token), null); } //#endregion diff --git a/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts b/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts index 5493783e71f..8feeb14711d 100644 --- a/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts +++ b/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts @@ -16,7 +16,7 @@ import { EditorAction, ServicesAccessor, registerEditorAction, registerEditorCon import { Position } from 'vs/editor/common/core/position'; import { IEditorContribution } from 'vs/editor/common/editorCommon'; import { ITextModel } from 'vs/editor/common/model'; -import { FontStyle, LanguageIdentifier, StandardTokenType, TokenMetadata, SemanticTokensProviderRegistry, SemanticTokensLegend, SemanticTokens } from 'vs/editor/common/modes'; +import { FontStyle, LanguageIdentifier, StandardTokenType, TokenMetadata, DocumentSemanticTokensProviderRegistry, SemanticTokensLegend, SemanticTokens } from 'vs/editor/common/modes'; import { IModeService } from 'vs/editor/common/services/modeService'; import { INotificationService } from 'vs/platform/notification/common/notification'; import { editorHoverBackground, editorHoverBorder } from 'vs/platform/theme/common/colorRegistry'; @@ -411,11 +411,10 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { } private async _computeSemanticTokens(): Promise { - const tokenProviders = SemanticTokensProviderRegistry.ordered(this._model); + const tokenProviders = DocumentSemanticTokensProviderRegistry.ordered(this._model); if (tokenProviders.length) { const provider = tokenProviders[0]; - const range = this._model.getFullModelRange(); - const tokens = await Promise.resolve(provider.provideSemanticTokens(this._model, null, [range], this._currentRequestCancellationTokenSource.token)); + const tokens = await Promise.resolve(provider.provideDocumentSemanticTokens(this._model, null, this._currentRequestCancellationTokenSource.token)); if (this.isSemanticTokens(tokens)) { return { tokens, legend: provider.getLegend() }; }