maxmustermann/vscode
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

SemanticTokens - implement feedback received in API call:

Browse source 
- extract a separate DocumentRangeSemanticTokensProvider that deals with a document range
- extract a separate provideDocumentSemanticTokensEdits that deals with updating via SemanticTokensEdits a previous result
This commit is contained in:
Alex Dima 2020-01-16 17:53:24 +01:00
parent 430de16fef
commit 9e1d730cf2
No known key found for this signature in database
GPG key ID: 6E58D7B045760DA0
12 changed files with 272 additions and 176 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
extensions/html-language-features/client/src/htmlMain.ts
View file

@ -11,7 +11,7 @@ const localize = nls.loadMessageBundle();
import { import {
languages, ExtensionContext, IndentAction, Position, TextDocument, Range, CompletionItem, CompletionItemKind, SnippetString, workspace, languages, ExtensionContext, IndentAction, Position, TextDocument, Range, CompletionItem, CompletionItemKind, SnippetString, workspace,
Disposable, FormattingOptions, CancellationToken, ProviderResult, TextEdit, CompletionContext, CompletionList, SemanticTokensLegend, Disposable, FormattingOptions, CancellationToken, ProviderResult, TextEdit, CompletionContext, CompletionList, SemanticTokensLegend,
SemanticTokensProvider, SemanticTokens DocumentSemanticTokensProvider, DocumentRangeSemanticTokensProvider, SemanticTokens
} from 'vscode'; } from 'vscode';
import { import {
LanguageClient, LanguageClientOptions, ServerOptions, TransportKind, RequestType, TextDocumentPositionParams, DocumentRangeFormattingParams, LanguageClient, LanguageClientOptions, ServerOptions, TransportKind, RequestType, TextDocumentPositionParams, DocumentRangeFormattingParams,
@ -153,18 +153,26 @@ export function activate(context: ExtensionContext) {
client.sendRequest(SemanticTokenLegendRequest.type).then(legend => { client.sendRequest(SemanticTokenLegendRequest.type).then(legend => {
if (legend) { if (legend) {
const provider: SemanticTokensProvider = { const provider: DocumentSemanticTokensProvider & DocumentRangeSemanticTokensProvider = {
provideSemanticTokens(doc, opts) { provideDocumentSemanticTokens(doc) {
const params: SemanticTokenParams = { const params: SemanticTokenParams = {
textDocument: client.code2ProtocolConverter.asTextDocumentIdentifier(doc), textDocument: client.code2ProtocolConverter.asTextDocumentIdentifier(doc),
ranges: opts.ranges?.map(r => client.code2ProtocolConverter.asRange(r)) };
return client.sendRequest(SemanticTokenRequest.type, params).then(data => {
return data && new SemanticTokens(new Uint32Array(data));
});
},
provideDocumentRangeSemanticTokens(doc, range) {
const params: SemanticTokenParams = {
textDocument: client.code2ProtocolConverter.asTextDocumentIdentifier(doc),
ranges: [client.code2ProtocolConverter.asRange(range)]
}; };
return client.sendRequest(SemanticTokenRequest.type, params).then(data => { return client.sendRequest(SemanticTokenRequest.type, params).then(data => {
return data && new SemanticTokens(new Uint32Array(data)); return data && new SemanticTokens(new Uint32Array(data));
}); });
} }
}; };
toDispose.push(languages.registerSemanticTokensProvider(documentSelector, provider, new SemanticTokensLegend(legend.types, legend.modifiers))); toDispose.push(languages.registerDocumentSemanticTokensProvider(documentSelector, provider, new SemanticTokensLegend(legend.types, legend.modifiers)));
} }
}); });

106
extensions/typescript-language-features/src/features/semanticTokens.ts
View file

@ -16,17 +16,20 @@ const minTypeScriptVersion = API.fromVersionString(`${VersionRequirement.major}.
export function register(selector: vscode.DocumentSelector, client: ITypeScriptServiceClient) { export function register(selector: vscode.DocumentSelector, client: ITypeScriptServiceClient) {
return new VersionDependentRegistration(client, minTypeScriptVersion, () => { return new VersionDependentRegistration(client, minTypeScriptVersion, () => {
const provider = new SemanticTokensProvider(client); const provider = new DocumentSemanticTokensProvider(client);
return vscode.languages.registerSemanticTokensProvider(selector, provider, provider.getLegend()); return vscode.Disposable.from(
vscode.languages.registerDocumentSemanticTokensProvider(selector, provider, provider.getLegend()),
vscode.languages.registerDocumentRangeSemanticTokensProvider(selector, provider, provider.getLegend()),
);
}); });
} }
/** /**
* Prototype of a SemanticTokensProvider, relying on the experimental `encodedSemanticClassifications-full` request from the TypeScript server. * Prototype of a DocumentSemanticTokensProvider, relying on the experimental `encodedSemanticClassifications-full` request from the TypeScript server.
* As the results retured by the TypeScript server are limited, we also add a Typescript plugin (typescript-vscode-sh-plugin) to enrich the returned token. * As the results retured by the TypeScript server are limited, we also add a Typescript plugin (typescript-vscode-sh-plugin) to enrich the returned token.
* See https://github.com/aeschli/typescript-vscode-sh-plugin. * See https://github.com/aeschli/typescript-vscode-sh-plugin.
*/ */
class SemanticTokensProvider implements vscode.SemanticTokensProvider { class DocumentSemanticTokensProvider implements vscode.DocumentSemanticTokensProvider, vscode.DocumentRangeSemanticTokensProvider {
constructor(private readonly client: ITypeScriptServiceClient) { constructor(private readonly client: ITypeScriptServiceClient) {
} }
@ -41,68 +44,65 @@ class SemanticTokensProvider implements vscode.SemanticTokensProvider {
return new vscode.SemanticTokensLegend(tokenTypes, tokenModifiers); return new vscode.SemanticTokensLegend(tokenTypes, tokenModifiers);
} }
async provideSemanticTokens(document: vscode.TextDocument, options: vscode.SemanticTokensRequestOptions, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> { async provideDocumentSemanticTokens(document: vscode.TextDocument, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
const file = this.client.toOpenedFilePath(document);
if (!file) {
return null;
}
return this._provideSemanticTokens(document, { file, start: 0, length: document.getText().length }, token);
}
async provideDocumentRangeSemanticTokens(document: vscode.TextDocument, range: vscode.Range, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
const file = this.client.toOpenedFilePath(document);
if (!file) {
return null;
}
const start = document.offsetAt(range.start);
const length = document.offsetAt(range.end) - start;
return this._provideSemanticTokens(document, { file, start, length }, token);
}
async _provideSemanticTokens(document: vscode.TextDocument, requestArg: ExperimentalProtocol.EncodedSemanticClassificationsRequestArgs, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
const file = this.client.toOpenedFilePath(document); const file = this.client.toOpenedFilePath(document);
if (!file) { if (!file) {
return null; return null;
} }
const versionBeforeRequest = document.version; const response = await (this.client as ExperimentalProtocol.IExtendedTypeScriptServiceClient).execute('encodedSemanticClassifications-full', requestArg, token);
if (response.type !== 'response' || !response.body) {
const allTokenSpans: number[][] = [];
let requestArgs: ExperimentalProtocol.EncodedSemanticClassificationsRequestArgs[] = [];
if (options.ranges) {
requestArgs = options.ranges.map(r => { const start = document.offsetAt(r.start); const length = document.offsetAt(r.end) - start; return { file, start, length }; });
requestArgs = requestArgs.sort((a1, a2) => a1.start - a2.start);
} else {
requestArgs = [{ file, start: 0, length: document.getText().length }]; // full document
}
for (const requestArg of requestArgs) {
const response = await (this.client as ExperimentalProtocol.IExtendedTypeScriptServiceClient).execute('encodedSemanticClassifications-full', requestArg, token);
if (response.type === 'response' && response.body) {
allTokenSpans.push(response.body.spans);
} else {
return null;
}
}
const versionAfterRequest = document.version;
if (versionBeforeRequest !== versionAfterRequest) {
// A new request will come in soon...
return null; return null;
} }
const tokenSpan = response.body.spans;
const builder = new vscode.SemanticTokensBuilder(); const builder = new vscode.SemanticTokensBuilder();
for (const tokenSpan of allTokenSpans) { let i = 0;
let i = 0; while (i < tokenSpan.length) {
while (i < tokenSpan.length) { const offset = tokenSpan[i++];
const offset = tokenSpan[i++]; const length = tokenSpan[i++];
const length = tokenSpan[i++]; const tsClassification = tokenSpan[i++];
const tsClassification = tokenSpan[i++];
let tokenModifiers = 0; let tokenModifiers = 0;
let tokenType = getTokenTypeFromClassification(tsClassification); let tokenType = getTokenTypeFromClassification(tsClassification);
if (tokenType !== undefined) { if (tokenType !== undefined) {
// it's a classification as returned by the typescript-vscode-sh-plugin // it's a classification as returned by the typescript-vscode-sh-plugin
tokenModifiers = getTokenModifierFromClassification(tsClassification); tokenModifiers = getTokenModifierFromClassification(tsClassification);
} else { } else {
// typescript-vscode-sh-plugin is not present // typescript-vscode-sh-plugin is not present
tokenType = tokenTypeMap[tsClassification]; tokenType = tokenTypeMap[tsClassification];
if (tokenType === undefined) { if (tokenType === undefined) {
continue; continue;
}
} }
}
// we can use the document's range conversion methods because the result is at the same version as the document // we can use the document's range conversion methods because the result is at the same version as the document
const startPos = document.positionAt(offset); const startPos = document.positionAt(offset);
const endPos = document.positionAt(offset + length); const endPos = document.positionAt(offset + length);
for (let line = startPos.line; line <= endPos.line; line++) { for (let line = startPos.line; line <= endPos.line; line++) {
const startCharacter = (line === startPos.line ? startPos.character : 0); const startCharacter = (line === startPos.line ? startPos.character : 0);
const endCharacter = (line === endPos.line ? endPos.character : document.lineAt(line).text.length); const endCharacter = (line === endPos.line ? endPos.character : document.lineAt(line).text.length);
builder.push(line, startCharacter, endCharacter - startCharacter, tokenType, tokenModifiers); builder.push(line, startCharacter, endCharacter - startCharacter, tokenType, tokenModifiers);
}
} }
} }
return new vscode.SemanticTokens(builder.build()); return new vscode.SemanticTokens(builder.build());

6
extensions/vscode-colorize-tests/src/colorizerTestMain.ts
View file

@ -15,8 +15,8 @@ export function activate(context: vscode.ExtensionContext): any {
const outputChannel = vscode.window.createOutputChannel('Semantic Tokens Test'); const outputChannel = vscode.window.createOutputChannel('Semantic Tokens Test');
const semanticHighlightProvider: vscode.SemanticTokensProvider = { const documentSemanticHighlightProvider: vscode.DocumentSemanticTokensProvider = {
provideSemanticTokens(document: vscode.TextDocument): vscode.ProviderResult<vscode.SemanticTokens> { provideDocumentSemanticTokens(document: vscode.TextDocument): vscode.ProviderResult<vscode.SemanticTokens> {
const builder = new vscode.SemanticTokensBuilder(); const builder = new vscode.SemanticTokensBuilder();
function addToken(value: string, startLine: number, startCharacter: number, length: number) { function addToken(value: string, startLine: number, startCharacter: number, length: number) {
@ -61,6 +61,6 @@ export function activate(context: vscode.ExtensionContext): any {
}; };
context.subscriptions.push(vscode.languages.registerSemanticTokensProvider({ pattern: '**/*semantic-test.json' }, semanticHighlightProvider, legend)); context.subscriptions.push(vscode.languages.registerDocumentSemanticTokensProvider({ pattern: '**/*semantic-test.json' }, documentSemanticHighlightProvider, legend));
} }

18
src/vs/editor/common/modes.ts
View file

@ -1500,10 +1500,15 @@ export interface SemanticTokensEdits {
readonly edits: SemanticTokensEdit[]; readonly edits: SemanticTokensEdit[];
} }
export interface SemanticTokensProvider { export interface DocumentSemanticTokensProvider {
getLegend(): SemanticTokensLegend; getLegend(): SemanticTokensLegend;
provideSemanticTokens(model: model.ITextModel, lastResultId: string | null, ranges: Range[] | null, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>; provideDocumentSemanticTokens(model: model.ITextModel, lastResultId: string | null, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
releaseSemanticTokens(resultId: string | undefined): void; releaseDocumentSemanticTokens(resultId: string | undefined): void;
}
export interface DocumentRangeSemanticTokensProvider {
getLegend(): SemanticTokensLegend;
provideDocumentRangeSemanticTokens(model: model.ITextModel, range: Range, token: CancellationToken): ProviderResult<SemanticTokens>;
} }
// --- feature registries ------ // --- feature registries ------
@ -1611,7 +1616,12 @@ export const FoldingRangeProviderRegistry = new LanguageFeatureRegistry<FoldingR
/** /**
* @internal * @internal
*/ */
export const SemanticTokensProviderRegistry = new LanguageFeatureRegistry<SemanticTokensProvider>(); export const DocumentSemanticTokensProviderRegistry = new LanguageFeatureRegistry<DocumentSemanticTokensProvider>();
/**
* @internal
*/
export const DocumentRangeSemanticTokensProviderRegistry = new LanguageFeatureRegistry<DocumentRangeSemanticTokensProvider>();
/** /**
* @internal * @internal

26
src/vs/editor/common/services/modelServiceImpl.ts
View file

@ -14,7 +14,7 @@ import { Range } from 'vs/editor/common/core/range';
import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model'; import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model';
import { TextModel, createTextBuffer } from 'vs/editor/common/model/textModel'; import { TextModel, createTextBuffer } from 'vs/editor/common/model/textModel';
import { IModelLanguageChangedEvent, IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents'; import { IModelLanguageChangedEvent, IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
import { LanguageIdentifier, SemanticTokensProviderRegistry, SemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits, TokenMetadata } from 'vs/editor/common/modes'; import { LanguageIdentifier, DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits, TokenMetadata } from 'vs/editor/common/modes';
import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry'; import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry';
import { ILanguageSelection } from 'vs/editor/common/services/modeService'; import { ILanguageSelection } from 'vs/editor/common/services/modeService';
import { IModelService } from 'vs/editor/common/services/modelService'; import { IModelService } from 'vs/editor/common/services/modelService';
@ -498,23 +498,23 @@ class SemanticColoringFeature extends Disposable {
class SemanticStyling extends Disposable { class SemanticStyling extends Disposable {
private _caches: WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>; private _caches: WeakMap<DocumentSemanticTokensProvider, SemanticColoringProviderStyling>;
constructor( constructor(
private readonly _themeService: IThemeService, private readonly _themeService: IThemeService,
private readonly _logService: ILogService private readonly _logService: ILogService
) { ) {
super(); super();
this._caches = new WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>(); this._caches = new WeakMap<DocumentSemanticTokensProvider, SemanticColoringProviderStyling>();
if (this._themeService) { if (this._themeService) {
// workaround for tests which use undefined... :/ // workaround for tests which use undefined... :/
this._register(this._themeService.onThemeChange(() => { this._register(this._themeService.onThemeChange(() => {
this._caches = new WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>(); this._caches = new WeakMap<DocumentSemanticTokensProvider, SemanticColoringProviderStyling>();
})); }));
} }
} }
public get(provider: SemanticTokensProvider): SemanticColoringProviderStyling { public get(provider: DocumentSemanticTokensProvider): SemanticColoringProviderStyling {
if (!this._caches.has(provider)) { if (!this._caches.has(provider)) {
this._caches.set(provider, new SemanticColoringProviderStyling(provider.getLegend(), this._themeService, this._logService)); this._caches.set(provider, new SemanticColoringProviderStyling(provider.getLegend(), this._themeService, this._logService));
} }
@ -676,13 +676,13 @@ const enum SemanticColoringConstants {
class SemanticTokensResponse { class SemanticTokensResponse {
constructor( constructor(
private readonly _provider: SemanticTokensProvider, private readonly _provider: DocumentSemanticTokensProvider,
public readonly resultId: string | undefined, public readonly resultId: string | undefined,
public readonly data: Uint32Array public readonly data: Uint32Array
) { } ) { }
public dispose(): void { public dispose(): void {
this._provider.releaseSemanticTokens(this.resultId); this._provider.releaseDocumentSemanticTokens(this.resultId);
} }
} }
@ -710,7 +710,7 @@ class ModelSemanticColoring extends Disposable {
this._fetchSemanticTokens.schedule(); this._fetchSemanticTokens.schedule();
} }
})); }));
this._register(SemanticTokensProviderRegistry.onDidChange(e => this._fetchSemanticTokens.schedule())); this._register(DocumentSemanticTokensProviderRegistry.onDidChange(e => this._fetchSemanticTokens.schedule()));
if (themeService) { if (themeService) {
// workaround for tests which use undefined... :/ // workaround for tests which use undefined... :/
this._register(themeService.onThemeChange(_ => { this._register(themeService.onThemeChange(_ => {
@ -756,7 +756,7 @@ class ModelSemanticColoring extends Disposable {
const styling = this._semanticStyling.get(provider); const styling = this._semanticStyling.get(provider);
const lastResultId = this._currentResponse ? this._currentResponse.resultId || null : null; const lastResultId = this._currentResponse ? this._currentResponse.resultId || null : null;
const request = Promise.resolve(provider.provideSemanticTokens(this._model, lastResultId, null, this._currentRequestCancellationTokenSource.token)); const request = Promise.resolve(provider.provideDocumentSemanticTokens(this._model, lastResultId, this._currentRequestCancellationTokenSource.token));
request.then((res) => { request.then((res) => {
this._currentRequestCancellationTokenSource = null; this._currentRequestCancellationTokenSource = null;
@ -784,7 +784,7 @@ class ModelSemanticColoring extends Disposable {
} }
} }
private _setSemanticTokens(provider: SemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticColoringProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void { private _setSemanticTokens(provider: DocumentSemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticColoringProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void {
const currentResponse = this._currentResponse; const currentResponse = this._currentResponse;
if (this._currentResponse) { if (this._currentResponse) {
this._currentResponse.dispose(); this._currentResponse.dispose();
@ -793,7 +793,7 @@ class ModelSemanticColoring extends Disposable {
if (this._isDisposed) { if (this._isDisposed) {
// disposed! // disposed!
if (provider && tokens) { if (provider && tokens) {
provider.releaseSemanticTokens(tokens.resultId); provider.releaseDocumentSemanticTokens(tokens.resultId);
} }
return; return;
} }
@ -954,8 +954,8 @@ class ModelSemanticColoring extends Disposable {
this._model.setSemanticTokens(null); this._model.setSemanticTokens(null);
} }
private _getSemanticColoringProvider(): SemanticTokensProvider | null { private _getSemanticColoringProvider(): DocumentSemanticTokensProvider | null {
const result = SemanticTokensProviderRegistry.ordered(this._model); const result = DocumentSemanticTokensProviderRegistry.ordered(this._model);
return (result.length > 0 ? result[0] : null); return (result.length > 0 ? result[0] : null);
} }
} }

11
src/vs/monaco.d.ts vendored
View file

@ -5948,10 +5948,15 @@ declare namespace monaco.languages {
readonly edits: SemanticTokensEdit[]; readonly edits: SemanticTokensEdit[];
} }
export interface SemanticTokensProvider { export interface DocumentSemanticTokensProvider {
getLegend(): SemanticTokensLegend; getLegend(): SemanticTokensLegend;
provideSemanticTokens(model: editor.ITextModel, lastResultId: string | null, ranges: Range[] | null, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>; provideDocumentSemanticTokens(model: editor.ITextModel, lastResultId: string | null, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
releaseSemanticTokens(resultId: string | undefined): void; releaseDocumentSemanticTokens(resultId: string | undefined): void;
}
export interface DocumentRangeSemanticTokensProvider {
getLegend(): SemanticTokensLegend;
provideDocumentRangeSemanticTokens(model: editor.ITextModel, range: Range, token: CancellationToken): ProviderResult<SemanticTokens>;
} }
export interface ILanguageExtensionPoint { export interface ILanguageExtensionPoint {

100
src/vs/vscode.proposed.d.ts vendored
View file

@ -180,8 +180,7 @@ declare module 'vscode' {
/** /**
* The result id of the tokens. * The result id of the tokens.
* *
* On a next call to `provideSemanticTokens`, if VS Code still holds in memory this result, * This is the id that will be passed to `DocumentSemanticTokensProvider.provideDocumentSemanticTokensEdits` (if implemented).
* the result id will be passed in as `SemanticTokensRequestOptions.previousResultId`.
*/ */
readonly resultId?: string; readonly resultId?: string;
readonly data: Uint32Array; readonly data: Uint32Array;
@ -193,8 +192,7 @@ declare module 'vscode' {
/** /**
* The result id of the tokens. * The result id of the tokens.
* *
* On a next call to `provideSemanticTokens`, if VS Code still holds in memory this result, * This is the id that will be passed to `DocumentSemanticTokensProvider.provideDocumentSemanticTokensEdits` (if implemented).
* the result id will be passed in as `SemanticTokensRequestOptions.previousResultId`.
*/ */
readonly resultId?: string; readonly resultId?: string;
readonly edits: SemanticTokensEdit[]; readonly edits: SemanticTokensEdit[];
@ -210,21 +208,11 @@ declare module 'vscode' {
constructor(start: number, deleteCount: number, data?: Uint32Array); constructor(start: number, deleteCount: number, data?: Uint32Array);
} }
export interface SemanticTokensRequestOptions {
readonly ranges?: readonly Range[];
/**
* The previous result id that the editor still holds in memory.
*
* Only when this is set it is safe for a `SemanticTokensProvider` to return `SemanticTokensEdits`.
*/
readonly previousResultId?: string;
}
/** /**
* The semantic tokens provider interface defines the contract between extensions and * The document semantic tokens provider interface defines the contract between extensions and
* semantic tokens. * semantic tokens.
*/ */
export interface SemanticTokensProvider { export interface DocumentSemanticTokensProvider {
/** /**
* A file can contain many tokens, perhaps even hundreds of thousands of tokens. Therefore, to improve * A file can contain many tokens, perhaps even hundreds of thousands of tokens. Therefore, to improve
* the memory consumption around describing semantic tokens, we have decided to avoid allocating an object * the memory consumption around describing semantic tokens, we have decided to avoid allocating an object
@ -232,21 +220,18 @@ declare module 'vscode' {
* of each token is expressed relative to the token before it because most tokens remain stable relative to * of each token is expressed relative to the token before it because most tokens remain stable relative to
* each other when edits are made in a file. * each other when edits are made in a file.
* *
*
* --- * ---
* In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following fields: * In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following array indices:
* - at index `5*i` - `deltaLine`: token line number, relative to the previous token * - at index `5*i` - `deltaLine`: token line number, relative to the previous token
* - at index `5*i+1` - `deltaStart`: token start character, relative to the previous token (relative to 0 or the previous token's start if they are on the same line) * - at index `5*i+1` - `deltaStart`: token start character, relative to the previous token (relative to 0 or the previous token's start if they are on the same line)
* - at index `5*i+2` - `length`: the length of the token. A token cannot be multiline. * - at index `5*i+2` - `length`: the length of the token. A token cannot be multiline.
* - at index `5*i+3` - `tokenType`: will be looked up in `SemanticTokensLegend.tokenTypes` * - at index `5*i+3` - `tokenType`: will be looked up in `SemanticTokensLegend.tokenTypes`
* - at index `5*i+4` - `tokenModifiers`: each set bit will be looked up in `SemanticTokensLegend.tokenModifiers` * - at index `5*i+4` - `tokenModifiers`: each set bit will be looked up in `SemanticTokensLegend.tokenModifiers`
* *
*
*
* --- * ---
* ### How to encode tokens * ### How to encode tokens
* *
* Here is an example for encoding a file with 3 tokens: * Here is an example for encoding a file with 3 tokens in a uint32 array:
* ``` * ```
* { line: 2, startChar: 5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] }, * { line: 2, startChar: 5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] },
* { line: 2, startChar: 10, length: 4, tokenType: "types", tokenModifiers: [] }, * { line: 2, startChar: 10, length: 4, tokenType: "types", tokenModifiers: [] },
@ -285,8 +270,12 @@ declare module 'vscode' {
* // 1st token, 2nd token, 3rd token * // 1st token, 2nd token, 3rd token
* [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] * [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ]
* ``` * ```
* */
* provideDocumentSemanticTokens(document: TextDocument, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
/**
* Instead of always returning all the tokens in a file, it is possible for a `DocumentSemanticTokensProvider` to implement
* this method (`updateSemanticTokens`) and then return incremental updates to the previously provided semantic tokens.
* *
* --- * ---
* ### How tokens change when the document changes * ### How tokens change when the document changes
@ -307,8 +296,8 @@ declare module 'vscode' {
* ``` * ```
* It is possible to express these new tokens in terms of an edit applied to the previous tokens: * It is possible to express these new tokens in terms of an edit applied to the previous tokens:
* ``` * ```
* [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] * [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] // old tokens
* [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] // new tokens
* *
* edit: { start: 0, deleteCount: 1, data: [3] } // replace integer at offset 0 with 3 * edit: { start: 0, deleteCount: 1, data: [3] } // replace integer at offset 0 with 3
* ``` * ```
@ -327,51 +316,56 @@ declare module 'vscode' {
* ``` * ```
* Again, it is possible to express these new tokens in terms of an edit applied to the previous tokens: * Again, it is possible to express these new tokens in terms of an edit applied to the previous tokens:
* ``` * ```
* [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] * [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] // old tokens
* [ 3,5,3,0,3, 0,5,4,1,0, 1,3,5,0,2, 2,2,7,2,0, ] * [ 3,5,3,0,3, 0,5,4,1,0, 1,3,5,0,2, 2,2,7,2,0, ] // new tokens
* *
* edit: { start: 10, deleteCount: 1, data: [1,3,5,0,2,2] } // replace integer at offset 10 with [1,3,5,0,2,2] * edit: { start: 10, deleteCount: 1, data: [1,3,5,0,2,2] } // replace integer at offset 10 with [1,3,5,0,2,2]
* ``` * ```
* *
* * *NOTE*: When doing edits, it is possible that multiple edits occur until VS Code decides to invoke the semantic tokens provider.
* * *NOTE*: If the provider cannot compute `SemanticTokensEdits`, it can "give up" and return all the tokens in the document again.
* --- * *NOTE*: All edits in `SemanticTokensEdits` contain indices in the old integers array, so they all refer to the previous result state.
* ### When to return `SemanticTokensEdits`
*
* When doing edits, it is possible that multiple edits occur until VS Code decides to invoke the semantic tokens provider.
* In principle, each call to `provideSemanticTokens` can return a full representations of the semantic tokens, and that would
* be a perfectly reasonable semantic tokens provider implementation.
*
* However, when having a language server running in a separate process, transferring all the tokens between processes
* might be slow, so VS Code allows to return the new tokens expressed in terms of multiple edits applied to the previous
* tokens.
*
* To clearly define what "previous tokens" means, it is possible to return a `resultId` with the semantic tokens. If the
* editor still has in memory the previous result, the editor will pass in options the previous `resultId` at
* `SemanticTokensRequestOptions.previousResultId`. Only when the editor passes in the previous `resultId`, it is allowed
* that a semantic tokens provider returns the new tokens expressed as edits to be applied to the previous result. Even in this
* case, the semantic tokens provider needs to return a new `resultId` that will identify these new tokens as a basis
* for the next request.
*
* *NOTE 1*: It is illegal to return `SemanticTokensEdits` if `options.previousResultId` is not set.
* *NOTE 2*: All edits in `SemanticTokensEdits` contain indices in the old integers array, so they all refer to the previous result state.
*/ */
provideSemanticTokens(document: TextDocument, options: SemanticTokensRequestOptions, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>; provideDocumentSemanticTokensEdits?(document: TextDocument, previousResultId: string, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
}
/**
* The document range semantic tokens provider interface defines the contract between extensions and
* semantic tokens.
*/
export interface DocumentRangeSemanticTokensProvider {
/**
* See [provideDocumentSemanticTokens](#DocumentSemanticTokensProvider.provideDocumentSemanticTokens).
*/
provideDocumentRangeSemanticTokens(document: TextDocument, range: Range, token: CancellationToken): ProviderResult<SemanticTokens>;
} }
export namespace languages { export namespace languages {
/** /**
* Register a semantic tokens provider. * Register a semantic tokens provider for a whole document.
* *
* Multiple providers can be registered for a language. In that case providers are sorted * Multiple providers can be registered for a language. In that case providers are sorted
* by their [score](#languages.match) and the best-matching provider is used. Failure * by their [score](#languages.match) and the best-matching provider is used. Failure
* of the selected provider will cause a failure of the whole operation. * of the selected provider will cause a failure of the whole operation.
* *
* @param selector A selector that defines the documents this provider is applicable to. * @param selector A selector that defines the documents this provider is applicable to.
* @param provider A semantic tokens provider. * @param provider A document semantic tokens provider.
* @return A [disposable](#Disposable) that unregisters this provider when being disposed. * @return A [disposable](#Disposable) that unregisters this provider when being disposed.
*/ */
export function registerSemanticTokensProvider(selector: DocumentSelector, provider: SemanticTokensProvider, legend: SemanticTokensLegend): Disposable; export function registerDocumentSemanticTokensProvider(selector: DocumentSelector, provider: DocumentSemanticTokensProvider, legend: SemanticTokensLegend): Disposable;
/**
* Register a semantic tokens provider for a document range.
*
* Multiple providers can be registered for a language. In that case providers are sorted
* by their [score](#languages.match) and the best-matching provider is used. Failure
* of the selected provider will cause a failure of the whole operation.
*
* @param selector A selector that defines the documents this provider is applicable to.
* @param provider A document range semantic tokens provider.
* @return A [disposable](#Disposable) that unregisters this provider when being disposed.
*/
export function registerDocumentRangeSemanticTokensProvider(selector: DocumentSelector, provider: DocumentRangeSemanticTokensProvider, legend: SemanticTokensLegend): Disposable;
} }
//#endregion //#endregion

50
src/vs/workbench/api/browser/mainThreadLanguageFeatures.ts
View file

@ -327,8 +327,12 @@ export class MainThreadLanguageFeatures implements MainThreadLanguageFeaturesSha
// --- semantic tokens // --- semantic tokens
$registerSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void { $registerDocumentSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void {
this._registrations.set(handle, modes.SemanticTokensProviderRegistry.register(selector, new MainThreadSemanticTokensProvider(this._proxy, handle, legend))); this._registrations.set(handle, modes.DocumentSemanticTokensProviderRegistry.register(selector, new MainThreadDocumentSemanticTokensProvider(this._proxy, handle, legend)));
}
$registerDocumentRangeSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void {
this._registrations.set(handle, modes.DocumentRangeSemanticTokensProviderRegistry.register(selector, new MainThreadDocumentRangeSemanticTokensProvider(this._proxy, handle, legend)));
} }
// --- suggest // --- suggest
@ -607,7 +611,7 @@ export class MainThreadLanguageFeatures implements MainThreadLanguageFeaturesSha
} }
export class MainThreadSemanticTokensProvider implements modes.SemanticTokensProvider { export class MainThreadDocumentSemanticTokensProvider implements modes.DocumentSemanticTokensProvider {
constructor( constructor(
private readonly _proxy: ExtHostLanguageFeaturesShape, private readonly _proxy: ExtHostLanguageFeaturesShape,
@ -616,9 +620,9 @@ export class MainThreadSemanticTokensProvider implements modes.SemanticTokensPro
) { ) {
} }
public releaseSemanticTokens(resultId: string | undefined): void { public releaseDocumentSemanticTokens(resultId: string | undefined): void {
if (resultId) { if (resultId) {
this._proxy.$releaseSemanticTokens(this._handle, parseInt(resultId, 10)); this._proxy.$releaseDocumentSemanticTokens(this._handle, parseInt(resultId, 10));
} }
} }
@ -626,9 +630,9 @@ export class MainThreadSemanticTokensProvider implements modes.SemanticTokensPro
return this._legend; return this._legend;
} }
async provideSemanticTokens(model: ITextModel, lastResultId: string | null, ranges: EditorRange[] | null, token: CancellationToken): Promise<modes.SemanticTokens | modes.SemanticTokensEdits | null> { async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise<modes.SemanticTokens | modes.SemanticTokensEdits | null> {
const nLastResultId = lastResultId ? parseInt(lastResultId, 10) : 0; const nLastResultId = lastResultId ? parseInt(lastResultId, 10) : 0;
const encodedDto = await this._proxy.$provideSemanticTokens(this._handle, model.uri, ranges, nLastResultId, token); const encodedDto = await this._proxy.$provideDocumentSemanticTokens(this._handle, model.uri, nLastResultId, token);
if (!encodedDto) { if (!encodedDto) {
return null; return null;
} }
@ -648,3 +652,35 @@ export class MainThreadSemanticTokensProvider implements modes.SemanticTokensPro
}; };
} }
} }
export class MainThreadDocumentRangeSemanticTokensProvider implements modes.DocumentRangeSemanticTokensProvider {
constructor(
private readonly _proxy: ExtHostLanguageFeaturesShape,
private readonly _handle: number,
private readonly _legend: modes.SemanticTokensLegend,
) {
}
public getLegend(): modes.SemanticTokensLegend {
return this._legend;
}
async provideDocumentRangeSemanticTokens(model: ITextModel, range: EditorRange, token: CancellationToken): Promise<modes.SemanticTokens | null> {
const encodedDto = await this._proxy.$provideDocumentRangeSemanticTokens(this._handle, model.uri, range, token);
if (!encodedDto) {
return null;
}
if (token.isCancellationRequested) {
return null;
}
const dto = decodeSemanticTokensDto(encodedDto);
if (dto.type === 'full') {
return {
resultId: String(dto.id),
data: dto.data
};
}
throw new Error(`Unexpected`);
}
}

8
src/vs/workbench/api/common/extHost.api.impl.ts
View file

@ -379,9 +379,13 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
registerOnTypeFormattingEditProvider(selector: vscode.DocumentSelector, provider: vscode.OnTypeFormattingEditProvider, firstTriggerCharacter: string, ...moreTriggerCharacters: string[]): vscode.Disposable { registerOnTypeFormattingEditProvider(selector: vscode.DocumentSelector, provider: vscode.OnTypeFormattingEditProvider, firstTriggerCharacter: string, ...moreTriggerCharacters: string[]): vscode.Disposable {
return extHostLanguageFeatures.registerOnTypeFormattingEditProvider(extension, checkSelector(selector), provider, [firstTriggerCharacter].concat(moreTriggerCharacters)); return extHostLanguageFeatures.registerOnTypeFormattingEditProvider(extension, checkSelector(selector), provider, [firstTriggerCharacter].concat(moreTriggerCharacters));
}, },
registerSemanticTokensProvider(selector: vscode.DocumentSelector, provider: vscode.SemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable { registerDocumentSemanticTokensProvider(selector: vscode.DocumentSelector, provider: vscode.DocumentSemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable {
checkProposedApiEnabled(extension); checkProposedApiEnabled(extension);
return extHostLanguageFeatures.registerSemanticTokensProvider(extension, checkSelector(selector), provider, legend); return extHostLanguageFeatures.registerDocumentSemanticTokensProvider(extension, checkSelector(selector), provider, legend);
},
registerDocumentRangeSemanticTokensProvider(selector: vscode.DocumentSelector, provider: vscode.DocumentRangeSemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable {
checkProposedApiEnabled(extension);
return extHostLanguageFeatures.registerDocumentRangeSemanticTokensProvider(extension, checkSelector(selector), provider, legend);
}, },
registerSignatureHelpProvider(selector: vscode.DocumentSelector, provider: vscode.SignatureHelpProvider, firstItem?: string | vscode.SignatureHelpProviderMetadata, ...remaining: string[]): vscode.Disposable { registerSignatureHelpProvider(selector: vscode.DocumentSelector, provider: vscode.SignatureHelpProvider, firstItem?: string | vscode.SignatureHelpProviderMetadata, ...remaining: string[]): vscode.Disposable {
if (typeof firstItem === 'object') { if (typeof firstItem === 'object') {

8
src/vs/workbench/api/common/extHost.protocol.ts
View file

@ -358,7 +358,8 @@ export interface MainThreadLanguageFeaturesShape extends IDisposable {
$registerOnTypeFormattingSupport(handle: number, selector: IDocumentFilterDto[], autoFormatTriggerCharacters: string[], extensionId: ExtensionIdentifier): void; $registerOnTypeFormattingSupport(handle: number, selector: IDocumentFilterDto[], autoFormatTriggerCharacters: string[], extensionId: ExtensionIdentifier): void;
$registerNavigateTypeSupport(handle: number): void; $registerNavigateTypeSupport(handle: number): void;
$registerRenameSupport(handle: number, selector: IDocumentFilterDto[], supportsResolveInitialValues: boolean): void; $registerRenameSupport(handle: number, selector: IDocumentFilterDto[], supportsResolveInitialValues: boolean): void;
$registerSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void; $registerDocumentSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void;
$registerDocumentRangeSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void;
$registerSuggestSupport(handle: number, selector: IDocumentFilterDto[], triggerCharacters: string[], supportsResolveDetails: boolean, extensionId: ExtensionIdentifier): void; $registerSuggestSupport(handle: number, selector: IDocumentFilterDto[], triggerCharacters: string[], supportsResolveDetails: boolean, extensionId: ExtensionIdentifier): void;
$registerSignatureHelpProvider(handle: number, selector: IDocumentFilterDto[], metadata: ISignatureHelpProviderMetadataDto): void; $registerSignatureHelpProvider(handle: number, selector: IDocumentFilterDto[], metadata: ISignatureHelpProviderMetadataDto): void;
$registerDocumentLinkProvider(handle: number, selector: IDocumentFilterDto[], supportsResolve: boolean): void; $registerDocumentLinkProvider(handle: number, selector: IDocumentFilterDto[], supportsResolve: boolean): void;
@ -1188,8 +1189,9 @@ export interface ExtHostLanguageFeaturesShape {
$releaseWorkspaceSymbols(handle: number, id: number): void; $releaseWorkspaceSymbols(handle: number, id: number): void;
$provideRenameEdits(handle: number, resource: UriComponents, position: IPosition, newName: string, token: CancellationToken): Promise<IWorkspaceEditDto | undefined>; $provideRenameEdits(handle: number, resource: UriComponents, position: IPosition, newName: string, token: CancellationToken): Promise<IWorkspaceEditDto | undefined>;
$resolveRenameLocation(handle: number, resource: UriComponents, position: IPosition, token: CancellationToken): Promise<modes.RenameLocation | undefined>; $resolveRenameLocation(handle: number, resource: UriComponents, position: IPosition, token: CancellationToken): Promise<modes.RenameLocation | undefined>;
$provideSemanticTokens(handle: number, resource: UriComponents, ranges: IRange[] | null, previousResultId: number, token: CancellationToken): Promise<VSBuffer | null>; $provideDocumentSemanticTokens(handle: number, resource: UriComponents, previousResultId: number, token: CancellationToken): Promise<VSBuffer | null>;
$releaseSemanticTokens(handle: number, semanticColoringResultId: number): void; $releaseDocumentSemanticTokens(handle: number, semanticColoringResultId: number): void;
$provideDocumentRangeSemanticTokens(handle: number, resource: UriComponents, range: IRange, token: CancellationToken): Promise<VSBuffer | null>;
$provideCompletionItems(handle: number, resource: UriComponents, position: IPosition, context: modes.CompletionContext, token: CancellationToken): Promise<ISuggestResultDto | undefined>; $provideCompletionItems(handle: number, resource: UriComponents, position: IPosition, context: modes.CompletionContext, token: CancellationToken): Promise<ISuggestResultDto | undefined>;
$resolveCompletionItem(handle: number, resource: UriComponents, position: IPosition, id: ChainedCacheId, token: CancellationToken): Promise<ISuggestDataDto | undefined>; $resolveCompletionItem(handle: number, resource: UriComponents, position: IPosition, id: ChainedCacheId, token: CancellationToken): Promise<ISuggestDataDto | undefined>;
$releaseCompletionItems(handle: number, id: number): void; $releaseCompletionItems(handle: number, id: number): void;

90
src/vs/workbench/api/common/extHostLanguageFeatures.ts
View file

@ -629,37 +629,38 @@ class SemanticTokensPreviousResult {
) { } ) { }
} }
export class SemanticTokensAdapter { export class DocumentSemanticTokensAdapter {
private readonly _previousResults: Map<number, SemanticTokensPreviousResult>; private readonly _previousResults: Map<number, SemanticTokensPreviousResult>;
private _nextResultId = 1; private _nextResultId = 1;
constructor( constructor(
private readonly _documents: ExtHostDocuments, private readonly _documents: ExtHostDocuments,
private readonly _provider: vscode.SemanticTokensProvider, private readonly _provider: vscode.DocumentSemanticTokensProvider,
) { ) {
this._previousResults = new Map<number, SemanticTokensPreviousResult>(); this._previousResults = new Map<number, SemanticTokensPreviousResult>();
} }
provideSemanticTokens(resource: URI, ranges: IRange[] | null, previousResultId: number, token: CancellationToken): Promise<VSBuffer | null> { provideDocumentSemanticTokens(resource: URI, previousResultId: number, token: CancellationToken): Promise<VSBuffer | null> {
const doc = this._documents.getDocument(resource); const doc = this._documents.getDocument(resource);
const previousResult = (previousResultId !== 0 ? this._previousResults.get(previousResultId) : null); const previousResult = (previousResultId !== 0 ? this._previousResults.get(previousResultId) : null);
const opts: vscode.SemanticTokensRequestOptions = { return asPromise(() => {
ranges: (Array.isArray(ranges) && ranges.length > 0 ? ranges.map<Range>(typeConvert.Range.to) : undefined), if (previousResult && typeof previousResult.resultId === 'string' && typeof this._provider.provideDocumentSemanticTokensEdits === 'function') {
previousResultId: (previousResult ? previousResult.resultId : undefined) return this._provider.provideDocumentSemanticTokensEdits(doc, previousResult.resultId, token);
};
return asPromise(() => this._provider.provideSemanticTokens(doc, opts, token)).then(value => {
if (!value) {
return null;
} }
return this._provider.provideDocumentSemanticTokens(doc, token);
}).then(value => {
if (previousResult) { if (previousResult) {
this._previousResults.delete(previousResultId); this._previousResults.delete(previousResultId);
} }
return this._send(SemanticTokensAdapter._convertToEdits(previousResult, value), value); if (!value) {
return null;
}
return this._send(DocumentSemanticTokensAdapter._convertToEdits(previousResult, value), value);
}); });
} }
async releaseSemanticColoring(semanticColoringResultId: number): Promise<void> { async releaseDocumentSemanticColoring(semanticColoringResultId: number): Promise<void> {
this._previousResults.delete(semanticColoringResultId); this._previousResults.delete(semanticColoringResultId);
} }
@ -672,7 +673,7 @@ export class SemanticTokensAdapter {
} }
private static _convertToEdits(previousResult: SemanticTokensPreviousResult | null | undefined, newResult: vscode.SemanticTokens | vscode.SemanticTokensEdits): vscode.SemanticTokens | vscode.SemanticTokensEdits { private static _convertToEdits(previousResult: SemanticTokensPreviousResult | null | undefined, newResult: vscode.SemanticTokens | vscode.SemanticTokensEdits): vscode.SemanticTokens | vscode.SemanticTokensEdits {
if (!SemanticTokensAdapter._isSemanticTokens(newResult)) { if (!DocumentSemanticTokensAdapter._isSemanticTokens(newResult)) {
return newResult; return newResult;
} }
if (!previousResult || !previousResult.tokens) { if (!previousResult || !previousResult.tokens) {
@ -708,7 +709,7 @@ export class SemanticTokensAdapter {
} }
private _send(value: vscode.SemanticTokens | vscode.SemanticTokensEdits, original: vscode.SemanticTokens | vscode.SemanticTokensEdits): VSBuffer | null { private _send(value: vscode.SemanticTokens | vscode.SemanticTokensEdits, original: vscode.SemanticTokens | vscode.SemanticTokensEdits): VSBuffer | null {
if (SemanticTokensAdapter._isSemanticTokens(value)) { if (DocumentSemanticTokensAdapter._isSemanticTokens(value)) {
const myId = this._nextResultId++; const myId = this._nextResultId++;
this._previousResults.set(myId, new SemanticTokensPreviousResult(value.resultId, value.data)); this._previousResults.set(myId, new SemanticTokensPreviousResult(value.resultId, value.data));
return encodeSemanticTokensDto({ return encodeSemanticTokensDto({
@ -718,9 +719,9 @@ export class SemanticTokensAdapter {
}); });
} }
if (SemanticTokensAdapter._isSemanticTokensEdits(value)) { if (DocumentSemanticTokensAdapter._isSemanticTokensEdits(value)) {
const myId = this._nextResultId++; const myId = this._nextResultId++;
if (SemanticTokensAdapter._isSemanticTokens(original)) { if (DocumentSemanticTokensAdapter._isSemanticTokens(original)) {
// store the original // store the original
this._previousResults.set(myId, new SemanticTokensPreviousResult(original.resultId, original.data)); this._previousResults.set(myId, new SemanticTokensPreviousResult(original.resultId, original.data));
} else { } else {
@ -737,6 +738,33 @@ export class SemanticTokensAdapter {
} }
} }
export class DocumentRangeSemanticTokensAdapter {
constructor(
private readonly _documents: ExtHostDocuments,
private readonly _provider: vscode.DocumentRangeSemanticTokensProvider,
) {
}
provideDocumentRangeSemanticTokens(resource: URI, range: IRange, token: CancellationToken): Promise<VSBuffer | null> {
const doc = this._documents.getDocument(resource);
return asPromise(() => this._provider.provideDocumentRangeSemanticTokens(doc, typeConvert.Range.to(range), token)).then(value => {
if (!value) {
return null;
}
return this._send(value);
});
}
private _send(value: vscode.SemanticTokens): VSBuffer | null {
return encodeSemanticTokensDto({
id: 0,
type: 'full',
data: value.data
});
}
}
class SuggestAdapter { class SuggestAdapter {
static supportsResolving(provider: vscode.CompletionItemProvider): boolean { static supportsResolving(provider: vscode.CompletionItemProvider): boolean {
@ -1324,9 +1352,9 @@ class CallHierarchyAdapter {
type Adapter = DocumentSymbolAdapter | CodeLensAdapter | DefinitionAdapter | HoverAdapter type Adapter = DocumentSymbolAdapter | CodeLensAdapter | DefinitionAdapter | HoverAdapter
| DocumentHighlightAdapter | ReferenceAdapter | CodeActionAdapter | DocumentFormattingAdapter | DocumentHighlightAdapter | ReferenceAdapter | CodeActionAdapter | DocumentFormattingAdapter
| RangeFormattingAdapter | OnTypeFormattingAdapter | NavigateTypeAdapter | RenameAdapter | RangeFormattingAdapter | OnTypeFormattingAdapter | NavigateTypeAdapter | RenameAdapter
| SemanticTokensAdapter | SuggestAdapter | SignatureHelpAdapter | LinkProviderAdapter | SuggestAdapter | SignatureHelpAdapter | LinkProviderAdapter | ImplementationAdapter
| ImplementationAdapter | TypeDefinitionAdapter | ColorProviderAdapter | FoldingProviderAdapter | TypeDefinitionAdapter | ColorProviderAdapter | FoldingProviderAdapter | DeclarationAdapter
| DeclarationAdapter | SelectionRangeAdapter | CallHierarchyAdapter; | SelectionRangeAdapter | CallHierarchyAdapter | DocumentSemanticTokensAdapter | DocumentRangeSemanticTokensAdapter;
class AdapterData { class AdapterData {
constructor( constructor(
@ -1657,18 +1685,28 @@ export class ExtHostLanguageFeatures implements extHostProtocol.ExtHostLanguageF
//#region semantic coloring //#region semantic coloring
registerSemanticTokensProvider(extension: IExtensionDescription, selector: vscode.DocumentSelector, provider: vscode.SemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable { registerDocumentSemanticTokensProvider(extension: IExtensionDescription, selector: vscode.DocumentSelector, provider: vscode.DocumentSemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable {
const handle = this._addNewAdapter(new SemanticTokensAdapter(this._documents, provider), extension); const handle = this._addNewAdapter(new DocumentSemanticTokensAdapter(this._documents, provider), extension);
this._proxy.$registerSemanticTokensProvider(handle, this._transformDocumentSelector(selector), legend); this._proxy.$registerDocumentSemanticTokensProvider(handle, this._transformDocumentSelector(selector), legend);
return this._createDisposable(handle); return this._createDisposable(handle);
} }
$provideSemanticTokens(handle: number, resource: UriComponents, ranges: IRange[] | null, previousResultId: number, token: CancellationToken): Promise<VSBuffer | null> { $provideDocumentSemanticTokens(handle: number, resource: UriComponents, previousResultId: number, token: CancellationToken): Promise<VSBuffer | null> {
return this._withAdapter(handle, SemanticTokensAdapter, adapter => adapter.provideSemanticTokens(URI.revive(resource), ranges, previousResultId, token), null); return this._withAdapter(handle, DocumentSemanticTokensAdapter, adapter => adapter.provideDocumentSemanticTokens(URI.revive(resource), previousResultId, token), null);
} }
$releaseSemanticTokens(handle: number, semanticColoringResultId: number): void { $releaseDocumentSemanticTokens(handle: number, semanticColoringResultId: number): void {
this._withAdapter(handle, SemanticTokensAdapter, adapter => adapter.releaseSemanticColoring(semanticColoringResultId), undefined); this._withAdapter(handle, DocumentSemanticTokensAdapter, adapter => adapter.releaseDocumentSemanticColoring(semanticColoringResultId), undefined);
}
registerDocumentRangeSemanticTokensProvider(extension: IExtensionDescription, selector: vscode.DocumentSelector, provider: vscode.DocumentRangeSemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable {
const handle = this._addNewAdapter(new DocumentRangeSemanticTokensAdapter(this._documents, provider), extension);
this._proxy.$registerDocumentRangeSemanticTokensProvider(handle, this._transformDocumentSelector(selector), legend);
return this._createDisposable(handle);
}
$provideDocumentRangeSemanticTokens(handle: number, resource: UriComponents, range: IRange, token: CancellationToken): Promise<VSBuffer | null> {
return this._withAdapter(handle, DocumentRangeSemanticTokensAdapter, adapter => adapter.provideDocumentRangeSemanticTokens(URI.revive(resource), range, token), null);
} }
//#endregion //#endregion

7
src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts
View file

@ -16,7 +16,7 @@ import { EditorAction, ServicesAccessor, registerEditorAction, registerEditorCon
import { Position } from 'vs/editor/common/core/position'; import { Position } from 'vs/editor/common/core/position';
import { IEditorContribution } from 'vs/editor/common/editorCommon'; import { IEditorContribution } from 'vs/editor/common/editorCommon';
import { ITextModel } from 'vs/editor/common/model'; import { ITextModel } from 'vs/editor/common/model';
import { FontStyle, LanguageIdentifier, StandardTokenType, TokenMetadata, SemanticTokensProviderRegistry, SemanticTokensLegend, SemanticTokens } from 'vs/editor/common/modes'; import { FontStyle, LanguageIdentifier, StandardTokenType, TokenMetadata, DocumentSemanticTokensProviderRegistry, SemanticTokensLegend, SemanticTokens } from 'vs/editor/common/modes';
import { IModeService } from 'vs/editor/common/services/modeService'; import { IModeService } from 'vs/editor/common/services/modeService';
import { INotificationService } from 'vs/platform/notification/common/notification'; import { INotificationService } from 'vs/platform/notification/common/notification';
import { editorHoverBackground, editorHoverBorder } from 'vs/platform/theme/common/colorRegistry'; import { editorHoverBackground, editorHoverBorder } from 'vs/platform/theme/common/colorRegistry';
@ -411,11 +411,10 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
} }
private async _computeSemanticTokens(): Promise<SemanticTokensResult | null> { private async _computeSemanticTokens(): Promise<SemanticTokensResult | null> {
const tokenProviders = SemanticTokensProviderRegistry.ordered(this._model); const tokenProviders = DocumentSemanticTokensProviderRegistry.ordered(this._model);
if (tokenProviders.length) { if (tokenProviders.length) {
const provider = tokenProviders[0]; const provider = tokenProviders[0];
const range = this._model.getFullModelRange(); const tokens = await Promise.resolve(provider.provideDocumentSemanticTokens(this._model, null, this._currentRequestCancellationTokenSource.token));
const tokens = await Promise.resolve(provider.provideSemanticTokens(this._model, null, [range], this._currentRequestCancellationTokenSource.token));
if (this.isSemanticTokens(tokens)) { if (this.isSemanticTokens(tokens)) {
return { tokens, legend: provider.getLegend() }; return { tokens, legend: provider.getLegend() };
} }