maxmustermann/vscode
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

SemanticTokens - implement feedback received in API call:

Browse source 
- extract a separate DocumentRangeSemanticTokensProvider that deals with a document range
- extract a separate provideDocumentSemanticTokensEdits that deals with updating via SemanticTokensEdits a previous result
This commit is contained in:
Alex Dima 2020-01-16 17:53:24 +01:00
parent 430de16fef
commit 9e1d730cf2
No known key found for this signature in database
GPG key ID: 6E58D7B045760DA0
12 changed files with 272 additions and 176 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
extensions/html-language-features/client/src/htmlMain.ts
View file

@ -11,7 +11,7 @@ const localize = nls.loadMessageBundle();
import {
languages, ExtensionContext, IndentAction, Position, TextDocument, Range, CompletionItem, CompletionItemKind, SnippetString, workspace,
Disposable, FormattingOptions, CancellationToken, ProviderResult, TextEdit, CompletionContext, CompletionList, SemanticTokensLegend,
SemanticTokensProvider, SemanticTokens
DocumentSemanticTokensProvider, DocumentRangeSemanticTokensProvider, SemanticTokens
} from 'vscode';
import {
LanguageClient, LanguageClientOptions, ServerOptions, TransportKind, RequestType, TextDocumentPositionParams, DocumentRangeFormattingParams,
@ -153,18 +153,26 @@ export function activate(context: ExtensionContext) {
client.sendRequest(SemanticTokenLegendRequest.type).then(legend => {
if (legend) {
const provider: SemanticTokensProvider = {
provideSemanticTokens(doc, opts) {
const provider: DocumentSemanticTokensProvider & DocumentRangeSemanticTokensProvider = {
provideDocumentSemanticTokens(doc) {
const params: SemanticTokenParams = {
textDocument: client.code2ProtocolConverter.asTextDocumentIdentifier(doc),
ranges: opts.ranges?.map(r => client.code2ProtocolConverter.asRange(r))
};
return client.sendRequest(SemanticTokenRequest.type, params).then(data => {
return data && new SemanticTokens(new Uint32Array(data));
});
},
provideDocumentRangeSemanticTokens(doc, range) {
const params: SemanticTokenParams = {
textDocument: client.code2ProtocolConverter.asTextDocumentIdentifier(doc),
ranges: [client.code2ProtocolConverter.asRange(range)]
};
return client.sendRequest(SemanticTokenRequest.type, params).then(data => {
return data && new SemanticTokens(new Uint32Array(data));
});
}
};
toDispose.push(languages.registerSemanticTokensProvider(documentSelector, provider, new SemanticTokensLegend(legend.types, legend.modifiers)));
toDispose.push(languages.registerDocumentSemanticTokensProvider(documentSelector, provider, new SemanticTokensLegend(legend.types, legend.modifiers)));
}
});

106
extensions/typescript-language-features/src/features/semanticTokens.ts
View file

@ -16,17 +16,20 @@ const minTypeScriptVersion = API.fromVersionString(`${VersionRequirement.major}.
export function register(selector: vscode.DocumentSelector, client: ITypeScriptServiceClient) {
return new VersionDependentRegistration(client, minTypeScriptVersion, () => {
const provider = new SemanticTokensProvider(client);
return vscode.languages.registerSemanticTokensProvider(selector, provider, provider.getLegend());
const provider = new DocumentSemanticTokensProvider(client);
return vscode.Disposable.from(
vscode.languages.registerDocumentSemanticTokensProvider(selector, provider, provider.getLegend()),
vscode.languages.registerDocumentRangeSemanticTokensProvider(selector, provider, provider.getLegend()),
);
});
}
/**
* Prototype of a SemanticTokensProvider, relying on the experimental `encodedSemanticClassifications-full` request from the TypeScript server.
* Prototype of a DocumentSemanticTokensProvider, relying on the experimental `encodedSemanticClassifications-full` request from the TypeScript server.
* As the results retured by the TypeScript server are limited, we also add a Typescript plugin (typescript-vscode-sh-plugin) to enrich the returned token.
* See https://github.com/aeschli/typescript-vscode-sh-plugin.
*/
class SemanticTokensProvider implements vscode.SemanticTokensProvider {
class DocumentSemanticTokensProvider implements vscode.DocumentSemanticTokensProvider, vscode.DocumentRangeSemanticTokensProvider {
constructor(private readonly client: ITypeScriptServiceClient) {
}
@ -41,68 +44,65 @@ class SemanticTokensProvider implements vscode.SemanticTokensProvider {
return new vscode.SemanticTokensLegend(tokenTypes, tokenModifiers);
}
async provideSemanticTokens(document: vscode.TextDocument, options: vscode.SemanticTokensRequestOptions, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
async provideDocumentSemanticTokens(document: vscode.TextDocument, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
const file = this.client.toOpenedFilePath(document);
if (!file) {
return null;
}
return this._provideSemanticTokens(document, { file, start: 0, length: document.getText().length }, token);
}
async provideDocumentRangeSemanticTokens(document: vscode.TextDocument, range: vscode.Range, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
const file = this.client.toOpenedFilePath(document);
if (!file) {
return null;
}
const start = document.offsetAt(range.start);
const length = document.offsetAt(range.end) - start;
return this._provideSemanticTokens(document, { file, start, length }, token);
}
async _provideSemanticTokens(document: vscode.TextDocument, requestArg: ExperimentalProtocol.EncodedSemanticClassificationsRequestArgs, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
const file = this.client.toOpenedFilePath(document);
if (!file) {
return null;
}
const versionBeforeRequest = document.version;
const allTokenSpans: number[][] = [];
let requestArgs: ExperimentalProtocol.EncodedSemanticClassificationsRequestArgs[] = [];
if (options.ranges) {
requestArgs = options.ranges.map(r => { const start = document.offsetAt(r.start); const length = document.offsetAt(r.end) - start; return { file, start, length }; });
requestArgs = requestArgs.sort((a1, a2) => a1.start - a2.start);
} else {
requestArgs = [{ file, start: 0, length: document.getText().length }]; // full document
}
for (const requestArg of requestArgs) {
const response = await (this.client as ExperimentalProtocol.IExtendedTypeScriptServiceClient).execute('encodedSemanticClassifications-full', requestArg, token);
if (response.type === 'response' && response.body) {
allTokenSpans.push(response.body.spans);
} else {
return null;
}
}
const versionAfterRequest = document.version;
if (versionBeforeRequest !== versionAfterRequest) {
// A new request will come in soon...
const response = await (this.client as ExperimentalProtocol.IExtendedTypeScriptServiceClient).execute('encodedSemanticClassifications-full', requestArg, token);
if (response.type !== 'response' || !response.body) {
return null;
}
const tokenSpan = response.body.spans;
const builder = new vscode.SemanticTokensBuilder();
for (const tokenSpan of allTokenSpans) {
let i = 0;
while (i < tokenSpan.length) {
const offset = tokenSpan[i++];
const length = tokenSpan[i++];
const tsClassification = tokenSpan[i++];
let i = 0;
while (i < tokenSpan.length) {
const offset = tokenSpan[i++];
const length = tokenSpan[i++];
const tsClassification = tokenSpan[i++];
let tokenModifiers = 0;
let tokenType = getTokenTypeFromClassification(tsClassification);
if (tokenType !== undefined) {
// it's a classification as returned by the typescript-vscode-sh-plugin
tokenModifiers = getTokenModifierFromClassification(tsClassification);
} else {
// typescript-vscode-sh-plugin is not present
tokenType = tokenTypeMap[tsClassification];
if (tokenType === undefined) {
continue;
}
let tokenModifiers = 0;
let tokenType = getTokenTypeFromClassification(tsClassification);
if (tokenType !== undefined) {
// it's a classification as returned by the typescript-vscode-sh-plugin
tokenModifiers = getTokenModifierFromClassification(tsClassification);
} else {
// typescript-vscode-sh-plugin is not present
tokenType = tokenTypeMap[tsClassification];
if (tokenType === undefined) {
continue;
}
}
// we can use the document's range conversion methods because the result is at the same version as the document
const startPos = document.positionAt(offset);
const endPos = document.positionAt(offset + length);
// we can use the document's range conversion methods because the result is at the same version as the document
const startPos = document.positionAt(offset);
const endPos = document.positionAt(offset + length);
for (let line = startPos.line; line <= endPos.line; line++) {
const startCharacter = (line === startPos.line ? startPos.character : 0);
const endCharacter = (line === endPos.line ? endPos.character : document.lineAt(line).text.length);
builder.push(line, startCharacter, endCharacter - startCharacter, tokenType, tokenModifiers);
}
for (let line = startPos.line; line <= endPos.line; line++) {
const startCharacter = (line === startPos.line ? startPos.character : 0);
const endCharacter = (line === endPos.line ? endPos.character : document.lineAt(line).text.length);
builder.push(line, startCharacter, endCharacter - startCharacter, tokenType, tokenModifiers);
}
}
return new vscode.SemanticTokens(builder.build());

6
extensions/vscode-colorize-tests/src/colorizerTestMain.ts
View file

@ -15,8 +15,8 @@ export function activate(context: vscode.ExtensionContext): any {
const outputChannel = vscode.window.createOutputChannel('Semantic Tokens Test');
const semanticHighlightProvider: vscode.SemanticTokensProvider = {
provideSemanticTokens(document: vscode.TextDocument): vscode.ProviderResult<vscode.SemanticTokens> {
const documentSemanticHighlightProvider: vscode.DocumentSemanticTokensProvider = {
provideDocumentSemanticTokens(document: vscode.TextDocument): vscode.ProviderResult<vscode.SemanticTokens> {
const builder = new vscode.SemanticTokensBuilder();
function addToken(value: string, startLine: number, startCharacter: number, length: number) {
@ -61,6 +61,6 @@ export function activate(context: vscode.ExtensionContext): any {
};
context.subscriptions.push(vscode.languages.registerSemanticTokensProvider({ pattern: '**/*semantic-test.json' }, semanticHighlightProvider, legend));
context.subscriptions.push(vscode.languages.registerDocumentSemanticTokensProvider({ pattern: '**/*semantic-test.json' }, documentSemanticHighlightProvider, legend));
}

18
src/vs/editor/common/modes.ts
View file

@ -1500,10 +1500,15 @@ export interface SemanticTokensEdits {
readonly edits: SemanticTokensEdit[];
}
export interface SemanticTokensProvider {
export interface DocumentSemanticTokensProvider {
getLegend(): SemanticTokensLegend;
provideSemanticTokens(model: model.ITextModel, lastResultId: string | null, ranges: Range[] | null, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
releaseSemanticTokens(resultId: string | undefined): void;
provideDocumentSemanticTokens(model: model.ITextModel, lastResultId: string | null, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
releaseDocumentSemanticTokens(resultId: string | undefined): void;
}
export interface DocumentRangeSemanticTokensProvider {
getLegend(): SemanticTokensLegend;
provideDocumentRangeSemanticTokens(model: model.ITextModel, range: Range, token: CancellationToken): ProviderResult<SemanticTokens>;
}
// --- feature registries ------
@ -1611,7 +1616,12 @@ export const FoldingRangeProviderRegistry = new LanguageFeatureRegistry<FoldingR
/**
* @internal
*/
export const SemanticTokensProviderRegistry = new LanguageFeatureRegistry<SemanticTokensProvider>();
export const DocumentSemanticTokensProviderRegistry = new LanguageFeatureRegistry<DocumentSemanticTokensProvider>();
/**
* @internal
*/
export const DocumentRangeSemanticTokensProviderRegistry = new LanguageFeatureRegistry<DocumentRangeSemanticTokensProvider>();
/**
* @internal

26
src/vs/editor/common/services/modelServiceImpl.ts
View file

@ -14,7 +14,7 @@ import { Range } from 'vs/editor/common/core/range';
import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model';
import { TextModel, createTextBuffer } from 'vs/editor/common/model/textModel';
import { IModelLanguageChangedEvent, IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
import { LanguageIdentifier, SemanticTokensProviderRegistry, SemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits, TokenMetadata } from 'vs/editor/common/modes';
import { LanguageIdentifier, DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits, TokenMetadata } from 'vs/editor/common/modes';
import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry';
import { ILanguageSelection } from 'vs/editor/common/services/modeService';
import { IModelService } from 'vs/editor/common/services/modelService';
@ -498,23 +498,23 @@ class SemanticColoringFeature extends Disposable {
class SemanticStyling extends Disposable {
private _caches: WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>;
private _caches: WeakMap<DocumentSemanticTokensProvider, SemanticColoringProviderStyling>;
constructor(
private readonly _themeService: IThemeService,
private readonly _logService: ILogService
) {
super();
this._caches = new WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>();
this._caches = new WeakMap<DocumentSemanticTokensProvider, SemanticColoringProviderStyling>();
if (this._themeService) {
// workaround for tests which use undefined... :/
this._register(this._themeService.onThemeChange(() => {
this._caches = new WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>();
this._caches = new WeakMap<DocumentSemanticTokensProvider, SemanticColoringProviderStyling>();
}));
}
}
public get(provider: SemanticTokensProvider): SemanticColoringProviderStyling {
public get(provider: DocumentSemanticTokensProvider): SemanticColoringProviderStyling {
if (!this._caches.has(provider)) {
this._caches.set(provider, new SemanticColoringProviderStyling(provider.getLegend(), this._themeService, this._logService));
}
@ -676,13 +676,13 @@ const enum SemanticColoringConstants {
class SemanticTokensResponse {
constructor(
private readonly _provider: SemanticTokensProvider,
private readonly _provider: DocumentSemanticTokensProvider,
public readonly resultId: string | undefined,
public readonly data: Uint32Array
) { }
public dispose(): void {
this._provider.releaseSemanticTokens(this.resultId);
this._provider.releaseDocumentSemanticTokens(this.resultId);
}
}
@ -710,7 +710,7 @@ class ModelSemanticColoring extends Disposable {
this._fetchSemanticTokens.schedule();
}
}));
this._register(SemanticTokensProviderRegistry.onDidChange(e => this._fetchSemanticTokens.schedule()));
this._register(DocumentSemanticTokensProviderRegistry.onDidChange(e => this._fetchSemanticTokens.schedule()));
if (themeService) {
// workaround for tests which use undefined... :/
this._register(themeService.onThemeChange(_ => {
@ -756,7 +756,7 @@ class ModelSemanticColoring extends Disposable {
const styling = this._semanticStyling.get(provider);
const lastResultId = this._currentResponse ? this._currentResponse.resultId || null : null;
const request = Promise.resolve(provider.provideSemanticTokens(this._model, lastResultId, null, this._currentRequestCancellationTokenSource.token));
const request = Promise.resolve(provider.provideDocumentSemanticTokens(this._model, lastResultId, this._currentRequestCancellationTokenSource.token));
request.then((res) => {
this._currentRequestCancellationTokenSource = null;
@ -784,7 +784,7 @@ class ModelSemanticColoring extends Disposable {
}
}
private _setSemanticTokens(provider: SemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticColoringProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void {
private _setSemanticTokens(provider: DocumentSemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticColoringProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void {
const currentResponse = this._currentResponse;
if (this._currentResponse) {
this._currentResponse.dispose();
@ -793,7 +793,7 @@ class ModelSemanticColoring extends Disposable {
if (this._isDisposed) {
// disposed!
if (provider && tokens) {
provider.releaseSemanticTokens(tokens.resultId);
provider.releaseDocumentSemanticTokens(tokens.resultId);
}
return;
}
@ -954,8 +954,8 @@ class ModelSemanticColoring extends Disposable {
this._model.setSemanticTokens(null);
}
private _getSemanticColoringProvider(): SemanticTokensProvider | null {
const result = SemanticTokensProviderRegistry.ordered(this._model);
private _getSemanticColoringProvider(): DocumentSemanticTokensProvider | null {
const result = DocumentSemanticTokensProviderRegistry.ordered(this._model);
return (result.length > 0 ? result[0] : null);
}
}

11
src/vs/monaco.d.ts vendored
View file

@ -5948,10 +5948,15 @@ declare namespace monaco.languages {
readonly edits: SemanticTokensEdit[];
}
export interface SemanticTokensProvider {
export interface DocumentSemanticTokensProvider {
getLegend(): SemanticTokensLegend;
provideSemanticTokens(model: editor.ITextModel, lastResultId: string | null, ranges: Range[] | null, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
releaseSemanticTokens(resultId: string | undefined): void;
provideDocumentSemanticTokens(model: editor.ITextModel, lastResultId: string | null, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
releaseDocumentSemanticTokens(resultId: string | undefined): void;
}
export interface DocumentRangeSemanticTokensProvider {
getLegend(): SemanticTokensLegend;
provideDocumentRangeSemanticTokens(model: editor.ITextModel, range: Range, token: CancellationToken): ProviderResult<SemanticTokens>;
}
export interface ILanguageExtensionPoint {

100
src/vs/vscode.proposed.d.ts vendored
View file

@ -180,8 +180,7 @@ declare module 'vscode' {
/**
* The result id of the tokens.
*
* On a next call to `provideSemanticTokens`, if VS Code still holds in memory this result,
* the result id will be passed in as `SemanticTokensRequestOptions.previousResultId`.
* This is the id that will be passed to `DocumentSemanticTokensProvider.provideDocumentSemanticTokensEdits` (if implemented).
*/
readonly resultId?: string;
readonly data: Uint32Array;
@ -193,8 +192,7 @@ declare module 'vscode' {
/**
* The result id of the tokens.
*
* On a next call to `provideSemanticTokens`, if VS Code still holds in memory this result,
* the result id will be passed in as `SemanticTokensRequestOptions.previousResultId`.
* This is the id that will be passed to `DocumentSemanticTokensProvider.provideDocumentSemanticTokensEdits` (if implemented).
*/
readonly resultId?: string;
readonly edits: SemanticTokensEdit[];
@ -210,21 +208,11 @@ declare module 'vscode' {
constructor(start: number, deleteCount: number, data?: Uint32Array);
}
export interface SemanticTokensRequestOptions {
readonly ranges?: readonly Range[];
/**
* The previous result id that the editor still holds in memory.
*
* Only when this is set it is safe for a `SemanticTokensProvider` to return `SemanticTokensEdits`.
*/
readonly previousResultId?: string;
}
/**
* The semantic tokens provider interface defines the contract between extensions and
* The document semantic tokens provider interface defines the contract between extensions and
* semantic tokens.
*/
export interface SemanticTokensProvider {
export interface DocumentSemanticTokensProvider {
/**
* A file can contain many tokens, perhaps even hundreds of thousands of tokens. Therefore, to improve
* the memory consumption around describing semantic tokens, we have decided to avoid allocating an object
@ -232,21 +220,18 @@ declare module 'vscode' {
* of each token is expressed relative to the token before it because most tokens remain stable relative to
* each other when edits are made in a file.
*
*
* ---
* In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following fields:
* In short, each token takes 5 integers to represent, so a specific token `i` in the file consists of the following array indices:
* - at index `5*i` - `deltaLine`: token line number, relative to the previous token
* - at index `5*i+1` - `deltaStart`: token start character, relative to the previous token (relative to 0 or the previous token's start if they are on the same line)
* - at index `5*i+2` - `length`: the length of the token. A token cannot be multiline.
* - at index `5*i+3` - `tokenType`: will be looked up in `SemanticTokensLegend.tokenTypes`
* - at index `5*i+4` - `tokenModifiers`: each set bit will be looked up in `SemanticTokensLegend.tokenModifiers`
*
*
*
* ---
* ### How to encode tokens
*
* Here is an example for encoding a file with 3 tokens:
* Here is an example for encoding a file with 3 tokens in a uint32 array:
* ```
* { line: 2, startChar: 5, length: 3, tokenType: "properties", tokenModifiers: ["private", "static"] },
* { line: 2, startChar: 10, length: 4, tokenType: "types", tokenModifiers: [] },
@ -285,8 +270,12 @@ declare module 'vscode' {
* // 1st token, 2nd token, 3rd token
* [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ]
* ```
*
*
*/
provideDocumentSemanticTokens(document: TextDocument, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
/**
* Instead of always returning all the tokens in a file, it is possible for a `DocumentSemanticTokensProvider` to implement
* this method (`updateSemanticTokens`) and then return incremental updates to the previously provided semantic tokens.
*
* ---
* ### How tokens change when the document changes
@ -307,8 +296,8 @@ declare module 'vscode' {
* ```
* It is possible to express these new tokens in terms of an edit applied to the previous tokens:
* ```
* [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ]
* [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ]
* [ 2,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] // old tokens
* [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] // new tokens
*
* edit: { start: 0, deleteCount: 1, data: [3] } // replace integer at offset 0 with 3
* ```
@ -327,51 +316,56 @@ declare module 'vscode' {
* ```
* Again, it is possible to express these new tokens in terms of an edit applied to the previous tokens:
* ```
* [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ]
* [ 3,5,3,0,3, 0,5,4,1,0, 1,3,5,0,2, 2,2,7,2,0, ]
* [ 3,5,3,0,3, 0,5,4,1,0, 3,2,7,2,0 ] // old tokens
* [ 3,5,3,0,3, 0,5,4,1,0, 1,3,5,0,2, 2,2,7,2,0, ] // new tokens
*
* edit: { start: 10, deleteCount: 1, data: [1,3,5,0,2,2] } // replace integer at offset 10 with [1,3,5,0,2,2]
* ```
*
*
*
* ---
* ### When to return `SemanticTokensEdits`
*
* When doing edits, it is possible that multiple edits occur until VS Code decides to invoke the semantic tokens provider.
* In principle, each call to `provideSemanticTokens` can return a full representations of the semantic tokens, and that would
* be a perfectly reasonable semantic tokens provider implementation.
*
* However, when having a language server running in a separate process, transferring all the tokens between processes
* might be slow, so VS Code allows to return the new tokens expressed in terms of multiple edits applied to the previous
* tokens.
*
* To clearly define what "previous tokens" means, it is possible to return a `resultId` with the semantic tokens. If the
* editor still has in memory the previous result, the editor will pass in options the previous `resultId` at
* `SemanticTokensRequestOptions.previousResultId`. Only when the editor passes in the previous `resultId`, it is allowed
* that a semantic tokens provider returns the new tokens expressed as edits to be applied to the previous result. Even in this
* case, the semantic tokens provider needs to return a new `resultId` that will identify these new tokens as a basis
* for the next request.
*
* *NOTE 1*: It is illegal to return `SemanticTokensEdits` if `options.previousResultId` is not set.
* *NOTE 2*: All edits in `SemanticTokensEdits` contain indices in the old integers array, so they all refer to the previous result state.
* *NOTE*: When doing edits, it is possible that multiple edits occur until VS Code decides to invoke the semantic tokens provider.
* *NOTE*: If the provider cannot compute `SemanticTokensEdits`, it can "give up" and return all the tokens in the document again.
* *NOTE*: All edits in `SemanticTokensEdits` contain indices in the old integers array, so they all refer to the previous result state.
*/
provideSemanticTokens(document: TextDocument, options: SemanticTokensRequestOptions, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
provideDocumentSemanticTokensEdits?(document: TextDocument, previousResultId: string, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
}
/**
* The document range semantic tokens provider interface defines the contract between extensions and
* semantic tokens.
*/
export interface DocumentRangeSemanticTokensProvider {
/**
* See [provideDocumentSemanticTokens](#DocumentSemanticTokensProvider.provideDocumentSemanticTokens).
*/
provideDocumentRangeSemanticTokens(document: TextDocument, range: Range, token: CancellationToken): ProviderResult<SemanticTokens>;
}
export namespace languages {
/**
* Register a semantic tokens provider.
* Register a semantic tokens provider for a whole document.
*
* Multiple providers can be registered for a language. In that case providers are sorted
* by their [score](#languages.match) and the best-matching provider is used. Failure
* of the selected provider will cause a failure of the whole operation.
*
* @param selector A selector that defines the documents this provider is applicable to.
* @param provider A semantic tokens provider.
* @param provider A document semantic tokens provider.
* @return A [disposable](#Disposable) that unregisters this provider when being disposed.
*/
export function registerSemanticTokensProvider(selector: DocumentSelector, provider: SemanticTokensProvider, legend: SemanticTokensLegend): Disposable;
export function registerDocumentSemanticTokensProvider(selector: DocumentSelector, provider: DocumentSemanticTokensProvider, legend: SemanticTokensLegend): Disposable;
/**
* Register a semantic tokens provider for a document range.
*
* Multiple providers can be registered for a language. In that case providers are sorted
* by their [score](#languages.match) and the best-matching provider is used. Failure
* of the selected provider will cause a failure of the whole operation.
*
* @param selector A selector that defines the documents this provider is applicable to.
* @param provider A document range semantic tokens provider.
* @return A [disposable](#Disposable) that unregisters this provider when being disposed.
*/
export function registerDocumentRangeSemanticTokensProvider(selector: DocumentSelector, provider: DocumentRangeSemanticTokensProvider, legend: SemanticTokensLegend): Disposable;
}
//#endregion

50
src/vs/workbench/api/browser/mainThreadLanguageFeatures.ts
View file

@ -327,8 +327,12 @@ export class MainThreadLanguageFeatures implements MainThreadLanguageFeaturesSha
// --- semantic tokens
$registerSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void {
this._registrations.set(handle, modes.SemanticTokensProviderRegistry.register(selector, new MainThreadSemanticTokensProvider(this._proxy, handle, legend)));
$registerDocumentSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void {
this._registrations.set(handle, modes.DocumentSemanticTokensProviderRegistry.register(selector, new MainThreadDocumentSemanticTokensProvider(this._proxy, handle, legend)));
}
$registerDocumentRangeSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void {
this._registrations.set(handle, modes.DocumentRangeSemanticTokensProviderRegistry.register(selector, new MainThreadDocumentRangeSemanticTokensProvider(this._proxy, handle, legend)));
}
// --- suggest
@ -607,7 +611,7 @@ export class MainThreadLanguageFeatures implements MainThreadLanguageFeaturesSha
}
export class MainThreadSemanticTokensProvider implements modes.SemanticTokensProvider {
export class MainThreadDocumentSemanticTokensProvider implements modes.DocumentSemanticTokensProvider {
constructor(
private readonly _proxy: ExtHostLanguageFeaturesShape,
@ -616,9 +620,9 @@ export class MainThreadSemanticTokensProvider implements modes.SemanticTokensPro
) {
}
public releaseSemanticTokens(resultId: string | undefined): void {
public releaseDocumentSemanticTokens(resultId: string | undefined): void {
if (resultId) {
this._proxy.$releaseSemanticTokens(this._handle, parseInt(resultId, 10));
this._proxy.$releaseDocumentSemanticTokens(this._handle, parseInt(resultId, 10));
}
}
@ -626,9 +630,9 @@ export class MainThreadSemanticTokensProvider implements modes.SemanticTokensPro
return this._legend;
}
async provideSemanticTokens(model: ITextModel, lastResultId: string | null, ranges: EditorRange[] | null, token: CancellationToken): Promise<modes.SemanticTokens | modes.SemanticTokensEdits | null> {
async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise<modes.SemanticTokens | modes.SemanticTokensEdits | null> {
const nLastResultId = lastResultId ? parseInt(lastResultId, 10) : 0;
const encodedDto = await this._proxy.$provideSemanticTokens(this._handle, model.uri, ranges, nLastResultId, token);
const encodedDto = await this._proxy.$provideDocumentSemanticTokens(this._handle, model.uri, nLastResultId, token);
if (!encodedDto) {
return null;
}
@ -648,3 +652,35 @@ export class MainThreadSemanticTokensProvider implements modes.SemanticTokensPro
};
}
}
export class MainThreadDocumentRangeSemanticTokensProvider implements modes.DocumentRangeSemanticTokensProvider {
constructor(
private readonly _proxy: ExtHostLanguageFeaturesShape,
private readonly _handle: number,
private readonly _legend: modes.SemanticTokensLegend,
) {
}
public getLegend(): modes.SemanticTokensLegend {
return this._legend;
}
async provideDocumentRangeSemanticTokens(model: ITextModel, range: EditorRange, token: CancellationToken): Promise<modes.SemanticTokens | null> {
const encodedDto = await this._proxy.$provideDocumentRangeSemanticTokens(this._handle, model.uri, range, token);
if (!encodedDto) {
return null;
}
if (token.isCancellationRequested) {
return null;
}
const dto = decodeSemanticTokensDto(encodedDto);
if (dto.type === 'full') {
return {
resultId: String(dto.id),
data: dto.data
};
}
throw new Error(`Unexpected`);
}
}

8
src/vs/workbench/api/common/extHost.api.impl.ts
View file

@ -379,9 +379,13 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
registerOnTypeFormattingEditProvider(selector: vscode.DocumentSelector, provider: vscode.OnTypeFormattingEditProvider, firstTriggerCharacter: string, ...moreTriggerCharacters: string[]): vscode.Disposable {
return extHostLanguageFeatures.registerOnTypeFormattingEditProvider(extension, checkSelector(selector), provider, [firstTriggerCharacter].concat(moreTriggerCharacters));
},
registerSemanticTokensProvider(selector: vscode.DocumentSelector, provider: vscode.SemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable {
registerDocumentSemanticTokensProvider(selector: vscode.DocumentSelector, provider: vscode.DocumentSemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable {
checkProposedApiEnabled(extension);
return extHostLanguageFeatures.registerSemanticTokensProvider(extension, checkSelector(selector), provider, legend);
return extHostLanguageFeatures.registerDocumentSemanticTokensProvider(extension, checkSelector(selector), provider, legend);
},
registerDocumentRangeSemanticTokensProvider(selector: vscode.DocumentSelector, provider: vscode.DocumentRangeSemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable {
checkProposedApiEnabled(extension);
return extHostLanguageFeatures.registerDocumentRangeSemanticTokensProvider(extension, checkSelector(selector), provider, legend);
},
registerSignatureHelpProvider(selector: vscode.DocumentSelector, provider: vscode.SignatureHelpProvider, firstItem?: string | vscode.SignatureHelpProviderMetadata, ...remaining: string[]): vscode.Disposable {
if (typeof firstItem === 'object') {

8
src/vs/workbench/api/common/extHost.protocol.ts
View file

@ -358,7 +358,8 @@ export interface MainThreadLanguageFeaturesShape extends IDisposable {
$registerOnTypeFormattingSupport(handle: number, selector: IDocumentFilterDto[], autoFormatTriggerCharacters: string[], extensionId: ExtensionIdentifier): void;
$registerNavigateTypeSupport(handle: number): void;
$registerRenameSupport(handle: number, selector: IDocumentFilterDto[], supportsResolveInitialValues: boolean): void;
$registerSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void;
$registerDocumentSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void;
$registerDocumentRangeSemanticTokensProvider(handle: number, selector: IDocumentFilterDto[], legend: modes.SemanticTokensLegend): void;
$registerSuggestSupport(handle: number, selector: IDocumentFilterDto[], triggerCharacters: string[], supportsResolveDetails: boolean, extensionId: ExtensionIdentifier): void;
$registerSignatureHelpProvider(handle: number, selector: IDocumentFilterDto[], metadata: ISignatureHelpProviderMetadataDto): void;
$registerDocumentLinkProvider(handle: number, selector: IDocumentFilterDto[], supportsResolve: boolean): void;
@ -1188,8 +1189,9 @@ export interface ExtHostLanguageFeaturesShape {
$releaseWorkspaceSymbols(handle: number, id: number): void;
$provideRenameEdits(handle: number, resource: UriComponents, position: IPosition, newName: string, token: CancellationToken): Promise<IWorkspaceEditDto | undefined>;
$resolveRenameLocation(handle: number, resource: UriComponents, position: IPosition, token: CancellationToken): Promise<modes.RenameLocation | undefined>;
$provideSemanticTokens(handle: number, resource: UriComponents, ranges: IRange[] | null, previousResultId: number, token: CancellationToken): Promise<VSBuffer | null>;
$releaseSemanticTokens(handle: number, semanticColoringResultId: number): void;
$provideDocumentSemanticTokens(handle: number, resource: UriComponents, previousResultId: number, token: CancellationToken): Promise<VSBuffer | null>;
$releaseDocumentSemanticTokens(handle: number, semanticColoringResultId: number): void;
$provideDocumentRangeSemanticTokens(handle: number, resource: UriComponents, range: IRange, token: CancellationToken): Promise<VSBuffer | null>;
$provideCompletionItems(handle: number, resource: UriComponents, position: IPosition, context: modes.CompletionContext, token: CancellationToken): Promise<ISuggestResultDto | undefined>;
$resolveCompletionItem(handle: number, resource: UriComponents, position: IPosition, id: ChainedCacheId, token: CancellationToken): Promise<ISuggestDataDto | undefined>;
$releaseCompletionItems(handle: number, id: number): void;

90
src/vs/workbench/api/common/extHostLanguageFeatures.ts
View file

@ -629,37 +629,38 @@ class SemanticTokensPreviousResult {
) { }
}
export class SemanticTokensAdapter {
export class DocumentSemanticTokensAdapter {
private readonly _previousResults: Map<number, SemanticTokensPreviousResult>;
private _nextResultId = 1;
constructor(
private readonly _documents: ExtHostDocuments,
private readonly _provider: vscode.SemanticTokensProvider,
private readonly _provider: vscode.DocumentSemanticTokensProvider,
) {
this._previousResults = new Map<number, SemanticTokensPreviousResult>();
}
provideSemanticTokens(resource: URI, ranges: IRange[] | null, previousResultId: number, token: CancellationToken): Promise<VSBuffer | null> {
provideDocumentSemanticTokens(resource: URI, previousResultId: number, token: CancellationToken): Promise<VSBuffer | null> {
const doc = this._documents.getDocument(resource);
const previousResult = (previousResultId !== 0 ? this._previousResults.get(previousResultId) : null);
const opts: vscode.SemanticTokensRequestOptions = {
ranges: (Array.isArray(ranges) && ranges.length > 0 ? ranges.map<Range>(typeConvert.Range.to) : undefined),
previousResultId: (previousResult ? previousResult.resultId : undefined)
};
return asPromise(() => this._provider.provideSemanticTokens(doc, opts, token)).then(value => {
if (!value) {
return null;
return asPromise(() => {
if (previousResult && typeof previousResult.resultId === 'string' && typeof this._provider.provideDocumentSemanticTokensEdits === 'function') {
return this._provider.provideDocumentSemanticTokensEdits(doc, previousResult.resultId, token);
}
return this._provider.provideDocumentSemanticTokens(doc, token);
}).then(value => {
if (previousResult) {
this._previousResults.delete(previousResultId);
}
return this._send(SemanticTokensAdapter._convertToEdits(previousResult, value), value);
if (!value) {
return null;
}
return this._send(DocumentSemanticTokensAdapter._convertToEdits(previousResult, value), value);
});
}
async releaseSemanticColoring(semanticColoringResultId: number): Promise<void> {
async releaseDocumentSemanticColoring(semanticColoringResultId: number): Promise<void> {
this._previousResults.delete(semanticColoringResultId);
}
@ -672,7 +673,7 @@ export class SemanticTokensAdapter {
}
private static _convertToEdits(previousResult: SemanticTokensPreviousResult | null | undefined, newResult: vscode.SemanticTokens | vscode.SemanticTokensEdits): vscode.SemanticTokens | vscode.SemanticTokensEdits {
if (!SemanticTokensAdapter._isSemanticTokens(newResult)) {
if (!DocumentSemanticTokensAdapter._isSemanticTokens(newResult)) {
return newResult;
}
if (!previousResult || !previousResult.tokens) {
@ -708,7 +709,7 @@ export class SemanticTokensAdapter {
}
private _send(value: vscode.SemanticTokens | vscode.SemanticTokensEdits, original: vscode.SemanticTokens | vscode.SemanticTokensEdits): VSBuffer | null {
if (SemanticTokensAdapter._isSemanticTokens(value)) {
if (DocumentSemanticTokensAdapter._isSemanticTokens(value)) {
const myId = this._nextResultId++;
this._previousResults.set(myId, new SemanticTokensPreviousResult(value.resultId, value.data));
return encodeSemanticTokensDto({
@ -718,9 +719,9 @@ export class SemanticTokensAdapter {
});
}
if (SemanticTokensAdapter._isSemanticTokensEdits(value)) {
if (DocumentSemanticTokensAdapter._isSemanticTokensEdits(value)) {
const myId = this._nextResultId++;
if (SemanticTokensAdapter._isSemanticTokens(original)) {
if (DocumentSemanticTokensAdapter._isSemanticTokens(original)) {
// store the original
this._previousResults.set(myId, new SemanticTokensPreviousResult(original.resultId, original.data));
} else {
@ -737,6 +738,33 @@ export class SemanticTokensAdapter {
}
}
export class DocumentRangeSemanticTokensAdapter {
constructor(
private readonly _documents: ExtHostDocuments,
private readonly _provider: vscode.DocumentRangeSemanticTokensProvider,
) {
}
provideDocumentRangeSemanticTokens(resource: URI, range: IRange, token: CancellationToken): Promise<VSBuffer | null> {
const doc = this._documents.getDocument(resource);
return asPromise(() => this._provider.provideDocumentRangeSemanticTokens(doc, typeConvert.Range.to(range), token)).then(value => {
if (!value) {
return null;
}
return this._send(value);
});
}
private _send(value: vscode.SemanticTokens): VSBuffer | null {
return encodeSemanticTokensDto({
id: 0,
type: 'full',
data: value.data
});
}
}
class SuggestAdapter {
static supportsResolving(provider: vscode.CompletionItemProvider): boolean {
@ -1324,9 +1352,9 @@ class CallHierarchyAdapter {
type Adapter = DocumentSymbolAdapter | CodeLensAdapter | DefinitionAdapter | HoverAdapter
| DocumentHighlightAdapter | ReferenceAdapter | CodeActionAdapter | DocumentFormattingAdapter
| RangeFormattingAdapter | OnTypeFormattingAdapter | NavigateTypeAdapter | RenameAdapter
| SemanticTokensAdapter | SuggestAdapter | SignatureHelpAdapter | LinkProviderAdapter
| ImplementationAdapter | TypeDefinitionAdapter | ColorProviderAdapter | FoldingProviderAdapter
| DeclarationAdapter | SelectionRangeAdapter | CallHierarchyAdapter;
| SuggestAdapter | SignatureHelpAdapter | LinkProviderAdapter | ImplementationAdapter
| TypeDefinitionAdapter | ColorProviderAdapter | FoldingProviderAdapter | DeclarationAdapter
| SelectionRangeAdapter | CallHierarchyAdapter | DocumentSemanticTokensAdapter | DocumentRangeSemanticTokensAdapter;
class AdapterData {
constructor(
@ -1657,18 +1685,28 @@ export class ExtHostLanguageFeatures implements extHostProtocol.ExtHostLanguageF
//#region semantic coloring
registerSemanticTokensProvider(extension: IExtensionDescription, selector: vscode.DocumentSelector, provider: vscode.SemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable {
const handle = this._addNewAdapter(new SemanticTokensAdapter(this._documents, provider), extension);
this._proxy.$registerSemanticTokensProvider(handle, this._transformDocumentSelector(selector), legend);
registerDocumentSemanticTokensProvider(extension: IExtensionDescription, selector: vscode.DocumentSelector, provider: vscode.DocumentSemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable {
const handle = this._addNewAdapter(new DocumentSemanticTokensAdapter(this._documents, provider), extension);
this._proxy.$registerDocumentSemanticTokensProvider(handle, this._transformDocumentSelector(selector), legend);
return this._createDisposable(handle);
}
$provideSemanticTokens(handle: number, resource: UriComponents, ranges: IRange[] | null, previousResultId: number, token: CancellationToken): Promise<VSBuffer | null> {
return this._withAdapter(handle, SemanticTokensAdapter, adapter => adapter.provideSemanticTokens(URI.revive(resource), ranges, previousResultId, token), null);
$provideDocumentSemanticTokens(handle: number, resource: UriComponents, previousResultId: number, token: CancellationToken): Promise<VSBuffer | null> {
return this._withAdapter(handle, DocumentSemanticTokensAdapter, adapter => adapter.provideDocumentSemanticTokens(URI.revive(resource), previousResultId, token), null);
}
$releaseSemanticTokens(handle: number, semanticColoringResultId: number): void {
this._withAdapter(handle, SemanticTokensAdapter, adapter => adapter.releaseSemanticColoring(semanticColoringResultId), undefined);
$releaseDocumentSemanticTokens(handle: number, semanticColoringResultId: number): void {
this._withAdapter(handle, DocumentSemanticTokensAdapter, adapter => adapter.releaseDocumentSemanticColoring(semanticColoringResultId), undefined);
}
registerDocumentRangeSemanticTokensProvider(extension: IExtensionDescription, selector: vscode.DocumentSelector, provider: vscode.DocumentRangeSemanticTokensProvider, legend: vscode.SemanticTokensLegend): vscode.Disposable {
const handle = this._addNewAdapter(new DocumentRangeSemanticTokensAdapter(this._documents, provider), extension);
this._proxy.$registerDocumentRangeSemanticTokensProvider(handle, this._transformDocumentSelector(selector), legend);
return this._createDisposable(handle);
}
$provideDocumentRangeSemanticTokens(handle: number, resource: UriComponents, range: IRange, token: CancellationToken): Promise<VSBuffer | null> {
return this._withAdapter(handle, DocumentRangeSemanticTokensAdapter, adapter => adapter.provideDocumentRangeSemanticTokens(URI.revive(resource), range, token), null);
}
//#endregion

7
src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts
View file

@ -16,7 +16,7 @@ import { EditorAction, ServicesAccessor, registerEditorAction, registerEditorCon
import { Position } from 'vs/editor/common/core/position';
import { IEditorContribution } from 'vs/editor/common/editorCommon';
import { ITextModel } from 'vs/editor/common/model';
import { FontStyle, LanguageIdentifier, StandardTokenType, TokenMetadata, SemanticTokensProviderRegistry, SemanticTokensLegend, SemanticTokens } from 'vs/editor/common/modes';
import { FontStyle, LanguageIdentifier, StandardTokenType, TokenMetadata, DocumentSemanticTokensProviderRegistry, SemanticTokensLegend, SemanticTokens } from 'vs/editor/common/modes';
import { IModeService } from 'vs/editor/common/services/modeService';
import { INotificationService } from 'vs/platform/notification/common/notification';
import { editorHoverBackground, editorHoverBorder } from 'vs/platform/theme/common/colorRegistry';
@ -411,11 +411,10 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
}
private async _computeSemanticTokens(): Promise<SemanticTokensResult | null> {
const tokenProviders = SemanticTokensProviderRegistry.ordered(this._model);
const tokenProviders = DocumentSemanticTokensProviderRegistry.ordered(this._model);
if (tokenProviders.length) {
const provider = tokenProviders[0];
const range = this._model.getFullModelRange();
const tokens = await Promise.resolve(provider.provideSemanticTokens(this._model, null, [range], this._currentRequestCancellationTokenSource.token));
const tokens = await Promise.resolve(provider.provideDocumentSemanticTokens(this._model, null, this._currentRequestCancellationTokenSource.token));
if (this.isSemanticTokens(tokens)) {
return { tokens, legend: provider.getLegend() };
}