Adopt latest vscode-textmate (fixes #84401)
This commit is contained in:
parent
2340d87072
commit
7521ebdeb4
|
@ -52,7 +52,7 @@
|
|||
"vscode-proxy-agent": "^0.5.2",
|
||||
"vscode-ripgrep": "^1.5.7",
|
||||
"vscode-sqlite3": "4.0.9",
|
||||
"vscode-textmate": "^4.3.0",
|
||||
"vscode-textmate": "^4.4.0",
|
||||
"xterm": "4.3.0-beta17",
|
||||
"xterm-addon-search": "0.4.0-beta4",
|
||||
"xterm-addon-web-links": "0.2.1",
|
||||
|
|
|
@ -19,7 +19,7 @@
|
|||
"vscode-nsfw": "1.2.8",
|
||||
"vscode-proxy-agent": "^0.5.2",
|
||||
"vscode-ripgrep": "^1.5.7",
|
||||
"vscode-textmate": "^4.3.0",
|
||||
"vscode-textmate": "^4.4.0",
|
||||
"xterm": "4.3.0-beta17",
|
||||
"xterm-addon-search": "0.4.0-beta4",
|
||||
"xterm-addon-web-links": "0.2.1",
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
"dependencies": {
|
||||
"onigasm-umd": "^2.2.4",
|
||||
"semver-umd": "^5.5.3",
|
||||
"vscode-textmate": "^4.3.0",
|
||||
"vscode-textmate": "^4.4.0",
|
||||
"xterm": "4.3.0-beta17",
|
||||
"xterm-addon-search": "0.4.0-beta4",
|
||||
"xterm-addon-web-links": "0.2.1",
|
||||
|
|
|
@ -24,10 +24,10 @@ semver-umd@^5.5.3:
|
|||
resolved "https://registry.yarnpkg.com/semver-umd/-/semver-umd-5.5.3.tgz#b64d7a2d4f5a717b369d56e31940a38e47e34d1e"
|
||||
integrity sha512-HOnQrn2iKnVe/xlqCTzMXQdvSz3rPbD0DmQXYuQ+oK1dpptGFfPghonQrx5JHl2O7EJwDqtQnjhE7ME23q6ngw==
|
||||
|
||||
vscode-textmate@^4.3.0:
|
||||
version "4.3.0"
|
||||
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.3.0.tgz#6e1f0f273d84148cfa1e9c7ed85bd16c974f9f61"
|
||||
integrity sha512-MhEZ3hvxOVuYGsrRzW/PZLDR2VdtG2+V6TIKPvmE9JT+RAq/OtPlrFd1+ZQwBefoHEhjRNuRJ0OktcFezuxPmg==
|
||||
vscode-textmate@^4.4.0:
|
||||
version "4.4.0"
|
||||
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.4.0.tgz#14032afeb50152e8f53258c95643e555f2948305"
|
||||
integrity sha512-dFpm2eK0HwEjeFSD1DDh3j0q47bDSVuZt20RiJWxGqjtm73Wu2jip3C2KaZI3dQx/fSeeXCr/uEN4LNaNj7Ytw==
|
||||
dependencies:
|
||||
oniguruma "^7.2.0"
|
||||
|
||||
|
|
|
@ -399,10 +399,10 @@ vscode-ripgrep@^1.5.7:
|
|||
resolved "https://registry.yarnpkg.com/vscode-ripgrep/-/vscode-ripgrep-1.5.7.tgz#acb6b548af488a4bca5d0f1bb5faf761343289ce"
|
||||
integrity sha512-/Vsz/+k8kTvui0q3O74pif9FK0nKopgFTiGNVvxicZANxtSA8J8gUE9GQ/4dpi7D/2yI/YVORszwVskFbz46hQ==
|
||||
|
||||
vscode-textmate@^4.3.0:
|
||||
version "4.3.0"
|
||||
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.3.0.tgz#6e1f0f273d84148cfa1e9c7ed85bd16c974f9f61"
|
||||
integrity sha512-MhEZ3hvxOVuYGsrRzW/PZLDR2VdtG2+V6TIKPvmE9JT+RAq/OtPlrFd1+ZQwBefoHEhjRNuRJ0OktcFezuxPmg==
|
||||
vscode-textmate@^4.4.0:
|
||||
version "4.4.0"
|
||||
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.4.0.tgz#14032afeb50152e8f53258c95643e555f2948305"
|
||||
integrity sha512-dFpm2eK0HwEjeFSD1DDh3j0q47bDSVuZt20RiJWxGqjtm73Wu2jip3C2KaZI3dQx/fSeeXCr/uEN4LNaNj7Ytw==
|
||||
dependencies:
|
||||
oniguruma "^7.2.0"
|
||||
|
||||
|
|
256
src/typings/vscode-textmate.d.ts
vendored
256
src/typings/vscode-textmate.d.ts
vendored
|
@ -1,256 +0,0 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
declare module "vscode-textmate" {
|
||||
/**
|
||||
* A single theme setting.
|
||||
*/
|
||||
export interface IRawThemeSetting {
|
||||
readonly name?: string;
|
||||
readonly scope?: string | string[];
|
||||
readonly settings: {
|
||||
readonly fontStyle?: string;
|
||||
readonly foreground?: string;
|
||||
readonly background?: string;
|
||||
};
|
||||
}
|
||||
/**
|
||||
* A TextMate theme.
|
||||
*/
|
||||
export interface IRawTheme {
|
||||
readonly name?: string;
|
||||
readonly settings: IRawThemeSetting[];
|
||||
}
|
||||
export interface Thenable<T> extends PromiseLike<T> {
|
||||
}
|
||||
/**
|
||||
* A registry helper that can locate grammar file paths given scope names.
|
||||
*/
|
||||
export interface RegistryOptions {
|
||||
theme?: IRawTheme;
|
||||
loadGrammar(scopeName: string): Thenable<IRawGrammar | undefined | null>;
|
||||
getInjections?(scopeName: string): string[];
|
||||
getOnigLib?(): Thenable<IOnigLib>;
|
||||
}
|
||||
/**
|
||||
* A map from scope name to a language id. Please do not use language id 0.
|
||||
*/
|
||||
export interface IEmbeddedLanguagesMap {
|
||||
[scopeName: string]: number;
|
||||
}
|
||||
/**
|
||||
* A map from selectors to token types.
|
||||
*/
|
||||
export interface ITokenTypeMap {
|
||||
[selector: string]: StandardTokenType;
|
||||
}
|
||||
export const enum StandardTokenType {
|
||||
Other = 0,
|
||||
Comment = 1,
|
||||
String = 2,
|
||||
RegEx = 4,
|
||||
}
|
||||
export interface IGrammarConfiguration {
|
||||
embeddedLanguages?: IEmbeddedLanguagesMap;
|
||||
tokenTypes?: ITokenTypeMap;
|
||||
}
|
||||
/**
|
||||
* The registry that will hold all grammars.
|
||||
*/
|
||||
export class Registry {
|
||||
private readonly _locator;
|
||||
private readonly _syncRegistry;
|
||||
constructor(locator?: RegistryOptions);
|
||||
/**
|
||||
* Change the theme. Once called, no previous `ruleStack` should be used anymore.
|
||||
*/
|
||||
setTheme(theme: IRawTheme): void;
|
||||
/**
|
||||
* Returns a lookup array for color ids.
|
||||
*/
|
||||
getColorMap(): string[];
|
||||
/**
|
||||
* Load the grammar for `scopeName` and all referenced included grammars asynchronously.
|
||||
* Please do not use language id 0.
|
||||
*/
|
||||
loadGrammarWithEmbeddedLanguages(initialScopeName: string, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap): Thenable<IGrammar>;
|
||||
/**
|
||||
* Load the grammar for `scopeName` and all referenced included grammars asynchronously.
|
||||
* Please do not use language id 0.
|
||||
*/
|
||||
loadGrammarWithConfiguration(initialScopeName: string, initialLanguage: number, configuration: IGrammarConfiguration): Thenable<IGrammar>;
|
||||
/**
|
||||
* Load the grammar for `scopeName` and all referenced included grammars asynchronously.
|
||||
*/
|
||||
loadGrammar(initialScopeName: string): Thenable<IGrammar>;
|
||||
private _loadGrammar;
|
||||
/**
|
||||
* Adds a rawGrammar.
|
||||
*/
|
||||
addGrammar(rawGrammar: IRawGrammar, injections?: string[], initialLanguage?: number, embeddedLanguages?: IEmbeddedLanguagesMap): Thenable<IGrammar>;
|
||||
/**
|
||||
* Get the grammar for `scopeName`. The grammar must first be created via `loadGrammar` or `addGrammar`.
|
||||
*/
|
||||
grammarForScopeName(scopeName: string, initialLanguage?: number, embeddedLanguages?: IEmbeddedLanguagesMap, tokenTypes?: ITokenTypeMap): Thenable<IGrammar>;
|
||||
}
|
||||
/**
|
||||
* A grammar
|
||||
*/
|
||||
export interface IGrammar {
|
||||
/**
|
||||
* Tokenize `lineText` using previous line state `prevState`.
|
||||
*/
|
||||
tokenizeLine(lineText: string, prevState: StackElement | null): ITokenizeLineResult;
|
||||
/**
|
||||
* Tokenize `lineText` using previous line state `prevState`.
|
||||
* The result contains the tokens in binary format, resolved with the following information:
|
||||
* - language
|
||||
* - token type (regex, string, comment, other)
|
||||
* - font style
|
||||
* - foreground color
|
||||
* - background color
|
||||
* e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET`
|
||||
*/
|
||||
tokenizeLine2(lineText: string, prevState: StackElement | null): ITokenizeLineResult2;
|
||||
}
|
||||
export interface ITokenizeLineResult {
|
||||
readonly tokens: IToken[];
|
||||
/**
|
||||
* The `prevState` to be passed on to the next line tokenization.
|
||||
*/
|
||||
readonly ruleStack: StackElement;
|
||||
}
|
||||
/**
|
||||
* Helpers to manage the "collapsed" metadata of an entire StackElement stack.
|
||||
* The following assumptions have been made:
|
||||
* - languageId < 256 => needs 8 bits
|
||||
* - unique color count < 512 => needs 9 bits
|
||||
*
|
||||
* The binary format is:
|
||||
* - -------------------------------------------
|
||||
* 3322 2222 2222 1111 1111 1100 0000 0000
|
||||
* 1098 7654 3210 9876 5432 1098 7654 3210
|
||||
* - -------------------------------------------
|
||||
* xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx
|
||||
* bbbb bbbb bfff ffff ffFF FTTT LLLL LLLL
|
||||
* - -------------------------------------------
|
||||
* - L = LanguageId (8 bits)
|
||||
* - T = StandardTokenType (3 bits)
|
||||
* - F = FontStyle (3 bits)
|
||||
* - f = foreground color (9 bits)
|
||||
* - b = background color (9 bits)
|
||||
*/
|
||||
export const enum MetadataConsts {
|
||||
LANGUAGEID_MASK = 255,
|
||||
TOKEN_TYPE_MASK = 1792,
|
||||
FONT_STYLE_MASK = 14336,
|
||||
FOREGROUND_MASK = 8372224,
|
||||
BACKGROUND_MASK = 4286578688,
|
||||
LANGUAGEID_OFFSET = 0,
|
||||
TOKEN_TYPE_OFFSET = 8,
|
||||
FONT_STYLE_OFFSET = 11,
|
||||
FOREGROUND_OFFSET = 14,
|
||||
BACKGROUND_OFFSET = 23,
|
||||
}
|
||||
export interface ITokenizeLineResult2 {
|
||||
/**
|
||||
* The tokens in binary format. Each token occupies two array indices. For token i:
|
||||
* - at offset 2*i => startIndex
|
||||
* - at offset 2*i + 1 => metadata
|
||||
*
|
||||
*/
|
||||
readonly tokens: Uint32Array;
|
||||
/**
|
||||
* The `prevState` to be passed on to the next line tokenization.
|
||||
*/
|
||||
readonly ruleStack: StackElement;
|
||||
}
|
||||
export interface IToken {
|
||||
startIndex: number;
|
||||
readonly endIndex: number;
|
||||
readonly scopes: string[];
|
||||
}
|
||||
/**
|
||||
* **IMPORTANT** - Immutable!
|
||||
*/
|
||||
export interface StackElement {
|
||||
_stackElementBrand: void;
|
||||
readonly depth: number;
|
||||
clone(): StackElement;
|
||||
equals(other: StackElement): boolean;
|
||||
}
|
||||
export const INITIAL: StackElement;
|
||||
export const parseRawGrammar: (content: string, filePath?: string) => IRawGrammar;
|
||||
export interface ILocation {
|
||||
readonly filename: string;
|
||||
readonly line: number;
|
||||
readonly char: number;
|
||||
}
|
||||
export interface ILocatable {
|
||||
readonly $vscodeTextmateLocation?: ILocation;
|
||||
}
|
||||
export interface IRawGrammar extends ILocatable {
|
||||
repository: IRawRepository;
|
||||
readonly scopeName: string;
|
||||
readonly patterns: IRawRule[];
|
||||
readonly injections?: {
|
||||
[expression: string]: IRawRule;
|
||||
};
|
||||
readonly injectionSelector?: string;
|
||||
readonly fileTypes?: string[];
|
||||
readonly name?: string;
|
||||
readonly firstLineMatch?: string;
|
||||
}
|
||||
export interface IRawRepositoryMap {
|
||||
[name: string]: IRawRule;
|
||||
$self: IRawRule;
|
||||
$base: IRawRule;
|
||||
}
|
||||
export type IRawRepository = IRawRepositoryMap & ILocatable;
|
||||
export interface IRawRule extends ILocatable {
|
||||
id?: number;
|
||||
readonly include?: string;
|
||||
readonly name?: string;
|
||||
readonly contentName?: string;
|
||||
readonly match?: string;
|
||||
readonly captures?: IRawCaptures;
|
||||
readonly begin?: string;
|
||||
readonly beginCaptures?: IRawCaptures;
|
||||
readonly end?: string;
|
||||
readonly endCaptures?: IRawCaptures;
|
||||
readonly while?: string;
|
||||
readonly whileCaptures?: IRawCaptures;
|
||||
readonly patterns?: IRawRule[];
|
||||
readonly repository?: IRawRepository;
|
||||
readonly applyEndPatternLast?: boolean;
|
||||
}
|
||||
export interface IRawCapturesMap {
|
||||
[captureId: string]: IRawRule;
|
||||
}
|
||||
export type IRawCaptures = IRawCapturesMap & ILocatable;
|
||||
export interface IOnigLib {
|
||||
createOnigScanner(sources: string[]): OnigScanner;
|
||||
createOnigString(sources: string): OnigString;
|
||||
}
|
||||
export interface IOnigCaptureIndex {
|
||||
start: number;
|
||||
end: number;
|
||||
length: number;
|
||||
}
|
||||
export interface IOnigMatch {
|
||||
index: number;
|
||||
captureIndices: IOnigCaptureIndex[];
|
||||
scanner: OnigScanner;
|
||||
}
|
||||
export interface OnigScanner {
|
||||
findNextMatchSync(string: string | OnigString, startPosition: number): IOnigMatch;
|
||||
}
|
||||
export interface OnigString {
|
||||
readonly content: string;
|
||||
readonly dispose?: () => void;
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -174,7 +174,7 @@ class InspectTMScopesWidget extends Disposable implements IContentWidget {
|
|||
private readonly _notificationService: INotificationService;
|
||||
private readonly _model: ITextModel;
|
||||
private readonly _domNode: HTMLElement;
|
||||
private readonly _grammar: Promise<IGrammar>;
|
||||
private readonly _grammar: Promise<IGrammar | null>;
|
||||
|
||||
constructor(
|
||||
editor: IActiveCodeEditor,
|
||||
|
@ -212,7 +212,12 @@ class InspectTMScopesWidget extends Disposable implements IContentWidget {
|
|||
dom.clearNode(this._domNode);
|
||||
this._domNode.appendChild(document.createTextNode(nls.localize('inspectTMScopesWidget.loading', "Loading...")));
|
||||
this._grammar.then(
|
||||
(grammar) => this._compute(grammar, position),
|
||||
(grammar) => {
|
||||
if (!grammar) {
|
||||
throw new Error(`Could not find grammar for language!`);
|
||||
}
|
||||
this._compute(grammar, position);
|
||||
},
|
||||
(err) => {
|
||||
this._notificationService.warn(err);
|
||||
setTimeout(() => {
|
||||
|
|
|
@ -216,6 +216,9 @@ class Snapper {
|
|||
public captureSyntaxTokens(fileName: string, content: string): Promise<IToken[]> {
|
||||
const modeId = this.modeService.getModeIdByFilepathOrFirstLine(URI.file(fileName));
|
||||
return this.textMateService.createGrammar(modeId!).then((grammar) => {
|
||||
if (!grammar) {
|
||||
return [];
|
||||
}
|
||||
let lines = content.split(/\r\n|\r|\n/);
|
||||
|
||||
let result = this._tokenize(grammar, lines);
|
||||
|
|
|
@ -218,6 +218,9 @@ export abstract class AbstractTextMateService extends Disposable implements ITex
|
|||
return null;
|
||||
}
|
||||
const r = await grammarFactory.createGrammar(languageId);
|
||||
if (!r.grammar) {
|
||||
return null;
|
||||
}
|
||||
const tokenization = new TMTokenization(r.grammar, r.initialState, r.containsEmbeddedLanguages);
|
||||
tokenization.onDidEncounterLanguage((languageId) => {
|
||||
if (!this._encounteredLanguages[languageId]) {
|
||||
|
@ -314,7 +317,7 @@ export abstract class AbstractTextMateService extends Disposable implements ITex
|
|||
return true;
|
||||
}
|
||||
|
||||
public async createGrammar(modeId: string): Promise<IGrammar> {
|
||||
public async createGrammar(modeId: string): Promise<IGrammar | null> {
|
||||
const grammarFactory = await this._getOrCreateGrammarFactory();
|
||||
const { grammar } = await grammarFactory.createGrammar(this._modeService.getLanguageIdentifier(modeId)!.id);
|
||||
return grammar;
|
||||
|
|
|
@ -18,7 +18,7 @@ interface ITMGrammarFactoryHost {
|
|||
|
||||
export interface ICreateGrammarResult {
|
||||
languageId: LanguageId;
|
||||
grammar: IGrammar;
|
||||
grammar: IGrammar | null;
|
||||
initialState: StackElement;
|
||||
containsEmbeddedLanguages: boolean;
|
||||
}
|
||||
|
|
|
@ -14,7 +14,7 @@ export interface ITextMateService {
|
|||
|
||||
onDidEncounterLanguage: Event<LanguageId>;
|
||||
|
||||
createGrammar(modeId: string): Promise<IGrammar>;
|
||||
createGrammar(modeId: string): Promise<IGrammar | null>;
|
||||
}
|
||||
|
||||
// -------------- Types "liberated" from vscode-textmate due to usage in /common/
|
||||
|
|
|
@ -9072,10 +9072,10 @@ vscode-sqlite3@4.0.9:
|
|||
dependencies:
|
||||
nan "^2.14.0"
|
||||
|
||||
vscode-textmate@^4.3.0:
|
||||
version "4.3.0"
|
||||
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.3.0.tgz#6e1f0f273d84148cfa1e9c7ed85bd16c974f9f61"
|
||||
integrity sha512-MhEZ3hvxOVuYGsrRzW/PZLDR2VdtG2+V6TIKPvmE9JT+RAq/OtPlrFd1+ZQwBefoHEhjRNuRJ0OktcFezuxPmg==
|
||||
vscode-textmate@^4.4.0:
|
||||
version "4.4.0"
|
||||
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.4.0.tgz#14032afeb50152e8f53258c95643e555f2948305"
|
||||
integrity sha512-dFpm2eK0HwEjeFSD1DDh3j0q47bDSVuZt20RiJWxGqjtm73Wu2jip3C2KaZI3dQx/fSeeXCr/uEN4LNaNj7Ytw==
|
||||
dependencies:
|
||||
oniguruma "^7.2.0"
|
||||
|
||||
|
|
Loading…
Reference in a new issue