diff --git a/package.json b/package.json index 4a3cc1e9002..30b567bb9d5 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "vscode-proxy-agent": "^0.5.2", "vscode-ripgrep": "^1.5.7", "vscode-sqlite3": "4.0.9", - "vscode-textmate": "^4.3.0", + "vscode-textmate": "^4.4.0", "xterm": "4.3.0-beta17", "xterm-addon-search": "0.4.0-beta4", "xterm-addon-web-links": "0.2.1", diff --git a/remote/package.json b/remote/package.json index 1a1c1a0e64a..94e15be6512 100644 --- a/remote/package.json +++ b/remote/package.json @@ -19,7 +19,7 @@ "vscode-nsfw": "1.2.8", "vscode-proxy-agent": "^0.5.2", "vscode-ripgrep": "^1.5.7", - "vscode-textmate": "^4.3.0", + "vscode-textmate": "^4.4.0", "xterm": "4.3.0-beta17", "xterm-addon-search": "0.4.0-beta4", "xterm-addon-web-links": "0.2.1", diff --git a/remote/web/package.json b/remote/web/package.json index c4a39c50bb3..3c8b489c130 100644 --- a/remote/web/package.json +++ b/remote/web/package.json @@ -4,7 +4,7 @@ "dependencies": { "onigasm-umd": "^2.2.4", "semver-umd": "^5.5.3", - "vscode-textmate": "^4.3.0", + "vscode-textmate": "^4.4.0", "xterm": "4.3.0-beta17", "xterm-addon-search": "0.4.0-beta4", "xterm-addon-web-links": "0.2.1", diff --git a/remote/web/yarn.lock b/remote/web/yarn.lock index d5d1fcdd499..a09ab549ac8 100644 --- a/remote/web/yarn.lock +++ b/remote/web/yarn.lock @@ -24,10 +24,10 @@ semver-umd@^5.5.3: resolved "https://registry.yarnpkg.com/semver-umd/-/semver-umd-5.5.3.tgz#b64d7a2d4f5a717b369d56e31940a38e47e34d1e" integrity sha512-HOnQrn2iKnVe/xlqCTzMXQdvSz3rPbD0DmQXYuQ+oK1dpptGFfPghonQrx5JHl2O7EJwDqtQnjhE7ME23q6ngw== -vscode-textmate@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.3.0.tgz#6e1f0f273d84148cfa1e9c7ed85bd16c974f9f61" - integrity sha512-MhEZ3hvxOVuYGsrRzW/PZLDR2VdtG2+V6TIKPvmE9JT+RAq/OtPlrFd1+ZQwBefoHEhjRNuRJ0OktcFezuxPmg== +vscode-textmate@^4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.4.0.tgz#14032afeb50152e8f53258c95643e555f2948305" + integrity sha512-dFpm2eK0HwEjeFSD1DDh3j0q47bDSVuZt20RiJWxGqjtm73Wu2jip3C2KaZI3dQx/fSeeXCr/uEN4LNaNj7Ytw== dependencies: oniguruma "^7.2.0" diff --git a/remote/yarn.lock b/remote/yarn.lock index c2f8da25470..3d08be2e5d0 100644 --- a/remote/yarn.lock +++ b/remote/yarn.lock @@ -399,10 +399,10 @@ vscode-ripgrep@^1.5.7: resolved "https://registry.yarnpkg.com/vscode-ripgrep/-/vscode-ripgrep-1.5.7.tgz#acb6b548af488a4bca5d0f1bb5faf761343289ce" integrity sha512-/Vsz/+k8kTvui0q3O74pif9FK0nKopgFTiGNVvxicZANxtSA8J8gUE9GQ/4dpi7D/2yI/YVORszwVskFbz46hQ== -vscode-textmate@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.3.0.tgz#6e1f0f273d84148cfa1e9c7ed85bd16c974f9f61" - integrity sha512-MhEZ3hvxOVuYGsrRzW/PZLDR2VdtG2+V6TIKPvmE9JT+RAq/OtPlrFd1+ZQwBefoHEhjRNuRJ0OktcFezuxPmg== +vscode-textmate@^4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.4.0.tgz#14032afeb50152e8f53258c95643e555f2948305" + integrity sha512-dFpm2eK0HwEjeFSD1DDh3j0q47bDSVuZt20RiJWxGqjtm73Wu2jip3C2KaZI3dQx/fSeeXCr/uEN4LNaNj7Ytw== dependencies: oniguruma "^7.2.0" diff --git a/src/typings/vscode-textmate.d.ts b/src/typings/vscode-textmate.d.ts deleted file mode 100644 index 835b33008ac..00000000000 --- a/src/typings/vscode-textmate.d.ts +++ /dev/null @@ -1,256 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -declare module "vscode-textmate" { - /** - * A single theme setting. - */ - export interface IRawThemeSetting { - readonly name?: string; - readonly scope?: string | string[]; - readonly settings: { - readonly fontStyle?: string; - readonly foreground?: string; - readonly background?: string; - }; - } - /** - * A TextMate theme. - */ - export interface IRawTheme { - readonly name?: string; - readonly settings: IRawThemeSetting[]; - } - export interface Thenable extends PromiseLike { - } - /** - * A registry helper that can locate grammar file paths given scope names. - */ - export interface RegistryOptions { - theme?: IRawTheme; - loadGrammar(scopeName: string): Thenable; - getInjections?(scopeName: string): string[]; - getOnigLib?(): Thenable; - } - /** - * A map from scope name to a language id. Please do not use language id 0. - */ - export interface IEmbeddedLanguagesMap { - [scopeName: string]: number; - } - /** - * A map from selectors to token types. - */ - export interface ITokenTypeMap { - [selector: string]: StandardTokenType; - } - export const enum StandardTokenType { - Other = 0, - Comment = 1, - String = 2, - RegEx = 4, - } - export interface IGrammarConfiguration { - embeddedLanguages?: IEmbeddedLanguagesMap; - tokenTypes?: ITokenTypeMap; - } - /** - * The registry that will hold all grammars. - */ - export class Registry { - private readonly _locator; - private readonly _syncRegistry; - constructor(locator?: RegistryOptions); - /** - * Change the theme. Once called, no previous `ruleStack` should be used anymore. - */ - setTheme(theme: IRawTheme): void; - /** - * Returns a lookup array for color ids. - */ - getColorMap(): string[]; - /** - * Load the grammar for `scopeName` and all referenced included grammars asynchronously. - * Please do not use language id 0. - */ - loadGrammarWithEmbeddedLanguages(initialScopeName: string, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap): Thenable; - /** - * Load the grammar for `scopeName` and all referenced included grammars asynchronously. - * Please do not use language id 0. - */ - loadGrammarWithConfiguration(initialScopeName: string, initialLanguage: number, configuration: IGrammarConfiguration): Thenable; - /** - * Load the grammar for `scopeName` and all referenced included grammars asynchronously. - */ - loadGrammar(initialScopeName: string): Thenable; - private _loadGrammar; - /** - * Adds a rawGrammar. - */ - addGrammar(rawGrammar: IRawGrammar, injections?: string[], initialLanguage?: number, embeddedLanguages?: IEmbeddedLanguagesMap): Thenable; - /** - * Get the grammar for `scopeName`. The grammar must first be created via `loadGrammar` or `addGrammar`. - */ - grammarForScopeName(scopeName: string, initialLanguage?: number, embeddedLanguages?: IEmbeddedLanguagesMap, tokenTypes?: ITokenTypeMap): Thenable; - } - /** - * A grammar - */ - export interface IGrammar { - /** - * Tokenize `lineText` using previous line state `prevState`. - */ - tokenizeLine(lineText: string, prevState: StackElement | null): ITokenizeLineResult; - /** - * Tokenize `lineText` using previous line state `prevState`. - * The result contains the tokens in binary format, resolved with the following information: - * - language - * - token type (regex, string, comment, other) - * - font style - * - foreground color - * - background color - * e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET` - */ - tokenizeLine2(lineText: string, prevState: StackElement | null): ITokenizeLineResult2; - } - export interface ITokenizeLineResult { - readonly tokens: IToken[]; - /** - * The `prevState` to be passed on to the next line tokenization. - */ - readonly ruleStack: StackElement; - } - /** - * Helpers to manage the "collapsed" metadata of an entire StackElement stack. - * The following assumptions have been made: - * - languageId < 256 => needs 8 bits - * - unique color count < 512 => needs 9 bits - * - * The binary format is: - * - ------------------------------------------- - * 3322 2222 2222 1111 1111 1100 0000 0000 - * 1098 7654 3210 9876 5432 1098 7654 3210 - * - ------------------------------------------- - * xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx - * bbbb bbbb bfff ffff ffFF FTTT LLLL LLLL - * - ------------------------------------------- - * - L = LanguageId (8 bits) - * - T = StandardTokenType (3 bits) - * - F = FontStyle (3 bits) - * - f = foreground color (9 bits) - * - b = background color (9 bits) - */ - export const enum MetadataConsts { - LANGUAGEID_MASK = 255, - TOKEN_TYPE_MASK = 1792, - FONT_STYLE_MASK = 14336, - FOREGROUND_MASK = 8372224, - BACKGROUND_MASK = 4286578688, - LANGUAGEID_OFFSET = 0, - TOKEN_TYPE_OFFSET = 8, - FONT_STYLE_OFFSET = 11, - FOREGROUND_OFFSET = 14, - BACKGROUND_OFFSET = 23, - } - export interface ITokenizeLineResult2 { - /** - * The tokens in binary format. Each token occupies two array indices. For token i: - * - at offset 2*i => startIndex - * - at offset 2*i + 1 => metadata - * - */ - readonly tokens: Uint32Array; - /** - * The `prevState` to be passed on to the next line tokenization. - */ - readonly ruleStack: StackElement; - } - export interface IToken { - startIndex: number; - readonly endIndex: number; - readonly scopes: string[]; - } - /** - * **IMPORTANT** - Immutable! - */ - export interface StackElement { - _stackElementBrand: void; - readonly depth: number; - clone(): StackElement; - equals(other: StackElement): boolean; - } - export const INITIAL: StackElement; - export const parseRawGrammar: (content: string, filePath?: string) => IRawGrammar; - export interface ILocation { - readonly filename: string; - readonly line: number; - readonly char: number; - } - export interface ILocatable { - readonly $vscodeTextmateLocation?: ILocation; - } - export interface IRawGrammar extends ILocatable { - repository: IRawRepository; - readonly scopeName: string; - readonly patterns: IRawRule[]; - readonly injections?: { - [expression: string]: IRawRule; - }; - readonly injectionSelector?: string; - readonly fileTypes?: string[]; - readonly name?: string; - readonly firstLineMatch?: string; - } - export interface IRawRepositoryMap { - [name: string]: IRawRule; - $self: IRawRule; - $base: IRawRule; - } - export type IRawRepository = IRawRepositoryMap & ILocatable; - export interface IRawRule extends ILocatable { - id?: number; - readonly include?: string; - readonly name?: string; - readonly contentName?: string; - readonly match?: string; - readonly captures?: IRawCaptures; - readonly begin?: string; - readonly beginCaptures?: IRawCaptures; - readonly end?: string; - readonly endCaptures?: IRawCaptures; - readonly while?: string; - readonly whileCaptures?: IRawCaptures; - readonly patterns?: IRawRule[]; - readonly repository?: IRawRepository; - readonly applyEndPatternLast?: boolean; - } - export interface IRawCapturesMap { - [captureId: string]: IRawRule; - } - export type IRawCaptures = IRawCapturesMap & ILocatable; - export interface IOnigLib { - createOnigScanner(sources: string[]): OnigScanner; - createOnigString(sources: string): OnigString; - } - export interface IOnigCaptureIndex { - start: number; - end: number; - length: number; - } - export interface IOnigMatch { - index: number; - captureIndices: IOnigCaptureIndex[]; - scanner: OnigScanner; - } - export interface OnigScanner { - findNextMatchSync(string: string | OnigString, startPosition: number): IOnigMatch; - } - export interface OnigString { - readonly content: string; - readonly dispose?: () => void; - } - - -} diff --git a/src/vs/workbench/contrib/codeEditor/browser/inspectTMScopes/inspectTMScopes.ts b/src/vs/workbench/contrib/codeEditor/browser/inspectTMScopes/inspectTMScopes.ts index aecdba4bd23..19fa1197cad 100644 --- a/src/vs/workbench/contrib/codeEditor/browser/inspectTMScopes/inspectTMScopes.ts +++ b/src/vs/workbench/contrib/codeEditor/browser/inspectTMScopes/inspectTMScopes.ts @@ -174,7 +174,7 @@ class InspectTMScopesWidget extends Disposable implements IContentWidget { private readonly _notificationService: INotificationService; private readonly _model: ITextModel; private readonly _domNode: HTMLElement; - private readonly _grammar: Promise; + private readonly _grammar: Promise; constructor( editor: IActiveCodeEditor, @@ -212,7 +212,12 @@ class InspectTMScopesWidget extends Disposable implements IContentWidget { dom.clearNode(this._domNode); this._domNode.appendChild(document.createTextNode(nls.localize('inspectTMScopesWidget.loading', "Loading..."))); this._grammar.then( - (grammar) => this._compute(grammar, position), + (grammar) => { + if (!grammar) { + throw new Error(`Could not find grammar for language!`); + } + this._compute(grammar, position); + }, (err) => { this._notificationService.warn(err); setTimeout(() => { diff --git a/src/vs/workbench/contrib/themes/test/electron-browser/themes.test.contribution.ts b/src/vs/workbench/contrib/themes/test/electron-browser/themes.test.contribution.ts index 392488ad4e7..d9e0970f7ca 100644 --- a/src/vs/workbench/contrib/themes/test/electron-browser/themes.test.contribution.ts +++ b/src/vs/workbench/contrib/themes/test/electron-browser/themes.test.contribution.ts @@ -216,6 +216,9 @@ class Snapper { public captureSyntaxTokens(fileName: string, content: string): Promise { const modeId = this.modeService.getModeIdByFilepathOrFirstLine(URI.file(fileName)); return this.textMateService.createGrammar(modeId!).then((grammar) => { + if (!grammar) { + return []; + } let lines = content.split(/\r\n|\r|\n/); let result = this._tokenize(grammar, lines); diff --git a/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts b/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts index dc9222e4b02..0b4e3be1f52 100644 --- a/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts +++ b/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts @@ -218,6 +218,9 @@ export abstract class AbstractTextMateService extends Disposable implements ITex return null; } const r = await grammarFactory.createGrammar(languageId); + if (!r.grammar) { + return null; + } const tokenization = new TMTokenization(r.grammar, r.initialState, r.containsEmbeddedLanguages); tokenization.onDidEncounterLanguage((languageId) => { if (!this._encounteredLanguages[languageId]) { @@ -314,7 +317,7 @@ export abstract class AbstractTextMateService extends Disposable implements ITex return true; } - public async createGrammar(modeId: string): Promise { + public async createGrammar(modeId: string): Promise { const grammarFactory = await this._getOrCreateGrammarFactory(); const { grammar } = await grammarFactory.createGrammar(this._modeService.getLanguageIdentifier(modeId)!.id); return grammar; diff --git a/src/vs/workbench/services/textMate/common/TMGrammarFactory.ts b/src/vs/workbench/services/textMate/common/TMGrammarFactory.ts index 1ea68789895..38e3101c34b 100644 --- a/src/vs/workbench/services/textMate/common/TMGrammarFactory.ts +++ b/src/vs/workbench/services/textMate/common/TMGrammarFactory.ts @@ -18,7 +18,7 @@ interface ITMGrammarFactoryHost { export interface ICreateGrammarResult { languageId: LanguageId; - grammar: IGrammar; + grammar: IGrammar | null; initialState: StackElement; containsEmbeddedLanguages: boolean; } diff --git a/src/vs/workbench/services/textMate/common/textMateService.ts b/src/vs/workbench/services/textMate/common/textMateService.ts index 2b61bb56cb8..0ff72dadff2 100644 --- a/src/vs/workbench/services/textMate/common/textMateService.ts +++ b/src/vs/workbench/services/textMate/common/textMateService.ts @@ -14,7 +14,7 @@ export interface ITextMateService { onDidEncounterLanguage: Event; - createGrammar(modeId: string): Promise; + createGrammar(modeId: string): Promise; } // -------------- Types "liberated" from vscode-textmate due to usage in /common/ diff --git a/yarn.lock b/yarn.lock index d9bdce0cd9f..1ec9dbf0ebb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9072,10 +9072,10 @@ vscode-sqlite3@4.0.9: dependencies: nan "^2.14.0" -vscode-textmate@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.3.0.tgz#6e1f0f273d84148cfa1e9c7ed85bd16c974f9f61" - integrity sha512-MhEZ3hvxOVuYGsrRzW/PZLDR2VdtG2+V6TIKPvmE9JT+RAq/OtPlrFd1+ZQwBefoHEhjRNuRJ0OktcFezuxPmg== +vscode-textmate@^4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.4.0.tgz#14032afeb50152e8f53258c95643e555f2948305" + integrity sha512-dFpm2eK0HwEjeFSD1DDh3j0q47bDSVuZt20RiJWxGqjtm73Wu2jip3C2KaZI3dQx/fSeeXCr/uEN4LNaNj7Ytw== dependencies: oniguruma "^7.2.0"