From f0ba210ef93504af548442b7428dc25ebab91e2e Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Fri, 24 May 2019 15:21:29 +0200 Subject: [PATCH] Add support for vscode-textmate in the browser --- package.json | 3 +- remote/package.json | 2 + remote/yarn.lock | 24 + src/typings/onigasm-umd.d.ts | 33 ++ src/vs/code/browser/workbench/workbench.html | 1 + src/vs/code/browser/workbench/workbench.js | 6 +- .../workbench/browser/web.simpleservices.ts | 23 - .../electron-browser/extensionService.ts | 2 +- .../browser/abstractTextMateService.ts | 518 ++++++++++++++++++ .../textMate/browser/textMateService.ts | 66 +++ .../cgmanifest.json | 0 .../electron-browser/textMateService.ts | 506 +---------------- src/vs/workbench/workbench.web.main.ts | 2 +- tslint.json | 4 +- yarn.lock | 13 +- 15 files changed, 669 insertions(+), 534 deletions(-) create mode 100644 src/typings/onigasm-umd.d.ts create mode 100644 src/vs/workbench/services/textMate/browser/abstractTextMateService.ts create mode 100644 src/vs/workbench/services/textMate/browser/textMateService.ts rename src/vs/workbench/services/textMate/{electron-browser => common}/cgmanifest.json (100%) diff --git a/package.json b/package.json index 3b946674fdb..0b8f8524c38 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "native-keymap": "1.2.5", "native-watchdog": "1.0.0", "node-pty": "0.9.0-beta9", + "onigasm-umd": "^2.2.2", "semver": "^5.5.0", "spdlog": "0.8.1", "sudo-prompt": "8.2.0", @@ -51,7 +52,7 @@ "vscode-proxy-agent": "0.4.0", "vscode-ripgrep": "^1.2.5", "vscode-sqlite3": "4.0.7", - "vscode-textmate": "^4.0.1", + "vscode-textmate": "^4.1.1", "vscode-xterm": "3.14.0-beta4", "yauzl": "^2.9.1", "yazl": "^2.4.3" diff --git a/remote/package.json b/remote/package.json index cd8b8c6673e..a845d99edc7 100644 --- a/remote/package.json +++ b/remote/package.json @@ -13,12 +13,14 @@ "minimist": "1.2.0", "native-watchdog": "1.0.0", "node-pty": "0.8.1", + "onigasm-umd": "^2.2.2", "semver": "^5.5.0", "spdlog": "0.8.1", "vscode-chokidar": "1.6.5", "vscode-nsfw": "1.1.1", "vscode-proxy-agent": "0.4.0", "vscode-ripgrep": "^1.2.5", + "vscode-textmate": "^4.1.1", "yauzl": "^2.9.1", "yazl": "^2.4.3" }, diff --git a/remote/yarn.lock b/remote/yarn.lock index 2df672960ff..c62128b5ed7 100644 --- a/remote/yarn.lock +++ b/remote/yarn.lock @@ -607,6 +607,11 @@ nan@^2.10.0: resolved "https://registry.yarnpkg.com/nan/-/nan-2.11.0.tgz#574e360e4d954ab16966ec102c0c049fd961a099" integrity sha512-F4miItu2rGnV2ySkXOQoA8FKz/SR2Q2sWP0sbTxNxz/tuokeC8WxOhPMcwi0qIyGtVn/rrSeLbvVkznqCdwYnw== +nan@^2.12.1: + version "2.14.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c" + integrity sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg== + native-watchdog@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/native-watchdog/-/native-watchdog-1.0.0.tgz#97344e83cd6815a8c8e6c44a52e7be05832e65ca" @@ -683,6 +688,18 @@ once@^1.3.1, once@^1.4.0: dependencies: wrappy "1" +onigasm-umd@^2.2.2: + version "2.2.2" + resolved "https://registry.yarnpkg.com/onigasm-umd/-/onigasm-umd-2.2.2.tgz#b989d762df61f899a3052ac794a50bd93fe20257" + integrity sha512-v2eMOJu7iE444L2iJN+U6s6s5S0y7oj/N0DAkrd6wokRtTVoq/v/yaDI1lIqFrTeJbNtqNzYvguDF5yNzW3Rvw== + +oniguruma@^7.0.0: + version "7.1.0" + resolved "https://registry.yarnpkg.com/oniguruma/-/oniguruma-7.1.0.tgz#106ddf7eb42507d0442ac68b187c4f7fdf052c83" + integrity sha512-mV+6HcDNQ38vM8HVKM+MJyXO4EtSigwIZhq023A4rA8Am4dMlGhUkPwudDykExYR45oLrssR/Ep7PZCQ1OM3pA== + dependencies: + nan "^2.12.1" + os-homedir@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" @@ -1079,6 +1096,13 @@ vscode-ripgrep@^1.2.5: resolved "https://registry.yarnpkg.com/vscode-ripgrep/-/vscode-ripgrep-1.2.5.tgz#2093c8f36d52bd2dab9eb45b003dd02533c5499c" integrity sha512-n5XBm9od5hahpljw9T8wbkuMnAY7LlAG1OyEEtcCZEX9aCHFuBKSP0IcvciGRTbtWRovNuT83A2iRjt6PL3bLg== +vscode-textmate@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.1.1.tgz#857e836fbc13a376ec624242437e1747d79610a9" + integrity sha512-xBjq9LH6fMhWDhIVkbKlB1JeCu6lT3FI/QKN24Xi4RKPBUm16IhHTqs6Q6SUGewkNsFZGkb1tJdZsuMnlmVpgw== + dependencies: + oniguruma "^7.0.0" + vscode-windows-ca-certs@0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/vscode-windows-ca-certs/-/vscode-windows-ca-certs-0.1.0.tgz#d58eeb40b536130918cfde2b01e6dc7e5c1bd757" diff --git a/src/typings/onigasm-umd.d.ts b/src/typings/onigasm-umd.d.ts new file mode 100644 index 00000000000..151cecebfdd --- /dev/null +++ b/src/typings/onigasm-umd.d.ts @@ -0,0 +1,33 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +declare module "onigasm-umd" { + + function loadWASM(data: string | ArrayBuffer): Promise; + + class OnigString { + constructor(content: string); + readonly content: string; + readonly dispose?: () => void; + } + + class OnigScanner { + constructor(patterns: string[]); + findNextMatchSync(string: string | OnigString, startPosition: number): IOnigMatch; + } + + export interface IOnigCaptureIndex { + index: number + start: number + end: number + length: number + } + + export interface IOnigMatch { + index: number + captureIndices: IOnigCaptureIndex[] + scanner: OnigScanner + } +} \ No newline at end of file diff --git a/src/vs/code/browser/workbench/workbench.html b/src/vs/code/browser/workbench/workbench.html index 92e4cba15f7..6a7cf302b79 100644 --- a/src/vs/code/browser/workbench/workbench.html +++ b/src/vs/code/browser/workbench/workbench.html @@ -14,6 +14,7 @@ diff --git a/src/vs/code/browser/workbench/workbench.js b/src/vs/code/browser/workbench/workbench.js index d20bee1092a..1561a516ffb 100644 --- a/src/vs/code/browser/workbench/workbench.js +++ b/src/vs/code/browser/workbench/workbench.js @@ -21,7 +21,11 @@ // @ts-ignore require.config({ - baseUrl: `${window.location.origin}/out` + baseUrl: `${window.location.origin}/out`, + paths: { + 'vscode-textmate': `${window.location.origin}/node_modules/vscode-textmate/release/main`, + 'onigasm-umd': `${window.location.origin}/node_modules/onigasm-umd/release/main`, + } }); // @ts-ignore diff --git a/src/vs/workbench/browser/web.simpleservices.ts b/src/vs/workbench/browser/web.simpleservices.ts index aa86b9fd6e3..bf262a39a90 100644 --- a/src/vs/workbench/browser/web.simpleservices.ts +++ b/src/vs/workbench/browser/web.simpleservices.ts @@ -40,8 +40,6 @@ import { Schemas } from 'vs/base/common/network'; import { editorMatchesToTextSearchResults, addContextToEditorMatches } from 'vs/workbench/services/search/common/searchHelpers'; import { Disposable, IDisposable } from 'vs/base/common/lifecycle'; import { InMemoryStorageService, IStorageService } from 'vs/platform/storage/common/storage'; -import { ITextMateService, IGrammar as ITextMategrammar } from 'vs/workbench/services/textMate/common/textMateService'; -import { LanguageId, TokenizationRegistry } from 'vs/editor/common/modes'; import { IUpdateService, State } from 'vs/platform/update/common/update'; import { IWindowConfiguration, IPath, IPathsToWaitFor, IWindowService, INativeOpenDialogOptions, IEnterWorkspaceResult, IURIToOpen, IMessageBoxResult, IWindowsService, IOpenSettings } from 'vs/platform/windows/common/windows'; import { IProcessEnvironment } from 'vs/base/common/platform'; @@ -54,7 +52,6 @@ import { IWorkspaceContextService, Workspace, toWorkspaceFolder, IWorkspaceFolde import { ITextResourcePropertiesService } from 'vs/editor/common/services/resourceConfiguration'; import { IKeybindingService } from 'vs/platform/keybinding/common/keybinding'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; -import { Color, RGBA } from 'vs/base/common/color'; import { ITunnelService } from 'vs/platform/remote/common/tunnel'; import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService'; import { IFileService } from 'vs/platform/files/common/files'; @@ -820,26 +817,6 @@ registerSingleton(ITelemetryService, SimpleTelemetryService); //#endregion -//#region Textmate - -TokenizationRegistry.setColorMap([null, new Color(new RGBA(212, 212, 212, 1)), new Color(new RGBA(30, 30, 30, 1))]); - -export class SimpleTextMateService implements ITextMateService { - - _serviceBrand: any; - - readonly onDidEncounterLanguage: Event = Event.None; - - createGrammar(modeId: string): Promise { - // @ts-ignore - return Promise.resolve(undefined); - } -} - -registerSingleton(ITextMateService, SimpleTextMateService, true); - -//#endregion - //#region Text Resource Properties export class SimpleTextResourcePropertiesService extends SimpleResourcePropertiesService { } diff --git a/src/vs/workbench/services/extensions/electron-browser/extensionService.ts b/src/vs/workbench/services/extensions/electron-browser/extensionService.ts index a714a03fc97..be2007b0449 100644 --- a/src/vs/workbench/services/extensions/electron-browser/extensionService.ts +++ b/src/vs/workbench/services/extensions/electron-browser/extensionService.ts @@ -129,7 +129,7 @@ export class ExtensionService extends AbstractExtensionService implements IExten // see https://github.com/Microsoft/vscode/issues/41322 this._lifecycleService.when(LifecyclePhase.Ready).then(() => { // reschedule to ensure this runs after restoring viewlets, panels, and editors - runWhenIdle(async () => { + runWhenIdle(() => { this._initialize(); }, 50 /*max delay*/); }); diff --git a/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts b/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts new file mode 100644 index 00000000000..af7555454f4 --- /dev/null +++ b/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts @@ -0,0 +1,518 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as nls from 'vs/nls'; +import * as dom from 'vs/base/browser/dom'; +import { Color } from 'vs/base/common/color'; +import { onUnexpectedError } from 'vs/base/common/errors'; +import { Emitter, Event } from 'vs/base/common/event'; +import * as resources from 'vs/base/common/resources'; +import * as types from 'vs/base/common/types'; +import { URI } from 'vs/base/common/uri'; +import { TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token'; +import { IState, ITokenizationSupport, LanguageId, TokenMetadata, TokenizationRegistry } from 'vs/editor/common/modes'; +import { nullTokenize2 } from 'vs/editor/common/modes/nullMode'; +import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization'; +import { IModeService } from 'vs/editor/common/services/modeService'; +import { IFileService } from 'vs/platform/files/common/files'; +import { ILogService } from 'vs/platform/log/common/log'; +import { INotificationService } from 'vs/platform/notification/common/notification'; +import { ExtensionMessageCollector } from 'vs/workbench/services/extensions/common/extensionsRegistry'; +import { IEmbeddedLanguagesMap, ITMSyntaxExtensionPoint, TokenTypesContribution, grammarsExtPoint } from 'vs/workbench/services/textMate/common/TMGrammars'; +import { ITextMateService } from 'vs/workbench/services/textMate/common/textMateService'; +import { ITokenColorizationRule, IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService'; +import { IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2, IGrammar, ITokenTypeMap, Registry, StackElement, StandardTokenType, RegistryOptions, IRawGrammar } from 'vscode-textmate'; +import { Disposable, IDisposable, dispose } from 'vs/base/common/lifecycle'; +import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; + +export class TMScopeRegistry { + + private _scopeNameToLanguageRegistration: { [scopeName: string]: TMLanguageRegistration; }; + private _encounteredLanguages: boolean[]; + + private readonly _onDidEncounterLanguage = new Emitter(); + public readonly onDidEncounterLanguage: Event = this._onDidEncounterLanguage.event; + + constructor() { + this.reset(); + } + + public reset(): void { + this._scopeNameToLanguageRegistration = Object.create(null); + this._encounteredLanguages = []; + } + + public register(scopeName: string, grammarLocation: URI, embeddedLanguages?: IEmbeddedLanguagesMap, tokenTypes?: TokenTypesContribution): void { + if (this._scopeNameToLanguageRegistration[scopeName]) { + const existingRegistration = this._scopeNameToLanguageRegistration[scopeName]; + if (!resources.isEqual(existingRegistration.grammarLocation, grammarLocation)) { + console.warn( + `Overwriting grammar scope name to file mapping for scope ${scopeName}.\n` + + `Old grammar file: ${existingRegistration.grammarLocation.toString()}.\n` + + `New grammar file: ${grammarLocation.toString()}` + ); + } + } + this._scopeNameToLanguageRegistration[scopeName] = new TMLanguageRegistration(scopeName, grammarLocation, embeddedLanguages, tokenTypes); + } + + public getLanguageRegistration(scopeName: string): TMLanguageRegistration { + return this._scopeNameToLanguageRegistration[scopeName] || null; + } + + public getGrammarLocation(scopeName: string): URI | null { + let data = this.getLanguageRegistration(scopeName); + return data ? data.grammarLocation : null; + } + + /** + * To be called when tokenization found/hit an embedded language. + */ + public onEncounteredLanguage(languageId: LanguageId): void { + if (!this._encounteredLanguages[languageId]) { + this._encounteredLanguages[languageId] = true; + this._onDidEncounterLanguage.fire(languageId); + } + } +} + +export class TMLanguageRegistration { + _topLevelScopeNameDataBrand: void; + + readonly scopeName: string; + readonly grammarLocation: URI; + readonly embeddedLanguages: IEmbeddedLanguagesMap; + readonly tokenTypes: ITokenTypeMap; + + constructor(scopeName: string, grammarLocation: URI, embeddedLanguages: IEmbeddedLanguagesMap | undefined, tokenTypes: TokenTypesContribution | undefined) { + this.scopeName = scopeName; + this.grammarLocation = grammarLocation; + + // embeddedLanguages handling + this.embeddedLanguages = Object.create(null); + + if (embeddedLanguages) { + // If embeddedLanguages are configured, fill in `this._embeddedLanguages` + let scopes = Object.keys(embeddedLanguages); + for (let i = 0, len = scopes.length; i < len; i++) { + let scope = scopes[i]; + let language = embeddedLanguages[scope]; + if (typeof language !== 'string') { + // never hurts to be too careful + continue; + } + this.embeddedLanguages[scope] = language; + } + } + + this.tokenTypes = Object.create(null); + if (tokenTypes) { + // If tokenTypes is configured, fill in `this._tokenTypes` + const scopes = Object.keys(tokenTypes); + for (const scope of scopes) { + const tokenType = tokenTypes[scope]; + switch (tokenType) { + case 'string': + this.tokenTypes[scope] = StandardTokenType.String; + break; + case 'other': + this.tokenTypes[scope] = StandardTokenType.Other; + break; + case 'comment': + this.tokenTypes[scope] = StandardTokenType.Comment; + break; + } + } + } + } +} + +interface ICreateGrammarResult { + languageId: LanguageId; + grammar: IGrammar; + initialState: StackElement; + containsEmbeddedLanguages: boolean; +} + +export abstract class AbstractTextMateService extends Disposable implements ITextMateService { + public _serviceBrand: any; + + private readonly _onDidEncounterLanguage: Emitter = this._register(new Emitter()); + public readonly onDidEncounterLanguage: Event = this._onDidEncounterLanguage.event; + + private readonly _styleElement: HTMLStyleElement; + private readonly _createdModes: string[]; + + protected _scopeRegistry: TMScopeRegistry; + private _injections: { [scopeName: string]: string[]; }; + private _injectedEmbeddedLanguages: { [scopeName: string]: IEmbeddedLanguagesMap[]; }; + protected _languageToScope: Map; + private _grammarRegistry: Promise<[Registry, StackElement]> | null; + private _tokenizersRegistrations: IDisposable[]; + private _currentTokenColors: ITokenColorizationRule[] | null; + private _themeListener: IDisposable | null; + + constructor( + @IModeService private readonly _modeService: IModeService, + @IWorkbenchThemeService private readonly _themeService: IWorkbenchThemeService, + @IFileService private readonly _fileService: IFileService, + @INotificationService private readonly _notificationService: INotificationService, + @ILogService private readonly _logService: ILogService, + @IConfigurationService private readonly _configurationService: IConfigurationService + ) { + super(); + this._styleElement = dom.createStyleSheet(); + this._styleElement.className = 'vscode-tokens-styles'; + this._createdModes = []; + this._scopeRegistry = new TMScopeRegistry(); + this._scopeRegistry.onDidEncounterLanguage((language) => this._onDidEncounterLanguage.fire(language)); + this._injections = {}; + this._injectedEmbeddedLanguages = {}; + this._languageToScope = new Map(); + this._grammarRegistry = null; + this._tokenizersRegistrations = []; + this._currentTokenColors = null; + this._themeListener = null; + + grammarsExtPoint.setHandler((extensions) => { + this._scopeRegistry.reset(); + this._injections = {}; + this._injectedEmbeddedLanguages = {}; + this._languageToScope = new Map(); + this._grammarRegistry = null; + this._tokenizersRegistrations = dispose(this._tokenizersRegistrations); + this._currentTokenColors = null; + if (this._themeListener) { + this._themeListener.dispose(); + this._themeListener = null; + } + + for (const extension of extensions) { + let grammars = extension.value; + for (const grammar of grammars) { + this._handleGrammarExtensionPointUser(extension.description.extensionLocation, grammar, extension.collector); + } + } + + for (const createMode of this._createdModes) { + this._registerDefinitionIfAvailable(createMode); + } + }); + + // Generate some color map until the grammar registry is loaded + let colorTheme = this._themeService.getColorTheme(); + let defaultForeground: Color = Color.transparent; + let defaultBackground: Color = Color.transparent; + for (let i = 0, len = colorTheme.tokenColors.length; i < len; i++) { + let rule = colorTheme.tokenColors[i]; + if (!rule.scope && rule.settings) { + if (rule.settings.foreground) { + defaultForeground = Color.fromHex(rule.settings.foreground); + } + if (rule.settings.background) { + defaultBackground = Color.fromHex(rule.settings.background); + } + } + } + TokenizationRegistry.setColorMap([null!, defaultForeground, defaultBackground]); + + this._modeService.onDidCreateMode((mode) => { + let modeId = mode.getId(); + this._createdModes.push(modeId); + this._registerDefinitionIfAvailable(modeId); + }); + } + + private _registerDefinitionIfAvailable(modeId: string): void { + if (this._languageToScope.has(modeId)) { + const promise = this._createGrammar(modeId).then((r) => { + return new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService, this._configurationService); + }, e => { + onUnexpectedError(e); + return null; + }); + this._tokenizersRegistrations.push(TokenizationRegistry.registerPromise(modeId, promise)); + } + } + + protected _getRegistryOptions(parseRawGrammar: (content: string, filePath: string) => IRawGrammar): RegistryOptions { + return { + loadGrammar: async (scopeName: string) => { + const location = this._scopeRegistry.getGrammarLocation(scopeName); + if (!location) { + this._logService.trace(`No grammar found for scope ${scopeName}`); + return null; + } + try { + const content = await this._fileService.readFile(location); + return parseRawGrammar(content.value.toString(), location.path); + } catch (e) { + this._logService.error(`Unable to load and parse grammar for scope ${scopeName} from ${location}`, e); + return null; + } + }, + getInjections: (scopeName: string) => { + const scopeParts = scopeName.split('.'); + let injections: string[] = []; + for (let i = 1; i <= scopeParts.length; i++) { + const subScopeName = scopeParts.slice(0, i).join('.'); + injections = [...injections, ...(this._injections[subScopeName] || [])]; + } + return injections; + } + }; + } + + private async _createGrammarRegistry(): Promise<[Registry, StackElement]> { + const { Registry, INITIAL, parseRawGrammar } = await this._loadVSCodeTextmate(); + const grammarRegistry = new Registry(this._getRegistryOptions(parseRawGrammar)); + this._updateTheme(grammarRegistry); + this._themeListener = this._themeService.onDidColorThemeChange((e) => this._updateTheme(grammarRegistry)); + return <[Registry, StackElement]>[grammarRegistry, INITIAL]; + } + + private _getOrCreateGrammarRegistry(): Promise<[Registry, StackElement]> { + if (!this._grammarRegistry) { + this._grammarRegistry = this._createGrammarRegistry(); + } + return this._grammarRegistry; + } + + private static _toColorMap(colorMap: string[]): Color[] { + let result: Color[] = [null!]; + for (let i = 1, len = colorMap.length; i < len; i++) { + result[i] = Color.fromHex(colorMap[i]); + } + return result; + } + + private _updateTheme(grammarRegistry: Registry): void { + let colorTheme = this._themeService.getColorTheme(); + if (!this.compareTokenRules(colorTheme.tokenColors)) { + return; + } + grammarRegistry.setTheme({ name: colorTheme.label, settings: colorTheme.tokenColors }); + let colorMap = AbstractTextMateService._toColorMap(grammarRegistry.getColorMap()); + let cssRules = generateTokensCSSForColorMap(colorMap); + this._styleElement.innerHTML = cssRules; + TokenizationRegistry.setColorMap(colorMap); + } + + private compareTokenRules(newRules: ITokenColorizationRule[]): boolean { + let currRules = this._currentTokenColors; + this._currentTokenColors = newRules; + if (!newRules || !currRules || newRules.length !== currRules.length) { + return true; + } + for (let i = newRules.length - 1; i >= 0; i--) { + let r1 = newRules[i]; + let r2 = currRules[i]; + if (r1.scope !== r2.scope) { + return true; + } + let s1 = r1.settings; + let s2 = r2.settings; + if (s1 && s2) { + if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground || s1.background !== s2.background) { + return true; + } + } else if (!s1 || !s2) { + return true; + } + } + return false; + } + + private _handleGrammarExtensionPointUser(extensionLocation: URI, syntax: ITMSyntaxExtensionPoint, collector: ExtensionMessageCollector): void { + if (syntax.language && ((typeof syntax.language !== 'string') || !this._modeService.isRegisteredMode(syntax.language))) { + collector.error(nls.localize('invalid.language', "Unknown language in `contributes.{0}.language`. Provided value: {1}", grammarsExtPoint.name, String(syntax.language))); + return; + } + if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) { + collector.error(nls.localize('invalid.scopeName', "Expected string in `contributes.{0}.scopeName`. Provided value: {1}", grammarsExtPoint.name, String(syntax.scopeName))); + return; + } + if (!syntax.path || (typeof syntax.path !== 'string')) { + collector.error(nls.localize('invalid.path.0', "Expected string in `contributes.{0}.path`. Provided value: {1}", grammarsExtPoint.name, String(syntax.path))); + return; + } + if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || syntax.injectTo.some(scope => typeof scope !== 'string'))) { + collector.error(nls.localize('invalid.injectTo', "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.injectTo))); + return; + } + if (syntax.embeddedLanguages && !types.isObject(syntax.embeddedLanguages)) { + collector.error(nls.localize('invalid.embeddedLanguages', "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.embeddedLanguages))); + return; + } + + if (syntax.tokenTypes && !types.isObject(syntax.tokenTypes)) { + collector.error(nls.localize('invalid.tokenTypes', "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.tokenTypes))); + return; + } + + const grammarLocation = resources.joinPath(extensionLocation, syntax.path); + if (!resources.isEqualOrParent(grammarLocation, extensionLocation)) { + collector.warn(nls.localize('invalid.path.1', "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.", grammarsExtPoint.name, grammarLocation.path, extensionLocation.path)); + } + + this._scopeRegistry.register(syntax.scopeName, grammarLocation, syntax.embeddedLanguages, syntax.tokenTypes); + + if (syntax.injectTo) { + for (let injectScope of syntax.injectTo) { + let injections = this._injections[injectScope]; + if (!injections) { + this._injections[injectScope] = injections = []; + } + injections.push(syntax.scopeName); + } + + if (syntax.embeddedLanguages) { + for (let injectScope of syntax.injectTo) { + let injectedEmbeddedLanguages = this._injectedEmbeddedLanguages[injectScope]; + if (!injectedEmbeddedLanguages) { + this._injectedEmbeddedLanguages[injectScope] = injectedEmbeddedLanguages = []; + } + injectedEmbeddedLanguages.push(syntax.embeddedLanguages); + } + } + } + + let modeId = syntax.language; + if (modeId) { + this._languageToScope.set(modeId, syntax.scopeName); + } + } + + private _resolveEmbeddedLanguages(embeddedLanguages: IEmbeddedLanguagesMap): IEmbeddedLanguagesMap2 { + let scopes = Object.keys(embeddedLanguages); + let result: IEmbeddedLanguagesMap2 = Object.create(null); + for (let i = 0, len = scopes.length; i < len; i++) { + let scope = scopes[i]; + let language = embeddedLanguages[scope]; + let languageIdentifier = this._modeService.getLanguageIdentifier(language); + if (languageIdentifier) { + result[scope] = languageIdentifier.id; + } + } + return result; + } + + public async createGrammar(modeId: string): Promise { + const { grammar } = await this._createGrammar(modeId); + return grammar; + } + + private async _createGrammar(modeId: string): Promise { + const scopeName = this._languageToScope.get(modeId); + if (typeof scopeName !== 'string') { + // No TM grammar defined + return Promise.reject(new Error(nls.localize('no-tm-grammar', "No TM Grammar registered for this language."))); + } + const languageRegistration = this._scopeRegistry.getLanguageRegistration(scopeName); + if (!languageRegistration) { + // No TM grammar defined + return Promise.reject(new Error(nls.localize('no-tm-grammar', "No TM Grammar registered for this language."))); + } + let embeddedLanguages = this._resolveEmbeddedLanguages(languageRegistration.embeddedLanguages); + let rawInjectedEmbeddedLanguages = this._injectedEmbeddedLanguages[scopeName]; + if (rawInjectedEmbeddedLanguages) { + let injectedEmbeddedLanguages: IEmbeddedLanguagesMap2[] = rawInjectedEmbeddedLanguages.map(this._resolveEmbeddedLanguages.bind(this)); + for (const injected of injectedEmbeddedLanguages) { + for (const scope of Object.keys(injected)) { + embeddedLanguages[scope] = injected[scope]; + } + } + } + + let languageId = this._modeService.getLanguageIdentifier(modeId)!.id; + let containsEmbeddedLanguages = (Object.keys(embeddedLanguages).length > 0); + + const [grammarRegistry, initialState] = await this._getOrCreateGrammarRegistry(); + const grammar = await grammarRegistry.loadGrammarWithConfiguration(scopeName, languageId, { embeddedLanguages, tokenTypes: languageRegistration.tokenTypes }); + return { + languageId: languageId, + grammar: grammar, + initialState: initialState, + containsEmbeddedLanguages: containsEmbeddedLanguages + }; + } + + protected abstract _loadVSCodeTextmate(): Promise; +} + +class TMTokenization implements ITokenizationSupport { + + private readonly _scopeRegistry: TMScopeRegistry; + private readonly _languageId: LanguageId; + private readonly _grammar: IGrammar; + private readonly _containsEmbeddedLanguages: boolean; + private readonly _seenLanguages: boolean[]; + private readonly _initialState: StackElement; + private _maxTokenizationLineLength: number; + private _tokenizationWarningAlreadyShown: boolean; + + constructor(scopeRegistry: TMScopeRegistry, languageId: LanguageId, grammar: IGrammar, initialState: StackElement, containsEmbeddedLanguages: boolean, @INotificationService private readonly notificationService: INotificationService, @IConfigurationService readonly configurationService: IConfigurationService) { + this._scopeRegistry = scopeRegistry; + this._languageId = languageId; + this._grammar = grammar; + this._initialState = initialState; + this._containsEmbeddedLanguages = containsEmbeddedLanguages; + this._seenLanguages = []; + this._maxTokenizationLineLength = configurationService.getValue('editor.maxTokenizationLineLength'); + } + + public getInitialState(): IState { + return this._initialState; + } + + public tokenize(line: string, state: IState, offsetDelta: number): TokenizationResult { + throw new Error('Not supported!'); + } + + public tokenize2(line: string, state: StackElement, offsetDelta: number): TokenizationResult2 { + if (offsetDelta !== 0) { + throw new Error('Unexpected: offsetDelta should be 0.'); + } + + // Do not attempt to tokenize if a line is too long + if (line.length >= this._maxTokenizationLineLength) { + if (!this._tokenizationWarningAlreadyShown) { + this._tokenizationWarningAlreadyShown = true; + this.notificationService.warn(nls.localize('too many characters', "Tokenization is skipped for long lines for performance reasons. The length of a long line can be configured via `editor.maxTokenizationLineLength`.")); + } + console.log(`Line (${line.substr(0, 15)}...): longer than ${this._maxTokenizationLineLength} characters, tokenization skipped.`); + return nullTokenize2(this._languageId, line, state, offsetDelta); + } + + let textMateResult = this._grammar.tokenizeLine2(line, state); + + if (this._containsEmbeddedLanguages) { + let seenLanguages = this._seenLanguages; + let tokens = textMateResult.tokens; + + // Must check if any of the embedded languages was hit + for (let i = 0, len = (tokens.length >>> 1); i < len; i++) { + let metadata = tokens[(i << 1) + 1]; + let languageId = TokenMetadata.getLanguageId(metadata); + + if (!seenLanguages[languageId]) { + seenLanguages[languageId] = true; + this._scopeRegistry.onEncounteredLanguage(languageId); + } + } + } + + let endState: StackElement; + // try to save an object if possible + if (state.equals(textMateResult.ruleStack)) { + endState = state; + } else { + endState = textMateResult.ruleStack; + + } + + return new TokenizationResult2(textMateResult.tokens, endState); + } +} diff --git a/src/vs/workbench/services/textMate/browser/textMateService.ts b/src/vs/workbench/services/textMate/browser/textMateService.ts new file mode 100644 index 00000000000..c126bf74d25 --- /dev/null +++ b/src/vs/workbench/services/textMate/browser/textMateService.ts @@ -0,0 +1,66 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ITextMateService } from 'vs/workbench/services/textMate/common/textMateService'; +import { registerSingleton } from 'vs/platform/instantiation/common/extensions'; +import { AbstractTextMateService } from 'vs/workbench/services/textMate/browser/abstractTextMateService'; +import * as vscodeTextmate from 'vscode-textmate'; +import * as onigasm from 'onigasm-umd'; +import { IModeService } from 'vs/editor/common/services/modeService'; +import { IFileService } from 'vs/platform/files/common/files'; +import { ILogService } from 'vs/platform/log/common/log'; +import { INotificationService } from 'vs/platform/notification/common/notification'; +import { IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService'; +import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; + +export class TextMateService extends AbstractTextMateService { + + constructor( + @IModeService modeService: IModeService, + @IWorkbenchThemeService themeService: IWorkbenchThemeService, + @IFileService fileService: IFileService, + @INotificationService notificationService: INotificationService, + @ILogService logService: ILogService, + @IConfigurationService configurationService: IConfigurationService + ) { + super(modeService, themeService, fileService, notificationService, logService, configurationService); + } + + protected _loadVSCodeTextmate(): Promise { + return import('vscode-textmate'); + } + + protected _getRegistryOptions(parseRawGrammar: (content: string, filePath: string) => vscodeTextmate.IRawGrammar): vscodeTextmate.RegistryOptions { + const result = super._getRegistryOptions(parseRawGrammar); + result.getOnigLib = () => loadOnigasm(); + return result; + } +} + +let onigasmPromise: Promise | null = null; +async function loadOnigasm(): Promise { + if (!onigasmPromise) { + onigasmPromise = doLoadOnigasm(); + } + return onigasmPromise; +} + +async function doLoadOnigasm(): Promise { + const wasmBytes = await loadOnigasmWASM(); + await onigasm.loadWASM(wasmBytes); + return { + createOnigScanner(patterns: string[]) { return new onigasm.OnigScanner(patterns); }, + createOnigString(s: string) { return new onigasm.OnigString(s); } + }; +} + +async function loadOnigasmWASM(): Promise { + const wasmPath = require.toUrl('onigasm-umd/../onigasm.wasm'); + const response = await fetch(wasmPath); + const bytes = await response.arrayBuffer(); + return bytes; +} + +registerSingleton(ITextMateService, TextMateService); diff --git a/src/vs/workbench/services/textMate/electron-browser/cgmanifest.json b/src/vs/workbench/services/textMate/common/cgmanifest.json similarity index 100% rename from src/vs/workbench/services/textMate/electron-browser/cgmanifest.json rename to src/vs/workbench/services/textMate/common/cgmanifest.json diff --git a/src/vs/workbench/services/textMate/electron-browser/textMateService.ts b/src/vs/workbench/services/textMate/electron-browser/textMateService.ts index 94e175490fa..4b5ec093cd9 100644 --- a/src/vs/workbench/services/textMate/electron-browser/textMateService.ts +++ b/src/vs/workbench/services/textMate/electron-browser/textMateService.ts @@ -3,512 +3,14 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import * as nls from 'vs/nls'; -import * as dom from 'vs/base/browser/dom'; -import { Color } from 'vs/base/common/color'; -import { onUnexpectedError } from 'vs/base/common/errors'; -import { Emitter, Event } from 'vs/base/common/event'; -import * as resources from 'vs/base/common/resources'; -import * as types from 'vs/base/common/types'; -import { URI } from 'vs/base/common/uri'; -import { TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token'; -import { IState, ITokenizationSupport, LanguageId, TokenMetadata, TokenizationRegistry } from 'vs/editor/common/modes'; -import { nullTokenize2 } from 'vs/editor/common/modes/nullMode'; -import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization'; -import { IModeService } from 'vs/editor/common/services/modeService'; -import { IFileService } from 'vs/platform/files/common/files'; -import { ILogService } from 'vs/platform/log/common/log'; -import { INotificationService } from 'vs/platform/notification/common/notification'; -import { ExtensionMessageCollector } from 'vs/workbench/services/extensions/common/extensionsRegistry'; -import { IEmbeddedLanguagesMap, ITMSyntaxExtensionPoint, TokenTypesContribution, grammarsExtPoint } from 'vs/workbench/services/textMate/common/TMGrammars'; import { ITextMateService } from 'vs/workbench/services/textMate/common/textMateService'; -import { ITokenColorizationRule, IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService'; -import { IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2, IGrammar, ITokenTypeMap, Registry, StackElement, StandardTokenType } from 'vscode-textmate'; -import { Disposable, IDisposable, dispose } from 'vs/base/common/lifecycle'; -import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { registerSingleton } from 'vs/platform/instantiation/common/extensions'; +import { AbstractTextMateService } from 'vs/workbench/services/textMate/browser/abstractTextMateService'; -export class TMScopeRegistry { +export class TextMateService extends AbstractTextMateService { - private _scopeNameToLanguageRegistration: { [scopeName: string]: TMLanguageRegistration; }; - private _encounteredLanguages: boolean[]; - - private readonly _onDidEncounterLanguage = new Emitter(); - public readonly onDidEncounterLanguage: Event = this._onDidEncounterLanguage.event; - - constructor() { - this.reset(); - } - - public reset(): void { - this._scopeNameToLanguageRegistration = Object.create(null); - this._encounteredLanguages = []; - } - - public register(scopeName: string, grammarLocation: URI, embeddedLanguages?: IEmbeddedLanguagesMap, tokenTypes?: TokenTypesContribution): void { - if (this._scopeNameToLanguageRegistration[scopeName]) { - const existingRegistration = this._scopeNameToLanguageRegistration[scopeName]; - if (!resources.isEqual(existingRegistration.grammarLocation, grammarLocation)) { - console.warn( - `Overwriting grammar scope name to file mapping for scope ${scopeName}.\n` + - `Old grammar file: ${existingRegistration.grammarLocation.toString()}.\n` + - `New grammar file: ${grammarLocation.toString()}` - ); - } - } - this._scopeNameToLanguageRegistration[scopeName] = new TMLanguageRegistration(scopeName, grammarLocation, embeddedLanguages, tokenTypes); - } - - public getLanguageRegistration(scopeName: string): TMLanguageRegistration { - return this._scopeNameToLanguageRegistration[scopeName] || null; - } - - public getGrammarLocation(scopeName: string): URI | null { - let data = this.getLanguageRegistration(scopeName); - return data ? data.grammarLocation : null; - } - - /** - * To be called when tokenization found/hit an embedded language. - */ - public onEncounteredLanguage(languageId: LanguageId): void { - if (!this._encounteredLanguages[languageId]) { - this._encounteredLanguages[languageId] = true; - this._onDidEncounterLanguage.fire(languageId); - } - } -} - -export class TMLanguageRegistration { - _topLevelScopeNameDataBrand: void; - - readonly scopeName: string; - readonly grammarLocation: URI; - readonly embeddedLanguages: IEmbeddedLanguagesMap; - readonly tokenTypes: ITokenTypeMap; - - constructor(scopeName: string, grammarLocation: URI, embeddedLanguages: IEmbeddedLanguagesMap | undefined, tokenTypes: TokenTypesContribution | undefined) { - this.scopeName = scopeName; - this.grammarLocation = grammarLocation; - - // embeddedLanguages handling - this.embeddedLanguages = Object.create(null); - - if (embeddedLanguages) { - // If embeddedLanguages are configured, fill in `this._embeddedLanguages` - let scopes = Object.keys(embeddedLanguages); - for (let i = 0, len = scopes.length; i < len; i++) { - let scope = scopes[i]; - let language = embeddedLanguages[scope]; - if (typeof language !== 'string') { - // never hurts to be too careful - continue; - } - this.embeddedLanguages[scope] = language; - } - } - - this.tokenTypes = Object.create(null); - if (tokenTypes) { - // If tokenTypes is configured, fill in `this._tokenTypes` - const scopes = Object.keys(tokenTypes); - for (const scope of scopes) { - const tokenType = tokenTypes[scope]; - switch (tokenType) { - case 'string': - this.tokenTypes[scope] = StandardTokenType.String; - break; - case 'other': - this.tokenTypes[scope] = StandardTokenType.Other; - break; - case 'comment': - this.tokenTypes[scope] = StandardTokenType.Comment; - break; - } - } - } - } -} - -interface ICreateGrammarResult { - languageId: LanguageId; - grammar: IGrammar; - initialState: StackElement; - containsEmbeddedLanguages: boolean; -} - -export class TextMateService extends Disposable implements ITextMateService { - public _serviceBrand: any; - - private readonly _onDidEncounterLanguage: Emitter = this._register(new Emitter()); - public readonly onDidEncounterLanguage: Event = this._onDidEncounterLanguage.event; - - private readonly _styleElement: HTMLStyleElement; - private readonly _createdModes: string[]; - - private _scopeRegistry: TMScopeRegistry; - private _injections: { [scopeName: string]: string[]; }; - private _injectedEmbeddedLanguages: { [scopeName: string]: IEmbeddedLanguagesMap[]; }; - private _languageToScope: Map; - private _grammarRegistry: Promise<[Registry, StackElement]> | null; - private _tokenizersRegistrations: IDisposable[]; - private _currentTokenColors: ITokenColorizationRule[] | null; - private _themeListener: IDisposable | null; - - constructor( - @IModeService private readonly _modeService: IModeService, - @IWorkbenchThemeService private readonly _themeService: IWorkbenchThemeService, - @IFileService private readonly _fileService: IFileService, - @INotificationService private readonly _notificationService: INotificationService, - @ILogService private readonly _logService: ILogService, - @IConfigurationService private readonly _configurationService: IConfigurationService - ) { - super(); - this._styleElement = dom.createStyleSheet(); - this._styleElement.className = 'vscode-tokens-styles'; - this._createdModes = []; - this._scopeRegistry = new TMScopeRegistry(); - this._scopeRegistry.onDidEncounterLanguage((language) => this._onDidEncounterLanguage.fire(language)); - this._injections = {}; - this._injectedEmbeddedLanguages = {}; - this._languageToScope = new Map(); - this._grammarRegistry = null; - this._tokenizersRegistrations = []; - this._currentTokenColors = null; - this._themeListener = null; - - grammarsExtPoint.setHandler((extensions) => { - this._scopeRegistry.reset(); - this._injections = {}; - this._injectedEmbeddedLanguages = {}; - this._languageToScope = new Map(); - this._grammarRegistry = null; - this._tokenizersRegistrations = dispose(this._tokenizersRegistrations); - this._currentTokenColors = null; - if (this._themeListener) { - this._themeListener.dispose(); - this._themeListener = null; - } - - for (const extension of extensions) { - let grammars = extension.value; - for (const grammar of grammars) { - this._handleGrammarExtensionPointUser(extension.description.extensionLocation, grammar, extension.collector); - } - } - - for (const createMode of this._createdModes) { - this._registerDefinitionIfAvailable(createMode); - } - }); - - // Generate some color map until the grammar registry is loaded - let colorTheme = this._themeService.getColorTheme(); - let defaultForeground: Color = Color.transparent; - let defaultBackground: Color = Color.transparent; - for (let i = 0, len = colorTheme.tokenColors.length; i < len; i++) { - let rule = colorTheme.tokenColors[i]; - if (!rule.scope && rule.settings) { - if (rule.settings.foreground) { - defaultForeground = Color.fromHex(rule.settings.foreground); - } - if (rule.settings.background) { - defaultBackground = Color.fromHex(rule.settings.background); - } - } - } - TokenizationRegistry.setColorMap([null!, defaultForeground, defaultBackground]); - - this._modeService.onDidCreateMode((mode) => { - let modeId = mode.getId(); - this._createdModes.push(modeId); - this._registerDefinitionIfAvailable(modeId); - }); - } - - private _registerDefinitionIfAvailable(modeId: string): void { - if (this._languageToScope.has(modeId)) { - const promise = this._createGrammar(modeId).then((r) => { - return new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService, this._configurationService); - }, e => { - onUnexpectedError(e); - return null; - }); - this._tokenizersRegistrations.push(TokenizationRegistry.registerPromise(modeId, promise)); - } - } - - private async _createGrammarRegistry(): Promise<[Registry, StackElement]> { - const { Registry, INITIAL, parseRawGrammar } = await import('vscode-textmate'); - const grammarRegistry = new Registry({ - loadGrammar: async (scopeName: string) => { - const location = this._scopeRegistry.getGrammarLocation(scopeName); - if (!location) { - this._logService.trace(`No grammar found for scope ${scopeName}`); - return null; - } - try { - const content = await this._fileService.readFile(location); - return parseRawGrammar(content.value.toString(), location.path); - } catch (e) { - this._logService.error(`Unable to load and parse grammar for scope ${scopeName} from ${location}`, e); - return null; - } - }, - getInjections: (scopeName: string) => { - const scopeParts = scopeName.split('.'); - let injections: string[] = []; - for (let i = 1; i <= scopeParts.length; i++) { - const subScopeName = scopeParts.slice(0, i).join('.'); - injections = [...injections, ...(this._injections[subScopeName] || [])]; - } - return injections; - } - }); - this._updateTheme(grammarRegistry); - this._themeListener = this._themeService.onDidColorThemeChange((e) => this._updateTheme(grammarRegistry)); - return <[Registry, StackElement]>[grammarRegistry, INITIAL]; - } - - private _getOrCreateGrammarRegistry(): Promise<[Registry, StackElement]> { - if (!this._grammarRegistry) { - this._grammarRegistry = this._createGrammarRegistry(); - } - return this._grammarRegistry; - } - - private static _toColorMap(colorMap: string[]): Color[] { - let result: Color[] = [null!]; - for (let i = 1, len = colorMap.length; i < len; i++) { - result[i] = Color.fromHex(colorMap[i]); - } - return result; - } - - private _updateTheme(grammarRegistry: Registry): void { - let colorTheme = this._themeService.getColorTheme(); - if (!this.compareTokenRules(colorTheme.tokenColors)) { - return; - } - grammarRegistry.setTheme({ name: colorTheme.label, settings: colorTheme.tokenColors }); - let colorMap = TextMateService._toColorMap(grammarRegistry.getColorMap()); - let cssRules = generateTokensCSSForColorMap(colorMap); - this._styleElement.innerHTML = cssRules; - TokenizationRegistry.setColorMap(colorMap); - } - - private compareTokenRules(newRules: ITokenColorizationRule[]): boolean { - let currRules = this._currentTokenColors; - this._currentTokenColors = newRules; - if (!newRules || !currRules || newRules.length !== currRules.length) { - return true; - } - for (let i = newRules.length - 1; i >= 0; i--) { - let r1 = newRules[i]; - let r2 = currRules[i]; - if (r1.scope !== r2.scope) { - return true; - } - let s1 = r1.settings; - let s2 = r2.settings; - if (s1 && s2) { - if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground || s1.background !== s2.background) { - return true; - } - } else if (!s1 || !s2) { - return true; - } - } - return false; - } - - private _handleGrammarExtensionPointUser(extensionLocation: URI, syntax: ITMSyntaxExtensionPoint, collector: ExtensionMessageCollector): void { - if (syntax.language && ((typeof syntax.language !== 'string') || !this._modeService.isRegisteredMode(syntax.language))) { - collector.error(nls.localize('invalid.language', "Unknown language in `contributes.{0}.language`. Provided value: {1}", grammarsExtPoint.name, String(syntax.language))); - return; - } - if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) { - collector.error(nls.localize('invalid.scopeName', "Expected string in `contributes.{0}.scopeName`. Provided value: {1}", grammarsExtPoint.name, String(syntax.scopeName))); - return; - } - if (!syntax.path || (typeof syntax.path !== 'string')) { - collector.error(nls.localize('invalid.path.0', "Expected string in `contributes.{0}.path`. Provided value: {1}", grammarsExtPoint.name, String(syntax.path))); - return; - } - if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || syntax.injectTo.some(scope => typeof scope !== 'string'))) { - collector.error(nls.localize('invalid.injectTo', "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.injectTo))); - return; - } - if (syntax.embeddedLanguages && !types.isObject(syntax.embeddedLanguages)) { - collector.error(nls.localize('invalid.embeddedLanguages', "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.embeddedLanguages))); - return; - } - - if (syntax.tokenTypes && !types.isObject(syntax.tokenTypes)) { - collector.error(nls.localize('invalid.tokenTypes', "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.tokenTypes))); - return; - } - - const grammarLocation = resources.joinPath(extensionLocation, syntax.path); - if (!resources.isEqualOrParent(grammarLocation, extensionLocation)) { - collector.warn(nls.localize('invalid.path.1', "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.", grammarsExtPoint.name, grammarLocation.path, extensionLocation.path)); - } - - this._scopeRegistry.register(syntax.scopeName, grammarLocation, syntax.embeddedLanguages, syntax.tokenTypes); - - if (syntax.injectTo) { - for (let injectScope of syntax.injectTo) { - let injections = this._injections[injectScope]; - if (!injections) { - this._injections[injectScope] = injections = []; - } - injections.push(syntax.scopeName); - } - - if (syntax.embeddedLanguages) { - for (let injectScope of syntax.injectTo) { - let injectedEmbeddedLanguages = this._injectedEmbeddedLanguages[injectScope]; - if (!injectedEmbeddedLanguages) { - this._injectedEmbeddedLanguages[injectScope] = injectedEmbeddedLanguages = []; - } - injectedEmbeddedLanguages.push(syntax.embeddedLanguages); - } - } - } - - let modeId = syntax.language; - if (modeId) { - this._languageToScope.set(modeId, syntax.scopeName); - } - } - - private _resolveEmbeddedLanguages(embeddedLanguages: IEmbeddedLanguagesMap): IEmbeddedLanguagesMap2 { - let scopes = Object.keys(embeddedLanguages); - let result: IEmbeddedLanguagesMap2 = Object.create(null); - for (let i = 0, len = scopes.length; i < len; i++) { - let scope = scopes[i]; - let language = embeddedLanguages[scope]; - let languageIdentifier = this._modeService.getLanguageIdentifier(language); - if (languageIdentifier) { - result[scope] = languageIdentifier.id; - } - } - return result; - } - - public async createGrammar(modeId: string): Promise { - const { grammar } = await this._createGrammar(modeId); - return grammar; - } - - private async _createGrammar(modeId: string): Promise { - const scopeName = this._languageToScope.get(modeId); - if (typeof scopeName !== 'string') { - // No TM grammar defined - return Promise.reject(new Error(nls.localize('no-tm-grammar', "No TM Grammar registered for this language."))); - } - const languageRegistration = this._scopeRegistry.getLanguageRegistration(scopeName); - if (!languageRegistration) { - // No TM grammar defined - return Promise.reject(new Error(nls.localize('no-tm-grammar', "No TM Grammar registered for this language."))); - } - let embeddedLanguages = this._resolveEmbeddedLanguages(languageRegistration.embeddedLanguages); - let rawInjectedEmbeddedLanguages = this._injectedEmbeddedLanguages[scopeName]; - if (rawInjectedEmbeddedLanguages) { - let injectedEmbeddedLanguages: IEmbeddedLanguagesMap2[] = rawInjectedEmbeddedLanguages.map(this._resolveEmbeddedLanguages.bind(this)); - for (const injected of injectedEmbeddedLanguages) { - for (const scope of Object.keys(injected)) { - embeddedLanguages[scope] = injected[scope]; - } - } - } - - let languageId = this._modeService.getLanguageIdentifier(modeId)!.id; - let containsEmbeddedLanguages = (Object.keys(embeddedLanguages).length > 0); - - const [grammarRegistry, initialState] = await this._getOrCreateGrammarRegistry(); - const grammar = await grammarRegistry.loadGrammarWithConfiguration(scopeName, languageId, { embeddedLanguages, tokenTypes: languageRegistration.tokenTypes }); - return { - languageId: languageId, - grammar: grammar, - initialState: initialState, - containsEmbeddedLanguages: containsEmbeddedLanguages - }; - } -} - -class TMTokenization implements ITokenizationSupport { - - private readonly _scopeRegistry: TMScopeRegistry; - private readonly _languageId: LanguageId; - private readonly _grammar: IGrammar; - private readonly _containsEmbeddedLanguages: boolean; - private readonly _seenLanguages: boolean[]; - private readonly _initialState: StackElement; - private _maxTokenizationLineLength: number; - private _tokenizationWarningAlreadyShown: boolean; - - constructor(scopeRegistry: TMScopeRegistry, languageId: LanguageId, grammar: IGrammar, initialState: StackElement, containsEmbeddedLanguages: boolean, @INotificationService private readonly notificationService: INotificationService, @IConfigurationService readonly configurationService: IConfigurationService) { - this._scopeRegistry = scopeRegistry; - this._languageId = languageId; - this._grammar = grammar; - this._initialState = initialState; - this._containsEmbeddedLanguages = containsEmbeddedLanguages; - this._seenLanguages = []; - this._maxTokenizationLineLength = configurationService.getValue('editor.maxTokenizationLineLength'); - } - - public getInitialState(): IState { - return this._initialState; - } - - public tokenize(line: string, state: IState, offsetDelta: number): TokenizationResult { - throw new Error('Not supported!'); - } - - public tokenize2(line: string, state: StackElement, offsetDelta: number): TokenizationResult2 { - if (offsetDelta !== 0) { - throw new Error('Unexpected: offsetDelta should be 0.'); - } - - // Do not attempt to tokenize if a line is too long - if (line.length >= this._maxTokenizationLineLength) { - if (!this._tokenizationWarningAlreadyShown) { - this._tokenizationWarningAlreadyShown = true; - this.notificationService.warn(nls.localize('too many characters', "Tokenization is skipped for long lines for performance reasons. The length of a long line can be configured via `editor.maxTokenizationLineLength`.")); - } - console.log(`Line (${line.substr(0, 15)}...): longer than ${this._maxTokenizationLineLength} characters, tokenization skipped.`); - return nullTokenize2(this._languageId, line, state, offsetDelta); - } - - let textMateResult = this._grammar.tokenizeLine2(line, state); - - if (this._containsEmbeddedLanguages) { - let seenLanguages = this._seenLanguages; - let tokens = textMateResult.tokens; - - // Must check if any of the embedded languages was hit - for (let i = 0, len = (tokens.length >>> 1); i < len; i++) { - let metadata = tokens[(i << 1) + 1]; - let languageId = TokenMetadata.getLanguageId(metadata); - - if (!seenLanguages[languageId]) { - seenLanguages[languageId] = true; - this._scopeRegistry.onEncounteredLanguage(languageId); - } - } - } - - let endState: StackElement; - // try to save an object if possible - if (state.equals(textMateResult.ruleStack)) { - endState = state; - } else { - endState = textMateResult.ruleStack; - - } - - return new TokenizationResult2(textMateResult.tokens, endState); + protected _loadVSCodeTextmate(): Promise { + return import('vscode-textmate'); } } diff --git a/src/vs/workbench/workbench.web.main.ts b/src/vs/workbench/workbench.web.main.ts index 4dc252c0c3f..5530494f8a2 100644 --- a/src/vs/workbench/workbench.web.main.ts +++ b/src/vs/workbench/workbench.web.main.ts @@ -101,7 +101,7 @@ import 'vs/platform/dialogs/browser/dialogService'; import 'vs/workbench/services/bulkEdit/browser/bulkEditService'; // import 'vs/workbench/services/integrity/node/integrityService'; import 'vs/workbench/services/keybinding/common/keybindingEditing'; -// import 'vs/workbench/services/textMate/electron-browser/textMateService'; +import 'vs/workbench/services/textMate/browser/textMateService'; // import 'vs/workbench/services/workspace/electron-browser/workspaceEditingService'; // import 'vs/workbench/services/extensions/electron-browser/inactiveExtensionUrlHandler'; import 'vs/workbench/services/decorations/browser/decorationsService'; diff --git a/tslint.json b/tslint.json index d92178d76bd..5afacd6023d 100644 --- a/tslint.json +++ b/tslint.json @@ -441,7 +441,9 @@ "**/vs/platform/**/{common,browser}/**", "**/vs/editor/{common,browser}/**", "**/vs/workbench/{common,browser}/**", - "**/vs/workbench/services/**/{common,browser}/**" + "**/vs/workbench/services/**/{common,browser}/**", + "vscode-textmate", + "onigasm-umd" ] }, { diff --git a/yarn.lock b/yarn.lock index a90e410d1ae..0b9411df990 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6402,6 +6402,11 @@ onetime@^2.0.0: dependencies: mimic-fn "^1.0.0" +onigasm-umd@^2.2.2: + version "2.2.2" + resolved "https://registry.yarnpkg.com/onigasm-umd/-/onigasm-umd-2.2.2.tgz#b989d762df61f899a3052ac794a50bd93fe20257" + integrity sha512-v2eMOJu7iE444L2iJN+U6s6s5S0y7oj/N0DAkrd6wokRtTVoq/v/yaDI1lIqFrTeJbNtqNzYvguDF5yNzW3Rvw== + oniguruma@^7.0.0: version "7.0.2" resolved "https://registry.yarnpkg.com/oniguruma/-/oniguruma-7.0.2.tgz#a5c922cf7066da1dbcc60f6385a90437a83f8d0b" @@ -9610,10 +9615,10 @@ vscode-sqlite3@4.0.7: dependencies: nan "~2.10.0" -vscode-textmate@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.0.1.tgz#6c36f28e9059ce12bc34907f7a33ea43166b26a8" - integrity sha512-gHTXTj04TUgbjB8y7pkVwxOiuCuD6aU5gnFzIByQuqdgFpe/bJaaEIS4geGjbjWbd1XJh6zG1EthLfpNaXEqUw== +vscode-textmate@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.1.1.tgz#857e836fbc13a376ec624242437e1747d79610a9" + integrity sha512-xBjq9LH6fMhWDhIVkbKlB1JeCu6lT3FI/QKN24Xi4RKPBUm16IhHTqs6Q6SUGewkNsFZGkb1tJdZsuMnlmVpgw== dependencies: oniguruma "^7.0.0"