Add support for vscode-textmate in the browser
This commit is contained in:
Alex Dima
parent
029d92b68d
commit
f0ba210ef9
|
|
@ -41,6 +41,7 @@
|
|||
"native-keymap": "1.2.5",
|
||||
"native-watchdog": "1.0.0",
|
||||
"node-pty": "0.9.0-beta9",
|
||||
"onigasm-umd": "^2.2.2",
|
||||
"semver": "^5.5.0",
|
||||
"spdlog": "0.8.1",
|
||||
"sudo-prompt": "8.2.0",
|
||||
|
|
@ -51,7 +52,7 @@
|
|||
"vscode-proxy-agent": "0.4.0",
|
||||
"vscode-ripgrep": "^1.2.5",
|
||||
"vscode-sqlite3": "4.0.7",
|
||||
"vscode-textmate": "^4.0.1",
|
||||
"vscode-textmate": "^4.1.1",
|
||||
"vscode-xterm": "3.14.0-beta4",
|
||||
"yauzl": "^2.9.1",
|
||||
"yazl": "^2.4.3"
|
||||
|
|
|
|||
|
|
@ -13,12 +13,14 @@
|
|||
"minimist": "1.2.0",
|
||||
"native-watchdog": "1.0.0",
|
||||
"node-pty": "0.8.1",
|
||||
"onigasm-umd": "^2.2.2",
|
||||
"semver": "^5.5.0",
|
||||
"spdlog": "0.8.1",
|
||||
"vscode-chokidar": "1.6.5",
|
||||
"vscode-nsfw": "1.1.1",
|
||||
"vscode-proxy-agent": "0.4.0",
|
||||
"vscode-ripgrep": "^1.2.5",
|
||||
"vscode-textmate": "^4.1.1",
|
||||
"yauzl": "^2.9.1",
|
||||
"yazl": "^2.4.3"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -607,6 +607,11 @@ nan@^2.10.0:
|
|||
resolved "https://registry.yarnpkg.com/nan/-/nan-2.11.0.tgz#574e360e4d954ab16966ec102c0c049fd961a099"
|
||||
integrity sha512-F4miItu2rGnV2ySkXOQoA8FKz/SR2Q2sWP0sbTxNxz/tuokeC8WxOhPMcwi0qIyGtVn/rrSeLbvVkznqCdwYnw==
|
||||
|
||||
nan@^2.12.1:
|
||||
version "2.14.0"
|
||||
resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c"
|
||||
integrity sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==
|
||||
|
||||
native-watchdog@1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/native-watchdog/-/native-watchdog-1.0.0.tgz#97344e83cd6815a8c8e6c44a52e7be05832e65ca"
|
||||
|
|
@ -683,6 +688,18 @@ once@^1.3.1, once@^1.4.0:
|
|||
dependencies:
|
||||
wrappy "1"
|
||||
|
||||
onigasm-umd@^2.2.2:
|
||||
version "2.2.2"
|
||||
resolved "https://registry.yarnpkg.com/onigasm-umd/-/onigasm-umd-2.2.2.tgz#b989d762df61f899a3052ac794a50bd93fe20257"
|
||||
integrity sha512-v2eMOJu7iE444L2iJN+U6s6s5S0y7oj/N0DAkrd6wokRtTVoq/v/yaDI1lIqFrTeJbNtqNzYvguDF5yNzW3Rvw==
|
||||
|
||||
oniguruma@^7.0.0:
|
||||
version "7.1.0"
|
||||
resolved "https://registry.yarnpkg.com/oniguruma/-/oniguruma-7.1.0.tgz#106ddf7eb42507d0442ac68b187c4f7fdf052c83"
|
||||
integrity sha512-mV+6HcDNQ38vM8HVKM+MJyXO4EtSigwIZhq023A4rA8Am4dMlGhUkPwudDykExYR45oLrssR/Ep7PZCQ1OM3pA==
|
||||
dependencies:
|
||||
nan "^2.12.1"
|
||||
|
||||
os-homedir@^1.0.1:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3"
|
||||
|
|
@ -1079,6 +1096,13 @@ vscode-ripgrep@^1.2.5:
|
|||
resolved "https://registry.yarnpkg.com/vscode-ripgrep/-/vscode-ripgrep-1.2.5.tgz#2093c8f36d52bd2dab9eb45b003dd02533c5499c"
|
||||
integrity sha512-n5XBm9od5hahpljw9T8wbkuMnAY7LlAG1OyEEtcCZEX9aCHFuBKSP0IcvciGRTbtWRovNuT83A2iRjt6PL3bLg==
|
||||
|
||||
vscode-textmate@^4.1.1:
|
||||
version "4.1.1"
|
||||
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.1.1.tgz#857e836fbc13a376ec624242437e1747d79610a9"
|
||||
integrity sha512-xBjq9LH6fMhWDhIVkbKlB1JeCu6lT3FI/QKN24Xi4RKPBUm16IhHTqs6Q6SUGewkNsFZGkb1tJdZsuMnlmVpgw==
|
||||
dependencies:
|
||||
oniguruma "^7.0.0"
|
||||
|
||||
vscode-windows-ca-certs@0.1.0:
|
||||
version "0.1.0"
|
||||
resolved "https://registry.yarnpkg.com/vscode-windows-ca-certs/-/vscode-windows-ca-certs-0.1.0.tgz#d58eeb40b536130918cfde2b01e6dc7e5c1bd757"
|
||||
|
|
|
|||
33
src/typings/onigasm-umd.d.ts
vendored
Normal file
33
src/typings/onigasm-umd.d.ts
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
declare module "onigasm-umd" {
|
||||
|
||||
function loadWASM(data: string | ArrayBuffer): Promise<void>;
|
||||
|
||||
class OnigString {
|
||||
constructor(content: string);
|
||||
readonly content: string;
|
||||
readonly dispose?: () => void;
|
||||
}
|
||||
|
||||
class OnigScanner {
|
||||
constructor(patterns: string[]);
|
||||
findNextMatchSync(string: string | OnigString, startPosition: number): IOnigMatch;
|
||||
}
|
||||
|
||||
export interface IOnigCaptureIndex {
|
||||
index: number
|
||||
start: number
|
||||
end: number
|
||||
length: number
|
||||
}
|
||||
|
||||
export interface IOnigMatch {
|
||||
index: number
|
||||
captureIndices: IOnigCaptureIndex[]
|
||||
scanner: OnigScanner
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@
|
|||
<script>
|
||||
self.CONNECTION_AUTH_TOKEN = '{{CONNECTION_AUTH_TOKEN}}';
|
||||
self.USER_HOME_DIR = '{{USER_HOME_DIR}}';
|
||||
self.SERVER_APP_ROOT = '{{SERVER_APP_ROOT}}';
|
||||
</script>
|
||||
|
||||
<!-- Startup via workbench.js -->
|
||||
|
|
|
|||
|
|
@ -21,7 +21,11 @@
|
|||
|
||||
// @ts-ignore
|
||||
require.config({
|
||||
baseUrl: `${window.location.origin}/out`
|
||||
baseUrl: `${window.location.origin}/out`,
|
||||
paths: {
|
||||
'vscode-textmate': `${window.location.origin}/node_modules/vscode-textmate/release/main`,
|
||||
'onigasm-umd': `${window.location.origin}/node_modules/onigasm-umd/release/main`,
|
||||
}
|
||||
});
|
||||
|
||||
// @ts-ignore
|
||||
|
|
|
|||
|
|
@ -40,8 +40,6 @@ import { Schemas } from 'vs/base/common/network';
|
|||
import { editorMatchesToTextSearchResults, addContextToEditorMatches } from 'vs/workbench/services/search/common/searchHelpers';
|
||||
import { Disposable, IDisposable } from 'vs/base/common/lifecycle';
|
||||
import { InMemoryStorageService, IStorageService } from 'vs/platform/storage/common/storage';
|
||||
import { ITextMateService, IGrammar as ITextMategrammar } from 'vs/workbench/services/textMate/common/textMateService';
|
||||
import { LanguageId, TokenizationRegistry } from 'vs/editor/common/modes';
|
||||
import { IUpdateService, State } from 'vs/platform/update/common/update';
|
||||
import { IWindowConfiguration, IPath, IPathsToWaitFor, IWindowService, INativeOpenDialogOptions, IEnterWorkspaceResult, IURIToOpen, IMessageBoxResult, IWindowsService, IOpenSettings } from 'vs/platform/windows/common/windows';
|
||||
import { IProcessEnvironment } from 'vs/base/common/platform';
|
||||
|
|
@ -54,7 +52,6 @@ import { IWorkspaceContextService, Workspace, toWorkspaceFolder, IWorkspaceFolde
|
|||
import { ITextResourcePropertiesService } from 'vs/editor/common/services/resourceConfiguration';
|
||||
import { IKeybindingService } from 'vs/platform/keybinding/common/keybinding';
|
||||
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
|
||||
import { Color, RGBA } from 'vs/base/common/color';
|
||||
import { ITunnelService } from 'vs/platform/remote/common/tunnel';
|
||||
import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
|
||||
import { IFileService } from 'vs/platform/files/common/files';
|
||||
|
|
@ -820,26 +817,6 @@ registerSingleton(ITelemetryService, SimpleTelemetryService);
|
|||
|
||||
//#endregion
|
||||
|
||||
//#region Textmate
|
||||
|
||||
TokenizationRegistry.setColorMap([<any>null, new Color(new RGBA(212, 212, 212, 1)), new Color(new RGBA(30, 30, 30, 1))]);
|
||||
|
||||
export class SimpleTextMateService implements ITextMateService {
|
||||
|
||||
_serviceBrand: any;
|
||||
|
||||
readonly onDidEncounterLanguage: Event<LanguageId> = Event.None;
|
||||
|
||||
createGrammar(modeId: string): Promise<ITextMategrammar> {
|
||||
// @ts-ignore
|
||||
return Promise.resolve(undefined);
|
||||
}
|
||||
}
|
||||
|
||||
registerSingleton(ITextMateService, SimpleTextMateService, true);
|
||||
|
||||
//#endregion
|
||||
|
||||
//#region Text Resource Properties
|
||||
|
||||
export class SimpleTextResourcePropertiesService extends SimpleResourcePropertiesService { }
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ export class ExtensionService extends AbstractExtensionService implements IExten
|
|||
// see https://github.com/Microsoft/vscode/issues/41322
|
||||
this._lifecycleService.when(LifecyclePhase.Ready).then(() => {
|
||||
// reschedule to ensure this runs after restoring viewlets, panels, and editors
|
||||
runWhenIdle(async () => {
|
||||
runWhenIdle(() => {
|
||||
this._initialize();
|
||||
}, 50 /*max delay*/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,518 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import * as nls from 'vs/nls';
|
||||
import * as dom from 'vs/base/browser/dom';
|
||||
import { Color } from 'vs/base/common/color';
|
||||
import { onUnexpectedError } from 'vs/base/common/errors';
|
||||
import { Emitter, Event } from 'vs/base/common/event';
|
||||
import * as resources from 'vs/base/common/resources';
|
||||
import * as types from 'vs/base/common/types';
|
||||
import { URI } from 'vs/base/common/uri';
|
||||
import { TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token';
|
||||
import { IState, ITokenizationSupport, LanguageId, TokenMetadata, TokenizationRegistry } from 'vs/editor/common/modes';
|
||||
import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
|
||||
import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization';
|
||||
import { IModeService } from 'vs/editor/common/services/modeService';
|
||||
import { IFileService } from 'vs/platform/files/common/files';
|
||||
import { ILogService } from 'vs/platform/log/common/log';
|
||||
import { INotificationService } from 'vs/platform/notification/common/notification';
|
||||
import { ExtensionMessageCollector } from 'vs/workbench/services/extensions/common/extensionsRegistry';
|
||||
import { IEmbeddedLanguagesMap, ITMSyntaxExtensionPoint, TokenTypesContribution, grammarsExtPoint } from 'vs/workbench/services/textMate/common/TMGrammars';
|
||||
import { ITextMateService } from 'vs/workbench/services/textMate/common/textMateService';
|
||||
import { ITokenColorizationRule, IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService';
|
||||
import { IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2, IGrammar, ITokenTypeMap, Registry, StackElement, StandardTokenType, RegistryOptions, IRawGrammar } from 'vscode-textmate';
|
||||
import { Disposable, IDisposable, dispose } from 'vs/base/common/lifecycle';
|
||||
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
|
||||
|
||||
export class TMScopeRegistry {
|
||||
|
||||
private _scopeNameToLanguageRegistration: { [scopeName: string]: TMLanguageRegistration; };
|
||||
private _encounteredLanguages: boolean[];
|
||||
|
||||
private readonly _onDidEncounterLanguage = new Emitter<LanguageId>();
|
||||
public readonly onDidEncounterLanguage: Event<LanguageId> = this._onDidEncounterLanguage.event;
|
||||
|
||||
constructor() {
|
||||
this.reset();
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
this._scopeNameToLanguageRegistration = Object.create(null);
|
||||
this._encounteredLanguages = [];
|
||||
}
|
||||
|
||||
public register(scopeName: string, grammarLocation: URI, embeddedLanguages?: IEmbeddedLanguagesMap, tokenTypes?: TokenTypesContribution): void {
|
||||
if (this._scopeNameToLanguageRegistration[scopeName]) {
|
||||
const existingRegistration = this._scopeNameToLanguageRegistration[scopeName];
|
||||
if (!resources.isEqual(existingRegistration.grammarLocation, grammarLocation)) {
|
||||
console.warn(
|
||||
`Overwriting grammar scope name to file mapping for scope ${scopeName}.\n` +
|
||||
`Old grammar file: ${existingRegistration.grammarLocation.toString()}.\n` +
|
||||
`New grammar file: ${grammarLocation.toString()}`
|
||||
);
|
||||
}
|
||||
}
|
||||
this._scopeNameToLanguageRegistration[scopeName] = new TMLanguageRegistration(scopeName, grammarLocation, embeddedLanguages, tokenTypes);
|
||||
}
|
||||
|
||||
public getLanguageRegistration(scopeName: string): TMLanguageRegistration {
|
||||
return this._scopeNameToLanguageRegistration[scopeName] || null;
|
||||
}
|
||||
|
||||
public getGrammarLocation(scopeName: string): URI | null {
|
||||
let data = this.getLanguageRegistration(scopeName);
|
||||
return data ? data.grammarLocation : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* To be called when tokenization found/hit an embedded language.
|
||||
*/
|
||||
public onEncounteredLanguage(languageId: LanguageId): void {
|
||||
if (!this._encounteredLanguages[languageId]) {
|
||||
this._encounteredLanguages[languageId] = true;
|
||||
this._onDidEncounterLanguage.fire(languageId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class TMLanguageRegistration {
|
||||
_topLevelScopeNameDataBrand: void;
|
||||
|
||||
readonly scopeName: string;
|
||||
readonly grammarLocation: URI;
|
||||
readonly embeddedLanguages: IEmbeddedLanguagesMap;
|
||||
readonly tokenTypes: ITokenTypeMap;
|
||||
|
||||
constructor(scopeName: string, grammarLocation: URI, embeddedLanguages: IEmbeddedLanguagesMap | undefined, tokenTypes: TokenTypesContribution | undefined) {
|
||||
this.scopeName = scopeName;
|
||||
this.grammarLocation = grammarLocation;
|
||||
|
||||
// embeddedLanguages handling
|
||||
this.embeddedLanguages = Object.create(null);
|
||||
|
||||
if (embeddedLanguages) {
|
||||
// If embeddedLanguages are configured, fill in `this._embeddedLanguages`
|
||||
let scopes = Object.keys(embeddedLanguages);
|
||||
for (let i = 0, len = scopes.length; i < len; i++) {
|
||||
let scope = scopes[i];
|
||||
let language = embeddedLanguages[scope];
|
||||
if (typeof language !== 'string') {
|
||||
// never hurts to be too careful
|
||||
continue;
|
||||
}
|
||||
this.embeddedLanguages[scope] = language;
|
||||
}
|
||||
}
|
||||
|
||||
this.tokenTypes = Object.create(null);
|
||||
if (tokenTypes) {
|
||||
// If tokenTypes is configured, fill in `this._tokenTypes`
|
||||
const scopes = Object.keys(tokenTypes);
|
||||
for (const scope of scopes) {
|
||||
const tokenType = tokenTypes[scope];
|
||||
switch (tokenType) {
|
||||
case 'string':
|
||||
this.tokenTypes[scope] = StandardTokenType.String;
|
||||
break;
|
||||
case 'other':
|
||||
this.tokenTypes[scope] = StandardTokenType.Other;
|
||||
break;
|
||||
case 'comment':
|
||||
this.tokenTypes[scope] = StandardTokenType.Comment;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface ICreateGrammarResult {
|
||||
languageId: LanguageId;
|
||||
grammar: IGrammar;
|
||||
initialState: StackElement;
|
||||
containsEmbeddedLanguages: boolean;
|
||||
}
|
||||
|
||||
export abstract class AbstractTextMateService extends Disposable implements ITextMateService {
|
||||
public _serviceBrand: any;
|
||||
|
||||
private readonly _onDidEncounterLanguage: Emitter<LanguageId> = this._register(new Emitter<LanguageId>());
|
||||
public readonly onDidEncounterLanguage: Event<LanguageId> = this._onDidEncounterLanguage.event;
|
||||
|
||||
private readonly _styleElement: HTMLStyleElement;
|
||||
private readonly _createdModes: string[];
|
||||
|
||||
protected _scopeRegistry: TMScopeRegistry;
|
||||
private _injections: { [scopeName: string]: string[]; };
|
||||
private _injectedEmbeddedLanguages: { [scopeName: string]: IEmbeddedLanguagesMap[]; };
|
||||
protected _languageToScope: Map<string, string>;
|
||||
private _grammarRegistry: Promise<[Registry, StackElement]> | null;
|
||||
private _tokenizersRegistrations: IDisposable[];
|
||||
private _currentTokenColors: ITokenColorizationRule[] | null;
|
||||
private _themeListener: IDisposable | null;
|
||||
|
||||
constructor(
|
||||
@IModeService private readonly _modeService: IModeService,
|
||||
@IWorkbenchThemeService private readonly _themeService: IWorkbenchThemeService,
|
||||
@IFileService private readonly _fileService: IFileService,
|
||||
@INotificationService private readonly _notificationService: INotificationService,
|
||||
@ILogService private readonly _logService: ILogService,
|
||||
@IConfigurationService private readonly _configurationService: IConfigurationService
|
||||
) {
|
||||
super();
|
||||
this._styleElement = dom.createStyleSheet();
|
||||
this._styleElement.className = 'vscode-tokens-styles';
|
||||
this._createdModes = [];
|
||||
this._scopeRegistry = new TMScopeRegistry();
|
||||
this._scopeRegistry.onDidEncounterLanguage((language) => this._onDidEncounterLanguage.fire(language));
|
||||
this._injections = {};
|
||||
this._injectedEmbeddedLanguages = {};
|
||||
this._languageToScope = new Map<string, string>();
|
||||
this._grammarRegistry = null;
|
||||
this._tokenizersRegistrations = [];
|
||||
this._currentTokenColors = null;
|
||||
this._themeListener = null;
|
||||
|
||||
grammarsExtPoint.setHandler((extensions) => {
|
||||
this._scopeRegistry.reset();
|
||||
this._injections = {};
|
||||
this._injectedEmbeddedLanguages = {};
|
||||
this._languageToScope = new Map<string, string>();
|
||||
this._grammarRegistry = null;
|
||||
this._tokenizersRegistrations = dispose(this._tokenizersRegistrations);
|
||||
this._currentTokenColors = null;
|
||||
if (this._themeListener) {
|
||||
this._themeListener.dispose();
|
||||
this._themeListener = null;
|
||||
}
|
||||
|
||||
for (const extension of extensions) {
|
||||
let grammars = extension.value;
|
||||
for (const grammar of grammars) {
|
||||
this._handleGrammarExtensionPointUser(extension.description.extensionLocation, grammar, extension.collector);
|
||||
}
|
||||
}
|
||||
|
||||
for (const createMode of this._createdModes) {
|
||||
this._registerDefinitionIfAvailable(createMode);
|
||||
}
|
||||
});
|
||||
|
||||
// Generate some color map until the grammar registry is loaded
|
||||
let colorTheme = this._themeService.getColorTheme();
|
||||
let defaultForeground: Color = Color.transparent;
|
||||
let defaultBackground: Color = Color.transparent;
|
||||
for (let i = 0, len = colorTheme.tokenColors.length; i < len; i++) {
|
||||
let rule = colorTheme.tokenColors[i];
|
||||
if (!rule.scope && rule.settings) {
|
||||
if (rule.settings.foreground) {
|
||||
defaultForeground = Color.fromHex(rule.settings.foreground);
|
||||
}
|
||||
if (rule.settings.background) {
|
||||
defaultBackground = Color.fromHex(rule.settings.background);
|
||||
}
|
||||
}
|
||||
}
|
||||
TokenizationRegistry.setColorMap([null!, defaultForeground, defaultBackground]);
|
||||
|
||||
this._modeService.onDidCreateMode((mode) => {
|
||||
let modeId = mode.getId();
|
||||
this._createdModes.push(modeId);
|
||||
this._registerDefinitionIfAvailable(modeId);
|
||||
});
|
||||
}
|
||||
|
||||
private _registerDefinitionIfAvailable(modeId: string): void {
|
||||
if (this._languageToScope.has(modeId)) {
|
||||
const promise = this._createGrammar(modeId).then((r) => {
|
||||
return new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService, this._configurationService);
|
||||
}, e => {
|
||||
onUnexpectedError(e);
|
||||
return null;
|
||||
});
|
||||
this._tokenizersRegistrations.push(TokenizationRegistry.registerPromise(modeId, promise));
|
||||
}
|
||||
}
|
||||
|
||||
protected _getRegistryOptions(parseRawGrammar: (content: string, filePath: string) => IRawGrammar): RegistryOptions {
|
||||
return {
|
||||
loadGrammar: async (scopeName: string) => {
|
||||
const location = this._scopeRegistry.getGrammarLocation(scopeName);
|
||||
if (!location) {
|
||||
this._logService.trace(`No grammar found for scope ${scopeName}`);
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const content = await this._fileService.readFile(location);
|
||||
return parseRawGrammar(content.value.toString(), location.path);
|
||||
} catch (e) {
|
||||
this._logService.error(`Unable to load and parse grammar for scope ${scopeName} from ${location}`, e);
|
||||
return null;
|
||||
}
|
||||
},
|
||||
getInjections: (scopeName: string) => {
|
||||
const scopeParts = scopeName.split('.');
|
||||
let injections: string[] = [];
|
||||
for (let i = 1; i <= scopeParts.length; i++) {
|
||||
const subScopeName = scopeParts.slice(0, i).join('.');
|
||||
injections = [...injections, ...(this._injections[subScopeName] || [])];
|
||||
}
|
||||
return injections;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private async _createGrammarRegistry(): Promise<[Registry, StackElement]> {
|
||||
const { Registry, INITIAL, parseRawGrammar } = await this._loadVSCodeTextmate();
|
||||
const grammarRegistry = new Registry(this._getRegistryOptions(parseRawGrammar));
|
||||
this._updateTheme(grammarRegistry);
|
||||
this._themeListener = this._themeService.onDidColorThemeChange((e) => this._updateTheme(grammarRegistry));
|
||||
return <[Registry, StackElement]>[grammarRegistry, INITIAL];
|
||||
}
|
||||
|
||||
private _getOrCreateGrammarRegistry(): Promise<[Registry, StackElement]> {
|
||||
if (!this._grammarRegistry) {
|
||||
this._grammarRegistry = this._createGrammarRegistry();
|
||||
}
|
||||
return this._grammarRegistry;
|
||||
}
|
||||
|
||||
private static _toColorMap(colorMap: string[]): Color[] {
|
||||
let result: Color[] = [null!];
|
||||
for (let i = 1, len = colorMap.length; i < len; i++) {
|
||||
result[i] = Color.fromHex(colorMap[i]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private _updateTheme(grammarRegistry: Registry): void {
|
||||
let colorTheme = this._themeService.getColorTheme();
|
||||
if (!this.compareTokenRules(colorTheme.tokenColors)) {
|
||||
return;
|
||||
}
|
||||
grammarRegistry.setTheme({ name: colorTheme.label, settings: colorTheme.tokenColors });
|
||||
let colorMap = AbstractTextMateService._toColorMap(grammarRegistry.getColorMap());
|
||||
let cssRules = generateTokensCSSForColorMap(colorMap);
|
||||
this._styleElement.innerHTML = cssRules;
|
||||
TokenizationRegistry.setColorMap(colorMap);
|
||||
}
|
||||
|
||||
private compareTokenRules(newRules: ITokenColorizationRule[]): boolean {
|
||||
let currRules = this._currentTokenColors;
|
||||
this._currentTokenColors = newRules;
|
||||
if (!newRules || !currRules || newRules.length !== currRules.length) {
|
||||
return true;
|
||||
}
|
||||
for (let i = newRules.length - 1; i >= 0; i--) {
|
||||
let r1 = newRules[i];
|
||||
let r2 = currRules[i];
|
||||
if (r1.scope !== r2.scope) {
|
||||
return true;
|
||||
}
|
||||
let s1 = r1.settings;
|
||||
let s2 = r2.settings;
|
||||
if (s1 && s2) {
|
||||
if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground || s1.background !== s2.background) {
|
||||
return true;
|
||||
}
|
||||
} else if (!s1 || !s2) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private _handleGrammarExtensionPointUser(extensionLocation: URI, syntax: ITMSyntaxExtensionPoint, collector: ExtensionMessageCollector): void {
|
||||
if (syntax.language && ((typeof syntax.language !== 'string') || !this._modeService.isRegisteredMode(syntax.language))) {
|
||||
collector.error(nls.localize('invalid.language', "Unknown language in `contributes.{0}.language`. Provided value: {1}", grammarsExtPoint.name, String(syntax.language)));
|
||||
return;
|
||||
}
|
||||
if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
|
||||
collector.error(nls.localize('invalid.scopeName', "Expected string in `contributes.{0}.scopeName`. Provided value: {1}", grammarsExtPoint.name, String(syntax.scopeName)));
|
||||
return;
|
||||
}
|
||||
if (!syntax.path || (typeof syntax.path !== 'string')) {
|
||||
collector.error(nls.localize('invalid.path.0', "Expected string in `contributes.{0}.path`. Provided value: {1}", grammarsExtPoint.name, String(syntax.path)));
|
||||
return;
|
||||
}
|
||||
if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || syntax.injectTo.some(scope => typeof scope !== 'string'))) {
|
||||
collector.error(nls.localize('invalid.injectTo', "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.injectTo)));
|
||||
return;
|
||||
}
|
||||
if (syntax.embeddedLanguages && !types.isObject(syntax.embeddedLanguages)) {
|
||||
collector.error(nls.localize('invalid.embeddedLanguages', "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.embeddedLanguages)));
|
||||
return;
|
||||
}
|
||||
|
||||
if (syntax.tokenTypes && !types.isObject(syntax.tokenTypes)) {
|
||||
collector.error(nls.localize('invalid.tokenTypes', "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.tokenTypes)));
|
||||
return;
|
||||
}
|
||||
|
||||
const grammarLocation = resources.joinPath(extensionLocation, syntax.path);
|
||||
if (!resources.isEqualOrParent(grammarLocation, extensionLocation)) {
|
||||
collector.warn(nls.localize('invalid.path.1', "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.", grammarsExtPoint.name, grammarLocation.path, extensionLocation.path));
|
||||
}
|
||||
|
||||
this._scopeRegistry.register(syntax.scopeName, grammarLocation, syntax.embeddedLanguages, syntax.tokenTypes);
|
||||
|
||||
if (syntax.injectTo) {
|
||||
for (let injectScope of syntax.injectTo) {
|
||||
let injections = this._injections[injectScope];
|
||||
if (!injections) {
|
||||
this._injections[injectScope] = injections = [];
|
||||
}
|
||||
injections.push(syntax.scopeName);
|
||||
}
|
||||
|
||||
if (syntax.embeddedLanguages) {
|
||||
for (let injectScope of syntax.injectTo) {
|
||||
let injectedEmbeddedLanguages = this._injectedEmbeddedLanguages[injectScope];
|
||||
if (!injectedEmbeddedLanguages) {
|
||||
this._injectedEmbeddedLanguages[injectScope] = injectedEmbeddedLanguages = [];
|
||||
}
|
||||
injectedEmbeddedLanguages.push(syntax.embeddedLanguages);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let modeId = syntax.language;
|
||||
if (modeId) {
|
||||
this._languageToScope.set(modeId, syntax.scopeName);
|
||||
}
|
||||
}
|
||||
|
||||
private _resolveEmbeddedLanguages(embeddedLanguages: IEmbeddedLanguagesMap): IEmbeddedLanguagesMap2 {
|
||||
let scopes = Object.keys(embeddedLanguages);
|
||||
let result: IEmbeddedLanguagesMap2 = Object.create(null);
|
||||
for (let i = 0, len = scopes.length; i < len; i++) {
|
||||
let scope = scopes[i];
|
||||
let language = embeddedLanguages[scope];
|
||||
let languageIdentifier = this._modeService.getLanguageIdentifier(language);
|
||||
if (languageIdentifier) {
|
||||
result[scope] = languageIdentifier.id;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public async createGrammar(modeId: string): Promise<IGrammar> {
|
||||
const { grammar } = await this._createGrammar(modeId);
|
||||
return grammar;
|
||||
}
|
||||
|
||||
private async _createGrammar(modeId: string): Promise<ICreateGrammarResult> {
|
||||
const scopeName = this._languageToScope.get(modeId);
|
||||
if (typeof scopeName !== 'string') {
|
||||
// No TM grammar defined
|
||||
return Promise.reject(new Error(nls.localize('no-tm-grammar', "No TM Grammar registered for this language.")));
|
||||
}
|
||||
const languageRegistration = this._scopeRegistry.getLanguageRegistration(scopeName);
|
||||
if (!languageRegistration) {
|
||||
// No TM grammar defined
|
||||
return Promise.reject(new Error(nls.localize('no-tm-grammar', "No TM Grammar registered for this language.")));
|
||||
}
|
||||
let embeddedLanguages = this._resolveEmbeddedLanguages(languageRegistration.embeddedLanguages);
|
||||
let rawInjectedEmbeddedLanguages = this._injectedEmbeddedLanguages[scopeName];
|
||||
if (rawInjectedEmbeddedLanguages) {
|
||||
let injectedEmbeddedLanguages: IEmbeddedLanguagesMap2[] = rawInjectedEmbeddedLanguages.map(this._resolveEmbeddedLanguages.bind(this));
|
||||
for (const injected of injectedEmbeddedLanguages) {
|
||||
for (const scope of Object.keys(injected)) {
|
||||
embeddedLanguages[scope] = injected[scope];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let languageId = this._modeService.getLanguageIdentifier(modeId)!.id;
|
||||
let containsEmbeddedLanguages = (Object.keys(embeddedLanguages).length > 0);
|
||||
|
||||
const [grammarRegistry, initialState] = await this._getOrCreateGrammarRegistry();
|
||||
const grammar = await grammarRegistry.loadGrammarWithConfiguration(scopeName, languageId, { embeddedLanguages, tokenTypes: languageRegistration.tokenTypes });
|
||||
return {
|
||||
languageId: languageId,
|
||||
grammar: grammar,
|
||||
initialState: initialState,
|
||||
containsEmbeddedLanguages: containsEmbeddedLanguages
|
||||
};
|
||||
}
|
||||
|
||||
protected abstract _loadVSCodeTextmate(): Promise<typeof import('vscode-textmate')>;
|
||||
}
|
||||
|
||||
class TMTokenization implements ITokenizationSupport {
|
||||
|
||||
private readonly _scopeRegistry: TMScopeRegistry;
|
||||
private readonly _languageId: LanguageId;
|
||||
private readonly _grammar: IGrammar;
|
||||
private readonly _containsEmbeddedLanguages: boolean;
|
||||
private readonly _seenLanguages: boolean[];
|
||||
private readonly _initialState: StackElement;
|
||||
private _maxTokenizationLineLength: number;
|
||||
private _tokenizationWarningAlreadyShown: boolean;
|
||||
|
||||
constructor(scopeRegistry: TMScopeRegistry, languageId: LanguageId, grammar: IGrammar, initialState: StackElement, containsEmbeddedLanguages: boolean, @INotificationService private readonly notificationService: INotificationService, @IConfigurationService readonly configurationService: IConfigurationService) {
|
||||
this._scopeRegistry = scopeRegistry;
|
||||
this._languageId = languageId;
|
||||
this._grammar = grammar;
|
||||
this._initialState = initialState;
|
||||
this._containsEmbeddedLanguages = containsEmbeddedLanguages;
|
||||
this._seenLanguages = [];
|
||||
this._maxTokenizationLineLength = configurationService.getValue<number>('editor.maxTokenizationLineLength');
|
||||
}
|
||||
|
||||
public getInitialState(): IState {
|
||||
return this._initialState;
|
||||
}
|
||||
|
||||
public tokenize(line: string, state: IState, offsetDelta: number): TokenizationResult {
|
||||
throw new Error('Not supported!');
|
||||
}
|
||||
|
||||
public tokenize2(line: string, state: StackElement, offsetDelta: number): TokenizationResult2 {
|
||||
if (offsetDelta !== 0) {
|
||||
throw new Error('Unexpected: offsetDelta should be 0.');
|
||||
}
|
||||
|
||||
// Do not attempt to tokenize if a line is too long
|
||||
if (line.length >= this._maxTokenizationLineLength) {
|
||||
if (!this._tokenizationWarningAlreadyShown) {
|
||||
this._tokenizationWarningAlreadyShown = true;
|
||||
this.notificationService.warn(nls.localize('too many characters', "Tokenization is skipped for long lines for performance reasons. The length of a long line can be configured via `editor.maxTokenizationLineLength`."));
|
||||
}
|
||||
console.log(`Line (${line.substr(0, 15)}...): longer than ${this._maxTokenizationLineLength} characters, tokenization skipped.`);
|
||||
return nullTokenize2(this._languageId, line, state, offsetDelta);
|
||||
}
|
||||
|
||||
let textMateResult = this._grammar.tokenizeLine2(line, state);
|
||||
|
||||
if (this._containsEmbeddedLanguages) {
|
||||
let seenLanguages = this._seenLanguages;
|
||||
let tokens = textMateResult.tokens;
|
||||
|
||||
// Must check if any of the embedded languages was hit
|
||||
for (let i = 0, len = (tokens.length >>> 1); i < len; i++) {
|
||||
let metadata = tokens[(i << 1) + 1];
|
||||
let languageId = TokenMetadata.getLanguageId(metadata);
|
||||
|
||||
if (!seenLanguages[languageId]) {
|
||||
seenLanguages[languageId] = true;
|
||||
this._scopeRegistry.onEncounteredLanguage(languageId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let endState: StackElement;
|
||||
// try to save an object if possible
|
||||
if (state.equals(textMateResult.ruleStack)) {
|
||||
endState = state;
|
||||
} else {
|
||||
endState = textMateResult.ruleStack;
|
||||
|
||||
}
|
||||
|
||||
return new TokenizationResult2(textMateResult.tokens, endState);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { ITextMateService } from 'vs/workbench/services/textMate/common/textMateService';
|
||||
import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
|
||||
import { AbstractTextMateService } from 'vs/workbench/services/textMate/browser/abstractTextMateService';
|
||||
import * as vscodeTextmate from 'vscode-textmate';
|
||||
import * as onigasm from 'onigasm-umd';
|
||||
import { IModeService } from 'vs/editor/common/services/modeService';
|
||||
import { IFileService } from 'vs/platform/files/common/files';
|
||||
import { ILogService } from 'vs/platform/log/common/log';
|
||||
import { INotificationService } from 'vs/platform/notification/common/notification';
|
||||
import { IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService';
|
||||
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
|
||||
|
||||
export class TextMateService extends AbstractTextMateService {
|
||||
|
||||
constructor(
|
||||
@IModeService modeService: IModeService,
|
||||
@IWorkbenchThemeService themeService: IWorkbenchThemeService,
|
||||
@IFileService fileService: IFileService,
|
||||
@INotificationService notificationService: INotificationService,
|
||||
@ILogService logService: ILogService,
|
||||
@IConfigurationService configurationService: IConfigurationService
|
||||
) {
|
||||
super(modeService, themeService, fileService, notificationService, logService, configurationService);
|
||||
}
|
||||
|
||||
protected _loadVSCodeTextmate(): Promise<typeof import('vscode-textmate')> {
|
||||
return import('vscode-textmate');
|
||||
}
|
||||
|
||||
protected _getRegistryOptions(parseRawGrammar: (content: string, filePath: string) => vscodeTextmate.IRawGrammar): vscodeTextmate.RegistryOptions {
|
||||
const result = super._getRegistryOptions(parseRawGrammar);
|
||||
result.getOnigLib = () => loadOnigasm();
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
let onigasmPromise: Promise<vscodeTextmate.IOnigLib> | null = null;
|
||||
async function loadOnigasm(): Promise<vscodeTextmate.IOnigLib> {
|
||||
if (!onigasmPromise) {
|
||||
onigasmPromise = doLoadOnigasm();
|
||||
}
|
||||
return onigasmPromise;
|
||||
}
|
||||
|
||||
async function doLoadOnigasm(): Promise<vscodeTextmate.IOnigLib> {
|
||||
const wasmBytes = await loadOnigasmWASM();
|
||||
await onigasm.loadWASM(wasmBytes);
|
||||
return {
|
||||
createOnigScanner(patterns: string[]) { return new onigasm.OnigScanner(patterns); },
|
||||
createOnigString(s: string) { return new onigasm.OnigString(s); }
|
||||
};
|
||||
}
|
||||
|
||||
async function loadOnigasmWASM(): Promise<ArrayBuffer> {
|
||||
const wasmPath = require.toUrl('onigasm-umd/../onigasm.wasm');
|
||||
const response = await fetch(wasmPath);
|
||||
const bytes = await response.arrayBuffer();
|
||||
return bytes;
|
||||
}
|
||||
|
||||
registerSingleton(ITextMateService, TextMateService);
|
||||
|
|
@ -3,512 +3,14 @@
|
|||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import * as nls from 'vs/nls';
|
||||
import * as dom from 'vs/base/browser/dom';
|
||||
import { Color } from 'vs/base/common/color';
|
||||
import { onUnexpectedError } from 'vs/base/common/errors';
|
||||
import { Emitter, Event } from 'vs/base/common/event';
|
||||
import * as resources from 'vs/base/common/resources';
|
||||
import * as types from 'vs/base/common/types';
|
||||
import { URI } from 'vs/base/common/uri';
|
||||
import { TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token';
|
||||
import { IState, ITokenizationSupport, LanguageId, TokenMetadata, TokenizationRegistry } from 'vs/editor/common/modes';
|
||||
import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
|
||||
import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization';
|
||||
import { IModeService } from 'vs/editor/common/services/modeService';
|
||||
import { IFileService } from 'vs/platform/files/common/files';
|
||||
import { ILogService } from 'vs/platform/log/common/log';
|
||||
import { INotificationService } from 'vs/platform/notification/common/notification';
|
||||
import { ExtensionMessageCollector } from 'vs/workbench/services/extensions/common/extensionsRegistry';
|
||||
import { IEmbeddedLanguagesMap, ITMSyntaxExtensionPoint, TokenTypesContribution, grammarsExtPoint } from 'vs/workbench/services/textMate/common/TMGrammars';
|
||||
import { ITextMateService } from 'vs/workbench/services/textMate/common/textMateService';
|
||||
import { ITokenColorizationRule, IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService';
|
||||
import { IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2, IGrammar, ITokenTypeMap, Registry, StackElement, StandardTokenType } from 'vscode-textmate';
|
||||
import { Disposable, IDisposable, dispose } from 'vs/base/common/lifecycle';
|
||||
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
|
||||
import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
|
||||
import { AbstractTextMateService } from 'vs/workbench/services/textMate/browser/abstractTextMateService';
|
||||
|
||||
export class TMScopeRegistry {
|
||||
export class TextMateService extends AbstractTextMateService {
|
||||
|
||||
private _scopeNameToLanguageRegistration: { [scopeName: string]: TMLanguageRegistration; };
|
||||
private _encounteredLanguages: boolean[];
|
||||
|
||||
private readonly _onDidEncounterLanguage = new Emitter<LanguageId>();
|
||||
public readonly onDidEncounterLanguage: Event<LanguageId> = this._onDidEncounterLanguage.event;
|
||||
|
||||
constructor() {
|
||||
this.reset();
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
this._scopeNameToLanguageRegistration = Object.create(null);
|
||||
this._encounteredLanguages = [];
|
||||
}
|
||||
|
||||
public register(scopeName: string, grammarLocation: URI, embeddedLanguages?: IEmbeddedLanguagesMap, tokenTypes?: TokenTypesContribution): void {
|
||||
if (this._scopeNameToLanguageRegistration[scopeName]) {
|
||||
const existingRegistration = this._scopeNameToLanguageRegistration[scopeName];
|
||||
if (!resources.isEqual(existingRegistration.grammarLocation, grammarLocation)) {
|
||||
console.warn(
|
||||
`Overwriting grammar scope name to file mapping for scope ${scopeName}.\n` +
|
||||
`Old grammar file: ${existingRegistration.grammarLocation.toString()}.\n` +
|
||||
`New grammar file: ${grammarLocation.toString()}`
|
||||
);
|
||||
}
|
||||
}
|
||||
this._scopeNameToLanguageRegistration[scopeName] = new TMLanguageRegistration(scopeName, grammarLocation, embeddedLanguages, tokenTypes);
|
||||
}
|
||||
|
||||
public getLanguageRegistration(scopeName: string): TMLanguageRegistration {
|
||||
return this._scopeNameToLanguageRegistration[scopeName] || null;
|
||||
}
|
||||
|
||||
public getGrammarLocation(scopeName: string): URI | null {
|
||||
let data = this.getLanguageRegistration(scopeName);
|
||||
return data ? data.grammarLocation : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* To be called when tokenization found/hit an embedded language.
|
||||
*/
|
||||
public onEncounteredLanguage(languageId: LanguageId): void {
|
||||
if (!this._encounteredLanguages[languageId]) {
|
||||
this._encounteredLanguages[languageId] = true;
|
||||
this._onDidEncounterLanguage.fire(languageId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class TMLanguageRegistration {
|
||||
_topLevelScopeNameDataBrand: void;
|
||||
|
||||
readonly scopeName: string;
|
||||
readonly grammarLocation: URI;
|
||||
readonly embeddedLanguages: IEmbeddedLanguagesMap;
|
||||
readonly tokenTypes: ITokenTypeMap;
|
||||
|
||||
constructor(scopeName: string, grammarLocation: URI, embeddedLanguages: IEmbeddedLanguagesMap | undefined, tokenTypes: TokenTypesContribution | undefined) {
|
||||
this.scopeName = scopeName;
|
||||
this.grammarLocation = grammarLocation;
|
||||
|
||||
// embeddedLanguages handling
|
||||
this.embeddedLanguages = Object.create(null);
|
||||
|
||||
if (embeddedLanguages) {
|
||||
// If embeddedLanguages are configured, fill in `this._embeddedLanguages`
|
||||
let scopes = Object.keys(embeddedLanguages);
|
||||
for (let i = 0, len = scopes.length; i < len; i++) {
|
||||
let scope = scopes[i];
|
||||
let language = embeddedLanguages[scope];
|
||||
if (typeof language !== 'string') {
|
||||
// never hurts to be too careful
|
||||
continue;
|
||||
}
|
||||
this.embeddedLanguages[scope] = language;
|
||||
}
|
||||
}
|
||||
|
||||
this.tokenTypes = Object.create(null);
|
||||
if (tokenTypes) {
|
||||
// If tokenTypes is configured, fill in `this._tokenTypes`
|
||||
const scopes = Object.keys(tokenTypes);
|
||||
for (const scope of scopes) {
|
||||
const tokenType = tokenTypes[scope];
|
||||
switch (tokenType) {
|
||||
case 'string':
|
||||
this.tokenTypes[scope] = StandardTokenType.String;
|
||||
break;
|
||||
case 'other':
|
||||
this.tokenTypes[scope] = StandardTokenType.Other;
|
||||
break;
|
||||
case 'comment':
|
||||
this.tokenTypes[scope] = StandardTokenType.Comment;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface ICreateGrammarResult {
|
||||
languageId: LanguageId;
|
||||
grammar: IGrammar;
|
||||
initialState: StackElement;
|
||||
containsEmbeddedLanguages: boolean;
|
||||
}
|
||||
|
||||
export class TextMateService extends Disposable implements ITextMateService {
|
||||
public _serviceBrand: any;
|
||||
|
||||
private readonly _onDidEncounterLanguage: Emitter<LanguageId> = this._register(new Emitter<LanguageId>());
|
||||
public readonly onDidEncounterLanguage: Event<LanguageId> = this._onDidEncounterLanguage.event;
|
||||
|
||||
private readonly _styleElement: HTMLStyleElement;
|
||||
private readonly _createdModes: string[];
|
||||
|
||||
private _scopeRegistry: TMScopeRegistry;
|
||||
private _injections: { [scopeName: string]: string[]; };
|
||||
private _injectedEmbeddedLanguages: { [scopeName: string]: IEmbeddedLanguagesMap[]; };
|
||||
private _languageToScope: Map<string, string>;
|
||||
private _grammarRegistry: Promise<[Registry, StackElement]> | null;
|
||||
private _tokenizersRegistrations: IDisposable[];
|
||||
private _currentTokenColors: ITokenColorizationRule[] | null;
|
||||
private _themeListener: IDisposable | null;
|
||||
|
||||
constructor(
|
||||
@IModeService private readonly _modeService: IModeService,
|
||||
@IWorkbenchThemeService private readonly _themeService: IWorkbenchThemeService,
|
||||
@IFileService private readonly _fileService: IFileService,
|
||||
@INotificationService private readonly _notificationService: INotificationService,
|
||||
@ILogService private readonly _logService: ILogService,
|
||||
@IConfigurationService private readonly _configurationService: IConfigurationService
|
||||
) {
|
||||
super();
|
||||
this._styleElement = dom.createStyleSheet();
|
||||
this._styleElement.className = 'vscode-tokens-styles';
|
||||
this._createdModes = [];
|
||||
this._scopeRegistry = new TMScopeRegistry();
|
||||
this._scopeRegistry.onDidEncounterLanguage((language) => this._onDidEncounterLanguage.fire(language));
|
||||
this._injections = {};
|
||||
this._injectedEmbeddedLanguages = {};
|
||||
this._languageToScope = new Map<string, string>();
|
||||
this._grammarRegistry = null;
|
||||
this._tokenizersRegistrations = [];
|
||||
this._currentTokenColors = null;
|
||||
this._themeListener = null;
|
||||
|
||||
grammarsExtPoint.setHandler((extensions) => {
|
||||
this._scopeRegistry.reset();
|
||||
this._injections = {};
|
||||
this._injectedEmbeddedLanguages = {};
|
||||
this._languageToScope = new Map<string, string>();
|
||||
this._grammarRegistry = null;
|
||||
this._tokenizersRegistrations = dispose(this._tokenizersRegistrations);
|
||||
this._currentTokenColors = null;
|
||||
if (this._themeListener) {
|
||||
this._themeListener.dispose();
|
||||
this._themeListener = null;
|
||||
}
|
||||
|
||||
for (const extension of extensions) {
|
||||
let grammars = extension.value;
|
||||
for (const grammar of grammars) {
|
||||
this._handleGrammarExtensionPointUser(extension.description.extensionLocation, grammar, extension.collector);
|
||||
}
|
||||
}
|
||||
|
||||
for (const createMode of this._createdModes) {
|
||||
this._registerDefinitionIfAvailable(createMode);
|
||||
}
|
||||
});
|
||||
|
||||
// Generate some color map until the grammar registry is loaded
|
||||
let colorTheme = this._themeService.getColorTheme();
|
||||
let defaultForeground: Color = Color.transparent;
|
||||
let defaultBackground: Color = Color.transparent;
|
||||
for (let i = 0, len = colorTheme.tokenColors.length; i < len; i++) {
|
||||
let rule = colorTheme.tokenColors[i];
|
||||
if (!rule.scope && rule.settings) {
|
||||
if (rule.settings.foreground) {
|
||||
defaultForeground = Color.fromHex(rule.settings.foreground);
|
||||
}
|
||||
if (rule.settings.background) {
|
||||
defaultBackground = Color.fromHex(rule.settings.background);
|
||||
}
|
||||
}
|
||||
}
|
||||
TokenizationRegistry.setColorMap([null!, defaultForeground, defaultBackground]);
|
||||
|
||||
this._modeService.onDidCreateMode((mode) => {
|
||||
let modeId = mode.getId();
|
||||
this._createdModes.push(modeId);
|
||||
this._registerDefinitionIfAvailable(modeId);
|
||||
});
|
||||
}
|
||||
|
||||
private _registerDefinitionIfAvailable(modeId: string): void {
|
||||
if (this._languageToScope.has(modeId)) {
|
||||
const promise = this._createGrammar(modeId).then((r) => {
|
||||
return new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService, this._configurationService);
|
||||
}, e => {
|
||||
onUnexpectedError(e);
|
||||
return null;
|
||||
});
|
||||
this._tokenizersRegistrations.push(TokenizationRegistry.registerPromise(modeId, promise));
|
||||
}
|
||||
}
|
||||
|
||||
private async _createGrammarRegistry(): Promise<[Registry, StackElement]> {
|
||||
const { Registry, INITIAL, parseRawGrammar } = await import('vscode-textmate');
|
||||
const grammarRegistry = new Registry({
|
||||
loadGrammar: async (scopeName: string) => {
|
||||
const location = this._scopeRegistry.getGrammarLocation(scopeName);
|
||||
if (!location) {
|
||||
this._logService.trace(`No grammar found for scope ${scopeName}`);
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const content = await this._fileService.readFile(location);
|
||||
return parseRawGrammar(content.value.toString(), location.path);
|
||||
} catch (e) {
|
||||
this._logService.error(`Unable to load and parse grammar for scope ${scopeName} from ${location}`, e);
|
||||
return null;
|
||||
}
|
||||
},
|
||||
getInjections: (scopeName: string) => {
|
||||
const scopeParts = scopeName.split('.');
|
||||
let injections: string[] = [];
|
||||
for (let i = 1; i <= scopeParts.length; i++) {
|
||||
const subScopeName = scopeParts.slice(0, i).join('.');
|
||||
injections = [...injections, ...(this._injections[subScopeName] || [])];
|
||||
}
|
||||
return injections;
|
||||
}
|
||||
});
|
||||
this._updateTheme(grammarRegistry);
|
||||
this._themeListener = this._themeService.onDidColorThemeChange((e) => this._updateTheme(grammarRegistry));
|
||||
return <[Registry, StackElement]>[grammarRegistry, INITIAL];
|
||||
}
|
||||
|
||||
private _getOrCreateGrammarRegistry(): Promise<[Registry, StackElement]> {
|
||||
if (!this._grammarRegistry) {
|
||||
this._grammarRegistry = this._createGrammarRegistry();
|
||||
}
|
||||
return this._grammarRegistry;
|
||||
}
|
||||
|
||||
private static _toColorMap(colorMap: string[]): Color[] {
|
||||
let result: Color[] = [null!];
|
||||
for (let i = 1, len = colorMap.length; i < len; i++) {
|
||||
result[i] = Color.fromHex(colorMap[i]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private _updateTheme(grammarRegistry: Registry): void {
|
||||
let colorTheme = this._themeService.getColorTheme();
|
||||
if (!this.compareTokenRules(colorTheme.tokenColors)) {
|
||||
return;
|
||||
}
|
||||
grammarRegistry.setTheme({ name: colorTheme.label, settings: colorTheme.tokenColors });
|
||||
let colorMap = TextMateService._toColorMap(grammarRegistry.getColorMap());
|
||||
let cssRules = generateTokensCSSForColorMap(colorMap);
|
||||
this._styleElement.innerHTML = cssRules;
|
||||
TokenizationRegistry.setColorMap(colorMap);
|
||||
}
|
||||
|
||||
private compareTokenRules(newRules: ITokenColorizationRule[]): boolean {
|
||||
let currRules = this._currentTokenColors;
|
||||
this._currentTokenColors = newRules;
|
||||
if (!newRules || !currRules || newRules.length !== currRules.length) {
|
||||
return true;
|
||||
}
|
||||
for (let i = newRules.length - 1; i >= 0; i--) {
|
||||
let r1 = newRules[i];
|
||||
let r2 = currRules[i];
|
||||
if (r1.scope !== r2.scope) {
|
||||
return true;
|
||||
}
|
||||
let s1 = r1.settings;
|
||||
let s2 = r2.settings;
|
||||
if (s1 && s2) {
|
||||
if (s1.fontStyle !== s2.fontStyle || s1.foreground !== s2.foreground || s1.background !== s2.background) {
|
||||
return true;
|
||||
}
|
||||
} else if (!s1 || !s2) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private _handleGrammarExtensionPointUser(extensionLocation: URI, syntax: ITMSyntaxExtensionPoint, collector: ExtensionMessageCollector): void {
|
||||
if (syntax.language && ((typeof syntax.language !== 'string') || !this._modeService.isRegisteredMode(syntax.language))) {
|
||||
collector.error(nls.localize('invalid.language', "Unknown language in `contributes.{0}.language`. Provided value: {1}", grammarsExtPoint.name, String(syntax.language)));
|
||||
return;
|
||||
}
|
||||
if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
|
||||
collector.error(nls.localize('invalid.scopeName', "Expected string in `contributes.{0}.scopeName`. Provided value: {1}", grammarsExtPoint.name, String(syntax.scopeName)));
|
||||
return;
|
||||
}
|
||||
if (!syntax.path || (typeof syntax.path !== 'string')) {
|
||||
collector.error(nls.localize('invalid.path.0', "Expected string in `contributes.{0}.path`. Provided value: {1}", grammarsExtPoint.name, String(syntax.path)));
|
||||
return;
|
||||
}
|
||||
if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || syntax.injectTo.some(scope => typeof scope !== 'string'))) {
|
||||
collector.error(nls.localize('invalid.injectTo', "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.injectTo)));
|
||||
return;
|
||||
}
|
||||
if (syntax.embeddedLanguages && !types.isObject(syntax.embeddedLanguages)) {
|
||||
collector.error(nls.localize('invalid.embeddedLanguages', "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.embeddedLanguages)));
|
||||
return;
|
||||
}
|
||||
|
||||
if (syntax.tokenTypes && !types.isObject(syntax.tokenTypes)) {
|
||||
collector.error(nls.localize('invalid.tokenTypes', "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}", grammarsExtPoint.name, JSON.stringify(syntax.tokenTypes)));
|
||||
return;
|
||||
}
|
||||
|
||||
const grammarLocation = resources.joinPath(extensionLocation, syntax.path);
|
||||
if (!resources.isEqualOrParent(grammarLocation, extensionLocation)) {
|
||||
collector.warn(nls.localize('invalid.path.1', "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.", grammarsExtPoint.name, grammarLocation.path, extensionLocation.path));
|
||||
}
|
||||
|
||||
this._scopeRegistry.register(syntax.scopeName, grammarLocation, syntax.embeddedLanguages, syntax.tokenTypes);
|
||||
|
||||
if (syntax.injectTo) {
|
||||
for (let injectScope of syntax.injectTo) {
|
||||
let injections = this._injections[injectScope];
|
||||
if (!injections) {
|
||||
this._injections[injectScope] = injections = [];
|
||||
}
|
||||
injections.push(syntax.scopeName);
|
||||
}
|
||||
|
||||
if (syntax.embeddedLanguages) {
|
||||
for (let injectScope of syntax.injectTo) {
|
||||
let injectedEmbeddedLanguages = this._injectedEmbeddedLanguages[injectScope];
|
||||
if (!injectedEmbeddedLanguages) {
|
||||
this._injectedEmbeddedLanguages[injectScope] = injectedEmbeddedLanguages = [];
|
||||
}
|
||||
injectedEmbeddedLanguages.push(syntax.embeddedLanguages);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let modeId = syntax.language;
|
||||
if (modeId) {
|
||||
this._languageToScope.set(modeId, syntax.scopeName);
|
||||
}
|
||||
}
|
||||
|
||||
private _resolveEmbeddedLanguages(embeddedLanguages: IEmbeddedLanguagesMap): IEmbeddedLanguagesMap2 {
|
||||
let scopes = Object.keys(embeddedLanguages);
|
||||
let result: IEmbeddedLanguagesMap2 = Object.create(null);
|
||||
for (let i = 0, len = scopes.length; i < len; i++) {
|
||||
let scope = scopes[i];
|
||||
let language = embeddedLanguages[scope];
|
||||
let languageIdentifier = this._modeService.getLanguageIdentifier(language);
|
||||
if (languageIdentifier) {
|
||||
result[scope] = languageIdentifier.id;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public async createGrammar(modeId: string): Promise<IGrammar> {
|
||||
const { grammar } = await this._createGrammar(modeId);
|
||||
return grammar;
|
||||
}
|
||||
|
||||
private async _createGrammar(modeId: string): Promise<ICreateGrammarResult> {
|
||||
const scopeName = this._languageToScope.get(modeId);
|
||||
if (typeof scopeName !== 'string') {
|
||||
// No TM grammar defined
|
||||
return Promise.reject(new Error(nls.localize('no-tm-grammar', "No TM Grammar registered for this language.")));
|
||||
}
|
||||
const languageRegistration = this._scopeRegistry.getLanguageRegistration(scopeName);
|
||||
if (!languageRegistration) {
|
||||
// No TM grammar defined
|
||||
return Promise.reject(new Error(nls.localize('no-tm-grammar', "No TM Grammar registered for this language.")));
|
||||
}
|
||||
let embeddedLanguages = this._resolveEmbeddedLanguages(languageRegistration.embeddedLanguages);
|
||||
let rawInjectedEmbeddedLanguages = this._injectedEmbeddedLanguages[scopeName];
|
||||
if (rawInjectedEmbeddedLanguages) {
|
||||
let injectedEmbeddedLanguages: IEmbeddedLanguagesMap2[] = rawInjectedEmbeddedLanguages.map(this._resolveEmbeddedLanguages.bind(this));
|
||||
for (const injected of injectedEmbeddedLanguages) {
|
||||
for (const scope of Object.keys(injected)) {
|
||||
embeddedLanguages[scope] = injected[scope];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let languageId = this._modeService.getLanguageIdentifier(modeId)!.id;
|
||||
let containsEmbeddedLanguages = (Object.keys(embeddedLanguages).length > 0);
|
||||
|
||||
const [grammarRegistry, initialState] = await this._getOrCreateGrammarRegistry();
|
||||
const grammar = await grammarRegistry.loadGrammarWithConfiguration(scopeName, languageId, { embeddedLanguages, tokenTypes: languageRegistration.tokenTypes });
|
||||
return {
|
||||
languageId: languageId,
|
||||
grammar: grammar,
|
||||
initialState: initialState,
|
||||
containsEmbeddedLanguages: containsEmbeddedLanguages
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
class TMTokenization implements ITokenizationSupport {
|
||||
|
||||
private readonly _scopeRegistry: TMScopeRegistry;
|
||||
private readonly _languageId: LanguageId;
|
||||
private readonly _grammar: IGrammar;
|
||||
private readonly _containsEmbeddedLanguages: boolean;
|
||||
private readonly _seenLanguages: boolean[];
|
||||
private readonly _initialState: StackElement;
|
||||
private _maxTokenizationLineLength: number;
|
||||
private _tokenizationWarningAlreadyShown: boolean;
|
||||
|
||||
constructor(scopeRegistry: TMScopeRegistry, languageId: LanguageId, grammar: IGrammar, initialState: StackElement, containsEmbeddedLanguages: boolean, @INotificationService private readonly notificationService: INotificationService, @IConfigurationService readonly configurationService: IConfigurationService) {
|
||||
this._scopeRegistry = scopeRegistry;
|
||||
this._languageId = languageId;
|
||||
this._grammar = grammar;
|
||||
this._initialState = initialState;
|
||||
this._containsEmbeddedLanguages = containsEmbeddedLanguages;
|
||||
this._seenLanguages = [];
|
||||
this._maxTokenizationLineLength = configurationService.getValue<number>('editor.maxTokenizationLineLength');
|
||||
}
|
||||
|
||||
public getInitialState(): IState {
|
||||
return this._initialState;
|
||||
}
|
||||
|
||||
public tokenize(line: string, state: IState, offsetDelta: number): TokenizationResult {
|
||||
throw new Error('Not supported!');
|
||||
}
|
||||
|
||||
public tokenize2(line: string, state: StackElement, offsetDelta: number): TokenizationResult2 {
|
||||
if (offsetDelta !== 0) {
|
||||
throw new Error('Unexpected: offsetDelta should be 0.');
|
||||
}
|
||||
|
||||
// Do not attempt to tokenize if a line is too long
|
||||
if (line.length >= this._maxTokenizationLineLength) {
|
||||
if (!this._tokenizationWarningAlreadyShown) {
|
||||
this._tokenizationWarningAlreadyShown = true;
|
||||
this.notificationService.warn(nls.localize('too many characters', "Tokenization is skipped for long lines for performance reasons. The length of a long line can be configured via `editor.maxTokenizationLineLength`."));
|
||||
}
|
||||
console.log(`Line (${line.substr(0, 15)}...): longer than ${this._maxTokenizationLineLength} characters, tokenization skipped.`);
|
||||
return nullTokenize2(this._languageId, line, state, offsetDelta);
|
||||
}
|
||||
|
||||
let textMateResult = this._grammar.tokenizeLine2(line, state);
|
||||
|
||||
if (this._containsEmbeddedLanguages) {
|
||||
let seenLanguages = this._seenLanguages;
|
||||
let tokens = textMateResult.tokens;
|
||||
|
||||
// Must check if any of the embedded languages was hit
|
||||
for (let i = 0, len = (tokens.length >>> 1); i < len; i++) {
|
||||
let metadata = tokens[(i << 1) + 1];
|
||||
let languageId = TokenMetadata.getLanguageId(metadata);
|
||||
|
||||
if (!seenLanguages[languageId]) {
|
||||
seenLanguages[languageId] = true;
|
||||
this._scopeRegistry.onEncounteredLanguage(languageId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let endState: StackElement;
|
||||
// try to save an object if possible
|
||||
if (state.equals(textMateResult.ruleStack)) {
|
||||
endState = state;
|
||||
} else {
|
||||
endState = textMateResult.ruleStack;
|
||||
|
||||
}
|
||||
|
||||
return new TokenizationResult2(textMateResult.tokens, endState);
|
||||
protected _loadVSCodeTextmate(): Promise<typeof import('vscode-textmate')> {
|
||||
return import('vscode-textmate');
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ import 'vs/platform/dialogs/browser/dialogService';
|
|||
import 'vs/workbench/services/bulkEdit/browser/bulkEditService';
|
||||
// import 'vs/workbench/services/integrity/node/integrityService';
|
||||
import 'vs/workbench/services/keybinding/common/keybindingEditing';
|
||||
// import 'vs/workbench/services/textMate/electron-browser/textMateService';
|
||||
import 'vs/workbench/services/textMate/browser/textMateService';
|
||||
// import 'vs/workbench/services/workspace/electron-browser/workspaceEditingService';
|
||||
// import 'vs/workbench/services/extensions/electron-browser/inactiveExtensionUrlHandler';
|
||||
import 'vs/workbench/services/decorations/browser/decorationsService';
|
||||
|
|
|
|||
|
|
@ -441,7 +441,9 @@
|
|||
"**/vs/platform/**/{common,browser}/**",
|
||||
"**/vs/editor/{common,browser}/**",
|
||||
"**/vs/workbench/{common,browser}/**",
|
||||
"**/vs/workbench/services/**/{common,browser}/**"
|
||||
"**/vs/workbench/services/**/{common,browser}/**",
|
||||
"vscode-textmate",
|
||||
"onigasm-umd"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
|
|||
13
yarn.lock
13
yarn.lock
|
|
@ -6402,6 +6402,11 @@ onetime@^2.0.0:
|
|||
dependencies:
|
||||
mimic-fn "^1.0.0"
|
||||
|
||||
onigasm-umd@^2.2.2:
|
||||
version "2.2.2"
|
||||
resolved "https://registry.yarnpkg.com/onigasm-umd/-/onigasm-umd-2.2.2.tgz#b989d762df61f899a3052ac794a50bd93fe20257"
|
||||
integrity sha512-v2eMOJu7iE444L2iJN+U6s6s5S0y7oj/N0DAkrd6wokRtTVoq/v/yaDI1lIqFrTeJbNtqNzYvguDF5yNzW3Rvw==
|
||||
|
||||
oniguruma@^7.0.0:
|
||||
version "7.0.2"
|
||||
resolved "https://registry.yarnpkg.com/oniguruma/-/oniguruma-7.0.2.tgz#a5c922cf7066da1dbcc60f6385a90437a83f8d0b"
|
||||
|
|
@ -9610,10 +9615,10 @@ vscode-sqlite3@4.0.7:
|
|||
dependencies:
|
||||
nan "~2.10.0"
|
||||
|
||||
vscode-textmate@^4.0.1:
|
||||
version "4.0.1"
|
||||
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.0.1.tgz#6c36f28e9059ce12bc34907f7a33ea43166b26a8"
|
||||
integrity sha512-gHTXTj04TUgbjB8y7pkVwxOiuCuD6aU5gnFzIByQuqdgFpe/bJaaEIS4geGjbjWbd1XJh6zG1EthLfpNaXEqUw==
|
||||
vscode-textmate@^4.1.1:
|
||||
version "4.1.1"
|
||||
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.1.1.tgz#857e836fbc13a376ec624242437e1747d79610a9"
|
||||
integrity sha512-xBjq9LH6fMhWDhIVkbKlB1JeCu6lT3FI/QKN24Xi4RKPBUm16IhHTqs6Q6SUGewkNsFZGkb1tJdZsuMnlmVpgw==
|
||||
dependencies:
|
||||
oniguruma "^7.0.0"
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue