From 45d383f26865432cb02d395e75a78a0525fafdca Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Fri, 16 Sep 2016 10:44:13 +0200 Subject: [PATCH] Move tokenizationSupport out of the mode instance and into TokenizationRegistry --- src/vs/editor/browser/standalone/colorizer.ts | 139 +++---- .../browser/standalone/standaloneEditor.ts | 2 +- .../browser/standalone/standaloneLanguages.ts | 10 +- src/vs/editor/common/commonCodeEditor.ts | 7 - src/vs/editor/common/editorCommon.ts | 27 +- .../editor/common/model/compatMirrorModel.ts | 6 +- .../editor/common/model/editableTextModel.ts | 6 +- src/vs/editor/common/model/model.ts | 17 +- .../common/model/textModelWithDecorations.ts | 6 +- .../common/model/textModelWithMarkers.ts | 6 +- .../common/model/textModelWithTokens.ts | 380 ++++-------------- .../model/textModelWithTrackedRanges.ts | 39 +- src/vs/editor/common/modes.ts | 79 ++-- src/vs/editor/common/modes/TMState.ts | 14 +- src/vs/editor/common/modes/abstractMode.ts | 74 ---- src/vs/editor/common/modes/abstractState.ts | 28 +- src/vs/editor/common/modes/modesRegistry.ts | 4 +- .../common/modes/monarch/monarchLexer.ts | 42 +- src/vs/editor/common/modes/nullMode.ts | 33 +- .../modes/supports/tokenizationSupport.ts | 183 ++++----- .../common/modes/textToHtmlTokenizer.ts | 17 +- .../services/compatWorkerServiceWorker.ts | 2 +- src/vs/editor/common/services/modeService.ts | 4 - .../editor/common/services/modeServiceImpl.ts | 72 +--- src/vs/editor/common/services/modelService.ts | 2 + .../common/services/modelServiceImpl.ts | 31 +- .../editor/common/viewModel/viewModelImpl.ts | 4 - .../test/common/blockCommentCommand.test.ts | 2 +- .../test/common/lineCommentCommand.test.ts | 8 +- .../test/common/tokenSelectionSupport.test.ts | 2 +- src/vs/editor/node/textMate/TMSyntax.ts | 16 +- .../test/common/commands/commandTestUtils.ts | 3 +- .../test/common/commands/shiftCommand.test.ts | 4 +- .../test/common/controller/cursor.test.ts | 36 +- src/vs/editor/test/common/mocks/mockMode.ts | 30 +- .../test/common/model/model.modes.test.ts | 210 +++++++--- .../common/model/textModelWithTokens.test.ts | 59 ++- .../common/modes/textToHtmlTokenizer.test.ts | 21 +- .../test/common/modes/tokenization.test.ts | 132 +++--- src/vs/editor/test/common/modesUtil.ts | 17 +- src/vs/editor/test/common/testModes.ts | 167 -------- .../languages/handlebars/common/handlebars.ts | 14 +- .../handlebars/test/common/handlebars.test.ts | 24 +- src/vs/languages/html/common/html.ts | 58 +-- .../html/test/common/html-worker.test.ts | 2 +- .../languages/html/test/common/html.test.ts | 32 +- src/vs/languages/php/common/php.ts | 88 ++-- src/vs/languages/php/test/common/php.test.ts | 56 ++- .../razor/common/csharpTokenization.ts | 80 ++-- src/vs/languages/razor/common/razor.ts | 18 +- src/vs/languages/razor/common/vsxml.ts | 43 +- .../languages/razor/test/common/razor.test.ts | 6 +- src/vs/monaco.d.ts | 4 - .../browser/parts/editor/editorStatus.ts | 34 +- .../common/editor/textEditorModel.ts | 2 +- .../emmet/test/common/editorAccessor.test.ts | 17 +- .../parts/git/test/common/stageRanges.test.ts | 4 +- .../themes.test.contribution.ts | 2 +- .../test/node/api/extHostApiCommands.test.ts | 1 + 59 files changed, 933 insertions(+), 1493 deletions(-) diff --git a/src/vs/editor/browser/standalone/colorizer.ts b/src/vs/editor/browser/standalone/colorizer.ts index 396882293f9..bd1691a5e1b 100644 --- a/src/vs/editor/browser/standalone/colorizer.ts +++ b/src/vs/editor/browser/standalone/colorizer.ts @@ -4,13 +4,12 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {RunOnceScheduler} from 'vs/base/common/async'; import {IDisposable} from 'vs/base/common/lifecycle'; import {TPromise} from 'vs/base/common/winjs.base'; import {IModel} from 'vs/editor/common/editorCommon'; -import {ILineTokens, IMode} from 'vs/editor/common/modes'; +import {TokenizationRegistry, ITokenizationSupport} from 'vs/editor/common/modes'; import {IModeService} from 'vs/editor/common/services/modeService'; -import {RenderLineOutput, renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer'; +import {renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer'; import {ViewLineToken} from 'vs/editor/common/core/viewLineToken'; export interface IColorizerOptions { @@ -40,7 +39,7 @@ export class Colorizer { return this.colorize(modeService, text, mimeType, options).then(render, (err) => console.error(err), render); } - private static _tokenizationSupportChangedPromise(target:IMode): TPromise { + private static _tokenizationSupportChangedPromise(languageId:string): TPromise { let listener: IDisposable = null; let stopListening = () => { if (listener) { @@ -50,8 +49,8 @@ export class Colorizer { }; return new TPromise((c, e, p) => { - listener = target.addSupportChangedListener((e) => { - if (e.tokenizationSupport) { + listener = TokenizationRegistry.onDidChange((e) => { + if (e.languageId === languageId) { stopListening(); c(void 0); } @@ -60,64 +59,30 @@ export class Colorizer { } public static colorize(modeService:IModeService, text:string, mimeType:string, options:IColorizerOptions): TPromise { + let lines = text.split('\n'); + let languageId = modeService.getModeId(mimeType); + options = options || {}; if (typeof options.tabSize === 'undefined') { options.tabSize = 4; } - let lines = text.split('\n'); - let c: (v:string)=>void; - let e: (err:any)=>void; - let p: (v:string)=>void; - let isCanceled = false; - let mode: IMode; + // Send out the event to create the mode + modeService.getOrCreateMode(languageId); - let result = new TPromise((_c, _e, _p) => { - c = _c; - e = _e; - p = _p; - }, () => { - isCanceled = true; + let tokenizationSupport = TokenizationRegistry.get(languageId); + if (tokenizationSupport) { + return TPromise.as(_colorize(lines, options.tabSize, tokenizationSupport)); + } + + // wait 500ms for mode to load, then give up + return TPromise.any([this._tokenizationSupportChangedPromise(languageId), TPromise.timeout(500)]).then(_ => { + let tokenizationSupport = TokenizationRegistry.get(languageId); + if (tokenizationSupport) { + return _colorize(lines, options.tabSize, tokenizationSupport); + } + return _fakeColorize(lines, options.tabSize); }); - - let colorize = new RunOnceScheduler(() => { - if (isCanceled) { - return; - } - let r = actualColorize(lines, mode, options.tabSize); - if (r.retokenize.length > 0) { - // There are retokenization requests - r.retokenize.forEach((p) => p.then(scheduleColorize)); - p(r.result); - } else { - // There are no (more) retokenization requests - c(r.result); - } - }, 0); - let scheduleColorize = () => colorize.schedule(); - - modeService.getOrCreateMode(mimeType).then((_mode) => { - if (!_mode) { - e('Mode not found: "' + mimeType + '".'); - return; - } - if (!_mode.tokenizationSupport) { - // wait 500ms for mode to load, then give up - TPromise.any([this._tokenizationSupportChangedPromise(_mode), TPromise.timeout(500)]).then(_ => { - if (!_mode.tokenizationSupport) { - e('Mode found ("' + _mode.getId() + '"), but does not support tokenization.'); - return; - } - mode = _mode; - scheduleColorize(); - }); - return; - } - mode = _mode; - scheduleColorize(); - }); - - return result; } public static colorizeLine(line:string, tokens:ViewLineToken[], tabSize:number = 4): string { @@ -141,32 +106,43 @@ export class Colorizer { } } - -interface IActualColorizeResult { - result:string; - retokenize:TPromise[]; +function _colorize(lines:string[], tabSize:number, tokenizationSupport: ITokenizationSupport): string { + return _actualColorize(lines, tabSize, tokenizationSupport); } -function actualColorize(lines:string[], mode:IMode, tabSize:number): IActualColorizeResult { - let tokenization = mode.tokenizationSupport, - html:string[] = [], - state = tokenization.getInitialState(), - i:number, - length:number, - line: string, - tokenizeResult: ILineTokens, - renderResult: RenderLineOutput, - retokenize: TPromise[] = []; +function _fakeColorize(lines:string[], tabSize:number): string { + let html:string[] = []; - for (i = 0, length = lines.length; i < length; i++) { - line = lines[i]; + for (let i = 0, length = lines.length; i < length; i++) { + let line = lines[i]; - tokenizeResult = tokenization.tokenize(line, state); - if (tokenizeResult.retokenize) { - retokenize.push(tokenizeResult.retokenize); - } + let renderResult = renderLine(new RenderLineInput( + line, + tabSize, + 0, + -1, + 'none', + false, + [] + )); - renderResult = renderLine(new RenderLineInput( + html = html.concat(renderResult.output); + html.push('
'); + } + + return html.join(''); +} + +function _actualColorize(lines:string[], tabSize:number, tokenizationSupport: ITokenizationSupport): string { + let html:string[] = []; + let state = tokenizationSupport.getInitialState(); + + for (let i = 0, length = lines.length; i < length; i++) { + let line = lines[i]; + + let tokenizeResult = tokenizationSupport.tokenize(line, state); + + let renderResult = renderLine(new RenderLineInput( line, tabSize, 0, @@ -182,8 +158,5 @@ function actualColorize(lines:string[], mode:IMode, tabSize:number): IActualColo state = tokenizeResult.endState; } - return { - result: html.join(''), - retokenize: retokenize - }; -} \ No newline at end of file + return html.join(''); +} diff --git a/src/vs/editor/browser/standalone/standaloneEditor.ts b/src/vs/editor/browser/standalone/standaloneEditor.ts index 69cbbd351bc..4c7a386de96 100644 --- a/src/vs/editor/browser/standalone/standaloneEditor.ts +++ b/src/vs/editor/browser/standalone/standaloneEditor.ts @@ -148,7 +148,7 @@ export function createModel(value:string, language?:string, uri?:URI): IModel { * Change the language for a model. */ export function setModelLanguage(model:IModel, language:string): void { - model.setMode(StaticServices.modeService.get().getOrCreateMode(language)); + StaticServices.modelService.get().setMode(model, StaticServices.modeService.get().getOrCreateMode(language)); } /** diff --git a/src/vs/editor/browser/standalone/standaloneLanguages.ts b/src/vs/editor/browser/standalone/standaloneLanguages.ts index 5faaca7524f..cb7e4442646 100644 --- a/src/vs/editor/browser/standalone/standaloneLanguages.ts +++ b/src/vs/editor/browser/standalone/standaloneLanguages.ts @@ -23,7 +23,7 @@ import {compile} from 'vs/editor/common/modes/monarch/monarchCompile'; import {createTokenizationSupport} from 'vs/editor/common/modes/monarch/monarchLexer'; import {LanguageConfigurationRegistry} from 'vs/editor/common/modes/languageConfigurationRegistry'; import {IMarkerData} from 'vs/platform/markers/common/markers'; - +import {TokenizationSupport2Adapter} from 'vs/editor/common/services/modeServiceImpl'; /** * Register information about a new language. */ @@ -67,7 +67,8 @@ export function setLanguageConfiguration(languageId:string, configuration:Langua * Set the tokens provider for a language (manual implementation). */ export function setTokensProvider(languageId:string, provider:modes.TokensProvider): IDisposable { - return StaticServices.modeService.get().registerTokenizationSupport2(languageId, provider); + let adapter = new TokenizationSupport2Adapter(languageId, provider); + return modes.TokenizationRegistry.register(languageId, adapter); } /** @@ -75,9 +76,8 @@ export function setTokensProvider(languageId:string, provider:modes.TokensProvid */ export function setMonarchTokensProvider(languageId:string, languageDef:IMonarchLanguage): IDisposable { let lexer = compile(languageId, languageDef); - return StaticServices.modeService.get().registerTokenizationSupport(languageId, (mode) => { - return createTokenizationSupport(StaticServices.modeService.get(), mode, lexer); - }); + let adapter = createTokenizationSupport(StaticServices.modeService.get(), languageId, lexer); + return modes.TokenizationRegistry.register(languageId, adapter); } /** diff --git a/src/vs/editor/common/commonCodeEditor.ts b/src/vs/editor/common/commonCodeEditor.ts index 75d7dc1e11a..28d4a2debea 100644 --- a/src/vs/editor/common/commonCodeEditor.ts +++ b/src/vs/editor/common/commonCodeEditor.ts @@ -46,9 +46,6 @@ export abstract class CommonCodeEditor extends EventEmitter implements editorCom public onDidChangeModelOptions(listener: (e:editorCommon.IModelOptionsChangedEvent)=>void): IDisposable { return this.addListener2(editorCommon.EventType.ModelOptionsChanged, listener); } - public onDidChangeModelModeSupport(listener: (e:editorCommon.IModeSupportChangedEvent)=>void): IDisposable { - return this.addListener2(editorCommon.EventType.ModelModeSupportChanged, listener); - } public onDidChangeModelDecorations(listener: (e:editorCommon.IModelDecorationsChangedEvent)=>void): IDisposable { return this.addListener2(editorCommon.EventType.ModelDecorationsChanged, listener); } @@ -856,10 +853,6 @@ export abstract class CommonCodeEditor extends EventEmitter implements editorCom this.emit(editorCommon.EventType.ModelModeChanged, e); break; - case editorCommon.EventType.ModelModeSupportChanged: - this.emit(editorCommon.EventType.ModelModeSupportChanged, e); - break; - case editorCommon.EventType.ModelRawContentChanged: this.emit(editorCommon.EventType.ModelRawContentChanged, e); break; diff --git a/src/vs/editor/common/editorCommon.ts b/src/vs/editor/common/editorCommon.ts index 717ecfa12b7..d84c061c0de 100644 --- a/src/vs/editor/common/editorCommon.ts +++ b/src/vs/editor/common/editorCommon.ts @@ -1029,14 +1029,6 @@ export interface IConfigurationChangedEvent { contribInfo: boolean; } -/** - * An event describing that one or more supports of a mode have changed. - * @internal - */ -export interface IModeSupportChangedEvent { - tokenizationSupport:boolean; -} - /** * Vertical Lane in the overview ruler of the editor. */ @@ -1857,17 +1849,9 @@ export interface ITokenizedModel extends ITextModel { /** * Set the current language mode associated with the model. - */ - setMode(newMode:IMode|TPromise): void; - - /** - * A mode can be currently pending loading if a promise is used when constructing a model or calling setMode(). - * - * If there is no currently pending loading mode, then the result promise will complete immediately. - * Otherwise, the result will complete once the currently pending loading mode is loaded. * @internal */ - whenModeIsReady(): TPromise; + setMode(languageId:string): void; /** * Returns the true (inner-most) language mode at a given position. @@ -2214,10 +2198,6 @@ export interface IModel extends IReadOnlyModel, IEditableTextModel, ITextModelWi * An event emitted when the contents of the model have changed. */ onDidChangeContent(listener: (e:IModelContentChangedEvent2)=>void): IDisposable; - /** - * @internal - */ - onDidChangeModeSupport(listener: (e:IModeSupportChangedEvent)=>void): IDisposable; /** * An event emitted when decorations of the model have changed. */ @@ -3901,10 +3881,6 @@ export interface ICommonCodeEditor extends IEditor { * An event emitted when the model of this editor has changed (e.g. `editor.setModel()`). */ onDidChangeModel(listener: (e:IModelChangedEvent)=>void): IDisposable; - /** - * @internal - */ - onDidChangeModelModeSupport(listener: (e:IModeSupportChangedEvent)=>void): IDisposable; /** * An event emitted when the decorations of the current model have changed. */ @@ -4188,7 +4164,6 @@ export var EventType = { ModelTokensChanged: 'modelTokensChanged', ModelModeChanged: 'modelsModeChanged', - ModelModeSupportChanged: 'modelsModeSupportChanged', ModelOptionsChanged: 'modelOptionsChanged', ModelRawContentChanged: 'contentChanged', ModelContentChanged2: 'contentChanged2', diff --git a/src/vs/editor/common/model/compatMirrorModel.ts b/src/vs/editor/common/model/compatMirrorModel.ts index d55fe353dc1..23fcc2abb06 100644 --- a/src/vs/editor/common/model/compatMirrorModel.ts +++ b/src/vs/editor/common/model/compatMirrorModel.ts @@ -5,11 +5,9 @@ 'use strict'; import URI from 'vs/base/common/uri'; -import {TPromise} from 'vs/base/common/winjs.base'; import * as editorCommon from 'vs/editor/common/editorCommon'; import {ModelLine} from 'vs/editor/common/model/modelLine'; import {TextModelWithTokens} from 'vs/editor/common/model/textModelWithTokens'; -import {IMode} from 'vs/editor/common/modes'; import {ICompatMirrorModel} from 'vs/editor/common/services/resourceService'; export interface ICompatMirrorModelEvents { @@ -22,8 +20,8 @@ export class CompatMirrorModel extends TextModelWithTokens implements ICompatMir protected _associatedResource:URI; - constructor(versionId:number, value:editorCommon.IRawText, mode:IMode|TPromise, associatedResource?:URI) { - super(['changed', editorCommon.EventType.ModelDispose], value, mode); + constructor(versionId:number, value:editorCommon.IRawText, languageId:string, associatedResource?:URI) { + super(['changed', editorCommon.EventType.ModelDispose], value, languageId); this._setVersionId(versionId); this._associatedResource = associatedResource; diff --git a/src/vs/editor/common/model/editableTextModel.ts b/src/vs/editor/common/model/editableTextModel.ts index c474407629e..1885e0eea52 100644 --- a/src/vs/editor/common/model/editableTextModel.ts +++ b/src/vs/editor/common/model/editableTextModel.ts @@ -4,13 +4,11 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {TPromise} from 'vs/base/common/winjs.base'; import {Range} from 'vs/editor/common/core/range'; import * as editorCommon from 'vs/editor/common/editorCommon'; import {EditStack} from 'vs/editor/common/model/editStack'; import {ILineEdit, ILineMarker, ModelLine} from 'vs/editor/common/model/modelLine'; import {DeferredEventsBuilder, TextModelWithDecorations} from 'vs/editor/common/model/textModelWithDecorations'; -import {IMode} from 'vs/editor/common/modes'; import * as strings from 'vs/base/common/strings'; import {Selection} from 'vs/editor/common/core/selection'; import {IDisposable} from 'vs/base/common/lifecycle'; @@ -50,10 +48,10 @@ export class EditableTextModel extends TextModelWithDecorations implements edito private _trimAutoWhitespaceLines: number[]; - constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, modeOrPromise:IMode|TPromise) { + constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, languageId:string) { allowedEventTypes.push(editorCommon.EventType.ModelRawContentChanged); allowedEventTypes.push(editorCommon.EventType.ModelContentChanged2); - super(allowedEventTypes, rawText, modeOrPromise); + super(allowedEventTypes, rawText, languageId); this._commandManager = new EditStack(this); diff --git a/src/vs/editor/common/model/model.ts b/src/vs/editor/common/model/model.ts index 93c8e9f14a6..7f8d2e82993 100644 --- a/src/vs/editor/common/model/model.ts +++ b/src/vs/editor/common/model/model.ts @@ -5,14 +5,12 @@ 'use strict'; import URI from 'vs/base/common/uri'; -import {TPromise} from 'vs/base/common/winjs.base'; import { - EventType, IModel, ITextModelCreationOptions, IModeSupportChangedEvent, IModelDecorationsChangedEvent, + EventType, IModel, ITextModelCreationOptions, IModelDecorationsChangedEvent, IModelOptionsChangedEvent, IModelModeChangedEvent, IRawText } from 'vs/editor/common/editorCommon'; import {EditableTextModel} from 'vs/editor/common/model/editableTextModel'; import {TextModel} from 'vs/editor/common/model/textModel'; -import {IMode} from 'vs/editor/common/modes'; import {IDisposable} from 'vs/base/common/lifecycle'; import {BulkListenerCallback} from 'vs/base/common/eventEmitter'; @@ -36,9 +34,6 @@ var aliveModels:{[modelId:string]:boolean;} = {}; export class Model extends EditableTextModel implements IModel { - public onDidChangeModeSupport(listener: (e:IModeSupportChangedEvent)=>void): IDisposable { - return this.addListener2(EventType.ModelModeSupportChanged, listener); - } public onDidChangeDecorations(listener: (e:IModelDecorationsChangedEvent)=>void): IDisposable { return this.addListener2(EventType.ModelDecorationsChanged, listener); } @@ -56,9 +51,9 @@ export class Model extends EditableTextModel implements IModel { return super.addBulkListener(listener); } - public static createFromString(text:string, options:ITextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, mode:IMode|TPromise = null, uri:URI = null): Model { + public static createFromString(text:string, options:ITextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, languageId:string = null, uri:URI = null): Model { let rawText = TextModel.toRawText(text, options); - return new Model(rawText, mode, uri); + return new Model(rawText, languageId, uri); } public id:string; @@ -79,10 +74,8 @@ export class Model extends EditableTextModel implements IModel { * The resource associated with this model. If the value is not provided an * unique in memory URL is constructed as the associated resource. */ - constructor(rawText:IRawText, modeOrPromise:IMode|TPromise, associatedResource:URI=null) { - super([ - EventType.ModelDispose - ], rawText, modeOrPromise); + constructor(rawText:IRawText, languageId:string, associatedResource:URI=null) { + super([ EventType.ModelDispose ], rawText, languageId); // Generate a new unique model id MODEL_ID++; diff --git a/src/vs/editor/common/model/textModelWithDecorations.ts b/src/vs/editor/common/model/textModelWithDecorations.ts index b008f2f3588..49a9f71ca32 100644 --- a/src/vs/editor/common/model/textModelWithDecorations.ts +++ b/src/vs/editor/common/model/textModelWithDecorations.ts @@ -7,12 +7,10 @@ import {onUnexpectedError} from 'vs/base/common/errors'; import {MarkedString, markedStringsEquals} from 'vs/base/common/htmlContent'; import * as strings from 'vs/base/common/strings'; -import {TPromise} from 'vs/base/common/winjs.base'; import {IdGenerator} from 'vs/base/common/idGenerator'; import {Range} from 'vs/editor/common/core/range'; import * as editorCommon from 'vs/editor/common/editorCommon'; import {TextModelWithTrackedRanges} from 'vs/editor/common/model/textModelWithTrackedRanges'; -import {IMode} from 'vs/editor/common/modes'; export class DeferredEventsBuilder { @@ -96,9 +94,9 @@ export class TextModelWithDecorations extends TextModelWithTrackedRanges impleme private decorations:IInternalDecorationsMap; private rangeIdToDecorationId:IRangeIdToDecorationIdMap; - constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, modeOrPromise:IMode|TPromise) { + constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, languageId:string) { allowedEventTypes.push(editorCommon.EventType.ModelDecorationsChanged); - super(allowedEventTypes, rawText, modeOrPromise); + super(allowedEventTypes, rawText, languageId); // Initialize decorations this._decorationIdGenerator = new IdGenerator((++_INSTANCE_COUNT) + ';'); diff --git a/src/vs/editor/common/model/textModelWithMarkers.ts b/src/vs/editor/common/model/textModelWithMarkers.ts index 427b4d465f9..534383b3232 100644 --- a/src/vs/editor/common/model/textModelWithMarkers.ts +++ b/src/vs/editor/common/model/textModelWithMarkers.ts @@ -4,13 +4,11 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {TPromise} from 'vs/base/common/winjs.base'; import {IdGenerator} from 'vs/base/common/idGenerator'; import {Position} from 'vs/editor/common/core/position'; import {IModelContentChangedFlushEvent, IRawText, IReadOnlyLineMarker, ITextModelWithMarkers} from 'vs/editor/common/editorCommon'; import {ILineMarker, ModelLine} from 'vs/editor/common/model/modelLine'; import {TextModelWithTokens} from 'vs/editor/common/model/textModelWithTokens'; -import {IMode} from 'vs/editor/common/modes'; export interface IMarkerIdToMarkerMap { [key:string]:ILineMarker; @@ -51,8 +49,8 @@ export class TextModelWithMarkers extends TextModelWithTokens implements ITextMo private _markerIdGenerator: IdGenerator; protected _markerIdToMarker: IMarkerIdToMarkerMap; - constructor(allowedEventTypes:string[], rawText:IRawText, modeOrPromise:IMode|TPromise) { - super(allowedEventTypes, rawText, modeOrPromise); + constructor(allowedEventTypes:string[], rawText:IRawText, languageId:string) { + super(allowedEventTypes, rawText, languageId); this._markerIdGenerator = new IdGenerator((++_INSTANCE_COUNT) + ';'); this._markerIdToMarker = {}; } diff --git a/src/vs/editor/common/model/textModelWithTokens.ts b/src/vs/editor/common/model/textModelWithTokens.ts index bef45d17dff..8345dd981be 100644 --- a/src/vs/editor/common/model/textModelWithTokens.ts +++ b/src/vs/editor/common/model/textModelWithTokens.ts @@ -5,20 +5,18 @@ 'use strict'; import * as nls from 'vs/nls'; -import {RunOnceScheduler} from 'vs/base/common/async'; import {onUnexpectedError} from 'vs/base/common/errors'; -import {IDisposable, dispose} from 'vs/base/common/lifecycle'; +import {IDisposable} from 'vs/base/common/lifecycle'; import {StopWatch} from 'vs/base/common/stopwatch'; import * as timer from 'vs/base/common/timer'; -import {TPromise} from 'vs/base/common/winjs.base'; import {Range} from 'vs/editor/common/core/range'; import * as editorCommon from 'vs/editor/common/editorCommon'; import {ModelLine} from 'vs/editor/common/model/modelLine'; import {TextModel} from 'vs/editor/common/model/textModel'; import {WordHelper} from 'vs/editor/common/model/textModelWithTokensHelpers'; import {TokenIterator} from 'vs/editor/common/model/tokenIterator'; -import {ILineContext, ILineTokens, IMode, IState} from 'vs/editor/common/modes'; -import {NullMode, NullState, nullTokenize} from 'vs/editor/common/modes/nullMode'; +import {ITokenizationSupport, ILineContext, ILineTokens, IMode, IState, TokenizationRegistry} from 'vs/editor/common/modes'; +import {NULL_MODE_ID, nullTokenize} from 'vs/editor/common/modes/nullMode'; import {ignoreBracketsInToken} from 'vs/editor/common/modes/supports'; import {BracketsUtils} from 'vs/editor/common/modes/supports/richEditBrackets'; import {ModeTransition} from 'vs/editor/common/core/modeTransition'; @@ -27,101 +25,16 @@ import {Position} from 'vs/editor/common/core/position'; import {LanguageConfigurationRegistry} from 'vs/editor/common/modes/languageConfigurationRegistry'; import {Token} from 'vs/editor/common/core/token'; -class ModeToModelBinder implements IDisposable { +class Mode implements IMode { - private _modePromise:TPromise; - private _externalModePromise:TPromise; - private _externalModePromise_c:(value:boolean)=>void; - private _externalModePromise_e:(err:any)=>void; - private _model:TextModelWithTokens; - private _isDisposed:boolean; + private _languageId:string; - constructor(modePromise:TPromise, model:TextModelWithTokens) { - this._modePromise = modePromise; - // Create an external mode promise that fires after the mode is set to the model - this._externalModePromise = new TPromise((c, e, p) => { - this._externalModePromise_c = c; - this._externalModePromise_e = e; - }, () => { - // this promise cannot be canceled - }); - this._model = model; - this._isDisposed = false; - - // Ensure asynchronicity - TPromise.timeout(0).then(() => { - return this._modePromise; - }).then((mode:IMode) => { - if (this._isDisposed) { - this._externalModePromise_c(false); - return; - } - var model = this._model; - this.dispose(); - model.setMode(mode); - model._warmUpTokens(); - this._externalModePromise_c(true); - }).done(null, (err) => { - this._externalModePromise_e(err); - onUnexpectedError(err); - }); + constructor(languageId:string) { + this._languageId = languageId; } - public getModePromise(): TPromise { - return this._externalModePromise; - } - - public dispose(): void { - this._modePromise = null; - this._model = null; - this._isDisposed = true; - } -} - -export interface IRetokenizeRequest extends IDisposable { - - isFulfilled: boolean; - - /** - * If null, the entire model will be retokenzied, use null with caution - */ - getRange(): editorCommon.IRange; -} - -export class FullModelRetokenizer implements IRetokenizeRequest { - - public isFulfilled: boolean; - - protected _model:TextModelWithTokens; - private _retokenizePromise:TPromise; - private _isDisposed: boolean; - - constructor(retokenizePromise:TPromise, model:TextModelWithTokens) { - this._retokenizePromise = retokenizePromise; - this._model = model; - this._isDisposed = false; - this.isFulfilled = false; - - // Ensure asynchronicity - TPromise.timeout(0).then(() => { - return this._retokenizePromise; - }).then(() => { - if (this._isDisposed) { - return; - } - this.isFulfilled = true; - this._model.onRetokenizerFulfilled(); - }).done(null, onUnexpectedError); - } - - public getRange(): editorCommon.IRange { - return null; - } - - public dispose(): void { - this._retokenizePromise = null; - this._model = null; - this._isDisposed = true; + getId(): string { + return this._languageId; } } @@ -162,75 +75,45 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke private static MODE_TOKENIZATION_FAILED_MSG = nls.localize('mode.tokenizationSupportFailed', "The mode has failed while tokenizing the input."); - private _mode: IMode; - private _modeListener: IDisposable; - private _modeToModelBinder:ModeToModelBinder; + private _languageId: string; + private _tokenizationListener: IDisposable; + private _tokenizationSupport: ITokenizationSupport; private _tokensInflatorMap:TokensInflatorMap; private _invalidLineStartIndex:number; private _lastState:IState; private _revalidateTokensTimeout:number; - private _scheduleRetokenizeNow: RunOnceScheduler; - private _retokenizers:IRetokenizeRequest[]; - constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, modeOrPromise:IMode|TPromise) { + constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, languageId:string) { allowedEventTypes.push(editorCommon.EventType.ModelTokensChanged); allowedEventTypes.push(editorCommon.EventType.ModelModeChanged); - allowedEventTypes.push(editorCommon.EventType.ModelModeSupportChanged); super(allowedEventTypes, rawText); - this._mode = null; - this._modeListener = null; - this._modeToModelBinder = null; + this._languageId = languageId || NULL_MODE_ID; + this._tokenizationListener = TokenizationRegistry.onDidChange((e) => { + if (e.languageId !== this._languageId) { + return; + } + + this._resetTokenizationState(); + this.emitModelTokensChangedEvent(1, this.getLineCount()); + }); this._tokensInflatorMap = null; this._invalidLineStartIndex = 0; this._lastState = null; this._revalidateTokensTimeout = -1; - this._scheduleRetokenizeNow = null; - this._retokenizers = null; - - if (!modeOrPromise) { - this._mode = new NullMode(); - } else if (TPromise.is(modeOrPromise)) { - // TODO@Alex: To avoid mode id changes, we check if this promise is resolved - let promiseValue = (modeOrPromise)._value; - - if (promiseValue && typeof promiseValue.getId === 'function') { - // The promise is already resolved - this._mode = this._massageMode(promiseValue); - this._resetModeListener(this._mode); - } else { - var modePromise = >modeOrPromise; - this._modeToModelBinder = new ModeToModelBinder(modePromise, this); - this._mode = new NullMode(); - } - } else { - this._mode = this._massageMode(modeOrPromise); - this._resetModeListener(this._mode); - } - - this._revalidateTokensTimeout = -1; - this._scheduleRetokenizeNow = new RunOnceScheduler(() => this._retokenizeNow(), 200); - this._retokenizers = []; this._resetTokenizationState(); } public dispose(): void { - if (this._modeToModelBinder) { - this._modeToModelBinder.dispose(); - this._modeToModelBinder = null; - } - this._resetModeListener(null); + this._tokenizationListener.dispose(); this._clearTimers(); - this._mode = null; this._lastState = null; this._tokensInflatorMap = null; - this._retokenizers = dispose(this._retokenizers); - this._scheduleRetokenizeNow.dispose(); super.dispose(); } @@ -239,130 +122,38 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke return false; } - private _massageMode(mode: IMode): IMode { - if (this.isTooLargeForHavingAMode()) { - return new NullMode(); - } - if (this.isTooLargeForHavingARichMode()) { - return mode.toSimplifiedMode(); - } - return mode; - } - - public whenModeIsReady(): TPromise { - if (this._modeToModelBinder) { - // Still waiting for some mode to load - return this._modeToModelBinder.getModePromise().then(() => this._mode); - } - return TPromise.as(this._mode); - } - - public onRetokenizerFulfilled(): void { - this._scheduleRetokenizeNow.schedule(); - } - - private _retokenizeNow(): void { - var fulfilled = this._retokenizers.filter(r => r.isFulfilled); - this._retokenizers = this._retokenizers.filter(r => !r.isFulfilled); - - var hasFullModel = false; - for (var i = 0; i < fulfilled.length; i++) { - if (!fulfilled[i].getRange()) { - hasFullModel = true; - } - } - - if (hasFullModel) { - // Just invalidate all the lines - for (var i = 0, len = this._lines.length; i < len; i++) { - this._lines[i].isInvalid = true; - } - this._invalidLineStartIndex = 0; - } else { - var minLineNumber = Number.MAX_VALUE; - for (var i = 0; i < fulfilled.length; i++) { - var range = fulfilled[i].getRange(); - minLineNumber = Math.min(minLineNumber, range.startLineNumber); - for (var lineNumber = range.startLineNumber; lineNumber <= range.endLineNumber; lineNumber++) { - this._lines[lineNumber - 1].isInvalid = true; - } - } - if (minLineNumber - 1 < this._invalidLineStartIndex) { - if (this._invalidLineStartIndex < this._lines.length) { - this._lines[this._invalidLineStartIndex].isInvalid = true; - } - this._invalidLineStartIndex = minLineNumber - 1; - } - } - - this._beginBackgroundTokenization(); - - for (var i = 0; i < fulfilled.length; i++) { - fulfilled[i].dispose(); - } - } - - protected _createRetokenizer(retokenizePromise:TPromise, lineNumber:number): IRetokenizeRequest { - return new FullModelRetokenizer(retokenizePromise, this); - } - protected _resetValue(e:editorCommon.IModelContentChangedFlushEvent, newValue:editorCommon.IRawText): void { super._resetValue(e, newValue); // Cancel tokenization, clear all tokens and begin tokenizing this._resetTokenizationState(); } - protected _resetMode(e:editorCommon.IModelModeChangedEvent, newMode:IMode): void { - // Cancel tokenization, clear all tokens and begin tokenizing - this._mode = newMode; - this._resetModeListener(newMode); - this._resetTokenizationState(); - - this.emitModelTokensChangedEvent(1, this.getLineCount()); - } - - private _resetModeListener(newMode: IMode): void { - if (this._modeListener) { - this._modeListener.dispose(); - this._modeListener = null; - } - if (newMode && typeof newMode.addSupportChangedListener === 'function') { - this._modeListener = newMode.addSupportChangedListener( (e) => this._onModeSupportChanged(e) ); - } - } - - private _onModeSupportChanged(e: editorCommon.IModeSupportChangedEvent): void { - this._emitModelModeSupportChangedEvent(e); - if (e.tokenizationSupport) { - this._resetTokenizationState(); - this.emitModelTokensChangedEvent(1, this.getLineCount()); - } - } - protected _resetTokenizationState(): void { - this._retokenizers = dispose(this._retokenizers); - this._scheduleRetokenizeNow.cancel(); this._clearTimers(); - for (var i = 0; i < this._lines.length; i++) { + for (let i = 0; i < this._lines.length; i++) { this._lines[i].resetTokenizationState(); } - // Initialize tokenization states - var initialState:IState = null; - if (this._mode.tokenizationSupport) { + this._tokenizationSupport = null; + if (!this.isTooLargeForHavingAMode()) { + this._tokenizationSupport = TokenizationRegistry.get(this._languageId); + } + + if (this._tokenizationSupport) { + let initialState:IState = null; try { - initialState = this._mode.tokenizationSupport.getInitialState(); + initialState = this._tokenizationSupport.getInitialState(); } catch (e) { e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG; onUnexpectedError(e); - this._mode = new NullMode(); + this._tokenizationSupport = null; + } + + if (initialState) { + this._lines[0].setState(initialState); } } - if (!initialState) { - initialState = new NullState(this._mode, null); - } - this._lines[0].setState(initialState); this._lastState = null; this._tokensInflatorMap = new TokensInflatorMap(); this._invalidLineStartIndex = 0; @@ -403,38 +194,37 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } public getMode(): IMode { - return this._mode; + return new Mode(this._languageId); } public getModeId(): string { return this.getMode().getId(); } - public setMode(newModeOrPromise:IMode|TPromise): void { - if (!newModeOrPromise) { + public setMode(languageId:string): void { + if (this._languageId === languageId) { // There's nothing to do return; } - if (this._modeToModelBinder) { - this._modeToModelBinder.dispose(); - this._modeToModelBinder = null; - } - if (TPromise.is(newModeOrPromise)) { - this._modeToModelBinder = new ModeToModelBinder(>newModeOrPromise, this); - } else { - var actualNewMode = this._massageMode(newModeOrPromise); - if (this._mode !== actualNewMode) { - var e2:editorCommon.IModelModeChangedEvent = { - oldMode: this._mode, - newMode: actualNewMode - }; - this._resetMode(e2, actualNewMode); - this._emitModelModeChangedEvent(e2); - } - } + + let e:editorCommon.IModelModeChangedEvent = { + oldMode: new Mode(this._languageId), + newMode: new Mode(languageId) + }; + + this._languageId = languageId; + + // Cancel tokenization, clear all tokens and begin tokenizing + this._resetTokenizationState(); + + this.emitModelTokensChangedEvent(1, this.getLineCount()); + this._emitModelModeChangedEvent(e); } public getModeIdAtPosition(_lineNumber:number, _column:number): string { + if (!this._tokenizationSupport) { + return this.getModeId(); + } var validPosition = this.validatePosition({ lineNumber: _lineNumber, column: _column @@ -444,9 +234,9 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke var column = validPosition.column; if (column === 1) { - return this.getStateBeforeLine(lineNumber).getMode().getId(); + return this.getStateBeforeLine(lineNumber).getModeId(); } else if (column === this.getLineMaxColumn(lineNumber)) { - return this.getStateAfterLine(lineNumber).getMode().getId(); + return this.getStateAfterLine(lineNumber).getModeId(); } else { var modeTransitions = this._getLineModeTransitions(lineNumber); var modeTransitionIndex = ModeTransition.findIndexInSegmentsArray(modeTransitions, column - 1); @@ -567,6 +357,11 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } private _updateTokensUntilLine(lineNumber:number, emitEvents:boolean): void { + if (!this._tokenizationSupport) { + this._invalidLineStartIndex = this._lines.length; + return; + } + var linesLength = this._lines.length; var endLineIndex = lineNumber - 1; var stopLineTokenizationAfter = 1000000000; // 1 billion, if a line is so long, you have other trouble :). @@ -578,32 +373,26 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke var endStateIndex = lineIndex + 1; var r:ILineTokens = null; var text = this._lines[lineIndex].text; - if (this._mode.tokenizationSupport) { - try { - // Tokenize only the first X characters - r = this._mode.tokenizationSupport.tokenize(this._lines[lineIndex].text, this._lines[lineIndex].getState(), 0, stopLineTokenizationAfter); - } catch (e) { - e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG; - onUnexpectedError(e); - } + try { + // Tokenize only the first X characters + r = this._tokenizationSupport.tokenize(this._lines[lineIndex].text, this._lines[lineIndex].getState(), 0, stopLineTokenizationAfter); + } catch (e) { + e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG; + onUnexpectedError(e); + } - if (r && r.retokenize) { - this._retokenizers.push(this._createRetokenizer(r.retokenize, lineIndex + 1)); - } + if (r && r.tokens && r.tokens.length > 0) { + // Cannot have a stop offset before the last token + r.actualStopOffset = Math.max(r.actualStopOffset, r.tokens[r.tokens.length - 1].startIndex + 1); + } - if (r && r.tokens && r.tokens.length > 0) { - // Cannot have a stop offset before the last token - r.actualStopOffset = Math.max(r.actualStopOffset, r.tokens[r.tokens.length - 1].startIndex + 1); - } + if (r && r.actualStopOffset < text.length) { + // Treat the rest of the line (if above limit) as one default token + r.tokens.push(new Token(r.actualStopOffset, '')); - if (r && r.actualStopOffset < text.length) { - // Treat the rest of the line (if above limit) as one default token - r.tokens.push(new Token(r.actualStopOffset, '')); - - // Use as end state the starting state - r.endState = this._lines[lineIndex].getState(); - } + // Use as end state the starting state + r.endState = this._lines[lineIndex].getState(); } if (!r) { @@ -617,10 +406,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke r.modeTransitions.push(new ModeTransition(0, this.getModeId())); } this._lines[lineIndex].setTokens(this._tokensInflatorMap, r.tokens, this.getModeId(), r.modeTransitions); - - if (this._lines[lineIndex].isInvalid) { - this._lines[lineIndex].isInvalid = false; - } + this._lines[lineIndex].isInvalid = false; if (endStateIndex < linesLength) { if (this._lines[endStateIndex].getState() !== null && r.endState.equals(this._lines[endStateIndex].getState())) { @@ -673,12 +459,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } } - private _emitModelModeSupportChangedEvent(e:editorCommon.IModeSupportChangedEvent): void { - if (!this._isDisposing) { - this.emit(editorCommon.EventType.ModelModeSupportChanged, e); - } - } - // Having tokens allows implementing additional helper methods _lineIsTokenized(lineNumber:number): boolean { diff --git a/src/vs/editor/common/model/textModelWithTrackedRanges.ts b/src/vs/editor/common/model/textModelWithTrackedRanges.ts index 0320a75678f..c188b1604a3 100644 --- a/src/vs/editor/common/model/textModelWithTrackedRanges.ts +++ b/src/vs/editor/common/model/textModelWithTrackedRanges.ts @@ -4,14 +4,11 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {TPromise} from 'vs/base/common/winjs.base'; import {IdGenerator} from 'vs/base/common/idGenerator'; import {Range} from 'vs/editor/common/core/range'; import * as editorCommon from 'vs/editor/common/editorCommon'; import {ILineMarker} from 'vs/editor/common/model/modelLine'; import {INewMarker, TextModelWithMarkers} from 'vs/editor/common/model/textModelWithMarkers'; -import {FullModelRetokenizer, IRetokenizeRequest} from 'vs/editor/common/model/textModelWithTokens'; -import {IMode} from 'vs/editor/common/modes'; import {Position} from 'vs/editor/common/core/position'; interface ITrackedRange { @@ -28,34 +25,6 @@ interface IMarkerIdToRangeIdMap { [key:string]:string; } -class TrackedRangeModelRetokenizer extends FullModelRetokenizer { - - private trackedRangeId: string; - - constructor(retokenizePromise:TPromise, lineNumber:number, model:TextModelWithTrackedRanges) { - super(retokenizePromise, model); - this.trackedRangeId = model.addTrackedRange({ - startLineNumber: lineNumber, - startColumn : 1, - endLineNumber: lineNumber, - endColumn: model.getLineMaxColumn(lineNumber) - }, editorCommon.TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges); - } - - public getRange(): editorCommon.IRange { - return (this._model).getTrackedRange(this.trackedRangeId); - } - - public dispose(): void { - var model = (this._model); - // if this .dispose() is being called as part of the model.dispose(), then the tracked ranges might no longer be available (e.g. throw exceptions) - if (model.isValidTrackedRange(this.trackedRangeId)) { - model.removeTrackedRange(this.trackedRangeId); - } - super.dispose(); - } -} - class TrackedRange implements ITrackedRange { id:string; startMarkerId:string; @@ -77,18 +46,14 @@ export class TextModelWithTrackedRanges extends TextModelWithMarkers implements private _markerIdToRangeId:IMarkerIdToRangeIdMap; private _multiLineTrackedRanges: { [key:string]: boolean; }; - constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, modeOrPromise:IMode|TPromise) { - super(allowedEventTypes, rawText, modeOrPromise); + constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, languageId:string) { + super(allowedEventTypes, rawText, languageId); this._rangeIdGenerator = new IdGenerator((++_INSTANCE_COUNT) + ';'); this._ranges = {}; this._markerIdToRangeId = {}; this._multiLineTrackedRanges = {}; } - protected _createRetokenizer(retokenizePromise:TPromise, lineNumber:number): IRetokenizeRequest { - return new TrackedRangeModelRetokenizer(retokenizePromise, lineNumber, this); - } - public dispose(): void { this._ranges = null; this._markerIdToRangeId = null; diff --git a/src/vs/editor/common/modes.ts b/src/vs/editor/common/modes.ts index 95c17b3f2f1..51d80b0471f 100644 --- a/src/vs/editor/common/modes.ts +++ b/src/vs/editor/common/modes.ts @@ -16,6 +16,7 @@ import LanguageFeatureRegistry from 'vs/editor/common/modes/languageFeatureRegis import {CancellationToken} from 'vs/base/common/cancellation'; import {Position} from 'vs/editor/common/core/position'; import {Range} from 'vs/editor/common/core/range'; +import Event, {Emitter} from 'vs/base/common/event'; /** * @internal @@ -32,7 +33,7 @@ export interface ITokenizationResult { export interface IState { clone():IState; equals(other:IState):boolean; - getMode():IMode; + getModeId():string; tokenize(stream:IStream):ITokenizationResult; getStateData(): IState; setStateData(state:IState):void; @@ -186,28 +187,6 @@ export interface IMode { getId(): string; - /** - * Return a mode "similar" to this one that strips any "smart" supports. - * @internal - */ - toSimplifiedMode(): IMode; - - /** - * @internal - */ - addSupportChangedListener?(callback: (e: editorCommon.IModeSupportChangedEvent) => void): IDisposable; - - /** - * Register a support by name. Only optional. - * @internal - */ - setTokenizationSupport?(callback:(mode:IMode)=>T): IDisposable; - - /** - * Optional adapter to support tokenization. - * @internal - */ - tokenizationSupport?: ITokenizationSupport; } /** @@ -218,7 +197,6 @@ export interface ILineTokens { actualStopOffset: number; endState: IState; modeTransitions: ModeTransition[]; - retokenize?:TPromise; } /** @@ -1022,3 +1000,56 @@ export const OnTypeFormattingEditProviderRegistry = new LanguageFeatureRegistry< * @internal */ export const LinkProviderRegistry = new LanguageFeatureRegistry(); + +/** + * @internal + */ +export interface ITokenizationSupportChangedEvent { + languageId: string; +} + +/** + * @internal + */ +export class TokenizationRegistryImpl { + + private _map: {[languageId:string]:ITokenizationSupport}; + + private _onDidChange: Emitter = new Emitter(); + public onDidChange: Event = this._onDidChange.event; + + constructor() { + this._map = Object.create(null); + } + + /** + * Fire a change event for a language. + * This is useful for languages that embed other languages. + */ + public fire(languageId:string): void { + this._onDidChange.fire({ languageId: languageId }); + } + + public register(languageId:string, support:ITokenizationSupport): IDisposable { + this._map[languageId] = support; + this.fire(languageId); + return { + dispose: () => { + if (this._map[languageId] !== support) { + return; + } + delete this._map[languageId]; + this.fire(languageId); + } + }; + } + + public get(languageId:string): ITokenizationSupport { + return (this._map[languageId] || null); + } +} + +/** + * @internal + */ +export const TokenizationRegistry = new TokenizationRegistryImpl(); diff --git a/src/vs/editor/common/modes/TMState.ts b/src/vs/editor/common/modes/TMState.ts index 96b0ff080f6..d2f225cc12e 100644 --- a/src/vs/editor/common/modes/TMState.ts +++ b/src/vs/editor/common/modes/TMState.ts @@ -4,25 +4,25 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {IMode, IState, ITokenizationResult} from 'vs/editor/common/modes'; +import {IState, ITokenizationResult} from 'vs/editor/common/modes'; import {AbstractState} from 'vs/editor/common/modes/abstractState'; import {StackElement} from 'vscode-textmate'; export class TMState implements IState { - private _mode: IMode; + private _modeId: string; private _parentEmbedderState: IState; private _ruleStack: StackElement; - constructor(mode: IMode, parentEmbedderState: IState, ruleStack: StackElement) { - this._mode = mode; + constructor(modeId: string, parentEmbedderState: IState, ruleStack: StackElement) { + this._modeId = modeId; this._parentEmbedderState = parentEmbedderState; this._ruleStack = ruleStack; } public clone():TMState { let parentEmbedderStateClone = AbstractState.safeClone(this._parentEmbedderState); - return new TMState(this._mode, parentEmbedderStateClone, this._ruleStack); + return new TMState(this._modeId, parentEmbedderStateClone, this._ruleStack); } public equals(other:IState):boolean { @@ -46,8 +46,8 @@ export class TMState implements IState { return this._ruleStack.equals(otherState._ruleStack); } - public getMode():IMode { - return this._mode; + public getModeId():string { + return this._modeId; } public tokenize(stream:any):ITokenizationResult { diff --git a/src/vs/editor/common/modes/abstractMode.ts b/src/vs/editor/common/modes/abstractMode.ts index c71674f767c..7ebd7fe1f9c 100644 --- a/src/vs/editor/common/modes/abstractMode.ts +++ b/src/vs/editor/common/modes/abstractMode.ts @@ -4,13 +4,10 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {EventEmitter} from 'vs/base/common/eventEmitter'; -import {IDisposable} from 'vs/base/common/lifecycle'; import {TPromise} from 'vs/base/common/winjs.base'; import {AsyncDescriptor1, createAsyncDescriptor1} from 'vs/platform/instantiation/common/descriptors'; import {IInstantiationService, optional} from 'vs/platform/instantiation/common/instantiation'; import {IConfigurationService} from 'vs/platform/configuration/common/configuration'; -import {IModeSupportChangedEvent} from 'vs/editor/common/editorCommon'; import * as modes from 'vs/editor/common/modes'; import {TextualSuggestSupport} from 'vs/editor/common/modes/supports/suggestSupport'; import {IEditorWorkerService} from 'vs/editor/common/services/editorWorkerService'; @@ -81,44 +78,14 @@ export class ModeWorkerManager { export abstract class AbstractMode implements modes.IMode { private _modeId: string; - private _eventEmitter: EventEmitter; - private _simplifiedMode: modes.IMode; constructor(modeId:string) { this._modeId = modeId; - this._eventEmitter = new EventEmitter(); - this._simplifiedMode = null; } public getId(): string { return this._modeId; } - - public toSimplifiedMode(): modes.IMode { - if (!this._simplifiedMode) { - this._simplifiedMode = new SimplifiedMode(this); - } - return this._simplifiedMode; - } - - public addSupportChangedListener(callback: (e: IModeSupportChangedEvent) => void) : IDisposable { - return this._eventEmitter.addListener2('modeSupportChanged', callback); - } - - public setTokenizationSupport(callback:(mode:modes.IMode) => T) : IDisposable { - let supportImpl = callback(this); - this['tokenizationSupport'] = supportImpl; - this._eventEmitter.emit('modeSupportChanged', _createModeSupportChangedEvent()); - - return { - dispose: () => { - if (this['tokenizationSupport'] === supportImpl) { - delete this['tokenizationSupport']; - this._eventEmitter.emit('modeSupportChanged', _createModeSupportChangedEvent()); - } - } - }; - } } export abstract class CompatMode extends AbstractMode implements ICompatMode { @@ -136,41 +103,6 @@ export abstract class CompatMode extends AbstractMode implements ICompatMode { } -class SimplifiedMode implements modes.IMode { - - tokenizationSupport: modes.ITokenizationSupport; - - private _sourceMode: modes.IMode; - private _eventEmitter: EventEmitter; - private _id: string; - - constructor(sourceMode: modes.IMode) { - this._sourceMode = sourceMode; - this._eventEmitter = new EventEmitter(); - this._id = 'vs.editor.modes.simplifiedMode:' + sourceMode.getId(); - this._assignSupports(); - - if (this._sourceMode.addSupportChangedListener) { - this._sourceMode.addSupportChangedListener((e) => { - this._assignSupports(); - this._eventEmitter.emit('modeSupportChanged', e); - }); - } - } - - public getId(): string { - return this._id; - } - - public toSimplifiedMode(): modes.IMode { - return this; - } - - private _assignSupports(): void { - this.tokenizationSupport = this._sourceMode.tokenizationSupport; - } -} - export function isDigit(character:string, base:number): boolean { let c = character.charCodeAt(0); switch (base) { @@ -223,9 +155,3 @@ export class FrankensteinMode extends AbstractMode { } } } - -function _createModeSupportChangedEvent(): IModeSupportChangedEvent { - return { - tokenizationSupport: true - }; -} diff --git a/src/vs/editor/common/modes/abstractState.ts b/src/vs/editor/common/modes/abstractState.ts index 12bd47c5842..6a6546b2440 100644 --- a/src/vs/editor/common/modes/abstractState.ts +++ b/src/vs/editor/common/modes/abstractState.ts @@ -4,33 +4,31 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {IMode, IState, IStream, ITokenizationResult} from 'vs/editor/common/modes'; +import {IState, IStream, ITokenizationResult} from 'vs/editor/common/modes'; -export class AbstractState implements IState { +export abstract class AbstractState implements IState { - private mode:IMode; + private modeId:string; private stateData:IState; - constructor(mode:IMode, stateData:IState = null) { - this.mode = mode; + constructor(modeId:string, stateData:IState = null) { + this.modeId = modeId; this.stateData = stateData; } - public getMode():IMode { - return this.mode; + public getModeId():string { + return this.modeId; } - public clone():IState { + public clone():AbstractState { var result:AbstractState = this.makeClone(); result.initializeFrom(this); return result; } - public makeClone():AbstractState { - throw new Error('Abstract Method'); - } + protected abstract makeClone():AbstractState; - public initializeFrom(other:AbstractState): void { + protected initializeFrom(other:AbstractState): void { this.stateData = other.stateData !== null ? other.stateData.clone() : null; } @@ -43,7 +41,7 @@ export class AbstractState implements IState { } public equals(other:IState):boolean { - if (other === null || this.mode !== other.getMode()) { + if (other === null || this.modeId !== other.getModeId()) { return false; } if (other instanceof AbstractState) { @@ -52,9 +50,7 @@ export class AbstractState implements IState { return false; } - public tokenize(stream:IStream):ITokenizationResult { - throw new Error('Abstract Method'); - } + public abstract tokenize(stream:IStream):ITokenizationResult; public static safeEquals(a: IState, b: IState): boolean { if (a === null && b === null) { diff --git a/src/vs/editor/common/modes/modesRegistry.ts b/src/vs/editor/common/modes/modesRegistry.ts index 462119e6ba5..cdb6d698095 100644 --- a/src/vs/editor/common/modes/modesRegistry.ts +++ b/src/vs/editor/common/modes/modesRegistry.ts @@ -75,8 +75,10 @@ export class EditorModesRegistry { export var ModesRegistry = new EditorModesRegistry(); Registry.add(Extensions.ModesRegistry, ModesRegistry); +export const PLAINTEXT_MODE_ID = 'plaintext'; + ModesRegistry.registerLanguage({ - id: 'plaintext', + id: PLAINTEXT_MODE_ID, extensions: ['.txt', '.gitignore'], aliases: [nls.localize('plainText.alias', "Plain Text"), 'text'], mimetypes: ['text/plain'] diff --git a/src/vs/editor/common/modes/monarch/monarchLexer.ts b/src/vs/editor/common/modes/monarch/monarchLexer.ts index b9b0ab3bfa4..324c10d4eb2 100644 --- a/src/vs/editor/common/modes/monarch/monarchLexer.ts +++ b/src/vs/editor/common/modes/monarch/monarchLexer.ts @@ -13,7 +13,7 @@ import * as modes from 'vs/editor/common/modes'; import {AbstractState} from 'vs/editor/common/modes/abstractState'; import {LineStream} from 'vs/editor/common/modes/lineStream'; import * as monarchCommon from 'vs/editor/common/modes/monarch/monarchCommon'; -import {IEnteringNestedModeData, TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport'; +import {IModeLocator, TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport'; import {IModeService} from 'vs/editor/common/services/modeService'; /** @@ -39,8 +39,8 @@ export class MonarchLexer extends AbstractState { private groupMatched: string[]; private groupRule: monarchCommon.IRule; - constructor(mode: modes.IMode, modeService:IModeService, lexer: monarchCommon.ILexer, stack?: string[], embeddedMode?: string) { - super(mode); + constructor(modeId: string, modeService:IModeService, lexer: monarchCommon.ILexer, stack?: string[], embeddedMode?: string) { + super(modeId); this.id = MonarchLexer.ID++; // for debugging, assigns unique id to each instance this.modeService = modeService; @@ -61,7 +61,7 @@ export class MonarchLexer extends AbstractState { } public makeClone(): MonarchLexer { - return new MonarchLexer(this.getMode(), this.modeService, this.lexer, this.stack.slice(0), this.embeddedMode); + return new MonarchLexer(this.getModeId(), this.modeService, this.lexer, this.stack.slice(0), this.embeddedMode); } public equals(other: modes.IState): boolean { @@ -386,10 +386,10 @@ function findBracket(lexer: monarchCommon.ILexer, matched: string) { return null; } -export function createTokenizationSupport(modeService:IModeService, mode:modes.IMode, lexer: monarchCommon.ILexer): modes.ITokenizationSupport { - return new TokenizationSupport(mode, { +export function createTokenizationSupport(_modeService:IModeService, modeId:string, lexer: monarchCommon.ILexer): modes.ITokenizationSupport { + return new TokenizationSupport(_modeService, modeId, { getInitialState: (): modes.IState => { - return new MonarchLexer(mode, modeService, lexer); + return new MonarchLexer(modeId, _modeService, lexer); }, enterNestedMode: (state: modes.IState): boolean => { @@ -399,31 +399,9 @@ export function createTokenizationSupport(modeService:IModeService, mode:modes.I return false; }, - getNestedMode: (rawState: modes.IState): IEnteringNestedModeData => { - var mime = (rawState).embeddedMode; - - if (!modeService.isRegisteredMode(mime)) { - // unknown mode - return { - mode: modeService.getMode('text/plain'), - missingModePromise: null - }; - } - - var mode = modeService.getMode(mime); - if (mode) { - // mode is available - return { - mode: mode, - missingModePromise: null - }; - } - - // mode is not yet loaded - return { - mode: modeService.getMode('text/plain'), - missingModePromise: modeService.getOrCreateMode(mime).then(() => null) - }; + getNestedMode: (rawState: modes.IState, locator:IModeLocator): modes.IMode => { + let mime = (rawState).embeddedMode; + return locator.getMode(mime); }, getLeavingNestedModeData: (line: string, state: modes.IState) => { diff --git a/src/vs/editor/common/modes/nullMode.ts b/src/vs/editor/common/modes/nullMode.ts index 8408dd23eb2..30a18927bb6 100644 --- a/src/vs/editor/common/modes/nullMode.ts +++ b/src/vs/editor/common/modes/nullMode.ts @@ -4,27 +4,27 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {IMode, IState, IStream, ITokenizationResult, ILineTokens} from 'vs/editor/common/modes'; +import {IState, IStream, ITokenizationResult, ILineTokens} from 'vs/editor/common/modes'; import {ModeTransition} from 'vs/editor/common/core/modeTransition'; import {Token} from 'vs/editor/common/core/token'; export class NullState implements IState { - private mode: IMode; + private modeId: string; private stateData: IState; - constructor(mode: IMode, stateData: IState) { - this.mode = mode; + constructor(modeId: string, stateData: IState) { + this.modeId = modeId; this.stateData = stateData; } public clone(): IState { let stateDataClone:IState = (this.stateData ? this.stateData.clone() : null); - return new NullState(this.mode, stateDataClone); + return new NullState(this.modeId, stateDataClone); } public equals(other:IState): boolean { - if (this.mode !== other.getMode()) { + if (this.modeId !== other.getModeId()) { return false; } let otherStateData = other.getStateData(); @@ -37,8 +37,8 @@ export class NullState implements IState { return false; } - public getMode(): IMode { - return this.mode; + public getModeId(): string { + return this.modeId; } public tokenize(stream:IStream):ITokenizationResult { @@ -55,22 +55,7 @@ export class NullState implements IState { } } -export class NullMode implements IMode { - - - public static ID = 'vs.editor.nullMode'; - - constructor() { - } - - public getId():string { - return NullMode.ID; - } - - public toSimplifiedMode(): IMode { - return this; - } -} +export const NULL_MODE_ID = 'vs.editor.nullMode'; export function nullTokenize(modeId: string, buffer:string, state: IState, deltaOffset:number = 0, stopAtOffset?:number): ILineTokens { let tokens:Token[] = [new Token(deltaOffset, '')]; diff --git a/src/vs/editor/common/modes/supports/tokenizationSupport.ts b/src/vs/editor/common/modes/supports/tokenizationSupport.ts index 46ab89fffab..0fc381a9466 100644 --- a/src/vs/editor/common/modes/supports/tokenizationSupport.ts +++ b/src/vs/editor/common/modes/supports/tokenizationSupport.ts @@ -5,12 +5,12 @@ 'use strict'; import {IDisposable} from 'vs/base/common/lifecycle'; -import {TPromise} from 'vs/base/common/winjs.base'; import * as modes from 'vs/editor/common/modes'; import {LineStream} from 'vs/editor/common/modes/lineStream'; -import {NullMode, NullState, nullTokenize} from 'vs/editor/common/modes/nullMode'; +import {NullState, nullTokenize, NULL_MODE_ID} from 'vs/editor/common/modes/nullMode'; import {Token} from 'vs/editor/common/core/token'; import {ModeTransition} from 'vs/editor/common/core/modeTransition'; +import {IModeService} from 'vs/editor/common/services/modeService'; export interface ILeavingNestedModeData { /** @@ -29,9 +29,8 @@ export interface ILeavingNestedModeData { stateAfterNestedMode: modes.IState; } -export interface IEnteringNestedModeData { - mode:modes.IMode; - missingModePromise:TPromise; +export interface IModeLocator { + getMode(mimetypeOrModeId: string): modes.IMode; } export interface ITokenizationCustomization { @@ -40,9 +39,9 @@ export interface ITokenizationCustomization { enterNestedMode?: (state:modes.IState) => boolean; - getNestedMode?: (state:modes.IState) => IEnteringNestedModeData; + getNestedMode?: (state:modes.IState, locator:IModeLocator) => modes.IMode; - getNestedModeInitialState?: (myState:modes.IState) => { state:modes.IState; missingModePromise:TPromise; }; + getNestedModeInitialState?: (myState:modes.IState) => modes.IState; /** * Return null if the line does not leave the nested mode @@ -74,23 +73,18 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa onReturningFromNestedMode: boolean; }; - public supportsNestedModes:boolean; + private supportsNestedModes:boolean; - private _mode:modes.IMode; + private _modeService:IModeService; private _modeId:string; - private _embeddedModesListeners: { [modeId:string]: IDisposable; }; + private _embeddedModes: {[modeId:string]:boolean;}; + private _tokenizationRegistryListener: IDisposable; - constructor(mode:modes.IMode, customization:ITokenizationCustomization, supportsNestedModes:boolean) { - this._mode = mode; - this._modeId = this._mode.getId(); + constructor(modeService:IModeService, modeId:string, customization:ITokenizationCustomization, supportsNestedModes:boolean) { + this._modeService = modeService; + this._modeId = modeId; this.customization = customization; this.supportsNestedModes = supportsNestedModes; - this._embeddedModesListeners = {}; - if (this.supportsNestedModes) { - if (!this._mode.setTokenizationSupport) { - throw new Error('Cannot be a mode with nested modes unless I can emit a tokenizationSupport changed event!'); - } - } this.defaults = { enterNestedMode: !isFunction(customization.enterNestedMode), getNestedMode: !isFunction(customization.getNestedMode), @@ -98,13 +92,26 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa getLeavingNestedModeData: !isFunction(customization.getLeavingNestedModeData), onReturningFromNestedMode: !isFunction(customization.onReturningFromNestedMode) }; + + this._embeddedModes = Object.create(null); + + // Set up listening for embedded modes + let emitting = false; + this._tokenizationRegistryListener = modes.TokenizationRegistry.onDidChange((e) => { + if (emitting) { + return; + } + let isOneOfMyEmbeddedModes = this._embeddedModes[e.languageId]; + if (isOneOfMyEmbeddedModes) { + emitting = true; + modes.TokenizationRegistry.fire(this._modeId); + emitting = false; + } + }); } - public dispose() : void { - for (let listener in this._embeddedModesListeners) { - this._embeddedModesListeners[listener].dispose(); - delete this._embeddedModesListeners[listener]; - } + public dispose(): void { + this._tokenizationRegistryListener.dispose(); } public getInitialState(): modes.IState { @@ -112,7 +119,7 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa } public tokenize(line:string, state:modes.IState, deltaOffset:number = 0, stopAtOffset:number = deltaOffset + line.length):modes.ILineTokens { - if (state.getMode() !== this._mode) { + if (state.getModeId() !== this._modeId) { return this._nestedTokenize(line, state, deltaOffset, stopAtOffset, [], []); } else { return this._myTokenize(line, state, deltaOffset, stopAtOffset, [], []); @@ -120,31 +127,32 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa } /** - * Precondition is: nestedModeState.getMode() !== this + * Precondition is: nestedModeState.getModeId() !== this._modeId * This means we are in a nested mode when parsing starts on this line. */ private _nestedTokenize(buffer:string, nestedModeState:modes.IState, deltaOffset:number, stopAtOffset:number, prependTokens:Token[], prependModeTransitions:ModeTransition[]):modes.ILineTokens { let myStateBeforeNestedMode = nestedModeState.getStateData(); - let leavingNestedModeData = this.getLeavingNestedModeData(buffer, myStateBeforeNestedMode); + let leavingNestedModeData = this._getLeavingNestedModeData(buffer, myStateBeforeNestedMode); // Be sure to give every embedded mode the // opportunity to leave nested mode. // i.e. Don't go straight to the most nested mode let stepOnceNestedState = nestedModeState; - while (stepOnceNestedState.getStateData() && stepOnceNestedState.getStateData().getMode() !== this._mode) { + while (stepOnceNestedState.getStateData() && stepOnceNestedState.getStateData().getModeId() !== this._modeId) { stepOnceNestedState = stepOnceNestedState.getStateData(); } - let nestedMode = stepOnceNestedState.getMode(); + let nestedModeId = stepOnceNestedState.getModeId(); if (!leavingNestedModeData) { // tokenization will not leave nested mode let result:modes.ILineTokens; - if (nestedMode.tokenizationSupport) { - result = nestedMode.tokenizationSupport.tokenize(buffer, nestedModeState, deltaOffset, stopAtOffset); + let tokenizationSupport = modes.TokenizationRegistry.get(nestedModeId); + if (tokenizationSupport) { + result = tokenizationSupport.tokenize(buffer, nestedModeState, deltaOffset, stopAtOffset); } else { // The nested mode doesn't have tokenization support, // unfortunatelly this means we have to fake it - result = nullTokenize(nestedMode.getId(), buffer, nestedModeState, deltaOffset); + result = nullTokenize(nestedModeId, buffer, nestedModeState, deltaOffset); } result.tokens = prependTokens.concat(result.tokens); result.modeTransitions = prependModeTransitions.concat(result.modeTransitions); @@ -155,12 +163,13 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa if (nestedModeBuffer.length > 0) { // Tokenize with the nested mode let nestedModeLineTokens:modes.ILineTokens; - if (nestedMode.tokenizationSupport) { - nestedModeLineTokens = nestedMode.tokenizationSupport.tokenize(nestedModeBuffer, nestedModeState, deltaOffset, stopAtOffset); + let tokenizationSupport = modes.TokenizationRegistry.get(nestedModeId); + if (tokenizationSupport) { + nestedModeLineTokens = tokenizationSupport.tokenize(nestedModeBuffer, nestedModeState, deltaOffset, stopAtOffset); } else { // The nested mode doesn't have tokenization support, // unfortunatelly this means we have to fake it - nestedModeLineTokens = nullTokenize(nestedMode.getId(), nestedModeBuffer, nestedModeState, deltaOffset); + nestedModeLineTokens = nullTokenize(nestedModeId, nestedModeBuffer, nestedModeState, deltaOffset); } // Save last state of nested mode @@ -174,7 +183,7 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa let bufferAfterNestedMode = leavingNestedModeData.bufferAfterNestedMode; let myStateAfterNestedMode = leavingNestedModeData.stateAfterNestedMode; myStateAfterNestedMode.setStateData(myStateBeforeNestedMode.getStateData()); - this.onReturningFromNestedMode(myStateAfterNestedMode, nestedModeState); + this._onReturningFromNestedMode(myStateAfterNestedMode, nestedModeState); return this._myTokenize(bufferAfterNestedMode, myStateAfterNestedMode, deltaOffset + nestedModeBuffer.length, stopAtOffset, prependTokens, prependModeTransitions); } @@ -187,7 +196,6 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa let lineStream = new LineStream(buffer); let tokenResult:modes.ITokenizationResult, beforeTokenizeStreamPos:number; let previousType:string = null; - let retokenize:TPromise = null; myState = myState.clone(); if (prependModeTransitions.length <= 0 || prependModeTransitions[prependModeTransitions.length-1].modeId !== this._modeId) { @@ -222,40 +230,19 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa previousType = tokenResult.type; - if (this.supportsNestedModes && this.enterNestedMode(myState)) { + if (this.supportsNestedModes && this._enterNestedMode(myState)) { let currentEmbeddedLevels = this._getEmbeddedLevel(myState); if (currentEmbeddedLevels < TokenizationSupport.MAX_EMBEDDED_LEVELS) { - let nestedModeState = this.getNestedModeInitialState(myState); - - // Re-emit tokenizationSupport change events from all modes that I ever embedded - let embeddedMode = nestedModeState.state.getMode(); - if (typeof embeddedMode.addSupportChangedListener === 'function' && !this._embeddedModesListeners.hasOwnProperty(embeddedMode.getId())) { - let emitting = false; - this._embeddedModesListeners[embeddedMode.getId()] = embeddedMode.addSupportChangedListener((e) => { - if (emitting) { - return; - } - if (e.tokenizationSupport) { - emitting = true; - this._mode.setTokenizationSupport((mode) => { - return mode.tokenizationSupport; - }); - emitting = false; - } - }); - } - + let nestedModeState = this._getNestedModeInitialState(myState); if (!lineStream.eos()) { // There is content from the embedded mode let restOfBuffer = buffer.substr(lineStream.pos()); - let result = this._nestedTokenize(restOfBuffer, nestedModeState.state, deltaOffset + lineStream.pos(), stopAtOffset, prependTokens, prependModeTransitions); - result.retokenize = result.retokenize || nestedModeState.missingModePromise; + let result = this._nestedTokenize(restOfBuffer, nestedModeState, deltaOffset + lineStream.pos(), stopAtOffset, prependTokens, prependModeTransitions); return result; } else { // Transition to the nested mode state - myState = nestedModeState.state; - retokenize = nestedModeState.missingModePromise; + myState = nestedModeState; } } } @@ -265,8 +252,7 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa tokens: prependTokens, actualStopOffset: lineStream.pos() + deltaOffset, modeTransitions: prependModeTransitions, - endState: myState, - retokenize: retokenize + endState: myState }; } @@ -279,7 +265,7 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa return result; } - private enterNestedMode(state:modes.IState): boolean { + private _enterNestedMode(state:modes.IState): boolean { if (this.defaults.enterNestedMode) { return false; } @@ -287,60 +273,63 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa } - private getNestedMode(state:modes.IState): IEnteringNestedModeData { + private _getNestedMode(state:modes.IState): modes.IMode { if (this.defaults.getNestedMode) { return null; } - return this.customization.getNestedMode(state); - } - private static _validatedNestedMode(input:IEnteringNestedModeData): IEnteringNestedModeData { - let mode: modes.IMode = new NullMode(), - missingModePromise: TPromise = null; + let locator:IModeLocator = { + getMode: (mimetypeOrModeId: string): modes.IMode => { + if (!mimetypeOrModeId || !this._modeService.isRegisteredMode(mimetypeOrModeId)) { + return null; + } - if (input && input.mode) { - mode = input.mode; - } - if (input && input.missingModePromise) { - missingModePromise = input.missingModePromise; - } + let modeId = this._modeService.getModeId(mimetypeOrModeId); - return { - mode: mode, - missingModePromise: missingModePromise + let mode = this._modeService.getMode(modeId); + if (mode) { + // Re-emit tokenizationSupport change events from all modes that I ever embedded + this._embeddedModes[modeId] = true; + return mode; + } + + // Fire mode loading event + this._modeService.getOrCreateMode(modeId); + + this._embeddedModes[modeId] = true; + + return null; + } }; + + return this.customization.getNestedMode(state, locator); } - private getNestedModeInitialState(state:modes.IState): { state:modes.IState; missingModePromise:TPromise; } { + private _getNestedModeInitialState(state:modes.IState): modes.IState { if (this.defaults.getNestedModeInitialState) { - let nestedMode = TokenizationSupport._validatedNestedMode(this.getNestedMode(state)); - let missingModePromise = nestedMode.missingModePromise; - let nestedModeState: modes.IState; - - if (nestedMode.mode.tokenizationSupport) { - nestedModeState = nestedMode.mode.tokenizationSupport.getInitialState(); - } else { - nestedModeState = new NullState(nestedMode.mode, null); + let nestedMode = this._getNestedMode(state); + if (nestedMode) { + let tokenizationSupport = modes.TokenizationRegistry.get(nestedMode.getId()); + if (tokenizationSupport) { + let nestedModeState = tokenizationSupport.getInitialState(); + nestedModeState.setStateData(state); + return nestedModeState; + } } - nestedModeState.setStateData(state); - - return { - state: nestedModeState, - missingModePromise: missingModePromise - }; + return new NullState(nestedMode ? nestedMode.getId() : NULL_MODE_ID, state); } return this.customization.getNestedModeInitialState(state); } - private getLeavingNestedModeData(line:string, state:modes.IState): ILeavingNestedModeData { + private _getLeavingNestedModeData(line:string, state:modes.IState): ILeavingNestedModeData { if (this.defaults.getLeavingNestedModeData) { return null; } return this.customization.getLeavingNestedModeData(line, state); } - private onReturningFromNestedMode(myStateAfterNestedMode:modes.IState, lastNestedModeState:modes.IState): void { + private _onReturningFromNestedMode(myStateAfterNestedMode:modes.IState, lastNestedModeState:modes.IState): void { if (this.defaults.onReturningFromNestedMode) { return null; } diff --git a/src/vs/editor/common/modes/textToHtmlTokenizer.ts b/src/vs/editor/common/modes/textToHtmlTokenizer.ts index 1b53fa7e318..829eb05ab41 100644 --- a/src/vs/editor/common/modes/textToHtmlTokenizer.ts +++ b/src/vs/editor/common/modes/textToHtmlTokenizer.ts @@ -6,20 +6,21 @@ import {IHTMLContentElement} from 'vs/base/common/htmlContent'; import * as strings from 'vs/base/common/strings'; -import {IMode, IState, ITokenizationSupport} from 'vs/editor/common/modes'; +import {IState, ITokenizationSupport, TokenizationRegistry} from 'vs/editor/common/modes'; import {NullState, nullTokenize} from 'vs/editor/common/modes/nullMode'; -export function tokenizeToHtmlContent(text: string, mode: IMode): IHTMLContentElement { - return _tokenizeToHtmlContent(text, _getSafeTokenizationSupport(mode)); +export function tokenizeToHtmlContent(text: string, languageId: string): IHTMLContentElement { + return _tokenizeToHtmlContent(text, _getSafeTokenizationSupport(languageId)); } -export function tokenizeToString(text: string, mode: IMode, extraTokenClass?: string): string { - return _tokenizeToString(text, _getSafeTokenizationSupport(mode), extraTokenClass); +export function tokenizeToString(text: string, languageId: string, extraTokenClass?: string): string { + return _tokenizeToString(text, _getSafeTokenizationSupport(languageId), extraTokenClass); } -function _getSafeTokenizationSupport(mode: IMode): ITokenizationSupport { - if (mode && mode.tokenizationSupport) { - return mode.tokenizationSupport; +function _getSafeTokenizationSupport(languageId: string): ITokenizationSupport { + let tokenizationSupport = TokenizationRegistry.get(languageId); + if (tokenizationSupport) { + return tokenizationSupport; } return { getInitialState: () => new NullState(null, null), diff --git a/src/vs/editor/common/services/compatWorkerServiceWorker.ts b/src/vs/editor/common/services/compatWorkerServiceWorker.ts index c683dd99bb4..f90ef7dc430 100644 --- a/src/vs/editor/common/services/compatWorkerServiceWorker.ts +++ b/src/vs/editor/common/services/compatWorkerServiceWorker.ts @@ -74,7 +74,7 @@ export class CompatWorkerServiceWorker implements ICompatWorkerService { this.resourceService.remove(mirrorModel.uri); // (2) Change mode - mirrorModel.setMode(mode); + mirrorModel.setMode(mode.getId()); // (3) Insert again to resource service (it will have the new mode) this.resourceService.insert(mirrorModel.uri, mirrorModel); diff --git a/src/vs/editor/common/services/modeService.ts b/src/vs/editor/common/services/modeService.ts index a8bee90a69f..c9b0cbdb7a3 100644 --- a/src/vs/editor/common/services/modeService.ts +++ b/src/vs/editor/common/services/modeService.ts @@ -5,7 +5,6 @@ 'use strict'; import Event from 'vs/base/common/event'; -import {IDisposable} from 'vs/base/common/lifecycle'; import {TPromise} from 'vs/base/common/winjs.base'; import {createDecorator} from 'vs/platform/instantiation/common/instantiation'; import * as modes from 'vs/editor/common/modes'; @@ -64,7 +63,4 @@ export interface IModeService { getOrCreateMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): TPromise; getOrCreateModeByLanguageName(languageName: string): TPromise; getOrCreateModeByFilenameOrFirstLine(filename: string, firstLine?:string): TPromise; - - registerTokenizationSupport(modeId: string, callback: (mode: modes.IMode) => modes.ITokenizationSupport): IDisposable; - registerTokenizationSupport2(modeId: string, support: modes.TokensProvider): IDisposable; } diff --git a/src/vs/editor/common/services/modeServiceImpl.ts b/src/vs/editor/common/services/modeServiceImpl.ts index 30fc407781f..771b428b676 100644 --- a/src/vs/editor/common/services/modeServiceImpl.ts +++ b/src/vs/editor/common/services/modeServiceImpl.ts @@ -7,7 +7,6 @@ import * as nls from 'vs/nls'; import {onUnexpectedError} from 'vs/base/common/errors'; import Event, {Emitter} from 'vs/base/common/event'; -import {IDisposable, empty as EmptyDisposable} from 'vs/base/common/lifecycle'; // TODO@Alex import * as paths from 'vs/base/common/paths'; import {TPromise} from 'vs/base/common/winjs.base'; import mime = require('vs/base/common/mime'); @@ -149,7 +148,10 @@ export class ModeServiceImpl implements IModeService { private _onDidCreateMode: Emitter = new Emitter(); public onDidCreateMode: Event = this._onDidCreateMode.event; - constructor(instantiationService:IInstantiationService, extensionService:IExtensionService) { + constructor( + instantiationService:IInstantiationService, + extensionService:IExtensionService + ) { this._instantiationService = instantiationService; this._extensionService = extensionService; @@ -349,58 +351,16 @@ export class ModeServiceImpl implements IModeService { }; } - private _registerTokenizationSupport(mode:modes.IMode, callback: (mode: modes.IMode) => T): IDisposable { - if (mode.setTokenizationSupport) { - return mode.setTokenizationSupport(callback); - } else { - console.warn('Cannot register tokenizationSupport on mode ' + mode.getId() + ' because it does not support it.'); - return EmptyDisposable; - } - } - - private registerModeSupport(modeId: string, callback: (mode: modes.IMode) => T): IDisposable { - if (this._instantiatedModes.hasOwnProperty(modeId)) { - return this._registerTokenizationSupport(this._instantiatedModes[modeId], callback); - } - - let cc: (disposable:IDisposable)=>void; - let promise = new TPromise((c, e) => { cc = c; }); - - let disposable = this.onDidCreateMode((mode) => { - if (mode.getId() !== modeId) { - return; - } - - cc(this._registerTokenizationSupport(mode, callback)); - disposable.dispose(); - }); - - return { - dispose: () => { - promise.done(disposable => disposable.dispose(), null); - } - }; - } - - public registerTokenizationSupport(modeId: string, callback: (mode: modes.IMode) => modes.ITokenizationSupport): IDisposable { - return this.registerModeSupport(modeId, callback); - } - - public registerTokenizationSupport2(modeId: string, support: modes.TokensProvider): IDisposable { - return this.registerModeSupport(modeId, (mode) => { - return new TokenizationSupport2Adapter(mode, support); - }); - } } export class TokenizationState2Adapter implements modes.IState { - private _mode: modes.IMode; + private _modeId: string; private _actual: modes.IState2; private _stateData: modes.IState; - constructor(mode: modes.IMode, actual: modes.IState2, stateData: modes.IState) { - this._mode = mode; + constructor(modeId: string, actual: modes.IState2, stateData: modes.IState) { + this._modeId = modeId; this._actual = actual; this._stateData = stateData; } @@ -408,7 +368,7 @@ export class TokenizationState2Adapter implements modes.IState { public get actual(): modes.IState2 { return this._actual; } public clone(): TokenizationState2Adapter { - return new TokenizationState2Adapter(this._mode, this._actual.clone(), AbstractState.safeClone(this._stateData)); + return new TokenizationState2Adapter(this._modeId, this._actual.clone(), AbstractState.safeClone(this._stateData)); } public equals(other:modes.IState): boolean { @@ -421,8 +381,8 @@ export class TokenizationState2Adapter implements modes.IState { return false; } - public getMode(): modes.IMode { - return this._mode; + public getModeId(): string { + return this._modeId; } public tokenize(stream:any): any { @@ -440,16 +400,16 @@ export class TokenizationState2Adapter implements modes.IState { export class TokenizationSupport2Adapter implements modes.ITokenizationSupport { - private _mode: modes.IMode; + private _modeId: string; private _actual: modes.TokensProvider; - constructor(mode: modes.IMode, actual: modes.TokensProvider) { - this._mode = mode; + constructor(modeId: string, actual: modes.TokensProvider) { + this._modeId = modeId; this._actual = actual; } public getInitialState(): modes.IState { - return new TokenizationState2Adapter(this._mode, this._actual.getInitialState(), null); + return new TokenizationState2Adapter(this._modeId, this._actual.getInitialState(), null); } public tokenize(line:string, state:modes.IState, offsetDelta: number = 0, stopAtOffset?: number): modes.ILineTokens { @@ -468,8 +428,8 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport { return { tokens: tokens, actualStopOffset: offsetDelta + line.length, - endState: new TokenizationState2Adapter(state.getMode(), actualResult.endState, state.getStateData()), - modeTransitions: [new ModeTransition(offsetDelta, state.getMode().getId())], + endState: new TokenizationState2Adapter(state.getModeId(), actualResult.endState, state.getStateData()), + modeTransitions: [new ModeTransition(offsetDelta, state.getModeId())], }; } throw new Error('Unexpected state to tokenize with!'); diff --git a/src/vs/editor/common/services/modelService.ts b/src/vs/editor/common/services/modelService.ts index fb9fe23b25a..ed0a1991075 100644 --- a/src/vs/editor/common/services/modelService.ts +++ b/src/vs/editor/common/services/modelService.ts @@ -18,6 +18,8 @@ export interface IModelService { createModel(value:string | IRawText, modeOrPromise:TPromise|IMode, resource: URI): IModel; + setMode(model:IModel, modeOrPromise:TPromise|IMode): void; + destroyModel(resource: URI): void; getModels(): IModel[]; diff --git a/src/vs/editor/common/services/modelServiceImpl.ts b/src/vs/editor/common/services/modelServiceImpl.ts index ae1690964f9..33ce3b5c61e 100644 --- a/src/vs/editor/common/services/modelServiceImpl.ts +++ b/src/vs/editor/common/services/modelServiceImpl.ts @@ -24,6 +24,7 @@ import * as platform from 'vs/base/common/platform'; import {IConfigurationService} from 'vs/platform/configuration/common/configuration'; import {DEFAULT_INDENTATION, DEFAULT_TRIM_AUTO_WHITESPACE} from 'vs/editor/common/config/defaultConfig'; import {IMessageService} from 'vs/platform/message/common/message'; +import {PLAINTEXT_MODE_ID} from 'vs/editor/common/modes/modesRegistry'; export interface IRawModelData { url: URI; @@ -352,13 +353,13 @@ export class ModelServiceImpl implements IModelService { // --- begin IModelService - private _createModelData(value: string | editorCommon.IRawText, modeOrPromise: TPromise | IMode, resource: URI): ModelData { + private _createModelData(value: string | editorCommon.IRawText, languageId:string, resource: URI): ModelData { // create & save the model let model:Model; if (typeof value === 'string') { - model = Model.createFromString(value, this._modelCreationOptions, modeOrPromise, resource); + model = Model.createFromString(value, this._modelCreationOptions, languageId, resource); } else { - model = new Model(value, modeOrPromise, resource); + model = new Model(value, languageId, resource); } let modelId = MODEL_ID(model.uri); @@ -374,7 +375,14 @@ export class ModelServiceImpl implements IModelService { } public createModel(value: string | editorCommon.IRawText, modeOrPromise: TPromise | IMode, resource: URI): editorCommon.IModel { - let modelData = this._createModelData(value, modeOrPromise, resource); + let modelData: ModelData; + + if (!modeOrPromise || TPromise.is(modeOrPromise)) { + modelData = this._createModelData(value, PLAINTEXT_MODE_ID, resource); + this.setMode(modelData.model, modeOrPromise); + } else { + modelData = this._createModelData(value, modeOrPromise.getId(), resource); + } // handle markers (marker service => model) if (this._markerService) { @@ -386,6 +394,21 @@ export class ModelServiceImpl implements IModelService { return modelData.model; } + public setMode(model:editorCommon.IModel, modeOrPromise:TPromise|IMode): void { + if (!modeOrPromise) { + return; + } + if (TPromise.is(modeOrPromise)) { + modeOrPromise.then((mode) => { + if (!model.isDisposed()) { + model.setMode(mode.getId()); + } + }); + } else { + model.setMode(modeOrPromise.getId()); + } + } + public destroyModel(resource: URI): void { // We need to support that not all models get disposed through this service (i.e. model.dispose() should work!) let modelData = this._models[MODEL_ID(resource)]; diff --git a/src/vs/editor/common/viewModel/viewModelImpl.ts b/src/vs/editor/common/viewModel/viewModelImpl.ts index e171e2fb20f..28faf1b60b0 100644 --- a/src/vs/editor/common/viewModel/viewModelImpl.ts +++ b/src/vs/editor/common/viewModel/viewModelImpl.ts @@ -214,10 +214,6 @@ export class ViewModel extends EventEmitter implements IViewModel { // That's ok, a model tokens changed event will follow shortly break; - case editorCommon.EventType.ModelModeSupportChanged: - // That's ok, no work to do - break; - case editorCommon.EventType.ModelContentChanged2: // Ignore break; diff --git a/src/vs/editor/contrib/comment/test/common/blockCommentCommand.test.ts b/src/vs/editor/contrib/comment/test/common/blockCommentCommand.test.ts index f9c436feadc..01c7a8b77db 100644 --- a/src/vs/editor/contrib/comment/test/common/blockCommentCommand.test.ts +++ b/src/vs/editor/contrib/comment/test/common/blockCommentCommand.test.ts @@ -11,7 +11,7 @@ import {CommentMode} from 'vs/editor/test/common/testModes'; function testBlockCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { var mode = new CommentMode({ lineComment: '!@#', blockComment: ['<0', '0>'] }); - testCommand(lines, mode, selection, (sel) => new BlockCommentCommand(sel), expectedLines, expectedSelection); + testCommand(lines, mode.getId(), selection, (sel) => new BlockCommentCommand(sel), expectedLines, expectedSelection); } suite('Editor Contrib - Block Comment Command', () => { diff --git a/src/vs/editor/contrib/comment/test/common/lineCommentCommand.test.ts b/src/vs/editor/contrib/comment/test/common/lineCommentCommand.test.ts index 3374bd80401..c910c5b1421 100644 --- a/src/vs/editor/contrib/comment/test/common/lineCommentCommand.test.ts +++ b/src/vs/editor/contrib/comment/test/common/lineCommentCommand.test.ts @@ -14,12 +14,12 @@ suite('Editor Contrib - Line Comment Command', () => { function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { var mode = new CommentMode({ lineComment: '!@#', blockComment: [''] }); - testCommand(lines, mode, selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection); + testCommand(lines, mode.getId(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection); } function testAddLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { var mode = new CommentMode({ lineComment: '!@#', blockComment: [''] }); - testCommand(lines, mode, selection, (sel) => new LineCommentCommand(sel, 4, Type.ForceAdd), expectedLines, expectedSelection); + testCommand(lines, mode.getId(), selection, (sel) => new LineCommentCommand(sel, 4, Type.ForceAdd), expectedLines, expectedSelection); } test('comment single line', function () { @@ -521,7 +521,7 @@ suite('Editor Contrib - Line Comment As Block Comment', () => { function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { var mode = new CommentMode({ lineComment: '', blockComment: ['(', ')'] }); - testCommand(lines, mode, selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection); + testCommand(lines, mode.getId(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection); } test('fall back to block comment command', function () { @@ -631,7 +631,7 @@ suite('Editor Contrib - Line Comment As Block Comment', () => { suite('Editor Contrib - Line Comment As Block Comment 2', () => { function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { var mode = new CommentMode({ lineComment: null, blockComment: [''] }); - testCommand(lines, mode, selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection); + testCommand(lines, mode.getId(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection); } test('no selection => uses indentation', function () { diff --git a/src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts b/src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts index b685bc6eb83..5d699a80935 100644 --- a/src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts +++ b/src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts @@ -17,7 +17,7 @@ import {LanguageConfigurationRegistry} from 'vs/editor/common/modes/languageConf class MockJSMode extends MockTokenizingMode { constructor() { - super('js-tokenSelectionSupport', 'mock-js'); + super('mock-js'); LanguageConfigurationRegistry.register(this.getId(), { brackets: [ diff --git a/src/vs/editor/node/textMate/TMSyntax.ts b/src/vs/editor/node/textMate/TMSyntax.ts index 3c58e4c4768..c7855961c53 100644 --- a/src/vs/editor/node/textMate/TMSyntax.ts +++ b/src/vs/editor/node/textMate/TMSyntax.ts @@ -8,7 +8,7 @@ import * as nls from 'vs/nls'; import {onUnexpectedError} from 'vs/base/common/errors'; import * as paths from 'vs/base/common/paths'; import {IExtensionMessageCollector, ExtensionsRegistry} from 'vs/platform/extensions/common/extensionsRegistry'; -import {ILineTokens, IMode, ITokenizationSupport} from 'vs/editor/common/modes'; +import {ILineTokens, ITokenizationSupport, TokenizationRegistry} from 'vs/editor/common/modes'; import {TMState} from 'vs/editor/common/modes/TMState'; import {LineTokens} from 'vs/editor/common/modes/supports'; import {IModeService} from 'vs/editor/common/services/modeService'; @@ -141,17 +141,15 @@ export class MainProcessTextMateSyntax { return; } - this._modeService.registerTokenizationSupport(modeId, (mode: IMode) => { - return createTokenizationSupport(mode, grammar); - }); + TokenizationRegistry.register(modeId, createTokenizationSupport(modeId, grammar)); }); } } -function createTokenizationSupport(mode: IMode, grammar: IGrammar): ITokenizationSupport { - var tokenizer = new Tokenizer(mode.getId(), grammar); +function createTokenizationSupport(modeId: string, grammar: IGrammar): ITokenizationSupport { + var tokenizer = new Tokenizer(modeId, grammar); return { - getInitialState: () => new TMState(mode, null, null), + getInitialState: () => new TMState(modeId, null, null), tokenize: (line, state, offsetDelta?, stopAtOffset?) => tokenizer.tokenize(line, state, offsetDelta, stopAtOffset) }; } @@ -252,7 +250,7 @@ class Tokenizer { if (line.length >= 20000 || depth(state.getRuleStack()) > 100) { return new LineTokens( [new Token(offsetDelta, '')], - [new ModeTransition(offsetDelta, state.getMode().getId())], + [new ModeTransition(offsetDelta, state.getModeId())], offsetDelta, state ); @@ -279,7 +277,7 @@ class Tokenizer { return new LineTokens( tokens, - [new ModeTransition(offsetDelta, freshState.getMode().getId())], + [new ModeTransition(offsetDelta, freshState.getModeId())], offsetDelta + line.length, freshState ); diff --git a/src/vs/editor/test/common/commands/commandTestUtils.ts b/src/vs/editor/test/common/commands/commandTestUtils.ts index 44a632e0f0a..be95d2ff0df 100644 --- a/src/vs/editor/test/common/commands/commandTestUtils.ts +++ b/src/vs/editor/test/common/commands/commandTestUtils.ts @@ -10,13 +10,12 @@ import {Range} from 'vs/editor/common/core/range'; import {Selection} from 'vs/editor/common/core/selection'; import * as editorCommon from 'vs/editor/common/editorCommon'; import {Model} from 'vs/editor/common/model/model'; -import {IMode} from 'vs/editor/common/modes'; import {MockConfiguration} from 'vs/editor/test/common/mocks/mockConfiguration'; import {viewModelHelper} from 'vs/editor/test/common/editorTestUtils'; export function testCommand( lines: string[], - mode: IMode, + mode: string, selection: Selection, commandFactory: (selection:Selection) => editorCommon.ICommand, expectedLines: string[], diff --git a/src/vs/editor/test/common/commands/shiftCommand.test.ts b/src/vs/editor/test/common/commands/shiftCommand.test.ts index df15da8e2ea..39188e1096b 100644 --- a/src/vs/editor/test/common/commands/shiftCommand.test.ts +++ b/src/vs/editor/test/common/commands/shiftCommand.test.ts @@ -74,7 +74,7 @@ class DocBlockCommentMode extends MockMode { } function testShiftCommandInDocBlockCommentMode(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { - testCommand(lines, new DocBlockCommentMode(), selection, (sel) => new ShiftCommand(sel, { + testCommand(lines, new DocBlockCommentMode().getId(), selection, (sel) => new ShiftCommand(sel, { isUnshift: false, tabSize: 4, oneIndent: '\t' @@ -82,7 +82,7 @@ function testShiftCommandInDocBlockCommentMode(lines: string[], selection: Selec } function testUnshiftCommandInDocBlockCommentMode(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { - testCommand(lines, new DocBlockCommentMode(), selection, (sel) => new ShiftCommand(sel, { + testCommand(lines, new DocBlockCommentMode().getId(), selection, (sel) => new ShiftCommand(sel, { isUnshift: true, tabSize: 4, oneIndent: '\t' diff --git a/src/vs/editor/test/common/controller/cursor.test.ts b/src/vs/editor/test/common/controller/cursor.test.ts index 6e0a26aff1b..1ff109d8c58 100644 --- a/src/vs/editor/test/common/controller/cursor.test.ts +++ b/src/vs/editor/test/common/controller/cursor.test.ts @@ -1145,7 +1145,7 @@ suite('Editor Controller - Regression tests', () => { text: [ 'var x = (3 + (5-7));' ], - mode: new BracketMode() + modeId: new BracketMode().getId() }, (model, cursor) => { // ensure is tokenized model.getLineContext(1); @@ -1180,7 +1180,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - mode: new OnEnterMode(IndentAction.Indent), + modeId: new OnEnterMode(IndentAction.Indent).getId(), }, (model, cursor) => { moveTo(cursor, 4, 1, false); assertCursor(cursor, new Selection(4, 1, 4, 1)); @@ -1207,7 +1207,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - mode: new OnEnterMode(IndentAction.Indent), + modeId: new OnEnterMode(IndentAction.Indent).getId(), }, (model, cursor) => { moveTo(cursor, 4, 2, false); assertCursor(cursor, new Selection(4, 2, 4, 2)); @@ -1235,7 +1235,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - mode: new OnEnterMode(IndentAction.Indent), + modeId: new OnEnterMode(IndentAction.Indent).getId(), }, (model, cursor) => { moveTo(cursor, 4, 1, false); assertCursor(cursor, new Selection(4, 1, 4, 1)); @@ -1262,7 +1262,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - mode: new OnEnterMode(IndentAction.Indent), + modeId: new OnEnterMode(IndentAction.Indent).getId(), }, (model, cursor) => { moveTo(cursor, 4, 3, false); assertCursor(cursor, new Selection(4, 3, 4, 3)); @@ -1289,7 +1289,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - mode: new OnEnterMode(IndentAction.Indent), + modeId: new OnEnterMode(IndentAction.Indent).getId(), }, (model, cursor) => { moveTo(cursor, 4, 4, false); assertCursor(cursor, new Selection(4, 4, 4, 4)); @@ -1329,7 +1329,7 @@ suite('Editor Controller - Regression tests', () => { text: [ ' function baz() {' ], - mode: new OnEnterMode(IndentAction.IndentOutdent), + modeId: new OnEnterMode(IndentAction.IndentOutdent).getId(), modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 1, 6, false); @@ -1432,7 +1432,7 @@ suite('Editor Controller - Regression tests', () => { text: [ 'hello' ], - mode: new SurroundingMode(), + modeId: new SurroundingMode().getId(), modelOpts: { tabSize: 4, insertSpaces: true, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 1, 3, false); @@ -1454,7 +1454,7 @@ suite('Editor Controller - Regression tests', () => { ' return 1;', '};' ], - mode: new SurroundingMode(), + modeId: new SurroundingMode().getId(), modelOpts: { tabSize: 4, insertSpaces: true, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 3, 2, false); @@ -1521,7 +1521,7 @@ suite('Editor Controller - Regression tests', () => { 'and more lines', 'just some text', ], - mode: null, + modeId: null, modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 1, 1, false); @@ -2034,7 +2034,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - mode: new OnEnterMode(IndentAction.None), + modeId: new OnEnterMode(IndentAction.None).getId(), }, (model, cursor) => { moveTo(cursor, 3, 8, false); moveTo(cursor, 2, 12, true); @@ -2061,7 +2061,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - mode: new OnEnterMode(IndentAction.None), + modeId: new OnEnterMode(IndentAction.None).getId(), }, (model, cursor) => { moveTo(cursor, 2, 12, false); moveTo(cursor, 3, 8, true); @@ -2195,7 +2195,7 @@ suite('Editor Controller - Cursor Configuration', () => { text: [ '\thello' ], - mode: new OnEnterMode(IndentAction.Indent), + modeId: new OnEnterMode(IndentAction.Indent).getId(), modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 1, 7, false); @@ -2211,7 +2211,7 @@ suite('Editor Controller - Cursor Configuration', () => { text: [ '\thello' ], - mode: new OnEnterMode(IndentAction.None), + modeId: new OnEnterMode(IndentAction.None).getId(), modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 1, 7, false); @@ -2227,7 +2227,7 @@ suite('Editor Controller - Cursor Configuration', () => { text: [ '\thell()' ], - mode: new OnEnterMode(IndentAction.IndentOutdent), + modeId: new OnEnterMode(IndentAction.IndentOutdent).getId(), modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 1, 7, false); @@ -2387,7 +2387,7 @@ suite('Editor Controller - Cursor Configuration', () => { defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true }, - mode: new OnEnterMode(IndentAction.IndentOutdent), + modeId: new OnEnterMode(IndentAction.IndentOutdent).getId(), }, (model, cursor) => { moveTo(cursor, 1, 32); @@ -2688,13 +2688,13 @@ suite('Editor Controller - Cursor Configuration', () => { interface ICursorOpts { text: string[]; - mode?: IMode; + modeId?: string; modelOpts?: ITextModelCreationOptions; editorOpts?: IEditorOptions; } function usingCursor(opts:ICursorOpts, callback:(model:Model, cursor:Cursor)=>void): void { - let model = Model.createFromString(opts.text.join('\n'), opts.modelOpts, opts.mode); + let model = Model.createFromString(opts.text.join('\n'), opts.modelOpts, opts.modeId); let config = new MockConfiguration(opts.editorOpts); let cursor = new Cursor(1, config, model, viewModelHelper(model), false); diff --git a/src/vs/editor/test/common/mocks/mockMode.ts b/src/vs/editor/test/common/mocks/mockMode.ts index 8b2eefaceb8..2e38ea910be 100644 --- a/src/vs/editor/test/common/mocks/mockMode.ts +++ b/src/vs/editor/test/common/mocks/mockMode.ts @@ -4,17 +4,21 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import {IMode, IState, IStream, ITokenizationResult, ITokenizationSupport} from 'vs/editor/common/modes'; +import {IMode, IState, IStream, ITokenizationResult, ITokenizationSupport, TokenizationRegistry} from 'vs/editor/common/modes'; import {AbstractState} from 'vs/editor/common/modes/abstractState'; import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport'; +let instanceCount = 0; +export function generateMockModeId(): string { + return 'mockMode' + (++instanceCount); +} + export class MockMode implements IMode { - private static instanceCount = 0; private _id:string; constructor(id?:string) { if (typeof id === 'undefined') { - id = 'mockMode' + (++MockMode.instanceCount); + id = generateMockModeId(); } this._id = id; } @@ -22,18 +26,14 @@ export class MockMode implements IMode { public getId():string { return this._id; } - - public toSimplifiedMode(): IMode { - return this; - } } export class StateForMockTokenizingMode extends AbstractState { private _tokenType: string; - constructor(mode:IMode, tokenType:string) { - super(mode); + constructor(modeId:string, tokenType:string) { + super(modeId); this._tokenType = tokenType; } @@ -53,13 +53,11 @@ export class StateForMockTokenizingMode extends AbstractState { export class MockTokenizingMode extends MockMode { - public tokenizationSupport: ITokenizationSupport; + constructor(tokenType:string) { + super(); - constructor(id:string, tokenType:string) { - super(id); - - this.tokenizationSupport = new TokenizationSupport(this, { - getInitialState: () => new StateForMockTokenizingMode(this, tokenType) - }, false); + TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), { + getInitialState: () => new StateForMockTokenizingMode(this.getId(), tokenType) + }, false)); } } diff --git a/src/vs/editor/test/common/model/model.modes.test.ts b/src/vs/editor/test/common/model/model.modes.test.ts index 1294924d2ff..0853fe937fb 100644 --- a/src/vs/editor/test/common/model/model.modes.test.ts +++ b/src/vs/editor/test/common/model/model.modes.test.ts @@ -9,38 +9,12 @@ import {EditOperation} from 'vs/editor/common/core/editOperation'; import {Position} from 'vs/editor/common/core/position'; import {Range} from 'vs/editor/common/core/range'; import {Model} from 'vs/editor/common/model/model'; -import {ModelMode1, ModelMode2, NMode} from 'vs/editor/test/common/testModes'; +import {AbstractState} from 'vs/editor/common/modes/abstractState'; +import * as modes from 'vs/editor/common/modes'; +import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport'; // --------- utils -function checkAndClear(highlighter, arr) { - assert.deepEqual(highlighter.calledFor, arr); - highlighter.calledFor = []; -} - -function invalidEqual(model, indexArray) { - var i, len, asHash = {}; - for (i = 0, len = indexArray.length; i < len; i++) { - asHash[indexArray[i]] = true; - } - for (i = 0, len = model.getLineCount(); i < len; i++) { - assert.equal(model._lines[i].isInvalid, asHash.hasOwnProperty(i)); - } -} - -function stateEqual(state, content) { - assert.equal(state.prevLineContent, content); -} - -function statesEqual(model:Model, states:string[]) { - var i, len = states.length - 1; - for (i = 0; i < len; i++) { - stateEqual(model._lines[i].getState(), states[i]); - } - stateEqual((model)._lastState, states[len]); -} - - var LINE1 = '1'; var LINE2 = '2'; var LINE3 = '3'; @@ -50,18 +24,45 @@ var LINE5 = '5'; suite('Editor Model - Model Modes 1', () => { - var thisHighlighter: ModelMode1; - var thisModel: Model; + const LANGUAGE_ID = 'modelModeTest1'; + + let calledState = { + calledFor: [] + }; + let thisModel: Model; + + class ModelState1 extends AbstractState { + public makeClone():ModelState1 { + return this; + } + public equals(other: modes.IState): boolean { + return this === other; + } + public tokenize(stream:modes.IStream): modes.ITokenizationResult { + calledState.calledFor.push(stream.next()); + stream.advanceToEOS(); + return { type: '' }; + } + } + + function checkAndClear(calledState: { calledFor: string[] }, arr:string[]) { + assert.deepEqual(calledState.calledFor, arr); + calledState.calledFor = []; + } + + modes.TokenizationRegistry.register(LANGUAGE_ID, new TokenizationSupport(null, LANGUAGE_ID, { + getInitialState: () => new ModelState1(LANGUAGE_ID) + }, false)); setup(() => { - thisHighlighter = new ModelMode1(); + calledState.calledFor = []; var text = LINE1 + '\r\n' + LINE2 + '\n' + LINE3 + '\n' + LINE4 + '\r\n' + LINE5; - thisModel = Model.createFromString(text, undefined, thisHighlighter); + thisModel = Model.createFromString(text, undefined, LANGUAGE_ID); }); teardown(() => { @@ -69,98 +70,98 @@ suite('Editor Model - Model Modes 1', () => { }); test('model calls syntax highlighter 1', () => { thisModel.getLineTokens(1); - checkAndClear(thisHighlighter, ['1']); + checkAndClear(calledState, ['1']); }); test('model calls syntax highlighter 2', () => { thisModel.getLineTokens(2); - checkAndClear(thisHighlighter, ['1', '2']); + checkAndClear(calledState, ['1', '2']); thisModel.getLineTokens(2); - checkAndClear(thisHighlighter, []); + checkAndClear(calledState, []); }); test('model caches states', () => { thisModel.getLineTokens(1); - checkAndClear(thisHighlighter, ['1']); + checkAndClear(calledState, ['1']); thisModel.getLineTokens(2); - checkAndClear(thisHighlighter, ['2']); + checkAndClear(calledState, ['2']); thisModel.getLineTokens(3); - checkAndClear(thisHighlighter, ['3']); + checkAndClear(calledState, ['3']); thisModel.getLineTokens(4); - checkAndClear(thisHighlighter, ['4']); + checkAndClear(calledState, ['4']); thisModel.getLineTokens(5); - checkAndClear(thisHighlighter, ['5']); + checkAndClear(calledState, ['5']); thisModel.getLineTokens(5); - checkAndClear(thisHighlighter, []); + checkAndClear(calledState, []); }); test('model invalidates states for one line insert', () => { thisModel.getLineTokens(5); - checkAndClear(thisHighlighter, ['1', '2', '3', '4', '5']); + checkAndClear(calledState, ['1', '2', '3', '4', '5']); thisModel.applyEdits([EditOperation.insert(new Position(1, 1), '-')]); thisModel.getLineTokens(5); - checkAndClear(thisHighlighter, ['-']); + checkAndClear(calledState, ['-']); thisModel.getLineTokens(5); - checkAndClear(thisHighlighter, []); + checkAndClear(calledState, []); }); test('model invalidates states for many lines insert', () => { thisModel.getLineTokens(5); - checkAndClear(thisHighlighter, ['1', '2', '3', '4', '5']); + checkAndClear(calledState, ['1', '2', '3', '4', '5']); thisModel.applyEdits([EditOperation.insert(new Position(1, 1), '0\n-\n+')]); assert.equal(thisModel.getLineCount(), 7); thisModel.getLineTokens(7); - checkAndClear(thisHighlighter, ['0', '-', '+']); + checkAndClear(calledState, ['0', '-', '+']); thisModel.getLineTokens(7); - checkAndClear(thisHighlighter, []); + checkAndClear(calledState, []); }); test('model invalidates states for one new line', () => { thisModel.getLineTokens(5); - checkAndClear(thisHighlighter, ['1', '2', '3', '4', '5']); + checkAndClear(calledState, ['1', '2', '3', '4', '5']); thisModel.applyEdits([EditOperation.insert(new Position(1, 2), '\n')]); thisModel.applyEdits([EditOperation.insert(new Position(2, 1), 'a')]); thisModel.getLineTokens(6); - checkAndClear(thisHighlighter, ['1', 'a']); + checkAndClear(calledState, ['1', 'a']); }); test('model invalidates states for one line delete', () => { thisModel.getLineTokens(5); - checkAndClear(thisHighlighter, ['1', '2', '3', '4', '5']); + checkAndClear(calledState, ['1', '2', '3', '4', '5']); thisModel.applyEdits([EditOperation.insert(new Position(1, 2), '-')]); thisModel.getLineTokens(5); - checkAndClear(thisHighlighter, ['1']); + checkAndClear(calledState, ['1']); thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 2))]); thisModel.getLineTokens(5); - checkAndClear(thisHighlighter, ['-']); + checkAndClear(calledState, ['-']); thisModel.getLineTokens(5); - checkAndClear(thisHighlighter, []); + checkAndClear(calledState, []); }); test('model invalidates states for many lines delete', () => { thisModel.getLineTokens(5); - checkAndClear(thisHighlighter, ['1', '2', '3', '4', '5']); + checkAndClear(calledState, ['1', '2', '3', '4', '5']); thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 3, 1))]); thisModel.getLineTokens(3); - checkAndClear(thisHighlighter, ['3']); + checkAndClear(calledState, ['3']); thisModel.getLineTokens(3); - checkAndClear(thisHighlighter, []); + checkAndClear(calledState, []); }); }); @@ -168,18 +169,70 @@ suite('Editor Model - Model Modes 1', () => { suite('Editor Model - Model Modes 2', () => { - var thisHighlighter: ModelMode1; + const LANGUAGE_ID = 'modelModeTest2'; + + class ModelState2 extends AbstractState { + + private prevLineContent:string; + + constructor(modeId:string, prevLineContent:string) { + super(modeId); + this.prevLineContent = prevLineContent; + } + + public makeClone():ModelState2 { + return new ModelState2(this.getModeId(), this.prevLineContent); + } + + public equals(other: modes.IState):boolean { + return (other instanceof ModelState2) && (this.prevLineContent === (other).prevLineContent); + } + + public tokenize(stream:modes.IStream):modes.ITokenizationResult { + var line= ''; + while (!stream.eos()) { + line+= stream.next(); + } + this.prevLineContent= line; + return { type: '' }; + } + } + modes.TokenizationRegistry.register(LANGUAGE_ID, new TokenizationSupport(null, LANGUAGE_ID, { + getInitialState: () => new ModelState2(LANGUAGE_ID, '') + }, false)); + + function invalidEqual(model, indexArray) { + var i, len, asHash = {}; + for (i = 0, len = indexArray.length; i < len; i++) { + asHash[indexArray[i]] = true; + } + for (i = 0, len = model.getLineCount(); i < len; i++) { + assert.equal(model._lines[i].isInvalid, asHash.hasOwnProperty(i)); + } + } + + function stateEqual(state, content) { + assert.equal(state.prevLineContent, content); + } + + function statesEqual(model:Model, states:string[]) { + var i, len = states.length - 1; + for (i = 0; i < len; i++) { + stateEqual(model._lines[i].getState(), states[i]); + } + stateEqual((model)._lastState, states[len]); + } + var thisModel: Model; setup(() => { - thisHighlighter = new ModelMode2(); var text = 'Line1' + '\r\n' + 'Line2' + '\n' + 'Line3' + '\n' + 'Line4' + '\r\n' + 'Line5'; - thisModel = Model.createFromString(text, undefined, thisHighlighter); + thisModel = Model.createFromString(text, undefined, LANGUAGE_ID); }); teardown(() => { @@ -251,15 +304,48 @@ suite('Editor Model - Model Modes 2', () => { suite('Editor Model - Token Iterator', () => { + const LANGUAGE_ID = 'modelModeTestTokenIterator'; + + class NState extends AbstractState { + + private n:number; + private allResults:modes.ITokenizationResult[]; + + constructor(modeId:string, n:number) { + super(modeId); + this.n = n; + this.allResults = null; + } + + public makeClone():NState { + return this; + } + + public equals(other: modes.IState):boolean { + return true; + } + + public tokenize(stream:modes.IStream):modes.ITokenizationResult { + var ndash = this.n, value = ''; + while(!stream.eos() && ndash > 0) { + value += stream.next(); + ndash--; + } + return { type: 'n-' + (this.n - ndash) + '-' + value }; + } + } + modes.TokenizationRegistry.register(LANGUAGE_ID, new TokenizationSupport(null, LANGUAGE_ID, { + getInitialState: () => new NState(LANGUAGE_ID, 3) + }, false)); + var thisModel: Model; setup(() => { - var nmode = new NMode(3); var text = 'foobarfoobar' + '\r\n' + 'foobarfoobar' + '\r\n' + 'foobarfoobar' + '\r\n'; - thisModel = Model.createFromString(text, undefined, nmode); + thisModel = Model.createFromString(text, undefined, LANGUAGE_ID); }); teardown(() => { diff --git a/src/vs/editor/test/common/model/textModelWithTokens.test.ts b/src/vs/editor/test/common/model/textModelWithTokens.test.ts index ca756915838..d183665001f 100644 --- a/src/vs/editor/test/common/model/textModelWithTokens.test.ts +++ b/src/vs/editor/test/common/model/textModelWithTokens.test.ts @@ -7,7 +7,7 @@ import * as assert from 'assert'; import {Model} from 'vs/editor/common/model/model'; import {ViewLineToken} from 'vs/editor/common/core/viewLineToken'; -import {ITokenizationSupport} from 'vs/editor/common/modes'; +import {TokenizationRegistry} from 'vs/editor/common/modes'; import {MockMode} from 'vs/editor/test/common/mocks/mockMode'; import {Token} from 'vs/editor/common/core/token'; import {Range} from 'vs/editor/common/core/range'; @@ -152,24 +152,21 @@ suite('TextModelWithTokens - bracket matching', () => { ], 'is matching brackets at ' + lineNumber1 + ', ' + column11); } - class BracketMode extends MockMode { - constructor() { - super(); - LanguageConfigurationRegistry.register(this.getId(), { - brackets: [ - ['{', '}'], - ['[', ']'], - ['(', ')'], - ] - }); - } - } + const LANGUAGE_ID = 'bracketMode1'; + + LanguageConfigurationRegistry.register(LANGUAGE_ID, { + brackets: [ + ['{', '}'], + ['[', ']'], + ['(', ')'], + ] + }); test('bracket matching 1', () => { let text = ')]}{[(' + '\n' + ')]}{[('; - let model = Model.createFromString(text, undefined, new BracketMode()); + let model = Model.createFromString(text, undefined, LANGUAGE_ID); isNotABracket(model, 1, 1); isNotABracket(model, 1, 2); @@ -197,7 +194,7 @@ suite('TextModelWithTokens - bracket matching', () => { '}, bar: {hallo: [{' + '\n' + '}, {' + '\n' + '}]}}'; - let model = Model.createFromString(text, undefined, new BracketMode()); + let model = Model.createFromString(text, undefined, LANGUAGE_ID); let brackets = [ [1, 11, 12, 5, 4, 5], @@ -258,11 +255,9 @@ suite('TextModelWithTokens regression tests', () => { let _tokenId = 0; class IndicisiveMode extends MockMode { - public tokenizationSupport:ITokenizationSupport; - constructor() { super(); - this.tokenizationSupport = { + TokenizationRegistry.register(this.getId(), { getInitialState: () => { return null; }, @@ -276,7 +271,7 @@ suite('TextModelWithTokens regression tests', () => { retokenize: null }; } - }; + }); } } let model = Model.createFromString('A model with\ntwo lines'); @@ -284,12 +279,12 @@ suite('TextModelWithTokens regression tests', () => { assertViewLineTokens(model, 1, true, [new ViewLineToken(0, '')]); assertViewLineTokens(model, 2, true, [new ViewLineToken(0, '')]); - model.setMode(new IndicisiveMode()); + model.setMode(new IndicisiveMode().getId()); assertViewLineTokens(model, 1, true, [new ViewLineToken(0, 'custom.1')]); assertViewLineTokens(model, 2, true, [new ViewLineToken(0, 'custom.2')]); - model.setMode(new IndicisiveMode()); + model.setMode(new IndicisiveMode().getId()); assertViewLineTokens(model, 1, false, [new ViewLineToken(0, '')]); assertViewLineTokens(model, 2, false, [new ViewLineToken(0, '')]); @@ -298,17 +293,15 @@ suite('TextModelWithTokens regression tests', () => { }); test('Microsoft/monaco-editor#133: Error: Cannot read property \'modeId\' of undefined', () => { - class BracketMode extends MockMode { - constructor() { - super(); - LanguageConfigurationRegistry.register(this.getId(), { - brackets: [ - ['module','end module'], - ['sub','end sub'] - ] - }); - } - } + + const LANGUAGE_ID = 'bracketMode2'; + + LanguageConfigurationRegistry.register(LANGUAGE_ID, { + brackets: [ + ['module','end module'], + ['sub','end sub'] + ] + }); let model = Model.createFromString([ 'Imports System', @@ -320,7 +313,7 @@ suite('TextModelWithTokens regression tests', () => { '\tEnd Sub', '', 'End Module', - ].join('\n'), undefined, new BracketMode()); + ].join('\n'), undefined, LANGUAGE_ID); let actual = model.matchBracket(new Position(4,1)); assert.deepEqual(actual, [new Range(4,1,4,7), new Range(9,1,9,11)]); diff --git a/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts b/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts index 493aca668f4..3503a64383c 100644 --- a/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts +++ b/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts @@ -5,7 +5,7 @@ 'use strict'; import * as assert from 'assert'; -import {IMode, IStream, ITokenizationResult, ITokenizationSupport} from 'vs/editor/common/modes'; +import {IStream, ITokenizationResult, TokenizationRegistry} from 'vs/editor/common/modes'; import {AbstractState} from 'vs/editor/common/modes/abstractState'; import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport'; import {tokenizeToHtmlContent} from 'vs/editor/common/modes/textToHtmlTokenizer'; @@ -14,7 +14,7 @@ import {MockMode} from 'vs/editor/test/common/mocks/mockMode'; suite('Editor Modes - textToHtmlTokenizer', () => { test('TextToHtmlTokenizer', () => { var mode = new Mode(); - var result = tokenizeToHtmlContent('.abc..def...gh', mode); + var result = tokenizeToHtmlContent('.abc..def...gh', mode.getId()); assert.ok(!!result); @@ -45,7 +45,7 @@ suite('Editor Modes - textToHtmlTokenizer', () => { assert.equal(children[5].className, 'token text'); assert.equal(children[5].tagName, 'span'); - result = tokenizeToHtmlContent('.abc..def...gh\n.abc..def...gh', mode); + result = tokenizeToHtmlContent('.abc..def...gh\n.abc..def...gh', mode.getId()); assert.ok(!!result); @@ -59,12 +59,12 @@ suite('Editor Modes - textToHtmlTokenizer', () => { class State extends AbstractState { - constructor(mode:IMode) { - super(mode); + constructor(modeId:string) { + super(modeId); } public makeClone() : AbstractState { - return new State(this.getMode()); + return new State(this.getModeId()); } public tokenize(stream:IStream):ITokenizationResult { @@ -73,13 +73,10 @@ class State extends AbstractState { } class Mode extends MockMode { - - public tokenizationSupport: ITokenizationSupport; - constructor() { super(); - this.tokenizationSupport = new TokenizationSupport(this, { - getInitialState: () => new State(this) - }, false); + TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), { + getInitialState: () => new State(this.getId()) + }, false)); } } diff --git a/src/vs/editor/test/common/modes/tokenization.test.ts b/src/vs/editor/test/common/modes/tokenization.test.ts index 1645f2c1782..3a706cf94fd 100644 --- a/src/vs/editor/test/common/modes/tokenization.test.ts +++ b/src/vs/editor/test/common/modes/tokenization.test.ts @@ -5,58 +5,15 @@ 'use strict'; import * as assert from 'assert'; -import {IDisposable, empty as EmptyDisposable} from 'vs/base/common/lifecycle'; -import {IModeSupportChangedEvent} from 'vs/editor/common/editorCommon'; import * as modes from 'vs/editor/common/modes'; import {AbstractState} from 'vs/editor/common/modes/abstractState'; import {handleEvent} from 'vs/editor/common/modes/supports'; -import {IEnteringNestedModeData, ILeavingNestedModeData, TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport'; +import {IModeLocator, ILeavingNestedModeData, TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport'; import {createMockLineContext} from 'vs/editor/test/common/modesTestUtils'; import {MockMode} from 'vs/editor/test/common/mocks/mockMode'; import {ModeTransition} from 'vs/editor/common/core/modeTransition'; import {Token} from 'vs/editor/common/core/token'; -export class State extends AbstractState { - - constructor(mode:modes.IMode) { - super(mode); - } - - public makeClone() : AbstractState { - return new State(this.getMode()); - } - - public tokenize(stream:modes.IStream):modes.ITokenizationResult { - return { type: stream.next() === '.' ? '' : 'text' }; - } -} - -export class Mode extends MockMode { - - public tokenizationSupport: modes.ITokenizationSupport; - - constructor() { - super(); - this.tokenizationSupport = new TokenizationSupport(this, { - getInitialState: () => new State(this) - }, false); - } -} - - - - -function checkTokens(actual, expected) { - assert.equal(actual.length, expected.length); - for (var i = 0; i < expected.length; i++) { - for (var key in expected[i]) { - assert.deepEqual(actual[i][key], expected[i][key]); - } - } -} - - - export interface IModeSwitchingDescriptor { [character:string]:{ endCharacter: string; @@ -69,14 +26,14 @@ export class StateMemorizingLastWord extends AbstractState { public lastWord:string; private descriptor:IModeSwitchingDescriptor; - constructor(mode:modes.IMode, descriptor:IModeSwitchingDescriptor, lastWord:string) { - super(mode); + constructor(modeId:string, descriptor:IModeSwitchingDescriptor, lastWord:string) { + super(modeId); this.lastWord = lastWord; this.descriptor = descriptor; } public makeClone() : AbstractState { - return new StateMemorizingLastWord(this.getMode(), this.descriptor, this.lastWord); + return new StateMemorizingLastWord(this.getModeId(), this.descriptor, this.lastWord); } public tokenize(stream:modes.IStream):modes.ITokenizationResult { @@ -88,8 +45,8 @@ export class StateMemorizingLastWord extends AbstractState { } var word = stream.nextToken(); return { - type: this.getMode().getId() + '.' + word, - nextState: new StateMemorizingLastWord(this.getMode(), this.descriptor, word) + type: this.getModeId() + '.' + word, + nextState: new StateMemorizingLastWord(this.getModeId(), this.descriptor, word) }; } } @@ -98,24 +55,14 @@ export class SwitchingMode extends MockMode { private _switchingModeDescriptor:IModeSwitchingDescriptor; - public tokenizationSupport: modes.ITokenizationSupport; - constructor(id:string, descriptor:IModeSwitchingDescriptor) { super(id); this._switchingModeDescriptor = descriptor; - this.tokenizationSupport = new TokenizationSupport(this, this, true); - } - - setTokenizationSupport(callback:(mode:modes.IMode)=>T): IDisposable { - return EmptyDisposable; - } - - public addSupportChangedListener(callback: (e: IModeSupportChangedEvent) => void): IDisposable { - return EmptyDisposable; + modes.TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), this, true)); } public getInitialState():modes.IState { - return new StateMemorizingLastWord(this, this._switchingModeDescriptor, null); + return new StateMemorizingLastWord(this.getId(), this._switchingModeDescriptor, null); } public enterNestedMode(state:modes.IState):boolean { @@ -125,12 +72,9 @@ export class SwitchingMode extends MockMode { } } - public getNestedMode(state:modes.IState): IEnteringNestedModeData { + public getNestedMode(state:modes.IState, locator:IModeLocator): modes.IMode { var s = state; - return { - mode: this._switchingModeDescriptor[s.lastWord].mode, - missingModePromise: null - }; + return this._switchingModeDescriptor[s.lastWord].mode; } public getLeavingNestedModeData(line:string, state:modes.IState): ILeavingNestedModeData { @@ -141,7 +85,7 @@ export class SwitchingMode extends MockMode { return { nestedModeBuffer: line.substring(0, endCharPosition), bufferAfterNestedMode: line.substring(endCharPosition), - stateAfterNestedMode: new StateMemorizingLastWord(this, this._switchingModeDescriptor, null) + stateAfterNestedMode: new StateMemorizingLastWord(this.getId(), this._switchingModeDescriptor, null) }; } return null; @@ -175,7 +119,7 @@ function assertModeTransitions(actual:ModeTransition[], expected:ITestModeTransi assert.deepEqual(massagedActual, expected, message); }; -function createMode():SwitchingMode { +let switchingMode = (function() { var modeB = new SwitchingMode('B', {}); var modeC = new SwitchingMode('C', {}); var modeD = new SwitchingMode('D', { @@ -199,23 +143,41 @@ function createMode():SwitchingMode { } }); return modeA; -} +})(); -function switchingModeTokenize(line:string, mode:modes.IMode = null, state:modes.IState = null) { - if (state && mode) { - return mode.tokenizationSupport.tokenize(line, state); +function switchingModeTokenize(line:string, state:modes.IState = null) { + let tokenizationSupport = modes.TokenizationRegistry.get(switchingMode.getId()); + if (state) { + return tokenizationSupport.tokenize(line, state); } else { - mode = createMode(); - return mode.tokenizationSupport.tokenize(line, mode.tokenizationSupport.getInitialState()); + return tokenizationSupport.tokenize(line, tokenizationSupport.getInitialState()); } } suite('Editor Modes - Tokenization', () => { test('Syntax engine merges sequential untyped tokens', () => { - var mode = new Mode(); - var lineTokens = mode.tokenizationSupport.tokenize('.abc..def...gh', mode.tokenizationSupport.getInitialState()); - checkTokens(lineTokens.tokens, [ + class State extends AbstractState { + + constructor(modeId:string) { + super(modeId); + } + + public makeClone() : AbstractState { + return new State(this.getModeId()); + } + + public tokenize(stream:modes.IStream):modes.ITokenizationResult { + return { type: stream.next() === '.' ? '' : 'text' }; + } + } + + let tokenizationSupport = new TokenizationSupport(null, 'test', { + getInitialState: () => new State('test') + }, false); + + var lineTokens = tokenizationSupport.tokenize('.abc..def...gh', tokenizationSupport.getInitialState()); + assertTokens(lineTokens.tokens, [ { startIndex: 0, type: '' }, { startIndex: 1, type: 'text' }, { startIndex: 4, type: '' }, @@ -282,15 +244,14 @@ suite('Editor Modes - Tokenization', () => { { startIndex:3, type: '' }, { startIndex:4, type: 'A.(' } ]); - assert.equal((lineTokens.endState).getMode().getId(), 'B'); + assert.equal((lineTokens.endState).getModeId(), 'B'); assertModeTransitions(lineTokens.modeTransitions, [ { startIndex: 0, id: 'A' } ]); }); test('One embedded over multiple lines 1', () => { - var mode = createMode(); - var lineTokens = switchingModeTokenize('abc (def', mode, mode.getInitialState()); + var lineTokens = switchingModeTokenize('abc (def'); assertTokens(lineTokens.tokens, [ { startIndex:0, type: 'A.abc' }, { startIndex:3, type: '' }, @@ -302,7 +263,7 @@ suite('Editor Modes - Tokenization', () => { { startIndex: 5, id: 'B' } ]); - lineTokens = switchingModeTokenize('ghi jkl', mode, lineTokens.endState); + lineTokens = switchingModeTokenize('ghi jkl', lineTokens.endState); assertTokens(lineTokens.tokens, [ { startIndex:0, type: 'B.ghi' }, { startIndex:3, type: '' }, @@ -312,7 +273,7 @@ suite('Editor Modes - Tokenization', () => { { startIndex: 0, id: 'B' } ]); - lineTokens = switchingModeTokenize('mno)pqr', mode, lineTokens.endState); + lineTokens = switchingModeTokenize('mno)pqr', lineTokens.endState); assertTokens(lineTokens.tokens, [ { startIndex:0, type: 'B.mno' }, { startIndex:3, type: 'A.)' }, @@ -325,8 +286,7 @@ suite('Editor Modes - Tokenization', () => { }); test('One embedded over multiple lines 2 with handleEvent', () => { - var mode = createMode(); - var lineTokens = switchingModeTokenize('abc (def', mode, mode.getInitialState()); + var lineTokens = switchingModeTokenize('abc (def'); assertTokens(lineTokens.tokens, [ { startIndex:0, type: 'A.abc' }, { startIndex:3, type: '' }, @@ -360,7 +320,7 @@ suite('Editor Modes - Tokenization', () => { assert.equal(context.getLineContent(), 'def'); }); - lineTokens = switchingModeTokenize('ghi jkl', mode, lineTokens.endState); + lineTokens = switchingModeTokenize('ghi jkl', lineTokens.endState); assertTokens(lineTokens.tokens, [ { startIndex:0, type: 'B.ghi' }, { startIndex:3, type: '' }, @@ -370,7 +330,7 @@ suite('Editor Modes - Tokenization', () => { { startIndex: 0, id: 'B' } ]); - lineTokens = switchingModeTokenize(')pqr', mode, lineTokens.endState); + lineTokens = switchingModeTokenize(')pqr', lineTokens.endState); assertTokens(lineTokens.tokens, [ { startIndex:0, type: 'A.)' }, { startIndex:1, type: 'A.pqr' } diff --git a/src/vs/editor/test/common/modesUtil.ts b/src/vs/editor/test/common/modesUtil.ts index 0fdcb35dd52..c57c8bf6b0f 100644 --- a/src/vs/editor/test/common/modesUtil.ts +++ b/src/vs/editor/test/common/modesUtil.ts @@ -7,9 +7,9 @@ import * as assert from 'assert'; import {Model} from 'vs/editor/common/model/model'; import * as modes from 'vs/editor/common/modes'; -import {MockMode} from 'vs/editor/test/common/mocks/mockMode'; import {RichEditSupport, LanguageConfiguration} from 'vs/editor/common/modes/languageConfigurationRegistry'; import {Token} from 'vs/editor/common/core/token'; +import {generateMockModeId} from 'vs/editor/test/common/mocks/mockMode'; export interface ITestToken { startIndex: number; @@ -45,15 +45,17 @@ export interface IOnEnterAsserter { indentsOutdents(oneLineAboveText:string, beforeText:string, afterText:string): void; } -export function createOnEnterAsserter(modeId:string, conf: LanguageConfiguration): IOnEnterAsserter { - var assertOne = (oneLineAboveText:string, beforeText:string, afterText:string, expected: modes.IndentAction) => { - var model = Model.createFromString( +export function createOnEnterAsserter(conf: LanguageConfiguration): IOnEnterAsserter { + const modeId = generateMockModeId(); + + const assertOne = (oneLineAboveText:string, beforeText:string, afterText:string, expected: modes.IndentAction) => { + let model = Model.createFromString( [ oneLineAboveText, beforeText + afterText ].join('\n'), undefined, - new MockMode(modeId) + modeId ); - var richEditSupport = new RichEditSupport(modeId, null, conf); - var actual = richEditSupport.onEnter.onEnter(model, { lineNumber: 2, column: beforeText.length + 1 }); + let richEditSupport = new RichEditSupport(modeId, null, conf); + let actual = richEditSupport.onEnter.onEnter(model, { lineNumber: 2, column: beforeText.length + 1 }); if (expected === modes.IndentAction.None) { assert.equal(actual, null, oneLineAboveText + '\\n' + beforeText + '|' + afterText); } else { @@ -61,6 +63,7 @@ export function createOnEnterAsserter(modeId:string, conf: LanguageConfiguration } model.dispose(); }; + return { nothing: (oneLineAboveText:string, beforeText:string, afterText:string): void => { assertOne(oneLineAboveText, beforeText, afterText, modes.IndentAction.None); diff --git a/src/vs/editor/test/common/testModes.ts b/src/vs/editor/test/common/testModes.ts index 23e4f6a0a86..2b22cd526b2 100644 --- a/src/vs/editor/test/common/testModes.ts +++ b/src/vs/editor/test/common/testModes.ts @@ -4,181 +4,14 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import * as modes from 'vs/editor/common/modes'; -import {AbstractState} from 'vs/editor/common/modes/abstractState'; import {LanguageConfigurationRegistry, CommentRule} from 'vs/editor/common/modes/languageConfigurationRegistry'; -import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport'; import {MockMode} from 'vs/editor/test/common/mocks/mockMode'; -export class CommentState extends AbstractState { - - constructor(mode:modes.IMode, stateCount:number) { - super(mode); - } - - public makeClone():CommentState { - return this; - } - - public equals(other:modes.IState):boolean { - return true; - } - - public tokenize(stream:modes.IStream):modes.ITokenizationResult { - stream.advanceToEOS(); - return { type: 'state' }; - } -} - export class CommentMode extends MockMode { - - public tokenizationSupport: modes.ITokenizationSupport; - constructor(commentsConfig:CommentRule) { super(); - this.tokenizationSupport = new TokenizationSupport(this, { - getInitialState: () => new CommentState(this, 0) - }, false); - LanguageConfigurationRegistry.register(this.getId(), { comments: commentsConfig }); } } - -export abstract class AbstractIndentingMode extends MockMode { - - public getElectricCharacters():string[] { - return null; - } - - public onElectricCharacter(context:modes.ILineContext, offset:number):modes.IElectricAction { - return null; - } - - public onEnter(context:modes.ILineContext, offset:number):modes.EnterAction { - return null; - } - -} - -export class ModelState1 extends AbstractState { - - constructor(mode:modes.IMode) { - super(mode); - } - - public makeClone():ModelState1 { - return this; - } - - public equals(other: modes.IState):boolean { - return this === other; - } - - public tokenize(stream:modes.IStream):modes.ITokenizationResult { - (this.getMode()).calledFor.push(stream.next()); - stream.advanceToEOS(); - return { type: '' }; - } -} - -export class ModelMode1 extends MockMode { - public calledFor:string[]; - - public tokenizationSupport: modes.ITokenizationSupport; - - constructor() { - super(); - this.calledFor = []; - this.tokenizationSupport = new TokenizationSupport(this, { - getInitialState: () => new ModelState1(this) - }, false); - } -} - -export class ModelState2 extends AbstractState { - - private prevLineContent:string; - - constructor(mode:ModelMode2, prevLineContent:string) { - super(mode); - this.prevLineContent = prevLineContent; - } - - public makeClone():ModelState2 { - return new ModelState2(this.getMode(), this.prevLineContent); - } - - public equals(other: modes.IState):boolean { - return (other instanceof ModelState2) && (this.prevLineContent === (other).prevLineContent); - } - - public tokenize(stream:modes.IStream):modes.ITokenizationResult { - var line= ''; - while (!stream.eos()) { - line+= stream.next(); - } - this.prevLineContent= line; - return { type: '' }; - } -} - -export class ModelMode2 extends MockMode { - public calledFor:any[]; - - public tokenizationSupport: modes.ITokenizationSupport; - - constructor() { - super(); - this.calledFor = null; - this.tokenizationSupport = new TokenizationSupport(this, { - getInitialState: () => new ModelState2(this, '') - }, false); - } -} - -export class NState extends AbstractState { - - private n:number; - private allResults:modes.ITokenizationResult[]; - - constructor(mode:modes.IMode, n:number) { - super(mode); - this.n = n; - this.allResults = null; - } - - - public makeClone():NState { - return this; - } - - public equals(other: modes.IState):boolean { - return true; - } - - public tokenize(stream:modes.IStream):modes.ITokenizationResult { - var ndash = this.n, value = ''; - while(!stream.eos() && ndash > 0) { - value += stream.next(); - ndash--; - } - return { type: 'n-' + (this.n - ndash) + '-' + value }; - } -} - -export class NMode extends MockMode { - - private n:number; - - public tokenizationSupport: modes.ITokenizationSupport; - - constructor(n:number) { - super(); - this.n = n; - this.tokenizationSupport = new TokenizationSupport(this, { - getInitialState: () => new NState(this, this.n) - }, false); - } -} \ No newline at end of file diff --git a/src/vs/languages/handlebars/common/handlebars.ts b/src/vs/languages/handlebars/common/handlebars.ts index 7434e07bbb4..4ccc898e1f1 100644 --- a/src/vs/languages/handlebars/common/handlebars.ts +++ b/src/vs/languages/handlebars/common/handlebars.ts @@ -12,11 +12,11 @@ import {IInstantiationService} from 'vs/platform/instantiation/common/instantiat import {IModeService} from 'vs/editor/common/services/modeService'; import {LanguageConfigurationRegistry, LanguageConfiguration} from 'vs/editor/common/modes/languageConfigurationRegistry'; import {createWordRegExp} from 'vs/editor/common/modes/abstractMode'; -import {ILeavingNestedModeData} from 'vs/editor/common/modes/supports/tokenizationSupport'; import {wireCancellationToken} from 'vs/base/common/async'; import {ICompatWorkerService} from 'vs/editor/common/services/compatWorkerService'; import {IWorkspaceContextService} from 'vs/platform/workspace/common/workspace'; import {IConfigurationService} from 'vs/platform/configuration/common/configuration'; +import {TokenizationSupport, ILeavingNestedModeData} from 'vs/editor/common/modes/supports/tokenizationSupport'; export enum States { HTML, @@ -26,7 +26,7 @@ export enum States { export class HandlebarsState extends htmlMode.State { - constructor(mode:modes.IMode, + constructor(modeId:string, kind:htmlMode.States, public handlebarsKind:States, lastTagName:string, @@ -35,11 +35,11 @@ export class HandlebarsState extends htmlMode.State { attributeValueQuote:string, attributeValueLength:number) { - super(mode, kind, lastTagName, lastAttributeName, embeddedContentType, attributeValueQuote, attributeValueLength); + super(modeId, kind, lastTagName, lastAttributeName, embeddedContentType, attributeValueQuote, attributeValueLength); } public makeClone(): HandlebarsState { - return new HandlebarsState(this.getMode(), this.kind, this.handlebarsKind, this.lastTagName, this.lastAttributeName, this.embeddedContentType, this.attributeValueQuote, this.attributeValueLength); + return new HandlebarsState(this.getModeId(), this.kind, this.handlebarsKind, this.lastTagName, this.lastAttributeName, this.embeddedContentType, this.attributeValueQuote, this.attributeValueLength); } public equals(other:modes.IState):boolean { @@ -184,16 +184,18 @@ export class HandlebarsMode extends htmlMode.HTMLMode { }, true); LanguageConfigurationRegistry.register(this.getId(), HandlebarsMode.LANG_CONFIG); + + modes.TokenizationRegistry.register(this.getId(), new TokenizationSupport(this._modeService, this.getId(), this, true)); } public getInitialState() : modes.IState { - return new HandlebarsState(this, htmlMode.States.Content, States.HTML, '', '', '', '', 0); + return new HandlebarsState(this.getId(), htmlMode.States.Content, States.HTML, '', '', '', '', 0); } public getLeavingNestedModeData(line:string, state:modes.IState):ILeavingNestedModeData { var leavingNestedModeData = super.getLeavingNestedModeData(line, state); if (leavingNestedModeData) { - leavingNestedModeData.stateAfterNestedMode = new HandlebarsState(this, htmlMode.States.Content, States.HTML, '', '', '', '', 0); + leavingNestedModeData.stateAfterNestedMode = new HandlebarsState(this.getId(), htmlMode.States.Content, States.HTML, '', '', '', '', 0); } return leavingNestedModeData; } diff --git a/src/vs/languages/handlebars/test/common/handlebars.test.ts b/src/vs/languages/handlebars/test/common/handlebars.test.ts index 8323eea6834..e25688093a7 100644 --- a/src/vs/languages/handlebars/test/common/handlebars.test.ts +++ b/src/vs/languages/handlebars/test/common/handlebars.test.ts @@ -35,11 +35,21 @@ class HandlebarsMockModeService extends MockModeService { throw new Error('Not implemented'); } - getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): Modes.IMode { - if (commaSeparatedMimetypesOrCommaSeparatedIds === 'text/javascript') { - return new MockTokenizingMode('js', 'mock-js'); + getModeId(mimetypeOrModeId: string): string { + if (mimetypeOrModeId === 'text/javascript') { + return 'js-mode-id'; } - if (commaSeparatedMimetypesOrCommaSeparatedIds === 'text/x-handlebars-template') { + if (mimetypeOrModeId === 'text/x-handlebars-template') { + return 'handlebars-mode-id'; + } + throw new Error('Not implemented'); + } + + getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): Modes.IMode { + if (commaSeparatedMimetypesOrCommaSeparatedIds === 'js-mode-id') { + return new MockTokenizingMode('mock-js'); + } + if (commaSeparatedMimetypesOrCommaSeparatedIds === 'handlebars-mode-id') { return this._handlebarsMode; } throw new Error('Not implemented'); @@ -49,7 +59,7 @@ class HandlebarsMockModeService extends MockModeService { suite('Handlebars', () => { var tokenizationSupport: Modes.ITokenizationSupport; - (function() { + suiteSetup(function() { let modeService = new HandlebarsMockModeService(); let mode = new HandlebarsMode( @@ -63,8 +73,8 @@ suite('Handlebars', () => { modeService.setHandlebarsMode(mode); - tokenizationSupport = mode.tokenizationSupport; - })(); + tokenizationSupport = Modes.TokenizationRegistry.get(mode.getId()); + }); test('Just HTML', () => { modesUtil.assertTokenization(tokenizationSupport, [{ diff --git a/src/vs/languages/html/common/html.ts b/src/vs/languages/html/common/html.ts index fbc9d4d7a63..0faa53bae8e 100644 --- a/src/vs/languages/html/common/html.ts +++ b/src/vs/languages/html/common/html.ts @@ -16,7 +16,7 @@ import {IInstantiationService} from 'vs/platform/instantiation/common/instantiat import * as htmlTokenTypes from 'vs/languages/html/common/htmlTokenTypes'; import {EMPTY_ELEMENTS} from 'vs/languages/html/common/htmlEmptyTagsShared'; import {LanguageConfigurationRegistry, LanguageConfiguration} from 'vs/editor/common/modes/languageConfigurationRegistry'; -import {TokenizationSupport, IEnteringNestedModeData, ILeavingNestedModeData, ITokenizationCustomization} from 'vs/editor/common/modes/supports/tokenizationSupport'; +import {TokenizationSupport, IModeLocator, ILeavingNestedModeData, ITokenizationCustomization} from 'vs/editor/common/modes/supports/tokenizationSupport'; import {wireCancellationToken} from 'vs/base/common/async'; import {ICompatWorkerService, CompatWorkerAttr} from 'vs/editor/common/services/compatWorkerService'; import {IWorkspaceContextService} from 'vs/platform/workspace/common/workspace'; @@ -52,8 +52,8 @@ export class State extends AbstractState { public attributeValueQuote:string; public attributeValueLength:number; - constructor(mode:modes.IMode, kind:States, lastTagName:string, lastAttributeName:string, embeddedContentType:string, attributeValueQuote:string, attributeValueLength:number) { - super(mode); + constructor(modeId:string, kind:States, lastTagName:string, lastAttributeName:string, embeddedContentType:string, attributeValueQuote:string, attributeValueLength:number) { + super(modeId); this.kind = kind; this.lastTagName = lastTagName; this.lastAttributeName = lastAttributeName; @@ -67,7 +67,7 @@ export class State extends AbstractState { } public makeClone():State { - return new State(this.getMode(), this.kind, this.lastTagName, this.lastAttributeName, this.embeddedContentType, this.attributeValueQuote, this.attributeValueLength); + return new State(this.getModeId(), this.kind, this.lastTagName, this.lastAttributeName, this.embeddedContentType, this.attributeValueQuote, this.attributeValueLength); } public equals(other:modes.IState):boolean { @@ -330,9 +330,7 @@ export class HTMLMode extends CompatMode implem ], }; - public tokenizationSupport: modes.ITokenizationSupport; - - private modeService:IModeService; + protected _modeService:IModeService; private _modeWorkerManager: ModeWorkerManager; constructor( @@ -346,9 +344,7 @@ export class HTMLMode extends CompatMode implem super(descriptor.id, compatWorkerService); this._modeWorkerManager = this._createModeWorkerManager(descriptor, instantiationService); - this.modeService = modeService; - - this.tokenizationSupport = new TokenizationSupport(this, this, true); + this._modeService = modeService; if (this.compatWorkerService && this.compatWorkerService.isInMainThread) { let updateConfiguration = () => { @@ -393,6 +389,8 @@ export class HTMLMode extends CompatMode implem }, true); LanguageConfigurationRegistry.register(this.getId(), HTMLMode.LANG_CONFIG); + + modes.TokenizationRegistry.register(this.getId(), new TokenizationSupport(this._modeService, this.getId(), this, true)); } protected _createModeWorkerManager(descriptor:modes.IModeDescriptor, instantiationService: IInstantiationService): ModeWorkerManager { @@ -406,43 +404,25 @@ export class HTMLMode extends CompatMode implem // TokenizationSupport public getInitialState():modes.IState { - return new State(this, States.Content, '', '', '', '', 0); + return new State(this.getId(), States.Content, '', '', '', '', 0); } public enterNestedMode(state:modes.IState):boolean { return state instanceof State && (state).kind === States.WithinEmbeddedContent; } - public getNestedMode(state:modes.IState): IEnteringNestedModeData { - var result:modes.IMode = null; - var htmlState:State = state; - var missingModePromise: winjs.Promise = null; - + public getNestedMode(state:modes.IState, locator:IModeLocator): modes.IMode { + let htmlState:State = state; if (htmlState.embeddedContentType !== null) { - if (this.modeService.isRegisteredMode(htmlState.embeddedContentType)) { - result = this.modeService.getMode(htmlState.embeddedContentType); - if (!result) { - missingModePromise = this.modeService.getOrCreateMode(htmlState.embeddedContentType); - } - } - } else { - var mimeType:string = null; - if ('script' === htmlState.lastTagName) { - mimeType = 'text/javascript'; - } else if ('style' === htmlState.lastTagName) { - mimeType = 'text/css'; - } else { - mimeType = 'text/plain'; - } - result = this.modeService.getMode(mimeType); + return locator.getMode(htmlState.embeddedContentType); } - if (result === null) { - result = this.modeService.getMode('text/plain'); + if ('script' === htmlState.lastTagName) { + return locator.getMode('text/javascript'); } - return { - mode: result, - missingModePromise: missingModePromise - }; + if ('style' === htmlState.lastTagName) { + return locator.getMode('text/css'); + } + return null; } public getLeavingNestedModeData(line:string, state:modes.IState):ILeavingNestedModeData { @@ -453,7 +433,7 @@ export class HTMLMode extends CompatMode implem return { nestedModeBuffer: line.substring(0, match.index), bufferAfterNestedMode: line.substring(match.index), - stateAfterNestedMode: new State(this, States.Content, '', '', '', '', 0) + stateAfterNestedMode: new State(this.getId(), States.Content, '', '', '', '', 0) }; } return null; diff --git a/src/vs/languages/html/test/common/html-worker.test.ts b/src/vs/languages/html/test/common/html-worker.test.ts index 5ea6f5a40be..51b6039de8d 100644 --- a/src/vs/languages/html/test/common/html-worker.test.ts +++ b/src/vs/languages/html/test/common/html-worker.test.ts @@ -16,7 +16,7 @@ import {MockModeService} from 'vs/editor/test/common/mocks/mockModeService'; import {TextModel} from 'vs/editor/common/model/textModel'; function createTestMirrorModelFromString(value:string, mode:Modes.IMode, associatedResource:URI): mm.CompatMirrorModel { - return new mm.CompatMirrorModel(0, TextModel.toRawText(value, TextModel.DEFAULT_CREATION_OPTIONS), mode, associatedResource); + return new mm.CompatMirrorModel(0, TextModel.toRawText(value, TextModel.DEFAULT_CREATION_OPTIONS), mode.getId(), associatedResource); } suite('HTML - worker', () => { diff --git a/src/vs/languages/html/test/common/html.test.ts b/src/vs/languages/html/test/common/html.test.ts index 200b30ad06f..3bf40c13412 100644 --- a/src/vs/languages/html/test/common/html.test.ts +++ b/src/vs/languages/html/test/common/html.test.ts @@ -21,7 +21,7 @@ import {LanguageConfigurationRegistry} from 'vs/editor/common/modes/languageConf class MockJSMode extends MockTokenizingMode { constructor() { - super('html-js-mock', 'mock-js'); + super('mock-js'); LanguageConfigurationRegistry.register(this.getId(), { brackets: [ @@ -63,6 +63,9 @@ class MockJSMode extends MockTokenizingMode { } class HTMLMockModeService extends MockModeService { + + private _mockJSMode = new MockJSMode(); + isRegisteredMode(mimetypeOrModeId: string): boolean { if (mimetypeOrModeId === 'text/javascript') { return true; @@ -73,12 +76,16 @@ class HTMLMockModeService extends MockModeService { throw new Error('Not implemented'); } - getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): Modes.IMode { - if (commaSeparatedMimetypesOrCommaSeparatedIds === 'text/javascript') { - return new MockJSMode(); + getModeId(mimetypeOrModeId: string): string { + if (mimetypeOrModeId === 'text/javascript') { + return 'js-mode-id'; } - if (commaSeparatedMimetypesOrCommaSeparatedIds === 'text/plain') { - return null; + throw new Error('Not implemented'); + } + + getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): Modes.IMode { + if (commaSeparatedMimetypesOrCommaSeparatedIds === 'js-mode-id') { + return this._mockJSMode; } throw new Error('Not implemented'); } @@ -90,7 +97,8 @@ suite('Colorizing - HTML', () => { let _mode: Modes.IMode; let onEnterSupport: Modes.IRichEditOnEnter; - (function() { + + suiteSetup(function() { _mode = new HTMLMode( { id: 'html' }, null, @@ -100,10 +108,10 @@ suite('Colorizing - HTML', () => { null ); - tokenizationSupport = _mode.tokenizationSupport; + tokenizationSupport = Modes.TokenizationRegistry.get(_mode.getId()); onEnterSupport = LanguageConfigurationRegistry.getOnEnterSupport(_mode.getId()); - })(); + }); test('Open Start Tag #1', () => { modesUtil.assertTokenization(tokenizationSupport, [{ @@ -694,7 +702,7 @@ suite('Colorizing - HTML', () => { }); test('onEnter 1', function() { - var model = Model.createFromString('