maxmustermann/vscode
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Move tokenizationSupport out of the mode instance and into TokenizationRegistry

Browse source
This commit is contained in:
Alex Dima 2016-09-16 10:44:13 +02:00
parent dff1153946
commit 45d383f268
59 changed files with 933 additions and 1493 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

139
src/vs/editor/browser/standalone/colorizer.ts
View file

@ -4,13 +4,12 @@
*--------------------------------------------------------------------------------------------*/
'use strict';
import {RunOnceScheduler} from 'vs/base/common/async';
import {IDisposable} from 'vs/base/common/lifecycle';
import {TPromise} from 'vs/base/common/winjs.base';
import {IModel} from 'vs/editor/common/editorCommon';
import {ILineTokens, IMode} from 'vs/editor/common/modes';
import {TokenizationRegistry, ITokenizationSupport} from 'vs/editor/common/modes';
import {IModeService} from 'vs/editor/common/services/modeService';
import {RenderLineOutput, renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer';
import {renderLine, RenderLineInput} from 'vs/editor/common/viewLayout/viewLineRenderer';
import {ViewLineToken} from 'vs/editor/common/core/viewLineToken';
export interface IColorizerOptions {
@ -40,7 +39,7 @@ export class Colorizer {
return this.colorize(modeService, text, mimeType, options).then(render, (err) => console.error(err), render);
}
private static _tokenizationSupportChangedPromise(target:IMode): TPromise<void> {
private static _tokenizationSupportChangedPromise(languageId:string): TPromise<void> {
let listener: IDisposable = null;
let stopListening = () => {
if (listener) {
@ -50,8 +49,8 @@ export class Colorizer {
};
return new TPromise<void>((c, e, p) => {
listener = target.addSupportChangedListener((e) => {
if (e.tokenizationSupport) {
listener = TokenizationRegistry.onDidChange((e) => {
if (e.languageId === languageId) {
stopListening();
c(void 0);
}
@ -60,64 +59,30 @@ export class Colorizer {
}
public static colorize(modeService:IModeService, text:string, mimeType:string, options:IColorizerOptions): TPromise<string> {
let lines = text.split('\n');
let languageId = modeService.getModeId(mimeType);
options = options || {};
if (typeof options.tabSize === 'undefined') {
options.tabSize = 4;
}
let lines = text.split('\n');
let c: (v:string)=>void;
let e: (err:any)=>void;
let p: (v:string)=>void;
let isCanceled = false;
let mode: IMode;
// Send out the event to create the mode
modeService.getOrCreateMode(languageId);
let result = new TPromise<string>((_c, _e, _p) => {
c = _c;
e = _e;
p = _p;
}, () => {
isCanceled = true;
let tokenizationSupport = TokenizationRegistry.get(languageId);
if (tokenizationSupport) {
return TPromise.as(_colorize(lines, options.tabSize, tokenizationSupport));
}
// wait 500ms for mode to load, then give up
return TPromise.any([this._tokenizationSupportChangedPromise(languageId), TPromise.timeout(500)]).then(_ => {
let tokenizationSupport = TokenizationRegistry.get(languageId);
if (tokenizationSupport) {
return _colorize(lines, options.tabSize, tokenizationSupport);
}
return _fakeColorize(lines, options.tabSize);
});
let colorize = new RunOnceScheduler(() => {
if (isCanceled) {
return;
}
let r = actualColorize(lines, mode, options.tabSize);
if (r.retokenize.length > 0) {
// There are retokenization requests
r.retokenize.forEach((p) => p.then(scheduleColorize));
p(r.result);
} else {
// There are no (more) retokenization requests
c(r.result);
}
}, 0);
let scheduleColorize = () => colorize.schedule();
modeService.getOrCreateMode(mimeType).then((_mode) => {
if (!_mode) {
e('Mode not found: "' + mimeType + '".');
return;
}
if (!_mode.tokenizationSupport) {
// wait 500ms for mode to load, then give up
TPromise.any([this._tokenizationSupportChangedPromise(_mode), TPromise.timeout(500)]).then(_ => {
if (!_mode.tokenizationSupport) {
e('Mode found ("' + _mode.getId() + '"), but does not support tokenization.');
return;
}
mode = _mode;
scheduleColorize();
});
return;
}
mode = _mode;
scheduleColorize();
});
return result;
}
public static colorizeLine(line:string, tokens:ViewLineToken[], tabSize:number = 4): string {
@ -141,32 +106,43 @@ export class Colorizer {
}
}
interface IActualColorizeResult {
result:string;
retokenize:TPromise<void>[];
function _colorize(lines:string[], tabSize:number, tokenizationSupport: ITokenizationSupport): string {
return _actualColorize(lines, tabSize, tokenizationSupport);
}
function actualColorize(lines:string[], mode:IMode, tabSize:number): IActualColorizeResult {
let tokenization = mode.tokenizationSupport,
html:string[] = [],
state = tokenization.getInitialState(),
i:number,
length:number,
line: string,
tokenizeResult: ILineTokens,
renderResult: RenderLineOutput,
retokenize: TPromise<void>[] = [];
function _fakeColorize(lines:string[], tabSize:number): string {
let html:string[] = [];
for (i = 0, length = lines.length; i < length; i++) {
line = lines[i];
for (let i = 0, length = lines.length; i < length; i++) {
let line = lines[i];
tokenizeResult = tokenization.tokenize(line, state);
if (tokenizeResult.retokenize) {
retokenize.push(tokenizeResult.retokenize);
}
let renderResult = renderLine(new RenderLineInput(
line,
tabSize,
0,
-1,
'none',
false,
[]
));
renderResult = renderLine(new RenderLineInput(
html = html.concat(renderResult.output);
html.push('<br/>');
}
return html.join('');
}
function _actualColorize(lines:string[], tabSize:number, tokenizationSupport: ITokenizationSupport): string {
let html:string[] = [];
let state = tokenizationSupport.getInitialState();
for (let i = 0, length = lines.length; i < length; i++) {
let line = lines[i];
let tokenizeResult = tokenizationSupport.tokenize(line, state);
let renderResult = renderLine(new RenderLineInput(
line,
tabSize,
0,
@ -182,8 +158,5 @@ function actualColorize(lines:string[], mode:IMode, tabSize:number): IActualColo
state = tokenizeResult.endState;
}
return {
result: html.join(''),
retokenize: retokenize
};
}
return html.join('');
}

2
src/vs/editor/browser/standalone/standaloneEditor.ts
View file

@ -148,7 +148,7 @@ export function createModel(value:string, language?:string, uri?:URI): IModel {
* Change the language for a model.
*/
export function setModelLanguage(model:IModel, language:string): void {
model.setMode(StaticServices.modeService.get().getOrCreateMode(language));
StaticServices.modelService.get().setMode(model, StaticServices.modeService.get().getOrCreateMode(language));
}
/**

10
src/vs/editor/browser/standalone/standaloneLanguages.ts
View file

@ -23,7 +23,7 @@ import {compile} from 'vs/editor/common/modes/monarch/monarchCompile';
import {createTokenizationSupport} from 'vs/editor/common/modes/monarch/monarchLexer';
import {LanguageConfigurationRegistry} from 'vs/editor/common/modes/languageConfigurationRegistry';
import {IMarkerData} from 'vs/platform/markers/common/markers';
import {TokenizationSupport2Adapter} from 'vs/editor/common/services/modeServiceImpl';
/**
* Register information about a new language.
*/
@ -67,7 +67,8 @@ export function setLanguageConfiguration(languageId:string, configuration:Langua
* Set the tokens provider for a language (manual implementation).
*/
export function setTokensProvider(languageId:string, provider:modes.TokensProvider): IDisposable {
return StaticServices.modeService.get().registerTokenizationSupport2(languageId, provider);
let adapter = new TokenizationSupport2Adapter(languageId, provider);
return modes.TokenizationRegistry.register(languageId, adapter);
}
/**
@ -75,9 +76,8 @@ export function setTokensProvider(languageId:string, provider:modes.TokensProvid
*/
export function setMonarchTokensProvider(languageId:string, languageDef:IMonarchLanguage): IDisposable {
let lexer = compile(languageId, languageDef);
return StaticServices.modeService.get().registerTokenizationSupport(languageId, (mode) => {
return createTokenizationSupport(StaticServices.modeService.get(), mode, lexer);
});
let adapter = createTokenizationSupport(StaticServices.modeService.get(), languageId, lexer);
return modes.TokenizationRegistry.register(languageId, adapter);
}
/**

7
src/vs/editor/common/commonCodeEditor.ts
View file

@ -46,9 +46,6 @@ export abstract class CommonCodeEditor extends EventEmitter implements editorCom
public onDidChangeModelOptions(listener: (e:editorCommon.IModelOptionsChangedEvent)=>void): IDisposable {
return this.addListener2(editorCommon.EventType.ModelOptionsChanged, listener);
}
public onDidChangeModelModeSupport(listener: (e:editorCommon.IModeSupportChangedEvent)=>void): IDisposable {
return this.addListener2(editorCommon.EventType.ModelModeSupportChanged, listener);
}
public onDidChangeModelDecorations(listener: (e:editorCommon.IModelDecorationsChangedEvent)=>void): IDisposable {
return this.addListener2(editorCommon.EventType.ModelDecorationsChanged, listener);
}
@ -856,10 +853,6 @@ export abstract class CommonCodeEditor extends EventEmitter implements editorCom
this.emit(editorCommon.EventType.ModelModeChanged, e);
break;
case editorCommon.EventType.ModelModeSupportChanged:
this.emit(editorCommon.EventType.ModelModeSupportChanged, e);
break;
case editorCommon.EventType.ModelRawContentChanged:
this.emit(editorCommon.EventType.ModelRawContentChanged, e);
break;

27
src/vs/editor/common/editorCommon.ts
View file

@ -1029,14 +1029,6 @@ export interface IConfigurationChangedEvent {
contribInfo: boolean;
}
/**
* An event describing that one or more supports of a mode have changed.
* @internal
*/
export interface IModeSupportChangedEvent {
tokenizationSupport:boolean;
}
/**
* Vertical Lane in the overview ruler of the editor.
*/
@ -1857,17 +1849,9 @@ export interface ITokenizedModel extends ITextModel {
/**
* Set the current language mode associated with the model.
*/
setMode(newMode:IMode|TPromise<IMode>): void;
/**
* A mode can be currently pending loading if a promise is used when constructing a model or calling setMode().
*
* If there is no currently pending loading mode, then the result promise will complete immediately.
* Otherwise, the result will complete once the currently pending loading mode is loaded.
* @internal
*/
whenModeIsReady(): TPromise<IMode>;
setMode(languageId:string): void;
/**
* Returns the true (inner-most) language mode at a given position.
@ -2214,10 +2198,6 @@ export interface IModel extends IReadOnlyModel, IEditableTextModel, ITextModelWi
* An event emitted when the contents of the model have changed.
*/
onDidChangeContent(listener: (e:IModelContentChangedEvent2)=>void): IDisposable;
/**
* @internal
*/
onDidChangeModeSupport(listener: (e:IModeSupportChangedEvent)=>void): IDisposable;
/**
* An event emitted when decorations of the model have changed.
*/
@ -3901,10 +3881,6 @@ export interface ICommonCodeEditor extends IEditor {
* An event emitted when the model of this editor has changed (e.g. `editor.setModel()`).
*/
onDidChangeModel(listener: (e:IModelChangedEvent)=>void): IDisposable;
/**
* @internal
*/
onDidChangeModelModeSupport(listener: (e:IModeSupportChangedEvent)=>void): IDisposable;
/**
* An event emitted when the decorations of the current model have changed.
*/
@ -4188,7 +4164,6 @@ export var EventType = {
ModelTokensChanged: 'modelTokensChanged',
ModelModeChanged: 'modelsModeChanged',
ModelModeSupportChanged: 'modelsModeSupportChanged',
ModelOptionsChanged: 'modelOptionsChanged',
ModelRawContentChanged: 'contentChanged',
ModelContentChanged2: 'contentChanged2',

6
src/vs/editor/common/model/compatMirrorModel.ts
View file

@ -5,11 +5,9 @@
'use strict';
import URI from 'vs/base/common/uri';
import {TPromise} from 'vs/base/common/winjs.base';
import * as editorCommon from 'vs/editor/common/editorCommon';
import {ModelLine} from 'vs/editor/common/model/modelLine';
import {TextModelWithTokens} from 'vs/editor/common/model/textModelWithTokens';
import {IMode} from 'vs/editor/common/modes';
import {ICompatMirrorModel} from 'vs/editor/common/services/resourceService';
export interface ICompatMirrorModelEvents {
@ -22,8 +20,8 @@ export class CompatMirrorModel extends TextModelWithTokens implements ICompatMir
protected _associatedResource:URI;
constructor(versionId:number, value:editorCommon.IRawText, mode:IMode|TPromise<IMode>, associatedResource?:URI) {
super(['changed', editorCommon.EventType.ModelDispose], value, mode);
constructor(versionId:number, value:editorCommon.IRawText, languageId:string, associatedResource?:URI) {
super(['changed', editorCommon.EventType.ModelDispose], value, languageId);
this._setVersionId(versionId);
this._associatedResource = associatedResource;

6
src/vs/editor/common/model/editableTextModel.ts
View file

@ -4,13 +4,11 @@
*--------------------------------------------------------------------------------------------*/
'use strict';
import {TPromise} from 'vs/base/common/winjs.base';
import {Range} from 'vs/editor/common/core/range';
import * as editorCommon from 'vs/editor/common/editorCommon';
import {EditStack} from 'vs/editor/common/model/editStack';
import {ILineEdit, ILineMarker, ModelLine} from 'vs/editor/common/model/modelLine';
import {DeferredEventsBuilder, TextModelWithDecorations} from 'vs/editor/common/model/textModelWithDecorations';
import {IMode} from 'vs/editor/common/modes';
import * as strings from 'vs/base/common/strings';
import {Selection} from 'vs/editor/common/core/selection';
import {IDisposable} from 'vs/base/common/lifecycle';
@ -50,10 +48,10 @@ export class EditableTextModel extends TextModelWithDecorations implements edito
private _trimAutoWhitespaceLines: number[];
constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, modeOrPromise:IMode|TPromise<IMode>) {
constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, languageId:string) {
allowedEventTypes.push(editorCommon.EventType.ModelRawContentChanged);
allowedEventTypes.push(editorCommon.EventType.ModelContentChanged2);
super(allowedEventTypes, rawText, modeOrPromise);
super(allowedEventTypes, rawText, languageId);
this._commandManager = new EditStack(this);

17
src/vs/editor/common/model/model.ts
View file

@ -5,14 +5,12 @@
'use strict';
import URI from 'vs/base/common/uri';
import {TPromise} from 'vs/base/common/winjs.base';
import {
EventType, IModel, ITextModelCreationOptions, IModeSupportChangedEvent, IModelDecorationsChangedEvent,
EventType, IModel, ITextModelCreationOptions, IModelDecorationsChangedEvent,
IModelOptionsChangedEvent, IModelModeChangedEvent, IRawText
} from 'vs/editor/common/editorCommon';
import {EditableTextModel} from 'vs/editor/common/model/editableTextModel';
import {TextModel} from 'vs/editor/common/model/textModel';
import {IMode} from 'vs/editor/common/modes';
import {IDisposable} from 'vs/base/common/lifecycle';
import {BulkListenerCallback} from 'vs/base/common/eventEmitter';
@ -36,9 +34,6 @@ var aliveModels:{[modelId:string]:boolean;} = {};
export class Model extends EditableTextModel implements IModel {
public onDidChangeModeSupport(listener: (e:IModeSupportChangedEvent)=>void): IDisposable {
return this.addListener2(EventType.ModelModeSupportChanged, listener);
}
public onDidChangeDecorations(listener: (e:IModelDecorationsChangedEvent)=>void): IDisposable {
return this.addListener2(EventType.ModelDecorationsChanged, listener);
}
@ -56,9 +51,9 @@ export class Model extends EditableTextModel implements IModel {
return super.addBulkListener(listener);
}
public static createFromString(text:string, options:ITextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, mode:IMode|TPromise<IMode> = null, uri:URI = null): Model {
public static createFromString(text:string, options:ITextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, languageId:string = null, uri:URI = null): Model {
let rawText = TextModel.toRawText(text, options);
return new Model(rawText, mode, uri);
return new Model(rawText, languageId, uri);
}
public id:string;
@ -79,10 +74,8 @@ export class Model extends EditableTextModel implements IModel {
* The resource associated with this model. If the value is not provided an
* unique in memory URL is constructed as the associated resource.
*/
constructor(rawText:IRawText, modeOrPromise:IMode|TPromise<IMode>, associatedResource:URI=null) {
super([
EventType.ModelDispose
], rawText, modeOrPromise);
constructor(rawText:IRawText, languageId:string, associatedResource:URI=null) {
super([ EventType.ModelDispose ], rawText, languageId);
// Generate a new unique model id
MODEL_ID++;

6
src/vs/editor/common/model/textModelWithDecorations.ts
View file

@ -7,12 +7,10 @@
import {onUnexpectedError} from 'vs/base/common/errors';
import {MarkedString, markedStringsEquals} from 'vs/base/common/htmlContent';
import * as strings from 'vs/base/common/strings';
import {TPromise} from 'vs/base/common/winjs.base';
import {IdGenerator} from 'vs/base/common/idGenerator';
import {Range} from 'vs/editor/common/core/range';
import * as editorCommon from 'vs/editor/common/editorCommon';
import {TextModelWithTrackedRanges} from 'vs/editor/common/model/textModelWithTrackedRanges';
import {IMode} from 'vs/editor/common/modes';
export class DeferredEventsBuilder {
@ -96,9 +94,9 @@ export class TextModelWithDecorations extends TextModelWithTrackedRanges impleme
private decorations:IInternalDecorationsMap;
private rangeIdToDecorationId:IRangeIdToDecorationIdMap;
constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, modeOrPromise:IMode|TPromise<IMode>) {
constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, languageId:string) {
allowedEventTypes.push(editorCommon.EventType.ModelDecorationsChanged);
super(allowedEventTypes, rawText, modeOrPromise);
super(allowedEventTypes, rawText, languageId);
// Initialize decorations
this._decorationIdGenerator = new IdGenerator((++_INSTANCE_COUNT) + ';');

6
src/vs/editor/common/model/textModelWithMarkers.ts
View file

@ -4,13 +4,11 @@
*--------------------------------------------------------------------------------------------*/
'use strict';
import {TPromise} from 'vs/base/common/winjs.base';
import {IdGenerator} from 'vs/base/common/idGenerator';
import {Position} from 'vs/editor/common/core/position';
import {IModelContentChangedFlushEvent, IRawText, IReadOnlyLineMarker, ITextModelWithMarkers} from 'vs/editor/common/editorCommon';
import {ILineMarker, ModelLine} from 'vs/editor/common/model/modelLine';
import {TextModelWithTokens} from 'vs/editor/common/model/textModelWithTokens';
import {IMode} from 'vs/editor/common/modes';
export interface IMarkerIdToMarkerMap {
[key:string]:ILineMarker;
@ -51,8 +49,8 @@ export class TextModelWithMarkers extends TextModelWithTokens implements ITextMo
private _markerIdGenerator: IdGenerator;
protected _markerIdToMarker: IMarkerIdToMarkerMap;
constructor(allowedEventTypes:string[], rawText:IRawText, modeOrPromise:IMode|TPromise<IMode>) {
super(allowedEventTypes, rawText, modeOrPromise);
constructor(allowedEventTypes:string[], rawText:IRawText, languageId:string) {
super(allowedEventTypes, rawText, languageId);
this._markerIdGenerator = new IdGenerator((++_INSTANCE_COUNT) + ';');
this._markerIdToMarker = {};
}

380
src/vs/editor/common/model/textModelWithTokens.ts
View file

@ -5,20 +5,18 @@
'use strict';
import * as nls from 'vs/nls';
import {RunOnceScheduler} from 'vs/base/common/async';
import {onUnexpectedError} from 'vs/base/common/errors';
import {IDisposable, dispose} from 'vs/base/common/lifecycle';
import {IDisposable} from 'vs/base/common/lifecycle';
import {StopWatch} from 'vs/base/common/stopwatch';
import * as timer from 'vs/base/common/timer';
import {TPromise} from 'vs/base/common/winjs.base';
import {Range} from 'vs/editor/common/core/range';
import * as editorCommon from 'vs/editor/common/editorCommon';
import {ModelLine} from 'vs/editor/common/model/modelLine';
import {TextModel} from 'vs/editor/common/model/textModel';
import {WordHelper} from 'vs/editor/common/model/textModelWithTokensHelpers';
import {TokenIterator} from 'vs/editor/common/model/tokenIterator';
import {ILineContext, ILineTokens, IMode, IState} from 'vs/editor/common/modes';
import {NullMode, NullState, nullTokenize} from 'vs/editor/common/modes/nullMode';
import {ITokenizationSupport, ILineContext, ILineTokens, IMode, IState, TokenizationRegistry} from 'vs/editor/common/modes';
import {NULL_MODE_ID, nullTokenize} from 'vs/editor/common/modes/nullMode';
import {ignoreBracketsInToken} from 'vs/editor/common/modes/supports';
import {BracketsUtils} from 'vs/editor/common/modes/supports/richEditBrackets';
import {ModeTransition} from 'vs/editor/common/core/modeTransition';
@ -27,101 +25,16 @@ import {Position} from 'vs/editor/common/core/position';
import {LanguageConfigurationRegistry} from 'vs/editor/common/modes/languageConfigurationRegistry';
import {Token} from 'vs/editor/common/core/token';
class ModeToModelBinder implements IDisposable {
class Mode implements IMode {
private _modePromise:TPromise<IMode>;
private _externalModePromise:TPromise<boolean>;
private _externalModePromise_c:(value:boolean)=>void;
private _externalModePromise_e:(err:any)=>void;
private _model:TextModelWithTokens;
private _isDisposed:boolean;
private _languageId:string;
constructor(modePromise:TPromise<IMode>, model:TextModelWithTokens) {
this._modePromise = modePromise;
// Create an external mode promise that fires after the mode is set to the model
this._externalModePromise = new TPromise<boolean>((c, e, p) => {
this._externalModePromise_c = c;
this._externalModePromise_e = e;
}, () => {
// this promise cannot be canceled
});
this._model = model;
this._isDisposed = false;
// Ensure asynchronicity
TPromise.timeout(0).then(() => {
return this._modePromise;
}).then((mode:IMode) => {
if (this._isDisposed) {
this._externalModePromise_c(false);
return;
}
var model = this._model;
this.dispose();
model.setMode(mode);
model._warmUpTokens();
this._externalModePromise_c(true);
}).done(null, (err) => {
this._externalModePromise_e(err);
onUnexpectedError(err);
});
constructor(languageId:string) {
this._languageId = languageId;
}
public getModePromise(): TPromise<boolean> {
return this._externalModePromise;
}
public dispose(): void {
this._modePromise = null;
this._model = null;
this._isDisposed = true;
}
}
export interface IRetokenizeRequest extends IDisposable {
isFulfilled: boolean;
/**
* If null, the entire model will be retokenzied, use null with caution
*/
getRange(): editorCommon.IRange;
}
export class FullModelRetokenizer implements IRetokenizeRequest {
public isFulfilled: boolean;
protected _model:TextModelWithTokens;
private _retokenizePromise:TPromise<void>;
private _isDisposed: boolean;
constructor(retokenizePromise:TPromise<void>, model:TextModelWithTokens) {
this._retokenizePromise = retokenizePromise;
this._model = model;
this._isDisposed = false;
this.isFulfilled = false;
// Ensure asynchronicity
TPromise.timeout(0).then(() => {
return this._retokenizePromise;
}).then(() => {
if (this._isDisposed) {
return;
}
this.isFulfilled = true;
this._model.onRetokenizerFulfilled();
}).done(null, onUnexpectedError);
}
public getRange(): editorCommon.IRange {
return null;
}
public dispose(): void {
this._retokenizePromise = null;
this._model = null;
this._isDisposed = true;
getId(): string {
return this._languageId;
}
}
@ -162,75 +75,45 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
private static MODE_TOKENIZATION_FAILED_MSG = nls.localize('mode.tokenizationSupportFailed', "The mode has failed while tokenizing the input.");
private _mode: IMode;
private _modeListener: IDisposable;
private _modeToModelBinder:ModeToModelBinder;
private _languageId: string;
private _tokenizationListener: IDisposable;
private _tokenizationSupport: ITokenizationSupport;
private _tokensInflatorMap:TokensInflatorMap;
private _invalidLineStartIndex:number;
private _lastState:IState;
private _revalidateTokensTimeout:number;
private _scheduleRetokenizeNow: RunOnceScheduler;
private _retokenizers:IRetokenizeRequest[];
constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, modeOrPromise:IMode|TPromise<IMode>) {
constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, languageId:string) {
allowedEventTypes.push(editorCommon.EventType.ModelTokensChanged);
allowedEventTypes.push(editorCommon.EventType.ModelModeChanged);
allowedEventTypes.push(editorCommon.EventType.ModelModeSupportChanged);
super(allowedEventTypes, rawText);
this._mode = null;
this._modeListener = null;
this._modeToModelBinder = null;
this._languageId = languageId || NULL_MODE_ID;
this._tokenizationListener = TokenizationRegistry.onDidChange((e) => {
if (e.languageId !== this._languageId) {
return;
}
this._resetTokenizationState();
this.emitModelTokensChangedEvent(1, this.getLineCount());
});
this._tokensInflatorMap = null;
this._invalidLineStartIndex = 0;
this._lastState = null;
this._revalidateTokensTimeout = -1;
this._scheduleRetokenizeNow = null;
this._retokenizers = null;
if (!modeOrPromise) {
this._mode = new NullMode();
} else if (TPromise.is(modeOrPromise)) {
// TODO@Alex: To avoid mode id changes, we check if this promise is resolved
let promiseValue = <IMode>(<any>modeOrPromise)._value;
if (promiseValue && typeof promiseValue.getId === 'function') {
// The promise is already resolved
this._mode = this._massageMode(promiseValue);
this._resetModeListener(this._mode);
} else {
var modePromise = <TPromise<IMode>>modeOrPromise;
this._modeToModelBinder = new ModeToModelBinder(modePromise, this);
this._mode = new NullMode();
}
} else {
this._mode = this._massageMode(<IMode>modeOrPromise);
this._resetModeListener(this._mode);
}
this._revalidateTokensTimeout = -1;
this._scheduleRetokenizeNow = new RunOnceScheduler(() => this._retokenizeNow(), 200);
this._retokenizers = [];
this._resetTokenizationState();
}
public dispose(): void {
if (this._modeToModelBinder) {
this._modeToModelBinder.dispose();
this._modeToModelBinder = null;
}
this._resetModeListener(null);
this._tokenizationListener.dispose();
this._clearTimers();
this._mode = null;
this._lastState = null;
this._tokensInflatorMap = null;
this._retokenizers = dispose(this._retokenizers);
this._scheduleRetokenizeNow.dispose();
super.dispose();
}
@ -239,130 +122,38 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
return false;
}
private _massageMode(mode: IMode): IMode {
if (this.isTooLargeForHavingAMode()) {
return new NullMode();
}
if (this.isTooLargeForHavingARichMode()) {
return mode.toSimplifiedMode();
}
return mode;
}
public whenModeIsReady(): TPromise<IMode> {
if (this._modeToModelBinder) {
// Still waiting for some mode to load
return this._modeToModelBinder.getModePromise().then(() => this._mode);
}
return TPromise.as(this._mode);
}
public onRetokenizerFulfilled(): void {
this._scheduleRetokenizeNow.schedule();
}
private _retokenizeNow(): void {
var fulfilled = this._retokenizers.filter(r => r.isFulfilled);
this._retokenizers = this._retokenizers.filter(r => !r.isFulfilled);
var hasFullModel = false;
for (var i = 0; i < fulfilled.length; i++) {
if (!fulfilled[i].getRange()) {
hasFullModel = true;
}
}
if (hasFullModel) {
// Just invalidate all the lines
for (var i = 0, len = this._lines.length; i < len; i++) {
this._lines[i].isInvalid = true;
}
this._invalidLineStartIndex = 0;
} else {
var minLineNumber = Number.MAX_VALUE;
for (var i = 0; i < fulfilled.length; i++) {
var range = fulfilled[i].getRange();
minLineNumber = Math.min(minLineNumber, range.startLineNumber);
for (var lineNumber = range.startLineNumber; lineNumber <= range.endLineNumber; lineNumber++) {
this._lines[lineNumber - 1].isInvalid = true;
}
}
if (minLineNumber - 1 < this._invalidLineStartIndex) {
if (this._invalidLineStartIndex < this._lines.length) {
this._lines[this._invalidLineStartIndex].isInvalid = true;
}
this._invalidLineStartIndex = minLineNumber - 1;
}
}
this._beginBackgroundTokenization();
for (var i = 0; i < fulfilled.length; i++) {
fulfilled[i].dispose();
}
}
protected _createRetokenizer(retokenizePromise:TPromise<void>, lineNumber:number): IRetokenizeRequest {
return new FullModelRetokenizer(retokenizePromise, this);
}
protected _resetValue(e:editorCommon.IModelContentChangedFlushEvent, newValue:editorCommon.IRawText): void {
super._resetValue(e, newValue);
// Cancel tokenization, clear all tokens and begin tokenizing
this._resetTokenizationState();
}
protected _resetMode(e:editorCommon.IModelModeChangedEvent, newMode:IMode): void {
// Cancel tokenization, clear all tokens and begin tokenizing
this._mode = newMode;
this._resetModeListener(newMode);
this._resetTokenizationState();
this.emitModelTokensChangedEvent(1, this.getLineCount());
}
private _resetModeListener(newMode: IMode): void {
if (this._modeListener) {
this._modeListener.dispose();
this._modeListener = null;
}
if (newMode && typeof newMode.addSupportChangedListener === 'function') {
this._modeListener = newMode.addSupportChangedListener( (e) => this._onModeSupportChanged(e) );
}
}
private _onModeSupportChanged(e: editorCommon.IModeSupportChangedEvent): void {
this._emitModelModeSupportChangedEvent(e);
if (e.tokenizationSupport) {
this._resetTokenizationState();
this.emitModelTokensChangedEvent(1, this.getLineCount());
}
}
protected _resetTokenizationState(): void {
this._retokenizers = dispose(this._retokenizers);
this._scheduleRetokenizeNow.cancel();
this._clearTimers();
for (var i = 0; i < this._lines.length; i++) {
for (let i = 0; i < this._lines.length; i++) {
this._lines[i].resetTokenizationState();
}
// Initialize tokenization states
var initialState:IState = null;
if (this._mode.tokenizationSupport) {
this._tokenizationSupport = null;
if (!this.isTooLargeForHavingAMode()) {
this._tokenizationSupport = TokenizationRegistry.get(this._languageId);
}
if (this._tokenizationSupport) {
let initialState:IState = null;
try {
initialState = this._mode.tokenizationSupport.getInitialState();
initialState = this._tokenizationSupport.getInitialState();
} catch (e) {
e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG;
onUnexpectedError(e);
this._mode = new NullMode();
this._tokenizationSupport = null;
}
if (initialState) {
this._lines[0].setState(initialState);
}
}
if (!initialState) {
initialState = new NullState(this._mode, null);
}
this._lines[0].setState(initialState);
this._lastState = null;
this._tokensInflatorMap = new TokensInflatorMap();
this._invalidLineStartIndex = 0;
@ -403,38 +194,37 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
}
public getMode(): IMode {
return this._mode;
return new Mode(this._languageId);
}
public getModeId(): string {
return this.getMode().getId();
}
public setMode(newModeOrPromise:IMode|TPromise<IMode>): void {
if (!newModeOrPromise) {
public setMode(languageId:string): void {
if (this._languageId === languageId) {
// There's nothing to do
return;
}
if (this._modeToModelBinder) {
this._modeToModelBinder.dispose();
this._modeToModelBinder = null;
}
if (TPromise.is(newModeOrPromise)) {
this._modeToModelBinder = new ModeToModelBinder(<TPromise<IMode>>newModeOrPromise, this);
} else {
var actualNewMode = this._massageMode(<IMode>newModeOrPromise);
if (this._mode !== actualNewMode) {
var e2:editorCommon.IModelModeChangedEvent = {
oldMode: this._mode,
newMode: actualNewMode
};
this._resetMode(e2, actualNewMode);
this._emitModelModeChangedEvent(e2);
}
}
let e:editorCommon.IModelModeChangedEvent = {
oldMode: new Mode(this._languageId),
newMode: new Mode(languageId)
};
this._languageId = languageId;
// Cancel tokenization, clear all tokens and begin tokenizing
this._resetTokenizationState();
this.emitModelTokensChangedEvent(1, this.getLineCount());
this._emitModelModeChangedEvent(e);
}
public getModeIdAtPosition(_lineNumber:number, _column:number): string {
if (!this._tokenizationSupport) {
return this.getModeId();
}
var validPosition = this.validatePosition({
lineNumber: _lineNumber,
column: _column
@ -444,9 +234,9 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
var column = validPosition.column;
if (column === 1) {
return this.getStateBeforeLine(lineNumber).getMode().getId();
return this.getStateBeforeLine(lineNumber).getModeId();
} else if (column === this.getLineMaxColumn(lineNumber)) {
return this.getStateAfterLine(lineNumber).getMode().getId();
return this.getStateAfterLine(lineNumber).getModeId();
} else {
var modeTransitions = this._getLineModeTransitions(lineNumber);
var modeTransitionIndex = ModeTransition.findIndexInSegmentsArray(modeTransitions, column - 1);
@ -567,6 +357,11 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
}
private _updateTokensUntilLine(lineNumber:number, emitEvents:boolean): void {
if (!this._tokenizationSupport) {
this._invalidLineStartIndex = this._lines.length;
return;
}
var linesLength = this._lines.length;
var endLineIndex = lineNumber - 1;
var stopLineTokenizationAfter = 1000000000; // 1 billion, if a line is so long, you have other trouble :).
@ -578,32 +373,26 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
var endStateIndex = lineIndex + 1;
var r:ILineTokens = null;
var text = this._lines[lineIndex].text;
if (this._mode.tokenizationSupport) {
try {
// Tokenize only the first X characters
r = this._mode.tokenizationSupport.tokenize(this._lines[lineIndex].text, this._lines[lineIndex].getState(), 0, stopLineTokenizationAfter);
} catch (e) {
e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG;
onUnexpectedError(e);
}
try {
// Tokenize only the first X characters
r = this._tokenizationSupport.tokenize(this._lines[lineIndex].text, this._lines[lineIndex].getState(), 0, stopLineTokenizationAfter);
} catch (e) {
e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG;
onUnexpectedError(e);
}
if (r && r.retokenize) {
this._retokenizers.push(this._createRetokenizer(r.retokenize, lineIndex + 1));
}
if (r && r.tokens && r.tokens.length > 0) {
// Cannot have a stop offset before the last token
r.actualStopOffset = Math.max(r.actualStopOffset, r.tokens[r.tokens.length - 1].startIndex + 1);
}
if (r && r.tokens && r.tokens.length > 0) {
// Cannot have a stop offset before the last token
r.actualStopOffset = Math.max(r.actualStopOffset, r.tokens[r.tokens.length - 1].startIndex + 1);
}
if (r && r.actualStopOffset < text.length) {
// Treat the rest of the line (if above limit) as one default token
r.tokens.push(new Token(r.actualStopOffset, ''));
if (r && r.actualStopOffset < text.length) {
// Treat the rest of the line (if above limit) as one default token
r.tokens.push(new Token(r.actualStopOffset, ''));
// Use as end state the starting state
r.endState = this._lines[lineIndex].getState();
}
// Use as end state the starting state
r.endState = this._lines[lineIndex].getState();
}
if (!r) {
@ -617,10 +406,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
r.modeTransitions.push(new ModeTransition(0, this.getModeId()));
}
this._lines[lineIndex].setTokens(this._tokensInflatorMap, r.tokens, this.getModeId(), r.modeTransitions);
if (this._lines[lineIndex].isInvalid) {
this._lines[lineIndex].isInvalid = false;
}
this._lines[lineIndex].isInvalid = false;
if (endStateIndex < linesLength) {
if (this._lines[endStateIndex].getState() !== null && r.endState.equals(this._lines[endStateIndex].getState())) {
@ -673,12 +459,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
}
}
private _emitModelModeSupportChangedEvent(e:editorCommon.IModeSupportChangedEvent): void {
if (!this._isDisposing) {
this.emit(editorCommon.EventType.ModelModeSupportChanged, e);
}
}
// Having tokens allows implementing additional helper methods
_lineIsTokenized(lineNumber:number): boolean {

39
src/vs/editor/common/model/textModelWithTrackedRanges.ts
View file

@ -4,14 +4,11 @@
*--------------------------------------------------------------------------------------------*/
'use strict';
import {TPromise} from 'vs/base/common/winjs.base';
import {IdGenerator} from 'vs/base/common/idGenerator';
import {Range} from 'vs/editor/common/core/range';
import * as editorCommon from 'vs/editor/common/editorCommon';
import {ILineMarker} from 'vs/editor/common/model/modelLine';
import {INewMarker, TextModelWithMarkers} from 'vs/editor/common/model/textModelWithMarkers';
import {FullModelRetokenizer, IRetokenizeRequest} from 'vs/editor/common/model/textModelWithTokens';
import {IMode} from 'vs/editor/common/modes';
import {Position} from 'vs/editor/common/core/position';
interface ITrackedRange {
@ -28,34 +25,6 @@ interface IMarkerIdToRangeIdMap {
[key:string]:string;
}
class TrackedRangeModelRetokenizer extends FullModelRetokenizer {
private trackedRangeId: string;
constructor(retokenizePromise:TPromise<void>, lineNumber:number, model:TextModelWithTrackedRanges) {
super(retokenizePromise, model);
this.trackedRangeId = model.addTrackedRange({
startLineNumber: lineNumber,
startColumn : 1,
endLineNumber: lineNumber,
endColumn: model.getLineMaxColumn(lineNumber)
}, editorCommon.TrackedRangeStickiness.AlwaysGrowsWhenTypingAtEdges);
}
public getRange(): editorCommon.IRange {
return (<TextModelWithTrackedRanges>this._model).getTrackedRange(this.trackedRangeId);
}
public dispose(): void {
var model = (<TextModelWithTrackedRanges>this._model);
// if this .dispose() is being called as part of the model.dispose(), then the tracked ranges might no longer be available (e.g. throw exceptions)
if (model.isValidTrackedRange(this.trackedRangeId)) {
model.removeTrackedRange(this.trackedRangeId);
}
super.dispose();
}
}
class TrackedRange implements ITrackedRange {
id:string;
startMarkerId:string;
@ -77,18 +46,14 @@ export class TextModelWithTrackedRanges extends TextModelWithMarkers implements
private _markerIdToRangeId:IMarkerIdToRangeIdMap;
private _multiLineTrackedRanges: { [key:string]: boolean; };
constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, modeOrPromise:IMode|TPromise<IMode>) {
super(allowedEventTypes, rawText, modeOrPromise);
constructor(allowedEventTypes:string[], rawText:editorCommon.IRawText, languageId:string) {
super(allowedEventTypes, rawText, languageId);
this._rangeIdGenerator = new IdGenerator((++_INSTANCE_COUNT) + ';');
this._ranges = {};
this._markerIdToRangeId = {};
this._multiLineTrackedRanges = {};
}
protected _createRetokenizer(retokenizePromise:TPromise<void>, lineNumber:number): IRetokenizeRequest {
return new TrackedRangeModelRetokenizer(retokenizePromise, lineNumber, this);
}
public dispose(): void {
this._ranges = null;
this._markerIdToRangeId = null;

79
src/vs/editor/common/modes.ts
View file

@ -16,6 +16,7 @@ import LanguageFeatureRegistry from 'vs/editor/common/modes/languageFeatureRegis
import {CancellationToken} from 'vs/base/common/cancellation';
import {Position} from 'vs/editor/common/core/position';
import {Range} from 'vs/editor/common/core/range';
import Event, {Emitter} from 'vs/base/common/event';
/**
* @internal
@ -32,7 +33,7 @@ export interface ITokenizationResult {
export interface IState {
clone():IState;
equals(other:IState):boolean;
getMode():IMode;
getModeId():string;
tokenize(stream:IStream):ITokenizationResult;
getStateData(): IState;
setStateData(state:IState):void;
@ -186,28 +187,6 @@ export interface IMode {
getId(): string;
/**
* Return a mode "similar" to this one that strips any "smart" supports.
* @internal
*/
toSimplifiedMode(): IMode;
/**
* @internal
*/
addSupportChangedListener?(callback: (e: editorCommon.IModeSupportChangedEvent) => void): IDisposable;
/**
* Register a support by name. Only optional.
* @internal
*/
setTokenizationSupport?<T>(callback:(mode:IMode)=>T): IDisposable;
/**
* Optional adapter to support tokenization.
* @internal
*/
tokenizationSupport?: ITokenizationSupport;
}
/**
@ -218,7 +197,6 @@ export interface ILineTokens {
actualStopOffset: number;
endState: IState;
modeTransitions: ModeTransition[];
retokenize?:TPromise<void>;
}
/**
@ -1022,3 +1000,56 @@ export const OnTypeFormattingEditProviderRegistry = new LanguageFeatureRegistry<
* @internal
*/
export const LinkProviderRegistry = new LanguageFeatureRegistry<LinkProvider>();
/**
* @internal
*/
export interface ITokenizationSupportChangedEvent {
languageId: string;
}
/**
* @internal
*/
export class TokenizationRegistryImpl {
private _map: {[languageId:string]:ITokenizationSupport};
private _onDidChange: Emitter<ITokenizationSupportChangedEvent> = new Emitter<ITokenizationSupportChangedEvent>();
public onDidChange: Event<ITokenizationSupportChangedEvent> = this._onDidChange.event;
constructor() {
this._map = Object.create(null);
}
/**
* Fire a change event for a language.
* This is useful for languages that embed other languages.
*/
public fire(languageId:string): void {
this._onDidChange.fire({ languageId: languageId });
}
public register(languageId:string, support:ITokenizationSupport): IDisposable {
this._map[languageId] = support;
this.fire(languageId);
return {
dispose: () => {
if (this._map[languageId] !== support) {
return;
}
delete this._map[languageId];
this.fire(languageId);
}
};
}
public get(languageId:string): ITokenizationSupport {
return (this._map[languageId] || null);
}
}
/**
* @internal
*/
export const TokenizationRegistry = new TokenizationRegistryImpl();

14
src/vs/editor/common/modes/TMState.ts
View file

@ -4,25 +4,25 @@
*--------------------------------------------------------------------------------------------*/
'use strict';
import {IMode, IState, ITokenizationResult} from 'vs/editor/common/modes';
import {IState, ITokenizationResult} from 'vs/editor/common/modes';
import {AbstractState} from 'vs/editor/common/modes/abstractState';
import {StackElement} from 'vscode-textmate';
export class TMState implements IState {
private _mode: IMode;
private _modeId: string;
private _parentEmbedderState: IState;
private _ruleStack: StackElement;
constructor(mode: IMode, parentEmbedderState: IState, ruleStack: StackElement) {
this._mode = mode;
constructor(modeId: string, parentEmbedderState: IState, ruleStack: StackElement) {
this._modeId = modeId;
this._parentEmbedderState = parentEmbedderState;
this._ruleStack = ruleStack;
}
public clone():TMState {
let parentEmbedderStateClone = AbstractState.safeClone(this._parentEmbedderState);
return new TMState(this._mode, parentEmbedderStateClone, this._ruleStack);
return new TMState(this._modeId, parentEmbedderStateClone, this._ruleStack);
}
public equals(other:IState):boolean {
@ -46,8 +46,8 @@ export class TMState implements IState {
return this._ruleStack.equals(otherState._ruleStack);
}
public getMode():IMode {
return this._mode;
public getModeId():string {
return this._modeId;
}
public tokenize(stream:any):ITokenizationResult {

74
src/vs/editor/common/modes/abstractMode.ts
View file

@ -4,13 +4,10 @@
*--------------------------------------------------------------------------------------------*/
'use strict';
import {EventEmitter} from 'vs/base/common/eventEmitter';
import {IDisposable} from 'vs/base/common/lifecycle';
import {TPromise} from 'vs/base/common/winjs.base';
import {AsyncDescriptor1, createAsyncDescriptor1} from 'vs/platform/instantiation/common/descriptors';
import {IInstantiationService, optional} from 'vs/platform/instantiation/common/instantiation';
import {IConfigurationService} from 'vs/platform/configuration/common/configuration';
import {IModeSupportChangedEvent} from 'vs/editor/common/editorCommon';
import * as modes from 'vs/editor/common/modes';
import {TextualSuggestSupport} from 'vs/editor/common/modes/supports/suggestSupport';
import {IEditorWorkerService} from 'vs/editor/common/services/editorWorkerService';
@ -81,44 +78,14 @@ export class ModeWorkerManager<W> {
export abstract class AbstractMode implements modes.IMode {
private _modeId: string;
private _eventEmitter: EventEmitter;
private _simplifiedMode: modes.IMode;
constructor(modeId:string) {
this._modeId = modeId;
this._eventEmitter = new EventEmitter();
this._simplifiedMode = null;
}
public getId(): string {
return this._modeId;
}
public toSimplifiedMode(): modes.IMode {
if (!this._simplifiedMode) {
this._simplifiedMode = new SimplifiedMode(this);
}
return this._simplifiedMode;
}
public addSupportChangedListener(callback: (e: IModeSupportChangedEvent) => void) : IDisposable {
return this._eventEmitter.addListener2('modeSupportChanged', callback);
}
public setTokenizationSupport<T>(callback:(mode:modes.IMode) => T) : IDisposable {
let supportImpl = callback(this);
this['tokenizationSupport'] = supportImpl;
this._eventEmitter.emit('modeSupportChanged', _createModeSupportChangedEvent());
return {
dispose: () => {
if (this['tokenizationSupport'] === supportImpl) {
delete this['tokenizationSupport'];
this._eventEmitter.emit('modeSupportChanged', _createModeSupportChangedEvent());
}
}
};
}
}
export abstract class CompatMode extends AbstractMode implements ICompatMode {
@ -136,41 +103,6 @@ export abstract class CompatMode extends AbstractMode implements ICompatMode {
}
class SimplifiedMode implements modes.IMode {
tokenizationSupport: modes.ITokenizationSupport;
private _sourceMode: modes.IMode;
private _eventEmitter: EventEmitter;
private _id: string;
constructor(sourceMode: modes.IMode) {
this._sourceMode = sourceMode;
this._eventEmitter = new EventEmitter();
this._id = 'vs.editor.modes.simplifiedMode:' + sourceMode.getId();
this._assignSupports();
if (this._sourceMode.addSupportChangedListener) {
this._sourceMode.addSupportChangedListener((e) => {
this._assignSupports();
this._eventEmitter.emit('modeSupportChanged', e);
});
}
}
public getId(): string {
return this._id;
}
public toSimplifiedMode(): modes.IMode {
return this;
}
private _assignSupports(): void {
this.tokenizationSupport = this._sourceMode.tokenizationSupport;
}
}
export function isDigit(character:string, base:number): boolean {
let c = character.charCodeAt(0);
switch (base) {
@ -223,9 +155,3 @@ export class FrankensteinMode extends AbstractMode {
}
}
}
function _createModeSupportChangedEvent(): IModeSupportChangedEvent {
return {
tokenizationSupport: true
};
}

28
src/vs/editor/common/modes/abstractState.ts
View file

@ -4,33 +4,31 @@
*--------------------------------------------------------------------------------------------*/
'use strict';
import {IMode, IState, IStream, ITokenizationResult} from 'vs/editor/common/modes';
import {IState, IStream, ITokenizationResult} from 'vs/editor/common/modes';
export class AbstractState implements IState {
export abstract class AbstractState implements IState {
private mode:IMode;
private modeId:string;
private stateData:IState;
constructor(mode:IMode, stateData:IState = null) {
this.mode = mode;
constructor(modeId:string, stateData:IState = null) {
this.modeId = modeId;
this.stateData = stateData;
}
public getMode():IMode {
return this.mode;
public getModeId():string {
return this.modeId;
}
public clone():IState {
public clone():AbstractState {
var result:AbstractState = this.makeClone();
result.initializeFrom(this);
return result;
}
public makeClone():AbstractState {
throw new Error('Abstract Method');
}
protected abstract makeClone():AbstractState;
public initializeFrom(other:AbstractState): void {
protected initializeFrom(other:AbstractState): void {
this.stateData = other.stateData !== null ? other.stateData.clone() : null;
}
@ -43,7 +41,7 @@ export class AbstractState implements IState {
}
public equals(other:IState):boolean {
if (other === null || this.mode !== other.getMode()) {
if (other === null || this.modeId !== other.getModeId()) {
return false;
}
if (other instanceof AbstractState) {
@ -52,9 +50,7 @@ export class AbstractState implements IState {
return false;
}
public tokenize(stream:IStream):ITokenizationResult {
throw new Error('Abstract Method');
}
public abstract tokenize(stream:IStream):ITokenizationResult;
public static safeEquals(a: IState, b: IState): boolean {
if (a === null && b === null) {

4
src/vs/editor/common/modes/modesRegistry.ts
View file

@ -75,8 +75,10 @@ export class EditorModesRegistry {
export var ModesRegistry = new EditorModesRegistry();
Registry.add(Extensions.ModesRegistry, ModesRegistry);
export const PLAINTEXT_MODE_ID = 'plaintext';
ModesRegistry.registerLanguage({
id: 'plaintext',
id: PLAINTEXT_MODE_ID,
extensions: ['.txt', '.gitignore'],
aliases: [nls.localize('plainText.alias', "Plain Text"), 'text'],
mimetypes: ['text/plain']

42
src/vs/editor/common/modes/monarch/monarchLexer.ts
View file

@ -13,7 +13,7 @@ import * as modes from 'vs/editor/common/modes';
import {AbstractState} from 'vs/editor/common/modes/abstractState';
import {LineStream} from 'vs/editor/common/modes/lineStream';
import * as monarchCommon from 'vs/editor/common/modes/monarch/monarchCommon';
import {IEnteringNestedModeData, TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
import {IModeLocator, TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
import {IModeService} from 'vs/editor/common/services/modeService';
/**
@ -39,8 +39,8 @@ export class MonarchLexer extends AbstractState {
private groupMatched: string[];
private groupRule: monarchCommon.IRule;
constructor(mode: modes.IMode, modeService:IModeService, lexer: monarchCommon.ILexer, stack?: string[], embeddedMode?: string) {
super(mode);
constructor(modeId: string, modeService:IModeService, lexer: monarchCommon.ILexer, stack?: string[], embeddedMode?: string) {
super(modeId);
this.id = MonarchLexer.ID++; // for debugging, assigns unique id to each instance
this.modeService = modeService;
@ -61,7 +61,7 @@ export class MonarchLexer extends AbstractState {
}
public makeClone(): MonarchLexer {
return new MonarchLexer(this.getMode(), this.modeService, this.lexer, this.stack.slice(0), this.embeddedMode);
return new MonarchLexer(this.getModeId(), this.modeService, this.lexer, this.stack.slice(0), this.embeddedMode);
}
public equals(other: modes.IState): boolean {
@ -386,10 +386,10 @@ function findBracket(lexer: monarchCommon.ILexer, matched: string) {
return null;
}
export function createTokenizationSupport(modeService:IModeService, mode:modes.IMode, lexer: monarchCommon.ILexer): modes.ITokenizationSupport {
return new TokenizationSupport(mode, {
export function createTokenizationSupport(_modeService:IModeService, modeId:string, lexer: monarchCommon.ILexer): modes.ITokenizationSupport {
return new TokenizationSupport(_modeService, modeId, {
getInitialState: (): modes.IState => {
return new MonarchLexer(mode, modeService, lexer);
return new MonarchLexer(modeId, _modeService, lexer);
},
enterNestedMode: (state: modes.IState): boolean => {
@ -399,31 +399,9 @@ export function createTokenizationSupport(modeService:IModeService, mode:modes.I
return false;
},
getNestedMode: (rawState: modes.IState): IEnteringNestedModeData => {
var mime = (<MonarchLexer>rawState).embeddedMode;
if (!modeService.isRegisteredMode(mime)) {
// unknown mode
return {
mode: modeService.getMode('text/plain'),
missingModePromise: null
};
}
var mode = modeService.getMode(mime);
if (mode) {
// mode is available
return {
mode: mode,
missingModePromise: null
};
}
// mode is not yet loaded
return {
mode: modeService.getMode('text/plain'),
missingModePromise: modeService.getOrCreateMode(mime).then(() => null)
};
getNestedMode: (rawState: modes.IState, locator:IModeLocator): modes.IMode => {
let mime = (<MonarchLexer>rawState).embeddedMode;
return locator.getMode(mime);
},
getLeavingNestedModeData: (line: string, state: modes.IState) => {

33
src/vs/editor/common/modes/nullMode.ts
View file

@ -4,27 +4,27 @@
*--------------------------------------------------------------------------------------------*/
'use strict';
import {IMode, IState, IStream, ITokenizationResult, ILineTokens} from 'vs/editor/common/modes';
import {IState, IStream, ITokenizationResult, ILineTokens} from 'vs/editor/common/modes';
import {ModeTransition} from 'vs/editor/common/core/modeTransition';
import {Token} from 'vs/editor/common/core/token';
export class NullState implements IState {
private mode: IMode;
private modeId: string;
private stateData: IState;
constructor(mode: IMode, stateData: IState) {
this.mode = mode;
constructor(modeId: string, stateData: IState) {
this.modeId = modeId;
this.stateData = stateData;
}
public clone(): IState {
let stateDataClone:IState = (this.stateData ? this.stateData.clone() : null);
return new NullState(this.mode, stateDataClone);
return new NullState(this.modeId, stateDataClone);
}
public equals(other:IState): boolean {
if (this.mode !== other.getMode()) {
if (this.modeId !== other.getModeId()) {
return false;
}
let otherStateData = other.getStateData();
@ -37,8 +37,8 @@ export class NullState implements IState {
return false;
}
public getMode(): IMode {
return this.mode;
public getModeId(): string {
return this.modeId;
}
public tokenize(stream:IStream):ITokenizationResult {
@ -55,22 +55,7 @@ export class NullState implements IState {
}
}
export class NullMode implements IMode {
public static ID = 'vs.editor.nullMode';
constructor() {
}
public getId():string {
return NullMode.ID;
}
public toSimplifiedMode(): IMode {
return this;
}
}
export const NULL_MODE_ID = 'vs.editor.nullMode';
export function nullTokenize(modeId: string, buffer:string, state: IState, deltaOffset:number = 0, stopAtOffset?:number): ILineTokens {
let tokens:Token[] = [new Token(deltaOffset, '')];

183
src/vs/editor/common/modes/supports/tokenizationSupport.ts
View file

@ -5,12 +5,12 @@
'use strict';
import {IDisposable} from 'vs/base/common/lifecycle';
import {TPromise} from 'vs/base/common/winjs.base';
import * as modes from 'vs/editor/common/modes';
import {LineStream} from 'vs/editor/common/modes/lineStream';
import {NullMode, NullState, nullTokenize} from 'vs/editor/common/modes/nullMode';
import {NullState, nullTokenize, NULL_MODE_ID} from 'vs/editor/common/modes/nullMode';
import {Token} from 'vs/editor/common/core/token';
import {ModeTransition} from 'vs/editor/common/core/modeTransition';
import {IModeService} from 'vs/editor/common/services/modeService';
export interface ILeavingNestedModeData {
/**
@ -29,9 +29,8 @@ export interface ILeavingNestedModeData {
stateAfterNestedMode: modes.IState;
}
export interface IEnteringNestedModeData {
mode:modes.IMode;
missingModePromise:TPromise<void>;
export interface IModeLocator {
getMode(mimetypeOrModeId: string): modes.IMode;
}
export interface ITokenizationCustomization {
@ -40,9 +39,9 @@ export interface ITokenizationCustomization {
enterNestedMode?: (state:modes.IState) => boolean;
getNestedMode?: (state:modes.IState) => IEnteringNestedModeData;
getNestedMode?: (state:modes.IState, locator:IModeLocator) => modes.IMode;
getNestedModeInitialState?: (myState:modes.IState) => { state:modes.IState; missingModePromise:TPromise<void>; };
getNestedModeInitialState?: (myState:modes.IState) => modes.IState;
/**
* Return null if the line does not leave the nested mode
@ -74,23 +73,18 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
onReturningFromNestedMode: boolean;
};
public supportsNestedModes:boolean;
private supportsNestedModes:boolean;
private _mode:modes.IMode;
private _modeService:IModeService;
private _modeId:string;
private _embeddedModesListeners: { [modeId:string]: IDisposable; };
private _embeddedModes: {[modeId:string]:boolean;};
private _tokenizationRegistryListener: IDisposable;
constructor(mode:modes.IMode, customization:ITokenizationCustomization, supportsNestedModes:boolean) {
this._mode = mode;
this._modeId = this._mode.getId();
constructor(modeService:IModeService, modeId:string, customization:ITokenizationCustomization, supportsNestedModes:boolean) {
this._modeService = modeService;
this._modeId = modeId;
this.customization = customization;
this.supportsNestedModes = supportsNestedModes;
this._embeddedModesListeners = {};
if (this.supportsNestedModes) {
if (!this._mode.setTokenizationSupport) {
throw new Error('Cannot be a mode with nested modes unless I can emit a tokenizationSupport changed event!');
}
}
this.defaults = {
enterNestedMode: !isFunction(customization.enterNestedMode),
getNestedMode: !isFunction(customization.getNestedMode),
@ -98,13 +92,26 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
getLeavingNestedModeData: !isFunction(customization.getLeavingNestedModeData),
onReturningFromNestedMode: !isFunction(customization.onReturningFromNestedMode)
};
this._embeddedModes = Object.create(null);
// Set up listening for embedded modes
let emitting = false;
this._tokenizationRegistryListener = modes.TokenizationRegistry.onDidChange((e) => {
if (emitting) {
return;
}
let isOneOfMyEmbeddedModes = this._embeddedModes[e.languageId];
if (isOneOfMyEmbeddedModes) {
emitting = true;
modes.TokenizationRegistry.fire(this._modeId);
emitting = false;
}
});
}
public dispose() : void {
for (let listener in this._embeddedModesListeners) {
this._embeddedModesListeners[listener].dispose();
delete this._embeddedModesListeners[listener];
}
public dispose(): void {
this._tokenizationRegistryListener.dispose();
}
public getInitialState(): modes.IState {
@ -112,7 +119,7 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
}
public tokenize(line:string, state:modes.IState, deltaOffset:number = 0, stopAtOffset:number = deltaOffset + line.length):modes.ILineTokens {
if (state.getMode() !== this._mode) {
if (state.getModeId() !== this._modeId) {
return this._nestedTokenize(line, state, deltaOffset, stopAtOffset, [], []);
} else {
return this._myTokenize(line, state, deltaOffset, stopAtOffset, [], []);
@ -120,31 +127,32 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
}
/**
* Precondition is: nestedModeState.getMode() !== this
* Precondition is: nestedModeState.getModeId() !== this._modeId
* This means we are in a nested mode when parsing starts on this line.
*/
private _nestedTokenize(buffer:string, nestedModeState:modes.IState, deltaOffset:number, stopAtOffset:number, prependTokens:Token[], prependModeTransitions:ModeTransition[]):modes.ILineTokens {
let myStateBeforeNestedMode = nestedModeState.getStateData();
let leavingNestedModeData = this.getLeavingNestedModeData(buffer, myStateBeforeNestedMode);
let leavingNestedModeData = this._getLeavingNestedModeData(buffer, myStateBeforeNestedMode);
// Be sure to give every embedded mode the
// opportunity to leave nested mode.
// i.e. Don't go straight to the most nested mode
let stepOnceNestedState = nestedModeState;
while (stepOnceNestedState.getStateData() && stepOnceNestedState.getStateData().getMode() !== this._mode) {
while (stepOnceNestedState.getStateData() && stepOnceNestedState.getStateData().getModeId() !== this._modeId) {
stepOnceNestedState = stepOnceNestedState.getStateData();
}
let nestedMode = stepOnceNestedState.getMode();
let nestedModeId = stepOnceNestedState.getModeId();
if (!leavingNestedModeData) {
// tokenization will not leave nested mode
let result:modes.ILineTokens;
if (nestedMode.tokenizationSupport) {
result = nestedMode.tokenizationSupport.tokenize(buffer, nestedModeState, deltaOffset, stopAtOffset);
let tokenizationSupport = modes.TokenizationRegistry.get(nestedModeId);
if (tokenizationSupport) {
result = tokenizationSupport.tokenize(buffer, nestedModeState, deltaOffset, stopAtOffset);
} else {
// The nested mode doesn't have tokenization support,
// unfortunatelly this means we have to fake it
result = nullTokenize(nestedMode.getId(), buffer, nestedModeState, deltaOffset);
result = nullTokenize(nestedModeId, buffer, nestedModeState, deltaOffset);
}
result.tokens = prependTokens.concat(result.tokens);
result.modeTransitions = prependModeTransitions.concat(result.modeTransitions);
@ -155,12 +163,13 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
if (nestedModeBuffer.length > 0) {
// Tokenize with the nested mode
let nestedModeLineTokens:modes.ILineTokens;
if (nestedMode.tokenizationSupport) {
nestedModeLineTokens = nestedMode.tokenizationSupport.tokenize(nestedModeBuffer, nestedModeState, deltaOffset, stopAtOffset);
let tokenizationSupport = modes.TokenizationRegistry.get(nestedModeId);
if (tokenizationSupport) {
nestedModeLineTokens = tokenizationSupport.tokenize(nestedModeBuffer, nestedModeState, deltaOffset, stopAtOffset);
} else {
// The nested mode doesn't have tokenization support,
// unfortunatelly this means we have to fake it
nestedModeLineTokens = nullTokenize(nestedMode.getId(), nestedModeBuffer, nestedModeState, deltaOffset);
nestedModeLineTokens = nullTokenize(nestedModeId, nestedModeBuffer, nestedModeState, deltaOffset);
}
// Save last state of nested mode
@ -174,7 +183,7 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
let bufferAfterNestedMode = leavingNestedModeData.bufferAfterNestedMode;
let myStateAfterNestedMode = leavingNestedModeData.stateAfterNestedMode;
myStateAfterNestedMode.setStateData(myStateBeforeNestedMode.getStateData());
this.onReturningFromNestedMode(myStateAfterNestedMode, nestedModeState);
this._onReturningFromNestedMode(myStateAfterNestedMode, nestedModeState);
return this._myTokenize(bufferAfterNestedMode, myStateAfterNestedMode, deltaOffset + nestedModeBuffer.length, stopAtOffset, prependTokens, prependModeTransitions);
}
@ -187,7 +196,6 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
let lineStream = new LineStream(buffer);
let tokenResult:modes.ITokenizationResult, beforeTokenizeStreamPos:number;
let previousType:string = null;
let retokenize:TPromise<void> = null;
myState = myState.clone();
if (prependModeTransitions.length <= 0 || prependModeTransitions[prependModeTransitions.length-1].modeId !== this._modeId) {
@ -222,40 +230,19 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
previousType = tokenResult.type;
if (this.supportsNestedModes && this.enterNestedMode(myState)) {
if (this.supportsNestedModes && this._enterNestedMode(myState)) {
let currentEmbeddedLevels = this._getEmbeddedLevel(myState);
if (currentEmbeddedLevels < TokenizationSupport.MAX_EMBEDDED_LEVELS) {
let nestedModeState = this.getNestedModeInitialState(myState);
// Re-emit tokenizationSupport change events from all modes that I ever embedded
let embeddedMode = nestedModeState.state.getMode();
if (typeof embeddedMode.addSupportChangedListener === 'function' && !this._embeddedModesListeners.hasOwnProperty(embeddedMode.getId())) {
let emitting = false;
this._embeddedModesListeners[embeddedMode.getId()] = embeddedMode.addSupportChangedListener((e) => {
if (emitting) {
return;
}
if (e.tokenizationSupport) {
emitting = true;
this._mode.setTokenizationSupport((mode) => {
return mode.tokenizationSupport;
});
emitting = false;
}
});
}
let nestedModeState = this._getNestedModeInitialState(myState);
if (!lineStream.eos()) {
// There is content from the embedded mode
let restOfBuffer = buffer.substr(lineStream.pos());
let result = this._nestedTokenize(restOfBuffer, nestedModeState.state, deltaOffset + lineStream.pos(), stopAtOffset, prependTokens, prependModeTransitions);
result.retokenize = result.retokenize || nestedModeState.missingModePromise;
let result = this._nestedTokenize(restOfBuffer, nestedModeState, deltaOffset + lineStream.pos(), stopAtOffset, prependTokens, prependModeTransitions);
return result;
} else {
// Transition to the nested mode state
myState = nestedModeState.state;
retokenize = nestedModeState.missingModePromise;
myState = nestedModeState;
}
}
}
@ -265,8 +252,7 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
tokens: prependTokens,
actualStopOffset: lineStream.pos() + deltaOffset,
modeTransitions: prependModeTransitions,
endState: myState,
retokenize: retokenize
endState: myState
};
}
@ -279,7 +265,7 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
return result;
}
private enterNestedMode(state:modes.IState): boolean {
private _enterNestedMode(state:modes.IState): boolean {
if (this.defaults.enterNestedMode) {
return false;
}
@ -287,60 +273,63 @@ export class TokenizationSupport implements modes.ITokenizationSupport, IDisposa
}
private getNestedMode(state:modes.IState): IEnteringNestedModeData {
private _getNestedMode(state:modes.IState): modes.IMode {
if (this.defaults.getNestedMode) {
return null;
}
return this.customization.getNestedMode(state);
}
private static _validatedNestedMode(input:IEnteringNestedModeData): IEnteringNestedModeData {
let mode: modes.IMode = new NullMode(),
missingModePromise: TPromise<void> = null;
let locator:IModeLocator = {
getMode: (mimetypeOrModeId: string): modes.IMode => {
if (!mimetypeOrModeId || !this._modeService.isRegisteredMode(mimetypeOrModeId)) {
return null;
}
if (input && input.mode) {
mode = input.mode;
}
if (input && input.missingModePromise) {
missingModePromise = input.missingModePromise;
}
let modeId = this._modeService.getModeId(mimetypeOrModeId);
return {
mode: mode,
missingModePromise: missingModePromise
let mode = this._modeService.getMode(modeId);
if (mode) {
// Re-emit tokenizationSupport change events from all modes that I ever embedded
this._embeddedModes[modeId] = true;
return mode;
}
// Fire mode loading event
this._modeService.getOrCreateMode(modeId);
this._embeddedModes[modeId] = true;
return null;
}
};
return this.customization.getNestedMode(state, locator);
}
private getNestedModeInitialState(state:modes.IState): { state:modes.IState; missingModePromise:TPromise<void>; } {
private _getNestedModeInitialState(state:modes.IState): modes.IState {
if (this.defaults.getNestedModeInitialState) {
let nestedMode = TokenizationSupport._validatedNestedMode(this.getNestedMode(state));
let missingModePromise = nestedMode.missingModePromise;
let nestedModeState: modes.IState;
if (nestedMode.mode.tokenizationSupport) {
nestedModeState = nestedMode.mode.tokenizationSupport.getInitialState();
} else {
nestedModeState = new NullState(nestedMode.mode, null);
let nestedMode = this._getNestedMode(state);
if (nestedMode) {
let tokenizationSupport = modes.TokenizationRegistry.get(nestedMode.getId());
if (tokenizationSupport) {
let nestedModeState = tokenizationSupport.getInitialState();
nestedModeState.setStateData(state);
return nestedModeState;
}
}
nestedModeState.setStateData(state);
return {
state: nestedModeState,
missingModePromise: missingModePromise
};
return new NullState(nestedMode ? nestedMode.getId() : NULL_MODE_ID, state);
}
return this.customization.getNestedModeInitialState(state);
}
private getLeavingNestedModeData(line:string, state:modes.IState): ILeavingNestedModeData {
private _getLeavingNestedModeData(line:string, state:modes.IState): ILeavingNestedModeData {
if (this.defaults.getLeavingNestedModeData) {
return null;
}
return this.customization.getLeavingNestedModeData(line, state);
}
private onReturningFromNestedMode(myStateAfterNestedMode:modes.IState, lastNestedModeState:modes.IState): void {
private _onReturningFromNestedMode(myStateAfterNestedMode:modes.IState, lastNestedModeState:modes.IState): void {
if (this.defaults.onReturningFromNestedMode) {
return null;
}

17
src/vs/editor/common/modes/textToHtmlTokenizer.ts
View file

@ -6,20 +6,21 @@
import {IHTMLContentElement} from 'vs/base/common/htmlContent';
import * as strings from 'vs/base/common/strings';
import {IMode, IState, ITokenizationSupport} from 'vs/editor/common/modes';
import {IState, ITokenizationSupport, TokenizationRegistry} from 'vs/editor/common/modes';
import {NullState, nullTokenize} from 'vs/editor/common/modes/nullMode';
export function tokenizeToHtmlContent(text: string, mode: IMode): IHTMLContentElement {
return _tokenizeToHtmlContent(text, _getSafeTokenizationSupport(mode));
export function tokenizeToHtmlContent(text: string, languageId: string): IHTMLContentElement {
return _tokenizeToHtmlContent(text, _getSafeTokenizationSupport(languageId));
}
export function tokenizeToString(text: string, mode: IMode, extraTokenClass?: string): string {
return _tokenizeToString(text, _getSafeTokenizationSupport(mode), extraTokenClass);
export function tokenizeToString(text: string, languageId: string, extraTokenClass?: string): string {
return _tokenizeToString(text, _getSafeTokenizationSupport(languageId), extraTokenClass);
}
function _getSafeTokenizationSupport(mode: IMode): ITokenizationSupport {
if (mode && mode.tokenizationSupport) {
return mode.tokenizationSupport;
function _getSafeTokenizationSupport(languageId: string): ITokenizationSupport {
let tokenizationSupport = TokenizationRegistry.get(languageId);
if (tokenizationSupport) {
return tokenizationSupport;
}
return {
getInitialState: () => new NullState(null, null),

2
src/vs/editor/common/services/compatWorkerServiceWorker.ts
View file

@ -74,7 +74,7 @@ export class CompatWorkerServiceWorker implements ICompatWorkerService {
this.resourceService.remove(mirrorModel.uri);
// (2) Change mode
mirrorModel.setMode(mode);
mirrorModel.setMode(mode.getId());
// (3) Insert again to resource service (it will have the new mode)
this.resourceService.insert(mirrorModel.uri, mirrorModel);

4
src/vs/editor/common/services/modeService.ts
View file

@ -5,7 +5,6 @@
'use strict';
import Event from 'vs/base/common/event';
import {IDisposable} from 'vs/base/common/lifecycle';
import {TPromise} from 'vs/base/common/winjs.base';
import {createDecorator} from 'vs/platform/instantiation/common/instantiation';
import * as modes from 'vs/editor/common/modes';
@ -64,7 +63,4 @@ export interface IModeService {
getOrCreateMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): TPromise<modes.IMode>;
getOrCreateModeByLanguageName(languageName: string): TPromise<modes.IMode>;
getOrCreateModeByFilenameOrFirstLine(filename: string, firstLine?:string): TPromise<modes.IMode>;
registerTokenizationSupport(modeId: string, callback: (mode: modes.IMode) => modes.ITokenizationSupport): IDisposable;
registerTokenizationSupport2(modeId: string, support: modes.TokensProvider): IDisposable;
}

72
src/vs/editor/common/services/modeServiceImpl.ts
View file

@ -7,7 +7,6 @@
import * as nls from 'vs/nls';
import {onUnexpectedError} from 'vs/base/common/errors';
import Event, {Emitter} from 'vs/base/common/event';
import {IDisposable, empty as EmptyDisposable} from 'vs/base/common/lifecycle'; // TODO@Alex
import * as paths from 'vs/base/common/paths';
import {TPromise} from 'vs/base/common/winjs.base';
import mime = require('vs/base/common/mime');
@ -149,7 +148,10 @@ export class ModeServiceImpl implements IModeService {
private _onDidCreateMode: Emitter<modes.IMode> = new Emitter<modes.IMode>();
public onDidCreateMode: Event<modes.IMode> = this._onDidCreateMode.event;
constructor(instantiationService:IInstantiationService, extensionService:IExtensionService) {
constructor(
instantiationService:IInstantiationService,
extensionService:IExtensionService
) {
this._instantiationService = instantiationService;
this._extensionService = extensionService;
@ -349,58 +351,16 @@ export class ModeServiceImpl implements IModeService {
};
}
private _registerTokenizationSupport<T>(mode:modes.IMode, callback: (mode: modes.IMode) => T): IDisposable {
if (mode.setTokenizationSupport) {
return mode.setTokenizationSupport(callback);
} else {
console.warn('Cannot register tokenizationSupport on mode ' + mode.getId() + ' because it does not support it.');
return EmptyDisposable;
}
}
private registerModeSupport<T>(modeId: string, callback: (mode: modes.IMode) => T): IDisposable {
if (this._instantiatedModes.hasOwnProperty(modeId)) {
return this._registerTokenizationSupport(this._instantiatedModes[modeId], callback);
}
let cc: (disposable:IDisposable)=>void;
let promise = new TPromise<IDisposable>((c, e) => { cc = c; });
let disposable = this.onDidCreateMode((mode) => {
if (mode.getId() !== modeId) {
return;
}
cc(this._registerTokenizationSupport(mode, callback));
disposable.dispose();
});
return {
dispose: () => {
promise.done(disposable => disposable.dispose(), null);
}
};
}
public registerTokenizationSupport(modeId: string, callback: (mode: modes.IMode) => modes.ITokenizationSupport): IDisposable {
return this.registerModeSupport(modeId, callback);
}
public registerTokenizationSupport2(modeId: string, support: modes.TokensProvider): IDisposable {
return this.registerModeSupport(modeId, (mode) => {
return new TokenizationSupport2Adapter(mode, support);
});
}
}
export class TokenizationState2Adapter implements modes.IState {
private _mode: modes.IMode;
private _modeId: string;
private _actual: modes.IState2;
private _stateData: modes.IState;
constructor(mode: modes.IMode, actual: modes.IState2, stateData: modes.IState) {
this._mode = mode;
constructor(modeId: string, actual: modes.IState2, stateData: modes.IState) {
this._modeId = modeId;
this._actual = actual;
this._stateData = stateData;
}
@ -408,7 +368,7 @@ export class TokenizationState2Adapter implements modes.IState {
public get actual(): modes.IState2 { return this._actual; }
public clone(): TokenizationState2Adapter {
return new TokenizationState2Adapter(this._mode, this._actual.clone(), AbstractState.safeClone(this._stateData));
return new TokenizationState2Adapter(this._modeId, this._actual.clone(), AbstractState.safeClone(this._stateData));
}
public equals(other:modes.IState): boolean {
@ -421,8 +381,8 @@ export class TokenizationState2Adapter implements modes.IState {
return false;
}
public getMode(): modes.IMode {
return this._mode;
public getModeId(): string {
return this._modeId;
}
public tokenize(stream:any): any {
@ -440,16 +400,16 @@ export class TokenizationState2Adapter implements modes.IState {
export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
private _mode: modes.IMode;
private _modeId: string;
private _actual: modes.TokensProvider;
constructor(mode: modes.IMode, actual: modes.TokensProvider) {
this._mode = mode;
constructor(modeId: string, actual: modes.TokensProvider) {
this._modeId = modeId;
this._actual = actual;
}
public getInitialState(): modes.IState {
return new TokenizationState2Adapter(this._mode, this._actual.getInitialState(), null);
return new TokenizationState2Adapter(this._modeId, this._actual.getInitialState(), null);
}
public tokenize(line:string, state:modes.IState, offsetDelta: number = 0, stopAtOffset?: number): modes.ILineTokens {
@ -468,8 +428,8 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
return {
tokens: tokens,
actualStopOffset: offsetDelta + line.length,
endState: new TokenizationState2Adapter(state.getMode(), actualResult.endState, state.getStateData()),
modeTransitions: [new ModeTransition(offsetDelta, state.getMode().getId())],
endState: new TokenizationState2Adapter(state.getModeId(), actualResult.endState, state.getStateData()),
modeTransitions: [new ModeTransition(offsetDelta, state.getModeId())],
};
}
throw new Error('Unexpected state to tokenize with!');

2
src/vs/editor/common/services/modelService.ts
View file

@ -18,6 +18,8 @@ export interface IModelService {
createModel(value:string | IRawText, modeOrPromise:TPromise<IMode>|IMode, resource: URI): IModel;
setMode(model:IModel, modeOrPromise:TPromise<IMode>|IMode): void;
destroyModel(resource: URI): void;
getModels(): IModel[];

31
src/vs/editor/common/services/modelServiceImpl.ts
View file

@ -24,6 +24,7 @@ import * as platform from 'vs/base/common/platform';
import {IConfigurationService} from 'vs/platform/configuration/common/configuration';
import {DEFAULT_INDENTATION, DEFAULT_TRIM_AUTO_WHITESPACE} from 'vs/editor/common/config/defaultConfig';
import {IMessageService} from 'vs/platform/message/common/message';
import {PLAINTEXT_MODE_ID} from 'vs/editor/common/modes/modesRegistry';
export interface IRawModelData {
url: URI;
@ -352,13 +353,13 @@ export class ModelServiceImpl implements IModelService {
// --- begin IModelService
private _createModelData(value: string | editorCommon.IRawText, modeOrPromise: TPromise<IMode> | IMode, resource: URI): ModelData {
private _createModelData(value: string | editorCommon.IRawText, languageId:string, resource: URI): ModelData {
// create & save the model
let model:Model;
if (typeof value === 'string') {
model = Model.createFromString(value, this._modelCreationOptions, modeOrPromise, resource);
model = Model.createFromString(value, this._modelCreationOptions, languageId, resource);
} else {
model = new Model(value, modeOrPromise, resource);
model = new Model(value, languageId, resource);
}
let modelId = MODEL_ID(model.uri);
@ -374,7 +375,14 @@ export class ModelServiceImpl implements IModelService {
}
public createModel(value: string | editorCommon.IRawText, modeOrPromise: TPromise<IMode> | IMode, resource: URI): editorCommon.IModel {
let modelData = this._createModelData(value, modeOrPromise, resource);
let modelData: ModelData;
if (!modeOrPromise || TPromise.is(modeOrPromise)) {
modelData = this._createModelData(value, PLAINTEXT_MODE_ID, resource);
this.setMode(modelData.model, modeOrPromise);
} else {
modelData = this._createModelData(value, modeOrPromise.getId(), resource);
}
// handle markers (marker service => model)
if (this._markerService) {
@ -386,6 +394,21 @@ export class ModelServiceImpl implements IModelService {
return modelData.model;
}
public setMode(model:editorCommon.IModel, modeOrPromise:TPromise<IMode>|IMode): void {
if (!modeOrPromise) {
return;
}
if (TPromise.is(modeOrPromise)) {
modeOrPromise.then((mode) => {
if (!model.isDisposed()) {
model.setMode(mode.getId());
}
});
} else {
model.setMode(modeOrPromise.getId());
}
}
public destroyModel(resource: URI): void {
// We need to support that not all models get disposed through this service (i.e. model.dispose() should work!)
let modelData = this._models[MODEL_ID(resource)];

4
src/vs/editor/common/viewModel/viewModelImpl.ts
View file

@ -214,10 +214,6 @@ export class ViewModel extends EventEmitter implements IViewModel {
// That's ok, a model tokens changed event will follow shortly
break;
case editorCommon.EventType.ModelModeSupportChanged:
// That's ok, no work to do
break;
case editorCommon.EventType.ModelContentChanged2:
// Ignore
break;

2
src/vs/editor/contrib/comment/test/common/blockCommentCommand.test.ts
View file

@ -11,7 +11,7 @@ import {CommentMode} from 'vs/editor/test/common/testModes';
function testBlockCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
var mode = new CommentMode({ lineComment: '!@#', blockComment: ['<0', '0>'] });
testCommand(lines, mode, selection, (sel) => new BlockCommentCommand(sel), expectedLines, expectedSelection);
testCommand(lines, mode.getId(), selection, (sel) => new BlockCommentCommand(sel), expectedLines, expectedSelection);
}
suite('Editor Contrib - Block Comment Command', () => {

8
src/vs/editor/contrib/comment/test/common/lineCommentCommand.test.ts
View file

@ -14,12 +14,12 @@ suite('Editor Contrib - Line Comment Command', () => {
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
var mode = new CommentMode({ lineComment: '!@#', blockComment: ['<!@#', '#@!>'] });
testCommand(lines, mode, selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection);
testCommand(lines, mode.getId(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection);
}
function testAddLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
var mode = new CommentMode({ lineComment: '!@#', blockComment: ['<!@#', '#@!>'] });
testCommand(lines, mode, selection, (sel) => new LineCommentCommand(sel, 4, Type.ForceAdd), expectedLines, expectedSelection);
testCommand(lines, mode.getId(), selection, (sel) => new LineCommentCommand(sel, 4, Type.ForceAdd), expectedLines, expectedSelection);
}
test('comment single line', function () {
@ -521,7 +521,7 @@ suite('Editor Contrib - Line Comment As Block Comment', () => {
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
var mode = new CommentMode({ lineComment: '', blockComment: ['(', ')'] });
testCommand(lines, mode, selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection);
testCommand(lines, mode.getId(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection);
}
test('fall back to block comment command', function () {
@ -631,7 +631,7 @@ suite('Editor Contrib - Line Comment As Block Comment', () => {
suite('Editor Contrib - Line Comment As Block Comment 2', () => {
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
var mode = new CommentMode({ lineComment: null, blockComment: ['<!@#', '#@!>'] });
testCommand(lines, mode, selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection);
testCommand(lines, mode.getId(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection);
}
test('no selection => uses indentation', function () {

2
src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts
View file

@ -17,7 +17,7 @@ import {LanguageConfigurationRegistry} from 'vs/editor/common/modes/languageConf
class MockJSMode extends MockTokenizingMode {
constructor() {
super('js-tokenSelectionSupport', 'mock-js');
super('mock-js');
LanguageConfigurationRegistry.register(this.getId(), {
brackets: [

16
src/vs/editor/node/textMate/TMSyntax.ts
View file

@ -8,7 +8,7 @@ import * as nls from 'vs/nls';
import {onUnexpectedError} from 'vs/base/common/errors';
import * as paths from 'vs/base/common/paths';
import {IExtensionMessageCollector, ExtensionsRegistry} from 'vs/platform/extensions/common/extensionsRegistry';
import {ILineTokens, IMode, ITokenizationSupport} from 'vs/editor/common/modes';
import {ILineTokens, ITokenizationSupport, TokenizationRegistry} from 'vs/editor/common/modes';
import {TMState} from 'vs/editor/common/modes/TMState';
import {LineTokens} from 'vs/editor/common/modes/supports';
import {IModeService} from 'vs/editor/common/services/modeService';
@ -141,17 +141,15 @@ export class MainProcessTextMateSyntax {
return;
}
this._modeService.registerTokenizationSupport(modeId, (mode: IMode) => {
return createTokenizationSupport(mode, grammar);
});
TokenizationRegistry.register(modeId, createTokenizationSupport(modeId, grammar));
});
}
}
function createTokenizationSupport(mode: IMode, grammar: IGrammar): ITokenizationSupport {
var tokenizer = new Tokenizer(mode.getId(), grammar);
function createTokenizationSupport(modeId: string, grammar: IGrammar): ITokenizationSupport {
var tokenizer = new Tokenizer(modeId, grammar);
return {
getInitialState: () => new TMState(mode, null, null),
getInitialState: () => new TMState(modeId, null, null),
tokenize: (line, state, offsetDelta?, stopAtOffset?) => tokenizer.tokenize(line, <TMState> state, offsetDelta, stopAtOffset)
};
}
@ -252,7 +250,7 @@ class Tokenizer {
if (line.length >= 20000 || depth(state.getRuleStack()) > 100) {
return new LineTokens(
[new Token(offsetDelta, '')],
[new ModeTransition(offsetDelta, state.getMode().getId())],
[new ModeTransition(offsetDelta, state.getModeId())],
offsetDelta,
state
);
@ -279,7 +277,7 @@ class Tokenizer {
return new LineTokens(
tokens,
[new ModeTransition(offsetDelta, freshState.getMode().getId())],
[new ModeTransition(offsetDelta, freshState.getModeId())],
offsetDelta + line.length,
freshState
);

3
src/vs/editor/test/common/commands/commandTestUtils.ts
View file

@ -10,13 +10,12 @@ import {Range} from 'vs/editor/common/core/range';
import {Selection} from 'vs/editor/common/core/selection';
import * as editorCommon from 'vs/editor/common/editorCommon';
import {Model} from 'vs/editor/common/model/model';
import {IMode} from 'vs/editor/common/modes';
import {MockConfiguration} from 'vs/editor/test/common/mocks/mockConfiguration';
import {viewModelHelper} from 'vs/editor/test/common/editorTestUtils';
export function testCommand(
lines: string[],
mode: IMode,
mode: string,
selection: Selection,
commandFactory: (selection:Selection) => editorCommon.ICommand,
expectedLines: string[],

4
src/vs/editor/test/common/commands/shiftCommand.test.ts
View file

@ -74,7 +74,7 @@ class DocBlockCommentMode extends MockMode {
}
function testShiftCommandInDocBlockCommentMode(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
testCommand(lines, new DocBlockCommentMode(), selection, (sel) => new ShiftCommand(sel, {
testCommand(lines, new DocBlockCommentMode().getId(), selection, (sel) => new ShiftCommand(sel, {
isUnshift: false,
tabSize: 4,
oneIndent: '\t'
@ -82,7 +82,7 @@ function testShiftCommandInDocBlockCommentMode(lines: string[], selection: Selec
}
function testUnshiftCommandInDocBlockCommentMode(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
testCommand(lines, new DocBlockCommentMode(), selection, (sel) => new ShiftCommand(sel, {
testCommand(lines, new DocBlockCommentMode().getId(), selection, (sel) => new ShiftCommand(sel, {
isUnshift: true,
tabSize: 4,
oneIndent: '\t'

36
src/vs/editor/test/common/controller/cursor.test.ts
View file

@ -1145,7 +1145,7 @@ suite('Editor Controller - Regression tests', () => {
text: [
'var x = (3 + (5-7));'
],
mode: new BracketMode()
modeId: new BracketMode().getId()
}, (model, cursor) => {
// ensure is tokenized
model.getLineContext(1);
@ -1180,7 +1180,7 @@ suite('Editor Controller - Regression tests', () => {
tabSize: 4,
trimAutoWhitespace: true
},
mode: new OnEnterMode(IndentAction.Indent),
modeId: new OnEnterMode(IndentAction.Indent).getId(),
}, (model, cursor) => {
moveTo(cursor, 4, 1, false);
assertCursor(cursor, new Selection(4, 1, 4, 1));
@ -1207,7 +1207,7 @@ suite('Editor Controller - Regression tests', () => {
tabSize: 4,
trimAutoWhitespace: true
},
mode: new OnEnterMode(IndentAction.Indent),
modeId: new OnEnterMode(IndentAction.Indent).getId(),
}, (model, cursor) => {
moveTo(cursor, 4, 2, false);
assertCursor(cursor, new Selection(4, 2, 4, 2));
@ -1235,7 +1235,7 @@ suite('Editor Controller - Regression tests', () => {
tabSize: 4,
trimAutoWhitespace: true
},
mode: new OnEnterMode(IndentAction.Indent),
modeId: new OnEnterMode(IndentAction.Indent).getId(),
}, (model, cursor) => {
moveTo(cursor, 4, 1, false);
assertCursor(cursor, new Selection(4, 1, 4, 1));
@ -1262,7 +1262,7 @@ suite('Editor Controller - Regression tests', () => {
tabSize: 4,
trimAutoWhitespace: true
},
mode: new OnEnterMode(IndentAction.Indent),
modeId: new OnEnterMode(IndentAction.Indent).getId(),
}, (model, cursor) => {
moveTo(cursor, 4, 3, false);
assertCursor(cursor, new Selection(4, 3, 4, 3));
@ -1289,7 +1289,7 @@ suite('Editor Controller - Regression tests', () => {
tabSize: 4,
trimAutoWhitespace: true
},
mode: new OnEnterMode(IndentAction.Indent),
modeId: new OnEnterMode(IndentAction.Indent).getId(),
}, (model, cursor) => {
moveTo(cursor, 4, 4, false);
assertCursor(cursor, new Selection(4, 4, 4, 4));
@ -1329,7 +1329,7 @@ suite('Editor Controller - Regression tests', () => {
text: [
' function baz() {'
],
mode: new OnEnterMode(IndentAction.IndentOutdent),
modeId: new OnEnterMode(IndentAction.IndentOutdent).getId(),
modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true }
}, (model, cursor) => {
moveTo(cursor, 1, 6, false);
@ -1432,7 +1432,7 @@ suite('Editor Controller - Regression tests', () => {
text: [
'hello'
],
mode: new SurroundingMode(),
modeId: new SurroundingMode().getId(),
modelOpts: { tabSize: 4, insertSpaces: true, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true }
}, (model, cursor) => {
moveTo(cursor, 1, 3, false);
@ -1454,7 +1454,7 @@ suite('Editor Controller - Regression tests', () => {
' return 1;',
'};'
],
mode: new SurroundingMode(),
modeId: new SurroundingMode().getId(),
modelOpts: { tabSize: 4, insertSpaces: true, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true }
}, (model, cursor) => {
moveTo(cursor, 3, 2, false);
@ -1521,7 +1521,7 @@ suite('Editor Controller - Regression tests', () => {
'and more lines',
'just some text',
],
mode: null,
modeId: null,
modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true }
}, (model, cursor) => {
moveTo(cursor, 1, 1, false);
@ -2034,7 +2034,7 @@ suite('Editor Controller - Regression tests', () => {
tabSize: 4,
trimAutoWhitespace: true
},
mode: new OnEnterMode(IndentAction.None),
modeId: new OnEnterMode(IndentAction.None).getId(),
}, (model, cursor) => {
moveTo(cursor, 3, 8, false);
moveTo(cursor, 2, 12, true);
@ -2061,7 +2061,7 @@ suite('Editor Controller - Regression tests', () => {
tabSize: 4,
trimAutoWhitespace: true
},
mode: new OnEnterMode(IndentAction.None),
modeId: new OnEnterMode(IndentAction.None).getId(),
}, (model, cursor) => {
moveTo(cursor, 2, 12, false);
moveTo(cursor, 3, 8, true);
@ -2195,7 +2195,7 @@ suite('Editor Controller - Cursor Configuration', () => {
text: [
'\thello'
],
mode: new OnEnterMode(IndentAction.Indent),
modeId: new OnEnterMode(IndentAction.Indent).getId(),
modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true }
}, (model, cursor) => {
moveTo(cursor, 1, 7, false);
@ -2211,7 +2211,7 @@ suite('Editor Controller - Cursor Configuration', () => {
text: [
'\thello'
],
mode: new OnEnterMode(IndentAction.None),
modeId: new OnEnterMode(IndentAction.None).getId(),
modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true }
}, (model, cursor) => {
moveTo(cursor, 1, 7, false);
@ -2227,7 +2227,7 @@ suite('Editor Controller - Cursor Configuration', () => {
text: [
'\thell()'
],
mode: new OnEnterMode(IndentAction.IndentOutdent),
modeId: new OnEnterMode(IndentAction.IndentOutdent).getId(),
modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true }
}, (model, cursor) => {
moveTo(cursor, 1, 7, false);
@ -2387,7 +2387,7 @@ suite('Editor Controller - Cursor Configuration', () => {
defaultEOL: DefaultEndOfLine.LF,
trimAutoWhitespace: true
},
mode: new OnEnterMode(IndentAction.IndentOutdent),
modeId: new OnEnterMode(IndentAction.IndentOutdent).getId(),
}, (model, cursor) => {
moveTo(cursor, 1, 32);
@ -2688,13 +2688,13 @@ suite('Editor Controller - Cursor Configuration', () => {
interface ICursorOpts {
text: string[];
mode?: IMode;
modeId?: string;
modelOpts?: ITextModelCreationOptions;
editorOpts?: IEditorOptions;
}
function usingCursor(opts:ICursorOpts, callback:(model:Model, cursor:Cursor)=>void): void {
let model = Model.createFromString(opts.text.join('\n'), opts.modelOpts, opts.mode);
let model = Model.createFromString(opts.text.join('\n'), opts.modelOpts, opts.modeId);
let config = new MockConfiguration(opts.editorOpts);
let cursor = new Cursor(1, config, model, viewModelHelper(model), false);

30
src/vs/editor/test/common/mocks/mockMode.ts
View file

@ -4,17 +4,21 @@
*--------------------------------------------------------------------------------------------*/
'use strict';
import {IMode, IState, IStream, ITokenizationResult, ITokenizationSupport} from 'vs/editor/common/modes';
import {IMode, IState, IStream, ITokenizationResult, ITokenizationSupport, TokenizationRegistry} from 'vs/editor/common/modes';
import {AbstractState} from 'vs/editor/common/modes/abstractState';
import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
let instanceCount = 0;
export function generateMockModeId(): string {
return 'mockMode' + (++instanceCount);
}
export class MockMode implements IMode {
private static instanceCount = 0;
private _id:string;
constructor(id?:string) {
if (typeof id === 'undefined') {
id = 'mockMode' + (++MockMode.instanceCount);
id = generateMockModeId();
}
this._id = id;
}
@ -22,18 +26,14 @@ export class MockMode implements IMode {
public getId():string {
return this._id;
}
public toSimplifiedMode(): IMode {
return this;
}
}
export class StateForMockTokenizingMode extends AbstractState {
private _tokenType: string;
constructor(mode:IMode, tokenType:string) {
super(mode);
constructor(modeId:string, tokenType:string) {
super(modeId);
this._tokenType = tokenType;
}
@ -53,13 +53,11 @@ export class StateForMockTokenizingMode extends AbstractState {
export class MockTokenizingMode extends MockMode {
public tokenizationSupport: ITokenizationSupport;
constructor(tokenType:string) {
super();
constructor(id:string, tokenType:string) {
super(id);
this.tokenizationSupport = new TokenizationSupport(this, {
getInitialState: () => new StateForMockTokenizingMode(this, tokenType)
}, false);
TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), {
getInitialState: () => new StateForMockTokenizingMode(this.getId(), tokenType)
}, false));
}
}

210
src/vs/editor/test/common/model/model.modes.test.ts
View file

@ -9,38 +9,12 @@ import {EditOperation} from 'vs/editor/common/core/editOperation';
import {Position} from 'vs/editor/common/core/position';
import {Range} from 'vs/editor/common/core/range';
import {Model} from 'vs/editor/common/model/model';
import {ModelMode1, ModelMode2, NMode} from 'vs/editor/test/common/testModes';
import {AbstractState} from 'vs/editor/common/modes/abstractState';
import * as modes from 'vs/editor/common/modes';
import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
// --------- utils
function checkAndClear(highlighter, arr) {
assert.deepEqual(highlighter.calledFor, arr);
highlighter.calledFor = [];
}
function invalidEqual(model, indexArray) {
var i, len, asHash = {};
for (i = 0, len = indexArray.length; i < len; i++) {
asHash[indexArray[i]] = true;
}
for (i = 0, len = model.getLineCount(); i < len; i++) {
assert.equal(model._lines[i].isInvalid, asHash.hasOwnProperty(i));
}
}
function stateEqual(state, content) {
assert.equal(state.prevLineContent, content);
}
function statesEqual(model:Model, states:string[]) {
var i, len = states.length - 1;
for (i = 0; i < len; i++) {
stateEqual(model._lines[i].getState(), states[i]);
}
stateEqual((<any>model)._lastState, states[len]);
}
var LINE1 = '1';
var LINE2 = '2';
var LINE3 = '3';
@ -50,18 +24,45 @@ var LINE5 = '5';
suite('Editor Model - Model Modes 1', () => {
var thisHighlighter: ModelMode1;
var thisModel: Model;
const LANGUAGE_ID = 'modelModeTest1';
let calledState = {
calledFor: <string[]>[]
};
let thisModel: Model;
class ModelState1 extends AbstractState {
public makeClone():ModelState1 {
return this;
}
public equals(other: modes.IState): boolean {
return this === other;
}
public tokenize(stream:modes.IStream): modes.ITokenizationResult {
calledState.calledFor.push(stream.next());
stream.advanceToEOS();
return { type: '' };
}
}
function checkAndClear(calledState: { calledFor: string[] }, arr:string[]) {
assert.deepEqual(calledState.calledFor, arr);
calledState.calledFor = [];
}
modes.TokenizationRegistry.register(LANGUAGE_ID, new TokenizationSupport(null, LANGUAGE_ID, {
getInitialState: () => new ModelState1(LANGUAGE_ID)
}, false));
setup(() => {
thisHighlighter = new ModelMode1();
calledState.calledFor = [];
var text =
LINE1 + '\r\n' +
LINE2 + '\n' +
LINE3 + '\n' +
LINE4 + '\r\n' +
LINE5;
thisModel = Model.createFromString(text, undefined, thisHighlighter);
thisModel = Model.createFromString(text, undefined, LANGUAGE_ID);
});
teardown(() => {
@ -69,98 +70,98 @@ suite('Editor Model - Model Modes 1', () => {
});
test('model calls syntax highlighter 1', () => {
thisModel.getLineTokens(1);
checkAndClear(thisHighlighter, ['1']);
checkAndClear(calledState, ['1']);
});
test('model calls syntax highlighter 2', () => {
thisModel.getLineTokens(2);
checkAndClear(thisHighlighter, ['1', '2']);
checkAndClear(calledState, ['1', '2']);
thisModel.getLineTokens(2);
checkAndClear(thisHighlighter, []);
checkAndClear(calledState, []);
});
test('model caches states', () => {
thisModel.getLineTokens(1);
checkAndClear(thisHighlighter, ['1']);
checkAndClear(calledState, ['1']);
thisModel.getLineTokens(2);
checkAndClear(thisHighlighter, ['2']);
checkAndClear(calledState, ['2']);
thisModel.getLineTokens(3);
checkAndClear(thisHighlighter, ['3']);
checkAndClear(calledState, ['3']);
thisModel.getLineTokens(4);
checkAndClear(thisHighlighter, ['4']);
checkAndClear(calledState, ['4']);
thisModel.getLineTokens(5);
checkAndClear(thisHighlighter, ['5']);
checkAndClear(calledState, ['5']);
thisModel.getLineTokens(5);
checkAndClear(thisHighlighter, []);
checkAndClear(calledState, []);
});
test('model invalidates states for one line insert', () => {
thisModel.getLineTokens(5);
checkAndClear(thisHighlighter, ['1', '2', '3', '4', '5']);
checkAndClear(calledState, ['1', '2', '3', '4', '5']);
thisModel.applyEdits([EditOperation.insert(new Position(1, 1), '-')]);
thisModel.getLineTokens(5);
checkAndClear(thisHighlighter, ['-']);
checkAndClear(calledState, ['-']);
thisModel.getLineTokens(5);
checkAndClear(thisHighlighter, []);
checkAndClear(calledState, []);
});
test('model invalidates states for many lines insert', () => {
thisModel.getLineTokens(5);
checkAndClear(thisHighlighter, ['1', '2', '3', '4', '5']);
checkAndClear(calledState, ['1', '2', '3', '4', '5']);
thisModel.applyEdits([EditOperation.insert(new Position(1, 1), '0\n-\n+')]);
assert.equal(thisModel.getLineCount(), 7);
thisModel.getLineTokens(7);
checkAndClear(thisHighlighter, ['0', '-', '+']);
checkAndClear(calledState, ['0', '-', '+']);
thisModel.getLineTokens(7);
checkAndClear(thisHighlighter, []);
checkAndClear(calledState, []);
});
test('model invalidates states for one new line', () => {
thisModel.getLineTokens(5);
checkAndClear(thisHighlighter, ['1', '2', '3', '4', '5']);
checkAndClear(calledState, ['1', '2', '3', '4', '5']);
thisModel.applyEdits([EditOperation.insert(new Position(1, 2), '\n')]);
thisModel.applyEdits([EditOperation.insert(new Position(2, 1), 'a')]);
thisModel.getLineTokens(6);
checkAndClear(thisHighlighter, ['1', 'a']);
checkAndClear(calledState, ['1', 'a']);
});
test('model invalidates states for one line delete', () => {
thisModel.getLineTokens(5);
checkAndClear(thisHighlighter, ['1', '2', '3', '4', '5']);
checkAndClear(calledState, ['1', '2', '3', '4', '5']);
thisModel.applyEdits([EditOperation.insert(new Position(1, 2), '-')]);
thisModel.getLineTokens(5);
checkAndClear(thisHighlighter, ['1']);
checkAndClear(calledState, ['1']);
thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 2))]);
thisModel.getLineTokens(5);
checkAndClear(thisHighlighter, ['-']);
checkAndClear(calledState, ['-']);
thisModel.getLineTokens(5);
checkAndClear(thisHighlighter, []);
checkAndClear(calledState, []);
});
test('model invalidates states for many lines delete', () => {
thisModel.getLineTokens(5);
checkAndClear(thisHighlighter, ['1', '2', '3', '4', '5']);
checkAndClear(calledState, ['1', '2', '3', '4', '5']);
thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 3, 1))]);
thisModel.getLineTokens(3);
checkAndClear(thisHighlighter, ['3']);
checkAndClear(calledState, ['3']);
thisModel.getLineTokens(3);
checkAndClear(thisHighlighter, []);
checkAndClear(calledState, []);
});
});
@ -168,18 +169,70 @@ suite('Editor Model - Model Modes 1', () => {
suite('Editor Model - Model Modes 2', () => {
var thisHighlighter: ModelMode1;
const LANGUAGE_ID = 'modelModeTest2';
class ModelState2 extends AbstractState {
private prevLineContent:string;
constructor(modeId:string, prevLineContent:string) {
super(modeId);
this.prevLineContent = prevLineContent;
}
public makeClone():ModelState2 {
return new ModelState2(this.getModeId(), this.prevLineContent);
}
public equals(other: modes.IState):boolean {
return (other instanceof ModelState2) && (this.prevLineContent === (<ModelState2>other).prevLineContent);
}
public tokenize(stream:modes.IStream):modes.ITokenizationResult {
var line= '';
while (!stream.eos()) {
line+= stream.next();
}
this.prevLineContent= line;
return { type: '' };
}
}
modes.TokenizationRegistry.register(LANGUAGE_ID, new TokenizationSupport(null, LANGUAGE_ID, {
getInitialState: () => new ModelState2(LANGUAGE_ID, '')
}, false));
function invalidEqual(model, indexArray) {
var i, len, asHash = {};
for (i = 0, len = indexArray.length; i < len; i++) {
asHash[indexArray[i]] = true;
}
for (i = 0, len = model.getLineCount(); i < len; i++) {
assert.equal(model._lines[i].isInvalid, asHash.hasOwnProperty(i));
}
}
function stateEqual(state, content) {
assert.equal(state.prevLineContent, content);
}
function statesEqual(model:Model, states:string[]) {
var i, len = states.length - 1;
for (i = 0; i < len; i++) {
stateEqual(model._lines[i].getState(), states[i]);
}
stateEqual((<any>model)._lastState, states[len]);
}
var thisModel: Model;
setup(() => {
thisHighlighter = new ModelMode2();
var text =
'Line1' + '\r\n' +
'Line2' + '\n' +
'Line3' + '\n' +
'Line4' + '\r\n' +
'Line5';
thisModel = Model.createFromString(text, undefined, thisHighlighter);
thisModel = Model.createFromString(text, undefined, LANGUAGE_ID);
});
teardown(() => {
@ -251,15 +304,48 @@ suite('Editor Model - Model Modes 2', () => {
suite('Editor Model - Token Iterator', () => {
const LANGUAGE_ID = 'modelModeTestTokenIterator';
class NState extends AbstractState {
private n:number;
private allResults:modes.ITokenizationResult[];
constructor(modeId:string, n:number) {
super(modeId);
this.n = n;
this.allResults = null;
}
public makeClone():NState {
return this;
}
public equals(other: modes.IState):boolean {
return true;
}
public tokenize(stream:modes.IStream):modes.ITokenizationResult {
var ndash = this.n, value = '';
while(!stream.eos() && ndash > 0) {
value += stream.next();
ndash--;
}
return { type: 'n-' + (this.n - ndash) + '-' + value };
}
}
modes.TokenizationRegistry.register(LANGUAGE_ID, new TokenizationSupport(null, LANGUAGE_ID, {
getInitialState: () => new NState(LANGUAGE_ID, 3)
}, false));
var thisModel: Model;
setup(() => {
var nmode = new NMode(3);
var text =
'foobarfoobar' + '\r\n' +
'foobarfoobar' + '\r\n' +
'foobarfoobar' + '\r\n';
thisModel = Model.createFromString(text, undefined, nmode);
thisModel = Model.createFromString(text, undefined, LANGUAGE_ID);
});
teardown(() => {

59
src/vs/editor/test/common/model/textModelWithTokens.test.ts
View file

@ -7,7 +7,7 @@
import * as assert from 'assert';
import {Model} from 'vs/editor/common/model/model';
import {ViewLineToken} from 'vs/editor/common/core/viewLineToken';
import {ITokenizationSupport} from 'vs/editor/common/modes';
import {TokenizationRegistry} from 'vs/editor/common/modes';
import {MockMode} from 'vs/editor/test/common/mocks/mockMode';
import {Token} from 'vs/editor/common/core/token';
import {Range} from 'vs/editor/common/core/range';
@ -152,24 +152,21 @@ suite('TextModelWithTokens - bracket matching', () => {
], 'is matching brackets at ' + lineNumber1 + ', ' + column11);
}
class BracketMode extends MockMode {
constructor() {
super();
LanguageConfigurationRegistry.register(this.getId(), {
brackets: [
['{', '}'],
['[', ']'],
['(', ')'],
]
});
}
}
const LANGUAGE_ID = 'bracketMode1';
LanguageConfigurationRegistry.register(LANGUAGE_ID, {
brackets: [
['{', '}'],
['[', ']'],
['(', ')'],
]
});
test('bracket matching 1', () => {
let text =
')]}{[(' + '\n' +
')]}{[(';
let model = Model.createFromString(text, undefined, new BracketMode());
let model = Model.createFromString(text, undefined, LANGUAGE_ID);
isNotABracket(model, 1, 1);
isNotABracket(model, 1, 2);
@ -197,7 +194,7 @@ suite('TextModelWithTokens - bracket matching', () => {
'}, bar: {hallo: [{' + '\n' +
'}, {' + '\n' +
'}]}}';
let model = Model.createFromString(text, undefined, new BracketMode());
let model = Model.createFromString(text, undefined, LANGUAGE_ID);
let brackets = [
[1, 11, 12, 5, 4, 5],
@ -258,11 +255,9 @@ suite('TextModelWithTokens regression tests', () => {
let _tokenId = 0;
class IndicisiveMode extends MockMode {
public tokenizationSupport:ITokenizationSupport;
constructor() {
super();
this.tokenizationSupport = {
TokenizationRegistry.register(this.getId(), {
getInitialState: () => {
return null;
},
@ -276,7 +271,7 @@ suite('TextModelWithTokens regression tests', () => {
retokenize: null
};
}
};
});
}
}
let model = Model.createFromString('A model with\ntwo lines');
@ -284,12 +279,12 @@ suite('TextModelWithTokens regression tests', () => {
assertViewLineTokens(model, 1, true, [new ViewLineToken(0, '')]);
assertViewLineTokens(model, 2, true, [new ViewLineToken(0, '')]);
model.setMode(new IndicisiveMode());
model.setMode(new IndicisiveMode().getId());
assertViewLineTokens(model, 1, true, [new ViewLineToken(0, 'custom.1')]);
assertViewLineTokens(model, 2, true, [new ViewLineToken(0, 'custom.2')]);
model.setMode(new IndicisiveMode());
model.setMode(new IndicisiveMode().getId());
assertViewLineTokens(model, 1, false, [new ViewLineToken(0, '')]);
assertViewLineTokens(model, 2, false, [new ViewLineToken(0, '')]);
@ -298,17 +293,15 @@ suite('TextModelWithTokens regression tests', () => {
});
test('Microsoft/monaco-editor#133: Error: Cannot read property \'modeId\' of undefined', () => {
class BracketMode extends MockMode {
constructor() {
super();
LanguageConfigurationRegistry.register(this.getId(), {
brackets: [
['module','end module'],
['sub','end sub']
]
});
}
}
const LANGUAGE_ID = 'bracketMode2';
LanguageConfigurationRegistry.register(LANGUAGE_ID, {
brackets: [
['module','end module'],
['sub','end sub']
]
});
let model = Model.createFromString([
'Imports System',
@ -320,7 +313,7 @@ suite('TextModelWithTokens regression tests', () => {
'\tEnd Sub',
'',
'End Module',
].join('\n'), undefined, new BracketMode());
].join('\n'), undefined, LANGUAGE_ID);
let actual = model.matchBracket(new Position(4,1));
assert.deepEqual(actual, [new Range(4,1,4,7), new Range(9,1,9,11)]);

21
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
View file

@ -5,7 +5,7 @@
'use strict';
import * as assert from 'assert';
import {IMode, IStream, ITokenizationResult, ITokenizationSupport} from 'vs/editor/common/modes';
import {IStream, ITokenizationResult, TokenizationRegistry} from 'vs/editor/common/modes';
import {AbstractState} from 'vs/editor/common/modes/abstractState';
import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
import {tokenizeToHtmlContent} from 'vs/editor/common/modes/textToHtmlTokenizer';
@ -14,7 +14,7 @@ import {MockMode} from 'vs/editor/test/common/mocks/mockMode';
suite('Editor Modes - textToHtmlTokenizer', () => {
test('TextToHtmlTokenizer', () => {
var mode = new Mode();
var result = tokenizeToHtmlContent('.abc..def...gh', mode);
var result = tokenizeToHtmlContent('.abc..def...gh', mode.getId());
assert.ok(!!result);
@ -45,7 +45,7 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
assert.equal(children[5].className, 'token text');
assert.equal(children[5].tagName, 'span');
result = tokenizeToHtmlContent('.abc..def...gh\n.abc..def...gh', mode);
result = tokenizeToHtmlContent('.abc..def...gh\n.abc..def...gh', mode.getId());
assert.ok(!!result);
@ -59,12 +59,12 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
class State extends AbstractState {
constructor(mode:IMode) {
super(mode);
constructor(modeId:string) {
super(modeId);
}
public makeClone() : AbstractState {
return new State(this.getMode());
return new State(this.getModeId());
}
public tokenize(stream:IStream):ITokenizationResult {
@ -73,13 +73,10 @@ class State extends AbstractState {
}
class Mode extends MockMode {
public tokenizationSupport: ITokenizationSupport;
constructor() {
super();
this.tokenizationSupport = new TokenizationSupport(this, {
getInitialState: () => new State(this)
}, false);
TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), {
getInitialState: () => new State(this.getId())
}, false));
}
}

132
src/vs/editor/test/common/modes/tokenization.test.ts
View file

@ -5,58 +5,15 @@
'use strict';
import * as assert from 'assert';
import {IDisposable, empty as EmptyDisposable} from 'vs/base/common/lifecycle';
import {IModeSupportChangedEvent} from 'vs/editor/common/editorCommon';
import * as modes from 'vs/editor/common/modes';
import {AbstractState} from 'vs/editor/common/modes/abstractState';
import {handleEvent} from 'vs/editor/common/modes/supports';
import {IEnteringNestedModeData, ILeavingNestedModeData, TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
import {IModeLocator, ILeavingNestedModeData, TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
import {createMockLineContext} from 'vs/editor/test/common/modesTestUtils';
import {MockMode} from 'vs/editor/test/common/mocks/mockMode';
import {ModeTransition} from 'vs/editor/common/core/modeTransition';
import {Token} from 'vs/editor/common/core/token';
export class State extends AbstractState {
constructor(mode:modes.IMode) {
super(mode);
}
public makeClone() : AbstractState {
return new State(this.getMode());
}
public tokenize(stream:modes.IStream):modes.ITokenizationResult {
return { type: stream.next() === '.' ? '' : 'text' };
}
}
export class Mode extends MockMode {
public tokenizationSupport: modes.ITokenizationSupport;
constructor() {
super();
this.tokenizationSupport = new TokenizationSupport(this, {
getInitialState: () => new State(this)
}, false);
}
}
function checkTokens(actual, expected) {
assert.equal(actual.length, expected.length);
for (var i = 0; i < expected.length; i++) {
for (var key in expected[i]) {
assert.deepEqual(actual[i][key], expected[i][key]);
}
}
}
export interface IModeSwitchingDescriptor {
[character:string]:{
endCharacter: string;
@ -69,14 +26,14 @@ export class StateMemorizingLastWord extends AbstractState {
public lastWord:string;
private descriptor:IModeSwitchingDescriptor;
constructor(mode:modes.IMode, descriptor:IModeSwitchingDescriptor, lastWord:string) {
super(mode);
constructor(modeId:string, descriptor:IModeSwitchingDescriptor, lastWord:string) {
super(modeId);
this.lastWord = lastWord;
this.descriptor = descriptor;
}
public makeClone() : AbstractState {
return new StateMemorizingLastWord(this.getMode(), this.descriptor, this.lastWord);
return new StateMemorizingLastWord(this.getModeId(), this.descriptor, this.lastWord);
}
public tokenize(stream:modes.IStream):modes.ITokenizationResult {
@ -88,8 +45,8 @@ export class StateMemorizingLastWord extends AbstractState {
}
var word = stream.nextToken();
return {
type: this.getMode().getId() + '.' + word,
nextState: new StateMemorizingLastWord(this.getMode(), this.descriptor, word)
type: this.getModeId() + '.' + word,
nextState: new StateMemorizingLastWord(this.getModeId(), this.descriptor, word)
};
}
}
@ -98,24 +55,14 @@ export class SwitchingMode extends MockMode {
private _switchingModeDescriptor:IModeSwitchingDescriptor;
public tokenizationSupport: modes.ITokenizationSupport;
constructor(id:string, descriptor:IModeSwitchingDescriptor) {
super(id);
this._switchingModeDescriptor = descriptor;
this.tokenizationSupport = new TokenizationSupport(this, this, true);
}
setTokenizationSupport<T>(callback:(mode:modes.IMode)=>T): IDisposable {
return EmptyDisposable;
}
public addSupportChangedListener(callback: (e: IModeSupportChangedEvent) => void): IDisposable {
return EmptyDisposable;
modes.TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), this, true));
}
public getInitialState():modes.IState {
return new StateMemorizingLastWord(this, this._switchingModeDescriptor, null);
return new StateMemorizingLastWord(this.getId(), this._switchingModeDescriptor, null);
}
public enterNestedMode(state:modes.IState):boolean {
@ -125,12 +72,9 @@ export class SwitchingMode extends MockMode {
}
}
public getNestedMode(state:modes.IState): IEnteringNestedModeData {
public getNestedMode(state:modes.IState, locator:IModeLocator): modes.IMode {
var s = <StateMemorizingLastWord>state;
return {
mode: this._switchingModeDescriptor[s.lastWord].mode,
missingModePromise: null
};
return this._switchingModeDescriptor[s.lastWord].mode;
}
public getLeavingNestedModeData(line:string, state:modes.IState): ILeavingNestedModeData {
@ -141,7 +85,7 @@ export class SwitchingMode extends MockMode {
return {
nestedModeBuffer: line.substring(0, endCharPosition),
bufferAfterNestedMode: line.substring(endCharPosition),
stateAfterNestedMode: new StateMemorizingLastWord(this, this._switchingModeDescriptor, null)
stateAfterNestedMode: new StateMemorizingLastWord(this.getId(), this._switchingModeDescriptor, null)
};
}
return null;
@ -175,7 +119,7 @@ function assertModeTransitions(actual:ModeTransition[], expected:ITestModeTransi
assert.deepEqual(massagedActual, expected, message);
};
function createMode():SwitchingMode {
let switchingMode = (function() {
var modeB = new SwitchingMode('B', {});
var modeC = new SwitchingMode('C', {});
var modeD = new SwitchingMode('D', {
@ -199,23 +143,41 @@ function createMode():SwitchingMode {
}
});
return modeA;
}
})();
function switchingModeTokenize(line:string, mode:modes.IMode = null, state:modes.IState = null) {
if (state && mode) {
return mode.tokenizationSupport.tokenize(line, state);
function switchingModeTokenize(line:string, state:modes.IState = null) {
let tokenizationSupport = modes.TokenizationRegistry.get(switchingMode.getId());
if (state) {
return tokenizationSupport.tokenize(line, state);
} else {
mode = createMode();
return mode.tokenizationSupport.tokenize(line, mode.tokenizationSupport.getInitialState());
return tokenizationSupport.tokenize(line, tokenizationSupport.getInitialState());
}
}
suite('Editor Modes - Tokenization', () => {
test('Syntax engine merges sequential untyped tokens', () => {
var mode = new Mode();
var lineTokens = mode.tokenizationSupport.tokenize('.abc..def...gh', mode.tokenizationSupport.getInitialState());
checkTokens(lineTokens.tokens, [
class State extends AbstractState {
constructor(modeId:string) {
super(modeId);
}
public makeClone() : AbstractState {
return new State(this.getModeId());
}
public tokenize(stream:modes.IStream):modes.ITokenizationResult {
return { type: stream.next() === '.' ? '' : 'text' };
}
}
let tokenizationSupport = new TokenizationSupport(null, 'test', {
getInitialState: () => new State('test')
}, false);
var lineTokens = tokenizationSupport.tokenize('.abc..def...gh', tokenizationSupport.getInitialState());
assertTokens(lineTokens.tokens, [
{ startIndex: 0, type: '' },
{ startIndex: 1, type: 'text' },
{ startIndex: 4, type: '' },
@ -282,15 +244,14 @@ suite('Editor Modes - Tokenization', () => {
{ startIndex:3, type: '' },
{ startIndex:4, type: 'A.(' }
]);
assert.equal((<StateMemorizingLastWord>lineTokens.endState).getMode().getId(), 'B');
assert.equal((<StateMemorizingLastWord>lineTokens.endState).getModeId(), 'B');
assertModeTransitions(lineTokens.modeTransitions, [
{ startIndex: 0, id: 'A' }
]);
});
test('One embedded over multiple lines 1', () => {
var mode = createMode();
var lineTokens = switchingModeTokenize('abc (def', mode, mode.getInitialState());
var lineTokens = switchingModeTokenize('abc (def');
assertTokens(lineTokens.tokens, [
{ startIndex:0, type: 'A.abc' },
{ startIndex:3, type: '' },
@ -302,7 +263,7 @@ suite('Editor Modes - Tokenization', () => {
{ startIndex: 5, id: 'B' }
]);
lineTokens = switchingModeTokenize('ghi jkl', mode, lineTokens.endState);
lineTokens = switchingModeTokenize('ghi jkl', lineTokens.endState);
assertTokens(lineTokens.tokens, [
{ startIndex:0, type: 'B.ghi' },
{ startIndex:3, type: '' },
@ -312,7 +273,7 @@ suite('Editor Modes - Tokenization', () => {
{ startIndex: 0, id: 'B' }
]);
lineTokens = switchingModeTokenize('mno)pqr', mode, lineTokens.endState);
lineTokens = switchingModeTokenize('mno)pqr', lineTokens.endState);
assertTokens(lineTokens.tokens, [
{ startIndex:0, type: 'B.mno' },
{ startIndex:3, type: 'A.)' },
@ -325,8 +286,7 @@ suite('Editor Modes - Tokenization', () => {
});
test('One embedded over multiple lines 2 with handleEvent', () => {
var mode = createMode();
var lineTokens = switchingModeTokenize('abc (def', mode, mode.getInitialState());
var lineTokens = switchingModeTokenize('abc (def');
assertTokens(lineTokens.tokens, [
{ startIndex:0, type: 'A.abc' },
{ startIndex:3, type: '' },
@ -360,7 +320,7 @@ suite('Editor Modes - Tokenization', () => {
assert.equal(context.getLineContent(), 'def');
});
lineTokens = switchingModeTokenize('ghi jkl', mode, lineTokens.endState);
lineTokens = switchingModeTokenize('ghi jkl', lineTokens.endState);
assertTokens(lineTokens.tokens, [
{ startIndex:0, type: 'B.ghi' },
{ startIndex:3, type: '' },
@ -370,7 +330,7 @@ suite('Editor Modes - Tokenization', () => {
{ startIndex: 0, id: 'B' }
]);
lineTokens = switchingModeTokenize(')pqr', mode, lineTokens.endState);
lineTokens = switchingModeTokenize(')pqr', lineTokens.endState);
assertTokens(lineTokens.tokens, [
{ startIndex:0, type: 'A.)' },
{ startIndex:1, type: 'A.pqr' }

17
src/vs/editor/test/common/modesUtil.ts
View file

@ -7,9 +7,9 @@
import * as assert from 'assert';
import {Model} from 'vs/editor/common/model/model';
import * as modes from 'vs/editor/common/modes';
import {MockMode} from 'vs/editor/test/common/mocks/mockMode';
import {RichEditSupport, LanguageConfiguration} from 'vs/editor/common/modes/languageConfigurationRegistry';
import {Token} from 'vs/editor/common/core/token';
import {generateMockModeId} from 'vs/editor/test/common/mocks/mockMode';
export interface ITestToken {
startIndex: number;
@ -45,15 +45,17 @@ export interface IOnEnterAsserter {
indentsOutdents(oneLineAboveText:string, beforeText:string, afterText:string): void;
}
export function createOnEnterAsserter(modeId:string, conf: LanguageConfiguration): IOnEnterAsserter {
var assertOne = (oneLineAboveText:string, beforeText:string, afterText:string, expected: modes.IndentAction) => {
var model = Model.createFromString(
export function createOnEnterAsserter(conf: LanguageConfiguration): IOnEnterAsserter {
const modeId = generateMockModeId();
const assertOne = (oneLineAboveText:string, beforeText:string, afterText:string, expected: modes.IndentAction) => {
let model = Model.createFromString(
[ oneLineAboveText, beforeText + afterText ].join('\n'),
undefined,
new MockMode(modeId)
modeId
);
var richEditSupport = new RichEditSupport(modeId, null, conf);
var actual = richEditSupport.onEnter.onEnter(model, { lineNumber: 2, column: beforeText.length + 1 });
let richEditSupport = new RichEditSupport(modeId, null, conf);
let actual = richEditSupport.onEnter.onEnter(model, { lineNumber: 2, column: beforeText.length + 1 });
if (expected === modes.IndentAction.None) {
assert.equal(actual, null, oneLineAboveText + '\\n' + beforeText + '|' + afterText);
} else {
@ -61,6 +63,7 @@ export function createOnEnterAsserter(modeId:string, conf: LanguageConfiguration
}
model.dispose();
};
return {
nothing: (oneLineAboveText:string, beforeText:string, afterText:string): void => {
assertOne(oneLineAboveText, beforeText, afterText, modes.IndentAction.None);

167
src/vs/editor/test/common/testModes.ts
View file

@ -4,181 +4,14 @@
*--------------------------------------------------------------------------------------------*/
'use strict';
import * as modes from 'vs/editor/common/modes';
import {AbstractState} from 'vs/editor/common/modes/abstractState';
import {LanguageConfigurationRegistry, CommentRule} from 'vs/editor/common/modes/languageConfigurationRegistry';
import {TokenizationSupport} from 'vs/editor/common/modes/supports/tokenizationSupport';
import {MockMode} from 'vs/editor/test/common/mocks/mockMode';
export class CommentState extends AbstractState {
constructor(mode:modes.IMode, stateCount:number) {
super(mode);
}
public makeClone():CommentState {
return this;
}
public equals(other:modes.IState):boolean {
return true;
}
public tokenize(stream:modes.IStream):modes.ITokenizationResult {
stream.advanceToEOS();
return { type: 'state' };
}
}
export class CommentMode extends MockMode {
public tokenizationSupport: modes.ITokenizationSupport;
constructor(commentsConfig:CommentRule) {
super();
this.tokenizationSupport = new TokenizationSupport(this, {
getInitialState: () => new CommentState(this, 0)
}, false);
LanguageConfigurationRegistry.register(this.getId(), {
comments: commentsConfig
});
}
}
export abstract class AbstractIndentingMode extends MockMode {
public getElectricCharacters():string[] {
return null;
}
public onElectricCharacter(context:modes.ILineContext, offset:number):modes.IElectricAction {
return null;
}
public onEnter(context:modes.ILineContext, offset:number):modes.EnterAction {
return null;
}
}
export class ModelState1 extends AbstractState {
constructor(mode:modes.IMode) {
super(mode);
}
public makeClone():ModelState1 {
return this;
}
public equals(other: modes.IState):boolean {
return this === other;
}
public tokenize(stream:modes.IStream):modes.ITokenizationResult {
(<ModelMode1>this.getMode()).calledFor.push(stream.next());
stream.advanceToEOS();
return { type: '' };
}
}
export class ModelMode1 extends MockMode {
public calledFor:string[];
public tokenizationSupport: modes.ITokenizationSupport;
constructor() {
super();
this.calledFor = [];
this.tokenizationSupport = new TokenizationSupport(this, {
getInitialState: () => new ModelState1(this)
}, false);
}
}
export class ModelState2 extends AbstractState {
private prevLineContent:string;
constructor(mode:ModelMode2, prevLineContent:string) {
super(mode);
this.prevLineContent = prevLineContent;
}
public makeClone():ModelState2 {
return new ModelState2(<ModelMode2>this.getMode(), this.prevLineContent);
}
public equals(other: modes.IState):boolean {
return (other instanceof ModelState2) && (this.prevLineContent === (<ModelState2>other).prevLineContent);
}
public tokenize(stream:modes.IStream):modes.ITokenizationResult {
var line= '';
while (!stream.eos()) {
line+= stream.next();
}
this.prevLineContent= line;
return { type: '' };
}
}
export class ModelMode2 extends MockMode {
public calledFor:any[];
public tokenizationSupport: modes.ITokenizationSupport;
constructor() {
super();
this.calledFor = null;
this.tokenizationSupport = new TokenizationSupport(this, {
getInitialState: () => new ModelState2(this, '')
}, false);
}
}
export class NState extends AbstractState {
private n:number;
private allResults:modes.ITokenizationResult[];
constructor(mode:modes.IMode, n:number) {
super(mode);
this.n = n;
this.allResults = null;
}
public makeClone():NState {
return this;
}
public equals(other: modes.IState):boolean {
return true;
}
public tokenize(stream:modes.IStream):modes.ITokenizationResult {
var ndash = this.n, value = '';
while(!stream.eos() && ndash > 0) {
value += stream.next();
ndash--;
}
return { type: 'n-' + (this.n - ndash) + '-' + value };
}
}
export class NMode extends MockMode {
private n:number;
public tokenizationSupport: modes.ITokenizationSupport;
constructor(n:number) {
super();
this.n = n;
this.tokenizationSupport = new TokenizationSupport(this, {
getInitialState: () => new NState(this, this.n)
}, false);
}
}

14
src/vs/languages/handlebars/common/handlebars.ts
View file

@ -12,11 +12,11 @@ import {IInstantiationService} from 'vs/platform/instantiation/common/instantiat
import {IModeService} from 'vs/editor/common/services/modeService';
import {LanguageConfigurationRegistry, LanguageConfiguration} from 'vs/editor/common/modes/languageConfigurationRegistry';
import {createWordRegExp} from 'vs/editor/common/modes/abstractMode';
import {ILeavingNestedModeData} from 'vs/editor/common/modes/supports/tokenizationSupport';
import {wireCancellationToken} from 'vs/base/common/async';
import {ICompatWorkerService} from 'vs/editor/common/services/compatWorkerService';
import {IWorkspaceContextService} from 'vs/platform/workspace/common/workspace';
import {IConfigurationService} from 'vs/platform/configuration/common/configuration';
import {TokenizationSupport, ILeavingNestedModeData} from 'vs/editor/common/modes/supports/tokenizationSupport';
export enum States {
HTML,
@ -26,7 +26,7 @@ export enum States {
export class HandlebarsState extends htmlMode.State {
constructor(mode:modes.IMode,
constructor(modeId:string,
kind:htmlMode.States,
public handlebarsKind:States,
lastTagName:string,
@ -35,11 +35,11 @@ export class HandlebarsState extends htmlMode.State {
attributeValueQuote:string,
attributeValueLength:number) {
super(mode, kind, lastTagName, lastAttributeName, embeddedContentType, attributeValueQuote, attributeValueLength);
super(modeId, kind, lastTagName, lastAttributeName, embeddedContentType, attributeValueQuote, attributeValueLength);
}
public makeClone(): HandlebarsState {
return new HandlebarsState(this.getMode(), this.kind, this.handlebarsKind, this.lastTagName, this.lastAttributeName, this.embeddedContentType, this.attributeValueQuote, this.attributeValueLength);
return new HandlebarsState(this.getModeId(), this.kind, this.handlebarsKind, this.lastTagName, this.lastAttributeName, this.embeddedContentType, this.attributeValueQuote, this.attributeValueLength);
}
public equals(other:modes.IState):boolean {
@ -184,16 +184,18 @@ export class HandlebarsMode extends htmlMode.HTMLMode<htmlWorker.HTMLWorker> {
}, true);
LanguageConfigurationRegistry.register(this.getId(), HandlebarsMode.LANG_CONFIG);
modes.TokenizationRegistry.register(this.getId(), new TokenizationSupport(this._modeService, this.getId(), this, true));
}
public getInitialState() : modes.IState {
return new HandlebarsState(this, htmlMode.States.Content, States.HTML, '', '', '', '', 0);
return new HandlebarsState(this.getId(), htmlMode.States.Content, States.HTML, '', '', '', '', 0);
}
public getLeavingNestedModeData(line:string, state:modes.IState):ILeavingNestedModeData {
var leavingNestedModeData = super.getLeavingNestedModeData(line, state);
if (leavingNestedModeData) {
leavingNestedModeData.stateAfterNestedMode = new HandlebarsState(this, htmlMode.States.Content, States.HTML, '', '', '', '', 0);
leavingNestedModeData.stateAfterNestedMode = new HandlebarsState(this.getId(), htmlMode.States.Content, States.HTML, '', '', '', '', 0);
}
return leavingNestedModeData;
}

24
src/vs/languages/handlebars/test/common/handlebars.test.ts
View file

@ -35,11 +35,21 @@ class HandlebarsMockModeService extends MockModeService {
throw new Error('Not implemented');
}
getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): Modes.IMode {
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'text/javascript') {
return new MockTokenizingMode('js', 'mock-js');
getModeId(mimetypeOrModeId: string): string {
if (mimetypeOrModeId === 'text/javascript') {
return 'js-mode-id';
}
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'text/x-handlebars-template') {
if (mimetypeOrModeId === 'text/x-handlebars-template') {
return 'handlebars-mode-id';
}
throw new Error('Not implemented');
}
getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): Modes.IMode {
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'js-mode-id') {
return new MockTokenizingMode('mock-js');
}
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'handlebars-mode-id') {
return this._handlebarsMode;
}
throw new Error('Not implemented');
@ -49,7 +59,7 @@ class HandlebarsMockModeService extends MockModeService {
suite('Handlebars', () => {
var tokenizationSupport: Modes.ITokenizationSupport;
(function() {
suiteSetup(function() {
let modeService = new HandlebarsMockModeService();
let mode = new HandlebarsMode(
@ -63,8 +73,8 @@ suite('Handlebars', () => {
modeService.setHandlebarsMode(mode);
tokenizationSupport = mode.tokenizationSupport;
})();
tokenizationSupport = Modes.TokenizationRegistry.get(mode.getId());
});
test('Just HTML', () => {
modesUtil.assertTokenization(tokenizationSupport, [{

58
src/vs/languages/html/common/html.ts
View file

@ -16,7 +16,7 @@ import {IInstantiationService} from 'vs/platform/instantiation/common/instantiat
import * as htmlTokenTypes from 'vs/languages/html/common/htmlTokenTypes';
import {EMPTY_ELEMENTS} from 'vs/languages/html/common/htmlEmptyTagsShared';
import {LanguageConfigurationRegistry, LanguageConfiguration} from 'vs/editor/common/modes/languageConfigurationRegistry';
import {TokenizationSupport, IEnteringNestedModeData, ILeavingNestedModeData, ITokenizationCustomization} from 'vs/editor/common/modes/supports/tokenizationSupport';
import {TokenizationSupport, IModeLocator, ILeavingNestedModeData, ITokenizationCustomization} from 'vs/editor/common/modes/supports/tokenizationSupport';
import {wireCancellationToken} from 'vs/base/common/async';
import {ICompatWorkerService, CompatWorkerAttr} from 'vs/editor/common/services/compatWorkerService';
import {IWorkspaceContextService} from 'vs/platform/workspace/common/workspace';
@ -52,8 +52,8 @@ export class State extends AbstractState {
public attributeValueQuote:string;
public attributeValueLength:number;
constructor(mode:modes.IMode, kind:States, lastTagName:string, lastAttributeName:string, embeddedContentType:string, attributeValueQuote:string, attributeValueLength:number) {
super(mode);
constructor(modeId:string, kind:States, lastTagName:string, lastAttributeName:string, embeddedContentType:string, attributeValueQuote:string, attributeValueLength:number) {
super(modeId);
this.kind = kind;
this.lastTagName = lastTagName;
this.lastAttributeName = lastAttributeName;
@ -67,7 +67,7 @@ export class State extends AbstractState {
}
public makeClone():State {
return new State(this.getMode(), this.kind, this.lastTagName, this.lastAttributeName, this.embeddedContentType, this.attributeValueQuote, this.attributeValueLength);
return new State(this.getModeId(), this.kind, this.lastTagName, this.lastAttributeName, this.embeddedContentType, this.attributeValueQuote, this.attributeValueLength);
}
public equals(other:modes.IState):boolean {
@ -330,9 +330,7 @@ export class HTMLMode<W extends htmlWorker.HTMLWorker> extends CompatMode implem
],
};
public tokenizationSupport: modes.ITokenizationSupport;
private modeService:IModeService;
protected _modeService:IModeService;
private _modeWorkerManager: ModeWorkerManager<W>;
constructor(
@ -346,9 +344,7 @@ export class HTMLMode<W extends htmlWorker.HTMLWorker> extends CompatMode implem
super(descriptor.id, compatWorkerService);
this._modeWorkerManager = this._createModeWorkerManager(descriptor, instantiationService);
this.modeService = modeService;
this.tokenizationSupport = new TokenizationSupport(this, this, true);
this._modeService = modeService;
if (this.compatWorkerService && this.compatWorkerService.isInMainThread) {
let updateConfiguration = () => {
@ -393,6 +389,8 @@ export class HTMLMode<W extends htmlWorker.HTMLWorker> extends CompatMode implem
}, true);
LanguageConfigurationRegistry.register(this.getId(), HTMLMode.LANG_CONFIG);
modes.TokenizationRegistry.register(this.getId(), new TokenizationSupport(this._modeService, this.getId(), this, true));
}
protected _createModeWorkerManager(descriptor:modes.IModeDescriptor, instantiationService: IInstantiationService): ModeWorkerManager<W> {
@ -406,43 +404,25 @@ export class HTMLMode<W extends htmlWorker.HTMLWorker> extends CompatMode implem
// TokenizationSupport
public getInitialState():modes.IState {
return new State(this, States.Content, '', '', '', '', 0);
return new State(this.getId(), States.Content, '', '', '', '', 0);
}
public enterNestedMode(state:modes.IState):boolean {
return state instanceof State && (<State>state).kind === States.WithinEmbeddedContent;
}
public getNestedMode(state:modes.IState): IEnteringNestedModeData {
var result:modes.IMode = null;
var htmlState:State = <State>state;
var missingModePromise: winjs.Promise = null;
public getNestedMode(state:modes.IState, locator:IModeLocator): modes.IMode {
let htmlState:State = <State>state;
if (htmlState.embeddedContentType !== null) {
if (this.modeService.isRegisteredMode(htmlState.embeddedContentType)) {
result = this.modeService.getMode(htmlState.embeddedContentType);
if (!result) {
missingModePromise = this.modeService.getOrCreateMode(htmlState.embeddedContentType);
}
}
} else {
var mimeType:string = null;
if ('script' === htmlState.lastTagName) {
mimeType = 'text/javascript';
} else if ('style' === htmlState.lastTagName) {
mimeType = 'text/css';
} else {
mimeType = 'text/plain';
}
result = this.modeService.getMode(mimeType);
return locator.getMode(htmlState.embeddedContentType);
}
if (result === null) {
result = this.modeService.getMode('text/plain');
if ('script' === htmlState.lastTagName) {
return locator.getMode('text/javascript');
}
return {
mode: result,
missingModePromise: missingModePromise
};
if ('style' === htmlState.lastTagName) {
return locator.getMode('text/css');
}
return null;
}
public getLeavingNestedModeData(line:string, state:modes.IState):ILeavingNestedModeData {
@ -453,7 +433,7 @@ export class HTMLMode<W extends htmlWorker.HTMLWorker> extends CompatMode implem
return {
nestedModeBuffer: line.substring(0, match.index),
bufferAfterNestedMode: line.substring(match.index),
stateAfterNestedMode: new State(this, States.Content, '', '', '', '', 0)
stateAfterNestedMode: new State(this.getId(), States.Content, '', '', '', '', 0)
};
}
return null;

2
src/vs/languages/html/test/common/html-worker.test.ts
View file

@ -16,7 +16,7 @@ import {MockModeService} from 'vs/editor/test/common/mocks/mockModeService';
import {TextModel} from 'vs/editor/common/model/textModel';
function createTestMirrorModelFromString(value:string, mode:Modes.IMode, associatedResource:URI): mm.CompatMirrorModel {
return new mm.CompatMirrorModel(0, TextModel.toRawText(value, TextModel.DEFAULT_CREATION_OPTIONS), mode, associatedResource);
return new mm.CompatMirrorModel(0, TextModel.toRawText(value, TextModel.DEFAULT_CREATION_OPTIONS), mode.getId(), associatedResource);
}
suite('HTML - worker', () => {

32
src/vs/languages/html/test/common/html.test.ts
View file

@ -21,7 +21,7 @@ import {LanguageConfigurationRegistry} from 'vs/editor/common/modes/languageConf
class MockJSMode extends MockTokenizingMode {
constructor() {
super('html-js-mock', 'mock-js');
super('mock-js');
LanguageConfigurationRegistry.register(this.getId(), {
brackets: [
@ -63,6 +63,9 @@ class MockJSMode extends MockTokenizingMode {
}
class HTMLMockModeService extends MockModeService {
private _mockJSMode = new MockJSMode();
isRegisteredMode(mimetypeOrModeId: string): boolean {
if (mimetypeOrModeId === 'text/javascript') {
return true;
@ -73,12 +76,16 @@ class HTMLMockModeService extends MockModeService {
throw new Error('Not implemented');
}
getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): Modes.IMode {
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'text/javascript') {
return new MockJSMode();
getModeId(mimetypeOrModeId: string): string {
if (mimetypeOrModeId === 'text/javascript') {
return 'js-mode-id';
}
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'text/plain') {
return null;
throw new Error('Not implemented');
}
getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): Modes.IMode {
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'js-mode-id') {
return this._mockJSMode;
}
throw new Error('Not implemented');
}
@ -90,7 +97,8 @@ suite('Colorizing - HTML', () => {
let _mode: Modes.IMode;
let onEnterSupport: Modes.IRichEditOnEnter;
(function() {
suiteSetup(function() {
_mode = new HTMLMode<htmlWorker.HTMLWorker>(
{ id: 'html' },
null,
@ -100,10 +108,10 @@ suite('Colorizing - HTML', () => {
null
);
tokenizationSupport = _mode.tokenizationSupport;
tokenizationSupport = Modes.TokenizationRegistry.get(_mode.getId());
onEnterSupport = LanguageConfigurationRegistry.getOnEnterSupport(_mode.getId());
})();
});
test('Open Start Tag #1', () => {
modesUtil.assertTokenization(tokenizationSupport, [{
@ -694,7 +702,7 @@ suite('Colorizing - HTML', () => {
});
test('onEnter 1', function() {
var model = Model.createFromString('<script type=\"text/javascript\">function f() { foo(); }', undefined, _mode);
var model = Model.createFromString('<script type=\"text/javascript\">function f() { foo(); }', undefined, _mode.getId());
var actual = onEnterSupport.onEnter(model, {
lineNumber: 1,
@ -708,7 +716,7 @@ suite('Colorizing - HTML', () => {
test('onEnter 2', function() {
function onEnter(line:string, offset:number): Modes.EnterAction {
let model = new TextModelWithTokens([], TextModel.toRawText(line, TextModel.DEFAULT_CREATION_OPTIONS), _mode);
let model = new TextModelWithTokens([], TextModel.toRawText(line, TextModel.DEFAULT_CREATION_OPTIONS), _mode.getId());
let result = LanguageConfigurationRegistry.getRawEnterActionAtPosition(model, 1, offset + 1);
model.dispose();
return result;
@ -751,7 +759,7 @@ suite('Colorizing - HTML', () => {
}
function assertBracket(lines:string[], lineNumber:number, column:number, expected:[Range, Range]): void {
let model = new TextModelWithTokens([], TextModel.toRawText(lines.join('\n'), TextModel.DEFAULT_CREATION_OPTIONS), _mode);
let model = new TextModelWithTokens([], TextModel.toRawText(lines.join('\n'), TextModel.DEFAULT_CREATION_OPTIONS), _mode.getId());
// force tokenization
model.getLineContext(model.getLineCount());
let actual = model.matchBracket({

88
src/vs/languages/php/common/php.ts
View file

@ -4,7 +4,6 @@
*--------------------------------------------------------------------------------------------*/
'use strict';
import WinJS = require('vs/base/common/winjs.base');
import objects = require('vs/base/common/objects');
import Modes = require('vs/editor/common/modes');
import {CompatMode, isDigit, createWordRegExp} from 'vs/editor/common/modes/abstractMode';
@ -110,14 +109,14 @@ var isVariable = (character:string) => {
return (character[0] === '$');
};
export class PHPState extends AbstractState {
export abstract class PHPState extends AbstractState {
private name:string;
private whitespaceTokenType:string;
public parent:Modes.IState;
constructor(mode:Modes.IMode, name:string, parent:Modes.IState, whitespaceTokenType:string='') {
super(mode);
constructor(modeId:string, name:string, parent:Modes.IState, whitespaceTokenType:string='') {
super(modeId);
this.name = name;
this.parent = parent;
this.whitespaceTokenType = whitespaceTokenType;
@ -154,14 +153,14 @@ export class PHPString extends PHPState {
private delimiter:string;
private isAtBeginning:boolean;
constructor(mode:Modes.IMode, parent:Modes.IState, delimiter:string, isAtBeginning:boolean=true) {
super(mode, 'string', parent, 'string.php');
constructor(modeId:string, parent:Modes.IState, delimiter:string, isAtBeginning:boolean=true) {
super(modeId, 'string', parent, 'string.php');
this.delimiter = delimiter;
this.isAtBeginning = isAtBeginning;
}
public makeClone():AbstractState {
return new PHPString(this.getMode(), AbstractState.safeClone(this.parent), this.delimiter, this.isAtBeginning);
return new PHPString(this.getModeId(), AbstractState.safeClone(this.parent), this.delimiter, this.isAtBeginning);
}
public equals(other:Modes.IState):boolean {
@ -205,13 +204,13 @@ export class PHPNumber extends PHPState {
private firstDigit:string;
constructor(mode:Modes.IMode, parent:Modes.IState, firstDigit:string) {
super(mode, 'number', parent);
constructor(modeId:string, parent:Modes.IState, firstDigit:string) {
super(modeId, 'number', parent);
this.firstDigit = firstDigit;
}
public makeClone():AbstractState {
return new PHPNumber(this.getMode(), AbstractState.safeClone(this.parent), this.firstDigit);
return new PHPNumber(this.getModeId(), AbstractState.safeClone(this.parent), this.firstDigit);
}
public equals(other:Modes.IState):boolean {
@ -276,12 +275,12 @@ export class PHPNumber extends PHPState {
export class PHPLineComment extends PHPState {
constructor(mode:Modes.IMode, parent:Modes.IState) {
super(mode, 'comment', parent, 'comment.php');
constructor(modeId:string, parent:Modes.IState) {
super(modeId, 'comment', parent, 'comment.php');
}
public makeClone():AbstractState {
return new PHPDocComment(this.getMode(), AbstractState.safeClone(this.parent));
return new PHPDocComment(this.getModeId(), AbstractState.safeClone(this.parent));
}
public equals(other:Modes.IState):boolean {
@ -307,12 +306,12 @@ export class PHPLineComment extends PHPState {
export class PHPDocComment extends PHPState {
constructor(mode:Modes.IMode, parent:Modes.IState) {
super(mode, 'comment', parent, 'comment.php');
constructor(modeId:string, parent:Modes.IState) {
super(modeId, 'comment', parent, 'comment.php');
}
public makeClone():AbstractState {
return new PHPDocComment(this.getMode(), AbstractState.safeClone(this.parent));
return new PHPDocComment(this.getModeId(), AbstractState.safeClone(this.parent));
}
public equals(other:Modes.IState):boolean {
@ -338,12 +337,12 @@ export class PHPDocComment extends PHPState {
export class PHPStatement extends PHPState {
constructor(mode:Modes.IMode, parent:Modes.IState) {
super(mode, 'expression', parent);
constructor(modeId:string, parent:Modes.IState) {
super(modeId, 'expression', parent);
}
public makeClone():AbstractState {
return new PHPStatement(this.getMode(), AbstractState.safeClone(this.parent));
return new PHPStatement(this.getModeId(), AbstractState.safeClone(this.parent));
}
public equals(other:Modes.IState):boolean {
@ -357,7 +356,7 @@ export class PHPStatement extends PHPState {
public stateTokenize(stream:Modes.IStream):Modes.ITokenizationResult {
if (isDigit(stream.peek(), 10)) {
return { nextState: new PHPNumber(this.getMode(), this, stream.next()) };
return { nextState: new PHPNumber(this.getModeId(), this, stream.next()) };
}
if (stream.advanceIfString('?>').length) {
return { type: 'metatag.php', nextState: this.parent };
@ -376,16 +375,16 @@ export class PHPStatement extends PHPState {
if (!stream.eos() && !stream.peekWhitespace()) {
switch(stream.peekToken()) {
case '/':
return { nextState: new PHPLineComment(this.getMode(), this) };
return { nextState: new PHPLineComment(this.getModeId(), this) };
case '*':
stream.nextToken();
return { nextState: new PHPDocComment(this.getMode(), this) };
return { nextState: new PHPDocComment(this.getModeId(), this) };
}
}
} else if (token === '#') {
return { nextState: new PHPLineComment(this.getMode(), this) };
return { nextState: new PHPLineComment(this.getModeId(), this) };
} else if (token === '"' || token === '\'') {
return { nextState: new PHPString(this.getMode(), this, token) };
return { nextState: new PHPString(this.getModeId(), this, token) };
} else if (brackets.stringIsBracket(token)) {
return {
type: brackets.tokenTypeFromString(token)
@ -399,12 +398,12 @@ export class PHPStatement extends PHPState {
export class PHPPlain extends PHPState {
constructor(mode:Modes.IMode, parent:Modes.IState) {
super(mode, 'plain', parent);
constructor(modeId:string, parent:Modes.IState) {
super(modeId, 'plain', parent);
}
public makeClone():AbstractState {
return new PHPPlain(this.getMode(), AbstractState.safeClone(this.parent));
return new PHPPlain(this.getModeId(), AbstractState.safeClone(this.parent));
}
public equals(other:Modes.IState):boolean {
@ -422,7 +421,7 @@ export class PHPPlain extends PHPState {
stream.advanceIfString('<?').length || stream.advanceIfString('<%').length) {
return {
type: 'metatag.php',
nextState: new PHPStatement(this.getMode(), new PHPEnterHTMLState(this.getMode(), this.parent))
nextState: new PHPStatement(this.getModeId(), new PHPEnterHTMLState(this.getModeId(), this.parent))
};
}
stream.next();
@ -432,12 +431,12 @@ export class PHPPlain extends PHPState {
export class PHPEnterHTMLState extends PHPState {
constructor(mode:Modes.IMode, parent:Modes.IState) {
super(mode, 'enterHTML', parent);
constructor(modeId:string, parent:Modes.IState) {
super(modeId, 'enterHTML', parent);
}
public makeClone():AbstractState {
return new PHPEnterHTMLState(this.getMode(), AbstractState.safeClone(this.parent));
return new PHPEnterHTMLState(this.getModeId(), AbstractState.safeClone(this.parent));
}
public equals(other:Modes.IState):boolean {
@ -476,9 +475,7 @@ export class PHPMode extends CompatMode implements ITokenizationCustomization {
]
};
public tokenizationSupport: Modes.ITokenizationSupport;
private modeService:IModeService;
private _modeService:IModeService;
constructor(
descriptor:Modes.IModeDescriptor,
@ -488,12 +485,12 @@ export class PHPMode extends CompatMode implements ITokenizationCustomization {
@ICompatWorkerService compatWorkerService: ICompatWorkerService
) {
super(descriptor.id, compatWorkerService);
this.modeService = modeService;
this.tokenizationSupport = new TokenizationSupport(this, this, true);
this._modeService = modeService;
LanguageConfigurationRegistry.register(this.getId(), PHPMode.LANG_CONFIG);
Modes.TokenizationRegistry.register(this.getId(), new TokenizationSupport(this._modeService, this.getId(), this, true));
if (editorWorkerService) {
Modes.SuggestRegistry.register(this.getId(), new TextualSuggestSupport(editorWorkerService, configurationService), true);
}
@ -502,9 +499,9 @@ export class PHPMode extends CompatMode implements ITokenizationCustomization {
public getInitialState():Modes.IState {
// Because AbstractMode doesn't allow the initial state to immediately enter a nested
// mode, we will enter a nested mode ourselves
var htmlMode = this.modeService.getMode('text/html');
var htmlState:Modes.IState = htmlMode.tokenizationSupport.getInitialState();
htmlState.setStateData(new PHPEnterHTMLState(this, null));
var htmlMode = this._modeService.getMode('text/html');
var htmlState:Modes.IState = Modes.TokenizationRegistry.get(htmlMode.getId()).getInitialState();
htmlState.setStateData(new PHPEnterHTMLState(this.getId(), null));
return htmlState;
}
@ -512,26 +509,23 @@ export class PHPMode extends CompatMode implements ITokenizationCustomization {
return state instanceof PHPEnterHTMLState;
}
public getNestedModeInitialState(myState:Modes.IState): { state:Modes.IState; missingModePromise:WinJS.Promise; } {
public getNestedModeInitialState(myState:Modes.IState): Modes.IState {
// Recall previous HTML state, that was saved in .parent, and carried over by the PHP states
// Also, prevent a .clone() endless loop by clearing the .parent pointer
// (the result will have its stateData point to myState)
var result = (<PHPState>myState).parent;
(<PHPState>myState).parent = null;
return {
state: result,
missingModePromise: null
};
return result;
}
public getLeavingNestedModeData(line:string, state:Modes.IState):ILeavingNestedModeData {
public getLeavingNestedModeData(line:string, state:Modes.IState): ILeavingNestedModeData {
// Leave HTML if <? is found on a line
var match:any = /<\?/i.exec(line);
if (match !== null) {
return {
nestedModeBuffer: line.substring(0, match.index),
bufferAfterNestedMode: line.substring(match.index),
stateAfterNestedMode: new PHPPlain(this, null)
stateAfterNestedMode: new PHPPlain(this.getId(), null)
};
}
return null;

56
src/vs/languages/php/test/common/php.test.ts
View file

@ -25,28 +25,54 @@ class PHPMockModeService extends MockModeService {
this._htmlMode = htmlMode;
}
getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): Modes.IMode {
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'text/html') {
return this._htmlMode;
isRegisteredMode(mimetypeOrModeId: string): boolean {
if (mimetypeOrModeId === 'text/html') {
return true;
}
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'text/javascript') {
return new MockTokenizingMode('js', 'mock-js');
if (mimetypeOrModeId === 'text/javascript') {
return true;
}
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'text/css') {
return new MockTokenizingMode('css', 'mock-css');
if (mimetypeOrModeId === 'text/css') {
return true;
}
throw new Error('Not implemented');
}
getModeId(mimetypeOrModeId: string): string {
if (mimetypeOrModeId === 'text/html') {
return 'html-mode-id';
}
if (mimetypeOrModeId === 'text/javascript') {
return 'js-mode-id';
}
if (mimetypeOrModeId === 'text/css') {
return 'css-mode-id';
}
throw new Error('Not implemented');
}
getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): Modes.IMode {
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'html-mode-id' || commaSeparatedMimetypesOrCommaSeparatedIds === 'text/html') {
return this._htmlMode;
}
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'js-mode-id') {
return new MockTokenizingMode('mock-js');
}
if (commaSeparatedMimetypesOrCommaSeparatedIds === 'css-mode-id') {
return new MockTokenizingMode('mock-css');
}
throw new Error('Not implemented: ' + commaSeparatedMimetypesOrCommaSeparatedIds);
}
}
suite('Syntax Highlighting - PHP', () => {
var wordDefinition:RegExp;
var assertWords = modesUtil.assertWords;
var tokenizationSupport: Modes.ITokenizationSupport;
var assertOnEnter: modesUtil.IOnEnterAsserter;
let wordDefinition:RegExp;
let assertWords = modesUtil.assertWords;
let tokenizationSupport: Modes.ITokenizationSupport;
let assertOnEnter: modesUtil.IOnEnterAsserter;
(function() {
suiteSetup(function() {
let modeService = new PHPMockModeService();
modeService.setHTMLMode(new HTMLMode<any>(
@ -66,10 +92,10 @@ suite('Syntax Highlighting - PHP', () => {
null
);
tokenizationSupport = mode.tokenizationSupport;
assertOnEnter = modesUtil.createOnEnterAsserter(mode.getId(), PHPMode.LANG_CONFIG);
tokenizationSupport = Modes.TokenizationRegistry.get(mode.getId());
assertOnEnter = modesUtil.createOnEnterAsserter(PHPMode.LANG_CONFIG);
wordDefinition = LanguageConfigurationRegistry.getWordDefinition(mode.getId());
})();
});
test('', () => {
modesUtil.executeTests(tokenizationSupport, [

80
src/vs/languages/razor/common/csharpTokenization.ts
View file

@ -82,13 +82,13 @@ var ispunctuation = (character:string) => {
return punctuations.indexOf(character) > -1;
};
export class CSState extends AbstractState {
export abstract class CSState extends AbstractState {
public name:string;
public parent:AbstractState;
constructor(mode:Modes.IMode, name:string, parent:AbstractState) {
super(mode);
constructor(modeId:string, name:string, parent:AbstractState) {
super(modeId);
this.name = name;
this.parent = parent;
}
@ -98,19 +98,7 @@ export class CSState extends AbstractState {
return false;
}
var otherCSState:CSState = <CSState>other;
return (other instanceof CSState) && (this.getMode() === otherCSState.getMode()) && (this.name === otherCSState.name) && ((this.parent === null && otherCSState.parent === null) || (this.parent !== null && this.parent.equals(otherCSState.parent)));
}
public tokenize(stream:Modes.IStream):Modes.ITokenizationResult {
stream.setTokenRules(separators, whitespace);
if (stream.skipWhitespace().length > 0) {
return { type: '' };
}
return this.stateTokenize(stream);
}
public stateTokenize(stream:Modes.IStream):Modes.ITokenizationResult {
throw new Error('To be implemented');
return (other instanceof CSState) && (this.getModeId() === otherCSState.getModeId()) && (this.name === otherCSState.name) && ((this.parent === null && otherCSState.parent === null) || (this.parent !== null && this.parent.equals(otherCSState.parent)));
}
}
@ -119,14 +107,14 @@ class CSString extends CSState {
private isAtBeginning:boolean;
private punctuation:string;
constructor(mode:Modes.IMode, parent:AbstractState, punctuation:string) {
super(mode, 'string', parent);
constructor(modeId:string, parent:AbstractState, punctuation:string) {
super(modeId, 'string', parent);
this.isAtBeginning = true;
this.punctuation = punctuation;
}
public makeClone():CSString {
return new CSString(this.getMode(), this.parent ? <AbstractState>this.parent.clone() : null, this.punctuation);
return new CSString(this.getModeId(), this.parent ? <AbstractState>this.parent.clone() : null, this.punctuation);
}
public equals(other:CSString):boolean {
@ -165,12 +153,12 @@ class CSString extends CSState {
class CSVerbatimString extends CSState {
constructor(mode:Modes.IMode, parent:AbstractState) {
super(mode, 'verbatimstring', parent);
constructor(modeId:string, parent:AbstractState) {
super(modeId, 'verbatimstring', parent);
}
public makeClone():CSVerbatimString {
return new CSVerbatimString(this.getMode(), this.parent ? <AbstractState>this.parent.clone() : null);
return new CSVerbatimString(this.getModeId(), this.parent ? <AbstractState>this.parent.clone() : null);
}
public tokenize(stream:Modes.IStream):Modes.ITokenizationResult {
@ -191,13 +179,13 @@ class CSVerbatimString extends CSState {
class CSNumber extends CSState {
private firstDigit:string;
constructor(mode:Modes.IMode, parent:AbstractState, firstDigit:string) {
super(mode, 'number', parent);
constructor(modeId:string, parent:AbstractState, firstDigit:string) {
super(modeId, 'number', parent);
this.firstDigit = firstDigit;
}
public makeClone():CSNumber {
return new CSNumber(this.getMode(), this.parent ? <AbstractState>this.parent.clone() : null, this.firstDigit);
return new CSNumber(this.getModeId(), this.parent ? <AbstractState>this.parent.clone() : null, this.firstDigit);
}
public tokenize(stream:Modes.IStream):Modes.ITokenizationResult {
@ -250,13 +238,13 @@ class CSNumber extends CSState {
export class CSComment extends CSState {
private commentChar:string;
constructor(mode:Modes.IMode, parent:AbstractState, commentChar:string) {
super(mode, 'comment', parent);
constructor(modeId:string, parent:AbstractState, commentChar:string) {
super(modeId, 'comment', parent);
this.commentChar = commentChar;
}
public makeClone():CSComment {
return new CSComment(this.getMode(), this.parent ? <AbstractState>this.parent.clone() : null, this.commentChar);
return new CSComment(this.getModeId(), this.parent ? <AbstractState>this.parent.clone() : null, this.commentChar);
}
public tokenize(stream:Modes.IStream):Modes.ITokenizationResult {
@ -280,14 +268,14 @@ export class CSStatement extends CSState implements VSXML.IVSXMLWrapperState {
private firstToken: boolean;
private firstTokenWasKeyword: boolean;
constructor(mode: Modes.IMode, parent: AbstractState, level: number, plevel: number, razorMode: boolean,
constructor(modeId:string, parent: AbstractState, level: number, plevel: number, razorMode: boolean,
expression: boolean, firstToken: boolean, firstTokenWasKeyword: boolean) {
super(mode, 'expression', parent);
super(modeId, 'expression', parent);
this.level = level;
this.plevel = plevel;
this.razorMode = razorMode;
this.expression = expression;
this.vsState = new VSXML.VSXMLExpression(mode, null);
this.vsState = new VSXML.VSXMLExpression(modeId, null);
this.firstToken = firstToken;
this.firstTokenWasKeyword = firstTokenWasKeyword;
}
@ -297,7 +285,7 @@ export class CSStatement extends CSState implements VSXML.IVSXMLWrapperState {
}
public makeClone():CSStatement {
var st = new CSStatement(this.getMode(), this.parent ? <AbstractState>this.parent.clone() : null, this.level,
var st = new CSStatement(this.getModeId(), this.parent ? <AbstractState>this.parent.clone() : null, this.level,
this.plevel, this.razorMode, this.expression, this.firstToken, this.firstTokenWasKeyword);
if (this.vsState !== null) {
st.setVSXMLState(<VSXML.VSXMLState>this.vsState.clone());
@ -312,11 +300,19 @@ export class CSStatement extends CSState implements VSXML.IVSXMLWrapperState {
(this.vsState !== null && this.vsState.equals((<CSStatement>other).vsState)));
}
public tokenize(stream:Modes.IStream):Modes.ITokenizationResult {
stream.setTokenRules(separators, whitespace);
if (stream.skipWhitespace().length > 0) {
return { type: '' };
}
return this.stateTokenize(stream);
}
public stateTokenize(stream:Modes.IStream):Modes.ITokenizationResult {
if (isDigit(stream.peek(), 10)) {
this.firstToken = false;
return { nextState: new CSNumber(this.getMode(), this, stream.next()) };
return { nextState: new CSNumber(this.getModeId(), this, stream.next()) };
}
var token = stream.nextToken();
@ -341,7 +337,7 @@ export class CSStatement extends CSState implements VSXML.IVSXMLWrapperState {
if (this.razorMode && token === '<' && acceptNestedModes) {
if (!stream.eos() && /[_:!\/\w]/.test(stream.peek())) {
return { nextState: new CSSimpleHTML(this.getMode(), this, htmlMode.States.Content) };
return { nextState: new CSSimpleHTML(this.getModeId(), this, htmlMode.States.Content) };
}
}
@ -367,7 +363,7 @@ export class CSStatement extends CSState implements VSXML.IVSXMLWrapperState {
if (stream.peekToken() !== '/') {
return {
type: 'comment.vs',
nextState: new VSXML.VSXMLEmbeddedState(this.getMode(), this.vsState, this)
nextState: new VSXML.VSXMLEmbeddedState(this.getModeId(), this.vsState, this)
};
}
}
@ -375,7 +371,7 @@ export class CSStatement extends CSState implements VSXML.IVSXMLWrapperState {
return { type: 'comment.cs' };
case '*':
stream.nextToken();
return { nextState: new CSComment(this.getMode(), this, '/') };
return { nextState: new CSComment(this.getModeId(), this, '/') };
}
}
return { type: 'punctuation.cs', nextState: nextStateAtEnd };
@ -385,10 +381,10 @@ export class CSStatement extends CSState implements VSXML.IVSXMLWrapperState {
switch(stream.peekToken()) {
case '"':
stream.nextToken();
return { nextState: new CSVerbatimString(this.getMode(), this) };
return { nextState: new CSVerbatimString(this.getModeId(), this) };
case '*':
stream.nextToken();
return { nextState: new CSComment(this.getMode(), this, '@') };
return { nextState: new CSComment(this.getModeId(), this, '@') };
}
}
}
@ -397,7 +393,7 @@ export class CSStatement extends CSState implements VSXML.IVSXMLWrapperState {
}
if (token === '"' || token === '\'') { // string or character
return { nextState: new CSString(this.getMode(), this, token) };
return { nextState: new CSString(this.getModeId(), this, token) };
}
if (brackets.stringIsBracket(token)) {
@ -465,13 +461,13 @@ export class CSStatement extends CSState implements VSXML.IVSXMLWrapperState {
class CSSimpleHTML extends CSState {
private state:htmlMode.States;
constructor(mode:Modes.IMode, parent:AbstractState, state:htmlMode.States) {
super(mode, 'number', parent);
constructor(modeId:string, parent:AbstractState, state:htmlMode.States) {
super(modeId, 'number', parent);
this.state = state;
}
public makeClone():CSSimpleHTML {
return new CSSimpleHTML(this.getMode(), this.parent ? <AbstractState>this.parent.clone() : null, this.state);
return new CSSimpleHTML(this.getModeId(), this.parent ? <AbstractState>this.parent.clone() : null, this.state);
}
private nextName(stream:Modes.IStream):string {

18
src/vs/languages/razor/common/razor.ts
View file

@ -13,22 +13,22 @@ import {RAZORWorker} from 'vs/languages/razor/common/razorWorker';
import {IInstantiationService} from 'vs/platform/instantiation/common/instantiation';
import {IModeService} from 'vs/editor/common/services/modeService';
import {LanguageConfigurationRegistry, LanguageConfiguration} from 'vs/editor/common/modes/languageConfigurationRegistry';
import {ILeavingNestedModeData} from 'vs/editor/common/modes/supports/tokenizationSupport';
import {wireCancellationToken} from 'vs/base/common/async';
import {ICompatWorkerService} from 'vs/editor/common/services/compatWorkerService';
import {IWorkspaceContextService} from 'vs/platform/workspace/common/workspace';
import {IConfigurationService} from 'vs/platform/configuration/common/configuration';
import {TokenizationSupport, ILeavingNestedModeData} from 'vs/editor/common/modes/supports/tokenizationSupport';
// for a brief description of the razor syntax see http://www.mikesdotnetting.com/Article/153/Inline-Razor-Syntax-Overview
class RAZORState extends htmlMode.State {
constructor(mode:modes.IMode, kind:htmlMode.States, lastTagName:string, lastAttributeName:string, embeddedContentType:string, attributeValueQuote:string, attributeValueLength:number) {
super(mode, kind, lastTagName, lastAttributeName, embeddedContentType, attributeValueQuote, attributeValueLength);
constructor(modeId:string, kind:htmlMode.States, lastTagName:string, lastAttributeName:string, embeddedContentType:string, attributeValueQuote:string, attributeValueLength:number) {
super(modeId, kind, lastTagName, lastAttributeName, embeddedContentType, attributeValueQuote, attributeValueLength);
}
public makeClone():RAZORState {
return new RAZORState(this.getMode(), this.kind, this.lastTagName, this.lastAttributeName, this.embeddedContentType, this.attributeValueQuote, this.attributeValueLength);
return new RAZORState(this.getModeId(), this.kind, this.lastTagName, this.lastAttributeName, this.embeddedContentType, this.attributeValueQuote, this.attributeValueLength);
}
public equals(other:modes.IState):boolean {
@ -45,10 +45,10 @@ class RAZORState extends htmlMode.State {
if (!stream.eos() && stream.peek() === '@') {
stream.next();
if (!stream.eos() && stream.peek() === '*') {
return { nextState: new csharpTokenization.CSComment(this.getMode(), this, '@') };
return { nextState: new csharpTokenization.CSComment(this.getModeId(), this, '@') };
}
if (stream.eos() || stream.peek() !== '@') {
return { type: razorTokenTypes.EMBED_CS, nextState: new csharpTokenization.CSStatement(this.getMode(), this, 0, 0, true, true, true, false) };
return { type: razorTokenTypes.EMBED_CS, nextState: new csharpTokenization.CSStatement(this.getModeId(), this, 0, 0, true, true, true, false) };
}
}
@ -132,6 +132,8 @@ export class RAZORMode extends htmlMode.HTMLMode<RAZORWorker> {
}, true);
LanguageConfigurationRegistry.register(this.getId(), RAZORMode.LANG_CONFIG);
modes.TokenizationRegistry.register(this.getId(), new TokenizationSupport(this._modeService, this.getId(), this, true));
}
protected _createModeWorkerManager(descriptor:modes.IModeDescriptor, instantiationService: IInstantiationService): ModeWorkerManager<RAZORWorker> {
@ -139,13 +141,13 @@ export class RAZORMode extends htmlMode.HTMLMode<RAZORWorker> {
}
public getInitialState(): modes.IState {
return new RAZORState(this, htmlMode.States.Content, '', '', '', '', 0);
return new RAZORState(this.getId(), htmlMode.States.Content, '', '', '', '', 0);
}
public getLeavingNestedModeData(line:string, state:modes.IState): ILeavingNestedModeData {
var leavingNestedModeData = super.getLeavingNestedModeData(line, state);
if (leavingNestedModeData) {
leavingNestedModeData.stateAfterNestedMode = new RAZORState(this, htmlMode.States.Content, '', '', '', '', 0);
leavingNestedModeData.stateAfterNestedMode = new RAZORState(this.getId(), htmlMode.States.Content, '', '', '', '', 0);
}
return leavingNestedModeData;
}

43
src/vs/languages/razor/common/vsxml.ts
View file

@ -12,7 +12,6 @@
'use strict';
import objects = require('vs/base/common/objects');
import errors = require('vs/base/common/errors');
import Modes = require('vs/editor/common/modes');
import {AbstractState} from 'vs/editor/common/modes/abstractState';
import vsxmlTokenTypes = require('vs/languages/razor/common/vsxmlTokenTypes');
@ -32,8 +31,8 @@ export class EmbeddedState extends AbstractState {
private state:Modes.IState;
private parentState:Modes.IState;
constructor(mode:Modes.IMode, state:Modes.IState, parentState:Modes.IState) {
super(mode);
constructor(modeId:string, state:Modes.IState, parentState:Modes.IState) {
super(modeId);
this.state = state;
this.parentState = parentState;
}
@ -43,7 +42,7 @@ export class EmbeddedState extends AbstractState {
}
public makeClone(): EmbeddedState {
return new EmbeddedState(this.getMode(), AbstractState.safeClone(this.state), AbstractState.safeClone(this.parentState));
return new EmbeddedState(this.getModeId(), AbstractState.safeClone(this.state), AbstractState.safeClone(this.parentState));
}
public equals(other:Modes.IState):boolean {
@ -77,8 +76,8 @@ export class EmbeddedState extends AbstractState {
export class VSXMLEmbeddedState extends EmbeddedState {
constructor(mode:Modes.IMode, state:Modes.IState, parentState:IVSXMLWrapperState) {
super(mode, state, parentState);
constructor(modeId:string, state:Modes.IState, parentState:IVSXMLWrapperState) {
super(modeId, state, parentState);
}
public equals(other:Modes.IState):boolean {
@ -103,14 +102,14 @@ export class VSXMLEmbeddedState extends EmbeddedState {
}
}
export class VSXMLState extends AbstractState {
export abstract class VSXMLState extends AbstractState {
public parent:Modes.IState;
public whitespaceTokenType:string;
private name:string;
constructor(mode:Modes.IMode, name:string, parent:Modes.IState, whitespaceTokenType:string='') {
super(mode);
constructor(modeId:string, name:string, parent:Modes.IState, whitespaceTokenType:string='') {
super(modeId);
this.name = name;
this.parent = parent;
this.whitespaceTokenType = whitespaceTokenType;
@ -136,19 +135,17 @@ export class VSXMLState extends AbstractState {
return this.stateTokenize(stream);
}
public stateTokenize(stream:Modes.IStream):Modes.ITokenizationResult {
throw errors.notImplemented();
}
public abstract stateTokenize(stream:Modes.IStream):Modes.ITokenizationResult;
}
export class VSXMLString extends VSXMLState {
constructor(mode:Modes.IMode, parent:Modes.IState) {
super(mode, 'string', parent, vsxmlTokenTypes.TOKEN_VALUE);
constructor(modeId:string, parent:Modes.IState) {
super(modeId, 'string', parent, vsxmlTokenTypes.TOKEN_VALUE);
}
public makeClone():VSXMLString {
return new VSXMLString(this.getMode(), this.parent ? this.parent.clone() : null);
return new VSXMLString(this.getModeId(), this.parent ? this.parent.clone() : null);
}
public equals(other:Modes.IState):boolean {
@ -173,12 +170,12 @@ export class VSXMLString extends VSXMLState {
export class VSXMLTag extends VSXMLState {
constructor(mode:Modes.IMode, parent:Modes.IState) {
super(mode, 'expression', parent, 'vs');
constructor(modeId:string, parent:Modes.IState) {
super(modeId, 'expression', parent, 'vs');
}
public makeClone():VSXMLTag {
return new VSXMLTag(this.getMode(), this.parent ? this.parent.clone() : null);
return new VSXMLTag(this.getModeId(), this.parent ? this.parent.clone() : null);
}
public equals(other:Modes.IState):boolean {
@ -196,7 +193,7 @@ export class VSXMLTag extends VSXMLState {
if (token === '>') {
return { type: 'punctuation.vs', nextState: this.parent };
} else if (token === '"') {
return { type: vsxmlTokenTypes.TOKEN_VALUE, nextState: new VSXMLString(this.getMode(), this) };
return { type: vsxmlTokenTypes.TOKEN_VALUE, nextState: new VSXMLString(this.getModeId(), this) };
} else if (isEntity(token)) {
tokenType = 'tag.vs';
} else if (isAttribute(token)) {
@ -210,12 +207,12 @@ export class VSXMLTag extends VSXMLState {
export class VSXMLExpression extends VSXMLState {
constructor(mode:Modes.IMode, parent:Modes.IState) {
super(mode, 'expression', parent, 'vs');
constructor(modeId:string, parent:Modes.IState) {
super(modeId, 'expression', parent, 'vs');
}
public makeClone():VSXMLExpression {
return new VSXMLExpression(this.getMode(), this.parent ? this.parent.clone() : null);
return new VSXMLExpression(this.getModeId(), this.parent ? this.parent.clone() : null);
}
public equals(other:Modes.IState):boolean {
@ -230,7 +227,7 @@ export class VSXMLExpression extends VSXMLState {
public stateTokenize(stream:Modes.IStream):Modes.ITokenizationResult {
var token = stream.nextToken();
if (token === '<') {
return { type: 'punctuation.vs', nextState: new VSXMLTag(this.getMode(), this) };
return { type: 'punctuation.vs', nextState: new VSXMLTag(this.getModeId(), this) };
}
return { type: this.whitespaceTokenType, nextState: this};
}

6
src/vs/languages/razor/test/common/razor.test.ts
View file

@ -15,7 +15,7 @@ suite('Syntax Highlighting - Razor', () => {
var tokenizationSupport: Modes.ITokenizationSupport;
(function() {
suiteSetup(function() {
let mode = new RAZORMode(
{ id: 'razor' },
null,
@ -25,8 +25,8 @@ suite('Syntax Highlighting - Razor', () => {
null
);
tokenizationSupport = mode.tokenizationSupport;
})();
tokenizationSupport = Modes.TokenizationRegistry.get(mode.getId());
});
test('', () => {
modesUtil.executeTests(tokenizationSupport,[

4
src/vs/monaco.d.ts vendored
View file

@ -2038,10 +2038,6 @@ declare module monaco.editor {
* Get the language associated with this model.
*/
getModeId(): string;
/**
* Set the current language mode associated with the model.
*/
setMode(newMode: languages.IMode | Promise<languages.IMode>): void;
/**
* Get the word under or besides `position`.
* @param position The position to look for a word.

34
src/vs/workbench/browser/parts/editor/editorStatus.ts
View file

@ -27,7 +27,7 @@ import {IConfigurationEditingService, ConfigurationTarget} from 'vs/workbench/se
import {IEditorAction, ICommonCodeEditor, IModelContentChangedEvent, IModelOptionsChangedEvent, IModelModeChangedEvent, ICursorPositionChangedEvent} from 'vs/editor/common/editorCommon';
import {ICodeEditor, IDiffEditor} from 'vs/editor/browser/editorBrowser';
import {TrimTrailingWhitespaceAction} from 'vs/editor/contrib/linesOperations/common/linesOperations';
import {EndOfLineSequence, ITokenizedModel, EditorType, ITextModel, IDiffEditorModel, IEditor} from 'vs/editor/common/editorCommon';
import {EndOfLineSequence, EditorType, IModel, IDiffEditorModel, IEditor} from 'vs/editor/common/editorCommon';
import {IndentUsingSpaces, IndentUsingTabs, DetectIndentation, IndentationToSpacesAction, IndentationToTabsAction} from 'vs/editor/contrib/indentation/common/indentation';
import {BaseTextEditor} from 'vs/workbench/browser/parts/editor/textEditor';
import {IEditor as IBaseEditor} from 'vs/platform/editor/common/editor';
@ -37,6 +37,7 @@ import {IWorkspaceConfigurationService} from 'vs/workbench/services/configuratio
import {IFilesConfiguration, SUPPORTED_ENCODINGS} from 'vs/platform/files/common/files';
import {IInstantiationService} from 'vs/platform/instantiation/common/instantiation';
import {IModeService} from 'vs/editor/common/services/modeService';
import {IModelService} from 'vs/editor/common/services/modelService';
import {StyleMutator} from 'vs/base/browser/styleMutator';
import {Selection} from 'vs/editor/common/core/selection';
import {IEditorGroupService} from 'vs/workbench/services/group/common/groupService';
@ -56,8 +57,8 @@ function getCodeEditor(editorWidget: IEditor): ICommonCodeEditor {
return null;
}
function getTextModel(editorWidget: IEditor): ITextModel {
let textModel: ITextModel;
function getTextModel(editorWidget: IEditor): IModel {
let textModel: IModel;
// Support for diff
const model = editorWidget.getModel();
@ -67,7 +68,7 @@ function getTextModel(editorWidget: IEditor): ITextModel {
// Normal editor
else {
textModel = <ITextModel>model;
textModel = <IModel>model;
}
return textModel;
@ -486,11 +487,9 @@ export class EditorStatus implements IStatusbarItem {
const textModel = getTextModel(editorWidget);
if (textModel) {
// Compute mode
if (!!(<ITokenizedModel>textModel).getMode) {
const mode = (<ITokenizedModel>textModel).getMode();
if (mode) {
info = { mode: this.modeService.getLanguageName(mode.getId()) };
}
const mode = textModel.getMode();
if (mode) {
info = { mode: this.modeService.getLanguageName(mode.getId()) };
}
}
}
@ -634,6 +633,7 @@ export class ChangeModeAction extends Action {
actionId: string,
actionLabel: string,
@IModeService private modeService: IModeService,
@IModelService private modelService: IModelService,
@IWorkbenchEditorService private editorService: IWorkbenchEditorService,
@IConfigurationEditingService private configurationEditingService: IConfigurationEditingService,
@IMessageService private messageService: IMessageService,
@ -656,11 +656,9 @@ export class ChangeModeAction extends Action {
// Compute mode
let currentModeId: string;
if (!!(<ITokenizedModel>textModel).getMode) {
const mode = (<ITokenizedModel>textModel).getMode();
if (mode) {
currentModeId = this.modeService.getLanguageName(mode.getId());
}
const mode = textModel.getMode();
if (mode) {
currentModeId = this.modeService.getLanguageName(mode.getId());
}
// All languages are valid picks
@ -707,7 +705,7 @@ export class ChangeModeAction extends Action {
activeEditor = this.editorService.getActiveEditor();
if (activeEditor instanceof BaseTextEditor) {
const editorWidget = activeEditor.getControl();
const models: ITextModel[] = [];
const models: IModel[] = [];
const textModel = getTextModel(editorWidget);
models.push(textModel);
@ -729,10 +727,8 @@ export class ChangeModeAction extends Action {
}
// Change mode
models.forEach(textModel => {
if (!!(<ITokenizedModel>textModel).getMode) {
(<ITokenizedModel>textModel).setMode(mode);
}
models.forEach((textModel) => {
this.modelService.setMode(textModel, mode);
});
}
}

2
src/vs/workbench/common/editor/textEditorModel.ts
View file

@ -92,7 +92,7 @@ export abstract class BaseTextEditorModel extends EditorModel implements ITextEd
model.setValueFromRawText(value);
}
model.setMode(mode);
this.modelService.setMode(model, mode);
}
this.textEditorModelHandle = model.uri;

17
src/vs/workbench/parts/emmet/test/common/editorAccessor.test.ts
View file

@ -7,7 +7,6 @@
import {EditorAccessor, IGrammarContributions} from 'vs/workbench/parts/emmet/node/editorAccessor';
import {withMockCodeEditor} from 'vs/editor/test/common/mocks/mockCodeEditor';
import {MockMode} from 'vs/editor/test/common/mocks/mockMode';
import assert = require('assert');
//
@ -50,7 +49,7 @@ suite('Emmet', () => {
withMockCodeEditor([], {}, (editor) => {
function testIsEnabled(mode: string, scopeName: string, isEnabled = true, profile = {}, excluded = []) {
editor.getModel().setMode(new MockMode(mode));
editor.getModel().setMode(mode);
let editorAccessor = new EditorAccessor(editor, profile, excluded, new MockGrammarContributions(scopeName));
assert.equal(editorAccessor.isEmmetEnabledMode(), isEnabled);
}
@ -84,6 +83,18 @@ suite('Emmet', () => {
testIsEnabled('java', 'source.java', true, {
'java': 'html'
});
});
withMockCodeEditor([
'<?'
], {}, (editor) => {
function testIsEnabled(mode: string, scopeName: string, isEnabled = true, profile = {}, excluded = []) {
editor.getModel().setMode(mode);
editor.setPosition({ lineNumber: 1, column: 3});
let editorAccessor = new EditorAccessor(editor, profile, excluded, new MockGrammarContributions(scopeName));
assert.equal(editorAccessor.isEmmetEnabledMode(), isEnabled);
}
// emmet enabled language that is disabled
testIsEnabled('php', 'text.html.php', false, {}, ['php']);
@ -94,7 +105,7 @@ suite('Emmet', () => {
withMockCodeEditor([], {}, (editor) => {
function testSyntax(mode: string, scopeName: string, expectedSyntax: string, profile = {}, excluded = []) {
editor.getModel().setMode(new MockMode(mode));
editor.getModel().setMode(mode);
let editorAccessor = new EditorAccessor(editor, profile, excluded, new MockGrammarContributions(scopeName));
assert.equal(editorAccessor.getSyntax(), expectedSyntax);
}

4
src/vs/workbench/parts/git/test/common/stageRanges.test.ts
View file

@ -8,7 +8,6 @@
import * as assert from 'assert';
import { getSelectedChanges, applyChangesToModel } from 'vs/workbench/parts/git/common/stageRanges';
import { Model } from 'vs/editor/common/model/model';
import { NullMode } from 'vs/editor/common/modes/nullMode';
import { IChange } from 'vs/editor/common/editorCommon';
import { Selection } from 'vs/editor/common/core/selection';
@ -34,7 +33,6 @@ function createChange(modifiedStart:number, modifiedEnd:number, originalStart:nu
}
suite('Git - Stage ranges', () => {
var mode = new NullMode();
test('Get selected changes test - no change selected (selection before changes)', () => {
var selections: Selection[] = [];
@ -142,7 +140,7 @@ suite('Git - Stage ranges', () => {
});
function createModel(text:string): Model {
return Model.createFromString(text, undefined, mode);
return Model.createFromString(text);
}
test('Apply changes to model - no changes', () => {

2
src/vs/workbench/parts/themes/test/electron-browser/themes.test.contribution.ts
View file

@ -117,7 +117,7 @@ class Snapper {
public captureSyntaxTokens(fileName: string, content: string) : TPromise<Data[]> {
return this.modeService.getOrCreateModeByFilenameOrFirstLine(fileName).then(mode => {
let result : Data[] = [];
let model = new TextModelWithTokens([], TextModel.toRawText(content, TextModel.DEFAULT_CREATION_OPTIONS), mode);
let model = new TextModelWithTokens([], TextModel.toRawText(content, TextModel.DEFAULT_CREATION_OPTIONS), mode.getId());
model.tokenIterator({lineNumber: 1, column: 1}, iterator => {
while (iterator.hasNext()) {
let tokenInfo = iterator.next();

1
src/vs/workbench/test/node/api/extHostApiCommands.test.ts
View file

@ -71,6 +71,7 @@ suite('ExtHostLanguageFeatureCommands', function() {
_serviceBrand: IModelService,
getModel(): any { return model; },
createModel(): any { throw new Error(); },
setMode(): any { throw new Error(); },
destroyModel(): any { throw new Error(); },
getModels(): any { throw new Error(); },
onModelAdded: undefined,