workbench theme service: provide tokenColorMap

src/vs/editor/standalone/browser/standaloneThemeServiceImpl.ts

@@ -14,7 +14,6 @@ import { IEnvironmentService } from 'vs/platform/environment/common/environment'
 import { Registry } from 'vs/platform/registry/common/platform';
 import { ColorIdentifier, Extensions, IColorRegistry } from 'vs/platform/theme/common/colorRegistry';
 import { Extensions as ThemingExtensions, ICssStyleCollector, IIconTheme, IThemingRegistry } from 'vs/platform/theme/common/themeService';
-import { TokenStyle, TokenClassification, ProbeScope } from 'vs/platform/theme/common/tokenClassificationRegistry';
 
 const VS_THEME_NAME = 'vs';
 const VS_DARK_THEME_NAME = 'vs-dark';
@@ -130,14 +129,6 @@ class StandaloneTheme implements IStandaloneTheme {
 		}
 		return this._tokenTheme;
 	}
-
-	getTokenStyle(classification: TokenClassification, useDefault?: boolean | undefined): TokenStyle | undefined {
-		return undefined;
-	}
-
-	resolveScopes(scopes: ProbeScope[]): TokenStyle | undefined {
-		return undefined;
-	}
 }
 
 function isBuiltinTheme(themeName: string): themeName is BuiltinTheme {

src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts

@@ -54,13 +54,10 @@ suite('TokenizationSupport2Adapter', () => {
 
 				defines: (color: ColorIdentifier): boolean => {
 					throw new Error('Not implemented');
-				},
-
-				getTokenStyle: () => undefined,
-				resolveScopes: () => undefined
-
+				}
 			};
 		}
+
 		public getIconTheme(): IIconTheme {
 			return {
 				hasFileIcons: false,

src/vs/platform/theme/common/themeService.ts

@@ -10,7 +10,6 @@ import * as platform from 'vs/platform/registry/common/platform';
 import { ColorIdentifier } from 'vs/platform/theme/common/colorRegistry';
 import { Event, Emitter } from 'vs/base/common/event';
 import { IEnvironmentService } from 'vs/platform/environment/common/environment';
-import { TokenStyle, TokenClassification, ProbeScope } from 'vs/platform/theme/common/tokenClassificationRegistry';
 
 export const IThemeService = createDecorator<IThemeService>('themeService');
 
@@ -60,10 +59,6 @@ export interface ITheme {
 	 * default color will be used.
 	 */
 	defines(color: ColorIdentifier): boolean;
-
-	getTokenStyle(classification: TokenClassification, useDefault?: boolean): TokenStyle | undefined;
-
-	resolveScopes(scopes: ProbeScope[]): TokenStyle | undefined;
 }
 
 export interface IIconTheme {

src/vs/platform/theme/common/tokenClassificationRegistry.ts

@@ -142,9 +142,9 @@ export interface ITokenClassificationRegistry {
 	getTokenModifiers(): TokenTypeOrModifierContribution[];
 
 	/**
-	 * Resolves a token classification against the given rules and default rules from the registry.
+	 * The styling rules to used when a schema does not define any styling rules.
 	 */
-	resolveTokenStyle(classification: TokenClassification, themingRules: TokenStylingRule[] | undefined, customThemingRules: TokenStylingRule[], theme: ITheme): TokenStyle | undefined;
+	getTokenStylingDefaultRules(): TokenStylingDefaultRule[];
 
 	/**
 	 * JSON schema for an object to assign styling to token classifications
@@ -271,87 +271,13 @@ class TokenClassificationRegistry implements ITokenClassificationRegistry {
 		return Object.keys(this.tokenModifierById).map(id => this.tokenModifierById[id]);
 	}
 
-	public resolveTokenStyle(classification: TokenClassification, themingRules: TokenStylingRule[] | undefined, customThemingRules: TokenStylingRule[], theme: ITheme): TokenStyle | undefined {
-		let result: any = {
-			foreground: undefined,
-			bold: undefined,
-			underline: undefined,
-			italic: undefined
-		};
-		let score = {
-			foreground: -1,
-			bold: -1,
-			underline: -1,
-			italic: -1
-		};
-
-		function _processStyle(matchScore: number, style: TokenStyle) {
-			if (style.foreground && score.foreground <= matchScore) {
-				score.foreground = matchScore;
-				result.foreground = style.foreground;
-			}
-			for (let p of ['bold', 'underline', 'italic']) {
-				const property = p as keyof TokenStyle;
-				const info = style[property];
-				if (info !== undefined) {
-					if (score[property] <= matchScore) {
-						score[property] = matchScore;
-						result[property] = info;
-					}
-				}
-			}
-		}
-		if (themingRules === undefined) {
-			for (const rule of this.tokenStylingDefaultRules) {
-				const matchScore = match(rule, classification);
-				if (matchScore >= 0) {
-					let style = theme.resolveScopes(rule.defaults.scopesToProbe);
-					if (!style) {
-						style = this.resolveTokenStyleValue(rule.defaults[theme.type], theme);
-					}
-					if (style) {
-						_processStyle(matchScore, style);
-					}
-				}
-			}
-		} else {
-			for (const rule of themingRules) {
-				const matchScore = match(rule, classification);
-				if (matchScore >= 0) {
-					_processStyle(matchScore, rule.value);
-				}
-			}
-		}
-		for (const rule of customThemingRules) {
-			const matchScore = match(rule, classification);
-			if (matchScore >= 0) {
-				_processStyle(matchScore, rule.value);
-			}
-		}
-		return TokenStyle.fromData(result);
-	}
-
-	/**
-	 * @param tokenStyleValue Resolve a tokenStyleValue in the context of a theme
-	 */
-	private resolveTokenStyleValue(tokenStyleValue: TokenStyleValue | null, theme: ITheme): TokenStyle | undefined {
-		if (tokenStyleValue === null) {
-			return undefined;
-		} else if (typeof tokenStyleValue === 'string') {
-			const classification = this.getTokenClassificationFromString(tokenStyleValue);
-			if (classification) {
-				return theme.getTokenStyle(classification);
-			}
-		} else if (typeof tokenStyleValue === 'object') {
-			return tokenStyleValue;
-		}
-		return undefined;
-	}
-
 	public getTokenStylingSchema(): IJSONSchema {
 		return this.tokenStylingSchema;
 	}
 
+	public getTokenStylingDefaultRules(): TokenStylingDefaultRule[] {
+		return this.tokenStylingDefaultRules;
+	}
 
 	public toString() {
 		let sorter = (a: string, b: string) => {
@@ -368,7 +294,7 @@ class TokenClassificationRegistry implements ITokenClassificationRegistry {
 
 }
 
-function match(themeSelector: TokenStylingRule | TokenStylingDefaultRule, classification: TokenClassification): number {
+export function matchTokenStylingRule(themeSelector: TokenStylingRule | TokenStylingDefaultRule, classification: TokenClassification): number {
 	const selectorType = themeSelector.classification.type;
 	if (selectorType !== TOKEN_TYPE_WILDCARD_NUM && selectorType !== classification.type) {
 		return -1;

src/vs/platform/theme/test/common/testThemeService.ts

@@ -6,7 +6,6 @@
 import { Event, Emitter } from 'vs/base/common/event';
 import { IThemeService, ITheme, DARK, IIconTheme } from 'vs/platform/theme/common/themeService';
 import { Color } from 'vs/base/common/color';
-import { TokenStyle, TokenClassification, ProbeScope } from 'vs/platform/theme/common/tokenClassificationRegistry';
 
 export class TestTheme implements ITheme {
 
@@ -24,14 +23,6 @@ export class TestTheme implements ITheme {
 	defines(color: string): boolean {
 		throw new Error('Method not implemented.');
 	}
-
-	getTokenStyle(classification: TokenClassification, useDefault?: boolean | undefined): TokenStyle | undefined {
-		throw new Error('Method not implemented.');
-	}
-
-	resolveScopes(scopes: ProbeScope[]): TokenStyle | undefined {
-		throw new Error('Method not implemented.');
-	}
 }
 
 export class TestIconTheme implements IIconTheme {

src/vs/workbench/services/themes/common/colorThemeData.ts

@@ -19,9 +19,10 @@ import { getParseErrorMessage } from 'vs/base/common/jsonErrorMessages';
 import { URI } from 'vs/base/common/uri';
 import { parse as parsePList } from 'vs/workbench/services/themes/common/plistParser';
 import { startsWith } from 'vs/base/common/strings';
-import { TokenStyle, TokenClassification, ProbeScope, TokenStylingRule, getTokenClassificationRegistry } from 'vs/platform/theme/common/tokenClassificationRegistry';
+import { TokenStyle, TokenClassification, ProbeScope, TokenStylingRule, getTokenClassificationRegistry, TokenStyleValue, matchTokenStylingRule } from 'vs/platform/theme/common/tokenClassificationRegistry';
 import { MatcherWithPriority, Matcher, createMatchers } from 'vs/workbench/services/themes/common/textMateScopeMatcher';
 import { IExtensionResourceLoaderService } from 'vs/workbench/services/extensionResourceLoader/common/extensionResourceLoader';
+import { FontStyle, ColorId, MetadataConsts } from 'vs/editor/common/modes';
 
 let colorRegistry = Registry.as<IColorRegistry>(ColorRegistryExtensions.ColorContribution);
 
@@ -60,6 +61,8 @@ export class ColorThemeData implements IColorTheme {
 	private themeTokenScopeMatchers: Matcher<ProbeScope>[] | undefined;
 	private customTokenScopeMatchers: Matcher<ProbeScope>[] | undefined;
 
+	private tokenColorIndex: TokenColorIndex | undefined = undefined; // created on demand
+
 	private constructor(id: string, label: string, settingsId: string) {
 		this.id = id;
 		this.label = label;
@@ -114,19 +117,141 @@ export class ColorThemeData implements IColorTheme {
 		return color;
 	}
 
-	public getTokenStyle(tokenClassification: TokenClassification, useDefault?: boolean): TokenStyle | undefined {
-		// todo: cache results
-		return tokenClassificationRegistry.resolveTokenStyle(tokenClassification, this.tokenStylingRules, this.customTokenStylingRules, this);
+	public getTokenStyle(classification: TokenClassification, useDefault?: boolean): TokenStyle | undefined {
+		let result: any = {
+			foreground: undefined,
+			bold: undefined,
+			underline: undefined,
+			italic: undefined
+		};
+		let score = {
+			foreground: -1,
+			bold: -1,
+			underline: -1,
+			italic: -1
+		};
+
+		function _processStyle(matchScore: number, style: TokenStyle) {
+			if (style.foreground && score.foreground <= matchScore) {
+				score.foreground = matchScore;
+				result.foreground = style.foreground;
+			}
+			for (let p of ['bold', 'underline', 'italic']) {
+				const property = p as keyof TokenStyle;
+				const info = style[property];
+				if (info !== undefined) {
+					if (score[property] <= matchScore) {
+						score[property] = matchScore;
+						result[property] = info;
+					}
+				}
+			}
+		}
+		if (this.tokenStylingRules === undefined) {
+			for (const rule of tokenClassificationRegistry.getTokenStylingDefaultRules()) {
+				const matchScore = matchTokenStylingRule(rule, classification);
+				if (matchScore >= 0) {
+					let style = this.resolveScopes(rule.defaults.scopesToProbe);
+					if (!style && useDefault !== false) {
+						style = this.resolveTokenStyleValue(rule.defaults[this.type]);
+					}
+					if (style) {
+						_processStyle(matchScore, style);
+					}
+				}
+			}
+		} else {
+			for (const rule of this.tokenStylingRules) {
+				const matchScore = matchTokenStylingRule(rule, classification);
+				if (matchScore >= 0) {
+					_processStyle(matchScore, rule.value);
+				}
+			}
+		}
+		for (const rule of this.customTokenStylingRules) {
+			const matchScore = matchTokenStylingRule(rule, classification);
+			if (matchScore >= 0) {
+				_processStyle(matchScore, rule.value);
+			}
+		}
+		return TokenStyle.fromData(result);
+
+	}
+
+	/**
+	 * @param tokenStyleValue Resolve a tokenStyleValue in the context of a theme
+	 */
+	private resolveTokenStyleValue(tokenStyleValue: TokenStyleValue | null): TokenStyle | undefined {
+		if (tokenStyleValue === null) {
+			return undefined;
+		} else if (typeof tokenStyleValue === 'string') {
+			const classification = tokenClassificationRegistry.getTokenClassificationFromString(tokenStyleValue);
+			if (classification) {
+				return this.getTokenStyle(classification);
+			}
+		} else if (typeof tokenStyleValue === 'object') {
+			return tokenStyleValue;
+		}
+		return undefined;
+	}
+
+	private getTokenColorIndex(): TokenColorIndex {
+		// collect all colors that tokens can have
+		if (!this.tokenColorIndex) {
+			const index = new TokenColorIndex();
+			this.tokenColors.forEach(rule => {
+				index.add(rule.settings.foreground);
+				index.add(rule.settings.background);
+			});
+
+			if (this.tokenStylingRules) {
+				this.tokenStylingRules.forEach(r => index.add(r.value.foreground));
+			} else {
+				tokenClassificationRegistry.getTokenStylingDefaultRules().forEach(r => {
+					const defaultColor = r.defaults[this.type];
+					if (defaultColor && typeof defaultColor === 'object') {
+						index.add(defaultColor.foreground);
+					}
+				});
+			}
+			this.customTokenStylingRules.forEach(r => index.add(r.value.foreground));
+
+			this.tokenColorIndex = index;
+		}
+		return this.tokenColorIndex;
+	}
+
+	public get tokenColorMap(): string[] {
+		return this.getTokenColorIndex().asArray();
+	}
+
+	public getTokenStyleMetadata(classification: TokenClassification, useDefault?: boolean): number {
+		const style = this.getTokenStyle(classification, useDefault);
+		let fontStyle = FontStyle.NotSet;
+		let foreground = 0;
+		if (style) {
+			if (style.bold === false && style.underline === false && style.italic === false) {
+				fontStyle = FontStyle.None;
+			} else {
+				if (style.bold) {
+					fontStyle |= FontStyle.Bold;
+				}
+				if (style.underline) {
+					fontStyle |= FontStyle.Underline;
+				}
+				if (style.italic) {
+					fontStyle |= FontStyle.Italic;
+				}
+			}
+			foreground = this.getTokenColorIndex().get(style.foreground);
+		}
+		return toMetadata(fontStyle, foreground, 0);
 	}
 
 	public getDefault(colorId: ColorIdentifier): Color | undefined {
 		return colorRegistry.resolveDefaultColor(colorId, this);
 	}
 
-	public getDefaultTokenStyle(tokenClassification: TokenClassification): TokenStyle | undefined {
-		return tokenClassificationRegistry.resolveTokenStyle(tokenClassification, undefined, [], this);
-	}
-
 	public resolveScopes(scopes: ProbeScope[]): TokenStyle | undefined {
 
 		if (!this.themeTokenScopeMatchers) {
@@ -177,6 +302,8 @@ export class ColorThemeData implements IColorTheme {
 		if (types.isObject(themeSpecificColors)) {
 			this.overwriteCustomColors(themeSpecificColors);
 		}
+
+		this.tokenColorIndex = undefined;
 	}
 
 	private overwriteCustomColors(colors: IColorCustomizations) {
@@ -209,6 +336,8 @@ export class ColorThemeData implements IColorTheme {
 		if (types.isObject(themeSpecificColors)) {
 			readCustomTokenStyleRules(themeSpecificColors, this.tokenStylingRules);
 		}
+
+		this.tokenColorIndex = undefined;
 	}
 
 	private addCustomTokenColors(customTokenColors: ITokenColorCustomizations) {
@@ -250,6 +379,7 @@ export class ColorThemeData implements IColorTheme {
 		}
 		this.themeTokenColors = [];
 		this.themeTokenScopeMatchers = undefined;
+		this.tokenColorIndex = undefined;
 
 		const result = {
 			colors: {},
@@ -587,3 +717,66 @@ function readCustomTokenStyleRules(tokenStylingRuleSection: IExperimentalTokenSt
 function isTokenColorizationSetting(style: any): style is ITokenColorizationSetting {
 	return style && (style.foreground || style.fontStyle);
 }
+
+
+class TokenColorIndex {
+
+	private _lastColorId: number;
+	private _id2color: string[];
+	private _color2id: { [color: string]: number; };
+
+	constructor() {
+		this._lastColorId = 0;
+		this._id2color = [];
+		this._color2id = Object.create(null);
+	}
+
+	public add(color: string | Color | undefined | null): number {
+		if (color === null || color === undefined) {
+			return 0;
+		}
+		color = normalizeColorForIndex(color);
+
+		let value = this._color2id[color];
+		if (value) {
+			return value;
+		}
+		value = ++this._lastColorId;
+		this._color2id[color] = value;
+		this._id2color[value] = color;
+		return value;
+	}
+
+	public get(color: string | Color | undefined): number {
+		if (color === undefined) {
+			return 0;
+		}
+		color = normalizeColorForIndex(color);
+		let value = this._color2id[color];
+		if (value) {
+			return value;
+		}
+		console.log(`Color ${color} not in index.`);
+		return 0;
+	}
+
+	public asArray(): string[] {
+		return this._id2color.slice(0);
+	}
+
+}
+
+function normalizeColorForIndex(color: string | Color): string {
+	if (typeof color !== 'string') {
+		color = Color.Format.CSS.formatHexA(color, true);
+	}
+	return color.toUpperCase();
+}
+
+function toMetadata(fontStyle: FontStyle, foreground: ColorId | number, background: ColorId | number) {
+	return (
+		(fontStyle << MetadataConsts.FONT_STYLE_OFFSET)
+		| (foreground << MetadataConsts.FOREGROUND_OFFSET)
+		| (background << MetadataConsts.BACKGROUND_OFFSET)
+	) >>> 0;
+}

src/vs/workbench/services/themes/common/workbenchThemeService.ts

@@ -8,6 +8,7 @@ import { Event } from 'vs/base/common/event';
 import { Color } from 'vs/base/common/color';
 import { ITheme, IThemeService, IIconTheme } from 'vs/platform/theme/common/themeService';
 import { ConfigurationTarget } from 'vs/platform/configuration/common/configuration';
+import { TokenClassification } from 'vs/platform/theme/common/tokenClassificationRegistry';
 
 export const IWorkbenchThemeService = createDecorator<IWorkbenchThemeService>('themeService');
 
@@ -32,6 +33,16 @@ export interface IColorTheme extends ITheme {
 	readonly description?: string;
 	readonly isLoaded: boolean;
 	readonly tokenColors: ITextMateThemingRule[];
+
+	/**
+	 * Returns the token style for a given classification. The result uses the <code>MetadataConsts</code> format
+	 */
+	getTokenStyleMetadata(classification: TokenClassification): number;
+
+	/**
+	 * List of all colors used with tokens. <code>getTokenStyleMetadata</code> references the colors by index into this list.
+	 */
+	readonly tokenColorMap: string[];
 }
 
 export interface IColorMap {