Move encoding to common for #79275 (#100539)

* move encoding.ts to common for #79275

* load iconv-lite-umd and jschardet in web version for #79275

* move EncodingOracle to the AbstractTextFileService for #79275

* review

* update to new iconv-lite-umd

* add workaround for jschardet types

* fix indentation

* add iconv-lite-umd and jschardet to workbench.html

* fix paths for modules

Co-authored-by: Benjamin Pasero <benjpas@microsoft.com>

.eslintrc.json

@@ -519,7 +519,8 @@
                     "**/vs/workbench/services/**/common/**",
                     "**/vs/workbench/api/**/common/**",
                     "vscode-textmate",
-                    "vscode-oniguruma"
+                    "vscode-oniguruma",
+                    "iconv-lite-umd"
                 ]
             },
             {

build/lib/i18n.js

@@ -1175,9 +1175,10 @@ function createIslFile(originalFilePath, messages, language, innoSetup) {
     });
     const basename = path.basename(originalFilePath);
     const filePath = `${basename}.${language.id}.isl`;
+    const encoded = iconv.encode(Buffer.from(content.join('\r\n'), 'utf8').toString(), innoSetup.codePage);
     return new File({
         path: filePath,
-        contents: iconv.encode(Buffer.from(content.join('\r\n'), 'utf8').toString(), innoSetup.codePage)
+        contents: Buffer.from(encoded),
     });
 }
 function encodeEntities(value) {

build/lib/i18n.ts

@@ -1339,10 +1339,11 @@ function createIslFile(originalFilePath: string, messages: Map<string>, language
 
 	const basename = path.basename(originalFilePath);
 	const filePath = `${basename}.${language.id}.isl`;
+	const encoded = iconv.encode(Buffer.from(content.join('\r\n'), 'utf8').toString(), innoSetup.codePage);
 
 	return new File({
 		path: filePath,
-		contents: iconv.encode(Buffer.from(content.join('\r\n'), 'utf8').toString(), innoSetup.codePage)
+		contents: Buffer.from(encoded),
 	});
 }
 

build/package.json

@@ -38,7 +38,7 @@
     "gulp-bom": "^1.0.0",
     "gulp-sourcemaps": "^1.11.0",
     "gulp-uglify": "^3.0.0",
-    "iconv-lite-umd": "0.6.2",
+    "iconv-lite-umd": "0.6.3",
     "mime": "^1.3.4",
     "minimatch": "3.0.4",
     "minimist": "^1.2.3",

build/yarn.lock

@@ -1415,10 +1415,10 @@ http-signature@~1.2.0:
     jsprim "^1.2.2"
     sshpk "^1.7.0"
 
-iconv-lite-umd@0.6.2:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.2.tgz#6410d3dc1bf5b0e0863f833d67e8168fcd52d47c"
-  integrity sha512-KOOIU5p4j/NOXybhgOF7ZMRMQ7+iOWwnr1+DSQaPCzCRfR1+vMzvEmjmrmUZ59kHkhcqZW7eABTa/axpc3i81w==
+iconv-lite-umd@0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.3.tgz#61307cab8ac29939992d0724d3ab8799467f0e97"
+  integrity sha512-fQ/8XE8reiCZ6t+SX4tX6/tQdV4tThJZv5qtMe5Sk+IWsExz0S2Zd+GiBS5IEPgDxnsmiJSpH67+qzN3FT4lKw==
 
 ignore@^5.1.1:
   version "5.1.2"

extensions/git/package.json

@@ -1878,7 +1878,7 @@
   "dependencies": {
     "byline": "^5.0.0",
     "file-type": "^7.2.0",
-    "iconv-lite-umd": "0.6.2",
+    "iconv-lite-umd": "0.6.3",
     "jschardet": "2.1.1",
     "vscode-extension-telemetry": "0.1.1",
     "vscode-nls": "^4.0.0",

extensions/git/yarn.lock

@@ -425,10 +425,10 @@ https-proxy-agent@^2.2.1:
     agent-base "^4.3.0"
     debug "^3.1.0"
 
-iconv-lite-umd@0.6.2:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.2.tgz#6410d3dc1bf5b0e0863f833d67e8168fcd52d47c"
-  integrity sha512-KOOIU5p4j/NOXybhgOF7ZMRMQ7+iOWwnr1+DSQaPCzCRfR1+vMzvEmjmrmUZ59kHkhcqZW7eABTa/axpc3i81w==
+iconv-lite-umd@0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.3.tgz#61307cab8ac29939992d0724d3ab8799467f0e97"
+  integrity sha512-fQ/8XE8reiCZ6t+SX4tX6/tQdV4tThJZv5qtMe5Sk+IWsExz0S2Zd+GiBS5IEPgDxnsmiJSpH67+qzN3FT4lKw==
 
 inflight@^1.0.4:
   version "1.0.6"

package.json

@@ -41,7 +41,7 @@
     "graceful-fs": "4.2.3",
     "http-proxy-agent": "^2.1.0",
     "https-proxy-agent": "^2.2.3",
-    "iconv-lite-umd": "0.6.2",
+    "iconv-lite-umd": "0.6.3",
     "jschardet": "2.1.1",
     "keytar": "^5.5.0",
     "minimist": "^1.2.5",

remote/package.json

@@ -8,7 +8,7 @@
     "graceful-fs": "4.2.3",
     "http-proxy-agent": "^2.1.0",
     "https-proxy-agent": "^2.2.3",
-    "iconv-lite-umd": "0.6.2",
+    "iconv-lite-umd": "0.6.3",
     "jschardet": "2.1.1",
     "minimist": "^1.2.5",
     "native-watchdog": "1.3.0",

remote/web/package.json

@@ -3,6 +3,8 @@
 	"version": "0.0.0",
 	"dependencies": {
 		"semver-umd": "^5.5.7",
+		"iconv-lite-umd": "0.6.3",
+		"jschardet": "2.1.1",
 		"vscode-oniguruma": "1.3.1",
 		"vscode-textmate": "5.1.1",
 		"xterm": "4.7.0-beta.3",

remote/web/yarn.lock

@@ -2,6 +2,16 @@
 # yarn lockfile v1
 
 
+iconv-lite-umd@0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.3.tgz#61307cab8ac29939992d0724d3ab8799467f0e97"
+  integrity sha512-fQ/8XE8reiCZ6t+SX4tX6/tQdV4tThJZv5qtMe5Sk+IWsExz0S2Zd+GiBS5IEPgDxnsmiJSpH67+qzN3FT4lKw==
+
+jschardet@2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/jschardet/-/jschardet-2.1.1.tgz#af6f8fd0b3b0f5d46a8fd9614a4fce490575c184"
+  integrity sha512-pA5qG9Zwm8CBpGlK/lo2GE9jPxwqRgMV7Lzc/1iaPccw6v4Rhj8Zg2BTyrdmHmxlJojnbLupLeRnaPLsq03x6Q==
+
 semver-umd@^5.5.7:
   version "5.5.7"
   resolved "https://registry.yarnpkg.com/semver-umd/-/semver-umd-5.5.7.tgz#966beb5e96c7da6fbf09c3da14c2872d6836c528"

remote/yarn.lock

@@ -176,10 +176,10 @@ https-proxy-agent@^2.2.3:
     agent-base "^4.3.0"
     debug "^3.1.0"
 
-iconv-lite-umd@0.6.2:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.2.tgz#6410d3dc1bf5b0e0863f833d67e8168fcd52d47c"
-  integrity sha512-KOOIU5p4j/NOXybhgOF7ZMRMQ7+iOWwnr1+DSQaPCzCRfR1+vMzvEmjmrmUZ59kHkhcqZW7eABTa/axpc3i81w==
+iconv-lite-umd@0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.3.tgz#61307cab8ac29939992d0724d3ab8799467f0e97"
+  integrity sha512-fQ/8XE8reiCZ6t+SX4tX6/tQdV4tThJZv5qtMe5Sk+IWsExz0S2Zd+GiBS5IEPgDxnsmiJSpH67+qzN3FT4lKw==
 
 ip@^1.1.5:
   version "1.1.5"

src/vs/code/browser/workbench/workbench-dev.html

@@ -37,6 +37,8 @@
 				'xterm-addon-unicode11': `${window.location.origin}/static/remote/web/node_modules/xterm-addon-unicode11/lib/xterm-addon-unicode11.js`,
 				'xterm-addon-webgl': `${window.location.origin}/static/remote/web/node_modules/xterm-addon-webgl/lib/xterm-addon-webgl.js`,
 				'semver-umd': `${window.location.origin}/static/remote/web/node_modules/semver-umd/lib/semver-umd.js`,
+				'iconv-lite-umd': `${window.location.origin}/static/remote/web/node_modules/iconv-lite-umd/lib/iconv-lite-umd.js`,
+				'jschardet': `${window.location.origin}/static/remote/web/node_modules/jschardet/dist/jschardet.min.js`,
 			}
 		};
 	</script>

src/vs/code/browser/workbench/workbench.html

@@ -23,6 +23,8 @@
 
 		<!-- Prefetch to avoid waterfall -->
 		<link rel="prefetch" href="./static/node_modules/semver-umd/lib/semver-umd.js">
+		<link rel="prefetch" href="./static/node_modules/iconv-lite-umd/lib/iconv-lite-umd.js">
+		<link rel="prefetch" href="./static/node_modules/jschardet/dist/jschardet.min.js">
 	</head>
 
 	<body aria-label="">
@@ -41,6 +43,8 @@
 				'xterm-addon-unicode11': `${window.location.origin}/static/node_modules/xterm-addon-unicode11/lib/xterm-addon-unicode11.js`,
 				'xterm-addon-webgl': `${window.location.origin}/static/node_modules/xterm-addon-webgl/lib/xterm-addon-webgl.js`,
 				'semver-umd': `${window.location.origin}/static/node_modules/semver-umd/lib/semver-umd.js`,
+				'iconv-lite-umd': `${window.location.origin}/static/node_modules/iconv-lite-umd/lib/iconv-lite-umd.js`,
+				'jschardet': `${window.location.origin}/static/node_modules/jschardet/dist/jschardet.min.js`,
 			}
 		};
 	</script>

src/vs/workbench/services/search/node/textSearchManager.ts

@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { toCanonicalName } from 'vs/base/node/encoding';
+import { toCanonicalName } from 'vs/workbench/services/textfile/common/encoding';
 import * as pfs from 'vs/base/node/pfs';
 import { ITextQuery } from 'vs/workbench/services/search/common/search';
 import { TextSearchProvider } from 'vs/workbench/services/search/common/searchExtTypes';

src/vs/workbench/services/textfile/browser/browserTextFileService.ts

@@ -3,18 +3,22 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { AbstractTextFileService } from 'vs/workbench/services/textfile/browser/textFileService';
-import { ITextFileService, IResourceEncodings, IResourceEncoding, TextFileEditorModelState } from 'vs/workbench/services/textfile/common/textfiles';
+import { AbstractTextFileService, EncodingOracle } from 'vs/workbench/services/textfile/browser/textFileService';
+import { ITextFileService, IResourceEncoding, TextFileEditorModelState } from 'vs/workbench/services/textfile/common/textfiles';
 import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
 import { ShutdownReason } from 'vs/platform/lifecycle/common/lifecycle';
 
 export class BrowserTextFileService extends AbstractTextFileService {
 
-	readonly encoding: IResourceEncodings = {
-		async getPreferredWriteEncoding(): Promise<IResourceEncoding> {
-			return { encoding: 'utf8', hasBOM: false };
+	private _browserEncoding: EncodingOracle | undefined;
+
+	get encoding(): EncodingOracle {
+		if (!this._browserEncoding) {
+			this._browserEncoding = this._register(this.instantiationService.createInstance(BrowserEncodingOracle));
 		}
-	};
+
+		return this._browserEncoding;
+	}
 
 	protected registerListeners(): void {
 		super.registerListeners();
@@ -34,4 +38,18 @@ export class BrowserTextFileService extends AbstractTextFileService {
 	}
 }
 
+class BrowserEncodingOracle extends EncodingOracle {
+	async getPreferredWriteEncoding(): Promise<IResourceEncoding> {
+		return { encoding: 'utf8', hasBOM: false };
+	}
+
+	async getWriteEncoding(): Promise<{ encoding: string, addBOM: boolean }> {
+		return { encoding: 'utf8', addBOM: false };
+	}
+
+	async getReadEncoding(): Promise<string> {
+		return 'utf8';
+	}
+}
+
 registerSingleton(ITextFileService, BrowserTextFileService);

src/vs/workbench/services/textfile/browser/textFileService.ts

@@ -6,7 +6,7 @@
 import * as nls from 'vs/nls';
 import { URI } from 'vs/base/common/uri';
 import { AsyncEmitter } from 'vs/base/common/event';
-import { ITextFileService, ITextFileStreamContent, ITextFileContent, IResourceEncodings, IReadTextFileOptions, IWriteTextFileOptions, toBufferOrReadable, TextFileOperationError, TextFileOperationResult, ITextFileSaveOptions, ITextFileEditorModelManager, TextFileCreateEvent } from 'vs/workbench/services/textfile/common/textfiles';
+import { ITextFileService, ITextFileStreamContent, ITextFileContent, IResourceEncodings, IReadTextFileOptions, IWriteTextFileOptions, toBufferOrReadable, TextFileOperationError, TextFileOperationResult, ITextFileSaveOptions, ITextFileEditorModelManager, TextFileCreateEvent, IResourceEncoding } from 'vs/workbench/services/textfile/common/textfiles';
 import { IRevertOptions, IEncodingSupport } from 'vs/workbench/common/editor';
 import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle';
 import { IFileService, FileOperationError, FileOperationResult, IFileStatWithMetadata, ICreateFileOptions, FileOperation } from 'vs/platform/files/common/files';
@@ -19,7 +19,7 @@ import { IInstantiationService } from 'vs/platform/instantiation/common/instanti
 import { Schemas } from 'vs/base/common/network';
 import { createTextBufferFactoryFromSnapshot, createTextBufferFactoryFromStream } from 'vs/editor/common/model/textModel';
 import { IModelService } from 'vs/editor/common/services/modelService';
-import { joinPath, dirname, basename, toLocalResource, extUri } from 'vs/base/common/resources';
+import { joinPath, dirname, basename, toLocalResource, extUri, extname, isEqualOrParent } from 'vs/base/common/resources';
 import { IDialogService, IFileDialogService, IConfirmation } from 'vs/platform/dialogs/common/dialogs';
 import { VSBuffer } from 'vs/base/common/buffer';
 import { ITextSnapshot, ITextModel } from 'vs/editor/common/model';
@@ -35,6 +35,10 @@ import { IPathService } from 'vs/workbench/services/path/common/pathService';
 import { isValidBasename } from 'vs/base/common/extpath';
 import { IWorkingCopyFileService } from 'vs/workbench/services/workingCopy/common/workingCopyFileService';
 import { IUriIdentityService } from 'vs/workbench/services/uriIdentity/common/uriIdentity';
+import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
+import { WORKSPACE_EXTENSION } from 'vs/platform/workspaces/common/workspaces';
+import { IEnvironmentService } from 'vs/platform/environment/common/environment';
+import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, UTF8_BOM, detectEncodingByBOMFromBuffer } from 'vs/workbench/services/textfile/common/encoding';
 
 /**
  * The workbench file service implementation implements the raw file service spec and adds additional methods on top.
@@ -54,8 +58,6 @@ export abstract class AbstractTextFileService extends Disposable implements ITex
 
 	readonly untitled: IUntitledTextEditorModelManager = this.untitledTextEditorService;
 
-	abstract get encoding(): IResourceEncodings;
-
 	constructor(
 		@IFileService protected readonly fileService: IFileService,
 		@IUntitledTextEditorService private untitledTextEditorService: IUntitledTextEditorService,
@@ -86,6 +88,16 @@ export abstract class AbstractTextFileService extends Disposable implements ITex
 
 	//#region text file read / write / create
 
+	private _encoding: EncodingOracle | undefined;
+
+	get encoding(): EncodingOracle {
+		if (!this._encoding) {
+			this._encoding = this._register(this.instantiationService.createInstance(EncodingOracle));
+		}
+
+		return this._encoding;
+	}
+
 	async read(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileContent> {
 		const content = await this.fileService.readFile(resource, options);
 
@@ -491,3 +503,150 @@ export abstract class AbstractTextFileService extends Disposable implements ITex
 
 	//#endregion
 }
+
+export interface IEncodingOverride {
+	parent?: URI;
+	extension?: string;
+	encoding: string;
+}
+
+export class EncodingOracle extends Disposable implements IResourceEncodings {
+
+	private _encodingOverrides: IEncodingOverride[];
+	protected get encodingOverrides(): IEncodingOverride[] { return this._encodingOverrides; }
+	protected set encodingOverrides(value: IEncodingOverride[]) { this._encodingOverrides = value; }
+
+	constructor(
+		@ITextResourceConfigurationService private textResourceConfigurationService: ITextResourceConfigurationService,
+		@IEnvironmentService private environmentService: IEnvironmentService,
+		@IWorkspaceContextService private contextService: IWorkspaceContextService,
+		@IFileService private fileService: IFileService
+	) {
+		super();
+
+		this._encodingOverrides = this.getDefaultEncodingOverrides();
+
+		this.registerListeners();
+	}
+
+	private registerListeners(): void {
+
+		// Workspace Folder Change
+		this._register(this.contextService.onDidChangeWorkspaceFolders(() => this.encodingOverrides = this.getDefaultEncodingOverrides()));
+	}
+
+	private getDefaultEncodingOverrides(): IEncodingOverride[] {
+		const defaultEncodingOverrides: IEncodingOverride[] = [];
+
+		// Global settings
+		defaultEncodingOverrides.push({ parent: this.environmentService.userRoamingDataHome, encoding: UTF8 });
+
+		// Workspace files (via extension and via untitled workspaces location)
+		defaultEncodingOverrides.push({ extension: WORKSPACE_EXTENSION, encoding: UTF8 });
+		defaultEncodingOverrides.push({ parent: this.environmentService.untitledWorkspacesHome, encoding: UTF8 });
+
+		// Folder Settings
+		this.contextService.getWorkspace().folders.forEach(folder => {
+			defaultEncodingOverrides.push({ parent: joinPath(folder.uri, '.vscode'), encoding: UTF8 });
+		});
+
+		return defaultEncodingOverrides;
+	}
+
+	async getWriteEncoding(resource: URI, options?: IWriteTextFileOptions): Promise<{ encoding: string, addBOM: boolean }> {
+		const { encoding, hasBOM } = await this.getPreferredWriteEncoding(resource, options ? options.encoding : undefined);
+
+		// Some encodings come with a BOM automatically
+		if (hasBOM) {
+			return { encoding, addBOM: true };
+		}
+
+		// Ensure that we preserve an existing BOM if found for UTF8
+		// unless we are instructed to overwrite the encoding
+		const overwriteEncoding = options?.overwriteEncoding;
+		if (!overwriteEncoding && encoding === UTF8) {
+			try {
+				const buffer = (await this.fileService.readFile(resource, { length: UTF8_BOM.length })).value;
+				if (detectEncodingByBOMFromBuffer(buffer, buffer.byteLength) === UTF8_with_bom) {
+					return { encoding, addBOM: true };
+				}
+			} catch (error) {
+				// ignore - file might not exist
+			}
+		}
+
+		return { encoding, addBOM: false };
+	}
+
+	async getPreferredWriteEncoding(resource: URI, preferredEncoding?: string): Promise<IResourceEncoding> {
+		const resourceEncoding = await this.getEncodingForResource(resource, preferredEncoding);
+
+		return {
+			encoding: resourceEncoding,
+			hasBOM: resourceEncoding === UTF16be || resourceEncoding === UTF16le || resourceEncoding === UTF8_with_bom // enforce BOM for certain encodings
+		};
+	}
+
+	getReadEncoding(resource: URI, options: IReadTextFileOptions | undefined, detectedEncoding: string | null): Promise<string> {
+		let preferredEncoding: string | undefined;
+
+		// Encoding passed in as option
+		if (options?.encoding) {
+			if (detectedEncoding === UTF8_with_bom && options.encoding === UTF8) {
+				preferredEncoding = UTF8_with_bom; // indicate the file has BOM if we are to resolve with UTF 8
+			} else {
+				preferredEncoding = options.encoding; // give passed in encoding highest priority
+			}
+		}
+
+		// Encoding detected
+		else if (detectedEncoding) {
+			preferredEncoding = detectedEncoding;
+		}
+
+		// Encoding configured
+		else if (this.textResourceConfigurationService.getValue(resource, 'files.encoding') === UTF8_with_bom) {
+			preferredEncoding = UTF8; // if we did not detect UTF 8 BOM before, this can only be UTF 8 then
+		}
+
+		return this.getEncodingForResource(resource, preferredEncoding);
+	}
+
+	private async getEncodingForResource(resource: URI, preferredEncoding?: string): Promise<string> {
+		let fileEncoding: string;
+
+		const override = this.getEncodingOverride(resource);
+		if (override) {
+			fileEncoding = override; // encoding override always wins
+		} else if (preferredEncoding) {
+			fileEncoding = preferredEncoding; // preferred encoding comes second
+		} else {
+			fileEncoding = this.textResourceConfigurationService.getValue(resource, 'files.encoding'); // and last we check for settings
+		}
+
+		if (!fileEncoding || !(await encodingExists(fileEncoding))) {
+			fileEncoding = UTF8; // the default is UTF 8
+		}
+
+		return fileEncoding;
+	}
+
+	private getEncodingOverride(resource: URI): string | undefined {
+		if (this.encodingOverrides && this.encodingOverrides.length) {
+			for (const override of this.encodingOverrides) {
+
+				// check if the resource is child of encoding override path
+				if (override.parent && isEqualOrParent(resource, override.parent)) {
+					return override.encoding;
+				}
+
+				// check if the resource extension is equal to encoding override
+				if (override.extension && extname(resource) === `.${override.extension}`) {
+					return override.encoding;
+				}
+			}
+		}
+
+		return undefined;
+	}
+}

src/vs/workbench/services/textfile/common/encoding.ts

@@ -65,7 +65,7 @@ export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeS
 				// decode and write buffered content
 				const iconv = await import('iconv-lite-umd');
 				decoder = iconv.getDecoder(toNodeEncoding(detected.encoding));
-				const decoded = decoder.write(Buffer.from(VSBuffer.concat(bufferedChunks).buffer));
+				const decoded = decoder.write(VSBuffer.concat(bufferedChunks).buffer);
 				target.write(decoded);
 
 				bufferedChunks.length = 0;
@@ -89,7 +89,7 @@ export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeS
 
 			// if the decoder is ready, we just write directly
 			if (decoder) {
-				target.write(decoder.write(Buffer.from(chunk.buffer)));
+				target.write(decoder.write(chunk.buffer));
 			}
 
 			// otherwise we need to buffer the data until the stream is ready
@@ -234,7 +234,13 @@ const IGNORE_ENCODINGS = ['ascii', 'utf-16', 'utf-32'];
 async function guessEncodingByBuffer(buffer: VSBuffer): Promise<string | null> {
 	const jschardet = await import('jschardet');
 
-	const guessed = jschardet.detect(Buffer.from(buffer.slice(0, AUTO_ENCODING_GUESS_MAX_BYTES).buffer)); // ensure to limit buffer for guessing due to https://github.com/aadsm/jschardet/issues/53
+	// ensure to limit buffer for guessing due to https://github.com/aadsm/jschardet/issues/53
+	const limitedBuffer = buffer.slice(0, AUTO_ENCODING_GUESS_MAX_BYTES);
+	// override type since jschardet expects Buffer even though can accept Uint8Array
+	// can be fixed once https://github.com/aadsm/jschardet/pull/58 is merged
+	const jschardetTypingsWorkaround = limitedBuffer.buffer as any;
+
+	const guessed = jschardet.detect(jschardetTypingsWorkaround);
 	if (!guessed || !guessed.encoding) {
 		return null;
 	}

src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts

@@ -5,7 +5,7 @@
 
 import { localize } from 'vs/nls';
 import { AbstractTextFileService } from 'vs/workbench/services/textfile/browser/textFileService';
-import { ITextFileService, ITextFileStreamContent, ITextFileContent, IResourceEncodings, IResourceEncoding, IReadTextFileOptions, IWriteTextFileOptions, stringToSnapshot, TextFileOperationResult, TextFileOperationError } from 'vs/workbench/services/textfile/common/textfiles';
+import { ITextFileService, ITextFileStreamContent, ITextFileContent, IReadTextFileOptions, IWriteTextFileOptions, stringToSnapshot, TextFileOperationResult, TextFileOperationError } from 'vs/workbench/services/textfile/common/textfiles';
 import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
 import { URI } from 'vs/base/common/uri';
 import { IFileStatWithMetadata, ICreateFileOptions, FileOperationError, FileOperationResult, IFileStreamContent, IFileService } from 'vs/platform/files/common/files';
@@ -15,12 +15,7 @@ import { join, dirname } from 'vs/base/common/path';
 import { isMacintosh } from 'vs/base/common/platform';
 import { IProductService } from 'vs/platform/product/common/productService';
 import { ITextResourceConfigurationService } from 'vs/editor/common/services/textResourceConfigurationService';
-import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
-import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, UTF8_BOM, toDecodeStream, toEncodeReadable, IDecodeStreamResult, detectEncodingByBOMFromBuffer } from 'vs/base/node/encoding';
-import { WORKSPACE_EXTENSION } from 'vs/platform/workspaces/common/workspaces';
-import { joinPath, extname, isEqualOrParent } from 'vs/base/common/resources';
-import { Disposable } from 'vs/base/common/lifecycle';
-import { IEnvironmentService } from 'vs/platform/environment/common/environment';
+import { UTF8, UTF8_with_bom, toDecodeStream, toEncodeReadable, IDecodeStreamResult } from 'vs/workbench/services/textfile/common/encoding';
 import { bufferToStream, VSBufferReadable } from 'vs/base/common/buffer';
 import { createTextBufferFactoryFromStream } from 'vs/editor/common/model/textModel';
 import { ITextSnapshot } from 'vs/editor/common/model';
@@ -64,15 +59,6 @@ export class NativeTextFileService extends AbstractTextFileService {
 		super(fileService, untitledTextEditorService, lifecycleService, instantiationService, modelService, environmentService, dialogService, fileDialogService, textResourceConfigurationService, filesConfigurationService, textModelService, codeEditorService, pathService, workingCopyFileService, uriIdentityService);
 	}
 
-	private _encoding: EncodingOracle | undefined;
-	get encoding(): EncodingOracle {
-		if (!this._encoding) {
-			this._encoding = this._register(this.instantiationService.createInstance(EncodingOracle));
-		}
-
-		return this._encoding;
-	}
-
 	async read(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileContent> {
 		const [bufferStream, decoder] = await this.doRead(resource, {
 			...options,
@@ -292,151 +278,4 @@ export class NativeTextFileService extends AbstractTextFileService {
 	}
 }
 
-export interface IEncodingOverride {
-	parent?: URI;
-	extension?: string;
-	encoding: string;
-}
-
-export class EncodingOracle extends Disposable implements IResourceEncodings {
-
-	private _encodingOverrides: IEncodingOverride[];
-	protected get encodingOverrides(): IEncodingOverride[] { return this._encodingOverrides; }
-	protected set encodingOverrides(value: IEncodingOverride[]) { this._encodingOverrides = value; }
-
-	constructor(
-		@ITextResourceConfigurationService private textResourceConfigurationService: ITextResourceConfigurationService,
-		@IEnvironmentService private environmentService: IEnvironmentService,
-		@IWorkspaceContextService private contextService: IWorkspaceContextService,
-		@IFileService private fileService: IFileService
-	) {
-		super();
-
-		this._encodingOverrides = this.getDefaultEncodingOverrides();
-
-		this.registerListeners();
-	}
-
-	private registerListeners(): void {
-
-		// Workspace Folder Change
-		this._register(this.contextService.onDidChangeWorkspaceFolders(() => this.encodingOverrides = this.getDefaultEncodingOverrides()));
-	}
-
-	private getDefaultEncodingOverrides(): IEncodingOverride[] {
-		const defaultEncodingOverrides: IEncodingOverride[] = [];
-
-		// Global settings
-		defaultEncodingOverrides.push({ parent: this.environmentService.userRoamingDataHome, encoding: UTF8 });
-
-		// Workspace files (via extension and via untitled workspaces location)
-		defaultEncodingOverrides.push({ extension: WORKSPACE_EXTENSION, encoding: UTF8 });
-		defaultEncodingOverrides.push({ parent: this.environmentService.untitledWorkspacesHome, encoding: UTF8 });
-
-		// Folder Settings
-		this.contextService.getWorkspace().folders.forEach(folder => {
-			defaultEncodingOverrides.push({ parent: joinPath(folder.uri, '.vscode'), encoding: UTF8 });
-		});
-
-		return defaultEncodingOverrides;
-	}
-
-	async getWriteEncoding(resource: URI, options?: IWriteTextFileOptions): Promise<{ encoding: string, addBOM: boolean }> {
-		const { encoding, hasBOM } = await this.getPreferredWriteEncoding(resource, options ? options.encoding : undefined);
-
-		// Some encodings come with a BOM automatically
-		if (hasBOM) {
-			return { encoding, addBOM: true };
-		}
-
-		// Ensure that we preserve an existing BOM if found for UTF8
-		// unless we are instructed to overwrite the encoding
-		const overwriteEncoding = options?.overwriteEncoding;
-		if (!overwriteEncoding && encoding === UTF8) {
-			try {
-				const buffer = (await this.fileService.readFile(resource, { length: UTF8_BOM.length })).value;
-				if (detectEncodingByBOMFromBuffer(buffer, buffer.byteLength) === UTF8_with_bom) {
-					return { encoding, addBOM: true };
-				}
-			} catch (error) {
-				// ignore - file might not exist
-			}
-		}
-
-		return { encoding, addBOM: false };
-	}
-
-	async getPreferredWriteEncoding(resource: URI, preferredEncoding?: string): Promise<IResourceEncoding> {
-		const resourceEncoding = await this.getEncodingForResource(resource, preferredEncoding);
-
-		return {
-			encoding: resourceEncoding,
-			hasBOM: resourceEncoding === UTF16be || resourceEncoding === UTF16le || resourceEncoding === UTF8_with_bom // enforce BOM for certain encodings
-		};
-	}
-
-	getReadEncoding(resource: URI, options: IReadTextFileOptions | undefined, detectedEncoding: string | null): Promise<string> {
-		let preferredEncoding: string | undefined;
-
-		// Encoding passed in as option
-		if (options?.encoding) {
-			if (detectedEncoding === UTF8_with_bom && options.encoding === UTF8) {
-				preferredEncoding = UTF8_with_bom; // indicate the file has BOM if we are to resolve with UTF 8
-			} else {
-				preferredEncoding = options.encoding; // give passed in encoding highest priority
-			}
-		}
-
-		// Encoding detected
-		else if (detectedEncoding) {
-			preferredEncoding = detectedEncoding;
-		}
-
-		// Encoding configured
-		else if (this.textResourceConfigurationService.getValue(resource, 'files.encoding') === UTF8_with_bom) {
-			preferredEncoding = UTF8; // if we did not detect UTF 8 BOM before, this can only be UTF 8 then
-		}
-
-		return this.getEncodingForResource(resource, preferredEncoding);
-	}
-
-	private async getEncodingForResource(resource: URI, preferredEncoding?: string): Promise<string> {
-		let fileEncoding: string;
-
-		const override = this.getEncodingOverride(resource);
-		if (override) {
-			fileEncoding = override; // encoding override always wins
-		} else if (preferredEncoding) {
-			fileEncoding = preferredEncoding; // preferred encoding comes second
-		} else {
-			fileEncoding = this.textResourceConfigurationService.getValue(resource, 'files.encoding'); // and last we check for settings
-		}
-
-		if (!fileEncoding || !(await encodingExists(fileEncoding))) {
-			fileEncoding = UTF8; // the default is UTF 8
-		}
-
-		return fileEncoding;
-	}
-
-	private getEncodingOverride(resource: URI): string | undefined {
-		if (this.encodingOverrides && this.encodingOverrides.length) {
-			for (const override of this.encodingOverrides) {
-
-				// check if the resource is child of encoding override path
-				if (override.parent && isEqualOrParent(resource, override.parent)) {
-					return override.encoding;
-				}
-
-				// check if the resource extension is equal to encoding override
-				if (override.extension && extname(resource) === `.${override.extension}`) {
-					return override.encoding;
-				}
-			}
-		}
-
-		return undefined;
-	}
-}
-
 registerSingleton(ITextFileService, NativeTextFileService);

src/vs/workbench/services/textfile/test/electron-browser/textFileService.io.test.ts

@@ -20,12 +20,12 @@ import { DiskFileSystemProvider } from 'vs/platform/files/node/diskFileSystemPro
 import { generateUuid } from 'vs/base/common/uuid';
 import { join, basename } from 'vs/base/common/path';
 import { getPathFromAmdModule } from 'vs/base/common/amd';
-import { UTF16be, UTF16le, UTF8_with_bom, UTF8 } from 'vs/base/node/encoding';
+import { UTF16be, UTF16le, UTF8_with_bom, UTF8 } from 'vs/workbench/services/textfile/common/encoding';
 import { DefaultEndOfLine, ITextSnapshot } from 'vs/editor/common/model';
 import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
 import { isWindows } from 'vs/base/common/platform';
 import { readFileSync, statSync } from 'fs';
-import { detectEncodingByBOM } from 'vs/base/test/node/encoding/encoding.test';
+import { detectEncodingByBOM } from 'vs/workbench/services/textfile/test/node/encoding/encoding.test';
 import { workbenchInstantiationService, TestNativeTextFileServiceWithEncodingOverrides } from 'vs/workbench/test/electron-browser/workbenchTestServices';
 
 suite('Files - TextFileService i/o', function () {

src/vs/workbench/services/textfile/test/node/encoding/encoding.test.ts

@@ -5,7 +5,7 @@
 
 import * as assert from 'assert';
 import * as fs from 'fs';
-import * as encoding from 'vs/base/node/encoding';
+import * as encoding from 'vs/workbench/services/textfile/common/encoding';
 import * as terminalEncoding from 'vs/base/node/terminalEncoding';
 import * as streams from 'vs/base/common/stream';
 import * as iconv from 'iconv-lite-umd';

src/vs/workbench/test/electron-browser/workbenchTestServices.ts

@@ -7,7 +7,8 @@ import { workbenchInstantiationService as browserWorkbenchInstantiationService,
 import { Event } from 'vs/base/common/event';
 import { ISharedProcessService } from 'vs/platform/ipc/electron-browser/sharedProcessService';
 import { NativeWorkbenchEnvironmentService, INativeWorkbenchEnvironmentService } from 'vs/workbench/services/environment/electron-browser/environmentService';
-import { NativeTextFileService, EncodingOracle, IEncodingOverride } from 'vs/workbench/services/textfile/electron-browser/nativeTextFileService';
+import { NativeTextFileService, } from 'vs/workbench/services/textfile/electron-browser/nativeTextFileService';
+import { EncodingOracle, IEncodingOverride } from 'vs/workbench/services/textfile/browser/textFileService';
 import { IElectronService } from 'vs/platform/electron/electron-sandbox/electron';
 import { FileOperationError, IFileService } from 'vs/platform/files/common/files';
 import { IUntitledTextEditorService } from 'vs/workbench/services/untitled/common/untitledTextEditorService';
@@ -29,7 +30,7 @@ import { parseArgs, OPTIONS } from 'vs/platform/environment/node/argv';
 import { LogLevel, ILogService } from 'vs/platform/log/common/log';
 import { IPathService } from 'vs/workbench/services/path/common/pathService';
 import { IWorkingCopyFileService } from 'vs/workbench/services/workingCopy/common/workingCopyFileService';
-import { UTF16le, UTF16be, UTF8_with_bom } from 'vs/base/node/encoding';
+import { UTF16le, UTF16be, UTF8_with_bom } from 'vs/workbench/services/textfile/common/encoding';
 import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
 import { ModelServiceImpl } from 'vs/editor/common/services/modelServiceImpl';
 import { IBackupFileService } from 'vs/workbench/services/backup/common/backup';

yarn.lock

@@ -4572,10 +4572,10 @@ husky@^0.13.1:
     is-ci "^1.0.9"
     normalize-path "^1.0.0"
 
-iconv-lite-umd@0.6.2:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.2.tgz#6410d3dc1bf5b0e0863f833d67e8168fcd52d47c"
-  integrity sha512-KOOIU5p4j/NOXybhgOF7ZMRMQ7+iOWwnr1+DSQaPCzCRfR1+vMzvEmjmrmUZ59kHkhcqZW7eABTa/axpc3i81w==
+iconv-lite-umd@0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite-umd/-/iconv-lite-umd-0.6.3.tgz#61307cab8ac29939992d0724d3ab8799467f0e97"
+  integrity sha512-fQ/8XE8reiCZ6t+SX4tX6/tQdV4tThJZv5qtMe5Sk+IWsExz0S2Zd+GiBS5IEPgDxnsmiJSpH67+qzN3FT4lKw==
 
 iconv-lite@^0.4.19:
   version "0.4.19"