maxmustermann/vscode
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Move read and write logic from nativeTextFileService to textFileService for #79275 (#100804)

Browse source 
* move encoding logic from NativeTextFileService to AbstractTextFileService for #79275

* some cleanup things - just cosmetic

* fix tests

* review

* use correct comparison

Co-authored-by: Benjamin Pasero <benjpas@microsoft.com>
This commit is contained in:
Fedor Nezhivoi 2020-07-03 23:55:44 +10:00 committed by GitHub
parent 1fb0d4424d
commit 05613d7a1c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 833 additions and 217 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/vs/platform/files/common/inMemoryFilesystemProvider.ts
View file

@ -11,7 +11,7 @@ import { URI } from 'vs/base/common/uri';
class File implements IStat {
type: FileType;
type: FileType.File;
ctime: number;
mtime: number;
size: number;
@ -30,7 +30,7 @@ class File implements IStat {
class Directory implements IStat {
type: FileType;
type: FileType.Directory;
ctime: number;
mtime: number;
size: number;

121
src/vs/workbench/services/textfile/browser/textFileService.ts
View file

@ -5,10 +5,10 @@
import * as nls from 'vs/nls';
import { URI } from 'vs/base/common/uri';
import { ITextFileService, ITextFileStreamContent, ITextFileContent, IResourceEncodings, IReadTextFileOptions, IWriteTextFileOptions, toBufferOrReadable, TextFileOperationError, TextFileOperationResult, ITextFileSaveOptions, ITextFileEditorModelManager, IResourceEncoding } from 'vs/workbench/services/textfile/common/textfiles';
import { ITextFileService, ITextFileStreamContent, ITextFileContent, IResourceEncodings, IReadTextFileOptions, IWriteTextFileOptions, toBufferOrReadable, TextFileOperationError, TextFileOperationResult, ITextFileSaveOptions, ITextFileEditorModelManager, IResourceEncoding, stringToSnapshot } from 'vs/workbench/services/textfile/common/textfiles';
import { IRevertOptions, IEncodingSupport } from 'vs/workbench/common/editor';
import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle';
import { IFileService, FileOperationError, FileOperationResult, IFileStatWithMetadata, ICreateFileOptions } from 'vs/platform/files/common/files';
import { IFileService, FileOperationError, FileOperationResult, IFileStatWithMetadata, ICreateFileOptions, IFileStreamContent } from 'vs/platform/files/common/files';
import { Disposable } from 'vs/base/common/lifecycle';
import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
import { IUntitledTextEditorService, IUntitledTextEditorModelManager } from 'vs/workbench/services/untitled/common/untitledTextEditorService';
@ -20,7 +20,7 @@ import { createTextBufferFactoryFromSnapshot, createTextBufferFactoryFromStream
import { IModelService } from 'vs/editor/common/services/modelService';
import { joinPath, dirname, basename, toLocalResource, extUri, extname, isEqualOrParent } from 'vs/base/common/resources';
import { IDialogService, IFileDialogService, IConfirmation } from 'vs/platform/dialogs/common/dialogs';
import { VSBuffer, VSBufferReadable } from 'vs/base/common/buffer';
import { VSBuffer, VSBufferReadable, bufferToStream } from 'vs/base/common/buffer';
import { ITextSnapshot, ITextModel } from 'vs/editor/common/model';
import { ITextResourceConfigurationService } from 'vs/editor/common/services/textResourceConfigurationService';
import { PLAINTEXT_MODE_ID } from 'vs/editor/common/modes/modesRegistry';
@ -36,7 +36,8 @@ import { IUriIdentityService } from 'vs/workbench/services/uriIdentity/common/ur
import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
import { WORKSPACE_EXTENSION } from 'vs/platform/workspaces/common/workspaces';
import { IEnvironmentService } from 'vs/platform/environment/common/environment';
import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, UTF8_BOM, detectEncodingByBOMFromBuffer } from 'vs/workbench/services/textfile/common/encoding';
import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, UTF8_BOM, detectEncodingByBOMFromBuffer, toEncodeReadable, toDecodeStream, IDecodeStreamResult } from 'vs/workbench/services/textfile/common/encoding';
import { consumeStream } from 'vs/base/common/stream';
/**
* The workbench file service implementation implements the raw file service spec and adds additional methods on top.
@ -90,75 +91,91 @@ export abstract class AbstractTextFileService extends Disposable implements ITex
}
async read(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileContent> {
const content = await this.fileService.readFile(resource, options);
// in case of acceptTextOnly: true, we check the first
// chunk for possibly being binary by looking for 0-bytes
// we limit this check to the first 512 bytes
this.validateBinary(content.value, options);
const [bufferStream, decoder] = await this.doRead(resource, {
...options,
// optimization: since we know that the caller does not
// care about buffering, we indicate this to the reader.
// this reduces all the overhead the buffered reading
// has (open, read, close) if the provider supports
// unbuffered reading.
preferUnbuffered: true
});
return {
...content,
encoding: 'utf8',
value: content.value.toString()
...bufferStream,
encoding: decoder.detected.encoding || UTF8,
value: await consumeStream(decoder.stream, strings => strings.join(''))
};
}
async readStream(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileStreamContent> {
const stream = await this.fileService.readFileStream(resource, options);
// in case of acceptTextOnly: true, we check the first
// chunk for possibly being binary by looking for 0-bytes
// we limit this check to the first 512 bytes
let checkedForBinary = false;
const throwOnBinary = (data: VSBuffer): Error | undefined => {
if (!checkedForBinary) {
checkedForBinary = true;
this.validateBinary(data, options);
}
return undefined;
};
const [bufferStream, decoder] = await this.doRead(resource, options);
return {
...stream,
encoding: 'utf8',
value: await createTextBufferFactoryFromStream(stream.value, undefined, options?.acceptTextOnly ? throwOnBinary : undefined)
...bufferStream,
encoding: decoder.detected.encoding || UTF8,
value: await createTextBufferFactoryFromStream(decoder.stream)
};
}
private validateBinary(buffer: VSBuffer, options?: IReadTextFileOptions): void {
if (!options || !options.acceptTextOnly) {
return; // no validation needed
private async doRead(resource: URI, options?: IReadTextFileOptions & { preferUnbuffered?: boolean }): Promise<[IFileStreamContent, IDecodeStreamResult]> {
// read stream raw (either buffered or unbuffered)
let bufferStream: IFileStreamContent;
if (options?.preferUnbuffered) {
const content = await this.fileService.readFile(resource, options);
bufferStream = {
...content,
value: bufferToStream(content.value)
};
} else {
bufferStream = await this.fileService.readFileStream(resource, options);
}
// in case of acceptTextOnly: true, we check the first
// chunk for possibly being binary by looking for 0-bytes
// we limit this check to the first 512 bytes
for (let i = 0; i < buffer.byteLength && i < 512; i++) {
if (buffer.readUInt8(i) === 0) {
throw new TextFileOperationError(nls.localize('fileBinaryError', "File seems to be binary and cannot be opened as text"), TextFileOperationResult.FILE_IS_BINARY, options);
}
// read through encoding library
const decoder = await toDecodeStream(bufferStream.value, {
guessEncoding: options?.autoGuessEncoding || this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'),
overwriteEncoding: detectedEncoding => this.encoding.getReadEncoding(resource, options, detectedEncoding)
});
// validate binary
if (options?.acceptTextOnly && decoder.detected.seemsBinary) {
throw new TextFileOperationError(nls.localize('fileBinaryError', "File seems to be binary and cannot be opened as text"), TextFileOperationResult.FILE_IS_BINARY, options);
}
return [bufferStream, decoder];
}
async create(resource: URI, value?: string | ITextSnapshot, options?: ICreateFileOptions): Promise<IFileStatWithMetadata> {
const encodedValue = await this.doEncodeText(resource, value);
const readable = await this.getEncodedReadable(resource, value);
return this.workingCopyFileService.create(resource, encodedValue, options);
}
protected async doEncodeText(resource: URI, value?: string | ITextSnapshot): Promise<VSBuffer | VSBufferReadable | undefined> {
if (!value) {
return undefined;
}
return toBufferOrReadable(value);
return this.workingCopyFileService.create(resource, readable, options);
}
async write(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {
return this.fileService.writeFile(resource, toBufferOrReadable(value), options);
const readable = await this.getEncodedReadable(resource, value, options);
return this.fileService.writeFile(resource, readable, options);
}
private async getEncodedReadable(resource: URI, value?: string | ITextSnapshot): Promise<VSBuffer | VSBufferReadable | undefined>;
private async getEncodedReadable(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<VSBuffer | VSBufferReadable>;
private async getEncodedReadable(resource: URI, value?: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<VSBuffer | VSBufferReadable | undefined> {
// check for encoding
const { encoding, addBOM } = await this.encoding.getWriteEncoding(resource, options);
// when encoding is standard skip encoding step
if (encoding === UTF8 && !addBOM) {
return typeof value === 'undefined'
? undefined
: toBufferOrReadable(value);
}
// otherwise create encoded readable
value = value || '';
const snapshot = typeof value === 'string' ? stringToSnapshot(value) : value;
return toEncodeReadable(snapshot, encoding, { addBOM });
}
//#endregion

19
src/vs/workbench/services/textfile/common/encoding.ts
View file

@ -236,9 +236,15 @@ async function guessEncodingByBuffer(buffer: VSBuffer): Promise<string | null> {
// ensure to limit buffer for guessing due to https://github.com/aadsm/jschardet/issues/53
const limitedBuffer = buffer.slice(0, AUTO_ENCODING_GUESS_MAX_BYTES);
// override type since jschardet expects Buffer even though can accept Uint8Array
// before guessing jschardet calls toString('binary') on input if it is a Buffer,
// since we are using it inside browser environment as well we do conversion ourselves
// https://github.com/aadsm/jschardet/blob/v2.1.1/src/index.js#L36-L40
const binaryString = encodeLatin1(limitedBuffer.buffer);
// override type since jschardet expects Buffer even though can accept string
// can be fixed once https://github.com/aadsm/jschardet/pull/58 is merged
const jschardetTypingsWorkaround = limitedBuffer.buffer as any;
const jschardetTypingsWorkaround = binaryString as any;
const guessed = jschardet.detect(jschardetTypingsWorkaround);
if (!guessed || !guessed.encoding) {
@ -265,6 +271,15 @@ function toIconvLiteEncoding(encodingName: string): string {
return mapped || normalizedEncodingName;
}
function encodeLatin1(buffer: Uint8Array): string {
let result = '';
for (let i = 0; i < buffer.length; i++) {
result += String.fromCharCode(buffer[i]);
}
return result;
}
/**
* The encodings that are allowed in a settings file don't match the canonical encoding labels specified by WHATWG.
* See https://encoding.spec.whatwg.org/#names-and-labels

107
src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts
View file

@ -5,21 +5,18 @@
import { localize } from 'vs/nls';
import { AbstractTextFileService } from 'vs/workbench/services/textfile/browser/textFileService';
import { ITextFileService, ITextFileStreamContent, ITextFileContent, IReadTextFileOptions, IWriteTextFileOptions, stringToSnapshot, TextFileOperationResult, TextFileOperationError } from 'vs/workbench/services/textfile/common/textfiles';
import { ITextFileService, ITextFileStreamContent, ITextFileContent, IReadTextFileOptions, IWriteTextFileOptions } from 'vs/workbench/services/textfile/common/textfiles';
import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
import { URI } from 'vs/base/common/uri';
import { IFileStatWithMetadata, FileOperationError, FileOperationResult, IFileStreamContent, IFileService } from 'vs/platform/files/common/files';
import { IFileStatWithMetadata, FileOperationError, FileOperationResult, IFileService } from 'vs/platform/files/common/files';
import { Schemas } from 'vs/base/common/network';
import { stat, chmod, MAX_FILE_SIZE, MAX_HEAP_SIZE } from 'vs/base/node/pfs';
import { join, dirname } from 'vs/base/common/path';
import { isMacintosh } from 'vs/base/common/platform';
import { IProductService } from 'vs/platform/product/common/productService';
import { ITextResourceConfigurationService } from 'vs/editor/common/services/textResourceConfigurationService';
import { UTF8, UTF8_with_bom, toDecodeStream, toEncodeReadable, IDecodeStreamResult } from 'vs/workbench/services/textfile/common/encoding';
import { bufferToStream, VSBufferReadable, VSBuffer } from 'vs/base/common/buffer';
import { createTextBufferFactoryFromStream } from 'vs/editor/common/model/textModel';
import { UTF8, UTF8_with_bom } from 'vs/workbench/services/textfile/common/encoding';
import { ITextSnapshot } from 'vs/editor/common/model';
import { consumeStream } from 'vs/base/common/stream';
import { IUntitledTextEditorService } from 'vs/workbench/services/untitled/common/untitledTextEditorService';
import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle';
import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
@ -60,62 +57,19 @@ export class NativeTextFileService extends AbstractTextFileService {
}
async read(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileContent> {
const [bufferStream, decoder] = await this.doRead(resource, {
...options,
// optimization: since we know that the caller does not
// care about buffering, we indicate this to the reader.
// this reduces all the overhead the buffered reading
// has (open, read, close) if the provider supports
// unbuffered reading.
preferUnbuffered: true
});
return {
...bufferStream,
encoding: decoder.detected.encoding || UTF8,
value: await consumeStream(decoder.stream, strings => strings.join(''))
};
// ensure size & memory limits
options = this.ensureLimits(options);
return super.read(resource, options);
}
async readStream(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileStreamContent> {
const [bufferStream, decoder] = await this.doRead(resource, options);
return {
...bufferStream,
encoding: decoder.detected.encoding || UTF8,
value: await createTextBufferFactoryFromStream(decoder.stream)
};
}
private async doRead(resource: URI, options?: IReadTextFileOptions & { preferUnbuffered?: boolean }): Promise<[IFileStreamContent, IDecodeStreamResult]> {
// ensure limits
// ensure size & memory limits
options = this.ensureLimits(options);
// read stream raw (either buffered or unbuffered)
let bufferStream: IFileStreamContent;
if (options.preferUnbuffered) {
const content = await this.fileService.readFile(resource, options);
bufferStream = {
...content,
value: bufferToStream(content.value)
};
} else {
bufferStream = await this.fileService.readFileStream(resource, options);
}
// read through encoding library
const decoder = await toDecodeStream(bufferStream.value, {
guessEncoding: options?.autoGuessEncoding || this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'),
overwriteEncoding: detectedEncoding => this.encoding.getReadEncoding(resource, options, detectedEncoding)
});
// validate binary
if (options?.acceptTextOnly && decoder.detected.seemsBinary) {
throw new TextFileOperationError(localize('fileBinaryError', "File seems to be binary and cannot be opened as text"), TextFileOperationResult.FILE_IS_BINARY, options);
}
return [bufferStream, decoder];
return super.readStream(resource, options);
}
private ensureLimits(options?: IReadTextFileOptions): IReadTextFileOptions {
@ -139,28 +93,17 @@ export class NativeTextFileService extends AbstractTextFileService {
}
if (typeof ensuredLimits.memory !== 'number') {
ensuredLimits.memory = Math.max(typeof this.environmentService.args['max-memory'] === 'string' ? parseInt(this.environmentService.args['max-memory']) * 1024 * 1024 || 0 : 0, MAX_HEAP_SIZE);
const maxMemory = this.environmentService.args['max-memory'];
ensuredLimits.memory = Math.max(
typeof maxMemory === 'string'
? parseInt(maxMemory) * 1024 * 1024 || 0
: 0, MAX_HEAP_SIZE
);
}
return ensuredOptions;
}
protected async doEncodeText(resource: URI, value?: string | ITextSnapshot): Promise<VSBuffer | VSBufferReadable | undefined> {
// check for encoding
const { encoding, addBOM } = await this.encoding.getWriteEncoding(resource);
// return to parent when encoding is standard
if (encoding === UTF8 && !addBOM) {
return super.doEncodeText(resource, value);
}
// otherwise create with encoding
const encodedReadable = await this.getEncodedReadable(value || '', encoding, addBOM);
return encodedReadable;
}
async write(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {
// check for overwriteReadonly property (only supported for local file://)
@ -181,20 +124,7 @@ export class NativeTextFileService extends AbstractTextFileService {
}
try {
// check for encoding
const { encoding, addBOM } = await this.encoding.getWriteEncoding(resource, options);
// return to parent when encoding is standard
if (encoding === UTF8 && !addBOM) {
return await super.write(resource, value, options);
}
// otherwise save with encoding
else {
const encodedReadable = await this.getEncodedReadable(value, encoding, addBOM);
return await this.fileService.writeFile(resource, encodedReadable, options);
}
return super.write(resource, value, options);
} catch (error) {
// In case of permission denied, we need to check for readonly
@ -218,11 +148,6 @@ export class NativeTextFileService extends AbstractTextFileService {
}
}
private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): Promise<VSBufferReadable> {
const snapshot = typeof value === 'string' ? stringToSnapshot(value) : value;
return toEncodeReadable(snapshot, encoding, { addBOM });
}
private async writeElevated(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {
// write into a tmp file first

115
src/vs/workbench/services/textfile/test/browser/browserTextFileService.io.test.ts Normal file
View file

@ -0,0 +1,115 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { workbenchInstantiationService, TestInMemoryFileSystemProvider, TestBrowserTextFileServiceWithEncodingOverrides } from 'vs/workbench/test/browser/workbenchTestServices';
import { NullLogService } from 'vs/platform/log/common/log';
import { FileService } from 'vs/platform/files/common/fileService';
import { Schemas } from 'vs/base/common/network';
import { ITextFileService } from 'vs/workbench/services/textfile/common/textfiles';
import { TextFileEditorModelManager } from 'vs/workbench/services/textfile/common/textFileEditorModelManager';
import { DisposableStore } from 'vs/base/common/lifecycle';
import { ServiceCollection } from 'vs/platform/instantiation/common/serviceCollection';
import { IFileService, IStat } from 'vs/platform/files/common/files';
import { URI } from 'vs/base/common/uri';
import { join } from 'vs/base/common/path';
import { UTF16le, detectEncodingByBOMFromBuffer, UTF8_with_bom, UTF16be, toCanonicalName } from 'vs/workbench/services/textfile/common/encoding';
import { VSBuffer } from 'vs/base/common/buffer';
import files from 'vs/workbench/services/textfile/test/browser/fixtures/files';
import createSuite from 'vs/workbench/services/textfile/test/common/textFileService.io.test';
import { isWeb } from 'vs/base/common/platform';
import { IWorkingCopyFileService, WorkingCopyFileService } from 'vs/workbench/services/workingCopy/common/workingCopyFileService';
import { TestWorkingCopyService } from 'vs/workbench/test/common/workbenchTestServices';
import { UriIdentityService } from 'vs/workbench/services/uriIdentity/common/uriIdentityService';
// optimization: we don't need to run this suite in native environment,
// because we have nativeTextFileService.io.test.ts for it,
// so our tests run faster
if (isWeb) {
suite('Files - BrowserTextFileService i/o', function () {
const disposables = new DisposableStore();
let service: ITextFileService;
let fileProvider: TestInMemoryFileSystemProvider;
const testDir = 'test';
createSuite({
setup: async () => {
const instantiationService = workbenchInstantiationService();
const logService = new NullLogService();
const fileService = new FileService(logService);
fileProvider = new TestInMemoryFileSystemProvider();
disposables.add(fileService.registerProvider(Schemas.file, fileProvider));
disposables.add(fileProvider);
const collection = new ServiceCollection();
collection.set(IFileService, fileService);
collection.set(IWorkingCopyFileService, new WorkingCopyFileService(fileService, new TestWorkingCopyService(), instantiationService, new UriIdentityService(fileService)));
service = instantiationService.createChild(collection).createInstance(TestBrowserTextFileServiceWithEncodingOverrides);
await fileProvider.mkdir(URI.file(testDir));
for (let fileName in files) {
await fileProvider.writeFile(
URI.file(join(testDir, fileName)),
files[fileName],
{ create: true, overwrite: false }
);
}
return { service, testDir };
},
teardown: async () => {
(<TextFileEditorModelManager>service.files).dispose();
disposables.clear();
},
exists,
stat,
readFile,
detectEncodingByBOM
});
async function exists(fsPath: string): Promise<boolean> {
try {
await fileProvider.readFile(URI.file(fsPath));
return true;
}
catch (e) {
return false;
}
}
async function readFile(fsPath: string): Promise<VSBuffer>;
async function readFile(fsPath: string, encoding: string): Promise<string>;
async function readFile(fsPath: string, encoding?: string): Promise<VSBuffer | string> {
const file = await fileProvider.readFile(URI.file(fsPath));
if (!encoding) {
return VSBuffer.wrap(file);
}
return new TextDecoder(toCanonicalName(encoding)).decode(file);
}
async function stat(fsPath: string): Promise<IStat> {
return fileProvider.stat(URI.file(fsPath));
}
async function detectEncodingByBOM(fsPath: string): Promise<typeof UTF16be | typeof UTF16le | typeof UTF8_with_bom | null> {
try {
const buffer = await readFile(fsPath);
return detectEncodingByBOMFromBuffer(buffer.slice(0, 3), 3);
} catch (error) {
return null; // ignore errors (like file not found)
}
}
});
}

421
src/vs/workbench/services/textfile/test/browser/fixtures/files.ts Normal file
View file

File diff suppressed because one or more lines are too long

109
src/vs/workbench/services/textfile/test/electron-browser/textFileService.io.test.ts → src/vs/workbench/services/textfile/test/common/textFileService.io.test.ts
View file

@ -4,79 +4,50 @@
*--------------------------------------------------------------------------------------------*/
import * as assert from 'assert';
import { ITextFileService, snapshotToString, TextFileOperationError, TextFileOperationResult, stringToSnapshot } from 'vs/workbench/services/textfile/common/textfiles';
import { URI } from 'vs/base/common/uri';
import { ITextFileService, snapshotToString, TextFileOperationResult, TextFileOperationError, stringToSnapshot } from 'vs/workbench/services/textfile/common/textfiles';
import { IFileService } from 'vs/platform/files/common/files';
import { TextFileEditorModelManager } from 'vs/workbench/services/textfile/common/textFileEditorModelManager';
import { Schemas } from 'vs/base/common/network';
import { ServiceCollection } from 'vs/platform/instantiation/common/serviceCollection';
import { rimraf, RimRafMode, copy, readFile, exists } from 'vs/base/node/pfs';
import { DisposableStore } from 'vs/base/common/lifecycle';
import { FileService } from 'vs/platform/files/common/fileService';
import { NullLogService } from 'vs/platform/log/common/log';
import { getRandomTestPath } from 'vs/base/test/node/testUtils';
import { tmpdir } from 'os';
import { DiskFileSystemProvider } from 'vs/platform/files/node/diskFileSystemProvider';
import { generateUuid } from 'vs/base/common/uuid';
import { join, basename } from 'vs/base/common/path';
import { getPathFromAmdModule } from 'vs/base/common/amd';
import { UTF16be, UTF16le, UTF8_with_bom, UTF8 } from 'vs/workbench/services/textfile/common/encoding';
import { DefaultEndOfLine, ITextSnapshot } from 'vs/editor/common/model';
import { UTF16le, UTF8_with_bom, UTF16be, UTF8 } from 'vs/workbench/services/textfile/common/encoding';
import { VSBuffer } from 'vs/base/common/buffer';
import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
import { ITextSnapshot, DefaultEndOfLine } from 'vs/editor/common/model';
import { isWindows } from 'vs/base/common/platform';
import { readFileSync, statSync } from 'fs';
import { detectEncodingByBOM } from 'vs/workbench/services/textfile/test/node/encoding/encoding.test';
import { workbenchInstantiationService, TestNativeTextFileServiceWithEncodingOverrides } from 'vs/workbench/test/electron-browser/workbenchTestServices';
import { WorkingCopyFileService, IWorkingCopyFileService } from 'vs/workbench/services/workingCopy/common/workingCopyFileService';
import { TestWorkingCopyService } from 'vs/workbench/test/common/workbenchTestServices';
import { UriIdentityService } from 'vs/workbench/services/uriIdentity/common/uriIdentityService';
suite('Files - TextFileService i/o', function () {
const parentDir = getRandomTestPath(tmpdir(), 'vsctests', 'textfileservice');
export interface Params {
setup(): Promise<{
service: ITextFileService,
testDir: string
}>
teardown(): Promise<void>
const disposables = new DisposableStore();
exists(fsPath: string): Promise<boolean>;
stat(fsPath: string): Promise<{ size: number }>;
readFile(fsPath: string): Promise<VSBuffer | Buffer>;
readFile(fsPath: string, encoding: string): Promise<string>;
readFile(fsPath: string, encoding?: string): Promise<VSBuffer | Buffer | string>;
detectEncodingByBOM(fsPath: string): Promise<typeof UTF16be | typeof UTF16le | typeof UTF8_with_bom | null>;
}
/**
* Allows us to reuse test suite across different environments.
*
* It introduces a bit of complexity with setup and teardown, however
* it helps us to ensure that tests are added for all environments at once,
* hence helps us catch bugs better.
*/
export default function createSuite(params: Params) {
let service: ITextFileService;
let testDir: string;
// Given issues such as https://github.com/microsoft/vscode/issues/78602
// and https://github.com/microsoft/vscode/issues/92334 we see random test
// failures when accessing the native file system. To diagnose further, we
// retry node.js file access tests up to 3 times to rule out any random disk
// issue and increase the timeout.
this.retries(3);
this.timeout(1000 * 10);
let testDir = '';
const { exists, stat, readFile, detectEncodingByBOM } = params;
setup(async () => {
const instantiationService = workbenchInstantiationService();
const logService = new NullLogService();
const fileService = new FileService(logService);
const fileProvider = new DiskFileSystemProvider(logService);
disposables.add(fileService.registerProvider(Schemas.file, fileProvider));
disposables.add(fileProvider);
const collection = new ServiceCollection();
collection.set(IFileService, fileService);
collection.set(IWorkingCopyFileService, new WorkingCopyFileService(fileService, new TestWorkingCopyService(), instantiationService, new UriIdentityService(fileService)));
service = instantiationService.createChild(collection).createInstance(TestNativeTextFileServiceWithEncodingOverrides);
const id = generateUuid();
testDir = join(parentDir, id);
const sourceDir = getPathFromAmdModule(require, './fixtures');
await copy(sourceDir, testDir);
const result = await params.setup();
service = result.service;
testDir = result.testDir;
});
teardown(async () => {
(<TextFileEditorModelManager>service.files).dispose();
disposables.clear();
await rimraf(parentDir, RimRafMode.MOVE);
await params.teardown();
});
test('create - no encoding - content empty', async () => {
@ -229,7 +200,7 @@ suite('Files - TextFileService i/o', function () {
});
test('write - use encoding (shiftjis)', async () => {
await testEncodingKeepsData(URI.file(join(testDir, 'some_shiftjs.txt')), 'shiftjis', '中文abc');
await testEncodingKeepsData(URI.file(join(testDir, 'some_shiftjis.txt')), 'shiftjis', '中文abc');
});
test('write - use encoding (gbk)', async () => {
@ -401,8 +372,12 @@ suite('Files - TextFileService i/o', function () {
const result = await service.readStream(resource);
assert.equal(result.name, basename(resource.fsPath));
assert.equal(result.size, statSync(resource.fsPath).size);
assert.equal(snapshotToString(result.value.create(DefaultEndOfLine.LF).createSnapshot(false)), snapshotToString(createTextModel(readFileSync(resource.fsPath).toString()).createSnapshot(false)));
assert.equal(result.size, (await stat(resource.fsPath)).size);
const content = (await readFile(resource.fsPath)).toString();
assert.equal(
snapshotToString(result.value.create(DefaultEndOfLine.LF).createSnapshot(false)),
snapshotToString(createTextModel(content).createSnapshot(false)));
}
test('read - small text', async () => {
@ -421,8 +396,8 @@ suite('Files - TextFileService i/o', function () {
const result = await service.read(resource);
assert.equal(result.name, basename(resource.fsPath));
assert.equal(result.size, statSync(resource.fsPath).size);
assert.equal(result.value, readFileSync(resource.fsPath).toString());
assert.equal(result.size, (await stat(resource.fsPath)).size);
assert.equal(result.value, (await readFile(resource.fsPath)).toString());
}
test('readStream - encoding picked up (CP1252)', async () => {
@ -506,7 +481,7 @@ suite('Files - TextFileService i/o', function () {
await testLargeEncoding('gbk', '中国abc');
});
test('readStream - large ShiftJS', async () => {
test('readStream - large ShiftJIS', async () => {
await testLargeEncoding('shiftjis', '中文abc');
});
@ -588,4 +563,4 @@ suite('Files - TextFileService i/o', function () {
const result = await service.read(URI.file(join(testDir, 'small.txt')), { acceptTextOnly: true });
assert.equal(result.name, 'small.txt');
});
});
}

0
src/vs/workbench/services/textfile/test/electron-browser/fixtures/some_shiftjs.txt → src/vs/workbench/services/textfile/test/electron-browser/fixtures/some_shiftjis.txt
View file

84
src/vs/workbench/services/textfile/test/electron-browser/nativeTextFileService.io.test.ts Normal file
View file

@ -0,0 +1,84 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { ITextFileService } from 'vs/workbench/services/textfile/common/textfiles';
import { IFileService } from 'vs/platform/files/common/files';
import { TextFileEditorModelManager } from 'vs/workbench/services/textfile/common/textFileEditorModelManager';
import { Schemas } from 'vs/base/common/network';
import { ServiceCollection } from 'vs/platform/instantiation/common/serviceCollection';
import { rimraf, RimRafMode, copy, readFile, exists, stat } from 'vs/base/node/pfs';
import { DisposableStore } from 'vs/base/common/lifecycle';
import { FileService } from 'vs/platform/files/common/fileService';
import { NullLogService } from 'vs/platform/log/common/log';
import { getRandomTestPath } from 'vs/base/test/node/testUtils';
import { tmpdir } from 'os';
import { DiskFileSystemProvider } from 'vs/platform/files/node/diskFileSystemProvider';
import { generateUuid } from 'vs/base/common/uuid';
import { join } from 'vs/base/common/path';
import { getPathFromAmdModule } from 'vs/base/common/amd';
import { detectEncodingByBOM } from 'vs/workbench/services/textfile/test/node/encoding/encoding.test';
import { workbenchInstantiationService, TestNativeTextFileServiceWithEncodingOverrides } from 'vs/workbench/test/electron-browser/workbenchTestServices';
import createSuite from 'vs/workbench/services/textfile/test/common/textFileService.io.test';
import { IWorkingCopyFileService, WorkingCopyFileService } from 'vs/workbench/services/workingCopy/common/workingCopyFileService';
import { TestWorkingCopyService } from 'vs/workbench/test/common/workbenchTestServices';
import { UriIdentityService } from 'vs/workbench/services/uriIdentity/common/uriIdentityService';
suite('Files - NativeTextFileService i/o', function () {
const parentDir = getRandomTestPath(tmpdir(), 'vsctests', 'textfileservice');
const disposables = new DisposableStore();
let service: ITextFileService;
let testDir: string;
// Given issues such as https://github.com/microsoft/vscode/issues/78602
// and https://github.com/microsoft/vscode/issues/92334 we see random test
// failures when accessing the native file system. To diagnose further, we
// retry node.js file access tests up to 3 times to rule out any random disk
// issue and increase the timeout.
this.retries(3);
this.timeout(1000 * 10);
createSuite({
setup: async () => {
const instantiationService = workbenchInstantiationService();
const logService = new NullLogService();
const fileService = new FileService(logService);
const fileProvider = new DiskFileSystemProvider(logService);
disposables.add(fileService.registerProvider(Schemas.file, fileProvider));
disposables.add(fileProvider);
const collection = new ServiceCollection();
collection.set(IFileService, fileService);
collection.set(IWorkingCopyFileService, new WorkingCopyFileService(fileService, new TestWorkingCopyService(), instantiationService, new UriIdentityService(fileService)));
service = instantiationService.createChild(collection).createInstance(TestNativeTextFileServiceWithEncodingOverrides);
const id = generateUuid();
testDir = join(parentDir, id);
const sourceDir = getPathFromAmdModule(require, './fixtures');
await copy(sourceDir, testDir);
return { service, testDir };
},
teardown: async () => {
(<TextFileEditorModelManager>service.files).dispose();
disposables.clear();
await rimraf(parentDir, RimRafMode.MOVE);
},
exists,
stat,
readFile,
detectEncodingByBOM
});
});

64
src/vs/workbench/test/browser/workbenchTestServices.ts
View file

@ -23,7 +23,7 @@ import { IUntitledTextEditorService, UntitledTextEditorService } from 'vs/workbe
import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
import { ILifecycleService, BeforeShutdownEvent, ShutdownReason, StartupKind, LifecyclePhase, WillShutdownEvent } from 'vs/platform/lifecycle/common/lifecycle';
import { ServiceCollection } from 'vs/platform/instantiation/common/serviceCollection';
import { FileOperationEvent, IFileService, IFileStat, IResolveFileResult, FileChangesEvent, IResolveFileOptions, ICreateFileOptions, IFileSystemProvider, FileSystemProviderCapabilities, IFileChange, IWatchOptions, IStat, FileType, FileDeleteOptions, FileOverwriteOptions, FileWriteOptions, FileOpenOptions, IFileStatWithMetadata, IResolveMetadataFileOptions, IWriteFileOptions, IReadFileOptions, IFileContent, IFileStreamContent, FileOperationError } from 'vs/platform/files/common/files';
import { FileOperationEvent, IFileService, IFileStat, IResolveFileResult, FileChangesEvent, IResolveFileOptions, ICreateFileOptions, IFileSystemProvider, FileSystemProviderCapabilities, IFileChange, IWatchOptions, IStat, FileType, FileDeleteOptions, FileOverwriteOptions, FileWriteOptions, FileOpenOptions, IFileStatWithMetadata, IResolveMetadataFileOptions, IWriteFileOptions, IReadFileOptions, IFileContent, IFileStreamContent, FileOperationError, IFileSystemProviderWithFileReadStreamCapability } from 'vs/platform/files/common/files';
import { IModelService } from 'vs/editor/common/services/modelService';
import { ModeServiceImpl } from 'vs/editor/common/services/modeServiceImpl';
import { ModelServiceImpl } from 'vs/editor/common/services/modelServiceImpl';
@ -110,6 +110,10 @@ import { IPaneComposite } from 'vs/workbench/common/panecomposite';
import { IUriIdentityService } from 'vs/workbench/services/uriIdentity/common/uriIdentity';
import { UriIdentityService } from 'vs/workbench/services/uriIdentity/common/uriIdentityService';
import { TextFileEditorModelManager } from 'vs/workbench/services/textfile/common/textFileEditorModelManager';
import { InMemoryFileSystemProvider } from 'vs/platform/files/common/inMemoryFilesystemProvider';
import { newWriteableStream, ReadableStreamEvents } from 'vs/base/common/stream';
import { EncodingOracle, IEncodingOverride } from 'vs/workbench/services/textfile/browser/textFileService';
import { UTF16le, UTF16be, UTF8_with_bom } from 'vs/workbench/services/textfile/common/encoding';
export function createFileEditorInput(instantiationService: IInstantiationService, resource: URI): FileEditorInput {
return instantiationService.createInstance(FileEditorInput, resource, undefined, undefined, undefined);
@ -273,6 +277,31 @@ export class TestTextFileService extends BrowserTextFileService {
}
}
export class TestBrowserTextFileServiceWithEncodingOverrides extends BrowserTextFileService {
private _testEncoding: TestEncodingOracle | undefined;
get encoding(): TestEncodingOracle {
if (!this._testEncoding) {
this._testEncoding = this._register(this.instantiationService.createInstance(TestEncodingOracle));
}
return this._testEncoding;
}
}
class TestEncodingOracle extends EncodingOracle {
protected get encodingOverrides(): IEncodingOverride[] {
return [
{ extension: 'utf16le', encoding: UTF16le },
{ extension: 'utf16be', encoding: UTF16be },
{ extension: 'utf8bom', encoding: UTF8_with_bom }
];
}
protected set encodingOverrides(overrides: IEncodingOverride[]) { }
}
class TestEnvironmentServiceWithArgs extends BrowserWorkbenchEnvironmentService {
args = [];
}
@ -946,6 +975,39 @@ export class RemoteFileSystemProvider implements IFileSystemProvider {
private toFileResource(resource: URI): URI { return resource.with({ scheme: Schemas.file, authority: '' }); }
}
export class TestInMemoryFileSystemProvider extends InMemoryFileSystemProvider implements IFileSystemProviderWithFileReadStreamCapability {
readonly capabilities: FileSystemProviderCapabilities =
FileSystemProviderCapabilities.FileReadWrite
| FileSystemProviderCapabilities.PathCaseSensitive
| FileSystemProviderCapabilities.FileReadStream;
readFileStream(resource: URI): ReadableStreamEvents<Uint8Array> {
const BUFFER_SIZE = 64 * 1024;
const stream = newWriteableStream<Uint8Array>(data => VSBuffer.concat(data.map(data => VSBuffer.wrap(data))).buffer);
(async () => {
try {
const data = await this.readFile(resource);
let offset = 0;
while (offset < data.length) {
await timeout(0);
await stream.write(data.subarray(offset, offset + BUFFER_SIZE));
offset += BUFFER_SIZE;
}
await timeout(0);
stream.end();
} catch (error) {
stream.end(error);
}
})();
return stream;
}
}
export const productService: IProductService = { _serviceBrand: undefined, ...product };
export class TestHostService implements IHostService {

6
test/unit/browser/renderer.html
View file

@ -51,10 +51,12 @@
catchError: true,
baseUrl: new URL('../../../src', baseUrl).href,
paths: {
'vs': new URL(`../../../${!!isBuild ? 'out-build' : 'out'}/vs`, baseUrl).href,
vs: new URL(`../../../${!!isBuild ? 'out-build' : 'out'}/vs`, baseUrl).href,
assert: new URL('../assert.js', baseUrl).href,
sinon: new URL('../../../node_modules/sinon/pkg/sinon-1.17.7.js', baseUrl).href,
xterm: new URL('../../../node_modules/xterm/lib/xterm.js', baseUrl).href
xterm: new URL('../../../node_modules/xterm/lib/xterm.js', baseUrl).href,
'iconv-lite-umd': new URL('../../../node_modules/iconv-lite-umd/lib/iconv-lite-umd.js', baseUrl).href,
jschardet: new URL('../../../node_modules/jschardet/dist/jschardet.min.js', baseUrl).href
}
});
</script>