From a6a19ffa648ec7de0b568fdc804cf8e31e24b4a9 Mon Sep 17 00:00:00 2001 From: Benjamin Pasero Date: Wed, 11 Aug 2021 10:35:31 +0200 Subject: [PATCH] Complete the browser local file system provider (#130555) * html fs - naive impls of create * html fs - cleanup handle registration flow * html fs - rewrite stat/readdir * cleanup * basic streaming support * implement stream with position and offset * implement simple rename * fix issues * error handling * empty line * simplify * :lipstick: --- .../files/browser/htmlFileSystemProvider.ts | 484 ++++++++++++------ src/vs/platform/files/common/fileService.ts | 10 +- .../contrib/files/browser/fileImportExport.ts | 6 +- .../dialogs/browser/fileDialogService.ts | 29 +- 4 files changed, 346 insertions(+), 183 deletions(-) diff --git a/src/vs/platform/files/browser/htmlFileSystemProvider.ts b/src/vs/platform/files/browser/htmlFileSystemProvider.ts index 8bff8ad321b..11756214abf 100644 --- a/src/vs/platform/files/browser/htmlFileSystemProvider.ts +++ b/src/vs/platform/files/browser/htmlFileSystemProvider.ts @@ -3,43 +3,36 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Emitter, Event } from 'vs/base/common/event'; +import { VSBuffer } from 'vs/base/common/buffer'; +import { CancellationToken } from 'vs/base/common/cancellation'; +import { Event } from 'vs/base/common/event'; import { Disposable, IDisposable } from 'vs/base/common/lifecycle'; +import { Schemas } from 'vs/base/common/network'; import { isLinux } from 'vs/base/common/platform'; -import { basename, extUri } from 'vs/base/common/resources'; +import { basename, extUri, isEqual } from 'vs/base/common/resources'; +import { newWriteableStream, ReadableStreamEvents } from 'vs/base/common/stream'; import { URI } from 'vs/base/common/uri'; -import { FileDeleteOptions, FileOverwriteOptions, FileSystemProviderCapabilities, FileType, FileWriteOptions, IFileChange, IFileSystemProviderWithFileReadWriteCapability, IStat, IWatchOptions } from 'vs/platform/files/common/files'; +import { generateUuid } from 'vs/base/common/uuid'; +import { createFileSystemProviderError, FileDeleteOptions, FileOverwriteOptions, FileReadStreamOptions, FileSystemProviderCapabilities, FileSystemProviderError, FileSystemProviderErrorCode, FileType, FileWriteOptions, IFileSystemProviderWithFileReadStreamCapability, IFileSystemProviderWithFileReadWriteCapability, IStat, IWatchOptions } from 'vs/platform/files/common/files'; -function split(path: string): [string, string] | undefined { - const match = /^(.*)\/([^/]+)$/.exec(path); +export class HTMLFileSystemProvider implements IFileSystemProviderWithFileReadWriteCapability, IFileSystemProviderWithFileReadStreamCapability { - if (!match) { - return undefined; - } + //#region Events (unsupported) - const [, parentPath, name] = match; - return [parentPath, name]; -} + readonly onDidChangeCapabilities = Event.None; + readonly onDidChangeFile = Event.None; + readonly onDidErrorOccur = Event.None; -function getRootUUID(uri: URI): string | undefined { - const match = /^\/([^/]+)\/[^/]+\/?$/.exec(uri.path); + //#endregion - if (!match) { - return undefined; - } - - return match[1]; -} - -export class HTMLFileSystemProvider implements IFileSystemProviderWithFileReadWriteCapability { - - private readonly files = new Map(); - private readonly directories = new Map(); + //#region File Capabilities private _capabilities: FileSystemProviderCapabilities | undefined; get capabilities(): FileSystemProviderCapabilities { if (!this._capabilities) { - this._capabilities = FileSystemProviderCapabilities.FileReadWrite; + this._capabilities = + FileSystemProviderCapabilities.FileReadWrite | + FileSystemProviderCapabilities.FileReadStream; if (isLinux) { this._capabilities |= FileSystemProviderCapabilities.PathCaseSensitive; @@ -49,49 +42,19 @@ export class HTMLFileSystemProvider implements IFileSystemProviderWithFileReadWr return this._capabilities; } - readonly onDidChangeCapabilities = Event.None; + //#endregion - private readonly _onDidChangeFile = new Emitter(); - readonly onDidChangeFile = this._onDidChangeFile.event; - - private readonly _onDidErrorOccur = new Emitter(); - readonly onDidErrorOccur = this._onDidErrorOccur.event; - - async readFile(resource: URI): Promise { - const handle = await this.getFileHandle(resource); - - if (!handle) { - throw new Error('File not found.'); - } - - const file = await handle.getFile(); - return new Uint8Array(await file.arrayBuffer()); - } - - async writeFile(resource: URI, content: Uint8Array, opts: FileWriteOptions): Promise { - const handle = await this.getFileHandle(resource); - - if (!handle) { - throw new Error('File not found.'); - } - - const writable = await handle.createWritable(); - await writable.write(content); - await writable.close(); - } - - watch(resource: URI, opts: IWatchOptions): IDisposable { - return Disposable.None; - } + //#region File Metadata Resolving async stat(resource: URI): Promise { - const rootUUID = getRootUUID(resource); + try { + const handle = await this.getHandle(resource); + if (!handle) { + throw createFileSystemProviderError(new Error(`No such file or directory, stat '${resource.toString(true)}'`), FileSystemProviderErrorCode.FileNotFound); + } - if (rootUUID) { - const fileHandle = this.files.get(rootUUID); - - if (fileHandle) { - const file = await fileHandle.getFile(); + if (handle.kind === 'file') { + const file = await handle.getFile(); return { type: FileType.File, @@ -101,126 +64,341 @@ export class HTMLFileSystemProvider implements IFileSystemProviderWithFileReadWr }; } - const directoryHandle = this.directories.get(rootUUID); + return { + type: FileType.Directory, + mtime: 0, + ctime: 0, + size: 0 + }; + } catch (error) { + throw this.toFileSystemProviderError(error); + } + } - if (directoryHandle) { - return { - type: FileType.Directory, - mtime: 0, - ctime: 0, - size: 0 - }; + async readdir(resource: URI): Promise<[string, FileType][]> { + try { + const handle = await this.getDirectoryHandle(resource); + if (!handle) { + throw createFileSystemProviderError(new Error(`No such file or directory, readdir '${resource.toString(true)}'`), FileSystemProviderErrorCode.FileNotFound); } + + const result: [string, FileType][] = []; + + for await (const [name, child] of handle) { + result.push([name, child.kind === 'file' ? FileType.File : FileType.Directory]); + } + + return result; + } catch (error) { + throw this.toFileSystemProviderError(error); } + } - const parent = await this.getParentDirectoryHandle(resource); + //#endregion - if (!parent) { - throw new Error('Stat error: no parent found'); + //#region File Reading/Writing + + readFileStream(resource: URI, opts: FileReadStreamOptions, token: CancellationToken): ReadableStreamEvents { + const stream = newWriteableStream(data => VSBuffer.concat(data.map(data => VSBuffer.wrap(data))).buffer, { + // Set a highWaterMark to prevent the stream + // for file upload to produce large buffers + // in-memory + highWaterMark: 10 + }); + + (async () => { + try { + const handle = await this.getFileHandle(resource); + if (!handle) { + throw createFileSystemProviderError(new Error(`No such file or directory, readFile '${resource.toString(true)}'`), FileSystemProviderErrorCode.FileNotFound); + } + + const file = await handle.getFile(); + + // Partial file: implemented simply via `readFile` + if (typeof opts.length === 'number' || typeof opts.position === 'number') { + let buffer = new Uint8Array(await file.arrayBuffer()); + + if (typeof opts?.position === 'number') { + buffer = buffer.slice(opts.position); + } + + if (typeof opts?.length === 'number') { + buffer = buffer.slice(0, opts.length); + } + + stream.end(buffer); + } + + // Entire file + else { + const reader: ReadableStreamDefaultReader = file.stream().getReader(); + + let res = await reader.read(); + while (!res.done) { + if (token.isCancellationRequested) { + break; + } + + // Write buffer into stream but make sure to wait + // in case the `highWaterMark` is reached + await stream.write(res.value); + + if (token.isCancellationRequested) { + break; + } + + res = await reader.read(); + } + stream.end(undefined); + } + } catch (error) { + stream.error(this.toFileSystemProviderError(error)); + stream.end(); + } + })(); + + return stream; + } + + async readFile(resource: URI): Promise { + try { + const handle = await this.getFileHandle(resource); + if (!handle) { + throw createFileSystemProviderError(new Error(`No such file or directory, readFile '${resource.toString(true)}'`), FileSystemProviderErrorCode.FileNotFound); + } + + const file = await handle.getFile(); + + return new Uint8Array(await file.arrayBuffer()); + } catch (error) { + throw this.toFileSystemProviderError(error); } + } - const name = extUri.basename(resource); - for await (const [childName, child] of parent) { - if (childName === name) { - if (child.kind === 'file') { - const file = await child.getFile(); + async writeFile(resource: URI, content: Uint8Array, opts: FileWriteOptions): Promise { + try { + let handle = await this.getFileHandle(resource); - return { - type: FileType.File, - mtime: file.lastModified, - ctime: 0, - size: file.size - }; + // Validate target unless { create: true, overwrite: true } + if (!opts.create || !opts.overwrite) { + if (handle) { + if (!opts.overwrite) { + throw createFileSystemProviderError(new Error(`File already exists, writeFile '${resource.toString(true)}'`), FileSystemProviderErrorCode.FileExists); + } } else { - return { - type: FileType.Directory, - mtime: 0, - ctime: 0, - size: 0 - }; + if (!opts.create) { + throw createFileSystemProviderError(new Error(`No such file, writeFile '${resource.toString(true)}'`), FileSystemProviderErrorCode.FileNotFound); + } + } + } + + // Create target as needed + if (!handle) { + const parent = await this.getParentHandle(resource); + if (!parent) { + throw createFileSystemProviderError(new Error(`No such parent directory, writeFile '${resource.toString(true)}'`), FileSystemProviderErrorCode.FileNotFound); + } + + handle = await parent.getFileHandle(basename(resource), { create: true }); + if (!handle) { + throw createFileSystemProviderError(new Error(`Unable to create file , writeFile '${resource.toString(true)}'`), FileSystemProviderErrorCode.Unknown); + } + } + + // Write to target overwriting any existing contents + const writable = await handle.createWritable(); + await writable.write(content); + await writable.close(); + } catch (error) { + throw this.toFileSystemProviderError(error); + } + } + + //#endregion + + //#region Move/Copy/Delete/Create Folder + + async mkdir(resource: URI): Promise { + try { + const parent = await this.getParentHandle(resource); + if (!parent) { + throw createFileSystemProviderError(new Error(`No such parent directory, mkdir '${resource.toString(true)}'`), FileSystemProviderErrorCode.FileNotFound); + } + + await parent.getDirectoryHandle(basename(resource), { create: true }); + } catch (error) { + throw this.toFileSystemProviderError(error); + } + } + + async delete(resource: URI, opts: FileDeleteOptions): Promise { + try { + const parent = await this.getParentHandle(resource); + if (!parent) { + throw createFileSystemProviderError(new Error(`No such parent directory, delete '${resource.toString(true)}'`), FileSystemProviderErrorCode.FileNotFound); + } + + return parent.removeEntry(basename(resource), { recursive: opts.recursive }); + } catch (error) { + throw this.toFileSystemProviderError(error); + } + } + + async rename(from: URI, to: URI, opts: FileOverwriteOptions): Promise { + try { + if (isEqual(from, to)) { + return; // no-op if the paths are the same + } + + // Implement file rename by write + delete + let fileHandle = await this.getFileHandle(from); + if (fileHandle) { + const file = await fileHandle.getFile(); + const contents = new Uint8Array(await file.arrayBuffer()); + + await this.writeFile(to, contents, { create: true, overwrite: opts.overwrite, unlock: false }); + await this.delete(from, { recursive: false, useTrash: false }); + } + + // File API does not support any real rename otherwise + else { + throw createFileSystemProviderError(new Error(`Rename is unsupported for folders`), FileSystemProviderErrorCode.Unavailable); + } + } catch (error) { + throw this.toFileSystemProviderError(error); + } + } + + //#endregion + + //#region File Watching (unsupported) + + watch(resource: URI, opts: IWatchOptions): IDisposable { + return Disposable.None; + } + + //#endregion + + //#region File/Directoy Handle Registry + + private readonly files = new Map(); + private readonly directories = new Map(); + + registerFileHandle(handle: FileSystemFileHandle): URI { + const handleId = generateUuid(); + this.files.set(handleId, handle); + + return this.toHandleUri(handle, handleId); + } + + registerDirectoryHandle(handle: FileSystemDirectoryHandle): URI { + const handleId = generateUuid(); + this.directories.set(handleId, handle); + + return this.toHandleUri(handle, handleId); + } + + private toHandleUri(handle: FileSystemHandle, handleId: string): URI { + return URI.from({ scheme: Schemas.file, path: `/${handleId}/${handle.name}` }); + } + + private async getHandle(resource: URI): Promise { + let handle: FileSystemHandle | undefined = undefined; + + // First: try to find a well known handle first + const handleId = this.findHandleId(resource); + if (handleId) { + handle = this.files.get(handleId) ?? this.directories.get(handleId); + } + + // Second: walk up parent directories and resolve handle if possible + if (!handle) { + const parent = await this.getParentHandle(resource); + if (parent) { + try { + handle = await parent.getFileHandle(extUri.basename(resource)); + } catch (error) { + try { + handle = await parent.getDirectoryHandle(extUri.basename(resource)); + } catch (error) { + // Ignore + } } } } - throw new Error('Stat error: entry not found'); + return handle; } - mkdir(resource: URI): Promise { - throw new Error('Method not implemented.'); - } - - async readdir(resource: URI): Promise<[string, FileType][]> { - const parent = await this.getDirectoryHandle(resource); - - if (!parent) { - throw new Error('Stat error: no parent found'); + private async getFileHandle(resource: URI): Promise { + const handleId = this.findHandleId(resource); + if (handleId) { + return this.files.get(handleId); } - const result: [string, FileType][] = []; + const parent = await this.getParentHandle(resource); - for await (const [name, child] of parent) { - result.push([name, child.kind === 'file' ? FileType.File : FileType.Directory]); + try { + return await parent?.getFileHandle(extUri.basename(resource)); + } catch (error) { + return undefined; // guard against possible DOMException } - - return result; } - async delete(resource: URI, opts: FileDeleteOptions): Promise { - const parent = await this.getParentDirectoryHandle(resource); - - if (!parent) { - throw new Error('Stat error: no parent found'); - } - - return parent.removeEntry(basename(resource), { recursive: opts.recursive }); - } - - rename(from: URI, to: URI, opts: FileOverwriteOptions): Promise { - throw new Error('Method not implemented: rename'); - } - - private async getDirectoryHandle(uri: URI): Promise { - const rootUUID = getRootUUID(uri); - - if (rootUUID) { - return this.directories.get(rootUUID); - } - - const splitResult = split(uri.path); - - if (!splitResult) { - return undefined; - } - - const parent = await this.getDirectoryHandle(URI.from({ ...uri, path: splitResult[0] })); - return await parent?.getDirectoryHandle(extUri.basename(uri)); - } - - private async getParentDirectoryHandle(uri: URI): Promise { + private async getParentHandle(uri: URI): Promise { return this.getDirectoryHandle(URI.from({ ...uri, path: extUri.dirname(uri).path })); } - private async getFileHandle(uri: URI): Promise { - const rootUUID = getRootUUID(uri); - - if (rootUUID) { - return this.files.get(rootUUID); + private async getDirectoryHandle(uri: URI): Promise { + const handleId = this.findHandleId(uri); + if (handleId) { + return this.directories.get(handleId); } - const parent = await this.getParentDirectoryHandle(uri); - const name = extUri.basename(uri); - return await parent?.getFileHandle(name); + const parentPath = this.findParent(uri.path); + if (!parentPath) { + return undefined; + } + + const parent = await this.getDirectoryHandle(URI.from({ ...uri, path: parentPath })); + + try { + return await parent?.getDirectoryHandle(extUri.basename(uri)); + } catch (error) { + return undefined; // guard against possible DOMException + } } - registerFileHandle(uuid: string, handle: FileSystemFileHandle): void { - this.files.set(uuid, handle); + private findParent(path: string): string | undefined { + const match = /^(.*)\/([^/]+)$/.exec(path); + if (!match) { + return undefined; + } + + const [, parentPath] = match; + return parentPath; } - registerDirectoryHandle(uuid: string, handle: FileSystemDirectoryHandle): void { - this.directories.set(uuid, handle); + private findHandleId(uri: URI): string | undefined { + // Given a path such as `/32b0b72b-ec76-4676-a621-0f8f4fe9a11f/ticino-playground` + // will match on the first path segment value (`32b0b72b-ec76-4676-a621-0f8f4fe9a11f) + // but only if the path component has exactly 2 segments (`//name`) + const match = /^\/([^/]+)\/[^/]+\/?$/.exec(uri.path); + if (!match) { + return undefined; + } + + return match[1]; } - dispose(): void { - this._onDidChangeFile.dispose(); + //#endregion + + private toFileSystemProviderError(error: Error): FileSystemProviderError { + if (error instanceof FileSystemProviderError) { + return error; // avoid double conversion + } + + return createFileSystemProviderError(error, FileSystemProviderErrorCode.Unknown); } } diff --git a/src/vs/platform/files/common/fileService.ts b/src/vs/platform/files/common/fileService.ts index 6faa14ff12c..0013a1eed1e 100644 --- a/src/vs/platform/files/common/fileService.ts +++ b/src/vs/platform/files/common/fileService.ts @@ -418,7 +418,7 @@ export class FileService extends Disposable implements IFileService { // but to the same length. This is a compromise we take to avoid having to produce checksums of // the file content for comparison which would be much slower to compute. if ( - options && typeof options.mtime === 'number' && typeof options.etag === 'string' && options.etag !== ETAG_DISABLED && + typeof options?.mtime === 'number' && typeof options.etag === 'string' && options.etag !== ETAG_DISABLED && typeof stat.mtime === 'number' && typeof stat.size === 'number' && options.mtime < stat.mtime && options.etag !== etag({ mtime: options.mtime /* not using stat.mtime for a reason, see above */, size: stat.size }) ) { @@ -496,7 +496,7 @@ export class FileService extends Disposable implements IFileService { // due to the likelihood of hitting a NOT_MODIFIED_SINCE result. // otherwise, we let it run in parallel to the file reading for // optimal startup performance. - if (options && typeof options.etag === 'string' && options.etag !== ETAG_DISABLED) { + if (typeof options?.etag === 'string' && options.etag !== ETAG_DISABLED) { await statPromise; } @@ -572,12 +572,12 @@ export class FileService extends Disposable implements IFileService { let buffer = await provider.readFile(resource); // respect position option - if (options && typeof options.position === 'number') { + if (typeof options?.position === 'number') { buffer = buffer.slice(options.position); } // respect length option - if (options && typeof options.length === 'number') { + if (typeof options?.length === 'number') { buffer = buffer.slice(0, options.length); } @@ -604,7 +604,7 @@ export class FileService extends Disposable implements IFileService { } // Throw if file not modified since (unless disabled) - if (options && typeof options.etag === 'string' && options.etag !== ETAG_DISABLED && options.etag === stat.etag) { + if (typeof options?.etag === 'string' && options.etag !== ETAG_DISABLED && options.etag === stat.etag) { throw new NotModifiedSinceFileOperationError(localize('fileNotModifiedError', "File not modified since"), stat, options); } diff --git a/src/vs/workbench/contrib/files/browser/fileImportExport.ts b/src/vs/workbench/contrib/files/browser/fileImportExport.ts index bf558d47c9b..23cd0bc1264 100644 --- a/src/vs/workbench/contrib/files/browser/fileImportExport.ts +++ b/src/vs/workbench/contrib/files/browser/fileImportExport.ts @@ -318,16 +318,16 @@ export class BrowserFileUpload { let res = await reader.read(); while (!res.done) { if (token.isCancellationRequested) { - return undefined; + break; } // Write buffer into stream but make sure to wait - // in case the highWaterMark is reached + // in case the `highWaterMark` is reached const buffer = VSBuffer.wrap(res.value); await writeableStream.write(buffer); if (token.isCancellationRequested) { - return undefined; + break; } // Report progress diff --git a/src/vs/workbench/services/dialogs/browser/fileDialogService.ts b/src/vs/workbench/services/dialogs/browser/fileDialogService.ts index c2301ca1d3e..5305a256b3f 100644 --- a/src/vs/workbench/services/dialogs/browser/fileDialogService.ts +++ b/src/vs/workbench/services/dialogs/browser/fileDialogService.ts @@ -10,7 +10,6 @@ import { AbstractFileDialogService } from 'vs/workbench/services/dialogs/browser import { Schemas } from 'vs/base/common/network'; import { memoize } from 'vs/base/common/decorators'; import { HTMLFileSystemProvider } from 'vs/platform/files/browser/htmlFileSystemProvider'; -import { generateUuid } from 'vs/base/common/uuid'; import { localize } from 'vs/nls'; import { getMediaOrTextMime } from 'vs/base/common/mime'; @@ -47,10 +46,8 @@ export class FileDialogService extends AbstractFileDialogService implements IFil } const [handle] = await window.showOpenFilePicker({ multiple: false }); - const uuid = generateUuid(); - const uri = URI.from({ scheme: Schemas.file, path: `/${uuid}/${handle.name}` }); - this.fileSystemProvider.registerFileHandle(uuid, handle); + const uri = this.fileSystemProvider.registerFileHandle(handle); await this.openerService.open(uri, { fromUserGesture: true, editorOptions: { pinned: true } }); } @@ -93,12 +90,8 @@ export class FileDialogService extends AbstractFileDialogService implements IFil } const handle = await window.showSaveFilePicker({ types: this.getFilePickerTypes(options.filters) }); - const uuid = generateUuid(); - const uri = URI.from({ scheme: Schemas.file, path: `/${uuid}/${handle.name}` }); - this.fileSystemProvider.registerFileHandle(uuid, handle); - - return uri; + return this.fileSystemProvider.registerFileHandle(handle); } private getFilePickerTypes(filters?: FileFilter[]): FilePickerAcceptType[] | undefined { @@ -123,12 +116,8 @@ export class FileDialogService extends AbstractFileDialogService implements IFil } const handle = await window.showSaveFilePicker({ types: this.getFilePickerTypes(options.filters) }); - const uuid = generateUuid(); - const uri = URI.from({ scheme: Schemas.file, path: `/${uuid}/${handle.name}` }); - this.fileSystemProvider.registerFileHandle(uuid, handle); - - return uri; + return this.fileSystemProvider.registerFileHandle(handle); } async showOpenDialog(options: IOpenDialogOptions): Promise { @@ -138,21 +127,17 @@ export class FileDialogService extends AbstractFileDialogService implements IFil return this.showOpenDialogSimplified(schema, options); } - let handleName: string | undefined; - const uuid = generateUuid(); + let uri: URI | undefined; if (options.canSelectFiles) { const handle = await window.showOpenFilePicker({ multiple: false, types: this.getFilePickerTypes(options.filters) }); if (handle.length === 1) { - handleName = handle[0].name; - this.fileSystemProvider.registerFileHandle(uuid, handle[0]); + uri = this.fileSystemProvider.registerFileHandle(handle[0]); } } else { const handle = await window.showDirectoryPicker(); - handleName = handle.name; - this.fileSystemProvider.registerDirectoryHandle(uuid, handle); + uri = this.fileSystemProvider.registerDirectoryHandle(handle); } - if (handleName) { - const uri = URI.from({ scheme: Schemas.file, path: `/${uuid}/${handleName}` }); + if (uri) { return [uri]; } return undefined;