[json] when downloading JSON Schema files, do Conditional GETs. Fixes #101050

extensions/json-language-features/client/src/browser/jsonClientMain.ts

@@ -5,9 +5,8 @@
 
 import { ExtensionContext, Uri } from 'vscode';
 import { LanguageClientOptions } from 'vscode-languageclient';
-import { startClient, LanguageClientConstructor } from '../jsonClient';
+import { startClient, LanguageClientConstructor, SchemaRequestService } from '../jsonClient';
 import { LanguageClient } from 'vscode-languageclient/browser';
-import { RequestService } from '../requests';
 
 declare const Worker: {
 	new(stringUrl: string): any;
@@ -24,7 +23,7 @@ export function activate(context: ExtensionContext) {
 			return new LanguageClient(id, name, clientOptions, worker);
 		};
 
-		const http: RequestService = {
+		const schemaRequests: SchemaRequestService = {
 			getContent(uri: string) {
 				return fetch(uri, { mode: 'cors' })
 					.then(function (response: any) {
@@ -32,7 +31,8 @@ export function activate(context: ExtensionContext) {
 					});
 			}
 		};
-		startClient(context, newLanguageClient, { http });
+
+		startClient(context, newLanguageClient, { schemaRequests });
 
 	} catch (e) {
 		console.log(e);

extensions/json-language-features/client/src/jsonClient.ts

@@ -20,7 +20,7 @@ import {
 } from 'vscode-languageclient';
 
 import { hash } from './utils/hash';
-import { RequestService, joinPath } from './requests';
+import { joinPath } from './requests';
 import { createLanguageStatusItem } from './languageStatus';
 
 namespace VSCodeContentRequest {
@@ -96,10 +96,16 @@ export interface TelemetryReporter {
 export type LanguageClientConstructor = (name: string, description: string, clientOptions: LanguageClientOptions) => CommonLanguageClient;
 
 export interface Runtime {
-	http: RequestService;
+	schemaRequests: SchemaRequestService;
 	telemetry?: TelemetryReporter
 }
 
+export interface SchemaRequestService {
+	getContent(uri: string): Promise<string>;
+}
+
+export const languageServerDescription = localize('jsonserver.name', 'JSON Language Server');
+
 export function startClient(context: ExtensionContext, newLanguageClient: LanguageClientConstructor, runtime: Runtime) {
 
 	const toDispose = context.subscriptions;
@@ -198,7 +204,7 @@ export function startClient(context: ExtensionContext, newLanguageClient: Langua
 	};
 
 	// Create the language client and start the client.
-	const client = newLanguageClient('json', localize('jsonserver.name', 'JSON Language Server'), clientOptions);
+	const client = newLanguageClient('json', languageServerDescription, clientOptions);
 	client.registerProposedFeatures();
 
 	const disposable = client.start();
@@ -228,7 +234,7 @@ export function startClient(context: ExtensionContext, newLanguageClient: Langua
 					 */
 					runtime.telemetry.sendTelemetryEvent('json.schema', { schemaURL: uriPath });
 				}
-				return runtime.http.getContent(uriPath).catch(e => {
+				return runtime.schemaRequests.getContent(uriPath).catch(e => {
 					return Promise.reject(new ResponseError(4, e.toString()));
 				});
 			} else {

extensions/json-language-features/client/src/node/jsonClientMain.ts

@@ -3,24 +3,26 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { ExtensionContext } from 'vscode';
-import { startClient, LanguageClientConstructor } from '../jsonClient';
+import { ExtensionContext, OutputChannel, window, workspace } from 'vscode';
+import { startClient, LanguageClientConstructor, SchemaRequestService, languageServerDescription } from '../jsonClient';
 import { ServerOptions, TransportKind, LanguageClientOptions, LanguageClient } from 'vscode-languageclient/node';
 
-import * as fs from 'fs';
-import { xhr, XHRResponse, getErrorStatusDescription } from 'request-light';
+import { promises as fs } from 'fs';
+import * as path from 'path';
+import { xhr, XHRResponse, getErrorStatusDescription, Headers } from 'request-light';
 
 import TelemetryReporter from 'vscode-extension-telemetry';
-import { RequestService } from '../requests';
+import { JSONSchemaCache } from './schemaCache';
 
 let telemetry: TelemetryReporter | undefined;
 
 // this method is called when vs code is activated
-export function activate(context: ExtensionContext) {
-
-	const clientPackageJSON = getPackageInfo(context);
+export async function activate(context: ExtensionContext) {
+	const clientPackageJSON = await getPackageInfo(context);
 	telemetry = new TelemetryReporter(clientPackageJSON.name, clientPackageJSON.version, clientPackageJSON.aiKey);
 
+	const outputChannel = window.createOutputChannel(languageServerDescription);
+
 	const serverMain = `./server/${clientPackageJSON.main.indexOf('/dist/') !== -1 ? 'dist' : 'out'}/node/jsonServerMain`;
 	const serverModule = context.asAbsolutePath(serverMain);
 
@@ -35,10 +37,15 @@ export function activate(context: ExtensionContext) {
 	};
 
 	const newLanguageClient: LanguageClientConstructor = (id: string, name: string, clientOptions: LanguageClientOptions) => {
+		clientOptions.outputChannel = outputChannel;
 		return new LanguageClient(id, name, serverOptions, clientOptions);
 	};
+	const log = getLog(outputChannel);
+	context.subscriptions.push(log);
 
-	startClient(context, newLanguageClient, { http: getHTTPRequestService(), telemetry });
+	const schemaRequests = await getSchemaRequestService(context, log);
+
+	startClient(context, newLanguageClient, { schemaRequests, telemetry });
 }
 
 export function deactivate(): Promise<any> {
@@ -52,23 +59,88 @@ interface IPackageInfo {
 	main: string;
 }
 
-function getPackageInfo(context: ExtensionContext): IPackageInfo {
+async function getPackageInfo(context: ExtensionContext): Promise<IPackageInfo> {
 	const location = context.asAbsolutePath('./package.json');
 	try {
-		return JSON.parse(fs.readFileSync(location).toString());
+		return JSON.parse((await fs.readFile(location)).toString());
 	} catch (e) {
 		console.log(`Problems reading ${location}: ${e}`);
 		return { name: '', version: '', aiKey: '', main: '' };
 	}
 }
 
-function getHTTPRequestService(): RequestService {
+interface Log {
+	trace(message: string): void;
+	dispose(): void;
+}
+
+const traceSetting = 'json.trace.server';
+function getLog(outputChannel: OutputChannel): Log {
+	let trace = workspace.getConfiguration().get(traceSetting) === 'verbose';
+	const configListener = workspace.onDidChangeConfiguration(e => {
+		if (e.affectsConfiguration(traceSetting)) {
+			trace = workspace.getConfiguration().get(traceSetting) === 'verbose';
+		}
+	});
 	return {
-		getContent(uri: string, _encoding?: string): Promise<string> {
-			const headers = { 'Accept-Encoding': 'gzip, deflate' };
-			return xhr({ url: uri, followRedirects: 5, headers }).then(response => {
-				return response.responseText;
-			}, (error: XHRResponse) => {
+		trace(message: string) {
+			if (trace) {
+				outputChannel.appendLine(message);
+			}
+		},
+		dispose: () => configListener.dispose()
+	};
+}
+
+const retryTimeoutInDays = 2; // 2 days
+const retryTimeoutInMs = retryTimeoutInDays * 24 * 60 * 60 * 1000;
+
+async function getSchemaRequestService(context: ExtensionContext, log: Log): Promise<SchemaRequestService> {
+	let cache: JSONSchemaCache | undefined = undefined;
+	const globalStorage = context.globalStorageUri;
+	if (globalStorage.scheme === 'file') {
+		const schemaCacheLocation = path.join(globalStorage.fsPath, 'json-schema-cache');
+		await fs.mkdir(schemaCacheLocation, { recursive: true });
+
+		cache = new JSONSchemaCache(schemaCacheLocation, context.globalState);
+		log.trace(`[json schema cache] initial state: ${JSON.stringify(cache.getCacheInfo(), null, ' ')}`);
+	}
+
+	const isXHRResponse = (error: any): error is XHRResponse => typeof error?.status === 'number';
+
+	const request = async (uri: string, etag?: string): Promise<string> => {
+		const headers: Headers = { 'Accept-Encoding': 'gzip, deflate' };
+		if (etag) {
+			headers['If-None-Match'] = etag;
+		}
+		try {
+			log.trace(`[json schema cache] Requesting schema ${uri} etag ${etag}...`);
+
+			const response = await xhr({ url: uri, followRedirects: 5, headers });
+			if (cache) {
+				const etag = response.headers['etag'];
+				if (typeof etag === 'string') {
+					log.trace(`[json schema cache] Storing schema ${uri} etag ${etag} in cache`);
+					await cache.putSchema(uri, etag, response.responseText);
+				} else {
+					log.trace(`[json schema cache] Response: schema ${uri} no etag`);
+				}
+			}
+			return response.responseText;
+		} catch (error: unknown) {
+			if (isXHRResponse(error)) {
+				if (error.status === 304 && etag && cache) {
+
+					log.trace(`[json schema cache] Response: schema ${uri} unchanged etag ${etag}`);
+
+					const content = await cache.getSchema(uri, etag);
+					if (content) {
+						log.trace(`[json schema cache] Get schema ${uri} etag ${etag} from cache`);
+						return content;
+					}
+					return request(uri);
+				}
+
 				let status = getErrorStatusDescription(error.status);
 				if (status && error.responseText) {
 					status = `${status}\n${error.responseText.substring(0, 200)}`;
@@ -76,8 +148,24 @@ function getHTTPRequestService(): RequestService {
 				if (!status) {
 					status = error.toString();
 				}
-				return Promise.reject(status);
-			});
+				log.trace(`[json schema cache] Respond schema ${uri} error ${status}`);
+
+				throw status;
+			}
+			throw error;
+		}
+	};
+
+	return {
+		getContent: async (uri: string) => {
+			if (cache && /^https?:\/\/json\.schemastore\.org\//.test(uri)) {
+				const content = await cache.getSchemaIfAccessedSince(uri, retryTimeoutInMs);
+				if (content) {
+					log.trace(`[json schema cache] Schema ${uri} from cache without request (last accessed less than ${retryTimeoutInDays} days ago)`);
+					return content;
+				}
+			}
+			return request(uri, cache?.getETag(uri));
 		}
 	};
 }

extensions/json-language-features/client/src/node/schemaCache.ts

@@ -0,0 +1,108 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { promises as fs } from 'fs';
+import * as path from 'path';
+import { createHash } from 'crypto';
+import { Memento } from 'vscode';
+
+interface CacheEntry {
+	etag: string;
+	fileName: string;
+	accessTime: number;
+}
+
+interface CacheInfo {
+	[schemaUri: string]: CacheEntry;
+}
+
+const MEMENTO_KEY = 'json-schema-cache';
+
+export class JSONSchemaCache {
+	private readonly cacheInfo: CacheInfo;
+
+	constructor(private readonly schemaCacheLocation: string, private readonly globalState: Memento) {
+		this.cacheInfo = globalState.get<CacheInfo>(MEMENTO_KEY, {});
+	}
+
+	getETag(schemaUri: string): string | undefined {
+		return this.cacheInfo[schemaUri]?.etag;
+	}
+
+	async putSchema(schemaUri: string, etag: string, schemaContent: string): Promise<void> {
+		try {
+			const fileName = getCacheFileName(schemaUri);
+			await fs.writeFile(path.join(this.schemaCacheLocation, fileName), schemaContent);
+			const entry: CacheEntry = { etag, fileName, accessTime: new Date().getTime() };
+			this.cacheInfo[schemaUri] = entry;
+		} catch (e) {
+			delete this.cacheInfo[schemaUri];
+		} finally {
+			await this.updateMemento();
+		}
+	}
+
+	async getSchemaIfAccessedSince(schemaUri: string, expirationDuration: number): Promise<string | undefined> {
+		const cacheEntry = this.cacheInfo[schemaUri];
+		if (cacheEntry && cacheEntry.accessTime + expirationDuration >= new Date().getTime()) {
+			return this.loadSchemaFile(schemaUri, cacheEntry);
+		}
+		return undefined;
+	}
+
+	async getSchema(schemaUri: string, etag: string): Promise<string | undefined> {
+		const cacheEntry = this.cacheInfo[schemaUri];
+		if (cacheEntry) {
+			if (cacheEntry.etag === etag) {
+				return this.loadSchemaFile(schemaUri, cacheEntry);
+			} else {
+				this.deleteSchemaFile(schemaUri, cacheEntry);
+			}
+		}
+		return undefined;
+	}
+
+	private async loadSchemaFile(schemaUri: string, cacheEntry: CacheEntry): Promise<string | undefined> {
+		const cacheLocation = path.join(this.schemaCacheLocation, cacheEntry.fileName);
+		try {
+			const content = (await fs.readFile(cacheLocation)).toString();
+			cacheEntry.accessTime = new Date().getTime();
+			return content;
+		} catch (e) {
+			delete this.cacheInfo[schemaUri];
+			return undefined;
+		} finally {
+			await this.updateMemento();
+		}
+	}
+
+	private async deleteSchemaFile(schemaUri: string, cacheEntry: CacheEntry): Promise<void> {
+		const cacheLocation = path.join(this.schemaCacheLocation, cacheEntry.fileName);
+		delete this.cacheInfo[schemaUri];
+		await this.updateMemento();
+		try {
+			await fs.rm(cacheLocation);
+		} catch (e) {
+			// ignore
+		}
+	}
+
+
+	// for debugging
+	public getCacheInfo() {
+		return this.cacheInfo;
+	}
+
+	private async updateMemento() {
+		try {
+			await this.globalState.update(MEMENTO_KEY, this.cacheInfo);
+		} catch (e) {
+			// ignore
+		}
+	}
+}
+function getCacheFileName(uri: string): string {
+	return `${createHash('MD5').update(uri).digest('hex')}.schema.json`;
+}

extensions/json-language-features/client/src/requests.ts

@@ -5,10 +5,6 @@
 
 import { Uri } from 'vscode';
 
-export interface RequestService {
-	getContent(uri: string, encoding?: string): Promise<string>;
-}
-
 export function getScheme(uri: string) {
 	return uri.substr(0, uri.indexOf(':'));
 }