Handle non-responsive pty host process

Fixes #116948

src/vs/platform/terminal/common/terminal.ts

@@ -19,7 +19,11 @@ export enum TerminalIpcChannels {
 	/**
 	 * Deals with logging from the pty host process.
 	 */
-	Log = 'log'
+	Log = 'log',
+	/**
+	 * Enables the detection of unresponsive pty hosts.
+	 */
+	Heartbeat = 'heartbeat'
 }
 
 export interface IPtyService {
@@ -27,6 +31,7 @@ export interface IPtyService {
 
 	readonly onPtyHostExit?: Event<number>;
 	readonly onPtyHostStart?: Event<void>;
+	readonly onPtyHostUnresponsive?: Event<void>;
 
 	readonly onProcessData: Event<{ id: number, event: IProcessDataEvent | string }>;
 	readonly onProcessExit: Event<{ id: number, event: number | undefined }>;
@@ -62,6 +67,36 @@ export interface IPtyService {
 	getCwd(id: number): Promise<string>;
 
 	getLatency(id: number): Promise<number>;
+
+	restartPtyHost?(): Promise<void>;
+}
+
+export enum HeartbeatConstants {
+	/**
+	 * The duration between heartbeats
+	 */
+	BeatInterval = 10000,
+	/**
+	 * Defines a multiplier for BeatInterval for how long to wait before starting the second wait
+	 * timer.
+	 */
+	FirstWaitMultiplier = 1.2,
+	/**
+	 * Defines a multiplier for BeatInterval for how long to wait before telling the user about
+	 * non-responsiveness. The second timer is to avoid informing the user incorrectly when waking
+	 * the computer up from sleep
+	 */
+	SecondWaitMultiplier = 1,
+	/**
+	 * How long to wait before telling the user about non-responsiveness when they try to create a
+	 * process. This short circuits the standard wait timeouts to tell the user sooner and only
+	 * create process is handled to avoid additional perf overhead.
+	 */
+	CreateProcessTimeout = 3000
+}
+
+export interface IHeartbeatService {
+	readonly onBeat: Event<void>;
 }
 
 export interface IShellLaunchConfig {

src/vs/platform/terminal/electron-browser/localPtyService.ts

@@ -5,7 +5,7 @@
 
 import { Disposable } from 'vs/base/common/lifecycle';
 import { ILogService } from 'vs/platform/log/common/log';
-import { IPtyService, IProcessDataEvent, IShellLaunchConfig, ITerminalDimensionsOverride, ITerminalLaunchError, TerminalIpcChannels } from 'vs/platform/terminal/common/terminal';
+import { IPtyService, IProcessDataEvent, IShellLaunchConfig, ITerminalDimensionsOverride, ITerminalLaunchError, TerminalIpcChannels, IHeartbeatService, HeartbeatConstants } from 'vs/platform/terminal/common/terminal';
 import { Client } from 'vs/base/parts/ipc/node/ipc.cp';
 import { FileAccess } from 'vs/base/common/network';
 import { ProxyChannel } from 'vs/base/parts/ipc/common/ipc';
@@ -20,16 +20,22 @@ enum Constants {
 export class LocalPtyService extends Disposable implements IPtyService {
 	declare readonly _serviceBrand: undefined;
 
+	private _client: Client;
 	// ProxyChannel is not used here because events get lost when forwarding across multiple proxies
 	private _proxy: IPtyService;
 
 	private _restartCount = 0;
 	private _isDisposed = false;
 
+	private _heartbeatFirstTimeout?: NodeJS.Timeout;
+	private _heartbeatSecondTimeout?: NodeJS.Timeout;
+
 	private readonly _onPtyHostExit = this._register(new Emitter<number>());
 	readonly onPtyHostExit = this._onPtyHostExit.event;
 	private readonly _onPtyHostStart = this._register(new Emitter<void>());
 	readonly onPtyHostStart = this._onPtyHostStart.event;
+	private readonly _onPtyHostUnresponsive = this._register(new Emitter<void>());
+	readonly onPtyHostUnresponsive = this._onPtyHostUnresponsive.event;
 
 	private readonly _onProcessData = this._register(new Emitter<{ id: number, event: IProcessDataEvent | string }>());
 	readonly onProcessData = this._onProcessData.event;
@@ -49,10 +55,10 @@ export class LocalPtyService extends Disposable implements IPtyService {
 	) {
 		super();
 
-		this._proxy = this._startPtyHost();
+		[this._client, this._proxy] = this._startPtyHost();
 	}
 
-	private _startPtyHost(): IPtyService {
+	private _startPtyHost(): [Client, IPtyService] {
 		const client = this._register(new Client(
 			FileAccess.asFileUri('bootstrap-fork', require).fsPath,
 			{
@@ -67,13 +73,13 @@ export class LocalPtyService extends Disposable implements IPtyService {
 		));
 		this._onPtyHostStart.fire();
 
+		const heartbeatService = ProxyChannel.toService<IHeartbeatService>(client.getChannel(TerminalIpcChannels.Heartbeat));
+		heartbeatService.onBeat(() => this._handleHeartbeat());
+
 		// Handle exit
 		this._register({
 			dispose: () => {
-				if (proxy.shutdownAll) {
-					proxy.shutdownAll();
-				}
-				client.dispose();
+				this._disposePtyHost();
 			}
 		});
 		this._register(client.onDidProcessExit(e => {
@@ -82,7 +88,7 @@ export class LocalPtyService extends Disposable implements IPtyService {
 				if (this._restartCount <= Constants.MaxRestarts) {
 					this._logService.error(`ptyHost terminated unexpectedly with code ${e.code}`);
 					this._restartCount++;
-					this._proxy = this._startPtyHost();
+					this.restartPtyHost();
 				} else {
 					this._logService.error(`ptyHost terminated unexpectedly with code ${e.code}, giving up`);
 				}
@@ -103,7 +109,7 @@ export class LocalPtyService extends Disposable implements IPtyService {
 		this._register(proxy.onProcessTitleChanged(e => this._onProcessTitleChanged.fire(e)));
 		this._register(proxy.onProcessOverrideDimensions(e => this._onProcessOverrideDimensions.fire(e)));
 		this._register(proxy.onProcessResolvedShellLaunchConfig(e => this._onProcessResolvedShellLaunchConfig.fire(e)));
-		return proxy;
+		return [client, proxy];
 	}
 
 	dispose() {
@@ -111,8 +117,11 @@ export class LocalPtyService extends Disposable implements IPtyService {
 		super.dispose();
 	}
 
-	createProcess(shellLaunchConfig: IShellLaunchConfig, cwd: string, cols: number, rows: number, env: IProcessEnvironment, executableEnv: IProcessEnvironment, windowsEnableConpty: boolean): Promise<number> {
-		return this._proxy.createProcess(shellLaunchConfig, cwd, cols, rows, env, executableEnv, windowsEnableConpty);
+	async createProcess(shellLaunchConfig: IShellLaunchConfig, cwd: string, cols: number, rows: number, env: IProcessEnvironment, executableEnv: IProcessEnvironment, windowsEnableConpty: boolean): Promise<number> {
+		const timeout = setTimeout(() => this._handleUnresponsiveCreateProcess(), HeartbeatConstants.CreateProcessTimeout);
+		const result = await this._proxy.createProcess(shellLaunchConfig, cwd, cols, rows, env, executableEnv, windowsEnableConpty);
+		clearTimeout(timeout);
+		return result;
 	}
 	start(id: number): Promise<ITerminalLaunchError | { remoteTerminalId: number; } | undefined> {
 		return this._proxy.start(id);
@@ -138,4 +147,51 @@ export class LocalPtyService extends Disposable implements IPtyService {
 	getLatency(id: number): Promise<number> {
 		return this._proxy.getLatency(id);
 	}
+
+	async restartPtyHost(): Promise<void> {
+		// TODO: Mark connection lost for all existing terminals on restart
+		this._disposePtyHost();
+		[this._client, this._proxy] = this._startPtyHost();
+	}
+
+	private _disposePtyHost(): void {
+		if (this._proxy.shutdownAll) {
+			this._proxy.shutdownAll();
+		}
+		this._client.dispose();
+	}
+
+	private _handleHeartbeat() {
+		this._clearHeartbeatTimeouts();
+		this._heartbeatFirstTimeout = setTimeout(() => this._handleHeartbeatFirstTimeout(), HeartbeatConstants.BeatInterval * HeartbeatConstants.FirstWaitMultiplier);
+	}
+
+	private _handleHeartbeatFirstTimeout() {
+		this._logService.warn(`No ptyHost heartbeat after ${HeartbeatConstants.BeatInterval * HeartbeatConstants.FirstWaitMultiplier}ms`);
+		this._heartbeatFirstTimeout = undefined;
+		this._heartbeatSecondTimeout = setTimeout(() => this._handleHeartbeatSecondTimeout(), HeartbeatConstants.BeatInterval * HeartbeatConstants.SecondWaitMultiplier);
+	}
+
+	private _handleHeartbeatSecondTimeout() {
+		this._logService.error(`No ptyHost heartbeat after ${HeartbeatConstants.BeatInterval * HeartbeatConstants.FirstWaitMultiplier}ms!`);
+		this._heartbeatSecondTimeout = undefined;
+		this._onPtyHostUnresponsive.fire();
+	}
+
+	private _handleUnresponsiveCreateProcess() {
+		this._clearHeartbeatTimeouts();
+		this._logService.error(`No ptyHost response to createProcess after ${HeartbeatConstants.CreateProcessTimeout}ms`);
+		this._onPtyHostUnresponsive.fire();
+	}
+
+	private _clearHeartbeatTimeouts() {
+		if (this._heartbeatFirstTimeout) {
+			clearTimeout(this._heartbeatFirstTimeout);
+			this._heartbeatFirstTimeout = undefined;
+		}
+		if (this._heartbeatSecondTimeout) {
+			clearTimeout(this._heartbeatSecondTimeout);
+			this._heartbeatSecondTimeout = undefined;
+		}
+	}
 }

src/vs/platform/terminal/node/heartbeatService.ts

@@ -0,0 +1,22 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Emitter } from 'vs/base/common/event';
+import { Disposable, toDisposable } from 'vs/base/common/lifecycle';
+import { HeartbeatConstants, IHeartbeatService } from 'vs/platform/terminal/common/terminal';
+
+export class HeartbeatService extends Disposable implements IHeartbeatService {
+	private readonly _onBeat = this._register(new Emitter<void>());
+	readonly onBeat = this._onBeat.event;
+
+	constructor() {
+		super();
+
+		const interval = setInterval(() => {
+			this._onBeat.fire();
+		}, HeartbeatConstants.BeatInterval);
+		this._register(toDisposable(() => clearInterval(interval)));
+	}
+}

src/vs/platform/terminal/node/ptyHostMain.ts

@@ -9,6 +9,7 @@ import { PtyService } from 'vs/platform/terminal/node/ptyService';
 import { TerminalIpcChannels } from 'vs/platform/terminal/common/terminal';
 import { ConsoleLogger, LogService } from 'vs/platform/log/common/log';
 import { LogLevelChannel } from 'vs/platform/log/common/logIpc';
+import { HeartbeatService } from 'vs/platform/terminal/node/heartbeatService';
 
 const server = new Server('ptyHost');
 
@@ -16,10 +17,14 @@ const logService = new LogService(new ConsoleLogger());
 const logChannel = new LogLevelChannel(logService);
 server.registerChannel(TerminalIpcChannels.Log, logChannel);
 
-const service = new PtyService(logService);
-server.registerChannel(TerminalIpcChannels.PtyHost, ProxyChannel.fromService(service));
+const heartbeatService = new HeartbeatService();
+server.registerChannel(TerminalIpcChannels.Heartbeat, ProxyChannel.fromService(heartbeatService));
+
+const ptyService = new PtyService(logService);
+server.registerChannel(TerminalIpcChannels.PtyHost, ProxyChannel.fromService(ptyService));
 
 process.once('exit', () => {
 	logService.dispose();
-	service.dispose();
+	heartbeatService.dispose();
+	ptyService.dispose();
 });

src/vs/platform/terminal/node/ptyService.ts

@@ -17,6 +17,9 @@ export class PtyService extends Disposable implements IPtyService {
 
 	private readonly _ptys: Map<number, ITerminalChildProcess> = new Map();
 
+	private readonly _onHeartbeat = this._register(new Emitter<void>());
+	readonly onHeartbeat = this._onHeartbeat.event;
+
 	private readonly _onProcessData = this._register(new Emitter<{ id: number, event: IProcessDataEvent | string }>());
 	readonly onProcessData = this._onProcessData.event;
 	private readonly _onProcessExit = this._register(new Emitter<{ id: number, event: number | undefined }>());

src/vs/workbench/contrib/terminal/electron-browser/terminalInstanceService.ts

@@ -26,7 +26,8 @@ import { IShellLaunchConfig, ITerminalChildProcess } from 'vs/platform/terminal/
 import { LocalPty } from 'vs/workbench/contrib/terminal/electron-sandbox/localPty';
 import { Emitter } from 'vs/base/common/event';
 import { Disposable } from 'vs/base/common/lifecycle';
-import { INotificationService } from 'vs/platform/notification/common/notification';
+import { INotificationService, IPromptChoice, Severity } from 'vs/platform/notification/common/notification';
+import { localize } from 'vs/nls';
 
 let Terminal: typeof XTermTerminal;
 let SearchAddon: typeof XTermSearchAddon;
@@ -62,6 +63,15 @@ export class TerminalInstanceService extends Disposable implements ITerminalInst
 				this._logService.info(`ptyHost restarted`);
 			});
 		}
+		if (this._localPtyService.onPtyHostUnresponsive) {
+			this._localPtyService.onPtyHostUnresponsive(() => {
+				const choices: IPromptChoice[] = [{
+					label: localize('restartPtyHost', "Restart pty host"),
+					run: () => this._localPtyService.restartPtyHost!()
+				}];
+				notificationService.prompt(Severity.Error, localize('nonResponsivePtyHost', "The connection to the terminal's pty host process is unresponsive, the terminals may stop working."), choices);
+			});
+		}
 	}
 
 	public async getXtermConstructor(): Promise<typeof XTermTerminal> {