From e996fadbb62b47f75ffa78575968a8e9e038418e Mon Sep 17 00:00:00 2001
From: "Dustin L. Howett" <duhowett@microsoft.com>
Date: Thu, 15 Oct 2020 11:50:27 -0700
Subject: [PATCH] conpty: fall back to conhost if OpenConsole is missing
 (#7741)

This commit is in support of WTU.

I initially added support for a new flag, `PSEUDOCONSOLE_UNDOCKED_PREFER_INBOX_CONHOST`,
which I liked because it was more explicit. We chose not to go that route.

### Automatic fallback
#### Pros
* It's easier on the consumer
* We can eventually expand it to support `$ARCH/openconsole.exe`
#### Cons
* Packaging the project wrong will result in a working-but-somewhat-broken experience (old conhost)
   * We ameliorated this by checking it in the packaging script.
* Implicit behavior may be bad
---
 build/scripts/Test-WindowsTerminalPackage.ps1 |  4 ++++
 src/winconpty/winconpty.cpp                   | 21 ++++++++++++++-----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/build/scripts/Test-WindowsTerminalPackage.ps1 b/build/scripts/Test-WindowsTerminalPackage.ps1
index 88cb14bbf..2f2b5cbc3 100644
--- a/build/scripts/Test-WindowsTerminalPackage.ps1
+++ b/build/scripts/Test-WindowsTerminalPackage.ps1
@@ -106,6 +106,10 @@ Try {
         Throw "Failed to find wt.exe/wtd.exe -- check the WAP packaging project"
     }
 
+    If ($null -eq (Get-Item "$AppxPackageRootPath\OpenConsole.exe" -EA:Ignore)) {
+        Throw "Failed to find OpenConsole.exe -- check the WAP packaging project"
+    }
+
 } Finally {
     Remove-Item -Recurse -Force $AppxPackageRootPath
 }
diff --git a/src/winconpty/winconpty.cpp b/src/winconpty/winconpty.cpp
index 53bebdb00..fc92b138b 100644
--- a/src/winconpty/winconpty.cpp
+++ b/src/winconpty/winconpty.cpp
@@ -22,23 +22,34 @@
 #pragma warning(disable : 4273) // inconsistent dll linkage (we are exporting things kernel32 also exports)
 #pragma warning(disable : 26485) // array-to-pointer decay is virtually impossible to avoid when we can't use STL.
 
+// Function Description:
+// - Returns the path to conhost.exe as a process heap string.
+static wil::unique_process_heap_string _InboxConsoleHostPath()
+{
+    wil::unique_process_heap_string systemDirectory;
+    wil::GetSystemDirectoryW<wil::unique_process_heap_string>(systemDirectory);
+    return wil::str_concat_failfast<wil::unique_process_heap_string>(L"\\\\?\\", systemDirectory, L"\\conhost.exe");
+}
+
 // Function Description:
 // - Returns the path to either conhost.exe or the side-by-side OpenConsole, depending on whether this
-//   module is building with Windows.
+//   module is building with Windows and OpenConsole could be found.
 // Return Value:
 // - A pointer to permanent storage containing the path to the console host.
 static wchar_t* _ConsoleHostPath()
 {
     // Use the magic of magic statics to only calculate this once.
     static wil::unique_process_heap_string consoleHostPath = []() {
-#ifdef __INSIDE_WINDOWS
-        wil::unique_process_heap_string systemDirectory;
-        wil::GetSystemDirectoryW<wil::unique_process_heap_string>(systemDirectory);
-        return wil::str_concat_failfast<wil::unique_process_heap_string>(L"\\\\?\\", systemDirectory, L"\\conhost.exe");
+#if defined(__INSIDE_WINDOWS)
+        return _InboxConsoleHostPath();
 #else
         // Use the STL only if we're not building in Windows.
         std::filesystem::path modulePath{ wil::GetModuleFileNameW<std::wstring>(wil::GetModuleInstanceHandle()) };
         modulePath.replace_filename(L"OpenConsole.exe");
+        if (!std::filesystem::exists(modulePath))
+        {
+            return _InboxConsoleHostPath();
+        }
         auto modulePathAsString{ modulePath.wstring() };
         return wil::make_process_heap_string_nothrow(modulePathAsString.data(), modulePathAsString.size());
 #endif // __INSIDE_WINDOWS