Honor -OutputFormat if specified in noninteractive, redirected, encoded command used with pwsh (#8115)

[Breaking Change]

There is specific code that sets the `OutputFormat` to xml if pwsh is run non-interactive, with redirected output, and the command was encoded.  However, it ignored whether OutputFormat was specified.  Fix is to track whether `-OutputFormat` was used and respect that value rather than defaulting to xml.

Fix https://github.com/PowerShell/PowerShell/issues/5912

src/Microsoft.PowerShell.ConsoleHost/host/msh/CommandLineParameterParser.cs

@@ -349,6 +349,16 @@ namespace Microsoft.PowerShell
             }
         }
 
+        internal bool OutputFormatSpecified
+        {
+            get
+            {
+                Dbg.Assert(_dirty, "Parse has not been called yet");
+
+                return _outputFormatSpecified;
+            }
+        }
+
         internal Serialization.DataFormat InputFormat
         {
             get
@@ -475,7 +485,7 @@ namespace Microsoft.PowerShell
             // Current user policy takes precedence.
             var consoleSessionSetting = Utils.GetPolicySetting<ConsoleSessionConfiguration>(Utils.CurrentUserThenSystemWideConfig);
 
-            return (consoleSessionSetting?.EnableConsoleSessionConfiguration == true && !string.IsNullOrEmpty(consoleSessionSetting?.ConsoleSessionConfigurationName)) ? 
+            return (consoleSessionSetting?.EnableConsoleSessionConfiguration == true && !string.IsNullOrEmpty(consoleSessionSetting?.ConsoleSessionConfigurationName)) ?
                     consoleSessionSetting.ConsoleSessionConfigurationName : string.Empty;
         }
 
@@ -493,7 +503,7 @@ namespace Microsoft.PowerShell
                 Dbg.Assert(args != null, "Argument 'args' to EarlyParseHelper should never be null");
                 return;
             }
-            
+
             bool noexitSeen = false;
             for (int i = 0; i < args.Length; ++i)
             {
@@ -587,7 +597,7 @@ namespace Microsoft.PowerShell
             Dbg.Assert(smallestUnambiguousMatch.Trim().ToLowerInvariant() == smallestUnambiguousMatch, "match should be normalized to lowercase w/ no outside whitespace");
             Dbg.Assert(match.Contains(smallestUnambiguousMatch), "sUM should be a substring of match");
 
-            return (match.Trim().ToLowerInvariant().IndexOf(switchKey, StringComparison.Ordinal) == 0 && 
+            return (match.Trim().ToLowerInvariant().IndexOf(switchKey, StringComparison.Ordinal) == 0 &&
                 switchKey.Length >= smallestUnambiguousMatch.Length);
         }
 
@@ -849,6 +859,7 @@ namespace Microsoft.PowerShell
                 else if (MatchSwitch(switchKey, "outputformat", "o") || MatchSwitch(switchKey, "of", "o"))
                 {
                     ParseFormat(args, ref i, ref _outFormat, CommandLineParameterParserStrings.MissingOutputFormatParameter);
+                    _outputFormatSpecified = true;
                 }
                 else if (MatchSwitch(switchKey, "inputformat", "in") || MatchSwitch(switchKey, "if", "if"))
                 {
@@ -1347,6 +1358,7 @@ namespace Microsoft.PowerShell
         private uint _exitCode = ConsoleHost.ExitCodeSuccess;
         private bool _dirty;
         private Serialization.DataFormat _outFormat = Serialization.DataFormat.Text;
+        private bool _outputFormatSpecified = false;
         private Serialization.DataFormat _inFormat = Serialization.DataFormat.Text;
         private Collection<CommandParameter> _collectedArgs = new Collection<CommandParameter>();
         private string _file;

src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs

@@ -1194,6 +1194,7 @@ namespace Microsoft.PowerShell
         }
 
         internal WrappedSerializer.DataFormat OutputFormat { get; private set; }
+        internal bool OutputFormatSpecified { get; private set; }
 
         internal WrappedSerializer.DataFormat InputFormat { get; private set; }
 
@@ -1203,9 +1204,9 @@ namespace Microsoft.PowerShell
             {
                 WrappedDeserializer.DataFormat format = OutputFormat;
 
-                //If this shell is invoked in minishell interop mode and error is redirected,
-                //always write data in error stream in xml format.
-                if (IsInteractive == false && Console.IsErrorRedirected && _wasInitialCommandEncoded)
+                // If this shell is invoked in non-interactive, error is redirected, and OutputFormat was not
+                // specified write data in error stream in xml format assuming PowerShell->PowerShell usage.
+                if (!OutputFormatSpecified && IsInteractive == false && Console.IsErrorRedirected && _wasInitialCommandEncoded)
                 {
                     format = Serialization.DataFormat.XML;
                 }
@@ -1315,6 +1316,7 @@ namespace Microsoft.PowerShell
                 }
 
                 OutputFormat = cpp.OutputFormat;
+                OutputFormatSpecified = cpp.OutputFormatSpecified;
                 InputFormat = cpp.InputFormat;
                 _wasInitialCommandEncoded = cpp.WasInitialCommandEncoded;
 

test/powershell/Host/ConsoleHost.Tests.ps1

@@ -298,6 +298,19 @@ Describe "ConsoleHost unit tests" -tags "Feature" {
             # Join (multiple lines) and remove whitespace (we don't care about spacing) to verify we converted to string (by generating a table)
             -join (& $powershell -noprofile -outputFormat text { [PSCustomObject]@{X=10;Y=20} }) -replace "\s","" | Should -Be "XY--1020"
         }
+
+        It "errors are in text if error is redirected, encoded command, non-interactive, and outputformat specified" {
+            $p = [Diagnostics.Process]::new()
+            $p.StartInfo.FileName = "pwsh"
+            $encoded = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes("throw 'boom'"))
+            $p.StartInfo.Arguments = "-EncodedCommand $encoded -ExecutionPolicy Bypass -NoLogo -NonInteractive -NoProfile -OutputFormat text"
+            $p.StartInfo.UseShellExecute = $false
+            $p.StartInfo.RedirectStandardError = $true
+            $p.Start() | Out-Null
+            $out = $p.StandardError.ReadToEnd()
+            $out | Should -Not -BeNullOrEmpty
+            $out.Split([Environment]::NewLine)[0] | Should -BeExactly "boom"
+        }
     }
 
     Context "Redirected standard output" {