From c0331053dbe7d1ae52627c32c9e1bf25b6357402 Mon Sep 17 00:00:00 2001 From: Hidetoshi Hirokawa <1086022+h-hirokawa@users.noreply.github.com> Date: Tue, 12 Nov 2019 14:58:57 +0900 Subject: [PATCH] Add encoding and codepage params to win_command/win_shell (#54896) (#54966) * Add output_encoding_override params to win_command/win_shell (#54896) This enhancement enables Ansible to parse the output of localized commands that ignore the prompt code page. * Added changelog and minor nits --- .../fragments/win_command-encoding.yaml | 2 ++ .../module_utils/csharp/Ansible.Process.cs | 32 ++++++++++++++----- .../Ansible.ModuleUtils.CommandUtil.psm1 | 8 +++-- lib/ansible/modules/windows/win_command.ps1 | 6 +++- lib/ansible/modules/windows/win_command.py | 9 ++++++ lib/ansible/modules/windows/win_shell.ps1 | 4 +++ lib/ansible/modules/windows/win_shell.py | 9 ++++++ .../targets/win_command/files/crt_setmode.c | 15 +++++++++ .../targets/win_command/tasks/main.yml | 18 +++++++++++ .../library/ansible_process_tests.ps1 | 8 ++++- .../targets/win_shell/tasks/main.yml | 15 +++++++++ 11 files changed, 114 insertions(+), 12 deletions(-) create mode 100644 changelogs/fragments/win_command-encoding.yaml create mode 100644 test/integration/targets/win_command/files/crt_setmode.c diff --git a/changelogs/fragments/win_command-encoding.yaml b/changelogs/fragments/win_command-encoding.yaml new file mode 100644 index 00000000000..1dabc82ae32 --- /dev/null +++ b/changelogs/fragments/win_command-encoding.yaml @@ -0,0 +1,2 @@ +minor_changes: +- win_command, win_shell - Add the ability to override the console output encoding with ``output_encoding_override`` - https://github.com/ansible/ansible/issues/54896 diff --git a/lib/ansible/module_utils/csharp/Ansible.Process.cs b/lib/ansible/module_utils/csharp/Ansible.Process.cs index 0ea20b0c093..f4c68f0529e 100644 --- a/lib/ansible/module_utils/csharp/Ansible.Process.cs +++ b/lib/ansible/module_utils/csharp/Ansible.Process.cs @@ -264,6 +264,18 @@ namespace Ansible.Process public static Result CreateProcess(string lpApplicationName, string lpCommandLine, string lpCurrentDirectory, IDictionary environment, string stdin) + { + return CreateProcess(lpApplicationName, lpCommandLine, lpCurrentDirectory, environment, stdin, null); + } + + public static Result CreateProcess(string lpApplicationName, string lpCommandLine, string lpCurrentDirectory, + IDictionary environment, byte[] stdin) + { + return CreateProcess(lpApplicationName, lpCommandLine, lpCurrentDirectory, environment, stdin, null); + } + + public static Result CreateProcess(string lpApplicationName, string lpCommandLine, string lpCurrentDirectory, + IDictionary environment, string stdin, string outputEncoding) { byte[] stdinBytes; if (String.IsNullOrEmpty(stdin)) @@ -274,7 +286,7 @@ namespace Ansible.Process stdin += Environment.NewLine; stdinBytes = new UTF8Encoding(false).GetBytes(stdin); } - return CreateProcess(lpApplicationName, lpCommandLine, lpCurrentDirectory, environment, stdinBytes); + return CreateProcess(lpApplicationName, lpCommandLine, lpCurrentDirectory, environment, stdinBytes, outputEncoding); } /// @@ -285,9 +297,10 @@ namespace Ansible.Process /// The full path to the current directory for the process, null will have the same cwd as the calling process /// A dictionary of key/value pairs to define the new process environment /// A byte array to send over the stdin pipe + /// The character encoding for decoding stdout/stderr output of the process. /// Result object that contains the command output and return code public static Result CreateProcess(string lpApplicationName, string lpCommandLine, string lpCurrentDirectory, - IDictionary environment, byte[] stdin) + IDictionary environment, byte[] stdin, string outputEncoding) { NativeHelpers.ProcessCreationFlags creationFlags = NativeHelpers.ProcessCreationFlags.CREATE_UNICODE_ENVIRONMENT | NativeHelpers.ProcessCreationFlags.EXTENDED_STARTUPINFO_PRESENT; @@ -337,7 +350,8 @@ namespace Ansible.Process } } - return WaitProcess(stdoutRead, stdoutWrite, stderrRead, stderrWrite, stdinStream, stdin, pi.hProcess); + return WaitProcess(stdoutRead, stdoutWrite, stderrRead, stderrWrite, stdinStream, stdin, pi.hProcess, + outputEncoding); } internal static void CreateStdioPipes(NativeHelpers.STARTUPINFOEX si, out SafeFileHandle stdoutRead, @@ -383,16 +397,18 @@ namespace Ansible.Process } internal static Result WaitProcess(SafeFileHandle stdoutRead, SafeFileHandle stdoutWrite, SafeFileHandle stderrRead, - SafeFileHandle stderrWrite, FileStream stdinStream, byte[] stdin, IntPtr hProcess) + SafeFileHandle stderrWrite, FileStream stdinStream, byte[] stdin, IntPtr hProcess, string outputEncoding = null) { - // Setup the output buffers and get stdout/stderr - UTF8Encoding utf8Encoding = new UTF8Encoding(false); + // Default to using UTF-8 as the output encoding, this should be a sane default for most scenarios. + outputEncoding = String.IsNullOrEmpty(outputEncoding) ? "utf-8" : outputEncoding; + Encoding encodingInstance = Encoding.GetEncoding(outputEncoding); + FileStream stdoutFS = new FileStream(stdoutRead, FileAccess.Read, 4096); - StreamReader stdout = new StreamReader(stdoutFS, utf8Encoding, true, 4096); + StreamReader stdout = new StreamReader(stdoutFS, encodingInstance, true, 4096); stdoutWrite.Close(); FileStream stderrFS = new FileStream(stderrRead, FileAccess.Read, 4096); - StreamReader stderr = new StreamReader(stderrFS, utf8Encoding, true, 4096); + StreamReader stderr = new StreamReader(stderrFS, encodingInstance, true, 4096); stderrWrite.Close(); stdinStream.Write(stdin, 0, stdin.Length); diff --git a/lib/ansible/module_utils/powershell/Ansible.ModuleUtils.CommandUtil.psm1 b/lib/ansible/module_utils/powershell/Ansible.ModuleUtils.CommandUtil.psm1 index a4a09743176..0e037e577f3 100644 --- a/lib/ansible/module_utils/powershell/Ansible.ModuleUtils.CommandUtil.psm1 +++ b/lib/ansible/module_utils/powershell/Ansible.ModuleUtils.CommandUtil.psm1 @@ -76,6 +76,9 @@ Function Run-Command { .PARAMETER environment A hashtable of key/value pairs to run with the command. If set, it will replace all other env vars. + .PARAMETER output_encoding_override + The character encoding name for decoding stdout/stderr output of the process. + .OUTPUT [Hashtable] [String]executable - The full path to the executable that was run @@ -87,7 +90,8 @@ Function Run-Command { [string]$command, [string]$working_directory = $null, [string]$stdin = "", - [hashtable]$environment = @{} + [hashtable]$environment = @{}, + [string]$output_encoding_override = $null ) # need to validate the working directory if it is set @@ -104,7 +108,7 @@ Function Run-Command { $executable = Get-ExecutablePath -executable $arguments[0] -directory $working_directory # run the command and get the results - $command_result = [Ansible.Process.ProcessUtil]::CreateProcess($executable, $command, $working_directory, $environment, $stdin) + $command_result = [Ansible.Process.ProcessUtil]::CreateProcess($executable, $command, $working_directory, $environment, $stdin, $output_encoding_override) return ,@{ executable = $executable diff --git a/lib/ansible/modules/windows/win_command.ps1 b/lib/ansible/modules/windows/win_command.ps1 index d021db3ac2d..e2a30650d29 100644 --- a/lib/ansible/modules/windows/win_command.ps1 +++ b/lib/ansible/modules/windows/win_command.ps1 @@ -18,7 +18,8 @@ $raw_command_line = Get-AnsibleParam -obj $params -name "_raw_params" -type "str $chdir = Get-AnsibleParam -obj $params -name "chdir" -type "path" $creates = Get-AnsibleParam -obj $params -name "creates" -type "path" $removes = Get-AnsibleParam -obj $params -name "removes" -type "path" -$stdin = Get-AnsibleParam -obj $params -name "stdin" -type 'str"' +$stdin = Get-AnsibleParam -obj $params -name "stdin" -type "str" +$output_encoding_override = Get-AnsibleParam -obj $params -name "output_encoding_override" -type "str" $raw_command_line = $raw_command_line.Trim() @@ -44,6 +45,9 @@ if ($chdir) { if ($stdin) { $command_args['stdin'] = $stdin } +if ($output_encoding_override) { + $command_args['output_encoding_override'] = $output_encoding_override +} $start_datetime = [DateTime]::UtcNow try { diff --git a/lib/ansible/modules/windows/win_command.py b/lib/ansible/modules/windows/win_command.py index 8fda93913ac..508419b28b1 100644 --- a/lib/ansible/modules/windows/win_command.py +++ b/lib/ansible/modules/windows/win_command.py @@ -44,6 +44,15 @@ options: - Set the stdin of the command directly to the specified value. type: str version_added: '2.5' + output_encoding_override: + description: + - This option overrides the encoding of stdout/stderr output. + - You can use this option when you need to run a command which ignore the console's codepage. + - You should only need to use this option in very rare circumstances. + - This value can be any valid encoding C(Name) based on the output of C([System.Text.Encoding]::GetEncodings()). + See U(https://docs.microsoft.com/dotnet/api/system.text.encoding.getencodings). + type: str + version_added: '2.10' notes: - If you want to run a command through a shell (say you are using C(<), C(>), C(|), etc), you actually want the M(win_shell) module instead. The diff --git a/lib/ansible/modules/windows/win_shell.ps1 b/lib/ansible/modules/windows/win_shell.ps1 index ce6b094bcca..54aef8de120 100644 --- a/lib/ansible/modules/windows/win_shell.ps1 +++ b/lib/ansible/modules/windows/win_shell.ps1 @@ -48,6 +48,7 @@ $creates = Get-AnsibleParam -obj $params -name "creates" -type "path" $removes = Get-AnsibleParam -obj $params -name "removes" -type "path" $stdin = Get-AnsibleParam -obj $params -name "stdin" -type "str" $no_profile = Get-AnsibleParam -obj $params -name "no_profile" -type "bool" -default $false +$output_encoding_override = Get-AnsibleParam -obj $params -name "output_encoding_override" -type "str" $raw_command_line = $raw_command_line.Trim() @@ -103,6 +104,9 @@ if ($chdir) { if ($stdin) { $run_command_arg['stdin'] = $stdin } +if ($output_encoding_override) { + $run_command_arg['output_encoding_override'] = $output_encoding_override +} $start_datetime = [DateTime]::UtcNow try { diff --git a/lib/ansible/modules/windows/win_shell.py b/lib/ansible/modules/windows/win_shell.py index 4f84743811d..ee2cd76240d 100644 --- a/lib/ansible/modules/windows/win_shell.py +++ b/lib/ansible/modules/windows/win_shell.py @@ -54,6 +54,15 @@ options: type: bool default: no version_added: '2.8' + output_encoding_override: + description: + - This option overrides the encoding of stdout/stderr output. + - You can use this option when you need to run a command which ignore the console's codepage. + - You should only need to use this option in very rare circumstances. + - This value can be any valid encoding C(Name) based on the output of C([System.Text.Encoding]::GetEncodings()). + See U(https://docs.microsoft.com/dotnet/api/system.text.encoding.getencodings). + type: str + version_added: '2.10' notes: - If you want to run an executable securely and predictably, it may be better to use the M(win_command) module instead. Best practices when writing diff --git a/test/integration/targets/win_command/files/crt_setmode.c b/test/integration/targets/win_command/files/crt_setmode.c new file mode 100644 index 00000000000..4067e717789 --- /dev/null +++ b/test/integration/targets/win_command/files/crt_setmode.c @@ -0,0 +1,15 @@ +// crt_setmode.c +// This program uses _setmode to change +// stdout from text mode to binary mode. +// Used to test output_encoding_override for win_command. + +#include +#include +#include + +int main(void) +{ + _setmode(_fileno(stdout), _O_BINARY); + // Translates to 日本 in shift_jis + printf("\x93\xFa\x96\x7B - Japan"); +} diff --git a/test/integration/targets/win_command/tasks/main.yml b/test/integration/targets/win_command/tasks/main.yml index 676d6eae4a8..bf72ffe21fa 100644 --- a/test/integration/targets/win_command/tasks/main.yml +++ b/test/integration/targets/win_command/tasks/main.yml @@ -203,6 +203,24 @@ - cmdout.stdout_lines[1] == 'ADDLOCAL=msi,example' - cmdout.stdout_lines[2] == 'two\\\\slashes' +- name: download binary that output shift_jis chars to console + win_get_url: + url: https://ansible-ci-files.s3.amazonaws.com/test/integration/targets/win_command/OutputEncodingOverride.exe + dest: C:\ansible testing\OutputEncodingOverride.exe + +- name: call binary with shift_jis output encoding override + win_command: '"C:\ansible testing\OutputEncodingOverride.exe"' + args: + output_encoding_override: shift_jis + register: cmdout + +- name: assert call to binary with shift_jis output + assert: + that: + - cmdout is changed + - cmdout.rc == 0 + - cmdout.stdout_lines[0] == '日本 - Japan' + - name: remove testing folder win_file: path: C:\ansible testing diff --git a/test/integration/targets/win_csharp_utils/library/ansible_process_tests.ps1 b/test/integration/targets/win_csharp_utils/library/ansible_process_tests.ps1 index f01c672e096..9b7b3cb36e7 100644 --- a/test/integration/targets/win_csharp_utils/library/ansible_process_tests.ps1 +++ b/test/integration/targets/win_csharp_utils/library/ansible_process_tests.ps1 @@ -217,6 +217,13 @@ $tests = @{ $actual.StandardError | Assert-Equals -Expected "" $actual.ExitCode | Assert-Equals -Expected 0 } + + "CreateProcess with unicode and us-ascii encoding" = { + $actual = [Ansible.Process.ProcessUtil]::CreateProcess($null, "cmd.exe /c echo 💩 café", $null, $null, '', 'us-ascii') + $actual.StandardOut | Assert-Equals -Expected "???? caf??`r`n" + $actual.StandardError | Assert-Equals -Expected "" + $actual.ExitCode | Assert-Equals -Expected 0 + } } foreach ($test_impl in $tests.GetEnumerator()) { @@ -226,4 +233,3 @@ foreach ($test_impl in $tests.GetEnumerator()) { $module.Result.data = "success" $module.ExitJson() - diff --git a/test/integration/targets/win_shell/tasks/main.yml b/test/integration/targets/win_shell/tasks/main.yml index 490caff0ee0..38387a30ab9 100644 --- a/test/integration/targets/win_shell/tasks/main.yml +++ b/test/integration/targets/win_shell/tasks/main.yml @@ -258,6 +258,21 @@ - nonascii_output.stdout_lines[0] == 'über den Fußgängerübergang gehen' - nonascii_output.stderr == '' +- name: echo some non ascii characters with us-ascii output encoding + win_shell: Write-Host über den Fußgängerübergang gehen + args: + output_encoding_override: us-ascii + register: nonascii_output_us_ascii_encoding + +- name: assert echo some non ascii characters with us-ascii output encoding + assert: + that: + - nonascii_output_us_ascii_encoding is changed + - nonascii_output_us_ascii_encoding.rc == 0 + - nonascii_output_us_ascii_encoding.stdout_lines|count == 1 + - nonascii_output_us_ascii_encoding.stdout_lines[0] == '??ber den Fu??g??nger??bergang gehen' + - nonascii_output_us_ascii_encoding.stderr == '' + - name: execute powershell without no_profile win_shell: '[System.Environment]::CommandLine' register: no_profile