PowerShell - Added coverage collector (#59009)

* Added coverage collection for PowerShell - ci_complete ci_coverage

* uncomment out coverage uploader call

* Generate XML for PowerShell coverage

* Use whitelist to exclude coverage run on non content plugins

* Remove uneeded ignore entry

* Try to reduce diff in cover.py

* Fix up coverage report package - ci_complete ci_coverage

lib/ansible/config/base.yml

@@ -342,6 +342,31 @@ CONDITIONAL_BARE_VARS:
   ini:
   - {key: conditional_bare_variables, section: defaults}
   version_added: "2.8"
+COVERAGE_REMOTE_OUTPUT:
+  name: Sets the output directory and filename prefix to generate coverage run info.
+  description:
+    - Sets the output directory on the remote host to generate coverage reports to.
+    - Currently only used for remote coverage on PowerShell modules.
+    - This is for internal use only.
+  env:
+  - {name: _ANSIBLE_COVERAGE_REMOTE_OUTPUT}
+  vars:
+  - {name: _ansible_coverage_remote_output}
+  type: str
+  version_added: '2.9'
+COVERAGE_REMOTE_WHITELIST:
+  name: Sets the list of paths to run coverage for.
+  description:
+  - A list of paths for files on the Ansible controller to run coverage for when executing on the remote host.
+  - Only files that match the path glob will have its coverage collected.
+  - Multiple path globs can be specified and are separated by ``:``.
+  - Currently only used for remote coverage on PowerShell modules.
+  - This is for internal use only.
+  default: '*'
+  env:
+  - {name: _ANSIBLE_COVERAGE_REMOTE_WHITELIST}
+  type: str
+  version_added: '2.9'
 ACTION_WARNINGS:
   name: Toggle action warnings
   default: True

lib/ansible/executor/module_common.py

@@ -1015,9 +1015,9 @@ def _find_module_utils(module_name, b_module_data, module_path, module_args, tas
         # create the common exec wrapper payload and set that as the module_data
         # bytes
         b_module_data = ps_manifest._create_powershell_wrapper(
-            b_module_data, module_args, environment, async_timeout, become,
+            b_module_data, module_path, module_args, environment,
-            become_method, become_user, become_password, become_flags,
+            async_timeout, become, become_method, become_user, become_password,
-            module_substyle
+            become_flags, module_substyle, task_vars
         )
 
     elif module_substyle == 'jsonargs':

lib/ansible/executor/powershell/coverage_wrapper.ps1

@@ -0,0 +1,190 @@
+# (c) 2019 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+param(
+    [Parameter(Mandatory=$true)][System.Collections.IDictionary]$Payload
+)
+
+#AnsibleRequires -Wrapper module_wrapper
+
+$ErrorActionPreference = "Stop"
+
+Write-AnsibleLog "INFO - starting coverage_wrapper" "coverage_wrapper"
+
+# Required to be set for psrp to we can set a breakpoint in the remote runspace
+if ($PSVersionTable.PSVersion -ge [Version]'4.0') {
+    $host.Runspace.Debugger.SetDebugMode([System.Management.Automation.DebugModes]::RemoteScript)
+}
+
+Function New-CoverageBreakpoint {
+    Param (
+        [String]$Path,
+        [ScriptBlock]$Code,
+        [String]$AnsiblePath
+    )
+
+    # It is quicker to pass in the code as a string instead of calling ParseFile as we already know the contents
+    $predicate = {
+        $args[0] -is [System.Management.Automation.Language.CommandBaseAst]
+    }
+    $script_cmds = $Code.Ast.FindAll($predicate, $true)
+
+    # Create an object that tracks the Ansible path of the file and the breakpoints that have been set in it
+    $info = [PSCustomObject]@{
+        Path = $AnsiblePath
+        Breakpoints = [System.Collections.Generic.List`1[System.Management.Automation.Breakpoint]]@()
+    }
+
+    # Keep track of lines that are already scanned. PowerShell can contains multiple commands in 1 line
+    $scanned_lines = [System.Collections.Generic.HashSet`1[System.Int32]]@()
+    foreach ($cmd in $script_cmds) {
+        if (-not $scanned_lines.Add($cmd.Extent.StartLineNumber)) {
+            continue
+        }
+
+        # Do not add any -Action value, even if it is $null or {}. Doing so will balloon the runtime.
+        $params = @{
+            Script = $Path
+            Line = $cmd.Extent.StartLineNumber
+            Column = $cmd.Extent.StartColumnNumber
+        }
+        $info.Breakpoints.Add((Set-PSBreakpoint @params))
+    }
+
+    $info
+}
+
+Function Compare-WhitelistPattern {
+    Param (
+        [String[]]$Patterns,
+        [String]$Path
+    )
+
+    foreach ($pattern in $Patterns) {
+        if ($Path -like $pattern) {
+            return $true
+        }
+    }
+    return $false
+}
+
+$module_name = $Payload.module_args["_ansible_module_name"]
+Write-AnsibleLog "INFO - building coverage payload for '$module_name'" "coverage_wrapper"
+
+# A PS Breakpoint needs an actual path to work properly, we create a temp directory that will store the module and
+# module_util code during execution
+$temp_path = Join-Path -Path ([System.IO.Path]::GetTempPath()) -ChildPath "ansible-coverage-$([System.IO.Path]::GetRandomFileName())"
+Write-AnsibleLog "INFO - Creating temp path for coverage files '$temp_path'" "coverage_wrapper"
+New-Item -Path $temp_path -ItemType Directory > $null
+$breakpoint_info = [System.Collections.Generic.List`1[PSObject]]@()
+
+try {
+    $scripts = [System.Collections.Generic.List`1[System.Object]]@($script:common_functions)
+
+    $coverage_whitelist = $Payload.coverage.whitelist.Split(":", [StringSplitOptions]::RemoveEmptyEntries)
+
+    # We need to track what utils have already been added to the script for loading. This is because the load
+    # order is important and can have module_utils that rely on other utils.
+    $loaded_utils = [System.Collections.Generic.HashSet`1[System.String]]@()
+    $parse_util = {
+        $util_name = $args[0]
+        if (-not $loaded_utils.Add($util_name)) {
+            return
+        }
+
+        $util_code = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.powershell_modules.$util_name))
+        $util_sb = [ScriptBlock]::Create($util_code)
+        $util_path = Join-Path -Path $temp_path -ChildPath "$($util_name).psm1"
+
+        Write-AnsibleLog "INFO - Outputting module_util $util_name to temp file '$util_path'" "coverage_wrapper"
+        Set-Content -LiteralPath $util_path -Value $util_code
+
+        $ansible_path = $Payload.coverage.module_util_paths.$util_name
+        if ((Compare-WhitelistPattern -Patterns $coverage_whitelist -Path $ansible_path)) {
+            $cov_params = @{
+                Path = $util_path
+                Code = $util_sb
+                AnsiblePath = $ansible_path
+            }
+            $breakpoints = New-CoverageBreakpoint @cov_params
+            $breakpoint_info.Add($breakpoints)
+        }
+
+        if ($null -ne $util_sb.Ast.ScriptRequirements) {
+            foreach ($required_util in $util_sb.Ast.ScriptRequirements.RequiredModules) {
+                &$parse_util $required_util.Name
+            }
+        }
+        Write-AnsibleLog "INFO - Adding util $util_name to scripts to run" "coverage_wrapper"
+        $scripts.Add("Import-Module -Name '$util_path'")
+    }
+    foreach ($util in $Payload.powershell_modules.Keys) {
+        &$parse_util $util
+    }
+
+    $module = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.module_entry))
+    $module_path = Join-Path -Path $temp_path -ChildPath "$($module_name).ps1"
+    Write-AnsibleLog "INFO - Ouputting module $module_name to temp file '$module_path'" "coverage_wrapper"
+    Set-Content -LiteralPath $module_path -Value $module
+    $scripts.Add($module_path)
+
+    $ansible_path = $Payload.coverage.module_path
+    if ((Compare-WhitelistPattern -Patterns $coverage_whitelist -Path $ansible_path)) {
+        $cov_params = @{
+            Path = $module_path
+            Code = [ScriptBlock]::Create($module)
+            AnsiblePath = $Payload.coverage.module_path
+        }
+        $breakpoints = New-CoverageBreakpoint @cov_params
+        $breakpoint_info.Add($breakpoints)
+    }
+
+    $variables = [System.Collections.ArrayList]@(@{ Name = "complex_args"; Value = $Payload.module_args; Scope = "Global" })
+    $entrypoint = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($payload.module_wrapper))
+    $entrypoint = [ScriptBlock]::Create($entrypoint)
+
+    $params = @{
+        Scripts = $scripts
+        Variables = $variables
+        Environment = $Payload.environment
+        ModuleName = $module_name
+    }
+    if ($breakpoint_info) {
+        $params.Breakpoints = $breakpoint_info.Breakpoints
+    }
+
+    try {
+        &$entrypoint @params
+    } finally {
+        # Processing here is kept to an absolute minimum to make sure each task runtime is kept as small as
+        # possible. Once all the tests have been run ansible-test will collect this info and process it locally in
+        # one go.
+        Write-AnsibleLog "INFO - Creating coverage result output" "coverage_wrapper"
+        $coverage_info = @{}
+        foreach ($info in $breakpoint_info) {
+            $coverage_info.($info.Path) = $info.Breakpoints | Select-Object -Property Line, HitCount
+        }
+
+        # The coverage.output value is a filename set by the Ansible controller. We append some more remote side
+        # info to the filename to make it unique and identify the remote host a bit more.
+        $ps_version = "$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)"
+        $coverage_output_path = "$($Payload.coverage.output)=powershell-$ps_version=coverage.$($env:COMPUTERNAME).$PID.$(Get-Random)"
+        $code_cov_json = ConvertTo-Json -InputObject $coverage_info -Compress
+
+        Write-AnsibleLog "INFO - Outputting coverage json to '$coverage_output_path'" "coverage_wrapper"
+        Set-Content -LiteralPath $coverage_output_path -Value $code_cov_json
+    }
+} finally {
+    try {
+        if ($breakpoint_info) {
+            foreach ($b in $breakpoint_info.Breakpoints) {
+                Remove-PSBreakpoint -Breakpoint $b
+            }
+        }
+    } finally {
+        Write-AnsibleLog "INFO - Remove temp coverage folder '$temp_path'" "coverage_wrapper"
+        Remove-Item -LiteralPath $temp_path -Force -Recurse
+    }
+}
+
+Write-AnsibleLog "INFO - ending coverage_wrapper" "coverage_wrapper"

lib/ansible/executor/powershell/module_manifest.py

@@ -134,13 +134,17 @@ class PSModuleDepFinder(object):
                                'for \'%s\'' % m)
 
         module_util_data = to_bytes(_slurp(mu_path))
+        util_info = {
+            'data': module_util_data,
+            'path': to_text(mu_path),
+        }
         if ext == ".psm1":
-            self.ps_modules[m] = module_util_data
+            self.ps_modules[m] = util_info
         else:
             if wrapper:
-                self.cs_utils_wrapper[m] = module_util_data
+                self.cs_utils_wrapper[m] = util_info
             else:
-                self.cs_utils_module[m] = module_util_data
+                self.cs_utils_module[m] = util_info
         self.scan_module(module_util_data, wrapper=wrapper,
                          powershell=(ext == ".psm1"))
 
@@ -202,10 +206,10 @@ def _strip_comments(source):
     return b'\n'.join(buf)
 
 
-def _create_powershell_wrapper(b_module_data, module_args, environment,
+def _create_powershell_wrapper(b_module_data, module_path, module_args,
-                               async_timeout, become, become_method,
+                               environment, async_timeout, become,
-                               become_user, become_password, become_flags,
+                               become_method, become_user, become_password,
-                               substyle):
+                               become_flags, substyle, task_vars):
     # creates the manifest/wrapper used in PowerShell/C# modules to enable
     # things like become and async - this is also called in action/script.py
 
@@ -227,7 +231,7 @@ def _create_powershell_wrapper(b_module_data, module_args, environment,
         module_args=module_args,
         actions=[module_wrapper],
         environment=environment,
-        encoded_output=False
+        encoded_output=False,
     )
     finder.scan_exec_script(module_wrapper)
 
@@ -261,6 +265,19 @@ def _create_powershell_wrapper(b_module_data, module_args, environment,
         exec_manifest['become_password'] = None
         exec_manifest['become_flags'] = None
 
+    coverage_manifest = dict(
+        module_path=module_path,
+        module_util_paths=dict(),
+        output=None,
+    )
+    coverage_output = C.config.get_config_value('COVERAGE_REMOTE_OUTPUT', variables=task_vars)
+    if coverage_output and substyle == 'powershell':
+        finder.scan_exec_script('coverage_wrapper')
+        coverage_manifest['output'] = coverage_output
+
+        coverage_whitelist = C.config.get_config_value('COVERAGE_REMOTE_WHITELIST', variables=task_vars)
+        coverage_manifest['whitelist'] = coverage_whitelist
+
     # make sure Ansible.ModuleUtils.AddType is added if any C# utils are used
     if len(finder.cs_utils_wrapper) > 0 or len(finder.cs_utils_module) > 0:
         finder._add_module((b"Ansible.ModuleUtils.AddType", ".psm1"),
@@ -283,16 +300,24 @@ def _create_powershell_wrapper(b_module_data, module_args, environment,
         exec_manifest[name] = b64_data
 
     for name, data in finder.ps_modules.items():
-        b64_data = to_text(base64.b64encode(data))
+        b64_data = to_text(base64.b64encode(data['data']))
         exec_manifest['powershell_modules'][name] = b64_data
+        coverage_manifest['module_util_paths'][name] = data['path']
+
+    cs_utils = {}
+    for cs_util in [finder.cs_utils_wrapper, finder.cs_utils_module]:
+        for name, data in cs_util.items():
+            cs_utils[name] = data['data']
 
-    cs_utils = finder.cs_utils_wrapper
-    cs_utils.update(finder.cs_utils_module)
     for name, data in cs_utils.items():
         b64_data = to_text(base64.b64encode(data))
         exec_manifest['csharp_utils'][name] = b64_data
     exec_manifest['csharp_utils_module'] = list(finder.cs_utils_module.keys())
 
+    # To save on the data we are sending across we only add the coverage info if coverage is being run
+    if 'coverage_wrapper' in exec_manifest:
+        exec_manifest['coverage'] = coverage_manifest
+
     b_json = to_bytes(json.dumps(exec_manifest))
     # delimit the payload JSON from the wrapper to keep sensitive contents out of scriptblocks (which can be logged)
     b_data = exec_wrapper + b'\0\0\0\0' + b_json

lib/ansible/executor/powershell/module_powershell_wrapper.ps1

@@ -32,16 +32,32 @@ if ($csharp_utils.Count -gt 0) {
     Add-CSharpType -References $csharp_utils -TempPath $new_tmp -IncludeDebugInfo
 }
 
+if ($Payload.ContainsKey("coverage") -and $null -ne $host.Runspace -and $null -ne $host.Runspace.Debugger) {
+    $entrypoint = $payload.coverage_wrapper
+
+    $params = @{
+        Payload = $Payload
+    }
+} else {
     # get the common module_wrapper code and invoke that to run the module
-$variables = [System.Collections.ArrayList]@(@{ Name = "complex_args"; Value = $Payload.module_args; Scope = "Global" })
     $module = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Payload.module_entry))
-$entrypoint = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($payload.module_wrapper))
+    $variables = [System.Collections.ArrayList]@(@{ Name = "complex_args"; Value = $Payload.module_args; Scope = "Global" })
+    $entrypoint = $Payload.module_wrapper
+
+    $params = @{
+        Scripts = @($script:common_functions, $module)
+        Variables = $variables
+        Environment = $Payload.environment
+        Modules = $Payload.powershell_modules
+        ModuleName = $module_name
+    }
+}
+
+$entrypoint = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($entrypoint))
 $entrypoint = [ScriptBlock]::Create($entrypoint)
 
 try {
-    &$entrypoint -Scripts $script:common_functions, $module -Variables $variables `
+    &$entrypoint @params
-        -Environment $Payload.environment -Modules $Payload.powershell_modules `
-        -ModuleName $module_name
 } catch {
     # failed to invoke the PowerShell module, capture the exception and
     # output a pretty error for Ansible to parse

lib/ansible/executor/powershell/module_wrapper.ps1

@@ -29,13 +29,18 @@ value is a base64 string of the module util code.
 
 .PARAMETER ModuleName
 [String] The name of the module that is being executed.
+
+.PARAMETER Breakpoints
+A list of line breakpoints to add to the runspace debugger. This is used to
+track module and module_utils coverage.
 #>
 param(
     [Object[]]$Scripts,
     [System.Collections.ArrayList][AllowEmptyCollection()]$Variables,
     [System.Collections.IDictionary]$Environment,
     [System.Collections.IDictionary]$Modules,
-    [String]$ModuleName
+    [String]$ModuleName,
+    [System.Management.Automation.LineBreakpoint[]]$Breakpoints = @()
 )
 
 Write-AnsibleLog "INFO - creating new PowerShell pipeline for $ModuleName" "module_wrapper"
@@ -92,6 +97,23 @@ foreach ($script in $Scripts) {
     $ps.AddScript($script).AddStatement() > $null
 }
 
+if ($Breakpoints.Count -gt 0) {
+    Write-AnsibleLog "INFO - adding breakpoint to runspace that will run the modules" "module_wrapper"
+    if ($PSVersionTable.PSVersion.Major -eq 3) {
+        # The SetBreakpoints method was only added in PowerShell v4+. We need to rely on a private method to
+        # achieve the same functionality in this older PowerShell version. This should be removed once we drop
+        # support for PowerShell v3.
+        $set_method = $ps.Runspace.Debugger.GetType().GetMethod(
+            'AddLineBreakpoint', [System.Reflection.BindingFlags]'Instance, NonPublic'
+        )
+        foreach ($b in $Breakpoints) {
+            $set_method.Invoke($ps.Runspace.Debugger, [Object[]]@(,$b)) > $null
+        }
+    } else {
+        $ps.Runspace.Debugger.SetBreakpoints($Breakpoints)
+    }
+}
+
 Write-AnsibleLog "INFO - start module exec with Invoke() - $ModuleName" "module_wrapper"
 
 # temporarily override the stdout stream and create our own in a StringBuilder

lib/ansible/plugins/action/script.py

@@ -130,9 +130,9 @@ class ActionModule(ActionBase):
                 # FUTURE: use a more public method to get the exec payload
                 pc = self._play_context
                 exec_data = ps_manifest._create_powershell_wrapper(
-                    to_bytes(script_cmd), {}, env_dict, self._task.async_val,
+                    to_bytes(script_cmd), source, {}, env_dict, self._task.async_val,
                     pc.become, pc.become_method, pc.become_user,
-                    pc.become_pass, pc.become_flags, substyle="script"
+                    pc.become_pass, pc.become_flags, "script", task_vars
                 )
                 # build the necessary exec wrapper command
                 # FUTURE: this still doesn't let script work on Windows with non-pipelined connections or

test/integration/targets/win_become/tasks/main.yml

@@ -195,7 +195,7 @@
     raw: |
       $dt=[datetime]"{{ test_starttime.stdout|trim }}"
       (Get-WinEvent -LogName Microsoft-Windows-Powershell/Operational |
-      ? { $_.TimeCreated -ge $dt -and $_.Message -match "{{ gen_pw }}|whoami" }).Count
+      ? { $_.TimeCreated -ge $dt -and $_.Message -match "{{ gen_pw }}" }).Count
     register: ps_log_count
 
   - name: assert no PS events contain password or module args

test/integration/targets/win_csharp_utils/library/ansible_basic_tests.ps1

@@ -89,7 +89,9 @@ $tmpdir = $module.Tmpdir
 # Override the Exit and WriteLine behaviour to throw an exception instead of exiting the module
 [Ansible.Basic.AnsibleModule]::Exit = {
     param([Int32]$rc)
-    throw "exit: $rc"
+    $exp = New-Object -TypeName System.Exception -ArgumentList "exit: $rc"
+    $exp | Add-Member -Type NoteProperty -Name Output -Value $_test_out
+    throw $exp
 }
 [Ansible.Basic.AnsibleModule]::WriteLine = {
     param([String]$line)
@@ -429,7 +431,7 @@ $tests = @{
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -529,7 +531,7 @@ $tests = @{
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -569,7 +571,7 @@ $tests = @{
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -643,10 +645,9 @@ $tests = @{
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
-        #$_test_out
 
         # verify no_log params are masked in invocation
         $expected = @{
@@ -728,7 +729,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -774,7 +775,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -816,7 +817,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -857,7 +858,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -895,7 +896,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -933,7 +934,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -987,7 +988,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1011,7 +1012,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $failed
 
@@ -1041,7 +1042,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $failed
 
@@ -1071,7 +1072,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $failed
 
@@ -1104,7 +1105,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $failed
 
@@ -1133,7 +1134,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $failed
 
@@ -1157,7 +1158,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $failed
 
@@ -1184,7 +1185,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $failed
 
@@ -1281,7 +1282,7 @@ test_no_log - Invoked with:
                 failed = $true
                 msg = "Unsupported parameters for (undefined win module) module: _ansible_invalid. Supported parameters include: "
             }
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
             $actual | Assert-DictionaryEquals -Expected $expected
         }
         $failed | Assert-Equals -Expected $true
@@ -1324,7 +1325,7 @@ test_no_log - Invoked with:
         try {
             $m.ExitJson()
         } catch [System.Management.Automation.RuntimeException] {
-            $output = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $output = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         (Test-Path -Path $actual_tmpdir -PathType Container) | Assert-Equals -Expected $false
         (Test-Path -Path $remote_tmp -PathType Container) | Assert-Equals -Expected $true
@@ -1369,7 +1370,7 @@ test_no_log - Invoked with:
         try {
             $m.ExitJson()
         } catch [System.Management.Automation.RuntimeException] {
-            $output = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $output = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         (Test-Path -Path $actual_tmpdir -PathType Container) | Assert-Equals -Expected $false
         (Test-Path -Path $remote_tmp -PathType Container) | Assert-Equals -Expected $true
@@ -1402,7 +1403,7 @@ test_no_log - Invoked with:
         try {
             $m.ExitJson()
         } catch [System.Management.Automation.RuntimeException] {
-            $output = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $output = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         (Test-Path -Path $actual_tmpdir -PathType Container) | Assert-Equals -Expected $true
         (Test-Path -Path $remote_tmp -PathType Container) | Assert-Equals -Expected $true
@@ -1420,7 +1421,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1452,7 +1453,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1477,7 +1478,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1504,7 +1505,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1532,7 +1533,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1561,7 +1562,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1593,7 +1594,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1626,7 +1627,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1679,7 +1680,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1718,7 +1719,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1750,7 +1751,7 @@ test_no_log - Invoked with:
         try {
             $m.ExitJson()
         } catch [System.Management.Automation.RuntimeException] {
-            $output = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $output = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $output.Keys.Count | Assert-Equals -Expected 2
         $output.changed | Assert-Equals -Expected $false
@@ -1773,7 +1774,7 @@ test_no_log - Invoked with:
         try {
             $m.ExitJson()
         } catch [System.Management.Automation.RuntimeException] {
-            $output = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $output = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $expected_warning = "value of option_key was a case insensitive match of one of: abc, def. "
         $expected_warning += "Checking of choices will be case sensitive in a future Ansible release. "
@@ -1802,7 +1803,7 @@ test_no_log - Invoked with:
         try {
             $m.ExitJson()
         } catch [System.Management.Automation.RuntimeException] {
-            $output = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $output = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $expected_warning = "value of option_key was a case insensitive match of one of: abc, def. "
         $expected_warning += "Checking of choices will be case sensitive in a future Ansible release. "
@@ -1832,7 +1833,7 @@ test_no_log - Invoked with:
         try {
             $m.ExitJson()
         } catch [System.Management.Automation.RuntimeException] {
-            $output = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $output = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $expected_warning = "value of option_key was a case insensitive match of one or more of: abc, def, ghi, JKL. "
         $expected_warning += "Checking of choices will be case sensitive in a future Ansible release. "
@@ -1862,7 +1863,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1894,7 +1895,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1926,7 +1927,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1958,7 +1959,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -1988,7 +1989,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -2022,7 +2023,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -2057,7 +2058,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -2088,7 +2089,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -2129,7 +2130,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -2161,7 +2162,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -2189,7 +2190,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -2222,7 +2223,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -2251,7 +2252,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -2283,7 +2284,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 1"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -2317,7 +2318,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -2338,7 +2339,7 @@ test_no_log - Invoked with:
         } catch [System.Management.Automation.RuntimeException] {
             $failed = $true
             $_.Exception.Message | Assert-Equals -Expected "exit: 0"
-            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_test_out)
+            $actual = [Ansible.Basic.AnsibleModule]::FromJson($_.Exception.InnerException.Output)
         }
         $failed | Assert-Equals -Expected $true
 
@@ -2370,7 +2371,6 @@ try {
     foreach ($test_impl in $tests.GetEnumerator()) {
         # Reset the variables before each test
         $complex_args = @{}
-        $_test_out = $null
 
         $test = $test_impl.Key
         &$test_impl.Value
@@ -2384,7 +2384,7 @@ try {
 
     if ($_.Exception.Message.StartSwith("exit: ")) {
         # The exception was caused by an unexpected Exit call, log that on the output
-        $module.Result.output = (ConvertFrom-Json -InputObject $_test_out)
+        $module.Result.output = (ConvertFrom-Json -InputObject $_.Exception.InnerException.Output)
         $module.Result.msg = "Uncaught AnsibleModule exit in tests, see output"
     } else {
         # Unrelated exception

test/integration/targets/win_exec_wrapper/tasks/main.yml

@@ -43,6 +43,10 @@
     data: error
   register: error_module
   ignore_errors: yes
+  vars:
+    # Running with coverage means the module is run from a script and not as a psuedo script in a pipeline. This
+    # results in a different error message being returned so we disable coverage collection for this task.
+    _ansible_coverage_remote_output: ''
 
 - name: assert test module with error msg
   assert:
@@ -82,6 +86,8 @@
     data: function_throw
   register: function_exception
   ignore_errors: yes
+  vars:
+    _ansible_coverage_remote_output: ''
 
 - name: assert test module with function exception
   assert:
@@ -257,7 +263,7 @@
   raw: |
     $dt=[datetime]"{{ test_starttime.stdout|trim }}"
     (Get-WinEvent -LogName Microsoft-Windows-Powershell/Operational |
-    ? { $_.TimeCreated -ge $dt -and $_.Message -match "test_fail|fail_module|hyphen-var" }).Count
+    ? { $_.TimeCreated -ge $dt -and $_.Message -match "fail_module|hyphen-var" }).Count
   register: ps_log_count
 
 - name: assert no PS events contain module args or envvars

test/integration/targets/win_module_utils/tasks/main.yml

@@ -51,6 +51,10 @@
 - name: call module that imports module_utils with further imports
   recursive_requires:
   register: recursive_requires
+  vars:
+    # Our coverage runner does not work with recursive required. This is a limitation on PowerShell so we need to
+    # disable coverage for this task
+    _ansible_coverage_remote_output: ''
 
 - assert:
     that:

test/lib/ansible_test/_data/playbooks/windows_coverage_setup.yml

@@ -0,0 +1,19 @@
+---
+- name: setup global coverage directory for Windows test targets
+  hosts: windows
+  gather_facts: no
+  tasks:
+  - name: create temp directory
+    win_file:
+      path: '{{ remote_temp_path }}'
+      state: directory
+
+  - name: allow everyone to write to coverage test dir
+    win_acl:
+      path: '{{ remote_temp_path }}'
+      user: Everyone
+      rights: Modify
+      inherit: ContainerInherit, ObjectInherit
+      propagation: 'None'
+      type: allow
+      state: present

test/lib/ansible_test/_data/playbooks/windows_coverage_teardown.yml

@@ -0,0 +1,77 @@
+---
+- name: collect the coverage files from the Windows host
+  hosts: windows
+  gather_facts: no
+  tasks:
+  - name: make sure all vars have been set
+    assert:
+      that:
+      - local_temp_path is defined
+      - remote_temp_path is defined
+
+  - name: zip up all coverage files in the
+    win_shell: |
+      $coverage_dir = '{{ remote_temp_path }}'
+      $zip_file = Join-Path -Path $coverage_dir -ChildPath 'coverage.zip'
+      if (Test-Path -LiteralPath $zip_file) {
+          Remove-Item -LiteralPath $zip_file -Force
+      }
+
+      $coverage_files = Get-ChildItem -LiteralPath $coverage_dir -Include '*=coverage*' -File
+
+      $legacy = $false
+      try {
+          # Requires .NET 4.5+ which isn't present on older WIndows versions. Remove once 2008/R2 is EOL.
+          # We also can't use the Shell.Application as it will fail on GUI-less servers (Server Core).
+          Add-Type -AssemblyName System.IO.Compression -ErrorAction Stop > $null
+      } catch {
+          $legacy = $true
+      }
+
+      if ($legacy) {
+          New-Item -Path $zip_file -ItemType File > $null
+          $shell = New-Object -ComObject Shell.Application
+          $zip = $shell.Namespace($zip_file)
+          foreach ($file in $coverage_files) {
+              $zip.CopyHere($file.FullName)
+          }
+      } else {
+          $fs = New-Object -TypeName System.IO.FileStream -ArgumentList $zip_file, 'CreateNew'
+          try {
+              $archive = New-Object -TypeName System.IO.Compression.ZipArchive -ArgumentList @(
+                  $fs,
+                  [System.IO.Compression.ZipArchiveMode]::Create
+              )
+              try  {
+                  foreach ($file in $coverage_files) {
+                      $archive_entry = $archive.CreateEntry($file.Name, 'Optimal')
+                      $entry_fs = $archive_entry.Open()
+                      try {
+                          $file_fs = [System.IO.File]::OpenRead($file.FullName)
+                          try {
+                              $file_fs.CopyTo($entry_fs)
+                          } finally {
+                              $file_fs.Dispose()
+                          }
+                      } finally {
+                          $entry_fs.Dispose()
+                      }
+                  }
+              } finally {
+                  $archive.Dispose()
+              }
+          } finally {
+              $fs.Dispose()
+          }
+      }
+
+  - name: fetch coverage zip file to localhost
+    fetch:
+      src: '{{ remote_temp_path }}\coverage.zip'
+      dest: '{{ local_temp_path }}/coverage-{{ inventory_hostname }}.zip'
+      flat: yes
+
+  - name: remove the temporary coverage directory
+    win_file:
+      path: '{{ remote_temp_path }}'
+      state: absent

test/lib/ansible_test/_internal/cli.py

@@ -732,11 +732,11 @@ def add_extra_coverage_options(parser):
 
     parser.add_argument('--all',
                         action='store_true',
-                        help='include all python source files')
+                        help='include all python/powershell source files')
 
     parser.add_argument('--stub',
                         action='store_true',
-                        help='generate empty report of all python source files')
+                        help='generate empty report of all python/powershell source files')
 
 
 def add_httptester_options(parser, argparse):

test/lib/ansible_test/_internal/cover.py

@@ -2,12 +2,26 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import json
 import os
 import re
+import time
+
+from xml.etree.ElementTree import (
+    Comment,
+    Element,
+    SubElement,
+    tostring,
+)
+
+from xml.dom import (
+    minidom,
+)
 
 from .target import (
     walk_module_targets,
     walk_compile_targets,
+    walk_powershell_targets,
 )
 
 from .util import (
@@ -15,6 +29,8 @@ from .util import (
     ApplicationError,
     common_environment,
     ANSIBLE_TEST_DATA_ROOT,
+    to_bytes,
+    to_text,
 )
 
 from .util_common import (
@@ -26,6 +42,10 @@ from .config import (
     CoverageReportConfig,
 )
 
+from .env import (
+    get_ansible_version,
+)
+
 from .executor import (
     Delegate,
     install_command_requirements,
@@ -44,49 +64,25 @@ def command_coverage_combine(args):
     :type args: CoverageConfig
     :rtype: list[str]
     """
+    return _command_coverage_combine_powershell(args) + _command_coverage_combine_python(args)
+
+
+def _command_coverage_combine_python(args):
+    """
+    :type args: CoverageConfig
+    :rtype: list[str]
+    """
     coverage = initialize_coverage(args)
 
     modules = dict((t.module, t.path) for t in list(walk_module_targets()) if t.path.endswith('.py'))
 
     coverage_dir = os.path.join(data_context().results, 'coverage')
-    coverage_files = [os.path.join(coverage_dir, f) for f in os.listdir(coverage_dir) if '=coverage.' in f]
+    coverage_files = [os.path.join(coverage_dir, f) for f in os.listdir(coverage_dir)
-
+                      if '=coverage.' in f and '=python' in f]
-    ansible_path = os.path.abspath('lib/ansible/') + '/'
-    root_path = data_context().content.root + '/'
 
     counter = 0
-    groups = {}
+    sources = _get_coverage_targets(args, walk_compile_targets)
-
+    groups = _build_stub_groups(args, sources, lambda line_count: set())
-    if args.all or args.stub:
-        # excludes symlinks of regular files to avoid reporting on the same file multiple times
-        # in the future it would be nice to merge any coverage for symlinks into the real files
-        sources = sorted(os.path.abspath(target.path) for target in walk_compile_targets(include_symlinks=False))
-    else:
-        sources = []
-
-    if args.stub:
-        stub_group = []
-        stub_groups = [stub_group]
-        stub_line_limit = 500000
-        stub_line_count = 0
-
-        for source in sources:
-            with open(source, 'r') as source_fd:
-                source_line_count = len(source_fd.read().splitlines())
-
-            stub_group.append(source)
-            stub_line_count += source_line_count
-
-            if stub_line_count > stub_line_limit:
-                stub_line_count = 0
-                stub_group = []
-                stub_groups.append(stub_group)
-
-        for stub_index, stub_group in enumerate(stub_groups):
-            if not stub_group:
-                continue
-
-            groups['=stub-%02d' % (stub_index + 1)] = dict((source, set()) for source in stub_group)
 
     if data_context().content.collection:
         collection_search_re = re.compile(r'/%s/' % data_context().content.collection.directory)
@@ -125,50 +121,10 @@ def command_coverage_combine(args):
                 display.warning('No arcs found for "%s" in coverage file: %s' % (filename, coverage_file))
                 continue
 
-            if '/ansible_modlib.zip/ansible/' in filename:
+            filename = _sanitise_filename(filename, modules=modules, collection_search_re=collection_search_re,
-                # Rewrite the module_utils path from the remote host to match the controller. Ansible 2.6 and earlier.
+                                          collection_sub_re=collection_sub_re)
-                new_name = re.sub('^.*/ansible_modlib.zip/ansible/', ansible_path, filename)
+            if not filename:
-                display.info('%s -> %s' % (filename, new_name), verbosity=3)
-                filename = new_name
-            elif collection_search_re and collection_search_re.search(filename):
-                new_name = os.path.abspath(collection_sub_re.sub('', filename))
-                display.info('%s -> %s' % (filename, new_name), verbosity=3)
-                filename = new_name
-            elif re.search(r'/ansible_[^/]+_payload\.zip/ansible/', filename):
-                # Rewrite the module_utils path from the remote host to match the controller. Ansible 2.7 and later.
-                new_name = re.sub(r'^.*/ansible_[^/]+_payload\.zip/ansible/', ansible_path, filename)
-                display.info('%s -> %s' % (filename, new_name), verbosity=3)
-                filename = new_name
-            elif '/ansible_module_' in filename:
-                # Rewrite the module path from the remote host to match the controller. Ansible 2.6 and earlier.
-                module_name = re.sub('^.*/ansible_module_(?P<module>.*).py$', '\\g<module>', filename)
-                if module_name not in modules:
-                    display.warning('Skipping coverage of unknown module: %s' % module_name)
                 continue
-                new_name = os.path.abspath(modules[module_name])
-                display.info('%s -> %s' % (filename, new_name), verbosity=3)
-                filename = new_name
-            elif re.search(r'/ansible_[^/]+_payload(_[^/]+|\.zip)/__main__\.py$', filename):
-                # Rewrite the module path from the remote host to match the controller. Ansible 2.7 and later.
-                # AnsiballZ versions using zipimporter will match the `.zip` portion of the regex.
-                # AnsiballZ versions not using zipimporter will match the `_[^/]+` portion of the regex.
-                module_name = re.sub(r'^.*/ansible_(?P<module>[^/]+)_payload(_[^/]+|\.zip)/__main__\.py$', '\\g<module>', filename).rstrip('_')
-                if module_name not in modules:
-                    display.warning('Skipping coverage of unknown module: %s' % module_name)
-                    continue
-                new_name = os.path.abspath(modules[module_name])
-                display.info('%s -> %s' % (filename, new_name), verbosity=3)
-                filename = new_name
-            elif re.search('^(/.*?)?/root/ansible/', filename):
-                # Rewrite the path of code running on a remote host or in a docker container as root.
-                new_name = re.sub('^(/.*?)?/root/ansible/', root_path, filename)
-                display.info('%s -> %s' % (filename, new_name), verbosity=3)
-                filename = new_name
-            elif '/.ansible/test/tmp/' in filename:
-                # Rewrite the path of code running from an integration test temporary directory.
-                new_name = re.sub(r'^.*/\.ansible/test/tmp/[^/]+/', root_path, filename)
-                display.info('%s -> %s' % (filename, new_name), verbosity=3)
-                filename = new_name
 
             if group not in groups:
                 groups[group] = {}
@@ -221,6 +177,127 @@ def command_coverage_combine(args):
     return sorted(output_files)
 
 
+def _get_coverage_targets(args, walk_func):
+    """
+    :type args: CoverageConfig
+    :type walk_func: Func
+    :rtype: list[tuple[str, int]]
+    """
+    sources = []
+
+    if args.all or args.stub:
+        # excludes symlinks of regular files to avoid reporting on the same file multiple times
+        # in the future it would be nice to merge any coverage for symlinks into the real files
+        for target in walk_func(include_symlinks=False):
+            target_path = os.path.abspath(target.path)
+
+            with open(target_path, 'r') as target_fd:
+                target_lines = len(target_fd.read().splitlines())
+
+            sources.append((target_path, target_lines))
+
+        sources.sort()
+
+    return sources
+
+
+def _build_stub_groups(args, sources, default_stub_value):
+    """
+    :type args: CoverageConfig
+    :type sources: List[tuple[str, int]]
+    :type default_stub_value: Func[int]
+    :rtype: dict
+    """
+    groups = {}
+
+    if args.stub:
+        stub_group = []
+        stub_groups = [stub_group]
+        stub_line_limit = 500000
+        stub_line_count = 0
+
+        for source, source_line_count in sources:
+            stub_group.append((source, source_line_count))
+            stub_line_count += source_line_count
+
+            if stub_line_count > stub_line_limit:
+                stub_line_count = 0
+                stub_group = []
+                stub_groups.append(stub_group)
+
+        for stub_index, stub_group in enumerate(stub_groups):
+            if not stub_group:
+                continue
+
+            groups['=stub-%02d' % (stub_index + 1)] = dict((source, default_stub_value(line_count))
+                                                           for source, line_count in stub_group)
+
+    return groups
+
+
+def _sanitise_filename(filename, modules=None, collection_search_re=None, collection_sub_re=None):
+    """
+    :type filename: str
+    :type modules: dict | None
+    :type collection_search_re: Pattern | None
+    :type collection_sub_re: Pattern | None
+    :rtype: str | None
+    """
+    ansible_path = os.path.abspath('lib/ansible/') + '/'
+    root_path = data_context().content.root + '/'
+
+    if modules is None:
+        modules = {}
+
+    if '/ansible_modlib.zip/ansible/' in filename:
+        # Rewrite the module_utils path from the remote host to match the controller. Ansible 2.6 and earlier.
+        new_name = re.sub('^.*/ansible_modlib.zip/ansible/', ansible_path, filename)
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+    elif collection_search_re and collection_search_re.search(filename):
+        new_name = os.path.abspath(collection_sub_re.sub('', filename))
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+    elif re.search(r'/ansible_[^/]+_payload\.zip/ansible/', filename):
+        # Rewrite the module_utils path from the remote host to match the controller. Ansible 2.7 and later.
+        new_name = re.sub(r'^.*/ansible_[^/]+_payload\.zip/ansible/', ansible_path, filename)
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+    elif '/ansible_module_' in filename:
+        # Rewrite the module path from the remote host to match the controller. Ansible 2.6 and earlier.
+        module_name = re.sub('^.*/ansible_module_(?P<module>.*).py$', '\\g<module>', filename)
+        if module_name not in modules:
+            display.warning('Skipping coverage of unknown module: %s' % module_name)
+            return None
+        new_name = os.path.abspath(modules[module_name])
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+    elif re.search(r'/ansible_[^/]+_payload(_[^/]+|\.zip)/__main__\.py$', filename):
+        # Rewrite the module path from the remote host to match the controller. Ansible 2.7 and later.
+        # AnsiballZ versions using zipimporter will match the `.zip` portion of the regex.
+        # AnsiballZ versions not using zipimporter will match the `_[^/]+` portion of the regex.
+        module_name = re.sub(r'^.*/ansible_(?P<module>[^/]+)_payload(_[^/]+|\.zip)/__main__\.py$',
+                             '\\g<module>', filename).rstrip('_')
+        if module_name not in modules:
+            display.warning('Skipping coverage of unknown module: %s' % module_name)
+            return None
+        new_name = os.path.abspath(modules[module_name])
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+    elif re.search('^(/.*?)?/root/ansible/', filename):
+        # Rewrite the path of code running on a remote host or in a docker container as root.
+        new_name = re.sub('^(/.*?)?/root/ansible/', root_path, filename)
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+    elif '/.ansible/test/tmp/' in filename:
+        # Rewrite the path of code running from an integration test temporary directory.
+        new_name = re.sub(r'^.*/\.ansible/test/tmp/[^/]+/', root_path, filename)
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+
+    return filename
+
+
 def command_coverage_report(args):
     """
     :type args: CoverageReportConfig
@@ -231,6 +308,9 @@ def command_coverage_report(args):
         if args.group_by or args.stub:
             display.info('>>> Coverage Group: %s' % ' '.join(os.path.basename(output_file).split('=')[1:]))
 
+        if output_file.endswith('-powershell'):
+            display.info(_generate_powershell_output_report(args, output_file))
+        else:
             options = []
 
             if args.show_missing:
@@ -254,6 +334,11 @@ def command_coverage_html(args):
     output_files = command_coverage_combine(args)
 
     for output_file in output_files:
+        if output_file.endswith('-powershell'):
+            # coverage.py does not support non-Python files so we just skip the local html report.
+            display.info("Skipping output file %s in html generation" % output_file, verbosity=3)
+            continue
+
         dir_name = os.path.join(data_context().results, 'reports', os.path.basename(output_file))
         env = common_environment()
         env.update(dict(COVERAGE_FILE=output_file))
@@ -268,6 +353,16 @@ def command_coverage_xml(args):
 
     for output_file in output_files:
         xml_name = os.path.join(data_context().results, 'reports', '%s.xml' % os.path.basename(output_file))
+        if output_file.endswith('-powershell'):
+            report = _generage_powershell_xml(output_file)
+
+            rough_string = tostring(report, 'utf-8')
+            reparsed = minidom.parseString(rough_string)
+            pretty = reparsed.toprettyxml(indent='    ')
+
+            with open(xml_name, 'w') as xml_fd:
+                xml_fd.write(pretty)
+        else:
             env = common_environment()
             env.update(dict(COVERAGE_FILE=output_file))
             run_command(args, env=env, cmd=['coverage', 'xml', '--rcfile', COVERAGE_CONFIG_PATH, '-i', '-o', xml_name])
@@ -338,3 +433,326 @@ def get_coverage_group(args, coverage_file):
             group += '=%s' % names[part]
 
     return group
+
+
+def _command_coverage_combine_powershell(args):
+    """
+    :type args: CoverageConfig
+    :rtype: list[str]
+    """
+    coverage_dir = os.path.join(data_context().results, 'coverage')
+    coverage_files = [os.path.join(coverage_dir, f) for f in os.listdir(coverage_dir)
+                      if '=coverage.' in f and '=powershell' in f]
+
+    def _default_stub_value(line_count):
+        val = {}
+        for line in range(line_count):
+            val[line] = 0
+        return val
+
+    counter = 0
+    sources = _get_coverage_targets(args, walk_powershell_targets)
+    groups = _build_stub_groups(args, sources, _default_stub_value)
+
+    for coverage_file in coverage_files:
+        counter += 1
+        display.info('[%4d/%4d] %s' % (counter, len(coverage_files), coverage_file), verbosity=2)
+
+        group = get_coverage_group(args, coverage_file)
+
+        if group is None:
+            display.warning('Unexpected name for coverage file: %s' % coverage_file)
+            continue
+
+        if os.path.getsize(coverage_file) == 0:
+            display.warning('Empty coverage file: %s' % coverage_file)
+            continue
+
+        try:
+            with open(coverage_file, 'rb') as original_fd:
+                coverage_run = json.loads(to_text(original_fd.read(), errors='replace'))
+        except Exception as ex:  # pylint: disable=locally-disabled, broad-except
+            display.error(u'%s' % ex)
+            continue
+
+        for filename, hit_info in coverage_run.items():
+            if group not in groups:
+                groups[group] = {}
+
+            coverage_data = groups[group]
+
+            filename = _sanitise_filename(filename)
+            if not filename:
+                continue
+
+            if filename not in coverage_data:
+                coverage_data[filename] = {}
+
+            file_coverage = coverage_data[filename]
+
+            if not isinstance(hit_info, list):
+                hit_info = [hit_info]
+
+            for hit_entry in hit_info:
+                if not hit_entry:
+                    continue
+
+                line_count = file_coverage.get(hit_entry['Line'], 0) + hit_entry['HitCount']
+                file_coverage[hit_entry['Line']] = line_count
+
+    output_files = []
+    invalid_path_count = 0
+    invalid_path_chars = 0
+
+    coverage_file = os.path.join(data_context().results, 'coverage', 'coverage')
+
+    for group in sorted(groups):
+        coverage_data = groups[group]
+
+        for filename in coverage_data:
+            if not os.path.isfile(filename):
+                invalid_path_count += 1
+                invalid_path_chars += len(filename)
+
+                if args.verbosity > 1:
+                    display.warning('Invalid coverage path: %s' % filename)
+
+                continue
+
+        if args.all:
+            # Add 0 line entries for files not in coverage_data
+            for source, source_line_count in sources:
+                if source in coverage_data:
+                    continue
+
+                coverage_data[source] = _default_stub_value(source_line_count)
+
+        if not args.explain:
+            output_file = coverage_file + group + '-powershell'
+            with open(output_file, 'wb') as output_file_fd:
+                output_file_fd.write(to_bytes(json.dumps(coverage_data)))
+
+            output_files.append(output_file)
+
+    if invalid_path_count > 0:
+        display.warning(
+            'Ignored %d characters from %d invalid coverage path(s).' % (invalid_path_chars, invalid_path_count))
+
+    return sorted(output_files)
+
+
+def _generage_powershell_xml(coverage_file):
+    """
+    :type input_path: str
+    :rtype: Element
+    """
+    with open(coverage_file, 'rb') as coverage_fd:
+        coverage_info = json.loads(to_text(coverage_fd.read()))
+
+    content_root = data_context().content.root
+    is_ansible = data_context().content.is_ansible
+
+    packages = {}
+    for path, results in coverage_info.items():
+        filename = os.path.splitext(os.path.basename(path))[0]
+
+        if filename.startswith('Ansible.ModuleUtils'):
+            package = 'ansible.module_utils'
+        elif is_ansible:
+            package = 'ansible.modules'
+        else:
+            rel_path = path[len(content_root) + 1:]
+            plugin_type = "modules" if rel_path.startswith("plugins/modules") else "module_utils"
+            package = 'ansible_collections.%splugins.%s' % (data_context().content.collection.prefix, plugin_type)
+
+        if package not in packages:
+            packages[package] = {}
+
+        packages[package][path] = results
+
+    elem_coverage = Element('coverage')
+    elem_coverage.append(
+        Comment(' Generated by ansible-test from the Ansible project: https://www.ansible.com/ '))
+    elem_coverage.append(
+        Comment(' Based on https://raw.githubusercontent.com/cobertura/web/master/htdocs/xml/coverage-04.dtd '))
+
+    elem_sources = SubElement(elem_coverage, 'sources')
+
+    elem_source = SubElement(elem_sources, 'source')
+    elem_source.text = data_context().content.root
+
+    elem_packages = SubElement(elem_coverage, 'packages')
+
+    total_lines_hit = 0
+    total_line_count = 0
+
+    for package_name, package_data in packages.items():
+        lines_hit, line_count = _add_cobertura_package(elem_packages, package_name, package_data)
+
+        total_lines_hit += lines_hit
+        total_line_count += line_count
+
+    elem_coverage.attrib.update({
+        'branch-rate': '0',
+        'branches-covered': '0',
+        'branches-valid': '0',
+        'complexity': '0',
+        'line-rate': str(round(total_lines_hit / total_line_count, 4)) if total_line_count else "0",
+        'lines-covered': str(total_line_count),
+        'lines-valid': str(total_lines_hit),
+        'timestamp': str(int(time.time())),
+        'version': get_ansible_version(),
+    })
+
+    return elem_coverage
+
+
+def _add_cobertura_package(packages, package_name, package_data):
+    """
+    :type packages: SubElement
+    :type package_name: str
+    :type package_data: Dict[str, Dict[str, int]]
+    :rtype: Tuple[int, int]
+    """
+    elem_package = SubElement(packages, 'package')
+    elem_classes = SubElement(elem_package, 'classes')
+
+    total_lines_hit = 0
+    total_line_count = 0
+
+    for path, results in package_data.items():
+        lines_hit = len([True for hits in results.values() if hits])
+        line_count = len(results)
+
+        total_lines_hit += lines_hit
+        total_line_count += line_count
+
+        elem_class = SubElement(elem_classes, 'class')
+
+        class_name = os.path.splitext(os.path.basename(path))[0]
+        if class_name.startswith("Ansible.ModuleUtils"):
+            class_name = class_name[20:]
+
+        content_root = data_context().content.root
+        filename = path
+        if filename.startswith(content_root):
+            filename = filename[len(content_root) + 1:]
+
+        elem_class.attrib.update({
+            'branch-rate': '0',
+            'complexity': '0',
+            'filename': filename,
+            'line-rate': str(round(lines_hit / line_count, 4)) if line_count else "0",
+            'name': class_name,
+        })
+
+        SubElement(elem_class, 'methods')
+
+        elem_lines = SubElement(elem_class, 'lines')
+
+        for number, hits in results.items():
+            elem_line = SubElement(elem_lines, 'line')
+            elem_line.attrib.update(
+                hits=str(hits),
+                number=str(number),
+            )
+
+    elem_package.attrib.update({
+        'branch-rate': '0',
+        'complexity': '0',
+        'line-rate': str(round(total_lines_hit / total_line_count, 4)) if total_line_count else "0",
+        'name': package_name,
+    })
+
+    return total_lines_hit, total_line_count
+
+
+def _generate_powershell_output_report(args, coverage_file):
+    """
+    :type args: CoverageConfig
+    :type coverage_file: str
+    :rtype: str
+    """
+    with open(coverage_file, 'rb') as coverage_fd:
+        coverage_info = json.loads(to_text(coverage_fd.read()))
+
+    root_path = data_context().content.root + '/'
+
+    name_padding = 7
+    cover_padding = 8
+
+    file_report = []
+    total_stmts = 0
+    total_miss = 0
+
+    for filename in sorted(coverage_info.keys()):
+        hit_info = coverage_info[filename]
+
+        if filename.startswith(root_path):
+            filename = filename[len(root_path):]
+
+        if args.omit and filename in args.omit:
+            continue
+        if args.include and filename not in args.include:
+            continue
+
+        stmts = len(hit_info)
+        miss = len([c for c in hit_info.values() if c == 0])
+
+        name_padding = max(name_padding, len(filename) + 3)
+
+        total_stmts += stmts
+        total_miss += miss
+
+        cover = "{0}%".format(int((stmts - miss) / stmts * 100))
+
+        missing = []
+        current_missing = None
+        sorted_lines = sorted([int(x) for x in hit_info.keys()])
+        for idx, line in enumerate(sorted_lines):
+            hit = hit_info[str(line)]
+            if hit == 0 and current_missing is None:
+                current_missing = line
+            elif hit != 0 and current_missing is not None:
+                end_line = sorted_lines[idx - 1]
+                if current_missing == end_line:
+                    missing.append(str(current_missing))
+                else:
+                    missing.append('%s-%s' % (current_missing, end_line))
+                current_missing = None
+
+        if current_missing is not None:
+            end_line = sorted_lines[-1]
+            if current_missing == end_line:
+                missing.append(str(current_missing))
+            else:
+                missing.append('%s-%s' % (current_missing, end_line))
+
+        file_report.append({'name': filename, 'stmts': stmts, 'miss': miss, 'cover': cover, 'missing': missing})
+
+    if total_stmts == 0:
+        return ''
+
+    total_percent = '{0}%'.format(int((total_stmts - total_miss) / total_stmts * 100))
+    stmts_padding = max(8, len(str(total_stmts)))
+    miss_padding = max(7, len(str(total_miss)))
+
+    line_length = name_padding + stmts_padding + miss_padding + cover_padding
+
+    header = 'Name'.ljust(name_padding) + 'Stmts'.rjust(stmts_padding) + 'Miss'.rjust(miss_padding) + \
+             'Cover'.rjust(cover_padding)
+
+    if args.show_missing:
+        header += 'Lines Missing'.rjust(16)
+        line_length += 16
+
+    line_break = '-' * line_length
+    lines = ['%s%s%s%s%s' % (f['name'].ljust(name_padding), str(f['stmts']).rjust(stmts_padding),
+                             str(f['miss']).rjust(miss_padding), f['cover'].rjust(cover_padding),
+                             '   ' + ', '.join(f['missing']) if args.show_missing else '')
+             for f in file_report]
+    totals = 'TOTAL'.ljust(name_padding) + str(total_stmts).rjust(stmts_padding) + \
+             str(total_miss).rjust(miss_padding) + total_percent.rjust(cover_padding)
+
+    report = '{0}\n{1}\n{2}\n{1}\n{3}'.format(header, line_break, "\n".join(lines), totals)
+    return report

test/lib/ansible_test/_internal/executor.py

@@ -62,6 +62,8 @@ from .util import (
     ANSIBLE_TEST_DATA_ROOT,
     ANSIBLE_TEST_CONFIG_ROOT,
     get_ansible_version,
+    tempdir,
+    open_zipfile,
 )
 
 from .util_common import (
@@ -679,16 +681,43 @@ def command_windows_integration(args):
             pre_target = forward_ssh_ports
             post_target = cleanup_ssh_ports
 
+    def run_playbook(playbook, playbook_vars):
+        playbook_path = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'playbooks', playbook)
+        command = ['ansible-playbook', '-i', inventory_path, playbook_path, '-e', json.dumps(playbook_vars)]
+        if args.verbosity:
+            command.append('-%s' % ('v' * args.verbosity))
+
+        env = ansible_environment(args)
+        intercept_command(args, command, '', env, disable_coverage=True)
+
+    remote_temp_path = None
+
+    if args.coverage and not args.coverage_check:
+        # Create the remote directory that is writable by everyone. Use Ansible to talk to the remote host.
+        remote_temp_path = 'C:\\ansible_test_coverage_%s' % time.time()
+        playbook_vars = {'remote_temp_path': remote_temp_path}
+        run_playbook('windows_coverage_setup.yml', playbook_vars)
+
     success = False
 
     try:
         command_integration_filtered(args, internal_targets, all_targets, inventory_path, pre_target=pre_target,
-                                     post_target=post_target)
+                                     post_target=post_target, remote_temp_path=remote_temp_path)
         success = True
     finally:
         if httptester_id:
             docker_rm(args, httptester_id)
 
+        if remote_temp_path:
+            # Zip up the coverage files that were generated and fetch it back to localhost.
+            with tempdir() as local_temp_path:
+                playbook_vars = {'remote_temp_path': remote_temp_path, 'local_temp_path': local_temp_path}
+                run_playbook('windows_coverage_teardown.yml', playbook_vars)
+
+                for filename in os.listdir(local_temp_path):
+                    with open_zipfile(os.path.join(local_temp_path, filename)) as coverage_zip:
+                        coverage_zip.extractall(os.path.join(data_context().results, 'coverage'))
+
         if args.remote_terminate == 'always' or (args.remote_terminate == 'success' and success):
             for instance in instances:
                 instance.result.stop()
@@ -878,7 +907,8 @@ def command_integration_filter(args,  # type: TIntegrationConfig
     return internal_targets
 
 
-def command_integration_filtered(args, targets, all_targets, inventory_path, pre_target=None, post_target=None):
+def command_integration_filtered(args, targets, all_targets, inventory_path, pre_target=None, post_target=None,
+                                 remote_temp_path=None):
     """
     :type args: IntegrationConfig
     :type targets: tuple[IntegrationTarget]
@@ -886,6 +916,7 @@ def command_integration_filtered(args, targets, all_targets, inventory_path, pre
     :type inventory_path: str
     :type pre_target: (IntegrationTarget) -> None | None
     :type post_target: (IntegrationTarget) -> None | None
+    :type remote_temp_path: str | None
     """
     found = False
     passed = []
@@ -986,9 +1017,11 @@ def command_integration_filtered(args, targets, all_targets, inventory_path, pre
 
                         try:
                             if target.script_path:
-                                command_integration_script(args, target, test_dir, inventory_path, common_temp_path)
+                                command_integration_script(args, target, test_dir, inventory_path, common_temp_path,
+                                                           remote_temp_path=remote_temp_path)
                             else:
-                                command_integration_role(args, target, start_at_task, test_dir, inventory_path, common_temp_path)
+                                command_integration_role(args, target, start_at_task, test_dir, inventory_path,
+                                                         common_temp_path, remote_temp_path=remote_temp_path)
                                 start_at_task = None
                         finally:
                             if post_target:
@@ -1275,13 +1308,14 @@ def integration_environment(args, target, test_dir, inventory_path, ansible_conf
     return env
 
 
-def command_integration_script(args, target, test_dir, inventory_path, temp_path):
+def command_integration_script(args, target, test_dir, inventory_path, temp_path, remote_temp_path=None):
     """
     :type args: IntegrationConfig
     :type target: IntegrationTarget
     :type test_dir: str
     :type inventory_path: str
     :type temp_path: str
+    :type remote_temp_path: str | None
     """
     display.info('Running %s integration test script' % target.name)
 
@@ -1310,10 +1344,11 @@ def command_integration_script(args, target, test_dir, inventory_path, temp_path
                 cmd += ['-e', '@%s' % config_path]
 
             module_coverage = 'non_local/' not in target.aliases
-            intercept_command(args, cmd, target_name=target.name, env=env, cwd=cwd, temp_path=temp_path, module_coverage=module_coverage)
+            intercept_command(args, cmd, target_name=target.name, env=env, cwd=cwd, temp_path=temp_path,
+                              remote_temp_path=remote_temp_path, module_coverage=module_coverage)
 
 
-def command_integration_role(args, target, start_at_task, test_dir, inventory_path, temp_path):
+def command_integration_role(args, target, start_at_task, test_dir, inventory_path, temp_path, remote_temp_path=None):
     """
     :type args: IntegrationConfig
     :type target: IntegrationTarget
@@ -1321,6 +1356,7 @@ def command_integration_role(args, target, start_at_task, test_dir, inventory_pa
     :type test_dir: str
     :type inventory_path: str
     :type temp_path: str
+    :type remote_temp_path: str | None
     """
     display.info('Running %s integration test role' % target.name)
 
@@ -1406,7 +1442,8 @@ def command_integration_role(args, target, start_at_task, test_dir, inventory_pa
             env['ANSIBLE_ROLES_PATH'] = os.path.abspath(os.path.join(test_env.integration_dir, 'targets'))
 
             module_coverage = 'non_local/' not in target.aliases
-            intercept_command(args, cmd, target_name=target.name, env=env, cwd=cwd, temp_path=temp_path, module_coverage=module_coverage)
+            intercept_command(args, cmd, target_name=target.name, env=env, cwd=cwd, temp_path=temp_path,
+                              remote_temp_path=remote_temp_path, module_coverage=module_coverage)
 
 
 def get_changes_filter(args):

test/lib/ansible_test/_internal/target.py

@@ -180,6 +180,13 @@ def walk_compile_targets(include_symlinks=True):
     return walk_test_targets(module_path=data_context().content.module_path, extensions=('.py',), extra_dirs=('bin',), include_symlinks=include_symlinks)
 
 
+def walk_powershell_targets(include_symlinks=True):
+    """
+    :rtype: collections.Iterable[TestTarget]
+    """
+    return walk_test_targets(module_path=data_context().content.module_path, extensions=('.ps1', '.psm1'), include_symlinks=include_symlinks)
+
+
 def walk_sanity_targets():
     """
     :rtype: collections.Iterable[TestTarget]

test/lib/ansible_test/_internal/util.py

@@ -16,7 +16,9 @@ import stat
 import string
 import subprocess
 import sys
+import tempfile
 import time
+import zipfile
 
 from struct import unpack, pack
 from termios import TIOCGWINSZ
@@ -894,4 +896,20 @@ def load_module(path, name):  # type: (str, str) -> None
             imp.load_module(name, module_file, path, ('.py', 'r', imp.PY_SOURCE))
 
 
+@contextlib.contextmanager
+def tempdir():
+    """Creates a temporary directory that is deleted outside the context scope."""
+    temp_path = tempfile.mkdtemp()
+    yield temp_path
+    shutil.rmtree(temp_path)
+
+
+@contextlib.contextmanager
+def open_zipfile(path, mode='r'):
+    """Opens a zip file and closes the file automatically."""
+    zib_obj = zipfile.ZipFile(path, mode=mode)
+    yield zib_obj
+    zib_obj.close()
+
+
 display = Display()  # pylint: disable=locally-disabled, invalid-name

test/lib/ansible_test/_internal/util_common.py

@@ -148,13 +148,14 @@ def cleanup_python_paths():
         shutil.rmtree(path)
 
 
-def get_coverage_environment(args, target_name, version, temp_path, module_coverage):
+def get_coverage_environment(args, target_name, version, temp_path, module_coverage, remote_temp_path=None):
     """
     :type args: TestConfig
     :type target_name: str
     :type version: str
     :type temp_path: str
     :type module_coverage: bool
+    :type remote_temp_path: str | None
     :rtype: dict[str, str]
     """
     if temp_path:
@@ -199,11 +200,18 @@ def get_coverage_environment(args, target_name, version, temp_path, module_cover
             _ANSIBLE_COVERAGE_OUTPUT=coverage_file,
         ))
 
+        if remote_temp_path:
+            # Include the command, target and label so the remote host can create a filename with that info. The remote
+            # is responsible for adding '={language version}=coverage.{hostname}.{pid}.{id}'
+            env['_ANSIBLE_COVERAGE_REMOTE_OUTPUT'] = os.path.join(remote_temp_path, '%s=%s=%s' % (
+                args.command, target_name, args.coverage_label or 'remote'))
+            env['_ANSIBLE_COVERAGE_REMOTE_WHITELIST'] = os.path.join(data_context().content.root, '*')
+
     return env
 
 
 def intercept_command(args, cmd, target_name, env, capture=False, data=None, cwd=None, python_version=None, temp_path=None, module_coverage=True,
-                      virtualenv=None, disable_coverage=False):
+                      virtualenv=None, disable_coverage=False, remote_temp_path=None):
     """
     :type args: TestConfig
     :type cmd: collections.Iterable[str]
@@ -217,6 +225,7 @@ def intercept_command(args, cmd, target_name, env, capture=False, data=None, cwd
     :type module_coverage: bool
     :type virtualenv: str | None
     :type disable_coverage: bool
+    :type remote_temp_path: str | None
     :rtype: str | None, str | None
     """
     if not env:
@@ -239,7 +248,8 @@ def intercept_command(args, cmd, target_name, env, capture=False, data=None, cwd
 
     if args.coverage and not disable_coverage:
         # add the necessary environment variables to enable code coverage collection
-        env.update(get_coverage_environment(args, target_name, version, temp_path, module_coverage))
+        env.update(get_coverage_environment(args, target_name, version, temp_path, module_coverage,
+                                            remote_temp_path=remote_temp_path))
 
     return run_command(args, cmd, capture=capture, env=env, data=data, cwd=cwd)
 

test/utils/shippable/shippable.sh

@@ -96,6 +96,7 @@ function cleanup
         if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then
             for file in test/results/reports/coverage=*.xml; do
                 flags="${file##*/coverage=}"
+                flags="${flags%-powershell.xml}"
                 flags="${flags%.xml}"
                 # remove numbered component from stub files when converting to tags
                 flags="${flags//stub-[0-9]*/stub}"