From ade85b4faa2f975d50c503aa606e12bf48d58f63 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 10 May 2019 10:56:12 -0700 Subject: [PATCH] Multiple test fixes and improved logging for fragile tests (#9569) --- test/powershell/Host/ConsoleHost.Tests.ps1 | 22 ++++++++++-------- test/powershell/Host/Read-Host.Tests.ps1 | 8 ++++++- .../Interop/DotNet/DotNetAPI.Tests.ps1 | 3 ++- .../FileSystem.Tests.ps1 | 3 ++- .../ExecutionPolicy.Tests.ps1 | 23 +++++++++++++------ .../Get-Command.Tests.ps1 | 1 - .../MarkdownCmdlets.Tests.ps1 | 16 ++++++++++--- .../engine/Api/BasicEngine.Tests.ps1 | 18 ++++++++------- .../Api/TaskBasedAsyncPowerShellAPI.Tests.ps1 | 6 ++++- .../Help/HelpSystem.OnlineHelp.Tests.ps1 | 2 +- 10 files changed, 69 insertions(+), 33 deletions(-) diff --git a/test/powershell/Host/ConsoleHost.Tests.ps1 b/test/powershell/Host/ConsoleHost.Tests.ps1 index 4e62fb462..17f91942f 100644 --- a/test/powershell/Host/ConsoleHost.Tests.ps1 +++ b/test/powershell/Host/ConsoleHost.Tests.ps1 @@ -772,16 +772,20 @@ public enum ShowWindowCommands : int @{WindowStyle="Maximized"} # hidden doesn't work in CI/Server Core ) { param ($WindowStyle) - $ps = Start-Process pwsh -ArgumentList "-WindowStyle $WindowStyle -noexit -interactive" -PassThru - $startTime = Get-Date - $showCmd = "Unknown" - while (((Get-Date) - $startTime).TotalSeconds -lt 10 -and $showCmd -ne $WindowStyle) - { - Start-Sleep -Milliseconds 100 - $showCmd = ([Test.User32]::GetPlacement($ps.MainWindowHandle)).showCmd + + try { + $ps = Start-Process pwsh -ArgumentList "-WindowStyle $WindowStyle -noexit -interactive" -PassThru + $startTime = Get-Date + $showCmd = "Unknown" + while (((Get-Date) - $startTime).TotalSeconds -lt 10 -and $showCmd -ne $WindowStyle) { + Start-Sleep -Milliseconds 100 + $showCmd = ([Test.User32]::GetPlacement($ps.MainWindowHandle)).showCmd + } + + $showCmd | Should -BeExactly $WindowStyle + } finally { + $ps | Stop-Process -Force } - $showCmd | Should -BeExactly $WindowStyle - $ps | Stop-Process -Force } It "Invalid -WindowStyle returns error" { diff --git a/test/powershell/Host/Read-Host.Tests.ps1 b/test/powershell/Host/Read-Host.Tests.ps1 index 196385efa..b85338ea4 100644 --- a/test/powershell/Host/Read-Host.Tests.ps1 +++ b/test/powershell/Host/Read-Host.Tests.ps1 @@ -12,10 +12,16 @@ Describe "Read-Host" -Tags "Slow","Feature" { } else { $ItArgs = @{ } } + + $expectFile = Join-Path $assetsDir "Read-Host.Output.expect" + + if (-not $IsWindows) { + chmod a+x $expectFile + } } It @ItArgs "Should output correctly" { - & (Join-Path $assetsDir "Read-Host.Output.expect") $powershell | Out-Null + & $expectFile $powershell | Out-Null $LASTEXITCODE | Should -Be 0 } } diff --git a/test/powershell/Language/Interop/DotNet/DotNetAPI.Tests.ps1 b/test/powershell/Language/Interop/DotNet/DotNetAPI.Tests.ps1 index d717a7bfb..be3cf81d1 100644 --- a/test/powershell/Language/Interop/DotNet/DotNetAPI.Tests.ps1 +++ b/test/powershell/Language/Interop/DotNet/DotNetAPI.Tests.ps1 @@ -27,6 +27,7 @@ Describe "DotNetAPI" -Tags "CI" { } It "Should access types in System.Console" { - [System.Console]::TreatControlCAsInput | Should -BeFalse + $type = "System.Console" -as [type] + $type.GetTypeInfo().FullName | Should -BeExactly "System.Console" } } diff --git a/test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1 index c6e1b6b6f..cb5bf66c0 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1 @@ -247,10 +247,11 @@ Describe "Basic FileSystem Provider Tests" -Tags "CI" { $protectedPath = Join-Path ([environment]::GetFolderPath("windows")) "appcompat" "Programs" $protectedPath2 = Join-Path $protectedPath "Install" $newItemPath = Join-Path $protectedPath "foo" + $shouldSkip = -not (Test-Path $protectedPath) } } - It "Access-denied test for " -Skip:(-not $IsWindows) -TestCases @( + It "Access-denied test for " -Skip:(-not $IsWindows -or $shouldSkip) -TestCases @( # NOTE: ensure the fileNameBase parameter is unique for each test case; it is used to generate a unique error and done file name. # The following test does not consistently work on windows # @{cmdline = "Get-Item $protectedPath2 -ErrorAction Stop"; expectedError = "ItemExistsUnauthorizedAccessError,Microsoft.PowerShell.Commands.GetItemCommand"} diff --git a/test/powershell/Modules/Microsoft.PowerShell.Security/ExecutionPolicy.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Security/ExecutionPolicy.Tests.ps1 index 5b22b7216..4e03919ee 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Security/ExecutionPolicy.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Security/ExecutionPolicy.Tests.ps1 @@ -654,7 +654,7 @@ ZoneId=$FileType Test-UnrestrictedExecutionPolicy $testScript $expected } - $error = "UnauthorizedAccess,Microsoft.PowerShell.Commands.ImportModuleCommand" + $expectedError = "UnauthorizedAccess,Microsoft.PowerShell.Commands.ImportModuleCommand" $testData = @( @{ @@ -666,8 +666,9 @@ ZoneId=$FileType if (Test-CanWriteToPsHome) { $testData += @( @{ + shouldMarkAsPending = $true module = $PSHomeUntrustedModule - error = $null + expectedError = $expectedError } @{ module = $PSHomeUnsignedModule @@ -678,15 +679,23 @@ ZoneId=$FileType $TestTypePrefix = "Test 'Unrestricted' execution policy." It "$TestTypePrefix Importing Module should throw ''" -TestCases $testData { - param([string]$module, [string]$error) - $testScript = {Import-Module -Name $module -Force} - if($error) + param([string]$module, [string]$expectedError, [bool]$shouldMarkAsPending) + + if ($shouldMarkAsPending) { - $testScript | Should -Throw -ErrorId $error + Set-ItResult -Pending -Because "Test is unreliable" + } + + $execPolicy = Get-ExecutionPolicy -List | Out-String + + $testScript = {Import-Module -Name $module -Force -ErrorAction Stop} + if($expectedError) + { + $testScript | Should -Throw -ErrorId $expectedError -Because "Untrusted modules should not be loaded even on unrestricted execution policy" } else { - {& $testScript} | Should -Not -Throw + $testScript | Should -Not -Throw -Because "Execution Policy is set as: $execPolicy" } } } diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Get-Command.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Get-Command.Tests.ps1 index c16fcffe5..fe46bc3b5 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Get-Command.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Get-Command.Tests.ps1 @@ -74,7 +74,6 @@ Describe "Get-Command Feature tests" -Tag Feature { It "Can return multiple results for cmdlets matching abbreviation" { # use mixed casing to validate case insensitivity $results = pwsh -outputformat xml -settingsfile $configFilePath -command "Get-Command i-C -UseAbbreviationExpansion" - $results.Count | Should -BeGreaterOrEqual 3 $results.Name | Should -Contain "Invoke-Command" $results.Name | Should -Contain "Import-Clixml" $results.Name | Should -Contain "Import-Csv" diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/MarkdownCmdlets.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/MarkdownCmdlets.Tests.ps1 index 431bc4929..96ad7cbd1 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/MarkdownCmdlets.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/MarkdownCmdlets.Tests.ps1 @@ -6,6 +6,8 @@ Describe 'ConvertFrom-Markdown tests' -Tags 'CI' { BeforeAll { $esc = [char]0x1b + $hostSupportsVT100 = $Host.UI.SupportsVirtualTerminal + function GetExpectedString { [CmdletBinding()] @@ -27,6 +29,10 @@ Describe 'ConvertFrom-Markdown tests' -Tags 'CI' { [bool] $VT100Support ) + # Force VT100Support to be false if the host does not support it. + # This makes the expected string to be correct. + $VT100Support = $VT100Support -and $hostSupportsVT100 + switch($elementType) { "Header1" { if($VT100Support) {"$esc[7m$text$esc[0m`n`n" } else {"$text`n`n"} } @@ -125,7 +131,11 @@ Describe 'ConvertFrom-Markdown tests' -Tags 'CI' { BeforeAll { $mdFile = New-Item -Path $TestDrive/input.md -Value "Some **test string** to write in a file" -Force $mdLiteralPath = New-Item -Path $TestDrive/LiteralPath.md -Value "Some **test string** to write in a file" -Force - $expectedStringFromFile = "Some $esc[1mtest string$esc[0m to write in a file`n`n" + $expectedStringFromFile = if ($hostSupportsVT100) { + "Some $esc[1mtest string$esc[0m to write in a file`n`n" + } else { + "Some test string to write in a file`n`n" + } $codeBlock = @' ``` @@ -294,8 +304,8 @@ bool function()`n{`n} @{Type = "Header4"; Markdown = "#### "; ExpectedOutput = ''} @{Type = "Header5"; Markdown = "##### "; ExpectedOutput = ''} @{Type = "Header6"; Markdown = "###### "; ExpectedOutput = ''} - @{Type = "Image"; Markdown = "'![]()'"; ExpectedOutput = "'$esc[33m[Image]$esc[0m'"} - @{Type = "Link"; Markdown = "'[]()'"; ExpectedOutput = "'$esc[4;38;5;117m`"`"$esc[0m'"} + @{Type = "Image"; Markdown = "'![]()'"; ExpectedOutput = if ($hostSupportsVT100) {"'$esc[33m[Image]$esc[0m'"} else {"'[Image]'"}} + @{Type = "Link"; Markdown = "'[]()'"; ExpectedOutput = if ($hostSupportsVT100) {"'$esc[4;38;5;117m`"`"$esc[0m'"} else {"'`"`"'"}} ) } diff --git a/test/powershell/engine/Api/BasicEngine.Tests.ps1 b/test/powershell/engine/Api/BasicEngine.Tests.ps1 index ddc971558..ccdb354a8 100644 --- a/test/powershell/engine/Api/BasicEngine.Tests.ps1 +++ b/test/powershell/engine/Api/BasicEngine.Tests.ps1 @@ -45,19 +45,21 @@ $rs.Open() $ps = [powershell]::Create() $ps.RunspacePool = $rs $null = $ps.AddScript(1).Invoke() -write-host should_not_stop_responding_at_exit +"should_not_stop_responding_at_exit" exit '@ - $process = Start-Process pwsh -ArgumentList $command -PassThru + $outputFile = New-Item -Path $TestDrive\output.txt -ItemType File + $process = Start-Process pwsh -ArgumentList $command -PassThru -RedirectStandardOutput $outputFile Wait-UntilTrue -sb { $process.HasExited } -TimeoutInMilliseconds 5000 -IntervalInMilliseconds 1000 | Should -BeTrue + $hasExited = $process.HasExited - $expect = "powershell process exits in 5 seconds" - if (-not $process.HasExited) { - Stop-Process -InputObject $process -Force -ErrorAction SilentlyContinue - "powershell process doesn't exit in 5 seconds" | Should -Be $expect - } else { - $expect | Should -Be $expect + $verboseMessage = Get-Content $outputFile + + if (-not $hasExited) { + Stop-Process $process -Force } + + $hasExited | Should -BeTrue -Because "Process did not exit in 5 seconds as: $verboseMessage" } } diff --git a/test/powershell/engine/Api/TaskBasedAsyncPowerShellAPI.Tests.ps1 b/test/powershell/engine/Api/TaskBasedAsyncPowerShellAPI.Tests.ps1 index f21293d38..be08c60fb 100644 --- a/test/powershell/engine/Api/TaskBasedAsyncPowerShellAPI.Tests.ps1 +++ b/test/powershell/engine/Api/TaskBasedAsyncPowerShellAPI.Tests.ps1 @@ -207,10 +207,14 @@ try { try { $ir = $ps.AddScript("Start-Sleep -Seconds 60").InvokeAsync() Wait-UntilTrue { $ps.InvocationStateInfo.State -eq [System.Management.Automation.PSInvocationState]::Running } | Should -BeTrue + $ps.InvocationStateInfo.State | Should -Be 'Running' + Start-Sleep -Seconds 1 # add a sleep to wait for pipeline to start executing the command. $sr = $ps.StopAsync($null, $null) [System.Threading.Tasks.Task]::WaitAll(@($sr)) + $ps.Streams.Error | Should -HaveCount 0 -Because ($ps.Streams.Error | Out-String) + $ps.Commands.Commands.commandtext | Should -Be "Start-Sleep -Seconds 60" $sr.IsCompletedSuccessfully | Should -Be $true - $ir.IsFaulted | Should -Be $true + $ir.IsFaulted | Should -Be $true -Because ($ir | Format-List -Force * | Out-String) $ir.Exception -is [System.AggregateException] | Should -Be $true $ir.Exception.InnerException -is [System.Management.Automation.PipelineStoppedException] | Should -Be $true $ps.InvocationStateInfo.State | Should -Be ([System.Management.Automation.PSInvocationState]::Stopped) diff --git a/test/powershell/engine/Help/HelpSystem.OnlineHelp.Tests.ps1 b/test/powershell/engine/Help/HelpSystem.OnlineHelp.Tests.ps1 index f4c001499..8acfad221 100644 --- a/test/powershell/engine/Help/HelpSystem.OnlineHelp.Tests.ps1 +++ b/test/powershell/engine/Help/HelpSystem.OnlineHelp.Tests.ps1 @@ -49,7 +49,7 @@ Describe 'Get-Help -Online opens the default web browser and navigates to the cm $skipTest = [System.Management.Automation.Platform]::IsIoT -or [System.Management.Automation.Platform]::IsNanoServer -or - $env:__InContainer -eq 1 + $env:__INCONTAINER -eq 1 # this code is a workaround for issue: https://github.com/PowerShell/PowerShell/issues/3079 if((-not ($skipTest)) -and $IsWindows)