maxmustermann/PowerShell
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Update CI scripts to support running tests for experimental features (#7419)

Browse source
This commit is contained in:
Dongbo Wang 2018-08-03 11:20:14 -07:00 committed by GitHub
parent ac8ac4f25b
commit 25c127c3bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 203 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
build.psm1
View file

@ -973,6 +973,18 @@ function Publish-PSTestTools {
}
}
function Get-ExperimentalFeatureTests {
$testMetadataFile = Join-Path $PSScriptRoot "test/tools/TestMetadata.json"
$metadata = Get-Content -Path $testMetadataFile -Raw | ConvertFrom-Json | ForEach-Object -MemberName ExperimentalFeatures
$features = $metadata | Get-Member -MemberType NoteProperty | ForEach-Object -MemberName Name
$featureTests = @{}
foreach ($featureName in $features) {
$featureTests[$featureName] = $metadata.$featureName
}
$featureTests
}
function Start-PSPester {
[CmdletBinding(DefaultParameterSetName='default')]
param(
@ -983,9 +995,9 @@ function Start-PSPester {
[string[]]$ExcludeTag = 'Slow',
[string[]]$Tag = @("CI","Feature"),
[switch]$ThrowOnFailure,
[string]$binDir = (Split-Path (Get-PSOptions -DefaultToNew).Output),
[string]$powershell = (Join-Path $binDir 'pwsh'),
[string]$Pester = ([IO.Path]::Combine($binDir, "Modules", "Pester")),
[string]$BinDir = (Split-Path (Get-PSOptions -DefaultToNew).Output),
[string]$powershell = (Join-Path $BinDir 'pwsh'),
[string]$Pester = ([IO.Path]::Combine($BinDir, "Modules", "Pester")),
[Parameter(ParameterSetName='Unelevate',Mandatory=$true)]
[switch]$Unelevate,
[switch]$Quiet,
@ -994,7 +1006,8 @@ function Start-PSPester {
[switch]$PassThru,
[Parameter(ParameterSetName='PassThru',HelpMessage='Run commands on Linux with sudo.')]
[switch]$Sudo,
[switch]$IncludeFailingTest
[switch]$IncludeFailingTest,
[string]$ExperimentalFeatureName
)
if (-not (Get-Module -ListAvailable -Name $Pester -ErrorAction SilentlyContinue | Where-Object { $_.Version -ge "4.2" } ))
@ -1151,6 +1164,37 @@ function Start-PSPester {
}
}
$PSFlags = @("-noprofile")
if (-not [string]::IsNullOrEmpty($ExperimentalFeatureName)) {
$configFile = [System.IO.Path]::GetTempFileName()
$configFile = [System.IO.Path]::ChangeExtension($configFile, ".json")
## Create the config.json file to enable the given experimental feature.
## On Windows, we need to have 'RemoteSigned' declared for ExecutionPolicy because the ExecutionPolicy is 'Restricted' by default.
## On Unix, ExecutionPolicy is not supported, so we don't need to declare it.
if ($Environment.IsWindows) {
$content = @"
{
"Microsoft.PowerShell:ExecutionPolicy":"RemoteSigned",
"ExperimentalFeatures": [
"$ExperimentalFeatureName"
]
}
"@
} else {
$content = @"
{
"ExperimentalFeatures": [
"$ExperimentalFeatureName"
]
}
"@
}
Set-Content -Path $configFile -Value $content -Encoding Ascii -Force
$PSFlags = @("-settings", $configFile, "-noprofile")
}
# To ensure proper testing, the module path must not be inherited by the spawned process
try {
$originalModulePath = $env:PSModulePath
@ -1158,7 +1202,7 @@ function Start-PSPester {
$env:POWERSHELL_TELEMETRY_OPTOUT = 1
if ($Unelevate)
{
Start-UnelevatedProcess -process $powershell -arguments @('-noprofile', '-c', $Command)
Start-UnelevatedProcess -process $powershell -arguments ($PSFlags + "-c $Command")
$currentLines = 0
while ($true)
{
@ -1197,11 +1241,11 @@ function Start-PSPester {
$passThruFile = [System.IO.Path]::GetTempFileName()
try
{
$command += "|Export-Clixml -Path '$passThruFile' -Force"
$command += "| Export-Clixml -Path '$passThruFile' -Force"
$passThruCommand = {& $powershell -noprofile -c $command }
$passThruCommand = { & $powershell $PSFlags -c $command }
if ($Sudo.IsPresent) {
$passThruCommand = {& sudo $powershell -noprofile -c $command }
$passThruCommand = { & sudo $powershell $PSFlags -c $command }
}
$writeCommand = { Write-Host $_ }
@ -1222,11 +1266,11 @@ function Start-PSPester {
{
if ($Terse)
{
Start-NativeExecution -sb {& $powershell -noprofile -c $command} | ForEach-Object { Write-Terse -line $_ }
Start-NativeExecution -sb {& $powershell $PSFlags -c $command} | ForEach-Object { Write-Terse -line $_ }
}
else
{
Start-NativeExecution -sb {& $powershell -noprofile -c $command}
Start-NativeExecution -sb {& $powershell $PSFlags -c $command}
}
}
}
@ -1352,12 +1396,17 @@ function Test-PSPesterResults
[CmdletBinding(DefaultParameterSetName='file')]
param(
[Parameter(ParameterSetName='file')]
[string]$TestResultsFile = "pester-tests.xml",
[string] $TestResultsFile = "pester-tests.xml",
[Parameter(ParameterSetName='file')]
[string]$TestArea = 'test/powershell',
[Parameter(ParameterSetName='PesterPassThruObject',Mandatory)]
[pscustomobject] $ResultObject
)
[string] $TestArea = 'test/powershell',
[Parameter(ParameterSetName='PesterPassThruObject', Mandatory)]
[pscustomobject] $ResultObject,
[Parameter(ParameterSetName='PesterPassThruObject')]
[switch] $CanHaveNoResult
)
if($PSCmdLet.ParameterSetName -eq 'file')
{
@ -1388,7 +1437,7 @@ function Test-PSPesterResults
}
elseif ($PSCmdLet.ParameterSetName -eq 'PesterPassThruObject')
{
if ($ResultObject.TotalCount -le 0)
if ($ResultObject.TotalCount -le 0 -and -not $CanHaveNoResult)
{
throw 'NO TESTS RUN'
}

5
test/tools/TestMetadata.json Normal file
View file

@ -0,0 +1,5 @@
{
"ExperimentalFeatures": {
"ExpTest.FeatureOne": [ "test/powershell/engine/ExperimentalFeature" ]
}
}

73
tools/appveyor.psm1
View file

@ -339,26 +339,67 @@ function Invoke-AppVeyorTest
# Pester doesn't allow Invoke-Pester -TagAll@('CI', 'RequireAdminOnWindows') currently
# https://github.com/pester/Pester/issues/608
# To work-around it, we exlude all categories, but 'CI' from the list
$ExcludeTag = @('Slow', 'Feature', 'Scenario')
if (Test-DailyBuild) {
$ExcludeTag = @()
Write-Host -Foreground Green 'Running all CoreCLR tests..'
}
else {
$ExcludeTag = @('Slow', 'Feature', 'Scenario')
Write-Host -Foreground Green 'Running "CI" CoreCLR tests..'
}
# Get the experimental feature names and the tests associated with them
$ExperimentalFeatureTests = Get-ExperimentalFeatureTests
if ($Purpose -eq 'UnelevatedPesterTests') {
Start-PSPester -Terse -bindir $env:CoreOutput -outputFile $testResultsNonAdminFile -Unelevate -Tag @() -ExcludeTag ($ExcludeTag + @('RequireAdminOnWindows'))
$arguments = @{
Bindir = $env:CoreOutput
OutputFile = $testResultsNonAdminFile
Unelevate = $true
Terse = $true
Tag = @()
ExcludeTag = $ExcludeTag + 'RequireAdminOnWindows'
}
Start-PSPester @arguments
Write-Host -Foreground Green 'Upload CoreCLR Non-Admin test results'
Update-AppVeyorTestResults -resultsFile $testResultsNonAdminFile
# Fail the build, if tests failed
Test-PSPesterResults -TestResultsFile $testResultsNonAdminFile
# Run tests with specified experimental features enabled
foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) {
$featureName = $entry.Key
$testFiles = $entry.Value
$expFeatureTestResultFile = "$pwd\TestsResultsNonAdmin.$featureName.xml"
$arguments['OutputFile'] = $expFeatureTestResultFile
$arguments['ExperimentalFeatureName'] = $featureName
if ($testFiles.Count -eq 0) {
# If an empty array is specified for the feature name, we run all tests with the feature enabled.
# This allows us to prevent regressions to a critical engine experimental feature.
$arguments.Remove('Path')
} else {
# If a non-empty string or array is specified for the feature name, we only run those test files.
$arguments['Path'] = $testFiles
}
Start-PSPester @arguments
Write-Host -ForegroundColor Green "Upload CoreCLR Non-Admin test results for experimental feature '$featureName'"
Update-AppVeyorTestResults -resultsFile $expFeatureTestResultFile
# Fail the build, if tests failed
Test-PSPesterResults -TestResultsFile $expFeatureTestResultFile
}
}
if ($Purpose -eq 'ElevatedPesterTests_xUnit_Packaging') {
Start-PSPester -Terse -bindir $env:CoreOutput -outputFile $testResultsAdminFile -Tag @('RequireAdminOnWindows') -ExcludeTag $ExcludeTag
$arguments = @{
Terse = $true
Bindir = $env:CoreOutput
OutputFile = $testResultsAdminFile
Tag = @('RequireAdminOnWindows')
ExcludeTag = $ExcludeTag
}
Start-PSPester @arguments
Write-Host -Foreground Green 'Upload CoreCLR Admin test results'
Update-AppVeyorTestResults -resultsFile $testResultsAdminFile
@ -375,6 +416,30 @@ function Invoke-AppVeyorTest
) | ForEach-Object {
Test-XUnitTestResults -TestResultsFile $_
}
# Run tests with specified experimental features enabled
foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) {
$featureName = $entry.Key
$testFiles = $entry.Value
$expFeatureTestResultFile = "$pwd\TestsResultsAdmin.$featureName.xml"
$arguments['OutputFile'] = $expFeatureTestResultFile
$arguments['ExperimentalFeatureName'] = $featureName
if ($testFiles.Count -eq 0) {
# If an empty array is specified for the feature name, we run all tests with the feature enabled.
# This allows us to prevent regressions to a critical engine experimental feature.
$arguments.Remove('Path')
} else {
# If a non-empty string or array is specified for the feature name, we only run those test files.
$arguments['Path'] = $testFiles
}
Start-PSPester @arguments
Write-Host -ForegroundColor Green "Upload CoreCLR Admin test results for experimental feature '$featureName'"
Update-AppVeyorTestResults -resultsFile $expFeatureTestResultFile
# Fail the build, if tests failed
Test-PSPesterResults -TestResultsFile $expFeatureTestResultFile
}
}
Set-BuildVariable -Name TestPassed -Value True

79
tools/travis.ps1
View file

@ -204,8 +204,8 @@ elseif($Stage -eq 'Build')
$testResultsNoSudo = "$pwd/TestResultsNoSudo.xml"
$testResultsSudo = "$pwd/TestResultsSudo.xml"
$pesterParam = @{
'binDir' = $output
$noSudoPesterParam = @{
'BinDir' = $output
'PassThru' = $true
'Terse' = $true
'Tag' = @()
@ -214,31 +214,80 @@ elseif($Stage -eq 'Build')
}
if ($isFullBuild) {
$pesterParam['Tag'] = @('CI','Feature','Scenario')
$noSudoPesterParam['Tag'] = @('CI','Feature','Scenario')
} else {
$pesterParam['Tag'] = @('CI')
$pesterParam['ThrowOnFailure'] = $true
$noSudoPesterParam['Tag'] = @('CI')
$noSudoPesterParam['ThrowOnFailure'] = $true
}
if ($hasRunFailingTestTag)
{
$pesterParam['IncludeFailingTest'] = $true
if ($hasRunFailingTestTag) {
$noSudoPesterParam['IncludeFailingTest'] = $true
}
# Get the experimental feature names and the tests associated with them
$ExperimentalFeatureTests = Get-ExperimentalFeatureTests
# Running tests which do not require sudo.
$pesterPassThruNoSudoObject = Start-PSPester @pesterParam
$pesterPassThruNoSudoObject = Start-PSPester @noSudoPesterParam
# Running tests that do not require sudo, with specified experimental features enabled
$noSudoResultsWithExpFeatures = @()
foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) {
$featureName = $entry.Key
$testFiles = $entry.Value
$expFeatureTestResultFile = "$pwd\TestResultsNoSudo.$featureName.xml"
$noSudoPesterParam['OutputFile'] = $expFeatureTestResultFile
$noSudoPesterParam['ExperimentalFeatureName'] = $featureName
if ($testFiles.Count -eq 0) {
# If an empty array is specified for the feature name, we run all tests with the feature enabled.
# This allows us to prevent regressions to a critical engine experimental feature.
$noSudoPesterParam.Remove('Path')
} else {
# If a non-empty string or array is specified for the feature name, we only run those test files.
$noSudoPesterParam['Path'] = $testFiles
}
$passThruResult = Start-PSPester @noSudoPesterParam
$noSudoResultsWithExpFeatures += $passThruResult
}
# Running tests, which require sudo.
$pesterParam['Tag'] = @('RequireSudoOnUnix')
$pesterParam['ExcludeTag'] = @()
$pesterParam['Sudo'] = $true
$pesterParam['OutputFile'] = $testResultsSudo
$pesterPassThruSudoObject = Start-PSPester @pesterParam
$sudoPesterParam = $noSudoPesterParam.Clone()
$sudoPesterParam.Remove('Path')
$sudoPesterParam['Tag'] = @('RequireSudoOnUnix')
$sudoPesterParam['ExcludeTag'] = @()
$sudoPesterParam['Sudo'] = $true
$sudoPesterParam['OutputFile'] = $testResultsSudo
$pesterPassThruSudoObject = Start-PSPester @sudoPesterParam
# Running tests that require sudo, with specified experimental features enabled
$sudoResultsWithExpFeatures = @()
foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) {
$featureName = $entry.Key
$testFiles = $entry.Value
$expFeatureTestResultFile = "$pwd\TestResultsSudo.$featureName.xml"
$sudoPesterParam['OutputFile'] = $expFeatureTestResultFile
$sudoPesterParam['ExperimentalFeatureName'] = $featureName
if ($testFiles.Count -eq 0) {
# If an empty array is specified for the feature name, we run all tests with the feature enabled.
# This allows us to prevent regressions to a critical engine experimental feature.
$sudoPesterParam.Remove('Path')
} else {
# If a non-empty string or array is specified for the feature name, we only run those test files.
$sudoPesterParam['Path'] = $testFiles
}
$passThruResult = Start-PSPester @sudoPesterParam
$sudoResultsWithExpFeatures += $passThruResult
}
# Determine whether the build passed
try {
$allTestResultsWithNoExpFeature = @($pesterPassThruNoSudoObject, $pesterPassThruSudoObject)
$allTestResultsWithExpFeatures = $noSudoResultsWithExpFeatures + $sudoResultsWithExpFeatures
# this throws if there was an error
@($pesterPassThruNoSudoObject, $pesterPassThruSudoObject) | ForEach-Object { Test-PSPesterResults -ResultObject $_ }
$allTestResultsWithNoExpFeature | ForEach-Object { Test-PSPesterResults -ResultObject $_ }
$allTestResultsWithExpFeatures | ForEach-Object { Test-PSPesterResults -ResultObject $_ -CanHaveNoResult }
$result = "PASS"
}
catch {