Add Linux VSTS CI (#7527)

Update Start-PSPester and Start-PSxUnit to upload test results when running in VSTS
Disable tests which require non-root using when running as root
remove verifications in Test-Connection tests which verify buggy behavior
Skip tests which don't behave correctly in VSTS (due to running in a container?)
- Test-Connection - #7528
- Set-Date - #7529
This commit is contained in:
Travis Plunk 2018-08-14 16:23:50 -07:00 committed by GitHub
parent f19c5dae1f
commit 965552873c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 179 additions and 109 deletions

78
.vsts-ci/linux.yml Normal file
View file

@ -0,0 +1,78 @@
name: PR-$(System.PullRequest.PullRequestNumber)-$(Date:yyyyMMdd)$(Rev:.rr)
variables:
DOTNET_CLI_TELEMETRY_OPTOUT: 1
POWERSHELL_TELEMETRY_OPTOUT: 1
# Avoid expensive initialization of dotnet cli, see: http://donovanbrown.com/post/Stop-wasting-time-during-NET-Core-builds
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
# Enable VSTS debug mode until stabilitized
system.debug: 'true'
resources:
- repo: self
clean: true
phases:
- phase: Linux_CI
queue:
name: Hosted Linux Preview
steps:
- powershell: Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhss"))"
displayName: Set Build Name for Non-PR
condition: ne(variables['Build.Reason'], 'PullRequest')
# Several scripts require TMPDIR to exist
# Set it to AGENT_TEMPDIRECTORY so that we know it exists
- powershell: |
Write-Host "##vso[task.setvariable variable=TEMP]$env:AGENT_TEMPDIRECTORY"
displayName: Set TEMP to AGENT_TEMPDIRECTORY
condition: succeededOrFailed()
- powershell: |
git submodule update --init
displayName: SubModule Init
condition: succeededOrFailed()
- powershell: |
apt-get update
apt-get install -y --no-install-recommends less
displayName: Install less
condition: succeededOrFailed()
- powershell: |
tools/travis.ps1 -Stage Bootstrap
displayName: Bootstrap
condition: succeeded()
- powershell: |
$ErrorActionPreference = 'continue'
tools/travis.ps1
displayName: Build and test
condition: succeeded()
- powershell: |
tools/travis.ps1 -Stage Failure
displayName: After Failure
condition: failed()
- powershell: |
tools/travis.ps1 -Stage Success
displayName: After Success
condition: succeeded()
# Uploads any packages as an artifact
- powershell: |
Get-ChildItem -Path *.rpm, *.deb, *.tar.gz -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object {
Write-Host "##vso[artifact.upload containerfolder=artifacts;artifactname=artifacts]$_"
}
displayName: Publish Artifacts
condition: succeededOrFailed()
continueOnError: true
# Uploads any Test results as an artifact
- powershell: |
Get-ChildItem -Path Test*.xml, *XUnitTestResults.xml -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object {
Write-Host "##vso[artifact.upload containerfolder=testResults;artifactname=testResults]$_"
}
displayName: Publish Test Results
condition: succeededOrFailed()
continueOnError: true

View file

@ -32,13 +32,6 @@ phases:
displayName: SubModule Init
condition: succeededOrFailed()
- powershell: |
rvm install ruby-2.3.3;
rvm --default use 2.3.3;
displayName: Install Ruby
condition: succeededOrFailed()
enabled: false
- powershell: |
tools/travis.ps1 -Stage Bootstrap
displayName: Bootstrap
@ -68,74 +61,3 @@ phases:
displayName: Publish Artifacts
condition: succeededOrFailed()
continueOnError: true
# Uploads any Test results as an artifact
- powershell: |
Get-ChildItem -Path Test*.xml, *XUnitTestResults.xml -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object {
Write-Host "##vso[artifact.upload containerfolder=testResults;artifactname=testResults]$_"
}
displayName: Publish Test Results
condition: succeededOrFailed()
continueOnError: true
# Publish the various Test results as Test results to VSTS
# Separate tasks are used because a failure will stop the task
- task: PublishTestResults@2
condition: succeededOrFailed()
displayName: Publish Test Results TestResultsSudo.xml
continueOnError: true
inputs:
testRunner: NUnit
testResultsFiles: '**\TestResultsSudo.xml'
testRunTitle: Sudo
mergeTestResults: true
- task: PublishTestResults@2
condition: succeededOrFailed()
displayName: Publish Test Results TestResultsNoSudo.xml
continueOnError: true
inputs:
testRunner: NUnit
testResultsFiles: '**\TestResultsNoSudo.xml'
testRunTitle: NoSudo
mergeTestResults: true
- task: PublishTestResults@2
condition: succeededOrFailed()
displayName: Publish Test Results TestResultsSudo.ExpTest.*.xml
continueOnError: true
inputs:
testRunner: NUnit
testResultsFiles: '**\TestResultsSudo.ExpTest.*.xml'
testRunTitle: Sudo Experimental Features
mergeTestResults: true
- task: PublishTestResults@2
condition: succeededOrFailed()
displayName: Publish Test Results TestResultsNoSudo.ExpTest.*.xml
continueOnError: true
inputs:
testRunner: NUnit
testResultsFiles: '**\TestResultsNoSudo.ExpTest.*.xml'
testRunTitle: Sudo Experimental Features
mergeTestResults: true
- task: PublishTestResults@2
condition: succeededOrFailed()
displayName: Publish Test Results ParallelXUnitTestResults.xml
continueOnError: true
inputs:
testRunner: XUnit
testResultsFiles: '**\ParallelXUnitTestResults.xml'
testRunTitle: Parallel XUnit
mergeTestResults: true
- task: PublishTestResults@2
condition: succeededOrFailed()
displayName: Publish Test Results SequentialXUnitTestResults.xml
continueOnError: true
inputs:
testRunner: XUnit
testResultsFiles: '**\SequentialXUnitTestResults.xml'
testRunTitle: Sequential XUnit
mergeTestResults: true

View file

@ -1022,7 +1022,9 @@ function Start-PSPester {
[Parameter(ParameterSetName='PassThru',HelpMessage='Run commands on Linux with sudo.')]
[switch]$Sudo,
[switch]$IncludeFailingTest,
[string]$ExperimentalFeatureName
[string]$ExperimentalFeatureName,
[Parameter(HelpMessage='Title to publish the results as.')]
[string]$Title = 'PowerShell Core Tests'
)
if (-not (Get-Module -ListAvailable -Name $Pester -ErrorAction SilentlyContinue | Where-Object { $_.Version -ge "4.2" } ))
@ -1260,7 +1262,8 @@ function Start-PSPester {
$passThruCommand = { & $powershell $PSFlags -c $command }
if ($Sudo.IsPresent) {
$passThruCommand = { & sudo $powershell $PSFlags -c $command }
# -E says to preserve the environment
$passThruCommand = { & sudo -E $powershell $PSFlags -c $command }
}
$writeCommand = { Write-Host $_ }
@ -1298,12 +1301,40 @@ function Start-PSPester {
}
}
Publish-TestResults -Path $OutputFile -Title $Title
if($ThrowOnFailure)
{
Test-PSPesterResults -TestResultsFile $OutputFile
}
}
function Publish-TestResults
{
param(
[Parameter(Mandatory)]
[string]
$Title,
[Parameter(Mandatory)]
[ValidateScript({Test-Path -Path $_})]
[string]
$Path,
[ValidateSet('NUnit','XUnit')]
[string]
$Type='NUnit'
)
# In VSTS publish Test Results
if($env:TF_BUILD)
{
$resolvedPath = (Resolve-Path -Path $Path).ProviderPath
Write-Host "##vso[results.publish type=$Type;mergeResults=true;runTitle=$Title;publishRunAttachments=true;resultFiles=$resolvedPath;]"
Write-Host "##vso[artifact.upload containerfolder=testResults;artifactname=testResults]$resolvedPath"
}
}
function script:Start-UnelevatedProcess
{
param(
@ -1528,6 +1559,9 @@ function Start-PSxUnit {
# Run sequential tests first, and then run the tests that can execute in parallel
dotnet xunit -configuration $Options.configuration -xml $SequentialTestResultsFile -namespace "PSTests.Sequential" -parallel none
Publish-TestResults -Path $SequentialTestResultsFile -Type 'XUnit' -Title 'Xunit Sequential'
$extraParams = @()
# we are having intermittent issues on macOS with these tests failing.
@ -1542,6 +1576,7 @@ function Start-PSxUnit {
}
dotnet xunit -configuration $Options.configuration -xml $ParallelTestResultsFile -namespace "PSTests.Parallel" -nobuild @extraParams
Publish-TestResults -Path $ParallelTestResultsFile -Type 'XUnit' -Title 'Xunit Parallel'
}
finally {
Pop-Location

View file

@ -1,5 +1,8 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
Import-Module HelpersCommon
function Clean-State
{
if (Test-Path $FullyQualifiedLink)
@ -263,7 +266,7 @@ Describe "New-Item with links fails for non elevated user." -Tags "CI" {
$FullyQualifiedFolder = Join-Path -Path $tmpDirectory -ChildPath $testfolder
}
It "Should error correctly when failing to create a symbolic link" {
It "Should error correctly when failing to create a symbolic link" -Skip:(Test-IsRoot) {
# This test expects that /sbin exists but is not writable by the user
{ New-Item -ItemType SymbolicLink -Path "/sbin/powershell-test" -Target $FullyQualifiedFolder -ErrorAction Stop } |
Should -Throw -ErrorId "NewItemSymbolicLinkElevationRequired,Microsoft.PowerShell.Commands.NewItemCommand"

View file

@ -1,6 +1,8 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
Import-Module HelpersCommon
Describe "Test-Connection" -tags "CI" {
BeforeAll {
$oldInformationPreference = $InformationPreference
@ -42,9 +44,7 @@ Describe "Test-Connection" -tags "CI" {
$replies[0].Address | Should -BeExactly $targetAddressIPv6
$replies[0].Status | Should -BeExactly "Success"
# TODO: Here and below we skip the check on Unix because .Net Core issue
if (!$isWindows) {
$replies[0].Buffer.Count | Should -Be 0
} else {
if ($isWindows) {
$replies[0].Buffer.Count | Should -Be 32
}
}
@ -78,19 +78,19 @@ Describe "Test-Connection" -tags "CI" {
}
}
It "Force IPv4 with implicit PingOptions" {
# In VSTS, address is 0.0.0.0
It "Force IPv4 with implicit PingOptions" -Skip:(Test-IsVstsLinux) {
$result = Test-Connection $realName -Count 1 -IPv4
$result.Replies[0].Address | Should -BeExactly $realAddress
$result.Replies[0].Options.Ttl | Should -BeLessThan 128
if (!$isWindows) {
$result.Replies[0].Options.DontFragment | Should -BeNullOrEmpty
} else {
if ($isWindows) {
$result.Replies[0].Options.DontFragment | Should -BeFalse
}
}
It "Force IPv4 with explicit PingOptions" {
# In VSTS, address is 0.0.0.0
It "Force IPv4 with explicit PingOptions" -Skip:(Test-IsVstsLinux) {
$result1 = Test-Connection $realName -Count 1 -IPv4 -MaxHops 10 -DontFragment
$result2 = Test-Connection $realName -Count 1 -IPv4 -MaxHops 1 -DontFragment
@ -147,12 +147,10 @@ Describe "Test-Connection" -tags "CI" {
{ Test-Connection $targetName -BufferSize 65501 } | Should -Throw -ErrorId "ParameterArgumentValidationError,Microsoft.PowerShell.Commands.TestConnectionCommand"
}
It "BufferSize works" {
It "BufferSize works" -Pending:(!$IsWindows) {
$result = Test-Connection $targetName -Count 1 -BufferSize 2
if (!$isWindows) {
$result.Replies[0].Buffer.Count | Should -Be 0
} else {
if ($isWindows) {
$result.Replies[0].Buffer.Count | Should -Be 2
}
}
@ -191,9 +189,7 @@ Describe "Test-Connection" -tags "CI" {
$result.Count | Should -BeGreaterThan 4
$result[0].Address | Should -BeExactly $targetAddress
$result[0].Status | Should -BeExactly "Success"
if (!$isWindows) {
$result[0].Buffer.Count | Should -Be 0
} else {
if ($isWindows) {
$result[0].Buffer.Count | Should -Be 32
}
}
@ -219,7 +215,8 @@ Describe "Test-Connection" -tags "CI" {
}
Context "TraceRoute" {
It "TraceRoute works" {
# Hangs in VSTS Linux
It "TraceRoute works" -skip:(Test-IsVstsLinux) {
$result = Test-Connection $realName -TraceRoute
$replies = $result.Replies
# Check target host reply.
@ -245,19 +242,20 @@ Describe "Test-Connection" -tags "CI" {
}
}
It "Quiet works" {
# Hangs in VSTS Linux
It "Quiet works" -skip:(Test-IsVstsLinux) {
$result = Test-Connection $realName -TraceRoute -Quiet
$result | Should -BeTrue
}
}
Context "Connection" {
BeforeAll {
# Ensure the local host listen on port 80
$WebListener = Start-WebListener
}
It "Test connection to local host port 80" {
Test-Connection '127.0.0.1' -TCPPort $WebListener.HttpPort | Should -BeTrue
}
@ -266,4 +264,4 @@ Describe "Test-Connection" -tags "CI" {
Test-Connection $UnreachableAddress -TCPPort 80 -TimeOut 1 | Should -BeFalse
}
}
}
}

View file

@ -1,5 +1,8 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
Import-Module HelpersCommon
Describe "Export-Alias DRT Unit Tests" -Tags "CI" {
BeforeAll {
@ -94,7 +97,7 @@ Describe "Export-Alias DRT Unit Tests" -Tags "CI" {
$fulltestpath| Should -FileContentMatchExactly '"abcd02","efgh02","","None"'
}
It "Export-Alias for Force ReadOnly Test" {
It "Export-Alias for Force ReadOnly Test" -Skip:(Test-IsRoot) {
Export-Alias $fulltestpath abcd01
if ( $IsWindows )
{

View file

@ -1,5 +1,8 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
Import-Module HelpersCommon
Describe "Export-Csv" -Tags "CI" {
BeforeAll {
$testObject = @("test","object","array")
@ -100,7 +103,7 @@ Describe "Export-Csv" -Tags "CI" {
$results.P1 | Should -BeExactly "first"
}
It "Should not overwrite read-only file without -Force parameter" {
It "Should not overwrite read-only file without -Force parameter" -Skip:(Test-IsRoot) {
$P1 | Export-Csv -Path $testCsv
Set-ItemProperty -Path $testCsv -Name IsReadOnly -Value $true

View file

@ -1,11 +1,16 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
Import-Module HelpersCommon
Describe "Set-Date for admin" -Tag @('CI', 'RequireAdminOnWindows', 'RequireSudoOnUnix') {
It "Set-Date should be able to set the date in an elevated context" {
# Fails in VSTS Linux with Operation not permitted
It "Set-Date should be able to set the date in an elevated context" -Skip:(Test-IsVstsLinux) {
{ Get-Date | Set-Date } | Should -Not -Throw
}
It "Set-Date should be able to set the date with -Date parameter" {
# Fails in VSTS Linux with Operation not permitted
It "Set-Date should be able to set the date with -Date parameter" -Skip:(Test-IsVstsLinux) {
$target = Get-Date
$expected = $target
Set-Date -Date $target | Should -Be $expected

View file

@ -24,6 +24,8 @@ FunctionsToExport = @(
'Send-VstsLogFile'
'Set-TesthookResult'
'Test-IsElevated'
'Test-IsRoot'
'Test-IsVstsLinux'
'Test-TesthookIsSet'
'Wait-FileToBePresent'
'Wait-UntilTrue'

View file

@ -236,3 +236,24 @@ function Send-VstsLogFile {
Write-Host "##vso[artifact.upload containerfolder=$name;artifactname=$name]$logFile"
Write-Verbose "Log file captured as $name" -Verbose
}
# Tests if the Linux or macOS user is root
function Test-IsRoot
{
if($IsLinux -or $IsMacOS)
{
$uid = &id -u
if($uid -eq 0)
{
return $true
}
}
return $false
}
# Tests if we are running is a VSTS Linux Build
function Test-IsVstsLinux
{
return ($env:TF_BUILD -and $IsLinux)
}

View file

@ -230,7 +230,7 @@ elseif($Stage -eq 'Build')
$ExperimentalFeatureTests = Get-ExperimentalFeatureTests
# Running tests which do not require sudo.
$pesterPassThruNoSudoObject = Start-PSPester @noSudoPesterParam
$pesterPassThruNoSudoObject = Start-PSPester @noSudoPesterParam -Title 'Pester No Sudo'
# Running tests that do not require sudo, with specified experimental features enabled
$noSudoResultsWithExpFeatures = @()
@ -249,7 +249,7 @@ elseif($Stage -eq 'Build')
# If a non-empty string or array is specified for the feature name, we only run those test files.
$noSudoPesterParam['Path'] = $testFiles
}
$passThruResult = Start-PSPester @noSudoPesterParam
$passThruResult = Start-PSPester @noSudoPesterParam -Title "Pester Experimental No Sudo - $featureName"
$noSudoResultsWithExpFeatures += $passThruResult
}
@ -260,7 +260,7 @@ elseif($Stage -eq 'Build')
$sudoPesterParam['ExcludeTag'] = @()
$sudoPesterParam['Sudo'] = $true
$sudoPesterParam['OutputFile'] = $testResultsSudo
$pesterPassThruSudoObject = Start-PSPester @sudoPesterParam
$pesterPassThruSudoObject = Start-PSPester @sudoPesterParam -Title 'Pester Sudo'
# Running tests that require sudo, with specified experimental features enabled
$sudoResultsWithExpFeatures = @()
@ -279,7 +279,7 @@ elseif($Stage -eq 'Build')
# If a non-empty string or array is specified for the feature name, we only run those test files.
$sudoPesterParam['Path'] = $testFiles
}
$passThruResult = Start-PSPester @sudoPesterParam
$passThruResult = Start-PSPester @sudoPesterParam -Title "Pester Experimental Sudo - $featureName"
$sudoResultsWithExpFeatures += $passThruResult
}