From 817f598e2016813bcb1f04b8673e6fc8e7f9199f Mon Sep 17 00:00:00 2001 From: Michael Niksa Date: Wed, 25 Aug 2021 15:58:06 -0700 Subject: [PATCH] Move PGO Helix pools (#11028) Moves PGO runs to supported Helix pools. We need to match Microsoft-UI-XAML on which Helix pools we used for each type of activities. ## PR Checklist * [x] Closes #10850 * [x] I work here * [x] If it builds, it sits. ## Validation Steps Performed * [x] Run PGO build against this branch --- build/Helix/OutputTestResults.ps1 | 4 ++-- build/Helix/ProcessHelixFiles.ps1 | 24 +++++++++++++++++-- build/Helix/UpdateUnreliableTests.ps1 | 9 +++---- .../pipelines/templates/build-console-pgo.yml | 2 +- .../helix-processtestresults-job.yml | 2 ++ .../templates/helix-runtests-job.yml | 16 ++++++++++++- .../pgo-build-and-publish-nuget-job.yml | 22 ++++++++++++----- 7 files changed, 63 insertions(+), 16 deletions(-) diff --git a/build/Helix/OutputTestResults.ps1 b/build/Helix/OutputTestResults.ps1 index 19aa8441b..eee4c5bae 100644 --- a/build/Helix/OutputTestResults.ps1 +++ b/build/Helix/OutputTestResults.ps1 @@ -21,7 +21,7 @@ Write-Host "Checking test results..." $queryUri = GetQueryTestRunsUri -CollectionUri $CollectionUri -TeamProject $TeamProject -BuildUri $BuildUri -IncludeRunDetails Write-Host "queryUri = $queryUri" -$testRuns = Invoke-RestMethod -Uri $queryUri -Method Get -Headers $azureDevOpsRestApiHeaders +$testRuns = Invoke-RestMethodWithRetries $queryUri -Headers $azureDevOpsRestApiHeaders [System.Collections.Generic.List[string]]$failingTests = @() [System.Collections.Generic.List[string]]$unreliableTests = @() [System.Collections.Generic.List[string]]$unexpectedResultTest = @() @@ -50,7 +50,7 @@ foreach ($testRun in ($testRuns.value | Sort-Object -Property "completedDate" -D $totalTestsExecutedCount += $testRun.totalTests $testRunResultsUri = "$($testRun.url)/results?api-version=5.0" - $testResults = Invoke-RestMethod -Uri "$($testRun.url)/results?api-version=5.0" -Method Get -Headers $azureDevOpsRestApiHeaders + $testResults = Invoke-RestMethodWithRetries "$($testRun.url)/results?api-version=5.0" -Headers $azureDevOpsRestApiHeaders foreach ($testResult in $testResults.value) { diff --git a/build/Helix/ProcessHelixFiles.ps1 b/build/Helix/ProcessHelixFiles.ps1 index f74187219..dcd3608a5 100644 --- a/build/Helix/ProcessHelixFiles.ps1 +++ b/build/Helix/ProcessHelixFiles.ps1 @@ -20,13 +20,31 @@ function Generate-File-Links Out-File -FilePath $helixLinkFile -Append -InputObject "" Out-File -FilePath $helixLinkFile -Append -InputObject "" } } +function Append-HelixAccessTokenToUrl +{ + Param ([string]$url, [string]$token) + if($token) + { + if($url.Contains("?")) + { + $url = "$($url)&access_token=$($token)" + } + else + { + $url = "$($url)?access_token=$($token)" + } + } + return $url +} + #Create output directory New-Item $OutputFolder -ItemType Directory @@ -63,7 +81,8 @@ foreach ($testRun in $testRuns.value) if (-not $workItems.Contains($workItem)) { $workItems.Add($workItem) - $filesQueryUri = "https://helix.dot.net/api/2019-06-17/jobs/$helixJobId/workitems/$helixWorkItemName/files$accessTokenParam" + $filesQueryUri = "https://helix.dot.net/api/2019-06-17/jobs/$helixJobId/workitems/$helixWorkItemName/files" + $filesQueryUri = Append-HelixAccessTokenToUrl $filesQueryUri $helixAccessToken $files = Invoke-RestMethodWithRetries $filesQueryUri $screenShots = $files | where { $_.Name.EndsWith(".jpg") } @@ -102,6 +121,7 @@ foreach ($testRun in $testRuns.value) Write-Host "Downloading $link to $destination" + $link = Append-HelixAccessTokenToUrl $link $HelixAccessToken Download-FileWithRetries $link $destination } } diff --git a/build/Helix/UpdateUnreliableTests.ps1 b/build/Helix/UpdateUnreliableTests.ps1 index 06af44f69..ecf4e8bf7 100644 --- a/build/Helix/UpdateUnreliableTests.ps1 +++ b/build/Helix/UpdateUnreliableTests.ps1 @@ -23,7 +23,7 @@ Write-Host "queryUri = $queryUri" # To account for unreliable tests, we'll iterate through all of the tests associated with this build, check to see any tests that were unreliable # (denoted by being marked as "skipped"), and if so, we'll instead mark those tests with a warning and enumerate all of the attempted runs # with their pass/fail states as well as any relevant error messages for failed attempts. -$testRuns = Invoke-RestMethod -Uri $queryUri -Method Get -Headers $azureDevOpsRestApiHeaders +$testRuns = Invoke-RestMethodWithRetries $queryUri -Headers $azureDevOpsRestApiHeaders $timesSeenByRunName = @{} @@ -32,10 +32,10 @@ foreach ($testRun in $testRuns.value) $testRunResultsUri = "$($testRun.url)/results?api-version=5.0" Write-Host "Marking test run `"$($testRun.name)`" as in progress so we can change its results to account for unreliable tests." - Invoke-RestMethod -Uri "$($testRun.url)?api-version=5.0" -Method Patch -Body (ConvertTo-Json @{ "state" = "InProgress" }) -Headers $azureDevOpsRestApiHeaders -ContentType "application/json" | Out-Null + Invoke-RestMethod "$($testRun.url)?api-version=5.0" -Method Patch -Body (ConvertTo-Json @{ "state" = "InProgress" }) -Headers $azureDevOpsRestApiHeaders -ContentType "application/json" | Out-Null Write-Host "Retrieving test results..." - $testResults = Invoke-RestMethod -Uri $testRunResultsUri -Method Get -Headers $azureDevOpsRestApiHeaders + $testResults = Invoke-RestMethodWithRetries $testRunResultsUri -Headers $azureDevOpsRestApiHeaders foreach ($testResult in $testResults.value) { @@ -54,7 +54,8 @@ foreach ($testRun in $testRuns.value) Write-Host " Test $($testResult.testCaseTitle) was detected as unreliable. Updating..." # The errorMessage field contains a link to the JSON-encoded rerun result data. - $rerunResults = ConvertFrom-Json (New-Object System.Net.WebClient).DownloadString($testResult.errorMessage) + $resultsJson = Download-StringWithRetries "Error results" $testResult.errorMessage + $rerunResults = ConvertFrom-Json $resultsJson [System.Collections.Generic.List[System.Collections.Hashtable]]$rerunDataList = @() $attemptCount = 0 $passCount = 0 diff --git a/build/pipelines/templates/build-console-pgo.yml b/build/pipelines/templates/build-console-pgo.yml index 1e33e82c8..d3dbaa99c 100644 --- a/build/pipelines/templates/build-console-pgo.yml +++ b/build/pipelines/templates/build-console-pgo.yml @@ -3,7 +3,7 @@ parameters: platform: '' additionalBuildArguments: '' minimumExpectedTestsExecutedCount: 1 # Sanity check for minimum expected tests to be reported - rerunPassesRequiredToAvoidFailure: 0 + rerunPassesRequiredToAvoidFailure: 5 jobs: - job: Build${{ parameters.platform }}${{ parameters.configuration }} diff --git a/build/pipelines/templates/helix-processtestresults-job.yml b/build/pipelines/templates/helix-processtestresults-job.yml index 9d301e9cb..1e5ff3c73 100644 --- a/build/pipelines/templates/helix-processtestresults-job.yml +++ b/build/pipelines/templates/helix-processtestresults-job.yml @@ -22,6 +22,7 @@ jobs: condition: succeededOrFailed() env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HelixAccessToken: $(HelixApiAccessToken) inputs: targetType: filePath filePath: build\Helix\UpdateUnreliableTests.ps1 @@ -32,6 +33,7 @@ jobs: condition: succeededOrFailed() env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HelixAccessToken: $(HelixApiAccessToken) inputs: targetType: filePath filePath: build\Helix\OutputTestResults.ps1 diff --git a/build/pipelines/templates/helix-runtests-job.yml b/build/pipelines/templates/helix-runtests-job.yml index d429687be..042c54c12 100644 --- a/build/pipelines/templates/helix-runtests-job.yml +++ b/build/pipelines/templates/helix-runtests-job.yml @@ -15,6 +15,7 @@ parameters: # if 'useBuildOutputFromBuildId' is set, we will default to using a build from this pipeline: useBuildOutputFromPipeline: $(System.DefinitionId) openHelixTargetQueues: 'windows.10.amd64.client19h1.open.xaml' + closedHelixTargetQueues: 'windows.10.amd64.client19h1.xaml' jobs: - job: ${{ parameters.name }} @@ -29,11 +30,11 @@ jobs: buildConfiguration: ${{ parameters.configuration }} buildPlatform: ${{ parameters.platform }} openHelixTargetQueues: ${{ parameters.openHelixTargetQueues }} + closedHelixTargetQueues: ${{ parameters.closedHelixTargetQueues }} artifactsDir: $(Build.SourcesDirectory)\Artifacts taefPath: $(Build.SourcesDirectory)\build\Helix\packages\Microsoft.Taef.10.60.210621002\build\Binaries\$(buildPlatform) helixCommonArgs: '/binaryLogger:$(Build.SourcesDirectory)/${{parameters.name}}.$(buildPlatform).$(buildConfiguration).binlog /p:HelixBuild=$(Build.BuildId).$(buildPlatform).$(buildConfiguration) /p:Platform=$(buildPlatform) /p:Configuration=$(buildConfiguration) /p:HelixType=${{parameters.helixType}} /p:TestSuite=${{parameters.testSuite}} /p:ProjFilesPath=$(Build.ArtifactStagingDirectory) /p:rerunPassesRequiredToAvoidFailure=${{parameters.rerunPassesRequiredToAvoidFailure}}' - steps: - task: CmdLine@1 displayName: 'Display build machine environment variables' @@ -140,6 +141,7 @@ jobs: - task: DotNetCoreCLI@2 displayName: 'Run tests in Helix (open queues)' + condition: and(succeeded(),eq(variables['System.CollectionUri'],'https://dev.azure.com/ms/')) env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) inputs: @@ -147,3 +149,15 @@ jobs: projects: build\Helix\RunTestsInHelix.proj custom: msbuild arguments: '$(helixCommonArgs) /p:IsExternal=true /p:Creator=Terminal /p:HelixTargetQueues=$(openHelixTargetQueues)' + + - task: DotNetCoreCLI@2 + displayName: 'Run tests in Helix (closed queues)' + condition: and(succeeded(),ne(variables['System.CollectionUri'],'https://dev.azure.com/ms/')) + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HelixAccessToken: $(HelixApiAccessToken) + inputs: + command: custom + projects: build\Helix\RunTestsInHelix.proj + custom: msbuild + arguments: '$(helixCommonArgs) /p:HelixTargetQueues=$(closedHelixTargetQueues)' diff --git a/build/pipelines/templates/pgo-build-and-publish-nuget-job.yml b/build/pipelines/templates/pgo-build-and-publish-nuget-job.yml index 85bc6c353..92484ebf3 100644 --- a/build/pipelines/templates/pgo-build-and-publish-nuget-job.yml +++ b/build/pipelines/templates/pgo-build-and-publish-nuget-job.yml @@ -20,11 +20,15 @@ jobs: inputs: artifactName: ${{ parameters.pgoArtifact }} downloadPath: $(artifactsPath) - - - task: NuGetToolInstaller@0 - displayName: 'Use NuGet 5.2.0' + + - task: NuGetAuthenticate@0 inputs: - versionSpec: 5.2.0 + nuGetServiceConnections: 'Terminal Public Artifact Feed' + + - task: NuGetToolInstaller@0 + displayName: 'Use NuGet 5.8.0' + inputs: + versionSpec: 5.8.0 - task: CopyFiles@2 displayName: 'Copy pgd files to NuGet build directory' @@ -58,5 +62,11 @@ jobs: displayName: 'NuGet push' inputs: command: push - publishVstsFeed: Terminal/TerminalDependencies - packagesToPush: $(Build.ArtifactStagingDirectory)/*.nupkg \ No newline at end of file + nuGetFeedType: external + packagesToPush: $(Build.ArtifactStagingDirectory)/*.nupkg + # The actual URL and PAT for this feed is configured at + # https://microsoft.visualstudio.com/Dart/_settings/adminservices + # This is the name of that connection + publishFeedCredentials: 'Terminal Public Artifact Feed' + feedsToUse: config + nugetConfigPath: '$(Build.SourcesDirectory)/NuGet.config' \ No newline at end of file