From f41b6b415eea8524ac04c004a4d90372753f8229 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 28 Oct 2021 11:44:06 -0700 Subject: [PATCH] Add GitHub Workflow to keep notices up to date (#16284) Co-authored-by: Robert Holt --- .github/workflows/update-cgmanifest.yml | 64 ++++++++ build.psm1 | 49 +----- tools/buildCommon/startNativeExecution.ps1 | 47 ++++++ tools/findMissingNotices.ps1 | 174 +++++++++++++++------ 4 files changed, 242 insertions(+), 92 deletions(-) create mode 100644 .github/workflows/update-cgmanifest.yml create mode 100644 tools/buildCommon/startNativeExecution.ps1 diff --git a/.github/workflows/update-cgmanifest.yml b/.github/workflows/update-cgmanifest.yml new file mode 100644 index 000000000..9e026b70f --- /dev/null +++ b/.github/workflows/update-cgmanifest.yml @@ -0,0 +1,64 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +name: Update cgmanifest +on: + workflow_dispatch: + schedule: + # At 13:00 UTC every day. + - cron: '0 13 * * *' + +defaults: + run: + shell: pwsh + +env: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + POWERSHELL_TELEMETRY_OPTOUT: 1 + +jobs: + update-cgmanifest: + name: Update cgmanifest + timeout-minutes: 15 + runs-on: windows-latest + if: github.repository == 'PowerShell/PowerShell' + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Sync tags + run: | + git fetch --prune --unshallow --tags + - name: Install Ships provider to deal with project.assets.json + run: | + Install-Module -Name dotnet.project.assets -force + - name: Bootstrap + run: | + Import-Module ./build.psm1 + Start-PSBootStrap + - name: Update Notices file + run: | + Invoke-WebRequest -Uri https://aka.ms/pwsh-daily-tpn -OutFile ./ThirdPartyNotices.txt + - name: Execute script to update cgmanifest + run: | + Import-Module ./build.psm1 + Find-Dotnet + ./tools/findMissingNotices.ps1 + - name: Microsoft Teams Notifier + uses: skitionek/notify-microsoft-teams@master + if: failure() + with: + webhook_url: ${{ secrets.PS_BUILD_TEAMS_CHANNEL }} + overwrite: "{title: `Failure in updating cgmanifest. Look at ${workflow_link}`}" + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + id: cpr + if: env.CREATE_PR == 'true' + with: + commit-message: "Update the cgmanifest with missing or updated components" + committer: GitHub + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + title: "Update ${{ env.FORMULA_NAME }} formula to version ${{ env.NEW_FORMULA_VERSION }}" + reviewers: travisez13 + base: master + draft: false + branch: update-cgmanifest diff --git a/build.psm1 b/build.psm1 index b8369dbaf..2900e1bc4 100644 --- a/build.psm1 +++ b/build.psm1 @@ -6,6 +6,8 @@ param( [parameter(Mandatory = $false)][switch]$SkipLinuxDistroCheck = $false ) +. "$PSScriptRoot\tools\buildCommon\startNativeExecution.ps1" + Set-StrictMode -Version 3.0 # On Unix paths is separated by colon @@ -2357,53 +2359,6 @@ function script:precheck([string]$command, [string]$missedMessage) { } } -# this function wraps native command Execution -# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/ -function script:Start-NativeExecution -{ - param( - [scriptblock]$sb, - [switch]$IgnoreExitcode, - [switch]$VerboseOutputOnError - ) - $backupEAP = $ErrorActionPreference - $ErrorActionPreference = "Continue" - try { - if($VerboseOutputOnError.IsPresent) - { - $output = & $sb 2>&1 - } - else - { - & $sb - } - - # note, if $sb doesn't have a native invocation, $LASTEXITCODE will - # point to the obsolete value - if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) { - if($VerboseOutputOnError.IsPresent -and $output) - { - $output | Out-String | Write-Verbose -Verbose - } - - # Get caller location for easier debugging - $caller = Get-PSCallStack -ErrorAction SilentlyContinue - if($caller) - { - $callerLocationParts = $caller[1].Location -split ":\s*line\s*" - $callerFile = $callerLocationParts[0] - $callerLine = $callerLocationParts[1] - - $errorMessage = "Execution of {$sb} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE" - throw $errorMessage - } - throw "Execution of {$sb} failed with exit code $LASTEXITCODE" - } - } finally { - $ErrorActionPreference = $backupEAP - } -} - # Cleans the PowerShell repo - everything but the root folder function Clear-PSRepo { diff --git a/tools/buildCommon/startNativeExecution.ps1 b/tools/buildCommon/startNativeExecution.ps1 new file mode 100644 index 000000000..ee7b00d04 --- /dev/null +++ b/tools/buildCommon/startNativeExecution.ps1 @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# this function wraps native command Execution +# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/ +function script:Start-NativeExecution { + param( + [Alias('sb')] + [Parameter(Mandatory=$true)] + [scriptblock]$ScriptBlock, + [switch]$IgnoreExitcode, + [switch]$VerboseOutputOnError + ) + + $backupEAP = $ErrorActionPreference + $ErrorActionPreference = "Continue" + Write-Verbose "Executing: $ScriptBlock" + try { + if ($VerboseOutputOnError.IsPresent) { + $output = & $ScriptBlock 2>&1 + } else { + & $ScriptBlock + } + + # note, if $ScriptBlock doesn't have a native invocation, $LASTEXITCODE will + # point to the obsolete value + if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) { + if ($VerboseOutputOnError.IsPresent -and $output) { + $output | Out-String | Write-Verbose -Verbose + } + + # Get caller location for easier debugging + $caller = Get-PSCallStack -ErrorAction SilentlyContinue + if ($caller) { + $callerLocationParts = $caller[1].Location -split ":\s*line\s*" + $callerFile = $callerLocationParts[0] + $callerLine = $callerLocationParts[1] + + $errorMessage = "Execution of {$ScriptBlock} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE" + throw $errorMessage + } + throw "Execution of {$ScriptBlock} failed with exit code $LASTEXITCODE" + } + } finally { + $ErrorActionPreference = $backupEAP + } +} diff --git a/tools/findMissingNotices.ps1 b/tools/findMissingNotices.ps1 index d1cf198f8..f2c806623 100644 --- a/tools/findMissingNotices.ps1 +++ b/tools/findMissingNotices.ps1 @@ -1,9 +1,23 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +# This script is used to completely rebuild the cgmanifgest.json file, +# which is used to generate the notice file. # Requires the module dotnet.project.assets from the PowerShell Gallery authored by @TravisEz13 -import-module dotnet.project.assets +Import-Module dotnet.project.assets +. "$PSScriptRoot\..\tools\buildCommon\startNativeExecution.ps1" + +$existingRegistrationTable = @{} +$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable +$existingRegistrationsJson.Registrations | ForEach-Object { + $registration = [Registration]$_ + if($registration.Component) { + $name = $registration.Component.Name() + $existingRegistrationTable.Add($name, $registration) + } +} + Class Registration { [Component]$Component [bool]$DevelopmentDependency @@ -79,58 +93,128 @@ function New-NugetComponent { return $registration } -$existingRegistrationTable = @{} -$newRegistrations = @() -$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable -$existingRegistrationsJson.Registrations | ForEach-Object { - $registration = [Registration]$_ - $existingRegistrationTable.Add($registration.Component.Name(), $registration) - $newRegistrations += $registration +$winDesktopSdk = 'Microsoft.NET.Sdk.WindowsDesktop' +if (!$IsWindows) { + $winDesktopSdk = 'Microsoft.NET.Sdk' + Write-Warning "Always using $winDesktopSdk since this is not windows!!!" } -Get-PSDrive -Name pwsh-win-core -ErrorAction Ignore | Remove-PSDrive -Push-Location $PSScriptRoot\..\src\powershell-win-core -$null = dotnet restore -$null = New-PADrive -Path $PSScriptRoot\..\src\powershell-win-core\obj\project.assets.json -Name pwsh-win-core -$targets = Get-ChildItem -Path 'pwsh-win-core:/targets/net6.0-windows7.0|win7-x64' | Where-Object { - $_.Type -eq 'package' -and - $_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and - $_.Name -notlike 'StyleCop*' -and - $_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and - $_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*' -} | select-object -ExpandProperty name -Pop-Location -Get-PSDrive -Name pwsh-win-core | Remove-PSDrive +Function Get-CGRegistrations { + param( + [Parameter(Mandatory)] + [ValidateSet( + "alpine-x64", + "linux-arm", + "linux-arm64", + "linux-x64", + "osx-arm64", + "osx-x64", + "win-arm", + "win-arm64", + "win7-x64", + "win7-x86", + "modules")] + [string]$Runtime, -$updateRegistrations = @() -$targets | ForEach-Object { - $target = $_ - $parts = ($target -split '\|') - $name = $parts[0] - $targetVersion = $parts[1] - $pattern = [regex]::Escape($name) + " " - $tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern - if (!$tpnMatch) { - if ($existingRegistrationTable.ContainsKey($name)) { - $registrationVersion = $existingRegistrationTable.$name.Component.Version() - if ($registrationVersion -ne $targetVersion) { - $registration = New-NugetComponent -Name $name -Version $targetVersion - $updateRegistrations += $registration - } else { - Write-Verbose "$target already registered: $registrationVersion" -Verbose - } - } else { + [Parameter(Mandatory)] + [System.Collections.Generic.Dictionary[string, Registration]] $RegistrationTable + ) + + $newRegistrations = $Registrations + + $dotnetTargetName = 'net6.0' + $dotnetTargetNameWin7 = 'net6.0-windows7.0' + $unixProjectName = 'powershell-unix' + $windowsProjectName = 'powershell-win-core' + $actualRuntime = $Runtime + + switch -regex ($Runtime) { + "alpine-.*" { + $folder = $unixProjectName + $target = "$dotnetTargetName|$Runtime" + } + "linux-.*" { + $folder = $unixProjectName + $target = "$dotnetTargetName|$Runtime" + } + "osx-.*" { + $folder = $unixProjectName + $target = "$dotnetTargetName|$Runtime" + } + "win7-.*" { + $sdkToUse = $winDesktopSdk + $folder = $windowsProjectName + $target = "$dotnetTargetNameWin7|$Runtime" + } + "win-.*" { + $folder = $windowsProjectName + $target = "$dotnetTargetNameWin7|$Runtime" + } + "modules" { + $folder = "modules" + $actualRuntime = 'linux-x64' + $target = "$dotnetTargetName|$actualRuntime" + } + Default { + throw "Invalid runtime name: $Runtime" + } + } + + Write-Verbose "Getting registrations for $folder - $actualRuntime ..." -Verbose + Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive + Push-Location $PSScriptRoot\..\src\$folder + try { + Start-NativeExecution -VerboseOutputOnError -sb { + dotnet restore --runtime $actualRuntime "/property:SDKToUse=$sdkToUse" + } + $null = New-PADrive -Path $PSScriptRoot\..\src\$folder\obj\project.assets.json -Name $folder + try { + $targets = Get-ChildItem -Path "${folder}:/targets/$target" -ErrorAction Stop | Where-Object { + $_.Type -eq 'package' -and + $_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and + $_.Name -notlike 'StyleCop*' -and + $_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and + $_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*' + } | select-object -ExpandProperty name + } catch { + Get-ChildItem -Path "${folder}:/targets" | Out-String | Write-Verbose -Verbose + throw + } + } finally { + Pop-Location + Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive + } + + $targets | ForEach-Object { + $target = $_ + $parts = ($target -split '\|') + $name = $parts[0] + $targetVersion = $parts[1] + $pattern = [regex]::Escape($name) + " " + $tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern + + # Add the registration to the cgmanifest if the TPN does not contain the name of the target OR + # the exisitng CG contains the registration, because if the existing CG contains the registration, + # that might be the only reason it is in the TPN. + if ((!$tpnMatch -or $existingRegistrationTable.ContainsKey($name)) -and !$RegistrationTable.ContainsKey($target)) { $registration = New-NugetComponent -Name $name -Version $targetVersion - $newRegistrations += $registration + $RegistrationTable.Add($target, $registration) } } } -if ($updateRegistrations.count -gt 0) { - #TODO delete existing and add new registration - throw "updating registrations is not implemented" +$registrations = [System.Collections.Generic.Dictionary[string, Registration]]::new() +$lastCount = 0 +foreach ($runtime in "win7-x64", "linux-x64", "osx-x64", "alpine-x64", "win-arm", "linux-arm", "linux-arm64", "osx-arm64", "win-arm64", "win7-x86") { + Get-CGRegistrations -Runtime $runtime -RegistrationTable $registrations + $count = $registrations.Count + $newCount = $count - $lastCount + $lastCount = $count + Write-Verbose "$newCount new registrations, $count total..." -Verbose } -$newCount = $newRegistrations.count - $existingRegistrationTable.count +$newRegistrations = $registrations.Keys | Sort-Object | ForEach-Object { $registrations[$_] } + +$count = $newRegistrations.Count @{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 | Set-Content $PSScriptRoot\..\cgmanifest.json -Write-Verbose "$newCount registrations added" -Verbose +Write-Verbose "$count registrations created!" -Verbose