Add GitHub Workflow to keep notices up to date (#16284)
Co-authored-by: Robert Holt <rjmholt_msft@outlook.com>
This commit is contained in:
parent
52b6d89407
commit
f41b6b415e
64
.github/workflows/update-cgmanifest.yml
vendored
Normal file
64
.github/workflows/update-cgmanifest.yml
vendored
Normal file
|
@ -0,0 +1,64 @@
|
||||||
|
# Copyright (c) Microsoft Corporation.
|
||||||
|
# Licensed under the MIT license.
|
||||||
|
|
||||||
|
name: Update cgmanifest
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
schedule:
|
||||||
|
# At 13:00 UTC every day.
|
||||||
|
- cron: '0 13 * * *'
|
||||||
|
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: pwsh
|
||||||
|
|
||||||
|
env:
|
||||||
|
DOTNET_CLI_TELEMETRY_OPTOUT: 1
|
||||||
|
POWERSHELL_TELEMETRY_OPTOUT: 1
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
update-cgmanifest:
|
||||||
|
name: Update cgmanifest
|
||||||
|
timeout-minutes: 15
|
||||||
|
runs-on: windows-latest
|
||||||
|
if: github.repository == 'PowerShell/PowerShell'
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v2
|
||||||
|
- name: Sync tags
|
||||||
|
run: |
|
||||||
|
git fetch --prune --unshallow --tags
|
||||||
|
- name: Install Ships provider to deal with project.assets.json
|
||||||
|
run: |
|
||||||
|
Install-Module -Name dotnet.project.assets -force
|
||||||
|
- name: Bootstrap
|
||||||
|
run: |
|
||||||
|
Import-Module ./build.psm1
|
||||||
|
Start-PSBootStrap
|
||||||
|
- name: Update Notices file
|
||||||
|
run: |
|
||||||
|
Invoke-WebRequest -Uri https://aka.ms/pwsh-daily-tpn -OutFile ./ThirdPartyNotices.txt
|
||||||
|
- name: Execute script to update cgmanifest
|
||||||
|
run: |
|
||||||
|
Import-Module ./build.psm1
|
||||||
|
Find-Dotnet
|
||||||
|
./tools/findMissingNotices.ps1
|
||||||
|
- name: Microsoft Teams Notifier
|
||||||
|
uses: skitionek/notify-microsoft-teams@master
|
||||||
|
if: failure()
|
||||||
|
with:
|
||||||
|
webhook_url: ${{ secrets.PS_BUILD_TEAMS_CHANNEL }}
|
||||||
|
overwrite: "{title: `Failure in updating cgmanifest. Look at ${workflow_link}`}"
|
||||||
|
- name: Create Pull Request
|
||||||
|
uses: peter-evans/create-pull-request@v3
|
||||||
|
id: cpr
|
||||||
|
if: env.CREATE_PR == 'true'
|
||||||
|
with:
|
||||||
|
commit-message: "Update the cgmanifest with missing or updated components"
|
||||||
|
committer: GitHub <noreply@github.com>
|
||||||
|
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
|
||||||
|
title: "Update ${{ env.FORMULA_NAME }} formula to version ${{ env.NEW_FORMULA_VERSION }}"
|
||||||
|
reviewers: travisez13
|
||||||
|
base: master
|
||||||
|
draft: false
|
||||||
|
branch: update-cgmanifest
|
49
build.psm1
49
build.psm1
|
@ -6,6 +6,8 @@ param(
|
||||||
[parameter(Mandatory = $false)][switch]$SkipLinuxDistroCheck = $false
|
[parameter(Mandatory = $false)][switch]$SkipLinuxDistroCheck = $false
|
||||||
)
|
)
|
||||||
|
|
||||||
|
. "$PSScriptRoot\tools\buildCommon\startNativeExecution.ps1"
|
||||||
|
|
||||||
Set-StrictMode -Version 3.0
|
Set-StrictMode -Version 3.0
|
||||||
|
|
||||||
# On Unix paths is separated by colon
|
# On Unix paths is separated by colon
|
||||||
|
@ -2357,53 +2359,6 @@ function script:precheck([string]$command, [string]$missedMessage) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# this function wraps native command Execution
|
|
||||||
# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/
|
|
||||||
function script:Start-NativeExecution
|
|
||||||
{
|
|
||||||
param(
|
|
||||||
[scriptblock]$sb,
|
|
||||||
[switch]$IgnoreExitcode,
|
|
||||||
[switch]$VerboseOutputOnError
|
|
||||||
)
|
|
||||||
$backupEAP = $ErrorActionPreference
|
|
||||||
$ErrorActionPreference = "Continue"
|
|
||||||
try {
|
|
||||||
if($VerboseOutputOnError.IsPresent)
|
|
||||||
{
|
|
||||||
$output = & $sb 2>&1
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
& $sb
|
|
||||||
}
|
|
||||||
|
|
||||||
# note, if $sb doesn't have a native invocation, $LASTEXITCODE will
|
|
||||||
# point to the obsolete value
|
|
||||||
if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) {
|
|
||||||
if($VerboseOutputOnError.IsPresent -and $output)
|
|
||||||
{
|
|
||||||
$output | Out-String | Write-Verbose -Verbose
|
|
||||||
}
|
|
||||||
|
|
||||||
# Get caller location for easier debugging
|
|
||||||
$caller = Get-PSCallStack -ErrorAction SilentlyContinue
|
|
||||||
if($caller)
|
|
||||||
{
|
|
||||||
$callerLocationParts = $caller[1].Location -split ":\s*line\s*"
|
|
||||||
$callerFile = $callerLocationParts[0]
|
|
||||||
$callerLine = $callerLocationParts[1]
|
|
||||||
|
|
||||||
$errorMessage = "Execution of {$sb} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE"
|
|
||||||
throw $errorMessage
|
|
||||||
}
|
|
||||||
throw "Execution of {$sb} failed with exit code $LASTEXITCODE"
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
$ErrorActionPreference = $backupEAP
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Cleans the PowerShell repo - everything but the root folder
|
# Cleans the PowerShell repo - everything but the root folder
|
||||||
function Clear-PSRepo
|
function Clear-PSRepo
|
||||||
{
|
{
|
||||||
|
|
47
tools/buildCommon/startNativeExecution.ps1
Normal file
47
tools/buildCommon/startNativeExecution.ps1
Normal file
|
@ -0,0 +1,47 @@
|
||||||
|
# Copyright (c) Microsoft Corporation.
|
||||||
|
# Licensed under the MIT License.
|
||||||
|
|
||||||
|
# this function wraps native command Execution
|
||||||
|
# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/
|
||||||
|
function script:Start-NativeExecution {
|
||||||
|
param(
|
||||||
|
[Alias('sb')]
|
||||||
|
[Parameter(Mandatory=$true)]
|
||||||
|
[scriptblock]$ScriptBlock,
|
||||||
|
[switch]$IgnoreExitcode,
|
||||||
|
[switch]$VerboseOutputOnError
|
||||||
|
)
|
||||||
|
|
||||||
|
$backupEAP = $ErrorActionPreference
|
||||||
|
$ErrorActionPreference = "Continue"
|
||||||
|
Write-Verbose "Executing: $ScriptBlock"
|
||||||
|
try {
|
||||||
|
if ($VerboseOutputOnError.IsPresent) {
|
||||||
|
$output = & $ScriptBlock 2>&1
|
||||||
|
} else {
|
||||||
|
& $ScriptBlock
|
||||||
|
}
|
||||||
|
|
||||||
|
# note, if $ScriptBlock doesn't have a native invocation, $LASTEXITCODE will
|
||||||
|
# point to the obsolete value
|
||||||
|
if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) {
|
||||||
|
if ($VerboseOutputOnError.IsPresent -and $output) {
|
||||||
|
$output | Out-String | Write-Verbose -Verbose
|
||||||
|
}
|
||||||
|
|
||||||
|
# Get caller location for easier debugging
|
||||||
|
$caller = Get-PSCallStack -ErrorAction SilentlyContinue
|
||||||
|
if ($caller) {
|
||||||
|
$callerLocationParts = $caller[1].Location -split ":\s*line\s*"
|
||||||
|
$callerFile = $callerLocationParts[0]
|
||||||
|
$callerLine = $callerLocationParts[1]
|
||||||
|
|
||||||
|
$errorMessage = "Execution of {$ScriptBlock} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE"
|
||||||
|
throw $errorMessage
|
||||||
|
}
|
||||||
|
throw "Execution of {$ScriptBlock} failed with exit code $LASTEXITCODE"
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
$ErrorActionPreference = $backupEAP
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,9 +1,23 @@
|
||||||
# Copyright (c) Microsoft Corporation.
|
# Copyright (c) Microsoft Corporation.
|
||||||
# Licensed under the MIT License.
|
# Licensed under the MIT License.
|
||||||
|
|
||||||
|
# This script is used to completely rebuild the cgmanifgest.json file,
|
||||||
|
# which is used to generate the notice file.
|
||||||
# Requires the module dotnet.project.assets from the PowerShell Gallery authored by @TravisEz13
|
# Requires the module dotnet.project.assets from the PowerShell Gallery authored by @TravisEz13
|
||||||
|
|
||||||
import-module dotnet.project.assets
|
Import-Module dotnet.project.assets
|
||||||
|
. "$PSScriptRoot\..\tools\buildCommon\startNativeExecution.ps1"
|
||||||
|
|
||||||
|
$existingRegistrationTable = @{}
|
||||||
|
$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable
|
||||||
|
$existingRegistrationsJson.Registrations | ForEach-Object {
|
||||||
|
$registration = [Registration]$_
|
||||||
|
if($registration.Component) {
|
||||||
|
$name = $registration.Component.Name()
|
||||||
|
$existingRegistrationTable.Add($name, $registration)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Class Registration {
|
Class Registration {
|
||||||
[Component]$Component
|
[Component]$Component
|
||||||
[bool]$DevelopmentDependency
|
[bool]$DevelopmentDependency
|
||||||
|
@ -79,58 +93,128 @@ function New-NugetComponent {
|
||||||
return $registration
|
return $registration
|
||||||
}
|
}
|
||||||
|
|
||||||
$existingRegistrationTable = @{}
|
$winDesktopSdk = 'Microsoft.NET.Sdk.WindowsDesktop'
|
||||||
$newRegistrations = @()
|
if (!$IsWindows) {
|
||||||
$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable
|
$winDesktopSdk = 'Microsoft.NET.Sdk'
|
||||||
$existingRegistrationsJson.Registrations | ForEach-Object {
|
Write-Warning "Always using $winDesktopSdk since this is not windows!!!"
|
||||||
$registration = [Registration]$_
|
|
||||||
$existingRegistrationTable.Add($registration.Component.Name(), $registration)
|
|
||||||
$newRegistrations += $registration
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Get-PSDrive -Name pwsh-win-core -ErrorAction Ignore | Remove-PSDrive
|
Function Get-CGRegistrations {
|
||||||
Push-Location $PSScriptRoot\..\src\powershell-win-core
|
param(
|
||||||
$null = dotnet restore
|
[Parameter(Mandatory)]
|
||||||
$null = New-PADrive -Path $PSScriptRoot\..\src\powershell-win-core\obj\project.assets.json -Name pwsh-win-core
|
[ValidateSet(
|
||||||
$targets = Get-ChildItem -Path 'pwsh-win-core:/targets/net6.0-windows7.0|win7-x64' | Where-Object {
|
"alpine-x64",
|
||||||
$_.Type -eq 'package' -and
|
"linux-arm",
|
||||||
$_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and
|
"linux-arm64",
|
||||||
$_.Name -notlike 'StyleCop*' -and
|
"linux-x64",
|
||||||
$_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and
|
"osx-arm64",
|
||||||
$_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*'
|
"osx-x64",
|
||||||
} | select-object -ExpandProperty name
|
"win-arm",
|
||||||
Pop-Location
|
"win-arm64",
|
||||||
Get-PSDrive -Name pwsh-win-core | Remove-PSDrive
|
"win7-x64",
|
||||||
|
"win7-x86",
|
||||||
|
"modules")]
|
||||||
|
[string]$Runtime,
|
||||||
|
|
||||||
$updateRegistrations = @()
|
[Parameter(Mandatory)]
|
||||||
$targets | ForEach-Object {
|
[System.Collections.Generic.Dictionary[string, Registration]] $RegistrationTable
|
||||||
$target = $_
|
)
|
||||||
$parts = ($target -split '\|')
|
|
||||||
$name = $parts[0]
|
$newRegistrations = $Registrations
|
||||||
$targetVersion = $parts[1]
|
|
||||||
$pattern = [regex]::Escape($name) + " "
|
$dotnetTargetName = 'net6.0'
|
||||||
$tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern
|
$dotnetTargetNameWin7 = 'net6.0-windows7.0'
|
||||||
if (!$tpnMatch) {
|
$unixProjectName = 'powershell-unix'
|
||||||
if ($existingRegistrationTable.ContainsKey($name)) {
|
$windowsProjectName = 'powershell-win-core'
|
||||||
$registrationVersion = $existingRegistrationTable.$name.Component.Version()
|
$actualRuntime = $Runtime
|
||||||
if ($registrationVersion -ne $targetVersion) {
|
|
||||||
$registration = New-NugetComponent -Name $name -Version $targetVersion
|
switch -regex ($Runtime) {
|
||||||
$updateRegistrations += $registration
|
"alpine-.*" {
|
||||||
} else {
|
$folder = $unixProjectName
|
||||||
Write-Verbose "$target already registered: $registrationVersion" -Verbose
|
$target = "$dotnetTargetName|$Runtime"
|
||||||
}
|
}
|
||||||
} else {
|
"linux-.*" {
|
||||||
|
$folder = $unixProjectName
|
||||||
|
$target = "$dotnetTargetName|$Runtime"
|
||||||
|
}
|
||||||
|
"osx-.*" {
|
||||||
|
$folder = $unixProjectName
|
||||||
|
$target = "$dotnetTargetName|$Runtime"
|
||||||
|
}
|
||||||
|
"win7-.*" {
|
||||||
|
$sdkToUse = $winDesktopSdk
|
||||||
|
$folder = $windowsProjectName
|
||||||
|
$target = "$dotnetTargetNameWin7|$Runtime"
|
||||||
|
}
|
||||||
|
"win-.*" {
|
||||||
|
$folder = $windowsProjectName
|
||||||
|
$target = "$dotnetTargetNameWin7|$Runtime"
|
||||||
|
}
|
||||||
|
"modules" {
|
||||||
|
$folder = "modules"
|
||||||
|
$actualRuntime = 'linux-x64'
|
||||||
|
$target = "$dotnetTargetName|$actualRuntime"
|
||||||
|
}
|
||||||
|
Default {
|
||||||
|
throw "Invalid runtime name: $Runtime"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-Verbose "Getting registrations for $folder - $actualRuntime ..." -Verbose
|
||||||
|
Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive
|
||||||
|
Push-Location $PSScriptRoot\..\src\$folder
|
||||||
|
try {
|
||||||
|
Start-NativeExecution -VerboseOutputOnError -sb {
|
||||||
|
dotnet restore --runtime $actualRuntime "/property:SDKToUse=$sdkToUse"
|
||||||
|
}
|
||||||
|
$null = New-PADrive -Path $PSScriptRoot\..\src\$folder\obj\project.assets.json -Name $folder
|
||||||
|
try {
|
||||||
|
$targets = Get-ChildItem -Path "${folder}:/targets/$target" -ErrorAction Stop | Where-Object {
|
||||||
|
$_.Type -eq 'package' -and
|
||||||
|
$_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and
|
||||||
|
$_.Name -notlike 'StyleCop*' -and
|
||||||
|
$_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and
|
||||||
|
$_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*'
|
||||||
|
} | select-object -ExpandProperty name
|
||||||
|
} catch {
|
||||||
|
Get-ChildItem -Path "${folder}:/targets" | Out-String | Write-Verbose -Verbose
|
||||||
|
throw
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
Pop-Location
|
||||||
|
Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive
|
||||||
|
}
|
||||||
|
|
||||||
|
$targets | ForEach-Object {
|
||||||
|
$target = $_
|
||||||
|
$parts = ($target -split '\|')
|
||||||
|
$name = $parts[0]
|
||||||
|
$targetVersion = $parts[1]
|
||||||
|
$pattern = [regex]::Escape($name) + " "
|
||||||
|
$tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern
|
||||||
|
|
||||||
|
# Add the registration to the cgmanifest if the TPN does not contain the name of the target OR
|
||||||
|
# the exisitng CG contains the registration, because if the existing CG contains the registration,
|
||||||
|
# that might be the only reason it is in the TPN.
|
||||||
|
if ((!$tpnMatch -or $existingRegistrationTable.ContainsKey($name)) -and !$RegistrationTable.ContainsKey($target)) {
|
||||||
$registration = New-NugetComponent -Name $name -Version $targetVersion
|
$registration = New-NugetComponent -Name $name -Version $targetVersion
|
||||||
$newRegistrations += $registration
|
$RegistrationTable.Add($target, $registration)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($updateRegistrations.count -gt 0) {
|
$registrations = [System.Collections.Generic.Dictionary[string, Registration]]::new()
|
||||||
#TODO delete existing and add new registration
|
$lastCount = 0
|
||||||
throw "updating registrations is not implemented"
|
foreach ($runtime in "win7-x64", "linux-x64", "osx-x64", "alpine-x64", "win-arm", "linux-arm", "linux-arm64", "osx-arm64", "win-arm64", "win7-x86") {
|
||||||
|
Get-CGRegistrations -Runtime $runtime -RegistrationTable $registrations
|
||||||
|
$count = $registrations.Count
|
||||||
|
$newCount = $count - $lastCount
|
||||||
|
$lastCount = $count
|
||||||
|
Write-Verbose "$newCount new registrations, $count total..." -Verbose
|
||||||
}
|
}
|
||||||
|
|
||||||
$newCount = $newRegistrations.count - $existingRegistrationTable.count
|
$newRegistrations = $registrations.Keys | Sort-Object | ForEach-Object { $registrations[$_] }
|
||||||
|
|
||||||
|
$count = $newRegistrations.Count
|
||||||
@{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 | Set-Content $PSScriptRoot\..\cgmanifest.json
|
@{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 | Set-Content $PSScriptRoot\..\cgmanifest.json
|
||||||
Write-Verbose "$newCount registrations added" -Verbose
|
Write-Verbose "$count registrations created!" -Verbose
|
||||||
|
|
Loading…
Reference in a new issue