maxmustermann/PowerShell
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add Software Bill of Materials to the main packages (#16202)

Browse source
This commit is contained in:
Travis Plunk 2021-10-12 15:40:43 -07:00 committed by GitHub
parent e4367386b3
commit bd5dc97087
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 735 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
.vsts-ci/windows/templates/windows-packaging.yml
View file

@ -1,20 +1,40 @@
parameters:
pool: 'Hosted VS2017'
jobName: 'win_packaging'
architecture: 'x64'
channel: 'preview'
parentJobs: []
- name: pool
default: 'Hosted VS2017'
- name: jobName
default: 'win_packaging'
- name: architecture
default: 'x64'
- name: channel
default: 'preview'
jobs:
- job: ${{ parameters.jobName }}_${{ parameters.channel }}_${{ parameters.architecture }}
dependsOn:
${{ parameters.parentJobs }}
variables:
- name: repoFolder
value: PowerShell
- name: repoPath
value: $(Agent.BuildDirectory)\$(repoFolder)
- name: complianceRepoFolder
value: compliance
- name: complianceRepoPath
value: $(Agent.BuildDirectory)\$(complianceRepoFolder)
pool:
name: ${{ parameters.pool }}
displayName: Windows Packaging - ${{ parameters.architecture }} - ${{ parameters.channel }}
steps:
- checkout: self
clean: true
path: $(repoFolder)
- checkout: ComplianceRepo
clean: true
path: $(complianceRepoFolder)
- powershell: |
Get-ChildItem -Path env:
displayName: Capture environment
@ -27,9 +47,24 @@ jobs:
Invoke-CIInstall -SkipUser
displayName: Bootstrap
condition: succeeded()
workingDirectory: $(repoPath)
- pwsh: |
Import-Module .\tools\ci.psm1
New-CodeCoverageAndTestPackage
Invoke-CIFinish -Runtime win7-${{ parameters.architecture }} -channel ${{ parameters.channel }}
displayName: Build and Test Package
Invoke-CIFinish -Runtime win7-${{ parameters.architecture }} -channel ${{ parameters.channel }} -Stage Build
displayName: Build
workingDirectory: $(repoPath)
- template: Sbom.yml@ComplianceRepo
parameters:
BuildDropPath: '$(System.ArtifactsDirectory)/mainBuild'
Build_Repository_Uri: $(build.repository.uri)
displayName: SBOM
- pwsh: |
Import-Module .\tools\ci.psm1
New-CodeCoverageAndTestPackage
Invoke-CIFinish -Runtime win7-${{ parameters.architecture }} -channel ${{ parameters.channel }} -Stage Package
displayName: Package and Test
workingDirectory: $(repoPath)

11
.vsts-ci/windows/windows-packaging.yml
View file

@ -67,10 +67,17 @@ variables:
- name: __SuppressAnsiEscapeSequences
value: 1
- group: fakeNugetKey
- name: SBOMGenerator_Formats
value: spdx:2.2
resources:
- repo: self
clean: true
repositories:
- repository: ComplianceRepo
type: github
endpoint: PowerShell
name: PowerShell/compliance
ref: master
stages:
- stage: PackagingWin
displayName: Packaging for Windows

12
assets/wix/files.wxs
View file

@ -3057,6 +3057,16 @@
<Component Id="cmp2EB55EB6B44F4CF186F0BAFD6B33BCD6">
<File Id="filC629C6EFCF314F9BB78C56E0876C08EB" KeyPath="yes" Source="$(var.ProductSourcePath)\mscordaccore_$(var.FileArchitecture)_$(var.FileArchitecture)_6.0.21.45113.dll" />
</Component>
<Directory Name="_manifest" Id="dir81094D6B916B4BC8B1BA0E1DADB93A02">
<Directory Name="spdx_2.2" Id="dirA36413FB3A534FDF8657D62728080E9F">
<Component Id="cmp408A3AF905EB47ADA35FBC5A6B1465A0">
<File Id="fil2146848B6ACB45FCA8E9C8FEE3BECDD8" KeyPath="yes" Source="$(var.ProductSourcePath)\_manifest\spdx_2.2\manifest.spdx.json" />
</Component>
</Directory>
</Directory>
<Component Id="cmpA6276F9EAB41411AAAEC496E67A3DBAE">
<File Id="filC840646CDE814D33B16DA3A5BBDDC88D" KeyPath="yes" Source="$(var.ProductSourcePath)\psoptions.json" />
</Component>
</DirectoryRef>
</Fragment>
<Fragment>
@ -4042,6 +4052,8 @@
<ComponentRef Id="cmpA75DEF5617C54DA1937CB37D5824BF79" />
<ComponentRef Id="cmp957498053E01454DA1E21E6D7317DCD6" />
<ComponentRef Id="cmp2EB55EB6B44F4CF186F0BAFD6B33BCD6" />
<ComponentRef Id="cmp408A3AF905EB47ADA35FBC5A6B1465A0" />
<ComponentRef Id="cmpA6276F9EAB41411AAAEC496E67A3DBAE" />
</ComponentGroup>
</Fragment>
</Wix>

12
build.psm1
View file

@ -321,7 +321,8 @@ function Start-PSBuild {
[string]$ReleaseTag,
[switch]$Detailed,
[switch]$InteractiveAuth,
[switch]$SkipRoslynAnalyzers
[switch]$SkipRoslynAnalyzers,
[string]$PSOptionsPath
)
if ($ReleaseTag -and $ReleaseTag -notmatch "^v\d+\.\d+\.\d+(-(preview|rc)(\.\d{1,2})?)?$") {
@ -668,6 +669,15 @@ Fix steps:
if ($CI) {
Restore-PSPester -Destination (Join-Path $publishPath "Modules")
}
if ($PSOptionsPath) {
$resolvedPSOptionsPath = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($PSOptionsPath)
$parent = Split-Path -Path $resolvedPSOptionsPath
if (!(Test-Path $parent)) {
$null = New-Item -ItemType Directory -Path $parent
}
Save-PSOptions -PSOptionsPath $PSOptionsPath -Options $Options
}
}
function Restore-PSPackage

5
tools/WindowsCI.psm1
View file

@ -1,11 +1,6 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
if($PSVersionTable.PSEdition -ne 'Desktop')
{
throw 'Must be run from Windows PowerShell'
}
function New-LocalUser
{
<#

219
tools/ci.psm1
View file

@ -442,111 +442,156 @@ function Invoke-CIFinish
{
param(
[string] $Runtime = 'win7-x64',
[string] $Channel = 'preview'
[string] $Channel = 'preview',
[Validateset('Build','Package')]
[string[]] $Stage = ('Build','Package')
)
if($PSEdition -eq 'Core' -and ($IsLinux -or $IsMacOS))
{
if ($PSEdition -eq 'Core' -and ($IsLinux -or $IsMacOS) -and $Stage -contains 'Build') {
return New-LinuxPackage
}
$artifacts = New-Object System.Collections.ArrayList
try {
$buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/mainBuild"
if($Channel -eq 'preview')
{
$releaseTag = Get-ReleaseTag
if ($Stage -contains "Build") {
if ($Channel -eq 'preview') {
$releaseTag = Get-ReleaseTag
$previewVersion = $releaseTag.Split('-')
$previewPrefix = $previewVersion[0]
$previewLabel = $previewVersion[1].replace('.','')
$previewVersion = $releaseTag.Split('-')
$previewPrefix = $previewVersion[0]
$previewLabel = $previewVersion[1].replace('.','')
if(Test-DailyBuild)
{
$previewLabel= "daily{0}" -f $previewLabel
}
$prereleaseIteration = (get-date).Day
$preReleaseVersion = "$previewPrefix-$previewLabel.$prereleaseIteration"
# Build clean before backing to remove files from testing
Start-PSBuild -CrossGen -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean -Runtime $Runtime -output $buildFolder -PSOptionsPath "${buildFolder}/psoptions.json"
$options = Get-PSOptions
# Remove symbol files.
$filter = Join-Path -Path (Split-Path $options.Output) -ChildPath '*.pdb'
Write-Verbose "Removing symbol files from $filter" -Verbose
Remove-Item $filter -Force -Recurse
}
else {
$releaseTag = Get-ReleaseTag
$releaseTagParts = $releaseTag.split('.')
$preReleaseVersion = $releaseTagParts[0]+ ".9.9"
Write-Verbose "newPSReleaseTag: $preReleaseVersion" -Verbose
Start-PSBuild -CrossGen -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean -Runtime $Runtime -output $buildFolder -PSOptionsPath "${buildFolder}/psoptions.json"
$options = Get-PSOptions
# Remove symbol files.
$filter = Join-Path -Path (Split-Path $options.Output) -ChildPath '*.pdb'
Write-Verbose "Removing symbol files from $filter" -Verbose
Remove-Item $filter -Force -Recurse
}
# Set a variable, both in the current process and in AzDevOps for the packaging stage to get the release tag
$env:CI_FINISH_RELASETAG=$preReleaseVersion
$vstsCommandString = "vso[task.setvariable variable=CI_FINISH_RELASETAG]$preReleaseVersion"
Write-Verbose -Message "$vstsCommandString" -Verbose
Write-Host -Object "##$vstsCommandString"
$armBuildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/releaseArm32"
# produce win-arm and win-arm64 packages if it is a daily build
Start-PSBuild -Restore -Runtime win-arm -PSModuleRestore -Configuration 'Release' -ReleaseTag $releaseTag -output $armBuildFolder -PSOptionsPath "${armBuildFolder}-meta/psoptions.json" -Crossgen
$options = Get-PSOptions
# Remove symbol files.
$filter = Join-Path -Path (Split-Path $options.Output) -ChildPath '*.pdb'
Write-Verbose "Removing symbol files from $filter" -Verbose
Remove-Item $filter -Force -Recurse
$armBuildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/releaseArm64"
Start-PSBuild -Restore -Runtime win-arm64 -PSModuleRestore -Configuration 'Release' -ReleaseTag $releaseTag -output $armBuildFolder -PSOptionsPath "${armBuildFolder}-meta/psoptions.json" -Crossgen
$options = Get-PSOptions
# Remove symbol files.
$filter = Join-Path -Path (Split-Path $options.Output) -ChildPath '*.pdb'
Write-Verbose "Removing symbol files from $filter" -Verbose
Remove-Item $filter -Force -Recurse
}
if ($Stage -contains "Package") {
Restore-PSOptions -PSOptionsPath "${buildFolder}-meta/psoptions.json"
$preReleaseVersion = $env:CI_FINISH_RELASETAG
# Build packages $preReleaseVersion = "$previewPrefix-$previewLabel.$prereleaseIteration"
$packages = Start-PSPackage -Type msi, nupkg, zip, zip-pdb -ReleaseTag $preReleaseVersion -SkipReleaseChecks -WindowsRuntime $Runtime
foreach ($package in $packages) {
if (Test-Path $package -ErrorAction Ignore)
{
Write-Log "Package found: $package"
}
else
{
Write-Warning -Message "Package NOT found: $package"
}
if($package -is [string])
{
$null = $artifacts.Add($package)
}
elseif($package -is [pscustomobject] -and $package.psobject.Properties['msi'])
{
$null = $artifacts.Add($package.msi)
$null = $artifacts.Add($package.wixpdb)
}
}
# the packaging tests find the MSI package using env:PSMsiX64Path
$env:PSMsiX64Path = $artifacts | Where-Object { $_.EndsWith(".msi")}
$architechture = $Runtime.Split('-')[1]
$exePath = New-ExePackage -ProductVersion ($preReleaseVersion -replace '^v') -ProductTargetArchitecture $architechture -MsiLocationPath $env:PSMsiX64Path
Write-Verbose "exe Path: $exePath" -Verbose
$artifacts.Add($exePath)
$env:PSExePath = $exePath
$env:PSMsiChannel = $Channel
$env:PSMsiRuntime = $Runtime
# Install the latest Pester and import it
$maximumPesterVersion = '4.99'
Install-Module Pester -Force -SkipPublisherCheck -MaximumVersion $maximumPesterVersion
Import-Module Pester -Force -MaximumVersion $maximumPesterVersion
$testResultPath = Join-Path -Path $env:TEMP -ChildPath "win-package-$channel-$runtime.xml"
# start the packaging tests and get the results
$packagingTestResult = Invoke-Pester -Script (Join-Path $repoRoot '.\test\packaging\windows\') -PassThru -OutputFormat NUnitXml -OutputFile $testResultPath
Publish-TestResults -Title "win-package-$channel-$runtime" -Path $testResultPath
# fail the CI job if the tests failed, or nothing passed
if(-not $packagingTestResult -is [pscustomobject] -or $packagingTestResult.FailedCount -ne 0 -or $packagingTestResult.PassedCount -eq 0)
{
throw "Packaging tests failed ($($packagingTestResult.FailedCount) failed/$($packagingTestResult.PassedCount) passed)"
}
# only publish assembly nuget packages if it is a daily build and tests passed
if(Test-DailyBuild)
{
$previewLabel= "daily{0}" -f $previewLabel
}
$prereleaseIteration = (get-date).Day
$preReleaseVersion = "$previewPrefix-$previewLabel.$prereleaseIteration"
# Build clean before backing to remove files from testing
Start-PSBuild -CrossGen -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean -Runtime $Runtime
}
else {
$releaseTag = Get-ReleaseTag
$releaseTagParts = $releaseTag.split('.')
$preReleaseVersion = $releaseTagParts[0]+ ".9.9"
Write-Verbose "newPSReleaseTag: $preReleaseVersion" -Verbose
Start-PSBuild -CrossGen -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean -Runtime $Runtime
}
# Build packages $preReleaseVersion = "$previewPrefix-$previewLabel.$prereleaseIteration"
$packages = Start-PSPackage -Type msi,nupkg,zip,zip-pdb -ReleaseTag $preReleaseVersion -SkipReleaseChecks -WindowsRuntime $Runtime
$artifacts = New-Object System.Collections.ArrayList
foreach ($package in $packages) {
if (Test-Path $package -ErrorAction Ignore)
{
Write-Log "Package found: $package"
}
else
{
Write-Warning -Message "Package NOT found: $package"
}
if($package -is [string])
{
$null = $artifacts.Add($package)
}
elseif($package -is [pscustomobject] -and $package.psobject.Properties['msi'])
{
$null = $artifacts.Add($package.msi)
$null = $artifacts.Add($package.wixpdb)
}
}
# the packaging tests find the MSI package using env:PSMsiX64Path
$env:PSMsiX64Path = $artifacts | Where-Object { $_.EndsWith(".msi")}
$architechture = $Runtime.Split('-')[1]
$exePath = New-ExePackage -ProductVersion ($preReleaseVersion -replace '^v') -ProductTargetArchitecture $architechture -MsiLocationPath $env:PSMsiX64Path
Write-Verbose "exe Path: $exePath" -Verbose
$artifacts.Add($exePath)
$env:PSExePath = $exePath
$env:PSMsiChannel = $Channel
$env:PSMsiRuntime = $Runtime
# Install the latest Pester and import it
$maximumPesterVersion = '4.99'
Install-Module Pester -Force -SkipPublisherCheck -MaximumVersion $maximumPesterVersion
Import-Module Pester -Force -MaximumVersion $maximumPesterVersion
$testResultPath = Join-Path -Path $env:TEMP -ChildPath "win-package-$channel-$runtime.xml"
# start the packaging tests and get the results
$packagingTestResult = Invoke-Pester -Script (Join-Path $repoRoot '.\test\packaging\windows\') -PassThru -OutputFormat NUnitXml -OutputFile $testResultPath
Publish-TestResults -Title "win-package-$channel-$runtime" -Path $testResultPath
# fail the CI job if the tests failed, or nothing passed
if(-not $packagingTestResult -is [pscustomobject] -or $packagingTestResult.FailedCount -ne 0 -or $packagingTestResult.PassedCount -eq 0)
{
throw "Packaging tests failed ($($packagingTestResult.FailedCount) failed/$($packagingTestResult.PassedCount) passed)"
}
# only publish assembly nuget packages if it is a daily build and tests passed
if(Test-DailyBuild)
{
$nugetArtifacts = Get-ChildItem $PSScriptRoot\packaging\nugetOutput -ErrorAction SilentlyContinue -Filter *.nupkg | Select-Object -ExpandProperty FullName
if($nugetArtifacts)
{
$artifacts.AddRange(@($nugetArtifacts))
$nugetArtifacts = Get-ChildItem $PSScriptRoot\packaging\nugetOutput -ErrorAction SilentlyContinue -Filter *.nupkg | Select-Object -ExpandProperty FullName
if($nugetArtifacts)
{
$artifacts.AddRange(@($nugetArtifacts))
}
}
}
# produce win-arm and win-arm64 packages if it is a daily build
Start-PSBuild -Restore -Runtime win-arm -PSModuleRestore -Configuration 'Release' -ReleaseTag $releaseTag
$armBuildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/releaseArm32"
Restore-PSOptions -PSOptionsPath "${armBuildFolder}-meta/psoptions.json"
$arm32Package = Start-PSPackage -Type zip -WindowsRuntime win-arm -ReleaseTag $releaseTag -SkipReleaseChecks
$artifacts.Add($arm32Package)
Start-PSBuild -Restore -Runtime win-arm64 -PSModuleRestore -Configuration 'Release' -ReleaseTag $releaseTag
$armBuildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/releaseArm64"
Restore-PSOptions -PSOptionsPath "${armBuildFolder}-meta/psoptions.json"
$arm64Package = Start-PSPackage -Type zip -WindowsRuntime win-arm64 -ReleaseTag $releaseTag -SkipReleaseChecks
$artifacts.Add($arm64Package)
}

5
tools/packaging/packaging.psd1
View file

@ -7,9 +7,11 @@
PowerShellVersion = "5.0"
CmdletsToExport = @()
FunctionsToExport = @(
'Compress-ExePackageEngine'
'Expand-ExePackageEngine'
'Expand-PSSignedBuild'
'Compress-ExePackageEngine'
'Invoke-AzDevOpsLinuxPackageBuild'
'Invoke-AzDevOpsLinuxPackageCreation'
'New-DotnetSdkContainerFxdPackage'
'New-ExePackage'
'New-GlobalToolNupkg'
@ -19,6 +21,7 @@
'New-PSSignedBuildZip'
'Publish-NugetToMyGet'
'Start-PSPackage'
'Test-PackageManifest'
'Update-PSSignedBuildFolder'
)
RootModule = "packaging.psm1"

273
tools/packaging/packaging.psm1
View file

@ -77,11 +77,11 @@ function Start-PSPackage {
$WindowsRuntime, "Release"
} elseif ($MacOSRuntime) {
$MacOSRuntime, "Release"
} elseif ($Type -eq "tar-alpine") {
} elseif ($Type.Count -eq 1 -and $Type[0] -eq "tar-alpine") {
New-PSOptions -Configuration "Release" -Runtime "alpine-x64" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration }
} elseif ($Type -eq "tar-arm") {
} elseif ($Type.Count -eq 1 -and $Type[0] -eq "tar-arm") {
New-PSOptions -Configuration "Release" -Runtime "Linux-ARM" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration }
} elseif ($Type -eq "tar-arm64") {
} elseif ($Type.Count -eq 1 -and $Type[0] -eq "tar-arm64") {
if ($IsMacOS) {
New-PSOptions -Configuration "Release" -Runtime "osx-arm64" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration }
} else {
@ -103,12 +103,13 @@ function Start-PSPackage {
if ($Type -eq 'fxdependent') {
$NameSuffix = "win-fxdependent"
Write-Log "Packaging : '$Type'; Packaging Configuration: '$Configuration'"
Write-Log "Packaging : '$Type'; Packaging Configuration: '$Configuration', Runtime: '$Runtime'"
} elseif ($Type -eq 'fxdependent-win-desktop') {
$NameSuffix = "win-fxdependentWinDesktop"
Write-Log "Packaging : '$Type'; Packaging Configuration: '$Configuration'"
Write-Log "Packaging : '$Type'; Packaging Configuration: '$Configuration', Runtime: '$Runtime'"
} elseif ($MacOSRuntime) {
$NameSuffix = $MacOSRuntime
Write-Log "Packaging : '$Type'; Packaging Configuration: '$Configuration', Runtime: '$Runtime'"
} else {
Write-Log "Packaging RID: '$Runtime'; Packaging Configuration: '$Configuration'"
}
@ -313,9 +314,6 @@ function Start-PSPackage {
}
}
"min-size" {
# Remove symbol files, xml document files.
Remove-Item "$Source\*.pdb", "$Source\*.xml" -Force
# Add suffix '-gc' because this package is for the Guest Config team.
if ($Environment.IsWindows) {
$Arguments = @{
@ -344,11 +342,6 @@ function Start-PSPackage {
}
}
{ $_ -like "fxdependent*" } {
## Remove PDBs from package to reduce size.
if(-not $IncludeSymbols.IsPresent) {
Get-ChildItem $Source -Filter *.pdb | Remove-Item -Force
}
if ($Environment.IsWindows) {
$Arguments = @{
PackageNameSuffix = $NameSuffix
@ -690,7 +683,8 @@ function Update-PSSignedBuildFolder
[Parameter(Mandatory)]
[string]$BuildPath,
[Parameter(Mandatory)]
[string]$SignedFilesPath
[string]$SignedFilesPath,
[string[]] $RemoveFilter = ('*.pdb', '*.zip')
)
# Replace unsigned binaries with signed
@ -701,6 +695,11 @@ function Update-PSSignedBuildFolder
Write-Log "replacing $destination with $_"
Copy-Item -Path $_ -Destination $destination -Force
}
foreach($filter in $RemoveFilter) {
$removePath = Join-Path -Path $BuildPath -ChildPath $filter
Remove-Item -Path $removePath -Recurse -Force
}
}
@ -735,7 +734,7 @@ function Expand-PSSignedBuild
Restore-PSModuleToBuild -PublishPath $buildPath
$psOptionsPath = Join-Path $buildPath -ChildPath 'psoptions.json'
Restore-PSOptions -PSOptionsPath $psOptionsPath -Remove
Restore-PSOptions -PSOptionsPath $psOptionsPath
$options = Get-PSOptions
@ -1606,8 +1605,6 @@ function New-ZipPackage
$staging = "$PSScriptRoot/staging"
New-StagingFolder -StagingPath $staging -PackageSourcePath $PackageSourcePath
Get-ChildItem $staging -Filter *.pdb -Recurse | Remove-Item -Force
Compress-Archive -Path $staging\* -DestinationPath $zipLocationPath
}
@ -1621,7 +1618,6 @@ function New-ZipPackage
throw "Failed to create $zipLocationPath"
}
}
#TODO: Use .NET Api to do compresss-archive equivalent if the pscmdlet is not present
else
{
Write-Error -Message "Compress-Archive cmdlet is missing in this PowerShell version"
@ -1693,7 +1689,6 @@ function New-PdbZipPackage
throw "Failed to create $zipLocationPath"
}
}
#TODO: Use .NET Api to do compresss-archive equivalent if the pscmdlet is not present
else
{
Write-Error -Message "Compress-Archive cmdlet is missing in this PowerShell version"
@ -3001,8 +2996,6 @@ function New-MSIPackage
$staging = "$PSScriptRoot/staging"
New-StagingFolder -StagingPath $staging -PackageSourcePath $ProductSourcePath
Get-ChildItem $staging -Filter *.pdb -Recurse | Remove-Item -Force
$assetsInSourcePath = Join-Path $staging 'assets'
New-Item $assetsInSourcePath -type directory -Force | Write-Verbose
@ -4023,3 +4016,241 @@ function New-GlobalToolNupkg
New-NugetPackage -NuSpecPath $_.RootFolder -PackageDestinationPath $DestinationPath
}
}
${mainLinuxBuildFolder} = 'pwshLinuxBuild'
${minSizeLinuxBuildFolder} = 'pwshLinuxBuildMinSize'
${arm32LinuxBuildFolder} = 'pwshLinuxBuildArm32'
${arm64LinuxBuildFolder} = 'pwshLinuxBuildArm64'
<#
Used in Azure DevOps Yaml to package all the linux packages for a channel.
#>
function Invoke-AzDevOpsLinuxPackageCreation {
param(
[switch]
$LTS,
[Parameter(Mandatory)]
[ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d{1,2})?)?$")]
[ValidateNotNullOrEmpty()]
[string]$ReleaseTag,
[Parameter(Mandatory)]
[ValidateSet('fxdependent', 'alpine', 'deb', 'rpm')]
[String]$BuildType
)
if (!${env:SYSTEM_ARTIFACTSDIRECTORY}) {
throw "Must be run in Azure DevOps"
}
try {
Write-Verbose "Packaging '$BuildType'-LTS:$LTS for $ReleaseTag ..." -Verbose
Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${mainLinuxBuildFolder}-meta\psoptions.json"
$releaseTagParam = @{ 'ReleaseTag' = $ReleaseTag }
switch ($BuildType) {
'fxdependent' {
Start-PSPackage -Type 'fxdependent' @releaseTagParam -LTS:$LTS
}
'alpine' {
Start-PSPackage -Type 'tar-alpine' @releaseTagParam -LTS:$LTS
}
'rpm' {
Start-PSPackage -Type 'rpm' @releaseTagParam -LTS:$LTS
}
default {
Start-PSPackage @releaseTagParam -LTS:$LTS -Type 'deb', 'tar'
}
}
if ($BuildType -eq 'deb') {
Start-PSPackage -Type tar @releaseTagParam -LTS:$LTS
Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${minSizeLinuxBuildFolder}-meta\psoptions.json"
Write-Verbose -Verbose "---- Min-Size ----"
Write-Verbose -Verbose "options.Output: $($options.Output)"
Write-Verbose -Verbose "options.Top $($options.Top)"
Start-PSPackage -Type min-size @releaseTagParam -LTS:$LTS
## Create 'linux-arm' 'tar.gz' package.
## Note that 'linux-arm' can only be built on Ubuntu environment.
Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${arm32LinuxBuildFolder}-meta\psoptions.json"
Start-PSPackage -Type tar-arm @releaseTagParam -LTS:$LTS
## Create 'linux-arm64' 'tar.gz' package.
## Note that 'linux-arm64' can only be built on Ubuntu environment.
Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${arm64LinuxBuildFolder}-meta\psoptions.json"
Start-PSPackage -Type tar-arm64 @releaseTagParam -LTS:$LTS
}
}
catch {
Get-Error
throw
}
}
<#
Used in Azure DevOps Yaml to do all the builds needed for all Linux packages for a channel.
#>
function Invoke-AzDevOpsLinuxPackageBuild {
param (
[Parameter(Mandatory)]
[ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d{1,2})?)?$")]
[ValidateNotNullOrEmpty()]
[string]$ReleaseTag,
[Parameter(Mandatory)]
[ValidateSet('fxdependent', 'alpine', 'deb', 'rpm')]
[String]$BuildType
)
if (!${env:SYSTEM_ARTIFACTSDIRECTORY}) {
throw "Must be run in Azure DevOps"
}
try {
Write-Verbose "Building '$BuildType' for $ReleaseTag ..." -Verbose
$releaseTagParam = @{ 'ReleaseTag' = $ReleaseTag }
$buildParams = @{ Configuration = 'Release'; PSModuleRestore = $true; Restore = $true }
switch ($BuildType) {
'fxdependent' {
$buildParams.Add("Runtime", "fxdependent")
}
'alpine' {
$buildParams.Add("Runtime", 'alpine-x64')
# We are cross compiling, so we can't generate experimental features
$buildParams.Add("SkipExperimentalFeatureGeneration", $true)
}
default {
$buildParams.Add("Crossgen", $true)
}
}
$buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${mainLinuxBuildFolder}"
Start-PSBuild @buildParams @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json"
# Remove symbol files.
Remove-Item "${buildFolder}\*.pdb" -Force
if ($BuildType -eq 'deb') {
## Build 'min-size'
$options = Get-PSOptions
Write-Verbose -Verbose "---- Min-Size ----"
Write-Verbose -Verbose "options.Output: $($options.Output)"
Write-Verbose -Verbose "options.Top $($options.Top)"
$binDir = Join-Path -Path $options.Top -ChildPath 'bin'
if (Test-Path -Path $binDir) {
Write-Verbose -Verbose "Remove $binDir, to get a clean build for min-size package"
Remove-Item -Path $binDir -Recurse -Force
}
$buildParams['Crossgen'] = $false
$buildParams['ForMinimalSize'] = $true
$buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${minSizeLinuxBuildFolder}"
Start-PSBuild -Clean @buildParams @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json"
# Remove symbol files, xml document files.
Remove-Item "${buildFolder}\*.pdb", "${buildFolder}\*.xml" -Force
## Build 'linux-arm' and create 'tar.gz' package for it.
## Note that 'linux-arm' can only be built on Ubuntu environment.
$buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${arm32LinuxBuildFolder}"
Start-PSBuild -Configuration Release -Restore -Runtime linux-arm -PSModuleRestore @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json"
# Remove symbol files.
Remove-Item "${buildFolder}\*.pdb" -Force
$buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${arm64LinuxBuildFolder}"
Start-PSBuild -Configuration Release -Restore -Runtime linux-arm64 -PSModuleRestore @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json"
# Remove symbol files.
Remove-Item "${buildFolder}\*.pdb" -Force
}
}
catch {
Get-Error
throw
}
}
enum PackageManifestResultStatus {
Mismatch
Match
MissingFromManifest
MissingFromPackage
}
class PackageManifestResult {
[string] $File
[string] $ExpectedHash
[string] $ActualHash
[PackageManifestResultStatus] $Status
}
function Test-PackageManifest {
param (
[Parameter(Mandatory)]
[string]
$PackagePath
)
Begin {
$spdxManifestPath = Join-Path $PackagePath -ChildPath "/_manifest/spdx_2.2/manifest.spdx.json"
$man = Get-Content $spdxManifestPath -ErrorAction Stop | convertfrom-json
$inManifest = @()
}
Process {
Write-Verbose "Processing $($man.files) files..." -verbose
$man.files | ForEach-Object {
$filePath = Join-Path $PackagePath -childPath $_.fileName
$checksumObj = $_.checksums | Where-Object {$_.algorithm -eq 'sha256'}
$sha256 = $checksumObj.checksumValue
$actualHash = $null
$actualHash = (Get-FileHash -Path $filePath -Algorithm sha256 -ErrorAction SilentlyContinue).Hash
$inManifest += $filePath
if($actualHash -ne $sha256) {
$status = [PackageManifestResultStatus]::Mismatch
if (!$actualHash) {
$status = [PackageManifestResultStatus]::MissingFromPackage
}
[PackageManifestResult] $result = @{
File = $filePath
ExpectedHash = $sha256
ActualHash = $actualHash
Status = $status
}
Write-Output $result
}
else {
[PackageManifestResult] $result = @{
File = $filePath
ExpectedHash = $sha256
ActualHash = $actualHash
Status = [PackageManifestResultStatus]::Match
}
Write-Output $result
}
}
Get-ChildItem $PackagePath -recurse | Select-Object -ExpandProperty FullName | foreach-object {
if(!$inManifest -contains $_) {
$actualHash = (get-filehash -Path $_ -algorithm sha256 -erroraction silentlycontinue).Hash
[PackageManifestResult] $result = @{
File = $_
ExpectedHash = $null
ActualHash = $actualHash
Status = [PackageManifestResultStatus]::MissingFromManifest
}
Write-Output $result
}
}
}
}

110
tools/packaging/releaseTests/sbom.tests.ps1 Normal file
View file

@ -0,0 +1,110 @@
Describe "Verify SBOMs" {
BeforeAll {
Write-Verbose "In Describe BeforeAll" -Verbose
Import-Module $PSScriptRoot/../../../build.psm1
Import-Module $PSScriptRoot/../packaging.psd1 -Force
$matchCases = @()
$testCases = @()
$missingFromPackageCases = @()
$missingFromManifestCases = @()
Write-Verbose "${env:PACKAGE_FOLDER}" -Verbose
Get-ChildItem $env:PACKAGE_FOLDER -Filter *.zip | ForEach-Object {
Write-Verbose "Found $($_.Name)..." -Verbose
$testCases += @{
FilePath = $_.FullName
Name = $_.Name
}
}
foreach($case in $testCases) {
$name = $case.Name
Write-Verbose "Testing $name..." -Verbose
$extractedPath = Join-Path Testdrive:\ -ChildPath ([System.io.path]::GetRandomFileName())
$null = New-Item -Path $extractedPath -ItemType Directory -Force
Expand-Archive -Path $case.FilePath -DestinationPath $extractedPath
$manifestPath = Join-Path $extractedPath -ChildPath '/_manifest/spdx_2.2/manifest.spdx.json'
It "$name has a BOM" {
$manifestPath | Should -Exist
}
Test-PackageManifest -PackagePath $extractedPath | ForEach-Object {
$status = $_.Status
$expectedHash = $_.ExpectedHash
$actual = $_.ActualHash
$file = $_.File
switch($status) {
# cover match and mismatch
default {
$matchCases += @{
Name = $name
File = $file
ActualHash = $actual
ExpectedHash = $ExpectedHash
Status = $status
}
}
"MissingFromPackage" {
$missingFromPackageCases = @{
Name = $name
File = $file
ActualHash = $actual
ExpectedHash = $ExpectedHash
Status = $status
}
}
"MissingFromManifest" {
$missingFromManifestCases = @{
Name = $name
File = $file
ActualHash = $actual
ExpectedHash = $ExpectedHash
Status = $status
}
}
}
}
}
}
Context "Zip files" {
BeforeAll {
Write-Verbose "In Context BeforeAll" -Verbose
}
It "<name> should have <file> with matching hash" -TestCases $matchCases {
param(
$Name,
$File,
$ActualHash,
$ExpectedHash,
$Status
)
$status | Should -Be "Match" -Because "$actualHash should be $expectedHash"
}
It "<name> should have <file> with matching hash" -TestCases $missingFromPackageCases -Skip:($missingFromPackageCases.Count -eq 0) {
param(
$Name,
$File,
$ActualHash,
$ExpectedHash,
$Status
)
$status | Should -Be "Match" -Because "All files in manifest should exist in package"
}
It "Manifest for <name> should have <file>" -TestCases $missingFromManifestCases -Skip:($missingFromManifestCases.Count -eq 0) {
param(
$Name,
$File,
$ActualHash,
$ExpectedHash,
$Status
)
$status | Should -Be "Match" -Because "All files in package should exist in manifest"
}
}
}

8
tools/releaseBuild/azureDevOps/releaseBuild.yml
View file

@ -13,6 +13,10 @@ pr:
parameters:
- name: ForceAzureBlobDelete
displayName: Delete Azure Blob
type: string
values:
- true
- false
default: false
resources:
@ -36,6 +40,10 @@ variables:
value: $[counter(variables['branchCounterKey'], 1)]
- name: ForceAzureBlobDelete
value: ${{ parameters.ForceAzureBlobDelete }}
- name: Github_Build_Repository_Uri
value: https://github.com/powershell/powershell
- name: SBOMGenerator_Formats
value: spdx:2.2
stages:
- stage: prep

17
tools/releaseBuild/azureDevOps/releasePipeline.yml
View file

@ -17,6 +17,8 @@ variables:
value : false
- name: NugetSecurityAnalysisWarningLevel
value: none
- name: skipComponentGovernanceDetection
value: true
stages:
- stage: MSIXBundle
@ -76,17 +78,26 @@ stages:
globalToolExeName: 'pwsh'
globalToolPackageName: 'PowerShell.Linux.x64'
- stage: ValidatePkgNames
- stage: StaticPkgValidation
dependsOn: []
displayName: Package Names validation
displayName: Static package validation
jobs:
- job: ValidatePkgNames
displayName: Validate Package Names
pool:
vmImage: windows-latest
variables:
- group: 'Azure Blob variable group'
steps:
- template: templates/release-ValidatePackageNames.yml
- job: ValidatePkgBOM
displayName: Validate Package BOM
pool:
vmImage: windows-latest
variables:
- group: 'Azure Blob variable group'
steps:
- template: templates/release-ValidatePackageBOM.yml
- stage: StartDocker
dependsOn: []
@ -127,7 +138,7 @@ stages:
- MSIXBundle
- ValidateSDK
- PRCreation
- ValidatePkgNames
- StaticPkgValidation
- StartDocker
- ManualValidation
- ReleaseAutomation

101
tools/releaseBuild/azureDevOps/templates/linux.yml
View file

@ -32,36 +32,103 @@ jobs:
parameters:
ReleaseTagVar: $(ReleaseTagVar)
- powershell: |
Write-Verbose -Verbose "$(build)"
- pwsh: |
# create folder
sudo mkdir /PowerShell
if ('$(build)' -in 'alpine', 'fxdependent' -and '$(ReleaseTagVar)' -match '6.0.*')
{
$vstsCommandString = "vso[task.setvariable variable=SkipBuild]true"
}
else
{
$vstsCommandString = "vso[task.setvariable variable=SkipBuild]false"
}
displayName: 'Skip Alpine or fxdependent for PS v6.0.*'
# make the current user the owner
sudo chown $env:USER /PowerShell
displayName: 'Create /PowerShell'
- template: cloneToOfficialPath.yml
- template: insert-nuget-config-azfeed.yml
parameters:
repoRoot: $(REPOROOT)
repoRoot: $(PowerShellRoot)
- powershell: |
import-module "$env:REPOROOT/build.psm1"
import-module "$env:POWERSHELLROOT/build.psm1"
Sync-PSTags -AddRemoteIfMissing
displayName: SyncTags
condition: and(succeeded(), ne(variables['SkipBuild'], 'true'))
workingDirectory: $(PowerShellRoot)
- powershell: |
$env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)'
& "$env:REPOROOT/tools/releaseBuild/vstsbuild.ps1" -ReleaseTag $(ReleaseTagVar) -Name '$(build)'
$env:AzDevOpsFeedPAT2 = $null
Import-Module "$env:POWERSHELLROOT/build.psm1"
displayName: 'Build and package'
Start-PSBootstrap -Package
displayName: 'Bootstrap'
condition: and(succeeded(), ne(variables['SkipBuild'], 'true'))
workingDirectory: $(PowerShellRoot)
- powershell: |
Import-Module "$env:POWERSHELLROOT/build.psm1"
Import-Module "$env:POWERSHELLROOT/tools/packaging"
Invoke-AzDevOpsLinuxPackageBuild -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)'
displayName: 'Build'
condition: and(succeeded(), ne(variables['SkipBuild'], 'true'))
workingDirectory: $(PowerShellRoot)
- template: Sbom.yml@ComplianceRepo
parameters:
BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuild'
Build_Repository_Uri: $(Github_Build_Repository_Uri)
displayName: ${{ parameters.buildName }} SBOM
- ${{ if eq(variables.build,'deb') }} :
- template: Sbom.yml@ComplianceRepo
parameters:
BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildMinSize'
Build_Repository_Uri: $(Github_Build_Repository_Uri)
displayName: MinSize SBOM
- ${{ if eq(variables.build,'deb') }} :
- template: Sbom.yml@ComplianceRepo
parameters:
BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm32'
Build_Repository_Uri: $(Github_Build_Repository_Uri)
displayName: Arm32 SBOM
- ${{ if eq(variables.build,'deb') }} :
- template: Sbom.yml@ComplianceRepo
parameters:
BuildDropPath: '$(System.ArtifactsDirectory)/pwshLinuxBuildArm64'
Build_Repository_Uri: $(Github_Build_Repository_Uri)
displayName: Arm64 SBOM
- powershell: |
Import-Module "$env:POWERSHELLROOT/build.psm1"
Import-Module "$env:POWERSHELLROOT/tools/packaging"
$metadata = Get-Content "$env:POWERSHELLROOT/tools/metadata.json" -Raw | ConvertFrom-Json
$LTS = $metadata.LTSRelease
Write-Verbose -Verbose -Message "LTS is set to: $LTS"
Invoke-AzDevOpsLinuxPackageCreation -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)'
if ($LTS) {
Write-Verbose -Verbose "Packaging LTS"
Invoke-AzDevOpsLinuxPackageCreation -LTS -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)'
}
displayName: 'Package'
condition: and(succeeded(), ne(variables['SkipBuild'], 'true'))
workingDirectory: $(PowerShellRoot)
- powershell: |
$linuxPackages = Get-ChildItem "$env:POWERSHELLROOT/powershell*" -Include *.deb,*.rpm,*.tar.gz
$bucket = 'release'
foreach ($linuxPackage in $linuxPackages)
{
$filePath = $linuxPackage.FullName
Write-Verbose "Publishing $filePath to $bucket" -Verbose
Write-Host "##vso[artifact.upload containerfolder=$bucket;artifactname=$bucket]$filePath"
}
displayName: Publish artifacts
condition: and(succeeded(), ne(variables['SkipBuild'], 'true'))
workingDirectory: $(PowerShellRoot)
- job: upload_${{ parameters.buildName }}
displayName: ${{ parameters.uploadDisplayName }} ${{ parameters.buildName }}

5
tools/releaseBuild/azureDevOps/templates/mac-package-build.yml
View file

@ -79,6 +79,11 @@ jobs:
displayName: Merge signed files with Build
condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true'))
- template: Sbom.yml@ComplianceRepo
parameters:
BuildDropPath: '$(System.ArtifactsDirectory)/$(SymbolsFolder)'
Build_Repository_Uri: $(Github_Build_Repository_Uri)
- pwsh: |
Import-Module $(PowerShellRoot)/build.psm1 -Force
Import-Module $(PowerShellRoot)/tools/packaging -Force

4
tools/releaseBuild/azureDevOps/templates/release-SetReleaseTagAndContainerName.yml
View file

@ -2,12 +2,12 @@ steps:
- pwsh: |
$variable = 'releaseTag'
$branch = $ENV:BUILD_SOURCEBRANCH
if($branch -notmatch '^.*(release[-/])')
if($branch -notmatch '^.*((release/|rebuild/.*rebuild))')
{
throw "Branch name is not in release format: '$branch'"
}
$releaseTag = $Branch -replace '^.*(release[-/])'
$releaseTag = $Branch -replace '^.*((release|rebuild)/)'
$vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag"
Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose
Write-Host -Object "##$vstsCommandString"

43
tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml Normal file
View file

@ -0,0 +1,43 @@
steps:
- checkout: self
clean: true
- pwsh: |
Get-ChildItem ENV:
displayName: Capture environment
- template: release-SetReleaseTagAndContainerName.yml
- pwsh: |
$name = "{0}_{1:x}" -f '$(releaseTag)', (Get-Date).Ticks
Write-Host $name
Write-Host "##vso[build.updatebuildnumber]$name"
displayName: Set Release Name
- pwsh: |
$azcopy = "C:\Program Files (x86)\Microsoft SDKs\Azure\AzCopy\AzCopy.exe"
& $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey)
displayName: Download Azure Artifacts
- pwsh: |
Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty Name
displayName: Capture Artifact Listing
- pwsh: |
Install-module Pester -Scope CurrentUser -Force -MaximumVersion 4.99
displayName: Install Pester
condition: succeededOrFailed()
- pwsh: |
Import-module './build.psm1'
Import-module './tools/packaging'
$env:PACKAGE_FOLDER = '$(System.ArtifactsDirectory)'
$path = Join-Path -Path $pwd -ChildPath './packageReleaseTests.xml'
$results = invoke-pester -Script './tools/packaging/releaseTests' -OutputFile $path -OutputFormat NUnitXml -PassThru
Write-Host "##vso[results.publish type=NUnit;mergeResults=true;runTitle=Package Release Tests;publishRunAttachments=true;resultFiles=$path;]"
if($results.TotalCount -eq 0 -or $results.FailedCount -gt 0)
{
throw "Package Release Tests failed"
}
displayName: Run packaging release tests

5
tools/releaseBuild/azureDevOps/templates/windows-packaging.yml
View file

@ -227,6 +227,11 @@ jobs:
displayName: Merge ThirdParty signed files with Build
condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true'))
- template: Sbom.yml@ComplianceRepo
parameters:
BuildDropPath: '$(System.ArtifactsDirectory)\$(SymbolsFolder)'
Build_Repository_Uri: $(Github_Build_Repository_Uri)
- powershell: |
Import-Module $(PowerShellRoot)/build.psm1 -Force
Import-Module $(PowerShellRoot)/tools/packaging -Force