Allow packaging from a zip package to allow for signing (#5418)

- Include a serialized version of PSOptions in an includesymbols zip
- Add a function which will create a zip package from the expanded includesymbols zip and a folder of signed files
- Add a function to restore an includesymbols zip as a build and populated PSOptions with the options

build.psm1

@@ -578,25 +578,37 @@ Fix steps:
 
     # download modules from powershell gallery.
     #   - PowerShellGet, PackageManagement, Microsoft.PowerShell.Archive
-    if ($PSModuleRestore) {
-        $ProgressPreference = "SilentlyContinue"
-        log "Restore PowerShell modules to $publishPath"
-
-        $modulesDir = Join-Path -Path $publishPath -ChildPath "Modules"
-
-        # Restore modules from myget feed
-        Restore-PSModule -Destination $modulesDir -Name @(
-            # PowerShellGet depends on PackageManagement module, so PackageManagement module will be installed with the PowerShellGet module.
-            'PowerShellGet'
-        )
-
-        # Restore modules from powershellgallery feed
-        Restore-PSModule -Destination $modulesDir -Name @(
-            'Microsoft.PowerShell.Archive'
-        ) -SourceLocation "https://www.powershellgallery.com/api/v2/"
+    if($PSModuleRestore)
+    {
+        Restore-PSModuleToBuild -PublishPath $publishPath
     }
 }
 
+function Restore-PSModuleToBuild
+{
+    param(
+        [Parameter(Mandatory)]
+        [string]
+        $PublishPath
+    )
+
+    $ProgressPreference = "SilentlyContinue"
+    log "Restore PowerShell modules to $publishPath"
+
+    $modulesDir = Join-Path -Path $publishPath -ChildPath "Modules"
+
+    # Restore modules from myget feed
+    Restore-PSModule -Destination $modulesDir -Name @(
+        # PowerShellGet depends on PackageManagement module, so PackageManagement module will be installed with the PowerShellGet module.
+        'PowerShellGet'
+    )
+
+    # Restore modules from powershellgallery feed
+    Restore-PSModule -Destination $modulesDir -Name @(
+        'Microsoft.PowerShell.Archive'
+    ) -SourceLocation "https://www.powershellgallery.com/api/v2/"
+}
+
 function Compress-TestContent {
     [CmdletBinding()]
     param(
@@ -745,13 +757,13 @@ function New-PSOptions {
     }
 
     return @{ RootInfo = [PSCustomObject]$RootInfo
-              Top = $Top;
-              Configuration = $Configuration;
-              Framework = $Framework;
-              Runtime = $Runtime;
-              Output = $Output;
-              CrossGen = $CrossGen
-              PSModuleRestore = $PSModuleRestore }
+              Top = $Top
+              Configuration = $Configuration
+              Framework = $Framework
+              Runtime = $Runtime
+              Output = $Output
+              CrossGen = $CrossGen.IsPresent
+              PSModuleRestore = $PSModuleRestore.IsPresent }
 }
 
 # Get the Options of the last build
@@ -759,6 +771,14 @@ function Get-PSOptions {
     return $script:Options
 }
 
+function Set-PSOptions {
+    param(
+        [PSObject]
+        $Options
+    )
+
+    $script:Options = $Options
+}
 
 function Get-PSOutput {
     [CmdletBinding()]param(

tools/.gitignore

@@ -1,2 +1,3 @@
 dotnet-install.sh
 dotnet-uninstall-debian-packages.sh
+ExpandedBuild/

tools/packaging/packaging.psd1

@@ -5,7 +5,7 @@ CompanyName="Microsoft Corporation"
 Copyright="Copyright (c) Microsoft Corporation. All rights reserved."
 ModuleVersion="1.0.0"
 PowerShellVersion="5.0"
-CmdletsToExport="Start-PSPackage"
+CmdletsToExport=@("Start-PSPackage",'New-PSSignedBuildZip')
 RootModule="packaging.psm1"
 RequiredModules = @("build")
 }

tools/packaging/packaging.psm1

@@ -180,6 +180,7 @@ function Start-PSPackage {
                 # Zip symbols.zip to the root package
                 $zipSource = Join-Path $symbolsSource -ChildPath '*'
                 $zipPath = Join-Path -Path $Source -ChildPath 'symbols.zip'
+                $Script:Options | ConvertTo-Json -Depth 3 | Out-File -Encoding utf8 -FilePath (Join-Path -Path $source -ChildPath 'psoptions.json')
                 Compress-Archive -Path $zipSource -DestinationPath $zipPath
             }
             finally
@@ -425,6 +426,75 @@ function New-TempFolder
 
     return $tempFolder
 }
+function New-PSSignedBuildZip
+{
+    param(
+        [Parameter(Mandatory)]
+        [string]$BuildPath,
+        [Parameter(Mandatory)]
+        [string]$SignedFilesPath,
+        [Parameter(Mandatory)]
+        [string]$DestinationFolder,
+        [parameter(HelpMessage='VSTS variable to set for path to zip')]
+        [string]$VstsVariableName
+    )
+
+    # Replace unsigned binaries with signed
+    $signedFilesFilter = Join-Path -Path $signedFilesPath -ChildPath '*'
+    Get-ChildItem -path $signedFilesFilter -Recurse | Select-Object -ExpandProperty FullName | Foreach-Object -Process {
+        $relativePath = $_.Replace($signedFilesPath,'')
+        $destination = Join-Path -Path $buildPath -ChildPath $relativePath
+        log "replacing $destination with $_"
+        Copy-Item -Path $_ -Destination $destination -force
+    }
+
+    $name = split-path -Path $BuildPath -Leaf
+    $zipLocationPath = Join-Path -Path $DestinationFolder -ChildPath "$name-signed.zip"
+    Compress-Archive -Path $BuildPath\* -DestinationPath $zipLocationPath
+    Write-Host "##vso[artifact.upload containerfolder=results;artifactname=$name]$zipLocationPath"
+    if ($VstsVariableName)
+    {
+        # set VSTS variable with path to package files
+        log "Setting $VstsVariableName to $zipLocationPath"
+        Write-Host "##vso[task.setvariable variable=$VstsVariableName]$zipLocationPath"
+    }
+    else 
+    {
+        return $zipLocationPath
+    }
+}
+
+function Expand-PSSignedBuild
+{
+    param(
+        [Parameter(Mandatory)]
+        [string]$BuildZip
+    )
+
+    $psModulePath = Split-Path -path $PSScriptRoot
+    # Expand unsigned build
+    $buildPath = Join-Path -path $psModulePath -childpath 'ExpandedBuild'
+    New-Item -path $buildPath -itemtype Directory -force
+    Expand-Archive -path $BuildZip -destinationpath $buildPath -Force
+    remove-item -Path (Join-Path -Path $buildPath -ChildPath '*.zip') -Recurse
+
+    $windowsExecutablePath = (Join-Path $buildPath -ChildPath 'pwsh.exe')
+
+    Restore-PSModuleToBuild -PublishPath $buildPath
+
+    $options = Get-Content -Path (Join-Path $buildPath -ChildPath 'psoptions.json') | ConvertFrom-Json
+
+    if(Test-Path -Path $windowsExecutablePath)
+    {
+        $options.Output = $windowsExecutablePath
+    }
+    else
+    {
+        throw 'Could not find pwsh'
+    }
+
+    Set-PSOptions -Options $options
+}
 
 function New-UnixPackage {
     [CmdletBinding(SupportsShouldProcess=$true)]

tools/releaseBuild/Images/microsoft_powershell_windowsservercore/PowerShellPackage.ps1

@@ -1,17 +1,29 @@
-[cmdletbinding()]
+[cmdletbinding(DefaultParameterSetName='default')]
 # PowerShell Script to clone, build and package PowerShell from specified fork and branch
 param (
     [string] $fork = 'powershell',
+
     [string] $branch = 'master',
+
     [string] $location = "$pwd\powershell",
+
     [string] $destination = "$env:WORKSPACE",
-    [ValidateSet("win7-x64", "win81-x64", "win10-x64", "win7-x86")]    
+
+    [ValidateSet("win7-x64", "win7-x86")]
     [string]$Runtime = 'win10-x64',
+
     [switch] $Wait,
+
     [ValidatePattern("^v\d+\.\d+\.\d+(-\w+\.\d+)?$")]
     [ValidateNotNullOrEmpty()]
     [string]$ReleaseTag,
-    [switch] $Symbols
+
+    [Parameter(Mandatory,ParameterSetName='IncludeSymbols')]
+    [switch] $Symbols,
+
+    [Parameter(Mandatory,ParameterSetName='packageSigned')]
+    [ValidatePattern("-signed.zip$")]
+    [string]$BuildZip
 )
 
 $releaseTagParam = @{}
@@ -59,15 +71,24 @@ try{
     Write-Verbose "Bootstrapping powershell build..." -verbose
     Start-PSBootstrap -Force -Package
 
-    Write-Verbose "Starting powershell build for RID: $Runtime and ReleaseTag: $ReleaseTag ..." -verbose
-    $buildParams = @{}
-    $buildParams['CrossGen'] = $true
-    if(!$Symbols.IsPresent)
+    if($PSCmdlet.ParameterSetName -eq 'packageSigned')
     {
-        $buildParams['PSModuleRestore'] = $true
+        Write-Verbose "Expanding signed build..." -verbose
+        Expand-PSSignedBuild -BuildZip $BuildZip
+        Remove-Item -Path $BuildZip
     }
+    else
+    {
+        Write-Verbose "Starting powershell build for RID: $Runtime and ReleaseTag: $ReleaseTag ..." -verbose
+        $buildParams = @{}
+        $buildParams['CrossGen'] = $true
+        if(!$Symbols.IsPresent)
+        {
+            $buildParams['PSModuleRestore'] = $true
+        }
 
-    Start-PSBuild -Clean -Runtime $Runtime -Configuration Release @releaseTagParam @buildParams
+        Start-PSBuild -Clean -Runtime $Runtime -Configuration Release @releaseTagParam @buildParams
+    }
 
     $pspackageParams = @{'Type'='msi'}
     if ($Runtime -ne 'win10-x64')
@@ -80,7 +101,7 @@ try{
         Write-Verbose "Starting powershell packaging(msi)..." -verbose
         Start-PSPackage @pspackageParams @releaseTagParam
     }
-    else 
+    else
     {
         $pspackageParams += @{'IncludeSymbols' = $true}
     }

tools/releaseBuild/build.json

@@ -55,6 +55,34 @@
         "BinaryBucket": "symbols",
         "BinariesExpected": 1,
         "VariableForExtractedBinariesPath": "Symbols_x86"
+    },
+    {
+        "Name": "win7-x64-package",
+        "RepoDestinationPath": "C:\\PowerShell",
+        "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x64 -ReleaseTag _ReleaseTag_",
+        "BuildDockerOptions": [
+            "-m",
+            "3968m"
+        ],
+        "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile",
+        "AdditionalContextFiles" :[ ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1"],
+        "DockerImageName": "ps-winsrvcore",
+        "BinaryBucket": "signed",
+        "BinariesExpected": 2
+    },
+    {
+        "Name": "win7-x86-package",
+        "RepoDestinationPath": "C:\\PowerShell",
+        "BuildCommand": "C:\\PowerShellPackage.ps1 -BuildZip _RepoDestinationPath_\\_BuildPackageName_ -location _RepoDestinationPath_ -destination _DockerVolume_ -Runtime win7-x86 -ReleaseTag _ReleaseTag_",
+        "BuildDockerOptions": [
+            "-m",
+            "3968m"
+        ],
+        "DockerFile": ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\DockerFile",
+        "AdditionalContextFiles" :[ ".\\tools\\releaseBuild\\Images\\microsoft_powershell_windowsservercore\\PowerShellPackage.ps1"],
+        "DockerImageName": "ps-winsrvcore",
+        "BinaryBucket": "signed",
+        "BinariesExpected": 2
     }
   ],
   "Linux": [

tools/releaseBuild/vstsbuild.ps1

@@ -1,7 +1,18 @@
+[cmdletbinding(DefaultParameterSetName='Build')]
 param(
+    [Parameter(ParameterSetName='packageSigned')]
     [Parameter(ParameterSetName='Build')]
     [ValidatePattern("^v\d+\.\d+\.\d+(-\w+\.\d+)?$")]
-    [string]$ReleaseTag
+    [string]$ReleaseTag,
+
+    # full paths to files to add to container to run the build
+    [Parameter(Mandatory,ParameterSetName='packageSigned')]
+    [string]
+    $BuildPath,
+
+    [Parameter(Mandatory,ParameterSetName='packageSigned')]
+    [string]
+    $SignedFilesPath
 )
 
 DynamicParam {
@@ -37,6 +48,29 @@ Begin {
 End {
     $ErrorActionPreference = 'Stop'
 
+    $additionalFiles = @()
+    $buildPackageName = $null
+    # If specified, Add package file to container
+    if ($BuildPath)
+    {
+        Import-Module (Join-Path -path $PSScriptRoot -childpath '..\..\build.psm1')
+        Import-Module (Join-Path -path $PSScriptRoot -childpath '..\packaging')
+
+        # Use temp as destination if not running in VSTS
+        $destFolder = $env:temp 
+        if($env:Build_ArtifactStagingDirectory)
+        {
+            # Use artifact staging if running in VSTS
+            $destFolder = $env:Build_ArtifactStagingDirectory
+        }
+
+        $BuildPackagePath = New-PSSignedBuildZip -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath -DestinationFolder $destFolder
+        Write-Host "##vso[artifact.upload containerfolder=results;artifactname=$name-singed.zip]$BuildPackagePath"
+        $buildPackageName = Split-Path -Path $BuildPackagePath -Leaf
+        $additionalFiles += $BuildPackagePath
+    }
+
+
     $psReleaseBranch = 'master'
     $psReleaseFork = 'PowerShell'
     $location = Join-Path -Path $PSScriptRoot -ChildPath 'PSRelease'
@@ -74,7 +108,12 @@ End {
         Import-Module "$location/dockerBasedBuild" -Force
         Clear-VstsTaskState
 
-        Invoke-Build -RepoPath $resolvedRepoRoot  -BuildJsonPath './tools/releaseBuild/build.json' -Name $Name -Parameters $PSBoundParameters
+        $buildParameters = @{
+            ReleaseTag = $ReleaseTag
+            BuildPackageName = $buildPackageName
+        }
+
+        Invoke-Build -RepoPath $resolvedRepoRoot  -BuildJsonPath './tools/releaseBuild/build.json' -Name $Name -Parameters $buildParameters -AdditionalFiles $AdditionalFiles
     }
     catch
     {