Add compliance to Coordinated build (#8798)

Add compliance to Coordinated build
  - Also switch to mac internal pool for release build
  - Also turn some duplicate tasks into templates
  - Also fix issue with vscode configuration which causes yaml files not to be recogized as yaml

.vscode/extensions.json

@@ -7,6 +7,7 @@
     "ms-vscode.csharp",
     "ms-vscode.PowerShell",
     "twxs.cmake",
-    "DavidAnson.vscode-markdownlint"
+    "DavidAnson.vscode-markdownlint",
+    "ms-azure-devops.azure-pipelines"
   ]
 }

.vscode/settings.json

@@ -22,19 +22,16 @@
 
   // Sets the codeformatting options to follow the given indent style in a way that is compatible with PowerShell syntax. For more information about the brace styles please refer to https://github.com/PoshCode/PowerShellPracticeAndStyle/issues/81.
   "powershell.codeFormatting.preset": "OTBS",
-  
+
   // Adds a space between a keyword and its associated scriptblock expression.
   "powershell.codeFormatting.whitespaceBeforeOpenBrace": true,
-  
+
   // Adds a space between a keyword (if, elseif, while, switch, etc) and its associated conditional expression.
   "powershell.codeFormatting.whitespaceBeforeOpenParen": true,
-  
+
   // Adds spaces before and after an operator ('=', '+', '-', etc.).
   "powershell.codeFormatting.whitespaceAroundOperator": true,
-  
+
   // Adds a space after a separator (',' and ';').
-  "powershell.codeFormatting.whitespaceAfterSeparator": true,
-  "files.associations": {
-    "*.yml": "azure-pipelines"
-  }
+  "powershell.codeFormatting.whitespaceAfterSeparator": true
 }

tools/releaseBuild/azureDevOps/diagram.puml

@@ -0,0 +1,61 @@
+@startuml
+
+' Define the build tasks as business processes
+archimate #Business "Build DEB"  as BuildDEB <<business-process>>
+archimate #Business "Build RPM"  as BuildRPM <<business-process>>
+archimate #Business "Build Alpine"  as BuildAlpine <<business-process>>
+archimate #Business "Build Linux-FxDependent"  as BuildLinuxFx <<business-process>>
+archimate #Business "Upload DEB"  as UploadDEB <<business-process>>
+archimate #Business "Upload RPM"  as UploadRPM <<business-process>>
+archimate #Business "Upload Alpine"  as UploadAlpine <<business-process>>
+archimate #Business "Upload Linux-FxDependent"  as UploadLinuxFx <<business-process>>
+
+archimate #Business "Build macOS"  as BuildMac <<business-process>>
+archimate #Business "macOS Package Signing and Upload"  as SignMac <<business-process>>
+
+archimate #Business "Build Windows - x64"  as BuildWinX64 <<business-process>>
+archimate #Business "Build Windows - x86"  as BuildWinX86 <<business-process>>
+archimate #Business "Build Windows - arm32"  as BuildWinArm32 <<business-process>>
+archimate #Business "Build Windows - arm64"  as BuildWinArm64 <<business-process>>
+archimate #Business "Build Windows - FxDependent"  as BuildWinFx <<business-process>>
+archimate #Business "Windows Package Signing and Upload"  as SignWin <<business-process>>
+
+archimate #Business "ComponentRegistration"  as BuildCG <<business-process>>
+
+archimate #Business "Compliance"  as Compliance <<business-process>>
+
+' Define finishing the build as a goal filled
+archimate #Goal "Finish"  as Finish <<goal-filled>>
+
+' map the various Upload task dependencies
+BuildDEB -down-> UploadDEB
+BuildRPM -down-> UploadRPM
+BuildLinuxFx -down-> UploadLinuxFx
+BuildAlpine -down-> UploadAlpine
+
+' map all of the SignMac task dependencies
+BuildMac -down-> SignMac
+
+' map all of the SignWin task dependencies
+BuildWinX64 -down-> SignWin
+BuildWinX86 -down-> SignWin
+BuildWinArm32 -down-> SignWin
+BuildWinArm64 -down-> SignWin
+BuildWinFx -down-> SignWin
+
+' map all of the Compliance task dependencies
+BuildWinX86 -down-> Compliance
+BuildWinX64 -down-> Compliance
+BuildWinFx -down-> Compliance
+
+' map all leafs to finish
+Compliance -down-> Finish
+UploadAlpine -down-> Finish
+UploadDEB -down-> Finish
+UploadRPM -down-> Finish
+UploadLinuxFx -down-> Finish
+SignWin -down-> Finish
+SignMac -down-> Finish
+BuildCG -down-> Finish
+
+@enduml

tools/releaseBuild/azureDevOps/releaseBuild.yml

@@ -85,3 +85,9 @@ jobs:
 
 - template: templates/mac-package-signing.yml
 
+- template: templates/compliance.yml
+  parameters:
+    parentJobs:
+      - build_windows_x64
+      - build_windows_x86
+      - build_windows_fxdependent

tools/releaseBuild/azureDevOps/templates/compliance.yml

@@ -0,0 +1,142 @@
+parameters:
+  parentJobs: []
+
+jobs:
+- job: compliance
+  displayName: Compliance
+  dependsOn:
+    ${{ parameters.parentJobs }}
+  pool:
+    name: Package ES CodeHub Lab E
+
+  steps:
+  - powershell: |
+      tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag $(ReleaseTagVar) -Variable "ReleaseTagVar"
+    displayName: 'Set ReleaseTag Variable'
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifacts'
+    inputs:
+      downloadType: specific
+      itemPattern: |
+        **/*.zip
+
+  - powershell: |
+      dir "$(System.ArtifactsDirectory)\*" -Recurse
+    displayName: 'Capture artifacts directory'
+    continueOnError: true
+
+  - powershell: |
+       $version = '$(ReleaseTagVar)'.Substring(1)
+       $vstsCommandString = "vso[task.setvariable variable=Version]$version"
+       Write-Host "sending " + $vstsCommandString
+       Write-Host "##$vstsCommandString"
+
+       $azureVersion = '$(ReleaseTagVar)'.ToLowerInvariant() -replace '\.', '-'
+       $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion"
+       Write-Host "sending " + $vstsCommandString
+       Write-Host "##$vstsCommandString"
+
+    displayName: 'Set Version Variable'
+
+  - template: expand-compliance.yml
+    parameters:
+      architecture: x86
+
+  - template: expand-compliance.yml
+    parameters:
+      architecture: x64
+
+  - template: expand-compliance.yml
+    parameters:
+      architecture: fxdependent
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3
+    displayName: 'Run MpCmdRun.exe'
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2
+    displayName: 'Run CredScan'
+    inputs:
+      suppressionsFile: tools/credScan/suppress.json
+      debugMode: false
+    continueOnError: true
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1
+    displayName: 'Run PoliCheck'
+    inputs:
+      targetType: F
+      optionsFC: 0
+      optionsXS: 0
+      optionsPE: '1|2|3|4'
+      optionsHMENABLE: 0
+      optionsRulesDBPath: '$(Build.SourcesDirectory)\tools\terms\PowerShell-Terms-Rules.mdb'
+      optionsFTPATH: '$(Build.SourcesDirectory)\tools\terms\FileTypeSet.xml'
+    continueOnError: true
+
+  # add RoslynAnalyzers
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-autoapplicability.AutoApplicability@1
+    displayName: 'Run AutoApplicability'
+    inputs:
+      ExternalRelease: true
+      IsSoftware: true
+      DataSensitivity: lbi
+    continueOnError: true
+
+  # add codeMetrics
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-vulnerabilityassessment.VulnerabilityAssessment@0
+    displayName: 'Run Vulnerability Assessment'
+    continueOnError: true
+
+  # FXCop is not applicable
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2
+    displayName: 'Publish Security Analysis Logs to Build Artifacts'
+    continueOnError: true
+
+  # PreFASt is not applicable
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1
+    displayName: 'Run PoliCheck'
+    inputs:
+      targetType: F
+      optionsFC: 0
+      optionsXS: 0
+      optionsPE: '1|2|3|4'
+      optionsHMENABLE: 0
+      optionsRulesDBPath: '$(Build.SourcesDirectory)\tools\terms\PowerShell-Terms-Rules.mdb'
+      optionsFTPATH: '$(Build.SourcesDirectory)\tools\terms\FileTypeSet.xml'
+    continueOnError: true
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-apiscan.APIScan@1
+    displayName: 'Run APIScan'
+    inputs:
+      softwareFolder: '$(CompliancePath)'
+      softwareName: PowerShell
+      softwareVersionNum: '$(ReleaseTagVar)'
+      isLargeApp: false
+      preserveTempFiles: true
+    continueOnError: true
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1
+    displayName: 'TSA upload to Codebase: PowerShellCore_201807 Stamp: Azure'
+    inputs:
+      tsaStamp: $(TsaStamp)
+      codeBaseName: $(CodeBaseName)
+      uploadFortifySCA: false
+      uploadFxCop: false
+      uploadModernCop: false
+      uploadPREfast: false
+      uploadRoslyn: false
+      uploadTSLint: false
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1
+    displayName: 'Create Security Analysis Report'
+    inputs:
+      TsvFile: false
+      APIScan: true
+      BinSkim: true
+      CredScan: true
+      PoliCheck: true
+      PoliCheckBreakOn: Severity2Above

tools/releaseBuild/azureDevOps/templates/expand-compliance.yml

@@ -0,0 +1,11 @@
+parameters:
+  architecture: x86
+
+steps:
+  - powershell: |
+      Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-${{ parameters.architecture }}.zip" -Destination "$(Build.StagingDirectory)\symbols\${{ parameters.architecture }}"
+    displayName: Expand symbols zip - ${{ parameters.architecture }}
+
+  - powershell: |
+      tools/releaseBuild/createComplianceFolder.ps1 -ArtifactFolder "$(Build.StagingDirectory)\symbols\${{ parameters.architecture }}" -VSTSVariableName 'CompliancePath'
+    displayName: Expand Compliance file - ${{ parameters.architecture }}

tools/releaseBuild/azureDevOps/templates/mac.yml

@@ -5,7 +5,7 @@ jobs:
 - job: ${{ parameters.jobName }}
   displayName: Build macOS
   condition: succeeded()
-  pool: Hosted macOS Preview
+  pool: Hosted Mac Internal
   steps:
   #- task: <task type name>@<version>
   #  inputs:

tools/releaseBuild/azureDevOps/templates/upload.yml

@@ -0,0 +1,40 @@
+parameters:
+  architecture: x86
+  msi: yes
+
+steps:
+- powershell: |
+      $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-${{ parameters.architecture }}.msi"
+      if(Test-Path -Path $packagePath)
+      {
+          Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath"
+      }
+  displayName: '[create script] upload signed msi - ${{ parameters.architecture }}'
+  condition: and(succeeded(), eq(variables['msi'], 'yes'))
+
+- task: AzureFileCopy@1
+  displayName: 'upload signed msi to Azure - ${{ parameters.architecture }}'
+  inputs:
+    SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-${{ parameters.architecture }}.msi'
+    azureSubscription: '$(AzureFileCopySubscription)'
+    Destination: AzureBlob
+    storage: '$(StorageAccount)'
+    ContainerName: '$(AzureVersion)'
+  condition: and(succeeded(), eq(variables['msi'], 'yes'))
+
+- powershell: |
+      $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-${{ parameters.architecture }}.zip"
+      if(Test-Path -Path $packagePath)
+      {
+          Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath"
+      }
+  displayName: '[create script] upload signed zip - ${{ parameters.architecture }}'
+
+- task: AzureFileCopy@1
+  displayName: 'upload signed zip to Azure - ${{ parameters.architecture }}'
+  inputs:
+    SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-${{ parameters.architecture }}.zip'
+    azureSubscription: '$(AzureFileCopySubscription)'
+    Destination: AzureBlob
+    storage: '$(StorageAccount)'
+    ContainerName: '$(AzureVersion)'

tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml

@@ -56,6 +56,16 @@ jobs:
     # Diagnostics is not critical it passes every time it runs
     continueOnError: true
 
+  - powershell: |
+       Import-Module $(Build.SourcesDirectory)/build.psm1 -Force
+       New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination $(Build.SourcesDirectory)/src/Modules
+
+       if(-not (Test-Path "$(Build.SourcesDirectory)/src/Modules/nuget.config"))
+       {
+           throw "nuget.config is not created"
+       }
+    displayName: 'Add nuget.config for AzDevOps feed for PSGallery modules '
+
   - powershell: |
        ./tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name win-x64-component-registration
     displayName: 'Build Windows Universal - Component Registration'

tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml

@@ -75,132 +75,28 @@ jobs:
       outPathRoot: '$(Build.StagingDirectory)\signedPackages'
     condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'))
 
-  - powershell: |
-       $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x64.msi"
-       if(Test-Path -Path $packagePath)
-       {
-           Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath"
-       }
-    displayName: '[Create script] upload signed msi - x64'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'))
+  - template: upload.yml
+    parameters:
+      architecture: x86
 
-  - task: AzureFileCopy@1
-    displayName: 'upload signed msi to Azure - x64'
-    inputs:
-      SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x64.msi'
-      azureSubscription: '$(AzureFileCopySubscription)'
-      Destination: AzureBlob
-      storage: '$(StorageAccount)'
-      ContainerName: '$(AzureVersion)'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'))
+  - template: upload.yml
+    parameters:
+      architecture: x64
 
-  - powershell: |
-       $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x86.msi"
-       if(Test-Path -Path $packagePath)
-       {
-           Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath"
-       }
-    displayName: '[create script] upload signed msi - x86'
+  - template: upload.yml
+    parameters:
+      architecture: arm32
+      msi: no
 
-  - task: AzureFileCopy@1
-    displayName: 'upload signed msi to Azure - x86'
-    inputs:
-      SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x86.msi'
-      azureSubscription: '$(AzureFileCopySubscription)'
-      Destination: AzureBlob
-      storage: '$(StorageAccount)'
-      ContainerName: '$(AzureVersion)'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'))
+  - template: upload.yml
+    parameters:
+      architecture: arm64
+      msi: no
 
-  - powershell: |
-       $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.zip"
-       if(Test-Path -Path $packagePath)
-       {
-           Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath"
-       }
-    displayName: '[Create script] upload signed zip - x64'
-
-  - task: AzureFileCopy@1
-    displayName: 'upload signed zip to Azure - x64'
-    inputs:
-      SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.zip'
-      azureSubscription: '$(AzureFileCopySubscription)'
-      Destination: AzureBlob
-      storage: '$(StorageAccount)'
-      ContainerName: '$(AzureVersion)'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'))
-
-  - powershell: |
-       $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.zip"
-       if(Test-Path -Path $packagePath)
-       {
-           Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath"
-       }
-    displayName: '[create script] upload signed zip - x86'
-
-  - task: AzureFileCopy@1
-    displayName: 'upload signed zip to Azure - x86'
-    inputs:
-      SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.zip'
-      azureSubscription: '$(AzureFileCopySubscription)'
-      Destination: AzureBlob
-      storage: '$(StorageAccount)'
-      ContainerName: '$(AzureVersion)'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'))
-
-  - powershell: |
-       $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.zip"
-       if(Test-Path -Path $packagePath)
-       {
-           Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath"
-       }
-    displayName: '[create script] upload signed zip - arm'
-
-  - task: AzureFileCopy@1
-    displayName: 'upload signed zip to Azure - arm'
-    inputs:
-      SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.zip'
-      azureSubscription: '$(AzureFileCopySubscription)'
-      Destination: AzureBlob
-      storage: '$(StorageAccount)'
-      ContainerName: '$(AzureVersion)'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'))
-
-  - powershell: |
-       $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.zip"
-       if(Test-Path -Path $packagePath)
-       {
-           Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath"
-       }
-    displayName: '[create script] upload signed zip - arm64'
-
-  - task: AzureFileCopy@1
-    displayName: 'upload signed zip to Azure - arm64'
-    inputs:
-      SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.zip'
-      azureSubscription: '$(AzureFileCopySubscription)'
-      Destination: AzureBlob
-      storage: '$(StorageAccount)'
-      ContainerName: '$(AzureVersion)'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'))
-
-  - powershell: |
-       $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-fxdependent.zip"
-       if(Test-Path -Path $packagePath)
-       {
-           Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath"
-       }
-    displayName: '[create script] upload signed zip - fxdependent'
-
-  - task: AzureFileCopy@1
-    displayName: 'upload signed zip to Azure - fxdependent'
-    inputs:
-      SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-fxdependent.zip'
-      azureSubscription: '$(AzureFileCopySubscription)'
-      Destination: AzureBlob
-      storage: '$(StorageAccount)'
-      ContainerName: '$(AzureVersion)'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'))
+  - template: upload.yml
+    parameters:
+      architecture: fxdependent
+      msi: no
 
   - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3
     displayName: 'Run MpCmdRun.exe'

tools/releaseBuild/macOS/vsts.yml

@@ -2,7 +2,7 @@ resources:
 - repo: self
   clean: true
 queue:
-  name: Hosted macOS Preview
+  name: Hosted Mac Internal
 name: macOS_$(Build.SourceBranchName)_$(Build.BuildId)
 steps:
 #- task: <task type name>@<version>