diff --git a/assets/additionalAttributions.txt b/assets/additionalAttributions.txt index c43aba0ef..6676ca99c 100644 --- a/assets/additionalAttributions.txt +++ b/assets/additionalAttributions.txt @@ -1,6 +1,9 @@ -## Used to generate a new TPN -## Copy this into the additional attributions fields -## Copy everything below here, but do not include this line + +------------------------------------------------------------------- + +------------------------------------------------------------------- + +Additional - ------------------------------------------------- Microsoft.PowerShell.Archive @@ -35,7 +38,7 @@ Microsoft.Management.Infrastructure Copyright (c) Microsoft Corporation -All rights reserved. +All rights reserved. MIT License diff --git a/tools/ComponentGovernance/ComponentGovernance.psm1 b/tools/ComponentGovernance/ComponentGovernance.psm1 new file mode 100644 index 000000000..53ec66d78 --- /dev/null +++ b/tools/ComponentGovernance/ComponentGovernance.psm1 @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +Class CgData { + [pscredential]$Pat + [string]$Organization + [string]$Project + + CgData ([pscredential]$Pat, [string]$Organization, [string]$Project) { + $this.Pat = $Pat + $this.Organization = $Organization + $this.Project = $Project + } +} + +# Get the Component Governance data needed to connect to the API +function Get-CgData { + if($Script:cgData){return $Script:cgData} + throw "First call Set-CgCredentials" +} + +# Get a Component Governance API URI +function Get-Uri { + param( + [Parameter(Mandatory=$true)] + [string]$PathPortion + ) + $cgData = Get-CgData + $baseUri = "https://governance.dev.azure.com/$($cgData.Organization)/$($cgData.Project)/_apis/componentgovernance/$PathPortion" + Write-Verbose "uri: $baseUri" -Verbose + return $baseUri +} + +# A class representing a Component Governance repository +class CgRepository { + #Json formatted summary information about this repository. Currently contains number of active nuget.config alerts. + [Object] $additionalInformation + + #The associations for this governed repository. For example, all service tree related entries. + [Object] $associations + + #Creator of the governed repository. + [Object] $createdBy + + [string] $createdDate + + [int] $id + + [string] $modifiedBy + + [string] $modifiedDate + + [string] $name + + #The policies that are configured in the governed repository. + [object[]] $policies + + [object] $projectReference + + [string] $repositoryMoniker + + [object] $repositoryOptions + + [object] $type + + [string] $url + + [object] $userRole +} + +# Gets a list of all repositories governed in the current project +function Get-CgRepositories { + $uri = Get-Uri -PathPortion "governedrepositories?api-version=6.1-preview.1" + $cgData = Get-CgData + [CgRepository[]] (Invoke-RestMethod -Uri $uri -Authentication Basic -Credential $cgData.Pat).value +} + +# Gets this PowerShell master repository +Function Get-CgPsRepository { + Get-CgRepositories | Where-Object {$_.name -eq 'PowerShell' -and $_.repositoryMoniker -notlike '*/*'} +} + +# Gets the Component Governance Snapshot Type (unique to each Pipeline and Job) in the repository +function Get-CgSnapshotType { + param( + [CgRepository] + $CgRepository + ) + + $id = $CgRepository.Id + $uri = Get-Uri -PathPortion "GovernedRepositories/$id/snapshottypes?api-version=6.1-preview.1" + $cgData = Get-CgData + (Invoke-RestMethod -Authentication Basic -Credential $cgData.Pat -Uri $uri).Value +} + +# Gets a Component Governance Notice for a given snapshot type +function Get-CgNotice { + param( + [CgRepository] + $CgRepository, + [int] + $SnapshotTypeId + ) + + $id = $CgRepository.Id + $uri = Get-Uri -PathPortion "GovernedRepositories/${id}/notice?snapshotTypeId=${SnapshotTypeId}&api-version=6.1-preview.1" + $cgData = Get-CgData + (Invoke-RestMethod -Authentication Basic -Credential $cgData.Pat -Uri $uri).content +} + +# Sets the Component Governance credentials used by other functions to connect to the API +function Set-CgCredentials { + param( + [Parameter(Mandatory=$true)] + [securestring] $Pat, + + [Parameter(Mandatory=$true)] + [string] $Organization, + + [Parameter(Mandatory=$true)] + [string] $Project + ) + + $pscred = [PSCredential]::new("PAT",$Pat) + $script:cgData = [CgData]::new($pscred, $Organization, $Project) +} + +Export-ModuleMember -Function @( + 'Get-CgRepositories' + 'Set-CgCredentials' + 'Get-CgRepository' + 'Get-CgPsRepository' + 'Get-CgSnapshotType' + 'Get-CgNotice' +) diff --git a/tools/findMissingNotices.ps1 b/tools/findMissingNotices.ps1 index 7e802e423..d1cf198f8 100644 --- a/tools/findMissingNotices.ps1 +++ b/tools/findMissingNotices.ps1 @@ -109,7 +109,7 @@ $targets | ForEach-Object { $name = $parts[0] $targetVersion = $parts[1] $pattern = [regex]::Escape($name) + " " - $tpnMatch = select-string -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern + $tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern if (!$tpnMatch) { if ($existingRegistrationTable.ContainsKey($name)) { $registrationVersion = $existingRegistrationTable.$name.Component.Version() diff --git a/tools/releaseBuild/azureDevOps/compliance.yml b/tools/releaseBuild/azureDevOps/compliance.yml new file mode 100644 index 000000000..d6aabe81d --- /dev/null +++ b/tools/releaseBuild/azureDevOps/compliance.yml @@ -0,0 +1,44 @@ +name: Compliance-$(Build.BuildId) + +trigger: none +pr: none + +schedules: + # Chrontab format, see https://en.wikipedia.org/wiki/Cron + # this is in UTC + - cron: '0 13 * * *' + branches: + include: + - master + +resources: + repositories: + - repository: ComplianceRepo + type: github + endpoint: ComplianceGHRepo + name: PowerShell/compliance + ref: master + +variables: + - name: DOTNET_CLI_TELEMETRY_OPTOUT + value: 1 + - name: POWERSHELL_TELEMETRY_OPTOUT + value: 1 + - name: NugetSecurityAnalysisWarningLevel + value: none + # Defines the variables AzureFileCopySubscription, StorageAccount, StorageAccountKey, StorageResourceGroup, StorageSubscriptionName + - group: 'Azure Blob variable group' + # Defines the variables CgPat, CgOrganization, and CgProject + - group: 'ComponentGovernance' + +stages: + - stage: compliance + dependsOn: [] + jobs: + - template: templates/compliance/compliance.yml + parameters: + parentJobs: [] + - template: templates/compliance/generateNotice.yml + parameters: + parentJobs: [] + diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index 280ae3551..7ab3b31b9 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -62,13 +62,6 @@ jobs: - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 displayName: 'Run Defender Scan' - - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 - displayName: 'Run CredScan' - inputs: - suppressionsFile: tools/credScan/suppress.json - debugMode: false - continueOnError: true - - task: securedevelopmentteam.vss-secure-development-tools.build-task-binskim.BinSkim@3 displayName: 'Run BinSkim ' inputs: @@ -80,18 +73,6 @@ jobs: AnalyzeStatistics: true continueOnError: true - - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1 - displayName: 'Run PoliCheck' - inputs: - targetType: F - optionsFC: 0 - optionsXS: 1 - optionsPE: '1|2|3|4' - optionsHMENABLE: 0 - optionsRulesDBPath: '$(Build.SourcesDirectory)\tools\terms\PowerShell-Terms-Rules.mdb' - optionsUEPath: $(Build.SourcesDirectory)\tools\terms\TermsExclusion.xml - continueOnError: true - # add RoslynAnalyzers - task: securedevelopmentteam.vss-secure-development-tools.build-task-autoapplicability.AutoApplicability@1 @@ -138,7 +119,10 @@ jobs: uploadModernCop: false uploadPREfast: false uploadRoslyn: false + uploadBinSkim: false uploadTSLint: false + uploadCredScan: false + uploadPoliCheck: false - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1 displayName: 'Create Security Analysis Report' diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml new file mode 100644 index 000000000..4d1007957 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml @@ -0,0 +1,90 @@ +parameters: + - name: parentJobs + type: jobList + +jobs: +- job: compliance + variables: + - name: runCodesignValidationInjection + value : false + - name: NugetSecurityAnalysisWarningLevel + value: none + + # Defines the variables APIScanClient, APIScanTenant and APIScanSecret + - group: PS-PS-APIScan + + displayName: Compliance + dependsOn: + ${{ parameters.parentJobs }} + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 + + # APIScan can take a long time + timeoutInMinutes: 180 + + steps: + - checkout: self + clean: true + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3 + displayName: 'Run CredScan' + inputs: + suppressionsFile: tools/credScan/suppress.json + debugMode: false + continueOnError: true + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1 + displayName: 'Run PoliCheck' + inputs: + # targetType F means file or folder and is the only applicable value and the default + targetType: F + # 1 to enable source code comment scanning, which is what we should do for open source + optionsFC: 1 + # recurse + optionsXS: 1 + # run for severity 1, 2, 3 and 4 issues + optionsPE: '1|2|3|4' + # disable history management + optionsHMENABLE: 0 + # Excluclusion access database + optionsRulesDBPath: '$(Build.SourcesDirectory)\tools\terms\PowerShell-Terms-Rules.mdb' + # Terms Exclusion xml file + optionsUEPath: $(Build.SourcesDirectory)\tools\terms\TermsExclusion.xml + continueOnError: true + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 + displayName: 'Publish Security Analysis Logs to Build Artifacts' + continueOnError: true + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1 + displayName: 'TSA upload to Codebase: PowerShellCore_201906' + inputs: + tsaVersion: TsaV2 + codeBaseName: 'PowerShellCore_201906' + uploadFortifySCA: false + uploadFxCop: false + uploadModernCop: false + uploadPREfast: false + uploadRoslyn: false + uploadTSLint: false + uploadCredScan: true + uploadPoliCheck: true + uploadBinSkim: false + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1 + displayName: 'Create Security Analysis Report' + inputs: + TsvFile: false + APIScan: false + BinSkim: false + CredScan: true + PoliCheck: true + PoliCheckBreakOn: Severity2Above + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(Build.SourcesDirectory)' + snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml b/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml new file mode 100644 index 000000000..c517a15b6 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml @@ -0,0 +1,64 @@ +parameters: + - name: parentJobs + type: jobList + +jobs: +- job: generateNotice + variables: + - name: runCodesignValidationInjection + value : false + - name: NugetSecurityAnalysisWarningLevel + value: none + + displayName: Generate Notice + dependsOn: + ${{ parameters.parentJobs }} + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 + + # APIScan can take a long time + timeoutInMinutes: 15 + + steps: + - checkout: self + clean: true + + - pwsh: | + Get-Content ./assets/additionalAttributions.txt | Out-File '$(System.ArtifactsDirectory)\additionalAttributions.txt' -Encoding utf8NoBOM -Force + Get-Content -Raw -Path '$(System.ArtifactsDirectory)\additionalAttributions.txt' + displayName: Get Additional Attributions + + - pwsh: | + Import-Module ./tools/ComponentGovernance + $pat = ConvertTo-SecureString -String "$(CgPat)" -AsPlainText -Force + Set-CgCredentials -Pat $pat -Organization $(CgOrganization) -Project $(CgProject) + $repo = Get-CgPsRepository + $typeId = (Get-CgSnapshotType -CgRepository $repo | where-object {$_.isTracked -eq 'True' -and $_.buildDisplayType -eq 'Coordinated Packages'}).typeId + $notice = Get-CgNotice -CgRepository $repo -SnapshotTypeId $typeId + $notice | Out-File '$(System.ArtifactsDirectory)\ThirdPartyNotices.txt' -Encoding utf8NoBOM -Force + Get-Content '$(System.ArtifactsDirectory)\additionalAttributions.txt' | Out-File '$(System.ArtifactsDirectory)\ThirdPartyNotices.txt' -Encoding utf8NoBOM -Force -Append + displayName: Get Notice + + - task: AzureFileCopy@4 + displayName: 'upload Notice' + inputs: + SourcePath: $(System.ArtifactsDirectory)\ThirdPartyNotices.txt + azureSubscription: '$(AzureFileCopySubscription)' + Destination: AzureBlob + storage: '$(StorageAccount)' + ContainerName: 'tpn' + resourceGroup: '$(StorageResourceGroup)' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(System.ArtifactsDirectory) + artifactName: notice + displayName: Publish notice artifacts + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(Build.SourcesDirectory)' + snapshotForceEnabled: true