From 5a03c96aa47d13e44dead66d9592770a89a9f434 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Fri, 23 Oct 2020 11:42:24 -0700 Subject: [PATCH 01/21] Add checkout step to release build templates (#13840) # Conflicts: # tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml # tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml # tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml --- tools/releaseBuild/azureDevOps/WindowsBuild.yml | 1 - tools/releaseBuild/azureDevOps/releaseBuild.yml | 5 ----- .../azureDevOps/templates/checkAzureContainer.yml | 2 ++ tools/releaseBuild/azureDevOps/templates/compliance.yml | 3 +++ tools/releaseBuild/azureDevOps/templates/json.yml | 3 +++ tools/releaseBuild/azureDevOps/templates/linux.yml | 2 ++ .../azureDevOps/templates/mac-file-signing.yml | 2 ++ .../azureDevOps/templates/mac-package-build.yml | 3 +++ .../azureDevOps/templates/mac-package-signing.yml | 2 ++ tools/releaseBuild/azureDevOps/templates/mac.yml | 2 ++ tools/releaseBuild/azureDevOps/templates/nuget.yml | 2 ++ .../azureDevOps/templates/release-GlobalToolTest.yml | 3 +++ .../azureDevOps/templates/release-SDKTests.yml | 3 +++ .../azureDevOps/templates/release-UpdateDepsJson.yml | 3 +++ tools/releaseBuild/azureDevOps/templates/testartifacts.yml | 3 +++ .../releaseBuild/azureDevOps/templates/vpackReleaseJob.yml | 2 ++ tools/releaseBuild/azureDevOps/templates/windows-build.yml | 1 - .../azureDevOps/templates/windows-component-governance.yml | 2 ++ .../azureDevOps/templates/windows-hosted-build.yml | 1 - .../azureDevOps/templates/windows-package-signing.yml | 2 ++ .../azureDevOps/templates/windows-packaging.yml | 1 - tools/releaseBuild/azureDevOps/vpackRelease.yml | 7 ++----- 22 files changed, 41 insertions(+), 14 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/WindowsBuild.yml b/tools/releaseBuild/azureDevOps/WindowsBuild.yml index 535e44dc8a7..ccd2a39029f 100644 --- a/tools/releaseBuild/azureDevOps/WindowsBuild.yml +++ b/tools/releaseBuild/azureDevOps/WindowsBuild.yml @@ -32,7 +32,6 @@ jobs: - checkout: self clean: true - persistCredentials: true - task: PkgESSetupBuild@10 displayName: 'Initialize build' diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index 2f38adc5367..59f5fe77312 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -14,11 +14,6 @@ variables: DOTNET_CLI_TELEMETRY_OPTOUT: 1 POWERSHELL_TELEMETRY_OPTOUT: 1 -# Set AzDevOps Agent to clean the machine after the end of the build -resources: -- repo: self - clean: true - stages: - stage: prep jobs: diff --git a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml index c320727cc13..65a89a8a79e 100644 --- a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml +++ b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml @@ -6,6 +6,8 @@ jobs: pool: vmImage: windows-latest steps: + - checkout: self + clean: true - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index f1ede279141..9157501ea3d 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -15,6 +15,9 @@ jobs: timeoutInMinutes: 180 steps: + - checkout: self + clean: true + - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/json.yml b/tools/releaseBuild/azureDevOps/templates/json.yml index 1a0bfba950d..b9949200549 100644 --- a/tools/releaseBuild/azureDevOps/templates/json.yml +++ b/tools/releaseBuild/azureDevOps/templates/json.yml @@ -17,6 +17,9 @@ jobs: # inputs: # # displayName: '' + - checkout: self + clean: true + - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index c959a72cd3b..1d99dbea44f 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -13,6 +13,8 @@ jobs: build: ${{ parameters.buildName }} runCodesignValidationInjection: false steps: + - checkout: self + clean: true - template: SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index 620247b10f0..eb54b93e6f7 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -13,6 +13,8 @@ jobs: BuildPlatform: any cpu steps: + - checkout: self + clean: true - template: shouldSign.yml diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml index 57208f26ce9..796215d2f6a 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -13,6 +13,9 @@ jobs: HOMEBREW_NO_ANALYTICS: 1 runCodesignValidationInjection: false steps: + - checkout: self + clean: true + - pwsh: | # create folder sudo mkdir /PowerShell diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index ef3fdc94e05..5d00adef9ba 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -13,6 +13,8 @@ jobs: BuildPlatform: any cpu steps: + - checkout: self + clean: true - template: shouldSign.yml diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index 4526c8d1267..a24214f4b35 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -15,6 +15,8 @@ jobs: # inputs: # # displayName: '' + - checkout: self + clean: true - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index 77b07cfd2de..08ef0bb404b 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -20,6 +20,8 @@ jobs: linuxFxdPath: '$(System.ArtifactsDirectory)/linuxFxd' steps: + - checkout: self + clean: true - powershell: | $content = Get-Content "$(Build.SourcesDirectory)/global.json" -Raw | ConvertFrom-Json diff --git a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml index b1465105af9..053354d34c0 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml @@ -12,6 +12,9 @@ jobs: pool: vmImage: ${{ parameters.imageName }} steps: + - checkout: self + clean: true + - task: DownloadPipelineArtifact@2 inputs: source: specific diff --git a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml index 08a450c9967..dd7c9618c9e 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml @@ -9,6 +9,9 @@ jobs: pool: vmImage: ${{ parameters.imageName }} steps: + - checkout: self + clean: true + - task: DownloadPipelineArtifact@2 inputs: source: specific diff --git a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml index 84f2d0f7772..4b469e281db 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml @@ -6,6 +6,9 @@ jobs: variables: - group: 'Azure Blob variable group' steps: + - checkout: self + clean: true + - task: DownloadPipelineArtifact@2 inputs: source: specific diff --git a/tools/releaseBuild/azureDevOps/templates/testartifacts.yml b/tools/releaseBuild/azureDevOps/templates/testartifacts.yml index dd51a95f57c..8967062f96d 100644 --- a/tools/releaseBuild/azureDevOps/templates/testartifacts.yml +++ b/tools/releaseBuild/azureDevOps/templates/testartifacts.yml @@ -6,6 +6,9 @@ jobs: condition: succeeded() pool: 'Hosted Ubuntu 1604' steps: + - checkout: self + clean: true + - pwsh: | Import-Module ./build.psm1 diff --git a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml index 4a333dfdadf..ce40d54feda 100644 --- a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml +++ b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml @@ -7,6 +7,8 @@ jobs: condition: succeeded() pool: Package ES CodeHub Lab E steps: + - checkout: self + clean: true - template: ./SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/windows-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-build.yml index cf263f43a7d..1d7449f077c 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-build.yml @@ -18,7 +18,6 @@ jobs: - checkout: self clean: true - persistCredentials: true - template: shouldSign.yml - template: SetVersionVariables.yml diff --git a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml index 6ae54703d87..d819aac1b79 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml @@ -10,6 +10,8 @@ jobs: name: Package ES CodeHub Lab E steps: + - checkout: self + clean: true - template: SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml index 39fbfe7a745..e0f7a963d58 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml @@ -21,7 +21,6 @@ jobs: - checkout: self clean: true - persistCredentials: true - template: SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index 473762bda10..33979304f26 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -14,6 +14,8 @@ jobs: BuildPlatform: any cpu steps: + - checkout: self + clean: true - template: shouldSign.yml - template: SetVersionVariables.yml diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index fcdc8b01c01..d59d88aa6de 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -21,7 +21,6 @@ jobs: - checkout: self clean: true - persistCredentials: true - template: shouldSign.yml - template: SetVersionVariables.yml diff --git a/tools/releaseBuild/azureDevOps/vpackRelease.yml b/tools/releaseBuild/azureDevOps/vpackRelease.yml index 48b9e833844..70091eac8b5 100644 --- a/tools/releaseBuild/azureDevOps/vpackRelease.yml +++ b/tools/releaseBuild/azureDevOps/vpackRelease.yml @@ -17,11 +17,6 @@ variables: value: 1 - group: Azure Blob variable group -# Set AzDevOps Agent to clean the machine after the end of the build -resources: -- repo: self - clean: true - jobs: - job: rename displayName: Name the build @@ -29,6 +24,8 @@ jobs: pool: vmImage: 'windows-latest' steps: + - checkout: self + clean: true - template: ./templates/SetVersionVariables.yml parameters: From f44ea9801e0d6ea535471e80a91cce42266515e4 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 5 Nov 2020 13:28:40 -0800 Subject: [PATCH 02/21] Move to ESRP signing for Windows files (#13988) Co-authored-by: Aditya Patwardhan # Conflicts: # tools/releaseBuild/azureDevOps/WindowsBuild.yml # tools/releaseBuild/azureDevOps/templates/windows-build.yml --- .../releaseBuild/azureDevOps/WindowsBuild.yml | 463 ------------------ .../releaseBuild/azureDevOps/releaseBuild.yml | 17 +- .../templates/SetVersionVariables.yml | 22 +- .../templates/checkAzureContainer.yml | 2 + .../templates/cloneToOfficialPath.yml | 16 + .../azureDevOps/templates/compliance.yml | 1 + .../azureDevOps/templates/json.yml | 1 + .../templates/mac-file-signing.yml | 1 + .../templates/mac-package-build.yml | 6 +- .../templates/mac-package-signing.yml | 1 + .../azureDevOps/templates/mac.yml | 5 +- .../templates/release-GlobalToolTest.yml | 1 + .../templates/release-SDKTests.yml | 1 + .../templates/release-UpdateDepsJson.yml | 1 + .../azureDevOps/templates/vpackReleaseJob.yml | 1 + .../azureDevOps/templates/windows-build.yml | 125 ----- .../templates/windows-hosted-build.yml | 19 +- .../templates/windows-package-signing.yml | 1 + .../templates/windows-packaging.yml | 125 +++-- .../releaseBuild/azureDevOps/vpackRelease.yml | 1 + tools/releaseBuild/macOS/vsts.yml | 47 -- 21 files changed, 154 insertions(+), 703 deletions(-) delete mode 100644 tools/releaseBuild/azureDevOps/WindowsBuild.yml create mode 100644 tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml delete mode 100644 tools/releaseBuild/azureDevOps/templates/windows-build.yml delete mode 100644 tools/releaseBuild/macOS/vsts.yml diff --git a/tools/releaseBuild/azureDevOps/WindowsBuild.yml b/tools/releaseBuild/azureDevOps/WindowsBuild.yml deleted file mode 100644 index ccd2a39029f..00000000000 --- a/tools/releaseBuild/azureDevOps/WindowsBuild.yml +++ /dev/null @@ -1,463 +0,0 @@ -jobs: - -- job: BuildJob - displayName: Build - condition: succeeded() - pool: - name: Package ES CodeHub Lab E - strategy: - matrix: - Build (x64,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x64 - Build (x86,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x86 - Build (arm,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: arm - Build (arm64,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: arm64 - Build (fxdependent,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: fxdependent - - steps: - - - checkout: self - clean: true - - - task: PkgESSetupBuild@10 - displayName: 'Initialize build' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - useDfs: false - productName: PowerShellCore - branchVersion: true - disableWorkspace: true - disableBuildTools: true - disableNugetPack: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag $(ReleaseTag) - displayName: 'Set ReleaseTag Variable' - - - powershell: | - Write-Verbose -Verbose "$(Architecture)" - - if ('$(Architecture)' -eq 'fxdependent' -and '$(ReleaseTag)' -match '6.0.*') - { - $vstsCommandString = "vso[task.setvariable variable=SkipFxDependent]true" - } - else - { - $vstsCommandString = "vso[task.setvariable variable=SkipFxDependent]false" - } - - Write-Verbose -Message "$vstsCommandString " -Verbose - Write-Host -Object "##$vstsCommandString" - displayName: 'Skip FxDependent for PS v6.0.*' - - - powershell: | - Import-Module $(Build.SourcesDirectory)/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination $(Build.SourcesDirectory)/src/Modules - - if(-not (Test-Path "$(Build.SourcesDirectory)/src/Modules/nuget.config")) - { - throw "nuget.config is not created" - } - displayName: 'Add nuget.config for AzDevOps feed for PSGallery modules ' - condition: ne(Variables['SkipFxDependent'], 'true') - - - powershell: | - $version = $env:ReleaseTag.Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Set Version Varibale' - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - docker container prune --force - docker container ls --all --format '{{ json .ID }}' | ConvertFrom-Json | ForEach-Object {docker container rm --force --volumes $_} - displayName: 'remove all containers [Port to PSRelease]' - continueOnError: true - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - docker image ls --format '{{ json .}}'|ConvertFrom-Json| ForEach-Object { - if($_.tag -eq '') - { - $formatString = 'yyyy-MM-dd HH:mm:ss zz00' - $createdAtString = $_.CreatedAt.substring(0,$_.CreatedAt.Length -4) - $createdAt = [DateTime]::ParseExact($createdAtString, $formatString,[System.Globalization.CultureInfo]::InvariantCulture) - if($createdAt -lt (Get-Date).adddays(-1)) - { - docker image rm $_.ID - } - } - } - exit 0 - displayName: 'remove old images [Port to PSRelease]' - continueOnError: true - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - Write-verbose "--docker info---" -verbose - docker info - Write-verbose "--docker image ls---" -verbose - docker image ls - Write-verbose "--docker container ls --all---" -verbose - docker container ls --all - Write-verbose "--git branch ---" -verbose - git branch - exit 0 - displayName: 'Get Environment' - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTag) -Name win-$(Architecture)-symbols - displayName: 'Build Windows Universal - $(Architecture) Symbols zip' - continueOnError: true - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - if ("$env:Architecture" -eq 'fxdependent') - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 -SkipPwshExe - } - else - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 - } - displayName: 'Update Signing Xml' - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - $vstsCommandString = "vso[task.setvariable variable=Symbols]${env:Symbols_$(Architecture)}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Get Symbols path [Update build.json]' - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(Architecture)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(Build.SourcesDirectory)\tools\releaseBuild\signing.xml' - inPathRoot: '$(Symbols)' - outPathRoot: '$(Symbols)\signed' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - New-Item -ItemType Directory -Path $(Symbols)\signed -Force - displayName: 'Create empty signed folder' - condition: and(succeeded(), ne(variables['Build.Reason'], 'Manual'), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTag) -Name win-$(Architecture)-package -BuildPath $(Symbols) -SignedFilesPath $(Symbols)\signed - displayName: 'Build Windows Universal - $(Architecture) Package' - continueOnError: true - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - -- job: ComponentRegistrationJob - displayName: Component Registration - dependsOn: BuildJob - condition: succeeded() - pool: - name: Package ES CodeHub Lab E - strategy: - matrix: - release-anycpu: - BuildConfiguration: release - BuildPlatform: any cpu - - steps: - - - powershell: | - ./tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag $(ReleaseTag) - displayName: 'Set ReleaseTag Variable' - - - powershell: | - $version = $env:ReleaseTag.Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Set Version Variable' - - - powershell: | - docker container prune --force - docker container ls --all --format '{{ json .ID }}' | ConvertFrom-Json | ForEach-Object {docker container rm --force --volumes $_} - displayName: 'Remove all containers' - continueOnError: true - - - powershell: | - docker image ls --format '{{ json .}}'|ConvertFrom-Json| ForEach-Object { - if($_.tag -eq '') - { - $formatString = 'yyyy-MM-dd HH:mm:ss zz00' - $createdAtString = $_.CreatedAt.substring(0,$_.CreatedAt.Length -4) - $createdAt = [DateTime]::ParseExact($createdAtString, $formatString,[System.Globalization.CultureInfo]::InvariantCulture) - if($createdAt -lt (Get-Date).adddays(-1)) - { - docker image rm $_.ID - } - } - } - exit 0 - displayName: 'Remove old images' - continueOnError: true - - - powershell: | - Write-verbose "--docker info---" -verbose - docker info - Write-verbose "--docker image ls---" -verbose - docker image ls - Write-verbose "--docker container ls --all---" -verbose - docker container ls --all - Write-verbose "--git branch ---" -verbose - git branch - exit 0 - displayName: 'Get Environment' - - - powershell: | - ./tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTag) -Name win-x64-component-registration - displayName: 'Build Windows Universal - Component Registration' - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(componentregistration)' - snapshotForceEnabled: true - -- job: PackageSigningJob - displayName: Package signing - dependsOn: BuildJob - condition: succeeded() - pool: - name: Package ES CodeHub Lab E - strategy: - matrix: - release-anycpu: - BuildConfiguration: release - BuildPlatform: any cpu - - steps: - - - powershell: | - tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag $(ReleaseTag) - displayName: 'Set ReleaseTag Variable' - continueOnError: true - - - powershell: | - $version = $env:ReleaseTag.Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - $azureVersion = $env:ReleaseTag.ToLowerInvariant() -replace '\.', '-' - $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - displayName: 'Set Version Variable' - continueOnError: true - - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifacts' - inputs: - downloadType: specific - continueOnError: true - - - powershell: | - dir "$(System.ArtifactsDirectory)\*" -Recurse - displayName: 'dir artifacts directory' - continueOnError: true - - - powershell: | - Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-x86.zip" -Destination "$(Build.StagingDirectory)\symbols\x86" - displayName: 'Expand symbols zip - x86' - continueOnError: true - - - powershell: | - Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-x64.zip" -Destination "$(Build.StagingDirectory)\symbols\x64" - displayName: 'Expand symbols zip - x64' - continueOnError: true - - - powershell: | - Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-fxdependent.zip" -Destination "$(Build.StagingDirectory)\symbols\fxdependent" - displayName: 'Expand symbols zip - fxdependent' - continueOnError: true - - - powershell: | - tools/releaseBuild/generatePackgeSigning.ps1 -AuthenticodeFiles "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.msi","$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.msi" -path "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Generate Package Signing Xml' - - - powershell: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'print signing xml' - continueOnError: true - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x64.msi" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[Create script] upload signed msi - x64' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - task: AzureFileCopy@4 - displayName: 'upload signed msi to Azure - x64' - inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x64.msi' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x86.msi" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed msi - x86' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed msi to Azure - x86' - inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x86.msi' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[Create script] upload signed zip - x64' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - x64' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - x86' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - x86' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - arm' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - arm' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - arm64' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - arm64' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-fxdependent.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - fxdependent' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - fxdependent' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-fxdependent.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index 59f5fe77312..48730050268 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -10,9 +10,18 @@ pr: - master - release* +resources: + repositories: + - repository: ComplianceRepo + type: github + endpoint: ComplianceGHRepo + name: PowerShell/compliance + variables: - DOTNET_CLI_TELEMETRY_OPTOUT: 1 - POWERSHELL_TELEMETRY_OPTOUT: 1 + - name: DOTNET_CLI_TELEMETRY_OPTOUT + value: 1 + - name: POWERSHELL_TELEMETRY_OPTOUT + value: 1 stages: - stage: prep @@ -146,6 +155,10 @@ stages: pool: vmImage: 'windows-latest' steps: + - checkout: self + clean: true + fetchDepth: 1 + - template: templates/SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml b/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml index ec688f8a91c..0b8e5f42417 100644 --- a/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml +++ b/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml @@ -4,9 +4,29 @@ parameters: CreateJson: 'no' steps: +- powershell: | + $path = "./build.psm1" + + if(Test-Path -Path $path) + { + $vstsCommandString = "vso[task.setvariable variable=repoRoot]." + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + } + else{ + $path = "./PowerShell/build.psm1" + if(Test-Path -Path $path) + { + $vstsCommandString = "vso[task.setvariable variable=repoRoot]./PowerShell" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + } + } + displayName: 'Set repo Root' + - powershell: | $createJson = ("${{ parameters.ReleaseTagVarName }}" -ne "no") - $releaseTag = tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag ${{ parameters.ReleaseTagVar }} -Variable "${{ parameters.ReleaseTagVarName }}" -CreateJson:$createJson + $releaseTag = & "$env:REPOROOT/tools/releaseBuild/setReleaseTag.ps1" -ReleaseTag ${{ parameters.ReleaseTagVar }} -Variable "${{ parameters.ReleaseTagVarName }}" -CreateJson:$createJson $version = $releaseTag.Substring(1) $vstsCommandString = "vso[task.setvariable variable=Version]$version" Write-Host ("sending " + $vstsCommandString) diff --git a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml index 65a89a8a79e..23d827bf95b 100644 --- a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml +++ b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml @@ -8,6 +8,8 @@ jobs: steps: - checkout: self clean: true + fetchDepth: 1 + - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml new file mode 100644 index 00000000000..0a52f887671 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml @@ -0,0 +1,16 @@ +steps: + - powershell: | + $dirSeparatorChar = [system.io.path]::DirectorySeparatorChar + $nativePath = "${dirSeparatorChar}PowerShell" + Write-Host "##vso[task.setvariable variable=PowerShellRoot]$nativePath" + + if ((Test-Path "$nativePath")) { + Remove-Item -Path "$nativePath" -Force -Recurse -Verbose -ErrorAction ignore + } + else { + Write-Verbose -Verbose -Message "No cleanup required." + } + + git clone --quiet $env:REPOROOT $nativePath + displayName: Clone PowerShell Repo to /PowerShell + errorActionPreference: silentlycontinue diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index 9157501ea3d..6fbe978bfbd 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -17,6 +17,7 @@ jobs: steps: - checkout: self clean: true + fetchDepth: 1 - template: SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/json.yml b/tools/releaseBuild/azureDevOps/templates/json.yml index b9949200549..44608f711f3 100644 --- a/tools/releaseBuild/azureDevOps/templates/json.yml +++ b/tools/releaseBuild/azureDevOps/templates/json.yml @@ -19,6 +19,7 @@ jobs: # displayName: '' - checkout: self clean: true + fetchDepth: 1 - template: SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index eb54b93e6f7..26e56b15044 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -15,6 +15,7 @@ jobs: steps: - checkout: self clean: true + fetchDepth: 1 - template: shouldSign.yml diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml index 796215d2f6a..0c819d0519f 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -24,11 +24,6 @@ jobs: sudo chown $env:USER /PowerShell displayName: 'Create /PowerShell' - - pwsh: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" - git clone $env:BUILD_REPOSITORY_LOCALPATH /PowerShell - displayName: Clone PowerShell Repo to /PowerShell - - template: shouldSign.yml #- task: @ @@ -39,6 +34,7 @@ jobs: parameters: ReleaseTagVar: $(ReleaseTagVar) + - template: cloneToOfficialPath.yml - task: DownloadBuildArtifacts@0 displayName: Download macosBinResults diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index 5d00adef9ba..67fe3fed79b 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -15,6 +15,7 @@ jobs: steps: - checkout: self clean: true + fetchDepth: 1 - template: shouldSign.yml diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index a24214f4b35..f38e9b63573 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -29,10 +29,7 @@ jobs: sudo chown $env:USER /PowerShell displayName: 'Create /PowerShell' - - pwsh: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" - git clone $env:BUILD_REPOSITORY_LOCALPATH /PowerShell - displayName: Clone PowerShell Repo to /PowerShell + - template: cloneToOfficialPath.yml - pwsh: | tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap diff --git a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml index 053354d34c0..24df4535f7e 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml @@ -14,6 +14,7 @@ jobs: steps: - checkout: self clean: true + fetchDepth: 1 - task: DownloadPipelineArtifact@2 inputs: diff --git a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml index dd7c9618c9e..ed2736b3197 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml @@ -11,6 +11,7 @@ jobs: steps: - checkout: self clean: true + fetchDepth: 1 - task: DownloadPipelineArtifact@2 inputs: diff --git a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml index 4b469e281db..c2eaf8ceda0 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml @@ -8,6 +8,7 @@ jobs: steps: - checkout: self clean: true + fetchDepth: 1 - task: DownloadPipelineArtifact@2 inputs: diff --git a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml index ce40d54feda..fdc6c684dfb 100644 --- a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml +++ b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml @@ -9,6 +9,7 @@ jobs: steps: - checkout: self clean: true + fetchDepth: 1 - template: ./SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/windows-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-build.yml deleted file mode 100644 index 1d7449f077c..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/windows-build.yml +++ /dev/null @@ -1,125 +0,0 @@ -parameters: - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x64 - -jobs: -- job: build_windows_${{ parameters.Architecture }} - displayName: Build Windows - ${{ parameters.Architecture }} - condition: succeeded() - pool: - name: Package ES CodeHub Lab E - variables: - BuildConfiguration: ${{ parameters.BuildConfiguration }} - BuildPlatform: ${{ parameters.BuildPlatform }} - Architecture: ${{ parameters.Architecture }} - - steps: - - - checkout: self - clean: true - - - template: shouldSign.yml - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - task: PkgESSetupBuild@10 - displayName: 'Initialize build' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - useDfs: false - productName: PowerShellCore - branchVersion: true - disableWorkspace: true - disableBuildTools: true - disableNugetPack: true - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - template: insert-nuget-config-azfeed.yml - - - powershell: | - docker container prune --force - docker container ls --all --format '{{ json .ID }}' | ConvertFrom-Json | ForEach-Object {docker container rm --force --volumes $_} - displayName: 'Remove all containers [Port to PSRelease]' - # Cleanup is not critical it passes every time it runs - continueOnError: true - - - powershell: | - docker image ls --format '{{ json .}}'|ConvertFrom-Json| ForEach-Object { - if($_.tag -eq '') - { - $formatString = 'yyyy-MM-dd HH:mm:ss zz00' - $createdAtString = $_.CreatedAt.substring(0,$_.CreatedAt.Length -4) - $createdAt = [DateTime]::ParseExact($createdAtString, $formatString,[System.Globalization.CultureInfo]::InvariantCulture) - if($createdAt -lt (Get-Date).adddays(-1)) - { - docker image rm $_.ID - } - } - } - exit 0 - displayName: 'Remove old images [Port to PSRelease]' - # Cleanup is not critical it passes every time it runs - continueOnError: true - - - powershell: | - Write-verbose "--docker info---" -verbose - docker info - Write-verbose "--docker image ls---" -verbose - docker image ls - Write-verbose "--docker container ls --all---" -verbose - docker container ls --all - exit 0 - displayName: 'Capture docker info' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name win-$(Architecture)-symbols - displayName: 'Build Windows Universal - $(Architecture) Symbols zip' - - - powershell: | - if ("$env:Architecture" -like 'fxdependent*') - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 -SkipPwshExe - } - else - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 - } - displayName: 'Update Signing Xml' - - - powershell: | - $vstsCommandString = "vso[task.setvariable variable=Symbols]${env:Symbols_$(Architecture)}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Get Symbols path [Update build.json]' - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(Architecture)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(Build.SourcesDirectory)\tools\releaseBuild\signing.xml' - inPathRoot: '$(Symbols)' - outPathRoot: '$(Symbols)\signed' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - powershell: | - New-Item -ItemType Directory -Path $(Symbols)\signed -Force - displayName: 'Create empty signed folder' - condition: and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')) - - - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name win-$(Architecture)-package -BuildPath $(Symbols) -SignedFilesPath $(Symbols)\signed - displayName: 'Build Windows Universal - $(Architecture) Package' - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(Build.SourcesDirectory)' - snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml index e0f7a963d58..3413896f0b2 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml @@ -1,8 +1,13 @@ parameters: - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x64 - parentJob: '' + - name: BuildConfiguration + default: release + - name: BuildPlatform + default: any cpu + - name: Architecture + default: x64 + - name: parentJob + default: '' + jobs: - job: build_windows_${{ parameters.Architecture }} displayName: Build Windows - ${{ parameters.Architecture }} @@ -28,11 +33,7 @@ jobs: - template: insert-nuget-config-azfeed.yml - - powershell: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" - $null = New-Item -ItemType Directory -Path /Powershell -Force - git clone $env:BUILD_REPOSITORY_LOCALPATH /PowerShell - displayName: Clone PowerShell Repo to /PowerShell + - template: cloneToOfficialPath.yml - powershell: | diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index 33979304f26..d5a1af4c76f 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -16,6 +16,7 @@ jobs: steps: - checkout: self clean: true + fetchDepth: 1 - template: shouldSign.yml - template: SetVersionVariables.yml diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index d59d88aa6de..95973693495 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -1,8 +1,12 @@ parameters: - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x64 - parentJob: '' + - name: BuildConfiguration + default: release + - name: BuildPlatform + default: any cpu + - name: Architecture + default: x64 + - name: parentJob + default: '' jobs: - job: sign_windows_${{ parameters.Architecture }} @@ -12,16 +16,24 @@ jobs: pool: name: Package ES CodeHub Lab E variables: - BuildConfiguration: ${{ parameters.BuildConfiguration }} - BuildPlatform: ${{ parameters.BuildPlatform }} - Architecture: ${{ parameters.Architecture }} - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + - name: BuildConfiguration + value: ${{ parameters.BuildConfiguration }} + - name: BuildPlatform + value: ${{ parameters.BuildPlatform }} + - name: Architecture + value: ${{ parameters.Architecture }} + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: ESRP steps: - checkout: self clean: true + - checkout: ComplianceRepo + clean: true + - template: shouldSign.yml - template: SetVersionVariables.yml parameters: @@ -59,20 +71,7 @@ jobs: itemPattern: '**/*$(PkgFilter).zip' downloadPath: '$(System.ArtifactsDirectory)\Symbols' - - powershell: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" - - if ((Test-Path "\PowerShell")) { - Remove-Item -Path "\PowerShell" -Force -Recurse -Verbose - } - else { - Write-Verbose -Verbose -Message "No cleanup required." - } - - git clone --quiet $env:BUILD_REPOSITORY_LOCALPATH '\PowerShell' - - displayName: Clone PowerShell Repo to /PowerShell - errorActionPreference: silentlycontinue + - template: cloneToOfficialPath.yml - powershell: | # cleanup previous install @@ -104,33 +103,65 @@ jobs: Expand-Archive -Path $zipPath -Destination "$(System.ArtifactsDirectory)\$expandedFolder" -Force displayName: Expand symbols zip - - powershell: | - if ("$env:Architecture" -like 'fxdependent*') - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 -SkipPwshExe - } - else - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 - } - displayName: 'Update Signing Xml' + - pwsh: | + $fullSymbolsFolder = "$(System.ArtifactsDirectory)\$($env:SYMBOLSFOLDER)" + + $filesToSignDirectory = "$(System.ArtifactsDirectory)\toBeSigned" + $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force + + $signedFilesDirectory = "$(System.ArtifactsDirectory)\signed" + $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force + + $itemsToCopyWithRecurse = @( + "$($fullSymbolsFolder)\*.ps1" + "$($fullSymbolsFolder)\Microsoft.PowerShell*.dll" + ) + + $itemsToCopy = @{ + "$($fullSymbolsFolder)\*.ps1" = "" + "$($fullSymbolsFolder)\Microsoft.Management.Infrastructure.CimCmdlets.dll" = "" + "$($fullSymbolsFolder)\Microsoft.WSMan.*.dll" = "" + "$($fullSymbolsFolder)\Modules\CimCmdlets\CimCmdlets.psd1" = "Modules\CimCmdlets" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Diagnostics.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Event.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\GetEvent.types.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\Microsoft.WSMan.Management.psd1" = "Modules\Microsoft.WSMan.Management" + "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\WSMan.format.ps1xml" = "Modules\Microsoft.WSMan.Management" + "$($fullSymbolsFolder)\Modules\PSDiagnostics\PSDiagnostics.ps?1" = "Modules\PSDiagnostics" + "$($fullSymbolsFolder)\pwsh.dll" = "" + "$($fullSymbolsFolder)\System.Management.Automation.dll" = "" + } - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(Architecture)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(Build.SourcesDirectory)\tools\releaseBuild\signing.xml' - inPathRoot: '$(System.ArtifactsDirectory)\$(SymbolsFolder)' - outPathRoot: '$(System.ArtifactsDirectory)\signed' - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + if ("$env:Architecture" -notlike 'fxdependent*') + { + $itemsToCopy += @{"$($fullSymbolsFolder)\pwsh.exe" = ""} + } - - pwsh: | - New-Item -ItemType Directory -Path $(System.ArtifactsDirectory)\signed -Force - displayName: 'Create empty signed folder' - condition: and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')) + Write-Verbose -verbose "recusively copying $($itemsToCopyWithRecurse | out-string) to $filesToSignDirectory" + Copy-Item -Path $itemsToCopyWithRecurse -Destination $filesToSignDirectory -Recurse -verbose - - powershell: | + foreach($pattern in $itemsToCopy.Keys) { + $destinationFolder = Join-Path $filesToSignDirectory -ChildPath $itemsToCopy.$pattern + $null = New-Item -ItemType Directory -Path $destinationFolder -Force + Write-Verbose -verbose "copying $pattern to $destinationFolder" + Copy-Item -Path $pattern -Destination $destinationFolder -Recurse -verbose + } + displayName: 'Prepare files to be signed' + + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\toBeSigned + signOutputPath: $(System.ArtifactsDirectory)\signed + certificateId: "CP-230012" + pattern: | + **\*.dll + **\*.psd1 + **\*.psm1 + **\*.ps1xml + **\*.ps1 + useMinimatch: true + + - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force $signedFilesPath = '$(System.ArtifactsDirectory)\signed\' @@ -140,7 +171,7 @@ jobs: $dlls = Get-ChildItem $BuildPath\*.dll -Recurse $signatures = $dlls | Get-AuthenticodeSignature $missingSignatures = $signatures | Where-Object { $_.status -eq 'notsigned'}| select-object -ExpandProperty Path - tools/releaseBuild/generatePackgeSigning.ps1 -ThirdPartyFiles $missingSignatures -path "$(System.ArtifactsDirectory)\thirtdparty.xml" + & "${env:REPOROOT}/tools/releaseBuild/generatePackgeSigning.ps1" -ThirdPartyFiles $missingSignatures -path "$(System.ArtifactsDirectory)\thirtdparty.xml" displayName: Create ThirdParty Signing Xml condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) diff --git a/tools/releaseBuild/azureDevOps/vpackRelease.yml b/tools/releaseBuild/azureDevOps/vpackRelease.yml index 70091eac8b5..1dc295fd181 100644 --- a/tools/releaseBuild/azureDevOps/vpackRelease.yml +++ b/tools/releaseBuild/azureDevOps/vpackRelease.yml @@ -26,6 +26,7 @@ jobs: steps: - checkout: self clean: true + fetchDepth: 1 - template: ./templates/SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/macOS/vsts.yml b/tools/releaseBuild/macOS/vsts.yml deleted file mode 100644 index b00fc9ae0b0..00000000000 --- a/tools/releaseBuild/macOS/vsts.yml +++ /dev/null @@ -1,47 +0,0 @@ -resources: -- repo: self - clean: true -queue: - name: Hosted Mac Internal -name: macOS_$(Build.SourceBranchName)_$(Build.BuildId) -steps: -#- task: @ -# inputs: -# -# displayName: '' -- task: ShellScript@2 - inputs: - scriptPath: 'tools/releaseBuild/setReleaseTag.sh' - args: '-ReleaseTag $(ReleaseTagVar) -Variable "ReleaseTagVar"' - displayName: 'Calculate Release Tag' -- task: ShellScript@2 - inputs: - scriptPath: 'tools/installpsh-osx.sh' - displayName: 'Install pwsh' -- task: ShellScript@2 - inputs: - scriptPath: 'tools/releaseBuild/macOS/createPowerShell.sh' - displayName: 'Create /PowerShell' -- powershell: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" - git clone $env:BUILD_REPOSITORY_LOCALPATH /PowerShell - displayName: Clone PowerShell Repo to /PowerShell -- task: ShellScript@2 - inputs: - scriptPath: 'tools/releaseBuild/macOS/PowerShellPackageVsts.sh' - args: '-location $(PowerShellRoot) -BootStrap' - displayName: 'Bootstrap VM' -- powershell: | - Import-Module $(Build.SourcesDirectory)/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination "$(PowerShellRoot)/src/Modules" - - if(-not (Test-Path "$(PowerShellRoot)/src/Modules/nuget.config")) - { - throw "nuget.config is not created" - } - displayName: 'Add nuget.config for AzDevOps feed for PSGallery modules ' -- powershell: | - $env:AZDEVOPSFEEDPAT = '$(AzDevOpsFeedPAT)' - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -ExtraPackage "tar" -location $(PowerShellRoot) -Build - $env:AZDEVOPSFEEDPAT = $null - displayName: 'Build and Package' From af4875934986f78b6f698b01f5c28fa575bc25ec Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Fri, 6 Nov 2020 13:55:39 -0800 Subject: [PATCH 03/21] Don't do a shallow checkout (#13992) --- tools/releaseBuild/azureDevOps/releaseBuild.yml | 2 -- .../releaseBuild/azureDevOps/templates/checkAzureContainer.yml | 2 -- tools/releaseBuild/azureDevOps/templates/compliance.yml | 1 - tools/releaseBuild/azureDevOps/templates/json.yml | 1 - tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml | 1 - .../releaseBuild/azureDevOps/templates/mac-package-signing.yml | 1 - .../azureDevOps/templates/release-GlobalToolTest.yml | 1 - tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml | 1 - .../azureDevOps/templates/release-UpdateDepsJson.yml | 1 - tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml | 1 - .../azureDevOps/templates/windows-package-signing.yml | 1 - tools/releaseBuild/azureDevOps/vpackRelease.yml | 1 - 12 files changed, 14 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index 48730050268..6df2fe18f09 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -157,8 +157,6 @@ stages: steps: - checkout: self clean: true - fetchDepth: 1 - - template: templates/SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml index 23d827bf95b..65a89a8a79e 100644 --- a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml +++ b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml @@ -8,8 +8,6 @@ jobs: steps: - checkout: self clean: true - fetchDepth: 1 - - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index 6fbe978bfbd..9157501ea3d 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -17,7 +17,6 @@ jobs: steps: - checkout: self clean: true - fetchDepth: 1 - template: SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/json.yml b/tools/releaseBuild/azureDevOps/templates/json.yml index 44608f711f3..b9949200549 100644 --- a/tools/releaseBuild/azureDevOps/templates/json.yml +++ b/tools/releaseBuild/azureDevOps/templates/json.yml @@ -19,7 +19,6 @@ jobs: # displayName: '' - checkout: self clean: true - fetchDepth: 1 - template: SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index 26e56b15044..eb54b93e6f7 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -15,7 +15,6 @@ jobs: steps: - checkout: self clean: true - fetchDepth: 1 - template: shouldSign.yml diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index 67fe3fed79b..5d00adef9ba 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -15,7 +15,6 @@ jobs: steps: - checkout: self clean: true - fetchDepth: 1 - template: shouldSign.yml diff --git a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml index 24df4535f7e..053354d34c0 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml @@ -14,7 +14,6 @@ jobs: steps: - checkout: self clean: true - fetchDepth: 1 - task: DownloadPipelineArtifact@2 inputs: diff --git a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml index ed2736b3197..dd7c9618c9e 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml @@ -11,7 +11,6 @@ jobs: steps: - checkout: self clean: true - fetchDepth: 1 - task: DownloadPipelineArtifact@2 inputs: diff --git a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml index c2eaf8ceda0..4b469e281db 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml @@ -8,7 +8,6 @@ jobs: steps: - checkout: self clean: true - fetchDepth: 1 - task: DownloadPipelineArtifact@2 inputs: diff --git a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml index fdc6c684dfb..ce40d54feda 100644 --- a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml +++ b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml @@ -9,7 +9,6 @@ jobs: steps: - checkout: self clean: true - fetchDepth: 1 - template: ./SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index d5a1af4c76f..33979304f26 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -16,7 +16,6 @@ jobs: steps: - checkout: self clean: true - fetchDepth: 1 - template: shouldSign.yml - template: SetVersionVariables.yml diff --git a/tools/releaseBuild/azureDevOps/vpackRelease.yml b/tools/releaseBuild/azureDevOps/vpackRelease.yml index 1dc295fd181..70091eac8b5 100644 --- a/tools/releaseBuild/azureDevOps/vpackRelease.yml +++ b/tools/releaseBuild/azureDevOps/vpackRelease.yml @@ -26,7 +26,6 @@ jobs: steps: - checkout: self clean: true - fetchDepth: 1 - template: ./templates/SetVersionVariables.yml parameters: From ca79a9b0c7d7552c43be0fbb38958824dcdc99ba Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 11 Nov 2020 12:09:55 -0800 Subject: [PATCH 04/21] Migrate 3rd party signing to ESRP (#14010) Co-authored-by: Robert Holt # Conflicts: # tools/releaseBuild/azureDevOps/templates/windows-packaging.yml --- .../templates/windows-packaging.yml | 62 ++++++++++--------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index 95973693495..a89a397f343 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -14,7 +14,7 @@ jobs: condition: succeeded() dependsOn: ${{ parameters.parentJob }} pool: - name: Package ES CodeHub Lab E + vmImage: windows-latest variables: - name: BuildConfiguration value: ${{ parameters.BuildConfiguration }} @@ -39,19 +39,6 @@ jobs: parameters: ReleaseTagVar: $(ReleaseTagVar) - - task: PkgESSetupBuild@10 - displayName: 'Initialize build' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - useDfs: false - productName: PowerShellCore - branchVersion: true - disableWorkspace: true - disableBuildTools: true - disableNugetPack: true - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - powershell: | $pkgFilter = if ( '$(Architecture)' -eq 'arm' ) { "arm32" @@ -113,8 +100,8 @@ jobs: $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force $itemsToCopyWithRecurse = @( - "$($fullSymbolsFolder)\*.ps1" - "$($fullSymbolsFolder)\Microsoft.PowerShell*.dll" + "$($fullSymbolsFolder)\*.ps1" + "$($fullSymbolsFolder)\Microsoft.PowerShell*.dll" ) $itemsToCopy = @{ @@ -125,6 +112,11 @@ jobs: "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Diagnostics.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Event.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\GetEvent.types.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Microsoft.PowerShell.Diagnostics.psd1" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Host\Microsoft.PowerShell.Host.psd1" = "Modules\Microsoft.PowerShell.Host" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Management\Microsoft.PowerShell.Management.psd1" = "Modules\Microsoft.PowerShell.Management" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Security\Microsoft.PowerShell.Security.psd1" = "Modules\Microsoft.PowerShell.Security" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Utility\Microsoft.PowerShell.Utility.psd1" = "Modules\Microsoft.PowerShell.Utility" "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\Microsoft.WSMan.Management.psd1" = "Modules\Microsoft.WSMan.Management" "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\WSMan.format.ps1xml" = "Modules\Microsoft.WSMan.Management" "$($fullSymbolsFolder)\Modules\PSDiagnostics\PSDiagnostics.ps?1" = "Modules\PSDiagnostics" @@ -159,6 +151,7 @@ jobs: **\*.psm1 **\*.ps1xml **\*.ps1 + **\*.exe useMinimatch: true - pwsh: | @@ -171,29 +164,40 @@ jobs: $dlls = Get-ChildItem $BuildPath\*.dll -Recurse $signatures = $dlls | Get-AuthenticodeSignature $missingSignatures = $signatures | Where-Object { $_.status -eq 'notsigned'}| select-object -ExpandProperty Path - & "${env:REPOROOT}/tools/releaseBuild/generatePackgeSigning.ps1" -ThirdPartyFiles $missingSignatures -path "$(System.ArtifactsDirectory)\thirtdparty.xml" - displayName: Create ThirdParty Signing Xml - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - task: PkgESCodeSign@10 - displayName: 'CodeSign ThirdParty $(Architecture)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\thirtdparty.xml' - inPathRoot: '$(System.ArtifactsDirectory)\$(SymbolsFolder)' - outPathRoot: '$(System.ArtifactsDirectory)\signedThirdParty' + Write-Verbose -verbose "to be signed:`r`n $($missingSignatures | Out-String)" + + $filesToSignDirectory = "$(System.ArtifactsDirectory)\thirdPartyToBeSigned" + $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force + + $signedFilesDirectory = "$(System.ArtifactsDirectory)\thirdPartySigned" + $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force + + $missingSignatures | ForEach-Object { + Copy-Item -Path $_ -Destination $filesToSignDirectory + } + + displayName: Create ThirdParty Signing Folder condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\thirdPartyToBeSigned + signOutputPath: $(System.ArtifactsDirectory)\thirdPartySigned + certificateId: "CP-231522" + pattern: | + **\*.dll + useMinimatch: true + - powershell: | - Get-ChildItem '$(System.ArtifactsDirectory)\signedThirdParty\*' + Get-ChildItem '$(System.ArtifactsDirectory)\thirdPartySigned\*' displayName: Captrue ThirdParty Signed files condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - powershell: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force - $signedFilesPath = '$(System.ArtifactsDirectory)\signedThirdParty\' + $signedFilesPath = '$(System.ArtifactsDirectory)\thirdPartySigned' $BuildPath = '$(System.ArtifactsDirectory)\$(SymbolsFolder)' Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath From 2edd191dde252d913001fb8448f3a015badf67b7 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 23 Nov 2020 14:50:00 -0800 Subject: [PATCH 05/21] Move Linux to Esrp signing (#14210) # Conflicts: # tools/releaseBuild/azureDevOps/templates/linux.yml --- .../templates/insert-nuget-config-azfeed.yml | 6 +- .../azureDevOps/templates/linux.yml | 77 +++++++++++-------- 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml index ae697e70e33..d768a876372 100644 --- a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml +++ b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml @@ -1,9 +1,9 @@ steps: - powershell: | - Import-Module $(Build.SourcesDirectory)/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination $(Build.SourcesDirectory)/src/Modules + Import-Module $env:REPOROOT/build.psm1 -Force + New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination $env:REPOROOT/src/Modules - if(-not (Test-Path "$(Build.SourcesDirectory)/src/Modules/nuget.config")) + if(-not (Test-Path "$env:REPOROOT/src/Modules/nuget.config")) { throw "nuget.config is not created" } diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 1d99dbea44f..5263217404e 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -7,11 +7,16 @@ jobs: - job: build_${{ parameters.buildName }} displayName: Build ${{ parameters.buildName }} condition: succeeded() - pool: Hosted Ubuntu 1604 + pool: + vmImage: ubuntu-16.04 dependsOn: ${{ parameters.parentJob }} variables: - build: ${{ parameters.buildName }} - runCodesignValidationInjection: false + - name: runCodesignValidationInjection + value: false + - name: build + value: ${{ parameters.buildName }} + - group: ESRP + steps: - checkout: self clean: true @@ -37,13 +42,13 @@ jobs: - powershell: | - import-module ./build.psm1 + import-module "$env:REPOROOT/build.psm1" Sync-PSTags -AddRemoteIfMissing displayName: SyncTags condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name '$(build)' + & "$env:REPOROOT/tools/releaseBuild/vstsbuild.ps1" -ReleaseTag $(ReleaseTagVar) -Name '$(build)' displayName: 'Build and package' condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) @@ -52,10 +57,19 @@ jobs: displayName: ${{ parameters.uploadDisplayName }} ${{ parameters.buildName }} dependsOn: build_${{ parameters.buildName }} condition: succeeded() - pool: Package ES CodeHub Lab E + pool: + vmImage: windows-latest variables: - buildName: ${{ parameters.buildName }} + - name: buildName + value: ${{ parameters.buildName }} + - group: ESRP + steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true - template: shouldSign.yml @@ -86,32 +100,29 @@ jobs: downloadPath: '$(System.ArtifactsDirectory)\rpm' condition: and(eq(variables['buildName'], 'RPM'),succeeded()) - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run Defender Scan' - - - powershell: | - $authenticodefiles = @() - Get-ChildItem -Path '$(System.ArtifactsDirectory)\rpm\*.rpm' -recurse | ForEach-Object { $authenticodefiles += $_.FullName} - tools/releaseBuild/generatePackgeSigning.ps1 -LinuxFiles $authenticodeFiles -path "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Generate RPM Signing Xml' - condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - - - powershell: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Capture RPM signing xml' - condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - - - task: PkgESCodeSign@10 - displayName: 'CodeSign RPM $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - task: SFP.build-tasks.custom-build-task-2.EsrpMalwareScanning@1 + displayName: 'Malware Scanning' inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - + ConnectedServiceName: pwshEsrpScanning + FolderPath: $(System.ArtifactsDirectory) + Pattern: | + **\*.rpm + **\*.deb + **\*.tar.gz + UseMinimatch: true + SessionTimeout: 30 + + - ${{ if eq(variables['buildName'], 'RPM') }}: + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\rpm + signOutputPath: $(Build.StagingDirectory)\signedPackages + certificateId: "CP-450779-Pgp" + pattern: | + **\*.rpm + useMinimatch: true + + # requires windows - task: AzureFileCopy@4 displayName: 'Upload to Azure - DEB and tar.gz' inputs: @@ -125,6 +136,7 @@ jobs: parameters: artifactPath: $(System.ArtifactsDirectory)\finished\release + # requires windows - task: AzureFileCopy@4 displayName: 'Upload to Azure - RPM - Unsigned' inputs: @@ -135,6 +147,7 @@ jobs: ContainerName: '$(AzureVersion)' condition: and(and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) + # requires windows - task: AzureFileCopy@4 displayName: 'Upload to Azure - RPM - Signed' inputs: From 296bde157a0a4ec797739bdffc57c9609495aa55 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Fri, 13 Nov 2020 14:16:06 -0800 Subject: [PATCH 06/21] Move windows package signing to use ESRP (#14060) # Conflicts: # tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml --- .../releaseBuild/azureDevOps/releaseBuild.yml | 1 + .../templates/windows-package-signing.yml | 62 ++++++++----------- 2 files changed, 28 insertions(+), 35 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index 6df2fe18f09..5e87ff58ec7 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -16,6 +16,7 @@ resources: type: github endpoint: ComplianceGHRepo name: PowerShell/compliance + ref: master variables: - name: DOTNET_CLI_TELEMETRY_OPTOUT diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index 33979304f26..3803226958a 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -8,15 +8,19 @@ jobs: ${{ parameters.parentJobs }} condition: succeeded() pool: - name: Package ES CodeHub Lab E + vmImage: windows-latest variables: - BuildConfiguration: release - BuildPlatform: any cpu + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: ESRP steps: - checkout: self clean: true + - checkout: ComplianceRepo + clean: true + - template: shouldSign.yml - template: SetVersionVariables.yml parameters: @@ -36,36 +40,15 @@ jobs: # Diagnostics is not critical it passes every time it runs continueOnError: true - - powershell: | - $authenticodefiles = @( - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.msi" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.msi" - ) - - $msixFiles = @( - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.msix" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.msix" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.msix" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.msix" - ) - - tools/releaseBuild/generatePackgeSigning.ps1 -AuthenticodeFiles $authenticodeFiles -path "$(System.ArtifactsDirectory)\package.xml" -MsixCertType $env:MSIX_TYPE -MsixFiles $msixFiles - displayName: 'Generate Package Signing Xml' - - - powershell: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Capture signing xml' - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\signed + signOutputPath: $(Build.StagingDirectory)\signedPackages + certificateId: "CP-230012" + pattern: | + **\*.msi + **\*.msix + useMinimatch: true - powershell: | new-item -itemtype Directory -path '$(Build.StagingDirectory)\signedPackages' @@ -110,8 +93,17 @@ jobs: msi: no msix: no - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run Defender Scan' + - task: SFP.build-tasks.custom-build-task-2.EsrpMalwareScanning@1 + displayName: 'Malware Scanning' + inputs: + ConnectedServiceName: pwshEsrpScanning + FolderPath: '$(Build.StagingDirectory)' + Pattern: | + **\*.msix + **\*.msi + **\*.zip + UseMinimatch: true + SessionTimeout: 30 - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' From eb6406ada8221e224247ee54cbd58d8a3b06857b Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 7 Dec 2020 12:11:54 -0800 Subject: [PATCH 07/21] Move macOS and nuget to esrp signing (#14324) --- .../azureDevOps/templates/linux.yml | 21 +++--- .../templates/mac-file-signing.yml | 42 +++++------ .../templates/mac-package-signing.yml | 38 ++++------ .../azureDevOps/templates/nuget.yml | 70 ++++++++++--------- .../templates/windows-package-signing.yml | 18 ++--- 5 files changed, 82 insertions(+), 107 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 5263217404e..53e9b4f1924 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -21,6 +21,9 @@ jobs: - checkout: self clean: true + - checkout: ComplianceRepo + clean: true + - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) @@ -100,17 +103,13 @@ jobs: downloadPath: '$(System.ArtifactsDirectory)\rpm' condition: and(eq(variables['buildName'], 'RPM'),succeeded()) - - task: SFP.build-tasks.custom-build-task-2.EsrpMalwareScanning@1 - displayName: 'Malware Scanning' - inputs: - ConnectedServiceName: pwshEsrpScanning - FolderPath: $(System.ArtifactsDirectory) - Pattern: | - **\*.rpm - **\*.deb - **\*.tar.gz - UseMinimatch: true - SessionTimeout: 30 + - template: EsrpScan.yml@ComplianceRepo + parameters: + scanPath: $(System.ArtifactsDirectory) + pattern: | + **\*.rpm + **\*.deb + **\*.tar.gz - ${{ if eq(variables['buildName'], 'RPM') }}: - template: EsrpSign.yml@ComplianceRepo diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index eb54b93e6f7..066841974e5 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -9,13 +9,15 @@ jobs: pool: name: Package ES CodeHub Lab E variables: - BuildConfiguration: release - BuildPlatform: any cpu + - group: ESRP steps: - checkout: self clean: true + - checkout: ComplianceRepo + clean: true + - template: shouldSign.yml - template: SetVersionVariables.yml @@ -63,27 +65,14 @@ jobs: Write-Host $zipFile displayName: 'Compress macOS binary files' - - pwsh: | - $pkgFiles = "$(Build.StagingDirectory)\macos\powershell-files-$(Version)-osx-x64.zip", "$(Build.StagingDirectory)\macos\powershell-lts-$(Version)-osx-x64.zip" - tools/releaseBuild/generatePackgeSigning.ps1 -MacDeveloperFiles $pkgFiles -path "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Generate macOS binary Signing Xml' - - - pwsh: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Capture macOS signing xml' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedMacOSPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(Build.StagingDirectory)\macos + signOutputPath: $(Build.StagingDirectory)\signedMacOSPackages + certificateId: "CP-401337-Apple" + pattern: | + **\*.zip + useMinimatch: true - pwsh: | $destination = "$(System.ArtifactsDirectory)\azureMacOs" @@ -100,8 +89,11 @@ jobs: artifactName: signedMacOsBins condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run Defender Scan' + - template: EsrpScan.yml@ComplianceRepo + parameters: + scanPath: $(System.ArtifactsDirectory)\azureMacOs + pattern: | + **\* - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index 5d00adef9ba..6c2fdb0cf18 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -9,13 +9,15 @@ jobs: pool: name: Package ES CodeHub Lab E variables: - BuildConfiguration: release - BuildPlatform: any cpu + - group: ESRP steps: - checkout: self clean: true + - checkout: ComplianceRepo + clean: true + - template: shouldSign.yml - template: SetVersionVariables.yml @@ -50,27 +52,14 @@ jobs: } displayName: 'Compress macOS Package' - - pwsh: | - $pkgFiles = "$(Build.StagingDirectory)\macos\powershell-$(Version)-osx-x64.zip", "$(Build.StagingDirectory)\macos\powershell-lts-$(Version)-osx-x64.zip" - tools/releaseBuild/generatePackgeSigning.ps1 -MacDeveloperFiles $pkgFiles -path "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Generate macOS Package Signing Xml' - - - pwsh: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Capture macOS signing xml' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedMacOSPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(Build.StagingDirectory)\macos + signOutputPath: $(Build.StagingDirectory)\signedMacOSPackages + certificateId: "CP-401337-Apple" + pattern: | + **\*.zip + useMinimatch: true - template: upload-final-results.yml parameters: @@ -120,9 +109,6 @@ jobs: ContainerName: '$(AzureVersion)' condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run Defender Scan' - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index 08ef0bb404b..0ec3d48547c 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -12,19 +12,35 @@ jobs: timeoutInMinutes: 90 variables: - runCodesignValidationInjection: false - GenAPIToolPath: '$(System.ArtifactsDirectory)/GenAPI' - PackagePath: '$(System.ArtifactsDirectory)/UnifiedPackagePath' - winFxdPath: '$(System.ArtifactsDirectory)/winFxd' - winFxdWinDesktopPath: '$(System.ArtifactsDirectory)/winFxdWinDesktop' - linuxFxdPath: '$(System.ArtifactsDirectory)/linuxFxd' + - name: runCodesignValidationInjection + value: false + - name: build + value: ${{ parameters.buildName }} + - group: ESRP + - name: GenAPIToolPath + value: '$(System.ArtifactsDirectory)/GenAPI' + - name: PackagePath + value: '$(System.ArtifactsDirectory)/UnifiedPackagePath' + - name: winFxdPath + value: '$(System.ArtifactsDirectory)/winFxd' + - name: winFxdWinDesktopPath + value: '$(System.ArtifactsDirectory)/winFxdWinDesktop' + - name: linuxFxdPath + value: '$(System.ArtifactsDirectory)/linuxFxd' steps: - checkout: self clean: true + - checkout: ComplianceRepo + clean: true + + - template: SetVersionVariables.yml + parameters: + ReleaseTagVar: $(ReleaseTagVar) + - powershell: | - $content = Get-Content "$(Build.SourcesDirectory)/global.json" -Raw | ConvertFrom-Json + $content = Get-Content "$env:REPOROOT/global.json" -Raw | ConvertFrom-Json $vstsCommandString = "vso[task.setvariable variable=SDKVersion]$($content.sdk.version)" Write-Host "sending " + $vstsCommandString Write-Host "##$vstsCommandString" @@ -97,8 +113,8 @@ jobs: displayName: 'Install NuGet.exe' - powershell: | - Import-Module $env:BUILD_SOURCESDIRECTORY\build.psm1 - Import-Module $env:BUILD_SOURCESDIRECTORY\tools\packaging + Import-Module $env:REPOROOT\build.psm1 + Import-Module $env:REPOROOT\tools\packaging Find-Dotnet New-ILNugetPackage -PackagePath "$(PackagePath)" -PackageVersion "$(Version)" -WinFxdBinPath '$(winFxdPath)' -LinuxFxdBinPath '$(linuxFxdPath)' -GenAPIToolPath "$(GenAPIToolPath)" displayName: 'Create Nuget Package Folders' @@ -110,8 +126,8 @@ jobs: displayName: Capture fxd folders - powershell: | - Import-Module $env:BUILD_SOURCESDIRECTORY\build.psm1 - Import-Module $env:BUILD_SOURCESDIRECTORY\tools\packaging + Import-Module $env:REPOROOT\build.psm1 + Import-Module $env:REPOROOT\tools\packaging Find-Dotnet # Create unified package first New-GlobalToolNupkg -UnifiedPackage -LinuxBinPath "$(linuxFxdPath)" -WindowsBinPath "$(winFxdPath)" -WindowsDesktopBinPath "$(winFxdWinDesktopPath)" -PackageVersion "$(Version)" -DestinationPath "$(PackagePath)\globaltool" @@ -124,28 +140,14 @@ jobs: Get-ChildItem "$(PackagePath)" -Recurse displayName: Capture generated packages - - powershell: | - $packages = Get-ChildItem "$(PackagePath)\*.nupkg", "$(PackagePath)\globaltool\*.nupkg" | Select-Object -ExpandProperty FullName - - if($packages.Count -lt 1) - { - throw "No packages created" - } - - $(Build.SourcesDirectory)\tools\releaseBuild\generatePackgeSigning.ps1 -Path $(PackagePath)\NugetSigning.xml -NuPkgFiles $packages - displayName: Create signing file - - - task: PkgESCodeSign@10 - displayName: 'CodeSign Nuget Packages' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(PackagePath)\NugetSigning.xml' - inPathRoot: '$(PackagePath)' - outPathRoot: '$(System.ArtifactsDirectory)\signed' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(PackagePath) + signOutputPath: $(System.ArtifactsDirectory)\signed + certificateId: "CP-401405" + pattern: | + **\*.nupkg + useMinimatch: true - pwsh: | if (-not (Test-Path '$(System.ArtifactsDirectory)\signed\')) { $null = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)\signed\' } @@ -155,7 +157,7 @@ jobs: condition: eq(variables['SHOULD_SIGN'], 'false') - powershell: | - Import-Module $(Build.SourcesDirectory)\build.psm1 -Force + Import-Module "${env:REPOROOT}\build.psm1" -Force Get-ChildItem -Recurse "$(System.ArtifactsDirectory)\signed\*.nupkg" -Verbose | ForEach-Object { Start-NativeExecution -sb { nuget.exe verify -All $_.FullName } } displayName: Verify all packages are signed condition: eq(variables['SHOULD_SIGN'], 'true') diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index 3803226958a..81c1a2228d9 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -93,17 +93,13 @@ jobs: msi: no msix: no - - task: SFP.build-tasks.custom-build-task-2.EsrpMalwareScanning@1 - displayName: 'Malware Scanning' - inputs: - ConnectedServiceName: pwshEsrpScanning - FolderPath: '$(Build.StagingDirectory)' - Pattern: | - **\*.msix - **\*.msi - **\*.zip - UseMinimatch: true - SessionTimeout: 30 + - template: EsrpScan.yml@ComplianceRepo + parameters: + scanPath: $(Build.StagingDirectory) + pattern: | + **\*.msix + **\*.msi + **\*.zip - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' From 02aa91ea1cdfacab9f69965aef495bfd279e1532 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 9 Dec 2020 14:51:43 -0800 Subject: [PATCH 08/21] Fix issue with unsigned build (#14367) --- tools/releaseBuild/azureDevOps/templates/linux.yml | 1 + .../azureDevOps/templates/mac-file-signing.yml | 12 +++++++----- .../azureDevOps/templates/mac-package-signing.yml | 1 + tools/releaseBuild/azureDevOps/templates/nuget.yml | 1 + .../templates/windows-package-signing.yml | 1 + .../azureDevOps/templates/windows-packaging.yml | 2 ++ tools/releaseBuild/setReleaseTag.ps1 | 2 +- 7 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 53e9b4f1924..f8cfd397a3d 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -120,6 +120,7 @@ jobs: pattern: | **\*.rpm useMinimatch: true + shouldSign: $(SHOULD_SIGN) # requires windows - task: AzureFileCopy@4 diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index 066841974e5..89f0c245879 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -73,6 +73,7 @@ jobs: pattern: | **\*.zip useMinimatch: true + shouldSign: $(SHOULD_SIGN) - pwsh: | $destination = "$(System.ArtifactsDirectory)\azureMacOs" @@ -89,11 +90,12 @@ jobs: artifactName: signedMacOsBins condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - template: EsrpScan.yml@ComplianceRepo - parameters: - scanPath: $(System.ArtifactsDirectory)\azureMacOs - pattern: | - **\* + - ${{ if eq(variables['SHOULD_SIGN'], 'true') }}: + - template: EsrpScan.yml@ComplianceRepo + parameters: + scanPath: $(System.ArtifactsDirectory)\azureMacOs + pattern: | + **\* - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index 6c2fdb0cf18..c405230e0b8 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -60,6 +60,7 @@ jobs: pattern: | **\*.zip useMinimatch: true + shouldSign: $(SHOULD_SIGN) - template: upload-final-results.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index 0ec3d48547c..e5983079832 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -148,6 +148,7 @@ jobs: pattern: | **\*.nupkg useMinimatch: true + shouldSign: $(SHOULD_SIGN) - pwsh: | if (-not (Test-Path '$(System.ArtifactsDirectory)\signed\')) { $null = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)\signed\' } diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index 81c1a2228d9..070a0840991 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -49,6 +49,7 @@ jobs: **\*.msi **\*.msix useMinimatch: true + shouldSign: $(SHOULD_SIGN) - powershell: | new-item -itemtype Directory -path '$(Build.StagingDirectory)\signedPackages' diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index a89a397f343..f5ff90a38ae 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -153,6 +153,7 @@ jobs: **\*.ps1 **\*.exe useMinimatch: true + shouldSign: $(SHOULD_SIGN) - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force @@ -188,6 +189,7 @@ jobs: pattern: | **\*.dll useMinimatch: true + shouldSign: $(SHOULD_SIGN) - powershell: | Get-ChildItem '$(System.ArtifactsDirectory)\thirdPartySigned\*' diff --git a/tools/releaseBuild/setReleaseTag.ps1 b/tools/releaseBuild/setReleaseTag.ps1 index 1927eb7ae60..121425d963b 100644 --- a/tools/releaseBuild/setReleaseTag.ps1 +++ b/tools/releaseBuild/setReleaseTag.ps1 @@ -79,7 +79,7 @@ if($ReleaseTag -eq 'fromBranch' -or !$ReleaseTag) New-BuildInfoJson -ReleaseTag $releaseTag } } - elseif($branchOnly -eq 'master' -or $branchOnly -like '*dailytest*') + elseif(($branchOnly -eq 'master' -and $env:BUILD_REASON -ne 'Manual') -or $branchOnly -like '*dailytest*') { $isDaily = $true Write-Verbose "daily build" -Verbose From 54ff9c67d90f38632bfdd9f01986245b6f45cfcf Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 9 Dec 2020 13:09:21 -0800 Subject: [PATCH 09/21] Make AppLocker Enforce mode take precedence over UMCI Audit mode (#14353) --- .../security/wldpNativeMethods.cs | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/System.Management.Automation/security/wldpNativeMethods.cs b/src/System.Management.Automation/security/wldpNativeMethods.cs index e9665f4dfa8..55cf43be260 100644 --- a/src/System.Management.Automation/security/wldpNativeMethods.cs +++ b/src/System.Management.Automation/security/wldpNativeMethods.cs @@ -79,28 +79,31 @@ public static SystemEnforcementMode GetSystemLockdownPolicy() /// An EnforcementMode that describes policy. public static SystemEnforcementMode GetLockdownPolicy(string path, SafeHandle handle) { - // Check the WLDP API - SystemEnforcementMode lockdownPolicy = GetWldpPolicy(path, handle); - if (lockdownPolicy == SystemEnforcementMode.Enforce) + // Check the WLDP File policy via API + var wldpFilePolicy = GetWldpPolicy(path, handle); + if (wldpFilePolicy == SystemEnforcementMode.Enforce) { - return lockdownPolicy; + return wldpFilePolicy; + } + + // Check the AppLocker File policy via API + // This needs to be checked before WLDP audit policy + // So, that we don't end up in Audit mode, + // when we should be enforce mode. + var appLockerFilePolicy = GetAppLockerPolicy(path, handle); + if (appLockerFilePolicy == SystemEnforcementMode.Enforce) + { + return appLockerFilePolicy; } // At this point, LockdownPolicy = Audit or Allowed. // If there was a WLDP policy, but WLDP didn't block it, // then it was explicitly allowed. Therefore, return the result for the file. SystemEnforcementMode systemWldpPolicy = s_cachedWldpSystemPolicy.GetValueOrDefault(SystemEnforcementMode.None); - if ((systemWldpPolicy == SystemEnforcementMode.Enforce) || - (systemWldpPolicy == SystemEnforcementMode.Audit)) - { - return lockdownPolicy; - } - - // Check the AppLocker API - lockdownPolicy = GetAppLockerPolicy(path, handle); - if (lockdownPolicy == SystemEnforcementMode.Enforce) + if ((systemWldpPolicy == SystemEnforcementMode.Audit) || + (systemWldpPolicy == SystemEnforcementMode.Enforce)) { - return lockdownPolicy; + return wldpFilePolicy; } // If there was a system-wide AppLocker policy, but AppLocker didn't block it, @@ -108,7 +111,7 @@ public static SystemEnforcementMode GetLockdownPolicy(string path, SafeHandle ha if (s_cachedSaferSystemPolicy.GetValueOrDefault(SaferPolicy.Allowed) == SaferPolicy.Disallowed) { - return lockdownPolicy; + return appLockerFilePolicy; } // If it's not set to 'Enforce' by the platform, allow debug overrides From 88478d66e65f9a7d5cd950a55459675966003faa Mon Sep 17 00:00:00 2001 From: Robert Holt Date: Thu, 10 Dec 2020 13:07:53 -0800 Subject: [PATCH 10/21] Fix syntax error in Windows packaging script (#14377) # Conflicts: # tools/releaseBuild/azureDevOps/templates/windows-packaging.yml --- .../releaseBuild/azureDevOps/templates/windows-packaging.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index f5ff90a38ae..3e959b62af1 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -124,13 +124,16 @@ jobs: "$($fullSymbolsFolder)\System.Management.Automation.dll" = "" } + $itemsToExclude = @( + ) + if ("$env:Architecture" -notlike 'fxdependent*') { $itemsToCopy += @{"$($fullSymbolsFolder)\pwsh.exe" = ""} } Write-Verbose -verbose "recusively copying $($itemsToCopyWithRecurse | out-string) to $filesToSignDirectory" - Copy-Item -Path $itemsToCopyWithRecurse -Destination $filesToSignDirectory -Recurse -verbose + Copy-Item -Path $itemsToCopyWithRecurse -Destination $filesToSignDirectory -Recurse -verbose -exclude $itemsToExclude foreach($pattern in $itemsToCopy.Keys) { $destinationFolder = Join-Path $filesToSignDirectory -ChildPath $itemsToCopy.$pattern From 57d4f578b850b125635279ea761bbb55cc9fca8b Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 14 Dec 2020 13:59:27 -0800 Subject: [PATCH 11/21] Fix path signed RPMs are uploaded from in release build (#14424) --- tools/releaseBuild/azureDevOps/templates/linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index f8cfd397a3d..cc1ea114506 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -151,7 +151,7 @@ jobs: - task: AzureFileCopy@4 displayName: 'Upload to Azure - RPM - Signed' inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\*' + SourcePath: '$(Build.StagingDirectory)\signedPackages\release\*' azureSubscription: '$(AzureFileCopySubscription)' Destination: AzureBlob storage: '$(StorageAccount)' @@ -165,7 +165,7 @@ jobs: - template: upload-final-results.yml parameters: - artifactPath: '$(Build.StagingDirectory)\signedPackages' + artifactPath: '$(Build.StagingDirectory)\signedPackages\release' condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 From caa6576ec9a8d4e19ff1813d3f04c6e3da934c44 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 14 Dec 2020 14:16:55 -0800 Subject: [PATCH 12/21] Use one feed in each nuget.config in official builds (#14363) Co-authored-by: Robert Holt --- build.psm1 | 18 ++++++++++----- nuget.config | 6 ++--- src/Modules/PSGalleryModules.csproj | 10 ++++++++- src/Modules/nuget.config | 11 ++++++++++ test/tools/Modules/nuget.config | 10 +++++++++ .../AzArtifactFeed/PSGalleryToAzArtifacts.yml | 2 +- .../templates/insert-nuget-config-azfeed.yml | 22 +++++++++++++++---- .../azureDevOps/templates/linux.yml | 5 ++++- .../azureDevOps/templates/mac.yml | 16 +++++--------- .../templates/windows-hosted-build.yml | 6 +++-- .../templates/windows-packaging.yml | 4 ++-- 11 files changed, 80 insertions(+), 30 deletions(-) create mode 100644 src/Modules/nuget.config create mode 100644 test/tools/Modules/nuget.config diff --git a/build.psm1 b/build.psm1 index 5e079e4fa2d..73319669cbf 100644 --- a/build.psm1 +++ b/build.psm1 @@ -337,7 +337,8 @@ function Start-PSBuild { try { # Excluded sqlite3 folder is due to this Roslyn issue: https://github.com/dotnet/roslyn/issues/23060 # Excluded src/Modules/nuget.config as this is required for release build. - git clean -fdX --exclude .vs/PowerShell/v16/Server/sqlite3 --exclude src/Modules/nuget.config + # Excluded nuget.config as this is required for release build. + git clean -fdX --exclude .vs/PowerShell/v16/Server/sqlite3 --exclude src/Modules/nuget.config --exclude nuget.config } finally { Pop-Location } @@ -628,7 +629,9 @@ function Restore-PSPackage [switch] $Force, - [switch] $InteractiveAuth + [switch] $InteractiveAuth, + + [switch] $PSModule ) if (-not $ProjectDirs) @@ -652,7 +655,10 @@ function Restore-PSPackage 'Microsoft.NET.Sdk' } - if ($Options.Runtime -notlike 'fxdependent*') { + if ($PSModule.IsPresent) { + $RestoreArguments = @("--verbosity") + } + elseif ($Options.Runtime -notlike 'fxdependent*') { $RestoreArguments = @("--runtime", $Options.Runtime, "/property:SDKToUse=$sdkToUse", "--verbosity") } else { $RestoreArguments = @("/property:SDKToUse=$sdkToUse", "--verbosity") @@ -2505,7 +2511,7 @@ function Copy-PSGalleryModules Find-DotNet - Restore-PSPackage -ProjectDirs (Split-Path $CsProjPath) -Force:$Force.IsPresent + Restore-PSPackage -ProjectDirs (Split-Path $CsProjPath) -Force:$Force.IsPresent -PSModule $cache = dotnet nuget locals global-packages -l if ($cache -match "global-packages: (.*)") { @@ -3287,8 +3293,10 @@ function New-NugetConfigFile - + + + <[FEEDNAME]> diff --git a/nuget.config b/nuget.config index a1713070c69..9b513db4b6b 100644 --- a/nuget.config +++ b/nuget.config @@ -5,8 +5,8 @@ - - - + + + diff --git a/src/Modules/PSGalleryModules.csproj b/src/Modules/PSGalleryModules.csproj index 3f7b71a07a3..63807dc1e50 100644 --- a/src/Modules/PSGalleryModules.csproj +++ b/src/Modules/PSGalleryModules.csproj @@ -1,6 +1,14 @@ - + + PowerShell + Microsoft Corporation + (c) Microsoft Corporation. + + net5.0 + + true + diff --git a/src/Modules/nuget.config b/src/Modules/nuget.config new file mode 100644 index 00000000000..f5a7f806a36 --- /dev/null +++ b/src/Modules/nuget.config @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/test/tools/Modules/nuget.config b/test/tools/Modules/nuget.config new file mode 100644 index 00000000000..b0fc73009da --- /dev/null +++ b/test/tools/Modules/nuget.config @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml index fab28643168..221a385b8b8 100644 --- a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml +++ b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml @@ -18,7 +18,7 @@ steps: - pwsh: | Import-Module -Force "$(Build.SourcesDirectory)/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1" - SyncGalleryToAzArtifacts -AzDevOpsFeedUserName $(AzDevOpsFeedUserName) -AzDevOpsPAT $(AzDevOpsFeedPAT) -Destination $(Build.ArtifactStagingDirectory) + SyncGalleryToAzArtifacts -AzDevOpsFeedUserName $(AzDevOpsFeedUserName) -AzDevOpsPAT $(AzDevOpsFeedPAT2) -Destination $(Build.ArtifactStagingDirectory) displayName: Download packages from PSGallery that need to be updated condition: succeededOrFailed() diff --git a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml index d768a876372..5fdb9f7d977 100644 --- a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml +++ b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml @@ -1,11 +1,25 @@ +parameters: + - name: "repoRoot" + default: $(REPOROOT) + steps: -- powershell: | - Import-Module $env:REPOROOT/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination $env:REPOROOT/src/Modules +- pwsh: | + Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force + New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination '${{ parameters.repoRoot }}/src/Modules' - if(-not (Test-Path "$env:REPOROOT/src/Modules/nuget.config")) + if(-not (Test-Path "${{ parameters.repoRoot }}/src/Modules/nuget.config")) { throw "nuget.config is not created" } displayName: 'Add nuget.config for Azure DevOps feed for PSGallery modules' condition: and(succeededOrFailed(), ne(variables['AzDevOpsFeed'], '')) +- pwsh: | + Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force + New-NugetConfigFile -NugetFeedUrl $(AzDevOpsPackageFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination '${{ parameters.repoRoot }}' + + if(-not (Test-Path "${{ parameters.repoRoot }}/nuget.config")) + { + throw "nuget.config is not created" + } + displayName: 'Add nuget.config for Azure DevOps feed for packages' + condition: and(succeededOrFailed(), ne(variables['AzDevOpsPackageFeed'], '')) diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index cc1ea114506..0fac7ae4f24 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -42,7 +42,8 @@ jobs: displayName: 'Skip Alpine or fxdependent for PS v6.0.*' - template: insert-nuget-config-azfeed.yml - + parameters: + repoRoot: $(REPOROOT) - powershell: | import-module "$env:REPOROOT/build.psm1" @@ -51,7 +52,9 @@ jobs: condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - powershell: | + $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)' & "$env:REPOROOT/tools/releaseBuild/vstsbuild.ps1" -ReleaseTag $(ReleaseTagVar) -Name '$(build)' + $env:AzDevOpsFeedPAT2 = $null displayName: 'Build and package' condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index f38e9b63573..6690f678888 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -35,20 +35,14 @@ jobs: tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap displayName: 'Bootstrap VM' - - pwsh: | - Import-Module $(Build.SourcesDirectory)/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination "$(PowerShellRoot)/src/Modules" - - if(-not (Test-Path "$(PowerShellRoot)/src/Modules/nuget.config")) - { - throw "nuget.config is not created" - } - displayName: 'Add nuget.config for AzDevOps feed for PSGallery modules ' + - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml + parameters: + repoRoot: $(PowerShellRoot) - pwsh: | - $env:AZDEVOPSFEEDPAT = '$(AzDevOpsFeedPAT)' + $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)' $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults - $env:AZDEVOPSFEEDPAT = $null + $env:AzDevOpsFeedPAT2 = $null displayName: 'Build' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml index 3413896f0b2..cc05417b17b 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml @@ -31,10 +31,12 @@ jobs: parameters: ReleaseTagVar: $(ReleaseTagVar) - - template: insert-nuget-config-azfeed.yml - - template: cloneToOfficialPath.yml + - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml + parameters: + repoRoot: $(PowerShellRoot) + - powershell: | $runtime = switch ($env:Architecture) diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index 3e959b62af1..4987514291f 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -60,7 +60,7 @@ jobs: - template: cloneToOfficialPath.yml - - powershell: | + - pwsh: | # cleanup previous install if((Test-Path "${env:ProgramFiles(x86)}\WiX Toolset xcopy")) { Remove-Item "${env:ProgramFiles(x86)}\WiX Toolset xcopy" -Recurse -Force @@ -68,7 +68,7 @@ jobs: $toolsDir = New-Item -ItemType Directory -Path '$(Build.ArtifactStagingDirectory)\tools' $wixUri = 'https://github.com/wixtoolset/wix3/releases/download/wix311rtm/wix311-binaries.zip' - Invoke-RestMethod -Uri $wixUri -OutFile '$(Build.ArtifactStagingDirectory)\tools\wix.zip' + Invoke-RestMethod -Uri $wixUri -OutFile '$(Build.ArtifactStagingDirectory)\tools\wix.zip' -MaximumRetryCount 5 -RetryIntervalSec 10 Import-Module '$(PowerShellRoot)/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/wix.psm1' Install-WixZip -zipPath '$(Build.ArtifactStagingDirectory)\tools\wix.zip' From f60b995148d566c8348532200ea589d3c4b80d4a Mon Sep 17 00:00:00 2001 From: Ilya Date: Fri, 4 Dec 2020 00:24:49 +0500 Subject: [PATCH 13/21] Avoid an exception if file system does not support reparse points (#13634) --- .../namespaces/FileSystemProvider.cs | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/System.Management.Automation/namespaces/FileSystemProvider.cs b/src/System.Management.Automation/namespaces/FileSystemProvider.cs index be4b95dc935..75921d838df 100644 --- a/src/System.Management.Automation/namespaces/FileSystemProvider.cs +++ b/src/System.Management.Automation/namespaces/FileSystemProvider.cs @@ -7840,8 +7840,6 @@ public static class InternalSymbolicLinkLinkCodeMethods // data is 16KB, plus there's a header. private const int MAX_REPARSE_SIZE = (16 * 1024) + REPARSE_GUID_DATA_BUFFER_HEADER_SIZE; - private const int ERROR_NOT_A_REPARSE_POINT = 4390; - private const int FSCTL_GET_REPARSE_POINT = 0x000900A8; private const int FSCTL_SET_REPARSE_POINT = 0x000900A4; @@ -8156,16 +8154,20 @@ private static string WinInternalGetLinkType(string filePath) // Get Buffer size IntPtr dangerousHandle = handle.DangerousGetHandle(); - bool result = DeviceIoControl(dangerousHandle, FSCTL_GET_REPARSE_POINT, - IntPtr.Zero, 0, outBuffer, outBufferSize, out bytesReturned, IntPtr.Zero); + bool result = DeviceIoControl( + dangerousHandle, + FSCTL_GET_REPARSE_POINT, + InBuffer: IntPtr.Zero, + nInBufferSize: 0, + outBuffer, + outBufferSize, + out bytesReturned, + lpOverlapped: IntPtr.Zero); if (!result) { - int lastError = Marshal.GetLastWin32Error(); - if (lastError == ERROR_NOT_A_REPARSE_POINT) - linkType = null; - else - throw new Win32Exception(lastError); + // It's not a reparse point or the file system doesn't support reparse points. + return IsHardLink(ref dangerousHandle) ? "HardLink" : null; } REPARSE_DATA_BUFFER_SYMBOLICLINK reparseDataBuffer = Marshal.PtrToStructure(outBuffer); @@ -8185,7 +8187,7 @@ private static string WinInternalGetLinkType(string filePath) break; default: - linkType = IsHardLink(ref dangerousHandle) ? "HardLink" : null; + linkType = null; break; } @@ -8402,16 +8404,20 @@ private static string WinInternalGetTarget(SafeFileHandle handle) // According to MSDN guidance DangerousAddRef() and DangerousRelease() have been used. handle.DangerousAddRef(ref success); - bool result = DeviceIoControl(handle.DangerousGetHandle(), FSCTL_GET_REPARSE_POINT, - IntPtr.Zero, 0, outBuffer, outBufferSize, out bytesReturned, IntPtr.Zero); + bool result = DeviceIoControl( + handle.DangerousGetHandle(), + FSCTL_GET_REPARSE_POINT, + InBuffer: IntPtr.Zero, + nInBufferSize: 0, + outBuffer, + outBufferSize, + out bytesReturned, + lpOverlapped: IntPtr.Zero); if (!result) { - int lastError = Marshal.GetLastWin32Error(); - if (lastError == ERROR_NOT_A_REPARSE_POINT) - return null; - - throw new Win32Exception(lastError); + // It's not a reparse point or the file system doesn't support reparse points. + return null; } string targetDir = null; From 05856acf1637e738cdc1c973784265c07349eefc Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 17 Dec 2020 20:40:01 +0000 Subject: [PATCH 14/21] Merged PR 13903: fix signing changes related issue in nuget fix signing changes related issue in nuget --- tools/releaseBuild/azureDevOps/templates/nuget.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index e5983079832..b78b294b5b9 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -53,7 +53,7 @@ jobs: condition: ne(variables['SDKVersion'], '5.0.100') - pwsh: | - Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force + Import-Module $env:REPOROOT\build.psm1 -Force Install-Dotnet -Version '5.0.100-rtm.20526.5' displayName: Install-DotNet condition: eq(variables['SDKVersion'], '5.0.100') From b4482f6209398aa9d99deb0da2c0bf5be78274bb Mon Sep 17 00:00:00 2001 From: Paul Higinbotham Date: Wed, 16 Dec 2020 17:35:35 -0800 Subject: [PATCH 15/21] Disable libmi dependent tests for macOS. (#14446) --- .../PSDesiredStateConfiguration.Tests.ps1 | 50 +++++++++++++++++++ .../configuration.Tests.ps1 | 3 ++ .../engine/Remoting/PSSession.Tests.ps1 | 12 +++-- .../Remoting/RemoteSession.Basic.Tests.ps1 | 27 ++++++---- 4 files changed, 80 insertions(+), 12 deletions(-) diff --git a/test/powershell/Modules/PSDesiredStateConfiguration/PSDesiredStateConfiguration.Tests.ps1 b/test/powershell/Modules/PSDesiredStateConfiguration/PSDesiredStateConfiguration.Tests.ps1 index c2fc8cf1e55..30a92d6a8ef 100644 --- a/test/powershell/Modules/PSDesiredStateConfiguration/PSDesiredStateConfiguration.Tests.ps1 +++ b/test/powershell/Modules/PSDesiredStateConfiguration/PSDesiredStateConfiguration.Tests.ps1 @@ -129,6 +129,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "https://github.com/PowerShell/PSDesiredStateConfiguration/issues/26" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + $resource = Get-DscResource -Name $name $resource | Should -Not -BeNullOrEmpty $resource.Name | Should -Be $Name @@ -148,6 +152,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "https://github.com/PowerShell/PSDesiredStateConfiguration/issues/26" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($PendingBecause) { Set-ItResult -Pending -Because $PendingBecause } @@ -225,6 +233,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Will only find script from PSDesiredStateConfiguration without modulename" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } @@ -254,6 +266,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "https://github.com/PowerShell/PSDesiredStateConfiguration/issues/12 and https://github.com/PowerShell/PowerShellGet/pull/529" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($PendingBecause) { Set-ItResult -Pending -Because $PendingBecause } @@ -315,6 +331,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($PendingBecause) { Set-ItResult -Pending -Because $PendingBecause } @@ -340,6 +360,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($PendingBecause) { Set-ItResult -Pending -Because $PendingBecause } @@ -402,6 +426,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if (!$IsLinux) { $result = Invoke-DscResource -Name PSModule -ModuleName $psGetModuleSpecification -Method set -Property @{ Name = 'PsDscResources' @@ -427,6 +455,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + # using create scriptBlock because $using: doesn't work with existing Invoke-DscResource # Verified in Windows PowerShell on 20190814 $result = Invoke-DscResource -Name Script -ModuleName PSDscResources -Method Set -Property @{TestScript = { Write-Output 'test'; return $false }; GetScript = { return @{ } }; SetScript = [scriptblock]::Create("`$global:DSCMachineStatus = $value;return") } @@ -438,6 +470,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $result = Invoke-DscResource -Name Script -ModuleName PSDscResources -Method Test -Property @{TestScript = { Write-Output 'test'; return $false }; GetScript = { return @{ } }; SetScript = { return } } $result | Should -Not -BeNullOrEmpty @@ -448,6 +483,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $result = Invoke-DscResource -Name Script -ModuleName PSDscResources -Method Test -Property @{TestScript = { Write-Verbose 'test'; return $true }; GetScript = { return @{ } }; SetScript = { return } } $result | Should -BeTrue -Because "Test method return true" @@ -457,6 +495,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $module = Get-Module PsDscResources -ListAvailable $moduleSpecification = @{ModuleName = $module.Name; ModuleVersion = $module.Version.ToString() } @@ -520,6 +561,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } { Invoke-DscResource -Name santoheusnaasonteuhsantoheu -Method Test -Property @{TestScript = { Write-Host 'test'; return $true }; GetScript = { return @{ } }; SetScript = { return } } -ErrorAction Stop @@ -531,6 +575,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($IsLinux) { Set-ItResult -Pending -Because "https://github.com/PowerShell/PSDesiredStateConfiguration/issues/12 and https://github.com/PowerShell/PowerShellGet/pull/529" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $result = Invoke-DscResource -Name PSModule -ModuleName $psGetModuleSpecification -Method Get -Property @{ Name = 'PsDscResources' } $result | Should -Not -BeNullOrEmpty @@ -577,6 +624,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $testString = '890574209347509120348' $result = Invoke-DscResource -Name XmlFileContentResource -ModuleName XmlContentDsc -Property @{Path = $resolvedXmlPath; XPath = '/configuration/appSetting/Test1'; Ensure = 'Present'; Attributes = @{ TestValue2 = $testString; Name = $testString } } -Method Set diff --git a/test/powershell/Modules/PSDesiredStateConfiguration/configuration.Tests.ps1 b/test/powershell/Modules/PSDesiredStateConfiguration/configuration.Tests.ps1 index 7b621578e4d..953b28acec7 100644 --- a/test/powershell/Modules/PSDesiredStateConfiguration/configuration.Tests.ps1 +++ b/test/powershell/Modules/PSDesiredStateConfiguration/configuration.Tests.ps1 @@ -15,6 +15,9 @@ Describe "DSC MOF Compilation" -tags "CI" { if ($IsLinux) { Set-ItResult -Pending -Because "https://github.com/PowerShell/PowerShellGet/pull/529" } + if ($IsMacOs) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } Write-Verbose "DSC_HOME: ${env:DSC_HOME}" -Verbose [Scriptblock]::Create(@" diff --git a/test/powershell/engine/Remoting/PSSession.Tests.ps1 b/test/powershell/engine/Remoting/PSSession.Tests.ps1 index e985ff6ab36..3b8259f3707 100644 --- a/test/powershell/engine/Remoting/PSSession.Tests.ps1 +++ b/test/powershell/engine/Remoting/PSSession.Tests.ps1 @@ -5,6 +5,11 @@ # PSSession tests for non-Windows platforms # +function GetRandomString() +{ + return [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetRandomFileName()) +} + Describe "New-PSSessionOption parameters for non-Windows platforms" -Tag "CI" { BeforeAll { @@ -36,12 +41,13 @@ Describe "SkipCACheck and SkipCNCheck PSSession options are required for New-PSS BeforeAll { $originalDefaultParameterValues = $PSDefaultParameterValues.Clone() - if ($IsWindows) { + # Skip this test for macOS because the latest OS release is incompatible with our shipped libmi for WinRM/OMI. + if ($IsWindows -or $IsMacOS) { $PSDefaultParameterValues['it:skip'] = $true } else { $userName = "User_$(Get-Random -Maximum 99999)" - $userPassword = "Password_$(Get-Random -Maximum 99999)" + $userPassword = GetRandomString $cred = [pscredential]::new($userName, (ConvertTo-SecureString -String $userPassword -AsPlainText -Force)) $soSkipCA = New-PSSessionOption -SkipCACheck $soSkipCN = New-PSSessionOption -SkipCNCheck @@ -54,7 +60,7 @@ Describe "SkipCACheck and SkipCNCheck PSSession options are required for New-PSS $testCases = @( @{ - Name = 'Verifies expected error when session options is missing' + Name = 'Verifies expected error when session option is missing' ScriptBlock = { New-PSSession -cn localhost -Credential $cred -Authentication Basic -UseSSL } ExpectedErrorCode = 825 }, diff --git a/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 b/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 index e48eaecb526..27d2fceb454 100644 --- a/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 +++ b/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 @@ -3,19 +3,25 @@ Import-Module HelpersCommon +function GetRandomString() +{ + return [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetRandomFileName()) +} + Describe "New-PSSession basic test" -Tag @("CI") { It "New-PSSession should not crash powershell" { $platformInfo = Get-PlatformInfo if ( ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') + ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($IsMacOS) ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, and CentOS 8" + Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" return } - { New-PSSession -ComputerName nonexistcomputer -Authentication Basic } | + { New-PSSession -ComputerName (GetRandomString) -Authentication Basic } | Should -Throw -ErrorId "InvalidOperation,Microsoft.PowerShell.Commands.NewPSSessionCommand" } } @@ -26,13 +32,14 @@ Describe "Basic Auth over HTTP not allowed on Unix" -Tag @("CI") { if ( ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') + ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($IsMacOS) ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, and CentOS 8" + Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" return } - $password = ConvertTo-SecureString -String "password" -AsPlainText -Force + $password = ConvertTo-SecureString -String (GetRandomString) -AsPlainText -Force $credential = [PSCredential]::new('username', $password) $err = ({New-PSSession -ComputerName 'localhost' -Credential $credential -Authentication Basic} | Should -Throw -PassThru -ErrorId 'System.Management.Automation.Remoting.PSRemotingDataStructureException,Microsoft.PowerShell.Commands.NewPSSessionCommand') @@ -42,18 +49,20 @@ Describe "Basic Auth over HTTP not allowed on Unix" -Tag @("CI") { $err.Exception.ErrorCode | Should -Be 801 } + # Skip this test for macOS because the latest OS release is incompatible with our shipped libmi for WinRM/OMI. It "New-PSSession should NOT throw a ConnectFailed exception when specifying Basic Auth over HTTPS on Unix" -Skip:($IsWindows) { $platformInfo = Get-PlatformInfo if ( ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') + ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($IsMacOS) ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, and CentOS 8" + Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" return } - $password = ConvertTo-SecureString -String "password" -AsPlainText -Force + $password = ConvertTo-SecureString -String (GetRandomString) -AsPlainText -Force $credential = [PSCredential]::new('username', $password) # use a Uri that specifies HTTPS to test Basic Auth logic. From 4b6cde094b4c07bf7f465503639461c0514760db Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Wed, 13 Jan 2021 17:29:24 +0000 Subject: [PATCH 16/21] Merged PR 14113: Update .NET SDK version and nuget package versions Update .NET SDK version and nuget package versions --- global.json | 2 +- src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj | 3 ++- .../System.Management.Automation.csproj | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/global.json b/global.json index ee8f712f95d..10c378d31f5 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "5.0.100" + "version": "5.0.102" } } diff --git a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj index f89ec614d82..502555d3716 100644 --- a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj +++ b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj @@ -16,7 +16,7 @@ - + @@ -30,6 +30,7 @@ + diff --git a/src/System.Management.Automation/System.Management.Automation.csproj b/src/System.Management.Automation/System.Management.Automation.csproj index e8d5ff5911d..5a3065af061 100644 --- a/src/System.Management.Automation/System.Management.Automation.csproj +++ b/src/System.Management.Automation/System.Management.Automation.csproj @@ -23,7 +23,7 @@ - + From adab3212a903db48a665eb29ccb0615b23a7d320 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Wed, 13 Jan 2021 18:34:00 +0000 Subject: [PATCH 17/21] Merged PR 14121: Update the file.wxs Update the `file.wxs` due to .NET runtime change. --- assets/files.wxs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/assets/files.wxs b/assets/files.wxs index 418a32db376..4aa9ab2dfd3 100644 --- a/assets/files.wxs +++ b/assets/files.wxs @@ -3057,8 +3057,8 @@ - - + + @@ -4045,7 +4045,7 @@ - + From 65766caa9543aadff15dac3224080f36cb90a072 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Wed, 13 Jan 2021 20:51:12 +0000 Subject: [PATCH 18/21] Merged PR 14128: Remove unneeded NuGet feeds for hosting SDK test Remove unneeded NuGet feeds for hosting SDK test, e.g. powershell.core/myget --- test/hosting/NuGet.Config | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/hosting/NuGet.Config b/test/hosting/NuGet.Config index fcdcf931709..765346e5343 100644 --- a/test/hosting/NuGet.Config +++ b/test/hosting/NuGet.Config @@ -2,9 +2,6 @@ - - - From 11278116ea260aa9a82d95e23a5898e50f21e620 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Wed, 13 Jan 2021 22:34:19 +0000 Subject: [PATCH 19/21] Merged PR 14132: Remove old .NET SDKs before SDK tests Remove old .NET SDKs before SDK tests --- .../azureDevOps/templates/release-SDKTests.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml index dd7c9618c9e..57e0a6905be 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml @@ -36,13 +36,6 @@ jobs: path: '$(Pipeline.Workspace)/releasePipeline/metadata' - pwsh: | - $dotnetMetadataPath = "$(Build.SourcesDirectory)/DotnetRuntimeMetadata.json" - $dotnetMetadataJson = Get-Content $dotnetMetadataPath -Raw | ConvertFrom-Json - - # Channel is like: $Channel = "5.0.1xx-preview2" - $Channel = $dotnetMetadataJson.sdk.channel - - $sdkVersion = (Get-Content "$(Build.SourcesDirectory)/global.json" -Raw | ConvertFrom-Json).sdk.version Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force Find-Dotnet @@ -54,9 +47,9 @@ jobs: Write-Verbose -Message "Register new package source 'dotnet5'" -verbose } - ## Install latest version from the channel + ## Remove old .NET SDKs + Remove-Item $Home/.dotnet -Recurse -Force - #Install-Dotnet -Channel "$Channel" -Version $sdkVersion Start-PSBootstrap Write-Verbose -Message "Installing .NET SDK completed." -Verbose From 24e2bbdde94623f50f4e3cf25d916dba3f6fda7b Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Wed, 13 Jan 2021 22:42:20 +0000 Subject: [PATCH 20/21] Merged PR 14135: Remove old .NET SDKs only if any exists Remove old .NET SDKs only if any exists --- .../azureDevOps/templates/release-SDKTests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml index 57e0a6905be..9ca03317a97 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml @@ -47,8 +47,11 @@ jobs: Write-Verbose -Message "Register new package source 'dotnet5'" -verbose } - ## Remove old .NET SDKs - Remove-Item $Home/.dotnet -Recurse -Force + ## Remove old .NET SDKs if any exists + if (Test-Path -Type Container $Home/.dotnet) + { + Remove-Item $Home/.dotnet -Recurse -Force + } Start-PSBootstrap From a2e7a5eab8d248ec8c7bd3a50d80d57defd07256 Mon Sep 17 00:00:00 2001 From: Andrew Menagarishvili Date: Thu, 14 Jan 2021 20:20:03 +0000 Subject: [PATCH 21/21] Merged PR 14143: Changelog for v7.1.1 Changelog for v7.1.1 --- .spelling | 1 + CHANGELOG/7.1.md | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/.spelling b/.spelling index f9406e0c2f0..cc0d9a735b3 100644 --- a/.spelling +++ b/.spelling @@ -757,6 +757,7 @@ rkitover robo210 ronn rpalo +RPMs runspace runspaceinit runspaces diff --git a/CHANGELOG/7.1.md b/CHANGELOG/7.1.md index a51cfcb7a41..6f92dd03b54 100644 --- a/CHANGELOG/7.1.md +++ b/CHANGELOG/7.1.md @@ -1,5 +1,37 @@ # 7.1 Changelog +## [7.1.1] - 2021-01-14 + +### General Cmdlet Updates and Fixes + +- Avoid an exception if file system does not support reparse points (#13634) (Thanks @iSazonov!) +- Make AppLocker Enforce mode take precedence over UMCI Audit mode (#14353) + +### Code Cleanup + +- Fix syntax error in Windows packaging script (#14377) + +### Build and Packaging Improvements + +
+ +
    +
  • Use one feed in each nuget.config in official builds (#14363)
    • +
  • Fix path signed RPMs are uploaded from in release build (#14424)
    • +
  • Fix issue with unsigned build (#14367)
    • +
  • Move macOS and NuGet packages to ESRP signing (#14324)
    • +
  • Move Windows packages signing to use ESRP (#14060)
    • +
  • Move Linux packages to ESRP signing (#14210)
    • +
  • Migrate 3rd party signing to ESRP (#14010)
    • +
  • Don't do a shallow checkout (#13992)
    • +
  • Move to ESRP signing for Windows files (#13988)
    • +
  • Add checkout step to release build templates (#13840)
    • +
+ +
+ +[7.1.1]: https://github.com/PowerShell/PowerShell/compare/v7.1.0...v7.1.1 + ## [7.1.0] - 2020-11-11 ### Engine Updates and Fixes