From 46fbeace7a863b447f2b9befe854069897895480 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 28 Oct 2024 18:51:53 +0000 Subject: [PATCH 001/316] Merged PR 33107: Change Microsoft.Windows.Compatibility version to CFS feed version changing Microsoft.Windows.Compatibility version to CFS feed version ---- #### AI description (iteration 1) #### PR Classification Dependency version update. #### PR Summary This pull request updates the version of the `Microsoft.Windows.Compatibility` package to align with the CFS feed version. - `src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj`: Changed `Microsoft.Windows.Compatibility` version from `9.0.0-rc.2.24510.7` to `9.0.0-rc.2.24474.4`. --- src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj index 5a1001ddc2c..0f1fef9362b 100644 --- a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj +++ b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj @@ -36,7 +36,7 @@ - + From 8e92318b5edd0788776eba2e160e7ebb2d35b8f7 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 29 Oct 2024 18:58:17 +0000 Subject: [PATCH 002/316] Merged PR 33114: Turned on deploy box for GitHub Release Stage Turned on deploy box for GitHub Release Stage ---- #### AI description (iteration 1) #### PR Classification Code cleanup and configuration update. #### PR Summary This pull request enables the deploy box for the GitHub Release Stage and updates related configurations. - `.pipelines/templates/release-githubtasks.yml`: Commented out the Azure RM module uninstallation and Az.Storage module installation steps, and added a new job to check `Pipeline.Workspace`. - `.pipelines/PowerShell-Release-Official.yml`: Updated the `WindowsHostVersion` to use `Version: 2022` and `Network: Netlock`, and added a `release` parameter. --- .pipelines/PowerShell-Release-Official.yml | 15 +- .pipelines/templates/release-githubtasks.yml | 156 ++++++++++++++----- 2 files changed, 126 insertions(+), 45 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 2538d1b5370..eedd2027b9c 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -79,12 +79,17 @@ resources: extends: template: v2/OneBranch.Official.CrossPlat.yml@templates parameters: + release: NonAzure # still using KS2 because we are not yet using a Box Product Deployment + # featureFlags: + # LinuxHostVersion: + # Network: KS2 + # WindowsHostVersion: + # Network: KS2 featureFlags: - LinuxHostVersion: - Network: KS2 WindowsHostVersion: - Network: KS2 + Version: 2022 + Network: Netlock cloudvault: enabled: false globalSdl: @@ -272,7 +277,9 @@ extends: - stage: PublishGitHubRelease displayName: Publish GitHub Release - dependsOn: BlobPublic + # dependsOn: BlobPublic + variables: + ob_release_environment: PPE jobs: - template: /.pipelines/templates/release-githubtasks.yml@self diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 039c171290f..ea2a7ead174 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -3,7 +3,73 @@ jobs: displayName: Create GitHub Release Draft condition: succeeded() pool: - type: windows + type: release + os: windows + templateContext: + inputs: + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_deb + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_fxdependent + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_mariner_arm64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_mariner_x64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_minSize + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_rpm + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_tar + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_tar_alpine + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_tar_alpine_fxd + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_tar_arm + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_tar_arm64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_mac_package_sign_package_macOS_arm64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_mac_package_sign_package_macOS_x64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_mac_package_sign_package_windows_x64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_windows_package_package_win_arm64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_windows_package_package_win_fxdendent + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_windows_package_package_win_fxdependentWinDesktop + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_windows_package_package_win_minsize + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_windows_package_package_win_x64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_windows_package_package_win_x86 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: macos-pkgs variables: - name: runCodesignValidationInjection value: false @@ -37,50 +103,58 @@ jobs: Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' - - pwsh: | - # Uninstall Azure RM modules - $azRmModules = Get-Module -Name AzureRM* -ListAvailable - if ($azRmModules) { - $azRmModules | Remove-Module -Force - } - - # Install Az.Storage module if not already installed - if (-not (Get-Module -Name Az.Storage -ListAvailable)) { - Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose - } - displayName: Install Az.Storage module - - - task: AzurePowerShell@5 - displayName: Download packages from Azure Storage - inputs: - azureSubscription: az-blob-cicd-infra - scriptType: inlineScript - azurePowerShellVersion: LatestVersion - pwsh: true - inline: | - $storageAccount = "$(StorageAccount)" - $containerName = "$(AzureVersion)" - $destinationPath = "$(System.ArtifactsDirectory)" - - # Get storage account context - $storageContext = New-AzStorageContext -StorageAccountName $storageAccount - - $blobList = Get-AzStorageBlob -Container $containerName -Context $storageContext - foreach ($blob in $blobList) { - $blobName = $blob.Name - $destinationFile = Join-Path -Path $destinationPath -ChildPath $blobName - Get-AzStorageBlobContent -Container $containerName -Blob $blobName -Destination $destinationFile -Context $storageContext -Force - Write-Output "Downloaded $blobName to $destinationFile" - } - - $packagesPath = Get-ChildItem -Path $destinationPath\*.deb -Recurse -File | Select-Object -First 1 -ExpandProperty DirectoryName - Write-Host "sending -- vso[task.setvariable variable=PackagesRoot]$packagesPath" - Write-Host "##vso[task.setvariable variable=PackagesRoot]$packagesPath" + # - pwsh: | + # # Uninstall Azure RM modules + # $azRmModules = Get-Module -Name AzureRM* -ListAvailable + # if ($azRmModules) { + # $azRmModules | Remove-Module -Force + # } + # + # # Install Az.Storage module if not already installed + # if (-not (Get-Module -Name Az.Storage -ListAvailable)) { + # Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose + # } + # displayName: Install Az.Storage module + + # - task: AzurePowerShell@5 + # displayName: Download packages from Azure Storage + # inputs: + # azureSubscription: az-blob-cicd-infra + # scriptType: inlineScript + # azurePowerShellVersion: LatestVersion + # pwsh: true + # inline: | + # $storageAccount = "$(StorageAccount)" + # $containerName = "$(AzureVersion)" + # $destinationPath = "$(System.ArtifactsDirectory)" + # + # # Get storage account context + # $storageContext = New-AzStorageContext -StorageAccountName $storageAccount + # + # $blobList = Get-AzStorageBlob -Container $containerName -Context $storageContext + # foreach ($blob in $blobList) { + # $blobName = $blob.Name + # $destinationFile = Join-Path -Path $destinationPath -ChildPath $blobName + # Get-AzStorageBlobContent -Container $containerName -Blob $blobName -Destination $destinationFile -Context $storageContext -Force + # Write-Output "Downloaded $blobName to $destinationFile" + # } + # + # $packagesPath = Get-ChildItem -Path $destinationPath\*.deb -Recurse -File | Select-Object -First 1 -ExpandProperty DirectoryName + # Write-Host "sending -- vso[task.setvariable variable=PackagesRoot]$packagesPath" + # Write-Host "##vso[task.setvariable variable=PackagesRoot]$packagesPath" - pwsh: | - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty FullName + Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName displayName: Capture downloaded artifacts + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Check Pipeline.Workspace + jobName: CheckPipelineWorkspace + dependsOnJob: GithubReleaseDraft + instructions: | + Check the Pipeline.Workspace for the downloaded artifacts + - pwsh: | git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' displayName: Clone Internal-Tools repository From 7575aa3d7b11c20b4dd975728f9bc35f055a2241 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 29 Oct 2024 14:02:45 -0500 Subject: [PATCH 003/316] removed approval before github --- .pipelines/templates/release-githubtasks.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index ea2a7ead174..f435706e405 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -147,14 +147,6 @@ jobs: Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName displayName: Capture downloaded artifacts - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Check Pipeline.Workspace - jobName: CheckPipelineWorkspace - dependsOnJob: GithubReleaseDraft - instructions: | - Check the Pipeline.Workspace for the downloaded artifacts - - pwsh: | git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' displayName: Clone Internal-Tools repository From 7e1d52e3aead7c0c4a4063e184f65d5fc54d5543 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 29 Oct 2024 14:07:03 -0500 Subject: [PATCH 004/316] manual validation changed from agentless to server --- .pipelines/PowerShell-Release-Official.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index eedd2027b9c..8d5d5d272a9 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -205,6 +205,8 @@ extends: - stage: ManualValidation dependsOn: [] displayName: Manual Validation + pool: + type: server jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: From a19e382a65bd354b32cb8ed783b569009cb2ac48 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 29 Oct 2024 14:13:45 -0500 Subject: [PATCH 005/316] added pool type server to approvalJob --- .pipelines/PowerShell-Release-Official.yml | 2 -- .pipelines/templates/approvalJob.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 8d5d5d272a9..eedd2027b9c 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -205,8 +205,6 @@ extends: - stage: ManualValidation dependsOn: [] displayName: Manual Validation - pool: - type: server jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: diff --git a/.pipelines/templates/approvalJob.yml b/.pipelines/templates/approvalJob.yml index ac3b8bc2ab2..d4cda2ad353 100644 --- a/.pipelines/templates/approvalJob.yml +++ b/.pipelines/templates/approvalJob.yml @@ -25,7 +25,7 @@ jobs: dependsOn: ${{ parameters.dependsOnJob }} displayName: ${{ parameters.displayName }} pool: - type: agentless + type: server timeoutInMinutes: 4320 # job times out in 3 days steps: - task: ManualValidation@0 From e3f1b94c0577e9824b17771857441015b9837cbb Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 29 Oct 2024 14:24:12 -0500 Subject: [PATCH 006/316] trying prefix PPE --- .pipelines/PowerShell-Release-Official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index eedd2027b9c..7c713d66fcf 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -275,7 +275,7 @@ extends: parameters: SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} - - stage: PublishGitHubRelease + - stage: PPE_PublishGitHubRelease displayName: Publish GitHub Release # dependsOn: BlobPublic variables: From 4df73f578554f0793782a1c851cb579b16d190d6 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 29 Oct 2024 14:29:25 -0500 Subject: [PATCH 007/316] category nonAzure --- .pipelines/PowerShell-Release-Official.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 7c713d66fcf..018050bd355 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -79,7 +79,8 @@ resources: extends: template: v2/OneBranch.Official.CrossPlat.yml@templates parameters: - release: NonAzure + release: + category: NonAzure # still using KS2 because we are not yet using a Box Product Deployment # featureFlags: # LinuxHostVersion: @@ -276,7 +277,7 @@ extends: SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} - stage: PPE_PublishGitHubRelease - displayName: Publish GitHub Release + displayName: PPE:Publish GitHub Release # dependsOn: BlobPublic variables: ob_release_environment: PPE From 10dfc479184cb82178cb064dfa0718b2791d17fc Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 29 Oct 2024 14:39:09 -0500 Subject: [PATCH 008/316] trying ob_release_environment --- .pipelines/PowerShell-Release-Official.yml | 2 ++ .pipelines/templates/release-githubtasks.yml | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 018050bd355..7c4e3367750 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -283,6 +283,8 @@ extends: ob_release_environment: PPE jobs: - template: /.pipelines/templates/release-githubtasks.yml@self + parameters: + ob_release_environemnt: $(ob_release_environment) - stage: PublishNuGet displayName: Publish NuGet diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index f435706e405..4eeb8bc9012 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -1,3 +1,8 @@ +parameters: + - name: ob_release_environemnt + type: string + default: PPE + jobs: - job: GithubReleaseDraft displayName: Create GitHub Release Draft From ffd1db37b680e101cb58b3a60d4eef9cd7f48551 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 30 Oct 2024 10:51:06 -0500 Subject: [PATCH 009/316] switch to nonofficial ob to test --- .pipelines/PowerShell-Release-Official.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 7c4e3367750..81d00ba5c22 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -77,7 +77,7 @@ resources: - releases/* extends: - template: v2/OneBranch.Official.CrossPlat.yml@templates + template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates parameters: release: category: NonAzure @@ -280,7 +280,7 @@ extends: displayName: PPE:Publish GitHub Release # dependsOn: BlobPublic variables: - ob_release_environment: PPE + ob_release_environment: Test jobs: - template: /.pipelines/templates/release-githubtasks.yml@self parameters: From 9294f9f394988ba1d375a638ef8cbe420a3541f0 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 30 Oct 2024 12:13:09 -0500 Subject: [PATCH 010/316] moved variables up --- .pipelines/PowerShell-Release-Official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 81d00ba5c22..8ea04507ae0 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -278,9 +278,9 @@ extends: - stage: PPE_PublishGitHubRelease displayName: PPE:Publish GitHub Release - # dependsOn: BlobPublic variables: ob_release_environment: Test + # dependsOn: BlobPublic jobs: - template: /.pipelines/templates/release-githubtasks.yml@self parameters: From 1d9e5153e0854811fbfcc198d2645eec485b891a Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 30 Oct 2024 12:14:34 -0500 Subject: [PATCH 011/316] trying name value --- .pipelines/PowerShell-Release-Official.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 8ea04507ae0..0ba6c7d4e58 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -279,7 +279,8 @@ extends: - stage: PPE_PublishGitHubRelease displayName: PPE:Publish GitHub Release variables: - ob_release_environment: Test + name: ob_release_environment + value: PPE # dependsOn: BlobPublic jobs: - template: /.pipelines/templates/release-githubtasks.yml@self From 4d9606bfa06addec46965b9878ceab78c10be7cf Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 30 Oct 2024 12:24:24 -0500 Subject: [PATCH 012/316] added ob for all approval jobs --- .pipelines/PowerShell-Release-Official.yml | 25 ++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 0ba6c7d4e58..f0538e776bb 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -276,11 +276,10 @@ extends: parameters: SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} - - stage: PPE_PublishGitHubRelease - displayName: PPE:Publish GitHub Release + - stage: PublishGitHubRelease + displayName: Publish GitHub Release variables: - name: ob_release_environment - value: PPE + ob_release_environment: Test # dependsOn: BlobPublic jobs: - template: /.pipelines/templates/release-githubtasks.yml@self @@ -306,6 +305,8 @@ extends: - stage: ReleaseDocker dependsOn: PublishGitHubRelease displayName: 'Docker Release' + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -325,6 +326,8 @@ extends: - stage: UpdateDotnetDocker dependsOn: PublishGitHubRelease displayName: Update DotNet SDK Docker images + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -340,6 +343,8 @@ extends: - stage: UpdateWinGet dependsOn: PublishGitHubRelease displayName: Add manifest entry to winget + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -351,6 +356,8 @@ extends: - stage: PublishMsix dependsOn: PublishGitHubRelease displayName: Publish MSIX to store + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -362,6 +369,8 @@ extends: - stage: PublishVPack dependsOn: PublishGitHubRelease displayName: Release vPack + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -392,6 +401,8 @@ extends: - stage: ChangesToMaster displayName: Ensure changes are in GH master dependsOn: ['PublishNuGet', 'PublishPMC'] + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -403,6 +414,8 @@ extends: - stage: ReleaseSnap displayName: Release Snap dependsOn: 'ChangesToMaster' + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -414,6 +427,8 @@ extends: - stage: ReleaseToMU displayName: Release to MU dependsOn: ['PublishNuGet', 'PublishPMC', 'ChangesToMaster'] + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -424,6 +439,8 @@ extends: - stage: ReleaseClose displayName: Finish Release dependsOn: ['ReleaseToMU', 'ReleaseSymbols', 'ReleaseSnap'] + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: From 31daeaadf63f533a2de2501e7bc9b3281c01da8c Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 30 Oct 2024 12:27:27 -0500 Subject: [PATCH 013/316] missed a couple added ob for all approval jobs --- .pipelines/PowerShell-Release-Official.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index f0538e776bb..c0a0bcefa43 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -206,6 +206,8 @@ extends: - stage: ManualValidation dependsOn: [] displayName: Manual Validation + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -224,6 +226,8 @@ extends: - stage: ReleaseAutomation dependsOn: [] displayName: 'Release Automation' + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -250,6 +254,8 @@ extends: - stage: UpdateChangeLog displayName: Update the changelog + variables: + ob_release_environment: Test dependsOn: - ManualValidation - ReleaseAutomation From 512486ee238e2d0e391ac6848cf4e8d73882e111 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 30 Oct 2024 14:25:57 -0500 Subject: [PATCH 014/316] got rid of template parameters and added os windows --- .pipelines/PowerShell-Release-Official.yml | 2 -- .pipelines/templates/approvalJob.yml | 1 + .pipelines/templates/release-githubtasks.yml | 5 ----- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index c0a0bcefa43..6f4e7e8a705 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -289,8 +289,6 @@ extends: # dependsOn: BlobPublic jobs: - template: /.pipelines/templates/release-githubtasks.yml@self - parameters: - ob_release_environemnt: $(ob_release_environment) - stage: PublishNuGet displayName: Publish NuGet diff --git a/.pipelines/templates/approvalJob.yml b/.pipelines/templates/approvalJob.yml index d4cda2ad353..594beb63f0a 100644 --- a/.pipelines/templates/approvalJob.yml +++ b/.pipelines/templates/approvalJob.yml @@ -26,6 +26,7 @@ jobs: displayName: ${{ parameters.displayName }} pool: type: server + os: windows timeoutInMinutes: 4320 # job times out in 3 days steps: - task: ManualValidation@0 diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 4eeb8bc9012..f435706e405 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -1,8 +1,3 @@ -parameters: - - name: ob_release_environemnt - type: string - default: PPE - jobs: - job: GithubReleaseDraft displayName: Create GitHub Release Draft From eae6e9d498a3a086e04702fcf2a5d82892d0111d Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 30 Oct 2024 14:27:49 -0500 Subject: [PATCH 015/316] trying official template --- .pipelines/PowerShell-Release-Official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 6f4e7e8a705..0706b284a5c 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -77,7 +77,7 @@ resources: - releases/* extends: - template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates + template: v2/OneBranch.Official.CrossPlat.yml@templates parameters: release: category: NonAzure From e259c76f7d9992d4861015c9bfa7f4eecc2f0233 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 30 Oct 2024 14:29:26 -0500 Subject: [PATCH 016/316] adding nonofficial back --- .pipelines/PowerShell-Release-Official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 0706b284a5c..6f4e7e8a705 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -77,7 +77,7 @@ resources: - releases/* extends: - template: v2/OneBranch.Official.CrossPlat.yml@templates + template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates parameters: release: category: NonAzure From c440aab260684bab3d16a7ad85e3a4493432a9fb Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 31 Oct 2024 14:48:57 -0500 Subject: [PATCH 017/316] removed os: windows form approvalJobs --- .pipelines/templates/approvalJob.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/templates/approvalJob.yml b/.pipelines/templates/approvalJob.yml index 594beb63f0a..d4cda2ad353 100644 --- a/.pipelines/templates/approvalJob.yml +++ b/.pipelines/templates/approvalJob.yml @@ -26,7 +26,6 @@ jobs: displayName: ${{ parameters.displayName }} pool: type: server - os: windows timeoutInMinutes: 4320 # job times out in 3 days steps: - task: ManualValidation@0 From a252aad18a1747f437ef5125bcbee936e409f70a Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 31 Oct 2024 14:50:36 -0500 Subject: [PATCH 018/316] changing msix type to server --- .pipelines/templates/release-create-msix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-create-msix.yml b/.pipelines/templates/release-create-msix.yml index 448a46c1194..c3a11175c68 100644 --- a/.pipelines/templates/release-create-msix.yml +++ b/.pipelines/templates/release-create-msix.yml @@ -2,7 +2,7 @@ jobs: - job: CreateMSIXBundle displayName: Create .msixbundle file pool: - type: windows + type: server variables: - group: msixTools From e230ddfb9162daf6694b69cc8f8efcb565e3efc4 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 31 Oct 2024 14:52:57 -0500 Subject: [PATCH 019/316] added ob release env to create msix --- .pipelines/PowerShell-Release-Official.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 6f4e7e8a705..f41e8b407e0 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -118,6 +118,8 @@ extends: stages: - stage: msixbundle displayName: 'Create MSIX Bundle' + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/release-create-msix.yml@self From 6ca5163b5a95647c80993caa1a9b27aea2427a04 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 31 Oct 2024 14:55:53 -0500 Subject: [PATCH 020/316] removed ob_output directory from msix undid last two commits turning it into release stage --- .pipelines/PowerShell-Release-Official.yml | 2 -- .pipelines/templates/release-create-msix.yml | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index f41e8b407e0..6f4e7e8a705 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -118,8 +118,6 @@ extends: stages: - stage: msixbundle displayName: 'Create MSIX Bundle' - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/release-create-msix.yml@self diff --git a/.pipelines/templates/release-create-msix.yml b/.pipelines/templates/release-create-msix.yml index c3a11175c68..5e27658aa1f 100644 --- a/.pipelines/templates/release-create-msix.yml +++ b/.pipelines/templates/release-create-msix.yml @@ -2,13 +2,13 @@ jobs: - job: CreateMSIXBundle displayName: Create .msixbundle file pool: - type: server + type: windows variables: - group: msixTools - group: 'Azure Blob variable group' - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + # - name: ob_outputDirectory + # value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' steps: - template: release-SetReleaseTagandContainerName.yml@self From d79fd0eef97525f3f80acd8534a017e1c6caa3bb Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 31 Oct 2024 17:23:35 -0500 Subject: [PATCH 021/316] trying to add ob_release_environment --- .pipelines/templates/release-create-msix.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-create-msix.yml b/.pipelines/templates/release-create-msix.yml index 5e27658aa1f..2cfe10bad15 100644 --- a/.pipelines/templates/release-create-msix.yml +++ b/.pipelines/templates/release-create-msix.yml @@ -7,8 +7,10 @@ jobs: variables: - group: msixTools - group: 'Azure Blob variable group' - # - name: ob_outputDirectory - # value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_release_environment + value: Test steps: - template: release-SetReleaseTagandContainerName.yml@self From ba62a21aa50198104450ad38fbab44f320603689 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 31 Oct 2024 17:24:33 -0500 Subject: [PATCH 022/316] trying server again --- .pipelines/templates/release-create-msix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-create-msix.yml b/.pipelines/templates/release-create-msix.yml index 2cfe10bad15..da5879a9676 100644 --- a/.pipelines/templates/release-create-msix.yml +++ b/.pipelines/templates/release-create-msix.yml @@ -2,7 +2,7 @@ jobs: - job: CreateMSIXBundle displayName: Create .msixbundle file pool: - type: windows + type: server variables: - group: msixTools From 3b3bf96ff34f954892eec14cf3009244ac3d33ac Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 31 Oct 2024 17:26:19 -0500 Subject: [PATCH 023/316] trying stage variable again --- .pipelines/PowerShell-Release-Official.yml | 2 ++ .pipelines/templates/release-create-msix.yml | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 6f4e7e8a705..f41e8b407e0 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -118,6 +118,8 @@ extends: stages: - stage: msixbundle displayName: 'Create MSIX Bundle' + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/release-create-msix.yml@self diff --git a/.pipelines/templates/release-create-msix.yml b/.pipelines/templates/release-create-msix.yml index da5879a9676..c3a11175c68 100644 --- a/.pipelines/templates/release-create-msix.yml +++ b/.pipelines/templates/release-create-msix.yml @@ -9,8 +9,6 @@ jobs: - group: 'Azure Blob variable group' - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_release_environment - value: Test steps: - template: release-SetReleaseTagandContainerName.yml@self From 970273aaa4f466cba10dce7aa0e77f37df9039e5 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 31 Oct 2024 17:39:30 -0500 Subject: [PATCH 024/316] trying to template entire stage and removed all ob_release_environments --- .pipelines/PowerShell-Release-Official.yml | 34 +- .pipelines/templates/release-create-msix.yml | 2 +- .pipelines/templates/release-githubtasks.yml | 464 ++++++++++--------- 3 files changed, 238 insertions(+), 262 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index f41e8b407e0..6f649306b96 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -118,8 +118,6 @@ extends: stages: - stage: msixbundle displayName: 'Create MSIX Bundle' - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/release-create-msix.yml@self @@ -208,8 +206,6 @@ extends: - stage: ManualValidation dependsOn: [] displayName: Manual Validation - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -228,8 +224,6 @@ extends: - stage: ReleaseAutomation dependsOn: [] displayName: 'Release Automation' - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -256,8 +250,6 @@ extends: - stage: UpdateChangeLog displayName: Update the changelog - variables: - ob_release_environment: Test dependsOn: - ManualValidation - ReleaseAutomation @@ -284,13 +276,9 @@ extends: parameters: SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} - - stage: PublishGitHubRelease - displayName: Publish GitHub Release - variables: - ob_release_environment: Test + + - template: /.pipelines/templates/release-githubtasks.yml@self # dependsOn: BlobPublic - jobs: - - template: /.pipelines/templates/release-githubtasks.yml@self - stage: PublishNuGet displayName: Publish NuGet @@ -311,8 +299,6 @@ extends: - stage: ReleaseDocker dependsOn: PublishGitHubRelease displayName: 'Docker Release' - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -332,8 +318,6 @@ extends: - stage: UpdateDotnetDocker dependsOn: PublishGitHubRelease displayName: Update DotNet SDK Docker images - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -349,8 +333,6 @@ extends: - stage: UpdateWinGet dependsOn: PublishGitHubRelease displayName: Add manifest entry to winget - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -362,8 +344,6 @@ extends: - stage: PublishMsix dependsOn: PublishGitHubRelease displayName: Publish MSIX to store - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -375,8 +355,6 @@ extends: - stage: PublishVPack dependsOn: PublishGitHubRelease displayName: Release vPack - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -407,8 +385,6 @@ extends: - stage: ChangesToMaster displayName: Ensure changes are in GH master dependsOn: ['PublishNuGet', 'PublishPMC'] - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -420,8 +396,6 @@ extends: - stage: ReleaseSnap displayName: Release Snap dependsOn: 'ChangesToMaster' - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -433,8 +407,6 @@ extends: - stage: ReleaseToMU displayName: Release to MU dependsOn: ['PublishNuGet', 'PublishPMC', 'ChangesToMaster'] - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -445,8 +417,6 @@ extends: - stage: ReleaseClose displayName: Finish Release dependsOn: ['ReleaseToMU', 'ReleaseSymbols', 'ReleaseSnap'] - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: diff --git a/.pipelines/templates/release-create-msix.yml b/.pipelines/templates/release-create-msix.yml index c3a11175c68..448a46c1194 100644 --- a/.pipelines/templates/release-create-msix.yml +++ b/.pipelines/templates/release-create-msix.yml @@ -2,7 +2,7 @@ jobs: - job: CreateMSIXBundle displayName: Create .msixbundle file pool: - type: server + type: windows variables: - group: msixTools diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index f435706e405..19050481ca9 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -1,230 +1,236 @@ -jobs: -- job: GithubReleaseDraft - displayName: Create GitHub Release Draft - condition: succeeded() - pool: - type: release - os: windows - templateContext: - inputs: - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_deb - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_fxdependent - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_mariner_arm64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_mariner_x64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_minSize - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_rpm - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_tar - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_tar_alpine - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_tar_alpine_fxd - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_tar_arm - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_tar_arm64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_mac_package_sign_package_macOS_arm64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_mac_package_sign_package_macOS_x64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_mac_package_sign_package_windows_x64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_arm64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_fxdendent - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_fxdependentWinDesktop - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_minsize - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_x64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_x86 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: macos-pkgs +stages: +- stage: PublishGitHubRelease + displayName: Publish GitHub Release variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - value: 1 - - group: 'mscodehub-code-read-akv' - - group: 'Azure Blob variable group' - - group: 'GitHubTokens' - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_sdl_codeSignValidation_enabled - value: false - - name: ob_sdl_binskim_enabled - value: false - - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - - steps: - - checkout: self - clean: true - env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - - - template: release-SetReleaseTagAndContainerName.yml - - - pwsh: | - Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose - displayName: 'Capture Environment Variables' - - # - pwsh: | - # # Uninstall Azure RM modules - # $azRmModules = Get-Module -Name AzureRM* -ListAvailable - # if ($azRmModules) { - # $azRmModules | Remove-Module -Force - # } - # - # # Install Az.Storage module if not already installed - # if (-not (Get-Module -Name Az.Storage -ListAvailable)) { - # Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose - # } - # displayName: Install Az.Storage module - - # - task: AzurePowerShell@5 - # displayName: Download packages from Azure Storage - # inputs: - # azureSubscription: az-blob-cicd-infra - # scriptType: inlineScript - # azurePowerShellVersion: LatestVersion - # pwsh: true - # inline: | - # $storageAccount = "$(StorageAccount)" - # $containerName = "$(AzureVersion)" - # $destinationPath = "$(System.ArtifactsDirectory)" - # - # # Get storage account context - # $storageContext = New-AzStorageContext -StorageAccountName $storageAccount - # - # $blobList = Get-AzStorageBlob -Container $containerName -Context $storageContext - # foreach ($blob in $blobList) { - # $blobName = $blob.Name - # $destinationFile = Join-Path -Path $destinationPath -ChildPath $blobName - # Get-AzStorageBlobContent -Container $containerName -Blob $blobName -Destination $destinationFile -Context $storageContext -Force - # Write-Output "Downloaded $blobName to $destinationFile" - # } - # - # $packagesPath = Get-ChildItem -Path $destinationPath\*.deb -Recurse -File | Select-Object -First 1 -ExpandProperty DirectoryName - # Write-Host "sending -- vso[task.setvariable variable=PackagesRoot]$packagesPath" - # Write-Host "##vso[task.setvariable variable=PackagesRoot]$packagesPath" - - - pwsh: | - Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName - displayName: Capture downloaded artifacts - - - pwsh: | - git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' - displayName: Clone Internal-Tools repository - - - pwsh: | - $Path = "$(System.ArtifactsDirectory)" - $OutputPath = Join-Path $Path 'hashes.sha256' - $srcPaths = @($Path) - $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File - $checksums = $packages | - ForEach-Object { - Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" - $packageName = $_.Name - $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() - # the '*' before the packagename signifies it is a binary - "$hash *$packageName" - } - $checksums | Out-File -FilePath $OutputPath -Force - $fileContent = Get-Content -Path $OutputPath -Raw | Out-String - Write-Verbose -Verbose -Message $fileContent - displayName: Add sha256 hashes - - - pwsh: | - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Set release version' - - - pwsh: | - Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion - - $isPreview = $semanticVersion.PreReleaseLabel -ne $null - - $fileName = if ($isPreview) { - "preview.md" - } - else { - $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" - } - - $filePath = "$env:BUILD_SOURCESDIRECTORY/PowerShell/CHANGELOG/$fileName" - Write-Verbose -Verbose "Selected Log file: $filePath" - - if (-not (Test-Path $filePath)) { - throw "$filePath not found" - } - - $changelog = Get-Content -Path $filePath - - $startPattern = "^## \[" + ([regex]::Escape($releaseVersion)) + "\]" - $endPattern = "^## \[{0}\.{1}\.{2}*" -f $semanticVersion.Major, $semanticVersion.Minor, $semanticVersion.Patch - - $clContent = $changelog | ForEach-Object { - if ($_ -match $startPattern) { $outputLine = $true } - elseif ($_ -match $endPattern) { $outputLine = $false } - if ($outputLine) { $_} - } | Out-String - - Write-Verbose -Verbose "Selected content: `n$clContent" - - Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $(PackagesRoot) -Token $(GitHubReleasePat) - displayName: Publish Release Draft - -- template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Push Git Tag - jobName: PushGitTag - dependsOnJob: GithubReleaseDraft - instructions: | - Push the git tag to upstream - -- template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Make Draft Public - jobName: DraftPublic - dependsOnJob: PushGitTag - instructions: | - Make the GitHub Release Draft Public + ob_release_environment: Test + jobs: + - job: GithubReleaseDraft + displayName: Create GitHub Release Draft + condition: succeeded() + pool: + type: release + os: windows + templateContext: + inputs: + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_deb + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_fxdependent + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_mariner_arm64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_mariner_x64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_minSize + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_rpm + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_tar + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_tar_alpine + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_tar_alpine_fxd + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_tar_arm + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_linux_package_tar_arm64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_mac_package_sign_package_macOS_arm64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_mac_package_sign_package_macOS_x64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_mac_package_sign_package_windows_x64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_windows_package_package_win_arm64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_windows_package_package_win_fxdendent + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_windows_package_package_win_fxdependentWinDesktop + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_windows_package_package_win_minsize + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_windows_package_package_win_x64 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_windows_package_package_win_x86 + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: macos-pkgs + variables: + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: 'mscodehub-code-read-akv' + - group: 'Azure Blob variable group' + - group: 'GitHubTokens' + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: release-SetReleaseTagAndContainerName.yml + + - pwsh: | + Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + displayName: 'Capture Environment Variables' + + # - pwsh: | + # # Uninstall Azure RM modules + # $azRmModules = Get-Module -Name AzureRM* -ListAvailable + # if ($azRmModules) { + # $azRmModules | Remove-Module -Force + # } + # + # # Install Az.Storage module if not already installed + # if (-not (Get-Module -Name Az.Storage -ListAvailable)) { + # Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose + # } + # displayName: Install Az.Storage module + + # - task: AzurePowerShell@5 + # displayName: Download packages from Azure Storage + # inputs: + # azureSubscription: az-blob-cicd-infra + # scriptType: inlineScript + # azurePowerShellVersion: LatestVersion + # pwsh: true + # inline: | + # $storageAccount = "$(StorageAccount)" + # $containerName = "$(AzureVersion)" + # $destinationPath = "$(System.ArtifactsDirectory)" + # + # # Get storage account context + # $storageContext = New-AzStorageContext -StorageAccountName $storageAccount + # + # $blobList = Get-AzStorageBlob -Container $containerName -Context $storageContext + # foreach ($blob in $blobList) { + # $blobName = $blob.Name + # $destinationFile = Join-Path -Path $destinationPath -ChildPath $blobName + # Get-AzStorageBlobContent -Container $containerName -Blob $blobName -Destination $destinationFile -Context $storageContext -Force + # Write-Output "Downloaded $blobName to $destinationFile" + # } + # + # $packagesPath = Get-ChildItem -Path $destinationPath\*.deb -Recurse -File | Select-Object -First 1 -ExpandProperty DirectoryName + # Write-Host "sending -- vso[task.setvariable variable=PackagesRoot]$packagesPath" + # Write-Host "##vso[task.setvariable variable=PackagesRoot]$packagesPath" + + - pwsh: | + Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName + displayName: Capture downloaded artifacts + + - pwsh: | + git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' + displayName: Clone Internal-Tools repository + + - pwsh: | + $Path = "$(System.ArtifactsDirectory)" + $OutputPath = Join-Path $Path 'hashes.sha256' + $srcPaths = @($Path) + $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File + $checksums = $packages | + ForEach-Object { + Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" + $packageName = $_.Name + $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() + # the '*' before the packagename signifies it is a binary + "$hash *$packageName" + } + $checksums | Out-File -FilePath $OutputPath -Force + $fileContent = Get-Content -Path $OutputPath -Raw | Out-String + Write-Verbose -Verbose -Message $fileContent + displayName: Add sha256 hashes + + - pwsh: | + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: 'Set release version' + + - pwsh: | + Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion + + $isPreview = $semanticVersion.PreReleaseLabel -ne $null + + $fileName = if ($isPreview) { + "preview.md" + } + else { + $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" + } + + $filePath = "$env:BUILD_SOURCESDIRECTORY/PowerShell/CHANGELOG/$fileName" + Write-Verbose -Verbose "Selected Log file: $filePath" + + if (-not (Test-Path $filePath)) { + throw "$filePath not found" + } + + $changelog = Get-Content -Path $filePath + + $startPattern = "^## \[" + ([regex]::Escape($releaseVersion)) + "\]" + $endPattern = "^## \[{0}\.{1}\.{2}*" -f $semanticVersion.Major, $semanticVersion.Minor, $semanticVersion.Patch + + $clContent = $changelog | ForEach-Object { + if ($_ -match $startPattern) { $outputLine = $true } + elseif ($_ -match $endPattern) { $outputLine = $false } + if ($outputLine) { $_} + } | Out-String + + Write-Verbose -Verbose "Selected content: `n$clContent" + + Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $(PackagesRoot) -Token $(GitHubReleasePat) + displayName: Publish Release Draft + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Push Git Tag + jobName: PushGitTag + dependsOnJob: GithubReleaseDraft + instructions: | + Push the git tag to upstream + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Make Draft Public + jobName: DraftPublic + dependsOnJob: PushGitTag + instructions: | + Make the GitHub Release Draft Public + \ No newline at end of file From 51604c0dac7759bbb4d436a3eb0dfc2572b7ccba Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 31 Oct 2024 17:41:50 -0500 Subject: [PATCH 025/316] Added path for SetReleaseTagAndContainerName yaml --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 19050481ca9..fce7f4d6616 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -102,7 +102,7 @@ stages: env: ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - - template: release-SetReleaseTagAndContainerName.yml + - template: /.pipelines/templates/release-SetReleaseTagAndContainerName.yml - pwsh: | Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose From 1ec1a7d488acd4ac424504262fde95fdb5c37a1a Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 31 Oct 2024 17:46:52 -0500 Subject: [PATCH 026/316] replaced template with actual steps --- .pipelines/templates/release-githubtasks.yml | 26 +++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index fce7f4d6616..eab6f2f7895 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -102,7 +102,31 @@ stages: env: ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - - template: /.pipelines/templates/release-SetReleaseTagAndContainerName.yml + - pwsh: | + $variable = 'releaseTag' + $branch = $ENV:BUILD_SOURCEBRANCH + if($branch -notmatch '^.*((release/|rebuild/.*rebuild))') + { + throw "Branch name is not in release format: '$branch'" + } + + $releaseTag = $Branch -replace '^.*((release|rebuild)/)' + $vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag" + Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose + Write-Host -Object "##$vstsCommandString" + displayName: Set Release Tag + + - pwsh: | + $azureVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '\.', '-' + $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + + $version = '$(ReleaseTag)'.ToLowerInvariant().Substring(1) + $vstsCommandString = "vso[task.setvariable variable=Version]$version" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: Set container name - pwsh: | Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose From 300dcb47cfcc5fcc8613c4434ef7fe8b7e55d4c0 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 1 Nov 2024 10:41:12 -0500 Subject: [PATCH 027/316] trying to add obreleaseenv to first stage --- .pipelines/PowerShell-Release-Official.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 6f649306b96..aca20ebb8ae 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -118,6 +118,8 @@ extends: stages: - stage: msixbundle displayName: 'Create MSIX Bundle' + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/release-create-msix.yml@self @@ -276,7 +278,6 @@ extends: parameters: SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} - - template: /.pipelines/templates/release-githubtasks.yml@self # dependsOn: BlobPublic From ad500f27ce65f9b1c7352831288a2ab46cd6e2fb Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 1 Nov 2024 10:52:29 -0500 Subject: [PATCH 028/316] untemplatized approval job in PublishGit --- .pipelines/templates/approvalJob.yml | 2 +- .pipelines/templates/release-githubtasks.yml | 56 +++++++++++++++----- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/.pipelines/templates/approvalJob.yml b/.pipelines/templates/approvalJob.yml index d4cda2ad353..ac3b8bc2ab2 100644 --- a/.pipelines/templates/approvalJob.yml +++ b/.pipelines/templates/approvalJob.yml @@ -25,7 +25,7 @@ jobs: dependsOn: ${{ parameters.dependsOnJob }} displayName: ${{ parameters.displayName }} pool: - type: server + type: agentless timeoutInMinutes: 4320 # job times out in 3 days steps: - task: ManualValidation@0 diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index eab6f2f7895..ecad2775f5b 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -242,19 +242,49 @@ stages: Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $(PackagesRoot) -Token $(GitHubReleasePat) displayName: Publish Release Draft - - template: /.pipelines/templates/approvalJob.yml@self - parameters: + - job: PushGitTag + dependsOn: GitHubReleaseDraft + displayName: Push Git Tag + pool: + type: server + timeoutInMinutes: 4320 # job times out in 3 days + steps: + - task: ManualValidation@0 displayName: Push Git Tag - jobName: PushGitTag - dependsOnJob: GithubReleaseDraft - instructions: | - Push the git tag to upstream - - - template: /.pipelines/templates/approvalJob.yml@self - parameters: + timeoutInMinutes: 2880 + inputs: + instructions: Push the git tag to upstream + onTimeout: reject + notifyUsers: '' + + - job: DraftPublic + dependsOn: PushGitTag + displayName: Make Draft Public + pool: + type: server + timeoutInMinutes: 4320 # job times out in 3 days + steps: + - task: ManualValidation@0 displayName: Make Draft Public - jobName: DraftPublic - dependsOnJob: PushGitTag - instructions: | - Make the GitHub Release Draft Public + timeoutInMinutes: 2880 + inputs: + instructions: Make the GitHub Release Draft Public + onTimeout: reject + notifyUsers: '' + + # - template: /.pipelines/templates/approvalJob.yml@self + # parameters: + # displayName: Push Git Tag + # jobName: PushGitTag + # dependsOnJob: GithubReleaseDraft + # instructions: | + # Push the git tag to upstream + + # - template: /.pipelines/templates/approvalJob.yml@self + # parameters: + # displayName: Make Draft Public + # jobName: DraftPublic + # dependsOnJob: PushGitTag + # instructions: | + # Make the GitHub Release Draft Public \ No newline at end of file From 1a87654048da6f55ee7302963fe1008aa55a08dc Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 1 Nov 2024 11:02:08 -0500 Subject: [PATCH 029/316] removed checkout step --- .pipelines/templates/release-githubtasks.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index ecad2775f5b..82f699fee10 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -97,10 +97,10 @@ stages: value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: - - checkout: self - clean: true - env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + # - checkout: self + # clean: true + # env: + # ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - pwsh: | $variable = 'releaseTag' From e0fb36cca594293a0d1dc2d0485ff77b9f62d4bc Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 1 Nov 2024 11:13:44 -0500 Subject: [PATCH 030/316] removed dependsOn for approval jobs --- .pipelines/templates/release-githubtasks.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 82f699fee10..c2b87ca6543 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -243,7 +243,6 @@ stages: displayName: Publish Release Draft - job: PushGitTag - dependsOn: GitHubReleaseDraft displayName: Push Git Tag pool: type: server @@ -258,7 +257,6 @@ stages: notifyUsers: '' - job: DraftPublic - dependsOn: PushGitTag displayName: Make Draft Public pool: type: server From b7b75f64cb47833b1664636ccf2c9155825bb4ac Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 1 Nov 2024 11:22:37 -0500 Subject: [PATCH 031/316] removed approval steps --- .pipelines/PowerShell-Release-Official.yml | 2 +- .pipelines/templates/release-githubtasks.yml | 54 ++++++++++---------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index aca20ebb8ae..a2ef0f414b0 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -279,7 +279,7 @@ extends: SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} - template: /.pipelines/templates/release-githubtasks.yml@self - # dependsOn: BlobPublic + # dependsOn: BlobPublic - stage: PublishNuGet displayName: Publish NuGet diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index c2b87ca6543..6321250dd2b 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -242,33 +242,33 @@ stages: Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $(PackagesRoot) -Token $(GitHubReleasePat) displayName: Publish Release Draft - - job: PushGitTag - displayName: Push Git Tag - pool: - type: server - timeoutInMinutes: 4320 # job times out in 3 days - steps: - - task: ManualValidation@0 - displayName: Push Git Tag - timeoutInMinutes: 2880 - inputs: - instructions: Push the git tag to upstream - onTimeout: reject - notifyUsers: '' - - - job: DraftPublic - displayName: Make Draft Public - pool: - type: server - timeoutInMinutes: 4320 # job times out in 3 days - steps: - - task: ManualValidation@0 - displayName: Make Draft Public - timeoutInMinutes: 2880 - inputs: - instructions: Make the GitHub Release Draft Public - onTimeout: reject - notifyUsers: '' + # - job: PushGitTag + # displayName: Push Git Tag + # pool: + # type: server + # timeoutInMinutes: 4320 # job times out in 3 days + # steps: + # - task: ManualValidation@0 + # displayName: Push Git Tag + # timeoutInMinutes: 2880 + # inputs: + # instructions: Push the git tag to upstream + # onTimeout: reject + # notifyUsers: '' +# + # - job: DraftPublic + # displayName: Make Draft Public + # pool: + # type: server + # timeoutInMinutes: 4320 # job times out in 3 days + # steps: + # - task: ManualValidation@0 + # displayName: Make Draft Public + # timeoutInMinutes: 2880 + # inputs: + # instructions: Make the GitHub Release Draft Public + # onTimeout: reject + # notifyUsers: '' # - template: /.pipelines/templates/approvalJob.yml@self # parameters: From 36d486246914feb6fb714b886e703a8cc286432a Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 1 Nov 2024 11:31:30 -0500 Subject: [PATCH 032/316] removed dependencies on githubrelease temporarily --- .pipelines/PowerShell-Release-Official.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index a2ef0f414b0..c2380cd0320 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -283,7 +283,7 @@ extends: - stage: PublishNuGet displayName: Publish NuGet - dependsOn: PublishGitHubRelease + # dependsOn: PublishGitHubRelease jobs: - template: /.pipelines/templates/release-publish-nuget.yml@self parameters: @@ -291,14 +291,14 @@ extends: - stage: PublishPMC displayName: Publish PMC - dependsOn: PublishGitHubRelease + # dependsOn: PublishGitHubRelease jobs: - template: /.pipelines/templates/release-publish-pmc.yml@self parameters: skipPublish: ${{ parameters.SkipPMCPublish }} - stage: ReleaseDocker - dependsOn: PublishGitHubRelease + # dependsOn: PublishGitHubRelease displayName: 'Docker Release' jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -317,7 +317,7 @@ extends: Kickoff docker release - stage: UpdateDotnetDocker - dependsOn: PublishGitHubRelease + # dependsOn: PublishGitHubRelease displayName: Update DotNet SDK Docker images jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -332,7 +332,7 @@ extends: 4. create PR targeting nightly branch - stage: UpdateWinGet - dependsOn: PublishGitHubRelease + # dependsOn: PublishGitHubRelease displayName: Add manifest entry to winget jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -343,7 +343,7 @@ extends: This is typically done by the community 1-2 days after the release. - stage: PublishMsix - dependsOn: PublishGitHubRelease + # dependsOn: PublishGitHubRelease displayName: Publish MSIX to store jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -354,7 +354,7 @@ extends: Ask Steve to release MSIX bundle package to Store - stage: PublishVPack - dependsOn: PublishGitHubRelease + # dependsOn: PublishGitHubRelease displayName: Release vPack jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -372,13 +372,13 @@ extends: # - template: templates/release-UpdateDepsJson.yml - stage: UploadBuildInfoJson - dependsOn: PublishGitHubRelease + # dependsOn: PublishGitHubRelease displayName: Upload BuildInfo.json jobs: - template: /.pipelines/templates/release-upload-buildinfo.yml@self - stage: ReleaseSymbols - dependsOn: PublishGitHubRelease + # dependsOn: PublishGitHubRelease displayName: Release Symbols jobs: - template: /.pipelines/templates/release-symbols.yml@self From ebbcfeeee3ef91c627dce27b0fedb2242a3743c1 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 1 Nov 2024 11:44:16 -0500 Subject: [PATCH 033/316] removed wrong package --- .pipelines/templates/release-githubtasks.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 6321250dd2b..8a15a7b862f 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -51,9 +51,6 @@ stages: - input: pipelineArtifact pipeline: PSPackagesOfficial artifactName: drop_mac_package_sign_package_macOS_x64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_mac_package_sign_package_windows_x64 - input: pipelineArtifact pipeline: PSPackagesOfficial artifactName: drop_windows_package_package_win_arm64 From 77eecf916eb3ea0e94df866ac678f84adb45f301 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 1 Nov 2024 11:49:24 -0500 Subject: [PATCH 034/316] fixed wrong package --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 8a15a7b862f..6ded9822fad 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -56,7 +56,7 @@ stages: artifactName: drop_windows_package_package_win_arm64 - input: pipelineArtifact pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_fxdendent + artifactName: drop_windows_package_package_win_fxdependent - input: pipelineArtifact pipeline: PSPackagesOfficial artifactName: drop_windows_package_package_win_fxdependentWinDesktop From d1baa92c407cf8f1480fde9a96cb82a99b6594a1 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 1 Nov 2024 11:58:49 -0500 Subject: [PATCH 035/316] changed pwsh to PowerShell@2 --- .pipelines/templates/release-githubtasks.yml | 192 +++++++++++-------- 1 file changed, 108 insertions(+), 84 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 6ded9822fad..870862105fc 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -99,34 +99,43 @@ stages: # env: # ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - - pwsh: | - $variable = 'releaseTag' - $branch = $ENV:BUILD_SOURCEBRANCH - if($branch -notmatch '^.*((release/|rebuild/.*rebuild))') - { - throw "Branch name is not in release format: '$branch'" - } - - $releaseTag = $Branch -replace '^.*((release|rebuild)/)' - $vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag" - Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose - Write-Host -Object "##$vstsCommandString" + - task: PowerShell@2 + inputs: + targetType: inline + script: | + $variable = 'releaseTag' + $branch = $ENV:BUILD_SOURCEBRANCH + if($branch -notmatch '^.*((release/|rebuild/.*rebuild))') + { + throw "Branch name is not in release format: '$branch'" + } + + $releaseTag = $Branch -replace '^.*((release|rebuild)/)' + $vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag" + Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose + Write-Host -Object "##$vstsCommandString" displayName: Set Release Tag - - pwsh: | - $azureVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '\.', '-' - $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - $version = '$(ReleaseTag)'.ToLowerInvariant().Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" + - task: PowerShell@2 + inputs: + targetType: inline + script: | + $azureVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '\.', '-' + $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + + $version = '$(ReleaseTag)'.ToLowerInvariant().Substring(1) + $vstsCommandString = "vso[task.setvariable variable=Version]$version" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" displayName: Set container name - - pwsh: | - Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + - task: PowerShell@2 + inputs: + targetType: inline + script: | + Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' # - pwsh: | @@ -169,74 +178,89 @@ stages: # Write-Host "sending -- vso[task.setvariable variable=PackagesRoot]$packagesPath" # Write-Host "##vso[task.setvariable variable=PackagesRoot]$packagesPath" - - pwsh: | - Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName + - task: PowerShell@2 + inputs: + targetType: inline + script: | + Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName displayName: Capture downloaded artifacts - - pwsh: | - git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' + - task: PowerShell@2 + inputs: + targetType: inline + script: | + git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' displayName: Clone Internal-Tools repository - - pwsh: | - $Path = "$(System.ArtifactsDirectory)" - $OutputPath = Join-Path $Path 'hashes.sha256' - $srcPaths = @($Path) - $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File - $checksums = $packages | - ForEach-Object { - Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" - $packageName = $_.Name - $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() - # the '*' before the packagename signifies it is a binary - "$hash *$packageName" - } - $checksums | Out-File -FilePath $OutputPath -Force - $fileContent = Get-Content -Path $OutputPath -Raw | Out-String - Write-Verbose -Verbose -Message $fileContent + - task: PowerShell@2 + inputs: + targetType: inline + script: | + $Path = "$(System.ArtifactsDirectory)" + $OutputPath = Join-Path $Path 'hashes.sha256' + $srcPaths = @($Path) + $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File + $checksums = $packages | + ForEach-Object { + Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" + $packageName = $_.Name + $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() + # the '*' before the packagename signifies it is a binary + "$hash *$packageName" + } + $checksums | Out-File -FilePath $OutputPath -Force + $fileContent = Get-Content -Path $OutputPath -Raw | Out-String + Write-Verbose -Verbose -Message $fileContent displayName: Add sha256 hashes - - pwsh: | - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" + - task: PowerShell@2 + inputs: + targetType: inline + script: | + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" displayName: 'Set release version' - - pwsh: | - Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion - - $isPreview = $semanticVersion.PreReleaseLabel -ne $null - - $fileName = if ($isPreview) { - "preview.md" - } - else { - $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" - } - - $filePath = "$env:BUILD_SOURCESDIRECTORY/PowerShell/CHANGELOG/$fileName" - Write-Verbose -Verbose "Selected Log file: $filePath" - - if (-not (Test-Path $filePath)) { - throw "$filePath not found" - } - - $changelog = Get-Content -Path $filePath - - $startPattern = "^## \[" + ([regex]::Escape($releaseVersion)) + "\]" - $endPattern = "^## \[{0}\.{1}\.{2}*" -f $semanticVersion.Major, $semanticVersion.Minor, $semanticVersion.Patch - - $clContent = $changelog | ForEach-Object { - if ($_ -match $startPattern) { $outputLine = $true } - elseif ($_ -match $endPattern) { $outputLine = $false } - if ($outputLine) { $_} - } | Out-String - - Write-Verbose -Verbose "Selected content: `n$clContent" - - Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $(PackagesRoot) -Token $(GitHubReleasePat) + - task: PowerShell@2 + inputs: + targetType: inline + script: | + Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion + + $isPreview = $semanticVersion.PreReleaseLabel -ne $null + + $fileName = if ($isPreview) { + "preview.md" + } + else { + $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" + } + + $filePath = "$env:BUILD_SOURCESDIRECTORY/PowerShell/CHANGELOG/$fileName" + Write-Verbose -Verbose "Selected Log file: $filePath" + + if (-not (Test-Path $filePath)) { + throw "$filePath not found" + } + + $changelog = Get-Content -Path $filePath + + $startPattern = "^## \[" + ([regex]::Escape($releaseVersion)) + "\]" + $endPattern = "^## \[{0}\.{1}\.{2}*" -f $semanticVersion.Major, $semanticVersion.Minor, $semanticVersion.Patch + + $clContent = $changelog | ForEach-Object { + if ($_ -match $startPattern) { $outputLine = $true } + elseif ($_ -match $endPattern) { $outputLine = $false } + if ($outputLine) { $_} + } | Out-String + + Write-Verbose -Verbose "Selected content: `n$clContent" + + Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $(PackagesRoot) -Token $(GitHubReleasePat) displayName: Publish Release Draft # - job: PushGitTag From ea577d6df8577ce34ced9a904d121f7e74b5fc42 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 1 Nov 2024 12:05:38 -0500 Subject: [PATCH 036/316] trying pwsh true --- .pipelines/templates/release-githubtasks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 870862105fc..ab993f6a9df 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -225,6 +225,7 @@ stages: - task: PowerShell@2 inputs: + pwsh: true targetType: inline script: | Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' From 67dafb1eaf82fdc13e245bef33a453bf5a7ca047 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 1 Nov 2024 12:11:09 -0500 Subject: [PATCH 037/316] switched system to linux and install pwsh --- .pipelines/templates/release-githubtasks.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index ab993f6a9df..7a9205465e0 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -9,7 +9,7 @@ stages: condition: succeeded() pool: type: release - os: windows + os: linux templateContext: inputs: - input: pipelineArtifact @@ -98,7 +98,12 @@ stages: # clean: true # env: # ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - + - task: Bash@3 + inputs: + targetType: inline + script: | + sudo tdnf install -y powershell + displayName: Install PowerShell - task: PowerShell@2 inputs: targetType: inline From 2269409dc6e6efffb9fd98e7ccff3267d98b2cf0 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 5 Nov 2024 15:37:06 -0600 Subject: [PATCH 038/316] removing linux going back to windows to test agent pool --- .pipelines/templates/release-githubtasks.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 7a9205465e0..fb86f09feed 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -9,7 +9,7 @@ stages: condition: succeeded() pool: type: release - os: linux + os: windows templateContext: inputs: - input: pipelineArtifact @@ -98,12 +98,13 @@ stages: # clean: true # env: # ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - - task: Bash@3 - inputs: - targetType: inline - script: | - sudo tdnf install -y powershell - displayName: Install PowerShell + # - task: Bash@3 + # inputs: + # targetType: inline + # script: | + # sudo tdnf install -y powershell + # displayName: Install PowerShell + - task: PowerShell@2 inputs: targetType: inline From 41248ac6cbe5a7ba300c146643060b2c8a3ef2da Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 5 Nov 2024 15:47:15 -0600 Subject: [PATCH 039/316] removed pwsh --- .pipelines/templates/release-githubtasks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index fb86f09feed..b7a02ed110d 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -231,7 +231,6 @@ stages: - task: PowerShell@2 inputs: - pwsh: true targetType: inline script: | Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' From b9f175779f9e7f943c002b760659c6dd054221e6 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 11:26:22 -0600 Subject: [PATCH 040/316] added semanticVersion class --- .pipelines/templates/release-githubtasks.yml | 32 +++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index b7a02ed110d..4da9d5bfd77 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -235,8 +235,38 @@ stages: script: | Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion + + class SemanticVersion { + [int]$Major + [int]$Minor + [int]$Patch + [string]$PreReleaseLabel + + SemanticVersion([string]$releaseVersion) { + $versionParts = $releaseVersion -split '[.-]' + for($i = 0; $i -lt $versionParts.Count; $i++) { + if ($i -eq 0) { + $this.Major = [int]$versionParts[$i] + } elseif ($i -eq 1) { + $this.Minor = [int]$versionParts[$i] + } elseif ($i -eq 2) { + $this.Patch = [int]$versionParts[$i] + } elseif ($i -eq 3) { + $this.PreReleaseLabel = $versionParts[$i..($versionParts.Count - 1)] -join '.' + } + } + } + [string] ToString() { + $version = "$($this.Major).$($this.Minor).$($this.Patch)" + if ($this.PreReleaseLabel) { + $version += "-$($this.PreReleaseLabel)" + } + return $version + } + } + + $semanticVersion = [SemanticVersion]::new($releaseVersion) $isPreview = $semanticVersion.PreReleaseLabel -ne $null $fileName = if ($isPreview) { From 5f7add3bf47d57ca8d6339f6a791399376ee7937 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 11:45:25 -0600 Subject: [PATCH 041/316] trying to added checkout step --- .pipelines/templates/release-githubtasks.yml | 22 +++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 4da9d5bfd77..443fcbf052b 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -104,6 +104,14 @@ stages: # script: | # sudo tdnf install -y powershell # displayName: Install PowerShell + - task: PowerShell@2 + inputs: + targetType: inline + script: | + git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShell/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' + cd $(Pipeline.Workspace)/PowerShell + git checkout $ENV:BUILD_SOURCEBRANCH + displayName: checkout - task: PowerShell@2 inputs: @@ -184,13 +192,6 @@ stages: # Write-Host "sending -- vso[task.setvariable variable=PackagesRoot]$packagesPath" # Write-Host "##vso[task.setvariable variable=PackagesRoot]$packagesPath" - - task: PowerShell@2 - inputs: - targetType: inline - script: | - Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName - displayName: Capture downloaded artifacts - - task: PowerShell@2 inputs: targetType: inline @@ -228,6 +229,13 @@ stages: Write-Host "sending " + $vstsCommandString Write-Host "##$vstsCommandString" displayName: 'Set release version' + + - task: PowerShell@2 + inputs: + targetType: inline + script: | + Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName + displayName: List all files in the workspace - task: PowerShell@2 inputs: From 6dbeecde56f5b1b8ccf61152d707692e57eb324a Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 11:54:45 -0600 Subject: [PATCH 042/316] fixed repo url typo --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 443fcbf052b..8a8964d7575 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -108,7 +108,7 @@ stages: inputs: targetType: inline script: | - git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShell/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' + git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' cd $(Pipeline.Workspace)/PowerShell git checkout $ENV:BUILD_SOURCEBRANCH displayName: checkout From f90d308833b72fe0de1fface650c622bde05daa4 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 12:22:23 -0600 Subject: [PATCH 043/316] added checkout logic to isolate branch name --- .pipelines/templates/release-githubtasks.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 8a8964d7575..98df4bd1e10 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -110,8 +110,10 @@ stages: script: | git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' cd $(Pipeline.Workspace)/PowerShell - git checkout $ENV:BUILD_SOURCEBRANCH - displayName: checkout + $branchParts = $ENV:BUILD_SOURCEBRANCH -split '/' + $branchName = "$($branchParts[-2])/$($branchParts[-1])" + git checkout $branchName + displayName: Checkout repository - task: PowerShell@2 inputs: From c96b45e4373f52fa6df14d5c5dff78df3ffcd1e8 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 12:36:37 -0600 Subject: [PATCH 044/316] file path change to pipelineworkspace --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 98df4bd1e10..57dec60e349 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -286,7 +286,7 @@ stages: $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" } - $filePath = "$env:BUILD_SOURCESDIRECTORY/PowerShell/CHANGELOG/$fileName" + $filePath = "$(Pipeline.Workspace)/PowerShell/CHANGELOG/$fileName" Write-Verbose -Verbose "Selected Log file: $filePath" if (-not (Test-Path $filePath)) { From 458906715bd4e6af0a14e1720875d3a9fd07e32e Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 13:14:05 -0600 Subject: [PATCH 045/316] moving files to staging directory --- .pipelines/templates/release-githubtasks.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 57dec60e349..3a49be1dccb 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -104,6 +104,7 @@ stages: # script: | # sudo tdnf install -y powershell # displayName: Install PowerShell + - task: PowerShell@2 inputs: targetType: inline @@ -245,7 +246,21 @@ stages: script: | Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' $releaseVersion = '$(ReleaseTag)' -replace '^v','' + Write-Verbose -Verbose "Release Version: $releaseVersion" + + Write-Verbose -Verbose "Moving packages to staging directory" + $packagesRoot = "$(Pipeline.Workspace)/PSPackagesOfficial" + New-Item -Path $packagesRoot -ItemType Directory -Force > $null + Get-ChildItem -Path $(Pipeline.Workspace) | + Where-Object { $_.Name -math "powershell-$(releaseVersion)-*" } + ForEach-Object { + Move-Item -Path $_.FullName -Destination $packagesRoot + } + + Write-Verbose -Verbose "List all packages in the staging directory" + Get-ChildItem -Path $packagesRoot -Recurse | Select-Object -ExpandProperty FullName + Write-Verbose -Verbose "Added Class SemanticVersion because pwsh 7+ is not available" class SemanticVersion { [int]$Major [int]$Minor @@ -306,7 +321,7 @@ stages: Write-Verbose -Verbose "Selected content: `n$clContent" - Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $(PackagesRoot) -Token $(GitHubReleasePat) + Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $packagesRoot -Token $(GitHubReleasePat) displayName: Publish Release Draft # - job: PushGitTag From 084c99b64318296722089a40fa5811c65797f112 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 13:20:56 -0600 Subject: [PATCH 046/316] fixed typos with file move --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 3a49be1dccb..03a764c3ec2 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -252,7 +252,7 @@ stages: $packagesRoot = "$(Pipeline.Workspace)/PSPackagesOfficial" New-Item -Path $packagesRoot -ItemType Directory -Force > $null Get-ChildItem -Path $(Pipeline.Workspace) | - Where-Object { $_.Name -math "powershell-$(releaseVersion)-*" } + Where-Object { $_.Name -match "powershell-$(releaseVersion)-*" } | ForEach-Object { Move-Item -Path $_.FullName -Destination $packagesRoot } From 9ecf09f82c66565072af72380308a567d3dc48bc Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 13:32:23 -0600 Subject: [PATCH 047/316] adding test to description, need to undo --- .pipelines/templates/release-githubtasks.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 03a764c3ec2..ddd66374b57 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -256,6 +256,7 @@ stages: ForEach-Object { Move-Item -Path $_.FullName -Destination $packagesRoot } + Move-Item -Path $(Pipeline.Workspace)/hashes.sha256 -Destination $packagesRoot Write-Verbose -Verbose "List all packages in the staging directory" Get-ChildItem -Path $packagesRoot -Recurse | Select-Object -ExpandProperty FullName @@ -320,6 +321,8 @@ stages: } | Out-String Write-Verbose -Verbose "Selected content: `n$clContent" + + $clContent = "Testing... DeployBox... `n`n" + $clContent Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $packagesRoot -Token $(GitHubReleasePat) displayName: Publish Release Draft From 117a5c854179ead75e3ca1cbb59674bcf4c23071 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 14:46:25 -0600 Subject: [PATCH 048/316] moved move task to before th sha so sha can be gneerated in the folder --- .pipelines/templates/release-githubtasks.yml | 36 ++++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index ddd66374b57..95dad24ffa0 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -201,12 +201,31 @@ stages: script: | git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' displayName: Clone Internal-Tools repository - + - task: PowerShell@2 inputs: targetType: inline script: | - $Path = "$(System.ArtifactsDirectory)" + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + Write-Verbose -Verbose "Moving packages to staging directory" + $packagesRoot = "$(Pipeline.Workspace)/PSPackagesOfficial" + New-Item -Path $packagesRoot -ItemType Directory -Force > $null + + Get-ChildItem -Path $(Pipeline.Workspace) | + Where-Object { $_.Name -match "powershell-$(releaseVersion)-*" } | + ForEach-Object { + Move-Item -Path $_.FullName -Destination $packagesRoot + } + + Write-Verbose -Verbose "List all packages in the staging directory" + Get-ChildItem -Path $packagesRoot -Recurse | Select-Object -ExpandProperty FullName + displayName: Move packages to staging directory + + - task: PowerShell@2 + inputs: + targetType: inline + script: | + $Path = "$(Pipeline.Workspace)/PSPackagesOfficial" $OutputPath = Join-Path $Path 'hashes.sha256' $srcPaths = @($Path) $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File @@ -247,19 +266,6 @@ stages: Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' $releaseVersion = '$(ReleaseTag)' -replace '^v','' Write-Verbose -Verbose "Release Version: $releaseVersion" - - Write-Verbose -Verbose "Moving packages to staging directory" - $packagesRoot = "$(Pipeline.Workspace)/PSPackagesOfficial" - New-Item -Path $packagesRoot -ItemType Directory -Force > $null - Get-ChildItem -Path $(Pipeline.Workspace) | - Where-Object { $_.Name -match "powershell-$(releaseVersion)-*" } | - ForEach-Object { - Move-Item -Path $_.FullName -Destination $packagesRoot - } - Move-Item -Path $(Pipeline.Workspace)/hashes.sha256 -Destination $packagesRoot - - Write-Verbose -Verbose "List all packages in the staging directory" - Get-ChildItem -Path $packagesRoot -Recurse | Select-Object -ExpandProperty FullName Write-Verbose -Verbose "Added Class SemanticVersion because pwsh 7+ is not available" class SemanticVersion { From 7f4bab795e7e4ecd9ba8bd23a2460d16d08a64fe Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 14:54:41 -0600 Subject: [PATCH 049/316] $(releaseVersion) -> $releaseVersion --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 95dad24ffa0..71463024933 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -212,7 +212,7 @@ stages: New-Item -Path $packagesRoot -ItemType Directory -Force > $null Get-ChildItem -Path $(Pipeline.Workspace) | - Where-Object { $_.Name -match "powershell-$(releaseVersion)-*" } | + Where-Object { $_.Name -match "powershell-$releaseVersion-*" } | ForEach-Object { Move-Item -Path $_.FullName -Destination $packagesRoot } From 8a856e9359e76fa9103ae06440b0dfe1ae990075 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 15:22:30 -0600 Subject: [PATCH 050/316] added rest of packages that didn't follow pattern --- .pipelines/templates/release-githubtasks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 71463024933..32cd9b96223 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -212,7 +212,7 @@ stages: New-Item -Path $packagesRoot -ItemType Directory -Force > $null Get-ChildItem -Path $(Pipeline.Workspace) | - Where-Object { $_.Name -match "powershell-$releaseVersion-*" } | + Where-Object { $_.Name -match "powershell-*" -or $_.Name -match "powershell_*" } | ForEach-Object { Move-Item -Path $_.FullName -Destination $packagesRoot } @@ -330,7 +330,7 @@ stages: $clContent = "Testing... DeployBox... `n`n" + $clContent - Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $packagesRoot -Token $(GitHubReleasePat) + Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder "$(Pipeline.Workspace)/PSPackagesOfficial" -Token $(GitHubReleasePat) displayName: Publish Release Draft # - job: PushGitTag From c6934bf673974b8f13720bb864a3e38bd171eae2 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 15:38:33 -0600 Subject: [PATCH 051/316] dont move the powershell repo --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 32cd9b96223..07021666b3f 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -212,7 +212,7 @@ stages: New-Item -Path $packagesRoot -ItemType Directory -Force > $null Get-ChildItem -Path $(Pipeline.Workspace) | - Where-Object { $_.Name -match "powershell-*" -or $_.Name -match "powershell_*" } | + Where-Object { $_.Name -match "powershell-*" -or $_.Name -match "powershell_*" -and -not $_.PSIsContainer } | ForEach-Object { Move-Item -Path $_.FullName -Destination $packagesRoot } From 03193cd1c2a76f3e6d5a26be834845a9e25c8f8a Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 16:57:42 -0600 Subject: [PATCH 052/316] Rewrote GithubRelease.psm1 in the pipeline --- .pipelines/templates/release-githubtasks.yml | 245 +++++++++++++++++++ 1 file changed, 245 insertions(+) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 07021666b3f..0b37700b19d 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -329,6 +329,251 @@ stages: Write-Verbose -Verbose "Selected content: `n$clContent" $clContent = "Testing... DeployBox... `n`n" + $clContent + + function Find-Release { + [CmdletBinding(DefaultParameterSetName="Default")] + param( + [Parameter(Mandatory)] + [string]$Tag + ) + + DynamicParam { + Get-CommonParameters + } + + Begin { + $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet + } + + End { + $maxRetries = 10 + $retryIntervalSec = 5 + $attempt = 0 + $success = $false + while (-not $success -and $attempt -lt $maxRetries) { + try { + $Result = Invoke-RestMethod -Headers $AuthHeader -Uri "https://api.github.com/repos/$User/$Repository/releases" + $success = $true + } catch { + $attempt++ + if ($attempt -lt $maxRetries) { + Start-Sleep -Seconds $retryIntervalSec + } else { + throw "Failed to retrieve releases after $maxRetries attempts." + } + } + } + + $Releases = $Result | ConvertFrom-Json + foreach ($Item in $Releases) { + if ($Item.tag_name -eq $Tag) { + Write-Verbose ("Release $Tag is found, upload_url=" + $Item.upload_url) -Verbose + return $Item.id + } + } + } + } + + function New-Release { + [CmdletBinding(DefaultParameterSetName="Default")] + param( + [Parameter(Mandatory)] + [string]$Tag, + + [Parameter(Mandatory)] + [string]$Name, + + [Parameter(Mandatory)] + [string]$Description + ) + + DynamicParam { + Get-CommonParameters + } + + Begin { + $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet + } + + End { + $maxRetries = 10 + $retryIntervalSec = 5 + $attempt = 0 + $success = $false + + $Body = @{ + tag_name = $Tag + name = $Name + body = $Description + draft = $true + prerelease = $false + } + + $BodyInJson = $Body | ConvertTo-Json + + while (-not $success -and $attempt -lt $maxRetries) { + try { + $Result = Invoke-RestMethod -Headers $AuthHeader -Method Post -Body $BodyInJson -Uri "https://api.github.com/repos/$User/$Repository/releases" + $success = $true + } catch { + $attempt++ + if ($attempt -lt $maxRetries) { + Start-Sleep -Seconds $retryIntervalSec + } else { + throw "Failed to create release after $maxRetries attempts." + } + } + } + + $Release = $Result | ConvertFrom-Json + return $Release.id + } + } + + function Find-ReleaseAsset { + [CmdletBinding(DefaultParameterSetName="Default")] + param( + [Parameter(Mandatory)] + [int]$ReleaseId, + + [Parameter(Mandatory)] + [string]$AssetName + ) + + DynamicParam { + Get-CommonParameters + } + + Begin { + $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet + } + + End { + $maxRetries = 10 + $retryIntervalSec = 5 + $attempt = 0 + $success = $false + + while (-not $success -and $attempt -lt $maxRetries) { + try { + $Result = Invoke-RestMethod -Headers $AuthHeader -Uri "https://api.github.com/repos/$User/$Repository/releases/$ReleaseId/assets" + $success = $true + } catch { + $attempt++ + if ($attempt -lt $maxRetries) { + Start-Sleep -Seconds $retryIntervalSec + } else { + throw "Failed to retrieve assets after $maxRetries attempts." + } + } + } + + $Assets = $Result | ConvertFrom-Json + foreach ($Item in $Assets) { + if ($Item.name -eq $AssetName) { + Write-Verbose "Asset $AssetName was already uploaded, id: $($Item.id)" -Verbose + Write-Verbose (" Asset is ready for download: " + $Item.browser_download_url) -Verbose + return $Item.id + } + } + } + } + + function Push-ReleaseAsset { + [CmdletBinding(DefaultParameterSetName="Default")] + param( + [Parameter(Mandatory)] + [int]$ReleaseId, + + [Parameter(Mandatory)] + [string]$AssetPath, + + [switch]$PassThru + ) + + DynamicParam { + Get-CommonParameters + } + + Begin { + $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet + } + + End { + $maxRetries = 10 + $retryIntervalSec = 5 + $attempt = 0 + $success = $false + + $type_7z = "application/octet-stream" + $type_exe = "application/x-msdownload" + + $AssetName = Split-Path $AssetPath -Leaf + + if ([System.IO.Path]::GetExtension($AssetName) -eq ".7z") { + $content_type = $type_7z + } else { + $content_type = $type_exe + } + + $Header = $AuthHeader + @{"Content-Type"=$content_type; "name"=$AssetName;} + $Uri = "https://uploads.github.com/repos/$User/$Repository/releases/$ReleaseId/assets?name=$AssetName" + + $Body = [System.IO.File]::ReadAllBytes($AssetPath) + + while (-not $success -and $attempt -lt $maxRetries) { + try { + $Result = Invoke-RestMethod -Headers $Header -Method Post -Body $Body -Uri $Uri + $success = $true + } catch { + $attempt++ + if ($attempt -lt $maxRetries) { + Start-Sleep -Seconds $retryIntervalSec + } else { + throw "Failed to upload asset after $maxRetries attempts." + } + } + } + + if ($Result.StatusCode -eq 201) { + $Content = $Result | ConvertFrom-Json + Write-Host "'$AssetName' -- Upload succeeded" -ForegroundColor Green + Write-Verbose -Message " Download URL: $($Content.browser_download_url)" + } else { + Write-Host "'$AssetName' -- Upload failed with StatusCode $($Result.StatusCode)" -ForegroundColor Red + } + + ## Pass through the HTTP result if '-PassThru' is specified + if ($PassThru) { return $Result } + } + } + + function Get-CommonParameters { + $ParamDict = [System.Management.Automation.RuntimeDefinedParameterDictionary]::new() + + ## Add the '-User' parameter + ## [Parameter(Mandatory)] + ## [string]$User + $UserAtts = [System.Collections.ObjectModel.Collection[System.Attribute]]::new() + $UserAtts.Add([System.Management.Automation.Parameter]@{Mandatory = $true}) > $null + $ParamDict.Add("User", [System.Management.Automation.RuntimeDefinedParameter]::new("User", [string], $UserAtts)) > $null + + ## Add the '-Repository' parameter + ## [Parameter(Mandatory)] + ## [string]$Repository + $RepositoryAtts = [System.Collections.ObjectModel.Collection[System.Attribute]]::new() + $RepositoryAtts.Add([System.Management.Automation.Parameter]@{Mandatory = $true}) > $null + $ParamDict.Add("Repository", [System.Management.Automation.RuntimeDefinedParameter]::new("Repository", [string], $RepositoryAtts)) > $null + + ## Add the '-Token' parameter + ## [Parameter(Mandatory, ParameterSetName="UseSpecifiedToken")] + ## [string]$Token + $TokenAtts = [System.Collections.ObjectModel.Collection[System.Attribute]]::new() + $TokenAtts.Add([System.Management.Automation.Parameter]@{Mandatory = $true; ParameterSetName="UseSpecifiedToken"}) > $null + $ParamDict.Add("Token", [System.Management.Automation.RuntimeDefinedParameter]::new("Token", [string], $TokenAtts)) > $null + + return $ParamDict + } Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder "$(Pipeline.Workspace)/PSPackagesOfficial" -Token $(GitHubReleasePat) displayName: Publish Release Draft From 1e50939a35bb6541d1e2940ac49c3d05dabd4fce Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 18:43:07 -0600 Subject: [PATCH 053/316] removed module githubrelease.psm1 --- .pipelines/templates/release-githubtasks.yml | 57 +++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 0b37700b19d..0e4266889b8 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -263,10 +263,10 @@ stages: inputs: targetType: inline script: | - Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' + # Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' $releaseVersion = '$(ReleaseTag)' -replace '^v','' Write-Verbose -Verbose "Release Version: $releaseVersion" - + # $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion Write-Verbose -Verbose "Added Class SemanticVersion because pwsh 7+ is not available" class SemanticVersion { [int]$Major @@ -574,6 +574,59 @@ stages: return $ParamDict } + + function Publish-ReleaseDraft { + [CmdletBinding(DefaultParameterSetName="Default")] + param( + [Parameter(Mandatory)] + [string]$Tag, + + [Parameter(Mandatory)] + [string]$Name, + + [Parameter(Mandatory)] + [string]$Description, + + [Parameter(Mandatory)] + [string]$PackageFolder + ) + + DynamicParam { Get-CommonParameters } + + Begin { + $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet + $Script:Auth = $AuthHeader + $CommonParams = @{User=$User; Repository=$Repository} + } + + End { + $ReleaseId = Find-Release -Tag $Tag @CommonParams + if ($ReleaseId) { + Write-Host "Release for $Tag already created, Release-Id: $ReleaseId" -ForegroundColor Green + } else { + $StringBuilder = [System.Text.StringBuilder]::new($Description, $Description.Length + 2kb) + $StringBuilder.AppendLine().AppendLine() > $null + $StringBuilder.AppendLine("### SHA256 Hashes of the release artifacts").AppendLine() > $null + Get-ChildItem -Path $PackageFolder -File | ForEach-Object { + $PackageName = $_.Name + $SHA256 = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash + $StringBuilder.AppendLine("- $PackageName").AppendLine(" - $SHA256") > $null + } + + $Description = $StringBuilder.ToString() + $ReleaseId = New-Release -Tag $Tag -Name $Name -Description $Description @CommonParams + Write-Host "Release for $Tag created, Release-Id: $ReleaseId" -ForegroundColor Green + } + + Get-ChildItem -Path $PackageFolder -File | ForEach-Object { + $PackageName = $_.Name + $PackageId = Find-ReleaseAsset -ReleaseId $ReleaseId -AssetName $PackageName @CommonParams + if (-not $PackageId) { + Push-ReleaseAsset -ReleaseId $ReleaseId -AssetPath $_.FullName @CommonParams + } + } + } + } Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder "$(Pipeline.Workspace)/PSPackagesOfficial" -Token $(GitHubReleasePat) displayName: Publish Release Draft From 1e90de358d1c9fffc9207b814898aa1b3bae249c Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 18:51:09 -0600 Subject: [PATCH 054/316] winPS5.1 Parameter -> ParameterAttribute --- .pipelines/templates/release-githubtasks.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 0e4266889b8..aca196786e2 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -555,21 +555,21 @@ stages: ## [Parameter(Mandatory)] ## [string]$User $UserAtts = [System.Collections.ObjectModel.Collection[System.Attribute]]::new() - $UserAtts.Add([System.Management.Automation.Parameter]@{Mandatory = $true}) > $null + $UserAtts.Add([System.Management.Automation.ParameterAttribute]@{Mandatory = $true}) > $null $ParamDict.Add("User", [System.Management.Automation.RuntimeDefinedParameter]::new("User", [string], $UserAtts)) > $null ## Add the '-Repository' parameter ## [Parameter(Mandatory)] ## [string]$Repository $RepositoryAtts = [System.Collections.ObjectModel.Collection[System.Attribute]]::new() - $RepositoryAtts.Add([System.Management.Automation.Parameter]@{Mandatory = $true}) > $null + $RepositoryAtts.Add([System.Management.Automation.ParameterAttribute]@{Mandatory = $true}) > $null $ParamDict.Add("Repository", [System.Management.Automation.RuntimeDefinedParameter]::new("Repository", [string], $RepositoryAtts)) > $null ## Add the '-Token' parameter ## [Parameter(Mandatory, ParameterSetName="UseSpecifiedToken")] ## [string]$Token $TokenAtts = [System.Collections.ObjectModel.Collection[System.Attribute]]::new() - $TokenAtts.Add([System.Management.Automation.Parameter]@{Mandatory = $true; ParameterSetName="UseSpecifiedToken"}) > $null + $TokenAtts.Add([System.Management.Automation.ParameterAttribute]@{Mandatory = $true; ParameterSetName="UseSpecifiedToken"}) > $null $ParamDict.Add("Token", [System.Management.Automation.RuntimeDefinedParameter]::new("Token", [string], $TokenAtts)) > $null return $ParamDict From ef8702cae969ab5707ce70883c5622ca57867714 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 18:57:34 -0600 Subject: [PATCH 055/316] Get-CommonParamValues --- .pipelines/templates/release-githubtasks.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index aca196786e2..bab0db18784 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -547,6 +547,8 @@ stages: if ($PassThru) { return $Result } } } + + $Script:Auth = $null function Get-CommonParameters { $ParamDict = [System.Management.Automation.RuntimeDefinedParameterDictionary]::new() @@ -575,6 +577,21 @@ stages: return $ParamDict } + function Get-CommonParamValues([PSCmdlet]$Cmdlet) { + $AuthHeader = $Script:Auth + if ($Cmdlet.ParameterSetName -eq "UseSpecifiedToken") { + $Token = $Cmdlet.MyInvocation.BoundParameters["Token"] + $AuthHeader = @{"Authorization"="token $Token"} + } + if (-not $AuthHeader) { + throw "Authentication token is not specified." + } + + $User = $Cmdlet.MyInvocation.BoundParameters["User"] + $Repository = $Cmdlet.MyInvocation.BoundParameters["Repository"] + return $User, $Repository, $AuthHeader + } + function Publish-ReleaseDraft { [CmdletBinding(DefaultParameterSetName="Default")] param( From ccd715e2709929f184c6514ab28c0eec0722933c Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 19:05:31 -0600 Subject: [PATCH 056/316] added SMA to PSCmdlet --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index bab0db18784..3a5b0742e54 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -577,7 +577,7 @@ stages: return $ParamDict } - function Get-CommonParamValues([PSCmdlet]$Cmdlet) { + function Get-CommonParamValues([System.Management.Automation.PSCmdlet]$Cmdlet) { $AuthHeader = $Script:Auth if ($Cmdlet.ParameterSetName -eq "UseSpecifiedToken") { $Token = $Cmdlet.MyInvocation.BoundParameters["Token"] From 76b4eb18fab13845427faf5c66e2ce4a66daffcf Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 6 Nov 2024 19:26:01 -0600 Subject: [PATCH 057/316] Json error handling --- .pipelines/templates/release-githubtasks.yml | 22 +++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 3a5b0742e54..74de6649b9d 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -350,6 +350,8 @@ stages: $retryIntervalSec = 5 $attempt = 0 $success = $false + $Result = $null + while (-not $success -and $attempt -lt $maxRetries) { try { $Result = Invoke-RestMethod -Headers $AuthHeader -Uri "https://api.github.com/repos/$User/$Repository/releases" @@ -364,12 +366,22 @@ stages: } } - $Releases = $Result | ConvertFrom-Json - foreach ($Item in $Releases) { - if ($Item.tag_name -eq $Tag) { - Write-Verbose ("Release $Tag is found, upload_url=" + $Item.upload_url) -Verbose - return $Item.id + if ($Result -and $Result.Content) { + try { + $Releases = $Result.Content | ConvertFrom-Json + } catch { + Write-Verbose -Message "This was the Response: $Result" + throw "Failed to convert response to JSON: $_" } + + foreach ($Item in $Releases) { + if ($Item.tag_name -eq $Tag) { + Write-Verbose ("Release $Tag is found, upload_url=" + $Item.upload_url) -Verbose + return $Item.id + } + } + } else { + throw "Response content is empty or invalid." } } } From 2bd95e5f3fdd037e3c2c3ae82484faf15e0bfa8e Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 09:56:39 -0600 Subject: [PATCH 058/316] added verbose logging for json convert error --- .pipelines/templates/release-githubtasks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 74de6649b9d..cd5e07082ce 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -368,6 +368,7 @@ stages: if ($Result -and $Result.Content) { try { + Write-Verbose -Message "This is the Response Content: $($Result.Content)" $Releases = $Result.Content | ConvertFrom-Json } catch { Write-Verbose -Message "This was the Response: $Result" From 353bb4912e3c8084f341658c46895ca8b9369c67 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 10:10:39 -0600 Subject: [PATCH 059/316] Added error handling for other convert jsons --- .pipelines/templates/release-githubtasks.yml | 40 +++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index cd5e07082ce..cf5d5763b81 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -437,9 +437,20 @@ stages: } } } - - $Release = $Result | ConvertFrom-Json - return $Release.id + + if ($Result -and $Result.Content) { + try { + Write-Verbose -Message "This is the Response Content: $($Result.Content)" + $Release = $Result | ConvertFrom-Json + } catch { + Write-Verbose -Message "This was the Response: $Result" + throw "Failed to convert response to JSON: $_" + } + return $Release.id + } + } else { + throw "Response content is empty or invalid." + } } } @@ -480,14 +491,23 @@ stages: } } } - - $Assets = $Result | ConvertFrom-Json - foreach ($Item in $Assets) { - if ($Item.name -eq $AssetName) { - Write-Verbose "Asset $AssetName was already uploaded, id: $($Item.id)" -Verbose - Write-Verbose (" Asset is ready for download: " + $Item.browser_download_url) -Verbose - return $Item.id + + if ($Result -and $Result.Content) { + try { + Write-Verbose -Message "This is the Response Content: $($Result.Content)" + $Assets = $Result | ConvertFrom-Json + foreach ($Item in $Assets) { + if ($Item.name -eq $AssetName) { + Write-Verbose "Asset $AssetName was already uploaded, id: $($Item.id)" -Verbose + Write-Verbose (" Asset is ready for download: " + $Item.browser_download_url) -Verbose + return $Item.id + } + } catch { + Write-Verbose -Message "This was the Response: $Result" + throw "Failed to convert response to JSON: $_" } + } else { + throw "Response content is empty or invalid." } } } From 353530ec0e562fd2d8a40dda44c7953932732f14 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 10:18:15 -0600 Subject: [PATCH 060/316] mssing bracket --- .pipelines/templates/release-githubtasks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index cf5d5763b81..1fa71bac1ba 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -447,7 +447,6 @@ stages: throw "Failed to convert response to JSON: $_" } return $Release.id - } } else { throw "Response content is empty or invalid." } From 1fa4060128f4af61a4c9baf2d048de688a9d23f2 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 11:25:14 -0600 Subject: [PATCH 061/316] another bracket missing --- .pipelines/templates/release-githubtasks.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 1fa71bac1ba..a775d9b98ca 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -496,10 +496,11 @@ stages: Write-Verbose -Message "This is the Response Content: $($Result.Content)" $Assets = $Result | ConvertFrom-Json foreach ($Item in $Assets) { - if ($Item.name -eq $AssetName) { - Write-Verbose "Asset $AssetName was already uploaded, id: $($Item.id)" -Verbose - Write-Verbose (" Asset is ready for download: " + $Item.browser_download_url) -Verbose - return $Item.id + if ($Item.name -eq $AssetName) { + Write-Verbose "Asset $AssetName was already uploaded, id: $($Item.id)" -Verbose + Write-Verbose (" Asset is ready for download: " + $Item.browser_download_url) -Verbose + return $Item.id + } } } catch { Write-Verbose -Message "This was the Response: $Result" From 826ff62fe6e637f2308d7108796085f3af4da9cd Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 11:39:41 -0600 Subject: [PATCH 062/316] added one more verbose logging before convert json --- .pipelines/templates/release-githubtasks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index a775d9b98ca..a95eb2149fc 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -569,6 +569,7 @@ stages: } if ($Result.StatusCode -eq 201) { + Write-Verbose -Message "This is the Response Content: $($Result.Content)" $Content = $Result | ConvertFrom-Json Write-Host "'$AssetName' -- Upload succeeded" -ForegroundColor Green Write-Verbose -Message " Download URL: $($Content.browser_download_url)" From a083b040ab42b00a9104bd380a41b052373c906e Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 11:55:14 -0600 Subject: [PATCH 063/316] added logging for each function --- .pipelines/templates/release-githubtasks.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index a95eb2149fc..7aa14df9d4b 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -328,8 +328,6 @@ stages: Write-Verbose -Verbose "Selected content: `n$clContent" - $clContent = "Testing... DeployBox... `n`n" + $clContent - function Find-Release { [CmdletBinding(DefaultParameterSetName="Default")] param( @@ -342,6 +340,7 @@ stages: } Begin { + Write-Verbose -Message "Finding release for tag '$Tag'" $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet } @@ -405,6 +404,7 @@ stages: } Begin { + Write-Verbose -Message "Creating a new release draft for $Tag" $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet } @@ -468,6 +468,7 @@ stages: } Begin { + Write-Verbose -Message "Searching for asset '$AssetName' in release $ReleaseId" $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet } @@ -529,6 +530,7 @@ stages: } Begin { + Write-Verbose -Message "Pushing asset '$AssetPath' to release $ReleaseId" $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet } @@ -645,6 +647,7 @@ stages: DynamicParam { Get-CommonParameters } Begin { + Write-Verbose -Message "Publishing release draft for $Tag" $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet $Script:Auth = $AuthHeader $CommonParams = @{User=$User; Repository=$Repository} @@ -679,6 +682,8 @@ stages: } } + $clContent = "Testing... DeployBox... `n`n" + $clContent + Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder "$(Pipeline.Workspace)/PSPackagesOfficial" -Token $(GitHubReleasePat) displayName: Publish Release Draft From beb41afe20f9395f52d665ca50d926fb44fb166e Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 12:05:06 -0600 Subject: [PATCH 064/316] added -Verbose to all Write-Verbose --- .pipelines/templates/release-githubtasks.yml | 32 ++++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 7aa14df9d4b..4fdbbc9f41c 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -340,7 +340,7 @@ stages: } Begin { - Write-Verbose -Message "Finding release for tag '$Tag'" + Write-Verbose -Message "Finding release for tag '$Tag'" -Verbose $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet } @@ -367,11 +367,11 @@ stages: if ($Result -and $Result.Content) { try { - Write-Verbose -Message "This is the Response Content: $($Result.Content)" + Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose $Releases = $Result.Content | ConvertFrom-Json } catch { - Write-Verbose -Message "This was the Response: $Result" - throw "Failed to convert response to JSON: $_" + Write-Verbose -Message "This was the Response: $Result" -Verbose + throw "Failed to convert response from JSON: $_" } foreach ($Item in $Releases) { @@ -404,7 +404,7 @@ stages: } Begin { - Write-Verbose -Message "Creating a new release draft for $Tag" + Write-Verbose -Message "Creating a new release draft for $Tag" -Verbose $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet } @@ -440,11 +440,11 @@ stages: if ($Result -and $Result.Content) { try { - Write-Verbose -Message "This is the Response Content: $($Result.Content)" + Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose $Release = $Result | ConvertFrom-Json } catch { - Write-Verbose -Message "This was the Response: $Result" - throw "Failed to convert response to JSON: $_" + Write-Verbose -Message "This was the Response: $Result" -Verbose + throw "Failed to convert response from JSON: $_" } return $Release.id } else { @@ -468,7 +468,7 @@ stages: } Begin { - Write-Verbose -Message "Searching for asset '$AssetName' in release $ReleaseId" + Write-Verbose -Message "Searching for asset '$AssetName' in release $ReleaseId" -Verbose $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet } @@ -494,7 +494,7 @@ stages: if ($Result -and $Result.Content) { try { - Write-Verbose -Message "This is the Response Content: $($Result.Content)" + Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose $Assets = $Result | ConvertFrom-Json foreach ($Item in $Assets) { if ($Item.name -eq $AssetName) { @@ -504,8 +504,8 @@ stages: } } } catch { - Write-Verbose -Message "This was the Response: $Result" - throw "Failed to convert response to JSON: $_" + Write-Verbose -Message "This was the Response: $Result" -Verbose + throw "Failed to convert response from JSON: $_" } } else { throw "Response content is empty or invalid." @@ -530,7 +530,7 @@ stages: } Begin { - Write-Verbose -Message "Pushing asset '$AssetPath' to release $ReleaseId" + Write-Verbose -Message "Pushing asset '$AssetPath' to release $ReleaseId" -Verbose $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet } @@ -571,10 +571,10 @@ stages: } if ($Result.StatusCode -eq 201) { - Write-Verbose -Message "This is the Response Content: $($Result.Content)" + Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose $Content = $Result | ConvertFrom-Json Write-Host "'$AssetName' -- Upload succeeded" -ForegroundColor Green - Write-Verbose -Message " Download URL: $($Content.browser_download_url)" + Write-Verbose -Message " Download URL: $($Content.browser_download_url)" -Verbose } else { Write-Host "'$AssetName' -- Upload failed with StatusCode $($Result.StatusCode)" -ForegroundColor Red } @@ -647,7 +647,7 @@ stages: DynamicParam { Get-CommonParameters } Begin { - Write-Verbose -Message "Publishing release draft for $Tag" + Write-Verbose -Message "Publishing release draft for $Tag" -Verbose $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet $Script:Auth = $AuthHeader $CommonParams = @{User=$User; Repository=$Repository} From 54006f6a0113e07447a2470e2a5afd294d6c8113 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 12:28:51 -0600 Subject: [PATCH 065/316] Trying to see why invoke-restmethod is returning null with verbose messages --- .pipelines/templates/release-githubtasks.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 4fdbbc9f41c..76ca4ec7897 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -367,7 +367,7 @@ stages: if ($Result -and $Result.Content) { try { - Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose + Write-Verbose -Message "This is the Response Content: $Result.Content" -Verbose $Releases = $Result.Content | ConvertFrom-Json } catch { Write-Verbose -Message "This was the Response: $Result" -Verbose @@ -440,7 +440,7 @@ stages: if ($Result -and $Result.Content) { try { - Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose + Write-Verbose -Message "This is the Response Content: $Result.Content" -Verbose $Release = $Result | ConvertFrom-Json } catch { Write-Verbose -Message "This was the Response: $Result" -Verbose @@ -480,7 +480,9 @@ stages: while (-not $success -and $attempt -lt $maxRetries) { try { + Write-Verbose -Message "Using AuthHeader: $AuthHeader" -Verbose $Result = Invoke-RestMethod -Headers $AuthHeader -Uri "https://api.github.com/repos/$User/$Repository/releases/$ReleaseId/assets" + Write-Verbose -Message "Result from Invoke-RestMethod: $Result" -Verbose $success = $true } catch { $attempt++ @@ -494,7 +496,7 @@ stages: if ($Result -and $Result.Content) { try { - Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose + Write-Verbose -Message "This is the Response Content: $Result.Content" -Verbose $Assets = $Result | ConvertFrom-Json foreach ($Item in $Assets) { if ($Item.name -eq $AssetName) { @@ -571,7 +573,7 @@ stages: } if ($Result.StatusCode -eq 201) { - Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose + Write-Verbose -Message "This is the Response Content: $Result.Content" -Verbose $Content = $Result | ConvertFrom-Json Write-Host "'$AssetName' -- Upload succeeded" -ForegroundColor Green Write-Verbose -Message " Download URL: $($Content.browser_download_url)" -Verbose From e8378c58dd526d0a2423c9f3e235281878da3060 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 12:39:54 -0600 Subject: [PATCH 066/316] Added verbose messaging to find-release --- .pipelines/templates/release-githubtasks.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 76ca4ec7897..b34659d43be 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -353,7 +353,9 @@ stages: while (-not $success -and $attempt -lt $maxRetries) { try { + Write-Verbose -Message "Using AuthHeader: $AuthHeader" -Verbose $Result = Invoke-RestMethod -Headers $AuthHeader -Uri "https://api.github.com/repos/$User/$Repository/releases" + Write-Verbose -Message "Result from Invoke-RestMethod: $Result" -Verbose $success = $true } catch { $attempt++ @@ -365,9 +367,9 @@ stages: } } - if ($Result -and $Result.Content) { + if (-not [string]::IsNullOrWhiteSpace($Result.Content)) { try { - Write-Verbose -Message "This is the Response Content: $Result.Content" -Verbose + Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose $Releases = $Result.Content | ConvertFrom-Json } catch { Write-Verbose -Message "This was the Response: $Result" -Verbose @@ -438,9 +440,9 @@ stages: } } - if ($Result -and $Result.Content) { + if (-not [string]::IsNullOrWhiteSpace($Result.Content)) { try { - Write-Verbose -Message "This is the Response Content: $Result.Content" -Verbose + Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose $Release = $Result | ConvertFrom-Json } catch { Write-Verbose -Message "This was the Response: $Result" -Verbose @@ -494,9 +496,9 @@ stages: } } - if ($Result -and $Result.Content) { + if (-not [string]::IsNullOrWhiteSpace($Result.Content)) { try { - Write-Verbose -Message "This is the Response Content: $Result.Content" -Verbose + Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose $Assets = $Result | ConvertFrom-Json foreach ($Item in $Assets) { if ($Item.name -eq $AssetName) { @@ -573,7 +575,7 @@ stages: } if ($Result.StatusCode -eq 201) { - Write-Verbose -Message "This is the Response Content: $Result.Content" -Verbose + Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose $Content = $Result | ConvertFrom-Json Write-Host "'$AssetName' -- Upload succeeded" -ForegroundColor Green Write-Verbose -Message " Download URL: $($Content.browser_download_url)" -Verbose From 3e4be19ee42e345149d13450b329a8d02b093a2c Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 13:07:04 -0600 Subject: [PATCH 067/316] fixed authheader verbose message --- .pipelines/templates/release-githubtasks.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index b34659d43be..45a87112395 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -353,7 +353,8 @@ stages: while (-not $success -and $attempt -lt $maxRetries) { try { - Write-Verbose -Message "Using AuthHeader: $AuthHeader" -Verbose + $AuthString = $AuthHeader | ConvertTo-Json + Write-Verbose -Message "Using AuthHeader: $AuthString" -Verbose $Result = Invoke-RestMethod -Headers $AuthHeader -Uri "https://api.github.com/repos/$User/$Repository/releases" Write-Verbose -Message "Result from Invoke-RestMethod: $Result" -Verbose $success = $true @@ -482,9 +483,7 @@ stages: while (-not $success -and $attempt -lt $maxRetries) { try { - Write-Verbose -Message "Using AuthHeader: $AuthHeader" -Verbose $Result = Invoke-RestMethod -Headers $AuthHeader -Uri "https://api.github.com/repos/$User/$Repository/releases/$ReleaseId/assets" - Write-Verbose -Message "Result from Invoke-RestMethod: $Result" -Verbose $success = $true } catch { $attempt++ From 733e7906597c1ed4c06280f43a9e4790b5152a61 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 13:33:31 -0600 Subject: [PATCH 068/316] checking variable values --- .pipelines/templates/release-githubtasks.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 45a87112395..b295a283d8d 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -353,8 +353,7 @@ stages: while (-not $success -and $attempt -lt $maxRetries) { try { - $AuthString = $AuthHeader | ConvertTo-Json - Write-Verbose -Message "Using AuthHeader: $AuthString" -Verbose + Write-Verbose -Message "Retrieving releases for $User/$Repository" -Verbose $Result = Invoke-RestMethod -Headers $AuthHeader -Uri "https://api.github.com/repos/$User/$Repository/releases" Write-Verbose -Message "Result from Invoke-RestMethod: $Result" -Verbose $success = $true From c1d2a85eb6fc4ff95ec38c3f5d97928db6965b3a Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 14:06:21 -0600 Subject: [PATCH 069/316] removed Headers --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index b295a283d8d..a7b8b97ade5 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -354,7 +354,7 @@ stages: while (-not $success -and $attempt -lt $maxRetries) { try { Write-Verbose -Message "Retrieving releases for $User/$Repository" -Verbose - $Result = Invoke-RestMethod -Headers $AuthHeader -Uri "https://api.github.com/repos/$User/$Repository/releases" + $Result = Invoke-RestMethod -Uri "https://api.github.com/repos/$User/$Repository/releases" Write-Verbose -Message "Result from Invoke-RestMethod: $Result" -Verbose $success = $true } catch { From cffd528ae4a1611afc9ef6c7243722f60c6fcfcf Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 14:29:24 -0600 Subject: [PATCH 070/316] changed from Result.Content to Result --- .pipelines/templates/release-githubtasks.yml | 58 ++++---------------- 1 file changed, 12 insertions(+), 46 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index a7b8b97ade5..911ffd6be26 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -354,7 +354,7 @@ stages: while (-not $success -and $attempt -lt $maxRetries) { try { Write-Verbose -Message "Retrieving releases for $User/$Repository" -Verbose - $Result = Invoke-RestMethod -Uri "https://api.github.com/repos/$User/$Repository/releases" + $Result = Invoke-RestMethod -Uri "https://api.github.com/repos/$User/$Repository/releases" -Headers $AuthHeader Write-Verbose -Message "Result from Invoke-RestMethod: $Result" -Verbose $success = $true } catch { @@ -367,23 +367,11 @@ stages: } } - if (-not [string]::IsNullOrWhiteSpace($Result.Content)) { - try { - Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose - $Releases = $Result.Content | ConvertFrom-Json - } catch { - Write-Verbose -Message "This was the Response: $Result" -Verbose - throw "Failed to convert response from JSON: $_" - } - - foreach ($Item in $Releases) { - if ($Item.tag_name -eq $Tag) { - Write-Verbose ("Release $Tag is found, upload_url=" + $Item.upload_url) -Verbose - return $Item.id - } + foreach ($Item in $Result) { + if ($Item.tag_name -eq $Tag) { + Write-Verbose ("Release $Tag is found, upload_url=" + $Item.upload_url) -Verbose + return $Item.id } - } else { - throw "Response content is empty or invalid." } } } @@ -440,18 +428,7 @@ stages: } } - if (-not [string]::IsNullOrWhiteSpace($Result.Content)) { - try { - Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose - $Release = $Result | ConvertFrom-Json - } catch { - Write-Verbose -Message "This was the Response: $Result" -Verbose - throw "Failed to convert response from JSON: $_" - } - return $Release.id - } else { - throw "Response content is empty or invalid." - } + return $Result.id } } @@ -494,24 +471,13 @@ stages: } } - if (-not [string]::IsNullOrWhiteSpace($Result.Content)) { - try { - Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose - $Assets = $Result | ConvertFrom-Json - foreach ($Item in $Assets) { - if ($Item.name -eq $AssetName) { - Write-Verbose "Asset $AssetName was already uploaded, id: $($Item.id)" -Verbose - Write-Verbose (" Asset is ready for download: " + $Item.browser_download_url) -Verbose - return $Item.id - } - } - } catch { - Write-Verbose -Message "This was the Response: $Result" -Verbose - throw "Failed to convert response from JSON: $_" + foreach ($Item in $Result) { + if ($Item.name -eq $AssetName) { + Write-Verbose "Asset $AssetName was already uploaded, id: $($Item.id)" -Verbose + Write-Verbose (" Asset is ready for download: " + $Item.browser_download_url) -Verbose + return $Item.id } - } else { - throw "Response content is empty or invalid." - } + } } } From 26f944ee35b33482fb80101bb83e58a58cbca054 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 7 Nov 2024 15:31:46 -0600 Subject: [PATCH 071/316] added verbose message for push asset --- .pipelines/templates/release-githubtasks.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 911ffd6be26..9926e400517 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -537,12 +537,12 @@ stages: } } } - - if ($Result.StatusCode -eq 201) { - Write-Verbose -Message "This is the Response Content: $($Result.Content)" -Verbose - $Content = $Result | ConvertFrom-Json + + Write-Verbose -Message "$($Result.state)" -Verbose + + if ($Result.state -eq "uploaded") { Write-Host "'$AssetName' -- Upload succeeded" -ForegroundColor Green - Write-Verbose -Message " Download URL: $($Content.browser_download_url)" -Verbose + Write-Verbose -Message " Download URL: $($Result.browser_download_url)" -Verbose } else { Write-Host "'$AssetName' -- Upload failed with StatusCode $($Result.StatusCode)" -ForegroundColor Red } From 68b19c6ec209a5d2ee17e3e1a5ee0001ef56a1fa Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 8 Nov 2024 11:17:30 -0600 Subject: [PATCH 072/316] trying to add pwsh7 --- .pipelines/templates/release-githubtasks.yml | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 9926e400517..6f7d135809b 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -105,6 +105,31 @@ stages: # sudo tdnf install -y powershell # displayName: Install PowerShell + - task: PowerShell@2 + inputs: + targetType: inline + script: | + $pwshRegPath = "HKLM:\SOFTWARE\Microsoft\PowerShellCore" + $previewPath = Join-Path -Path $pwshRegPath -ChildPath "InstalledVersions\31ab5147-9a97-4452-8443-d9709f0516e1" + if (!(Test-Path -Path $previewPath)) { + $null = New-Item -Path $previewPath -ItemType Directory -Force + } + Set-ItemProperty -Path $pwshRegPath -Name UseMU -Value 1 -Type DWord + Set-ItemProperty -Path $previewPath -Name Install -Value 1 -Type DWord + + $usoClientCommands = "USOClient StartInteractiveScan; USOClient ScanInstallWait; USOClient StartInstall" + + Start-Process powershell -ArgumentList "-NoProfile -ExecutionPolicy Bypass -Command `"$usoClientCommands`"" -Verb RunAs + displayName: Install pwsh 7 + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Write-Verbose -Message "PowerShell 7 installed" -Verbose + displayName: Check pwsh 7 installation + - task: PowerShell@2 inputs: targetType: inline From 822d9d16acabae1cfbcbd157f4f6045670d526ef Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 8 Nov 2024 11:52:18 -0600 Subject: [PATCH 073/316] added checks to see if pwsh will install --- .pipelines/templates/release-githubtasks.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 6f7d135809b..7bf786f90ee 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -111,15 +111,34 @@ stages: script: | $pwshRegPath = "HKLM:\SOFTWARE\Microsoft\PowerShellCore" $previewPath = Join-Path -Path $pwshRegPath -ChildPath "InstalledVersions\31ab5147-9a97-4452-8443-d9709f0516e1" + if (!(Test-Path -Path $previewPath)) { + Write-Verbose -Message "Creating preview path: $previewPath" -Verbose $null = New-Item -Path $previewPath -ItemType Directory -Force + } else { + Write-Verbose -Message "Preview path already exists: $previewPath" -Verbose } + Set-ItemProperty -Path $pwshRegPath -Name UseMU -Value 1 -Type DWord Set-ItemProperty -Path $previewPath -Name Install -Value 1 -Type DWord + $useMUValue = Get-ItemProperty -Path $pwshRegPath -Name UseMU + Write-Verbose -Message "UseMU is set to: $($useMUValue.UseMU)" -Verbose + $installValue = Get-ItemProperty -Path $previewPath -Name Install + Write-Verbose -Message "Install is set to: $($installValue.Install)" -Verbose + $usoClientCommands = "USOClient StartInteractiveScan; USOClient ScanInstallWait; USOClient StartInstall" Start-Process powershell -ArgumentList "-NoProfile -ExecutionPolicy Bypass -Command `"$usoClientCommands`"" -Verb RunAs + + # Verify installation + $previewPath = "C:\Program Files\PowerShell\7-preview\pwsh.exe" + while (-not (Test-Path -Path $previewPath)) { + Write-Verbose -Message "PowerShell Preview is still installing..." -Verbose + Start-Sleep -Seconds 10 + } + + Write-Verbose -Message "PowerShell Preview is installed." -Verbose displayName: Install pwsh 7 - task: PowerShell@2 From 822c25e295572b5cd9eea6ef821fff2013110826 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 8 Nov 2024 14:34:23 -0600 Subject: [PATCH 074/316] Upload download folder to artifacts --- .pipelines/templates/release-githubtasks.yml | 33 ++------------------ .pipelines/templates/uploadToAzure.yml | 5 +++ 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 7bf786f90ee..2cb9a5fcd2e 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -105,40 +105,13 @@ stages: # sudo tdnf install -y powershell # displayName: Install PowerShell + # Get-ChildItem -Directory -Path 'C:\Program Files\PowerShell\7*' | Select-Object -First 1 -ExpandProperty FullName + # x64 msi installer - task: PowerShell@2 inputs: targetType: inline script: | - $pwshRegPath = "HKLM:\SOFTWARE\Microsoft\PowerShellCore" - $previewPath = Join-Path -Path $pwshRegPath -ChildPath "InstalledVersions\31ab5147-9a97-4452-8443-d9709f0516e1" - - if (!(Test-Path -Path $previewPath)) { - Write-Verbose -Message "Creating preview path: $previewPath" -Verbose - $null = New-Item -Path $previewPath -ItemType Directory -Force - } else { - Write-Verbose -Message "Preview path already exists: $previewPath" -Verbose - } - - Set-ItemProperty -Path $pwshRegPath -Name UseMU -Value 1 -Type DWord - Set-ItemProperty -Path $previewPath -Name Install -Value 1 -Type DWord - - $useMUValue = Get-ItemProperty -Path $pwshRegPath -Name UseMU - Write-Verbose -Message "UseMU is set to: $($useMUValue.UseMU)" -Verbose - $installValue = Get-ItemProperty -Path $previewPath -Name Install - Write-Verbose -Message "Install is set to: $($installValue.Install)" -Verbose - - $usoClientCommands = "USOClient StartInteractiveScan; USOClient ScanInstallWait; USOClient StartInstall" - - Start-Process powershell -ArgumentList "-NoProfile -ExecutionPolicy Bypass -Command `"$usoClientCommands`"" -Verb RunAs - - # Verify installation - $previewPath = "C:\Program Files\PowerShell\7-preview\pwsh.exe" - while (-not (Test-Path -Path $previewPath)) { - Write-Verbose -Message "PowerShell Preview is still installing..." -Verbose - Start-Sleep -Seconds 10 - } - - Write-Verbose -Message "PowerShell Preview is installed." -Verbose + Start-Process -FilePath msiexec -ArgumentList "/package $localInstallerPath /quiet REGISTER_MANIFEST=1" -Wait -NoNewWindow displayName: Install pwsh 7 - task: PowerShell@2 diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index 174b5810de0..1fb96785c57 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -233,6 +233,11 @@ jobs: Get-ChildItem '$(Build.ArtifactStagingDirectory)/downloads' | Select-Object -ExpandProperty FullName displayName: 'Capture downloads' + - pwsh: | + Write-Verbose -Message "Copying all files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" + Copy-Item -Path $(Build.SourcesDirectory)\downloads -Destination $(ob_outputDirectory) + displayName: Copy downloads to Artifacts + - pwsh: | # Create output directory for packages which have been uploaded to blob storage New-Item -Path $(Build.ArtifactStagingDirectory)/uploaded -ItemType Directory -Force From 3f1dd2deeeb569e8d50ff069a6f3d61e4667204d Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 8 Nov 2024 16:52:02 -0600 Subject: [PATCH 075/316] Build.ArtifactStagingDirectory --- .pipelines/templates/uploadToAzure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index 1fb96785c57..49302a484a4 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -235,7 +235,7 @@ jobs: - pwsh: | Write-Verbose -Message "Copying all files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" - Copy-Item -Path $(Build.SourcesDirectory)\downloads -Destination $(ob_outputDirectory) + Copy-Item -Path $(Build.ArtifactStagingDirectory)/downloads -Destination $(ob_outputDirectory) displayName: Copy downloads to Artifacts - pwsh: | From 9337f9fd609df9a6b314674aefcdab99ed2cfe95 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 8 Nov 2024 16:53:17 -0600 Subject: [PATCH 076/316] added verbose --- .pipelines/templates/uploadToAzure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index 49302a484a4..943f8bb93d5 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -234,7 +234,7 @@ jobs: displayName: 'Capture downloads' - pwsh: | - Write-Verbose -Message "Copying all files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" + Write-Verbose -Verbose "Copying all files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" Copy-Item -Path $(Build.ArtifactStagingDirectory)/downloads -Destination $(ob_outputDirectory) displayName: Copy downloads to Artifacts From 6975141124fb90b3240498c5844e6de4f2480259 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 8 Nov 2024 18:01:53 -0600 Subject: [PATCH 077/316] download folder copy-item -recurse --- .pipelines/templates/uploadToAzure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index 943f8bb93d5..36c16f47f82 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -235,7 +235,7 @@ jobs: - pwsh: | Write-Verbose -Verbose "Copying all files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" - Copy-Item -Path $(Build.ArtifactStagingDirectory)/downloads -Destination $(ob_outputDirectory) + Copy-Item -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Destination $(ob_outputDirectory) -Recurse displayName: Copy downloads to Artifacts - pwsh: | From 19fcfd0a594a0251cd7bd7de98aadb10707dcb57 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 11 Nov 2024 15:01:10 -0600 Subject: [PATCH 078/316] Filter copy-item to exclude msix and nupkg --- .pipelines/templates/release-githubtasks.yml | 60 +------------------- .pipelines/templates/uploadToAzure.yml | 7 ++- 2 files changed, 7 insertions(+), 60 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 2cb9a5fcd2e..875fb892334 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -14,64 +14,7 @@ stages: inputs: - input: pipelineArtifact pipeline: PSPackagesOfficial - artifactName: drop_linux_package_deb - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_fxdependent - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_mariner_arm64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_mariner_x64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_minSize - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_rpm - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_tar - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_tar_alpine - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_tar_alpine_fxd - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_tar_arm - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_linux_package_tar_arm64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_mac_package_sign_package_macOS_arm64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_mac_package_sign_package_macOS_x64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_arm64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_fxdependent - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_fxdependentWinDesktop - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_minsize - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_x64 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_windows_package_package_win_x86 - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: macos-pkgs + artifactName: drop_upload_upload_packages variables: - name: runCodesignValidationInjection value: false @@ -111,6 +54,7 @@ stages: inputs: targetType: inline script: | + $localInstallerPath = Get-ChildItem -Path $(Pipeline.Workspace) -Filter '*win-x64.msi' | Select-Object -First 1 -ExpandProperty FullName Start-Process -FilePath msiexec -ArgumentList "/package $localInstallerPath /quiet REGISTER_MANIFEST=1" -Wait -NoNewWindow displayName: Install pwsh 7 diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index 36c16f47f82..8f281818800 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -234,8 +234,11 @@ jobs: displayName: 'Capture downloads' - pwsh: | - Write-Verbose -Verbose "Copying all files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" - Copy-Item -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Destination $(ob_outputDirectory) -Recurse + Write-Verbose -Verbose "Copying Github Release files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" + Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | + Where-Object { $_.Extension -notin '.msix', '.nupkg' } | + Where-Object { Write-Verbose -Verbose "$_.Name" } | + Copy-Item -Destination $(ob_outputDirectory) -Recurse displayName: Copy downloads to Artifacts - pwsh: | From 29adf894d5b2e39291d75f85f3a5e91e07e16c58 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 11 Nov 2024 16:00:04 -0600 Subject: [PATCH 079/316] fixed error with piping objects along --- .pipelines/templates/uploadToAzure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index 8f281818800..c2de1caebe1 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -237,7 +237,7 @@ jobs: Write-Verbose -Verbose "Copying Github Release files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | Where-Object { $_.Extension -notin '.msix', '.nupkg' } | - Where-Object { Write-Verbose -Verbose "$_.Name" } | + ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | Copy-Item -Destination $(ob_outputDirectory) -Recurse displayName: Copy downloads to Artifacts From 856c14dbb24448d9f11f376fdff3031d25612c5b Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 11 Nov 2024 17:03:31 -0600 Subject: [PATCH 080/316] adding pwsh to path, added checks to see if it was installed --- .pipelines/templates/release-githubtasks.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 875fb892334..f3375861bea 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -55,7 +55,23 @@ stages: targetType: inline script: | $localInstallerPath = Get-ChildItem -Path $(Pipeline.Workspace) -Filter '*win-x64.msi' | Select-Object -First 1 -ExpandProperty FullName + if (Test-Path -Path $localInstallerPath) { + Write-Verbose -Verbose "Installer found at $localInstallerPath" + } else { + throw "Installer not found" + } + Write-Verbose -Verbose "Installing PowerShell via msiexec" Start-Process -FilePath msiexec -ArgumentList "/package $localInstallerPath /quiet REGISTER_MANIFEST=1" -Wait -NoNewWindow + + $pwshPath = Get-ChildItem -Directory -Path 'C:\Program Files\PowerShell\7*' | Select-Object -First 1 -ExpandProperty FullName + + if (Test-Path -Path $pwshPath) { + Write-Verbose -Verbose "PowerShell installed at $pwshPath" + $setPwshPath = "vso[task.setvariable variable=PATH]${env:PATH}:$pwshPath" + Write-Verbose -Verbose "sending " + $setPwshPath + } else { + throw "PowerShell not installed" + } displayName: Install pwsh 7 - task: PowerShell@2 From 874af1c96994c447c2ef518183d8f3b8eed9deb8 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 11 Nov 2024 17:09:29 -0600 Subject: [PATCH 081/316] removed error with write Verbose --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index f3375861bea..d670ae961f4 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -68,7 +68,7 @@ stages: if (Test-Path -Path $pwshPath) { Write-Verbose -Verbose "PowerShell installed at $pwshPath" $setPwshPath = "vso[task.setvariable variable=PATH]${env:PATH}:$pwshPath" - Write-Verbose -Verbose "sending " + $setPwshPath + Write-Host "sending " + $setPwshPath } else { throw "PowerShell not installed" } From fc36532edc4c1c891438b06a8d2e635fa04b6cc0 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 11 Nov 2024 18:09:03 -0600 Subject: [PATCH 082/316] Write-Host ##vstsCommandString --- .pipelines/templates/release-githubtasks.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index d670ae961f4..a1ed473a352 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -68,7 +68,9 @@ stages: if (Test-Path -Path $pwshPath) { Write-Verbose -Verbose "PowerShell installed at $pwshPath" $setPwshPath = "vso[task.setvariable variable=PATH]${env:PATH}:$pwshPath" - Write-Host "sending " + $setPwshPath + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + Write-Verbose -Verbose $env:PATH } else { throw "PowerShell not installed" } From 3e64ca17a43b8e3f01a56b367f99232217d6153f Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 11 Nov 2024 19:40:44 -0600 Subject: [PATCH 083/316] vsts command string --- .pipelines/templates/release-githubtasks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index a1ed473a352..ff98ba64332 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -68,8 +68,8 @@ stages: if (Test-Path -Path $pwshPath) { Write-Verbose -Verbose "PowerShell installed at $pwshPath" $setPwshPath = "vso[task.setvariable variable=PATH]${env:PATH}:$pwshPath" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" + Write-Host "sending " + $setPwshPath + Write-Host "##$setPwshPath" Write-Verbose -Verbose $env:PATH } else { throw "PowerShell not installed" From 7f05c14e916c03021048242a9bd5211e4f0d2171 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 11:20:46 -0600 Subject: [PATCH 084/316] trying again --- .pipelines/templates/release-githubtasks.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index ff98ba64332..aa7a5095267 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -67,9 +67,8 @@ stages: if (Test-Path -Path $pwshPath) { Write-Verbose -Verbose "PowerShell installed at $pwshPath" - $setPwshPath = "vso[task.setvariable variable=PATH]${env:PATH}:$pwshPath" - Write-Host "sending " + $setPwshPath - Write-Host "##$setPwshPath" + Write-Host "vso[task.setvariable variable=$pwsh_path]$pwshPath" + Write-Host "vso[task.setvariable variable=PATH]${env:PATH}:$pwshPath" Write-Verbose -Verbose $env:PATH } else { throw "PowerShell not installed" From 094ee298f26d775d0824f1707cf7bd5f86771bfb Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 11:29:47 -0600 Subject: [PATCH 085/316] added path in a new way --- .pipelines/templates/release-githubtasks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index aa7a5095267..9f93c3ee9e8 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -68,7 +68,8 @@ stages: if (Test-Path -Path $pwshPath) { Write-Verbose -Verbose "PowerShell installed at $pwshPath" Write-Host "vso[task.setvariable variable=$pwsh_path]$pwshPath" - Write-Host "vso[task.setvariable variable=PATH]${env:PATH}:$pwshPath" + $env:PATH += ";$pwshPath" + Write-Host "vso[task.setvariable variable=PATH]$env:PATH" Write-Verbose -Verbose $env:PATH } else { throw "PowerShell not installed" From bddf275d75a2ce8c9731cb7498678211cabad8e4 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 11:45:11 -0600 Subject: [PATCH 086/316] checking if the folder contain pwsh.exe --- .pipelines/templates/release-githubtasks.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 9f93c3ee9e8..c92274a139f 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -64,9 +64,12 @@ stages: Start-Process -FilePath msiexec -ArgumentList "/package $localInstallerPath /quiet REGISTER_MANIFEST=1" -Wait -NoNewWindow $pwshPath = Get-ChildItem -Directory -Path 'C:\Program Files\PowerShell\7*' | Select-Object -First 1 -ExpandProperty FullName - + if (Test-Path -Path $pwshPath) { Write-Verbose -Verbose "PowerShell installed at $pwshPath" + Get-ChildItem -Path $pwshPath -Recurse | + Where-Object { $_.Name -eq 'pwsh.exe' } | + Select-Object -ExpandProperty FullName Write-Host "vso[task.setvariable variable=$pwsh_path]$pwshPath" $env:PATH += ";$pwshPath" Write-Host "vso[task.setvariable variable=PATH]$env:PATH" From 7d64f1823447bc8f7f6e804a6c6993f0d8d84810 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 11:50:00 -0600 Subject: [PATCH 087/316] added \ --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index c92274a139f..96c4830c503 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -71,7 +71,7 @@ stages: Where-Object { $_.Name -eq 'pwsh.exe' } | Select-Object -ExpandProperty FullName Write-Host "vso[task.setvariable variable=$pwsh_path]$pwshPath" - $env:PATH += ";$pwshPath" + $env:PATH += ";$pwshPath\" Write-Host "vso[task.setvariable variable=PATH]$env:PATH" Write-Verbose -Verbose $env:PATH } else { From c7c7cc7892260e1845212805ee9efc632c864266 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 12:02:04 -0600 Subject: [PATCH 088/316] checking the next step with out pwsh --- .pipelines/templates/release-githubtasks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 96c4830c503..6a2a818c8b3 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -82,9 +82,9 @@ stages: - task: PowerShell@2 inputs: targetType: inline - pwsh: true script: | - Write-Verbose -Message "PowerShell 7 installed" -Verbose + Write-Verbose -Verbose $env:PATH + Get-ChildItem -Directory -Path $pwsh_Path displayName: Check pwsh 7 installation - task: PowerShell@2 From f840e46537daaa6841bed00b02a749b57c5b4e4c Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 12:03:29 -0600 Subject: [PATCH 089/316] checking the next step with out pwsh --- .pipelines/templates/release-githubtasks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 6a2a818c8b3..be93519db07 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -85,6 +85,8 @@ stages: script: | Write-Verbose -Verbose $env:PATH Get-ChildItem -Directory -Path $pwsh_Path + pwsh.exe -c { $PSVersionTable } + displayName: Check pwsh 7 installation - task: PowerShell@2 From 73b9d47d7178c261efa56312c995fde97868b63a Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 12:23:41 -0600 Subject: [PATCH 090/316] trying ## --- .pipelines/templates/release-githubtasks.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index be93519db07..89831baaddc 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -70,9 +70,10 @@ stages: Get-ChildItem -Path $pwshPath -Recurse | Where-Object { $_.Name -eq 'pwsh.exe' } | Select-Object -ExpandProperty FullName - Write-Host "vso[task.setvariable variable=$pwsh_path]$pwshPath" + Write-Host "##vso[task.setvariable variable=$pwsh_path]$pwshPath" $env:PATH += ";$pwshPath\" - Write-Host "vso[task.setvariable variable=PATH]$env:PATH" + Write-Verbose -Verbose "Adding pwsh to env:PATH" + Write-Host "##vso[task.setvariable variable=PATH]$env:PATH" Write-Verbose -Verbose $env:PATH } else { throw "PowerShell not installed" @@ -84,7 +85,9 @@ stages: targetType: inline script: | Write-Verbose -Verbose $env:PATH - Get-ChildItem -Directory -Path $pwsh_Path + Get-ChildItem -Directory -Path $pwsh_Path | ForEach-Object { + Write-Verbose -Verbose $_.FullName + } pwsh.exe -c { $PSVersionTable } displayName: Check pwsh 7 installation From 92efa1a710c4f960c9b3fbc6c865c446cc8577fa Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 12:46:56 -0600 Subject: [PATCH 091/316] ##vso[task.prependpath]$env:PATH --- .pipelines/templates/release-githubtasks.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 89831baaddc..4c8e34d8ad9 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -71,9 +71,8 @@ stages: Where-Object { $_.Name -eq 'pwsh.exe' } | Select-Object -ExpandProperty FullName Write-Host "##vso[task.setvariable variable=$pwsh_path]$pwshPath" - $env:PATH += ";$pwshPath\" Write-Verbose -Verbose "Adding pwsh to env:PATH" - Write-Host "##vso[task.setvariable variable=PATH]$env:PATH" + Write-Host "##vso[task.prependpath]$env:PATH" Write-Verbose -Verbose $env:PATH } else { throw "PowerShell not installed" From bbde19e167a19500d0e19c55b20233d166dd15fb Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 13:18:35 -0600 Subject: [PATCH 092/316] fixed variable $ --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 4c8e34d8ad9..36d37570aeb 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -70,7 +70,7 @@ stages: Get-ChildItem -Path $pwshPath -Recurse | Where-Object { $_.Name -eq 'pwsh.exe' } | Select-Object -ExpandProperty FullName - Write-Host "##vso[task.setvariable variable=$pwsh_path]$pwshPath" + Write-Host "##vso[task.setvariable variable=pwsh_path;]$pwshPath" Write-Verbose -Verbose "Adding pwsh to env:PATH" Write-Host "##vso[task.prependpath]$env:PATH" Write-Verbose -Verbose $env:PATH From 0c9bddf9915fd0aae98aa6b76ef93ed6718f6c0e Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 13:19:47 -0600 Subject: [PATCH 093/316] fixed path --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 36d37570aeb..fa02a4d713c 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -84,7 +84,7 @@ stages: targetType: inline script: | Write-Verbose -Verbose $env:PATH - Get-ChildItem -Directory -Path $pwsh_Path | ForEach-Object { + Get-ChildItem -Directory -Path $(pwsh_Path) | ForEach-Object { Write-Verbose -Verbose $_.FullName } pwsh.exe -c { $PSVersionTable } From 8b71575843154d3dfe58b61752f73bdff7e9ef40 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 13:32:51 -0600 Subject: [PATCH 094/316] fixed string path issue, and env:Path --- .pipelines/templates/release-githubtasks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index fa02a4d713c..f3894239bb8 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -72,7 +72,7 @@ stages: Select-Object -ExpandProperty FullName Write-Host "##vso[task.setvariable variable=pwsh_path;]$pwshPath" Write-Verbose -Verbose "Adding pwsh to env:PATH" - Write-Host "##vso[task.prependpath]$env:PATH" + Write-Host "##vso[task.prependpath]$pwshPath" Write-Verbose -Verbose $env:PATH } else { throw "PowerShell not installed" @@ -84,7 +84,7 @@ stages: targetType: inline script: | Write-Verbose -Verbose $env:PATH - Get-ChildItem -Directory -Path $(pwsh_Path) | ForEach-Object { + Get-ChildItem -Directory -Path "$(pwsh_Path)" | ForEach-Object { Write-Verbose -Verbose $_.FullName } pwsh.exe -c { $PSVersionTable } From cfdc0488b8bf5d2fcf319575e1e9bfc58527cc85 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 15:03:29 -0600 Subject: [PATCH 095/316] trying pwsh: true --- .pipelines/templates/release-githubtasks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index f3894239bb8..b734ea13c47 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -82,7 +82,9 @@ stages: - task: PowerShell@2 inputs: targetType: inline + pwsh: true script: | + Write-Verbose -Verbose "Pwsh 7 Installed" Write-Verbose -Verbose $env:PATH Get-ChildItem -Directory -Path "$(pwsh_Path)" | ForEach-Object { Write-Verbose -Verbose $_.FullName From 6a451eeabf5fbbce68fd6242a4e707b457bc8c4f Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 15:26:56 -0600 Subject: [PATCH 096/316] added pwsh true, refactored publish release draft step to original --- .pipelines/templates/release-githubtasks.yml | 443 ++----------------- 1 file changed, 26 insertions(+), 417 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index b734ea13c47..246bd5954e3 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -67,13 +67,8 @@ stages: if (Test-Path -Path $pwshPath) { Write-Verbose -Verbose "PowerShell installed at $pwshPath" - Get-ChildItem -Path $pwshPath -Recurse | - Where-Object { $_.Name -eq 'pwsh.exe' } | - Select-Object -ExpandProperty FullName - Write-Host "##vso[task.setvariable variable=pwsh_path;]$pwshPath" Write-Verbose -Verbose "Adding pwsh to env:PATH" Write-Host "##vso[task.prependpath]$pwshPath" - Write-Verbose -Verbose $env:PATH } else { throw "PowerShell not installed" } @@ -85,17 +80,16 @@ stages: pwsh: true script: | Write-Verbose -Verbose "Pwsh 7 Installed" - Write-Verbose -Verbose $env:PATH - Get-ChildItem -Directory -Path "$(pwsh_Path)" | ForEach-Object { - Write-Verbose -Verbose $_.FullName + Write-Verbose -Verbose "env:Path: " + $env:PATH -split ';' | ForEach-Object { + Write-Verbose -Verbose $_ } - pwsh.exe -c { $PSVersionTable } - displayName: Check pwsh 7 installation - task: PowerShell@2 inputs: targetType: inline + pwsh: true script: | git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' cd $(Pipeline.Workspace)/PowerShell @@ -107,6 +101,7 @@ stages: - task: PowerShell@2 inputs: targetType: inline + pwsh: true script: | $variable = 'releaseTag' $branch = $ENV:BUILD_SOURCEBRANCH @@ -124,6 +119,7 @@ stages: - task: PowerShell@2 inputs: targetType: inline + pwsh: true script: | $azureVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '\.', '-' $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" @@ -139,53 +135,15 @@ stages: - task: PowerShell@2 inputs: targetType: inline + pwsh: true script: | Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' - - # - pwsh: | - # # Uninstall Azure RM modules - # $azRmModules = Get-Module -Name AzureRM* -ListAvailable - # if ($azRmModules) { - # $azRmModules | Remove-Module -Force - # } - # - # # Install Az.Storage module if not already installed - # if (-not (Get-Module -Name Az.Storage -ListAvailable)) { - # Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose - # } - # displayName: Install Az.Storage module - - # - task: AzurePowerShell@5 - # displayName: Download packages from Azure Storage - # inputs: - # azureSubscription: az-blob-cicd-infra - # scriptType: inlineScript - # azurePowerShellVersion: LatestVersion - # pwsh: true - # inline: | - # $storageAccount = "$(StorageAccount)" - # $containerName = "$(AzureVersion)" - # $destinationPath = "$(System.ArtifactsDirectory)" - # - # # Get storage account context - # $storageContext = New-AzStorageContext -StorageAccountName $storageAccount - # - # $blobList = Get-AzStorageBlob -Container $containerName -Context $storageContext - # foreach ($blob in $blobList) { - # $blobName = $blob.Name - # $destinationFile = Join-Path -Path $destinationPath -ChildPath $blobName - # Get-AzStorageBlobContent -Container $containerName -Blob $blobName -Destination $destinationFile -Context $storageContext -Force - # Write-Output "Downloaded $blobName to $destinationFile" - # } - # - # $packagesPath = Get-ChildItem -Path $destinationPath\*.deb -Recurse -File | Select-Object -First 1 -ExpandProperty DirectoryName - # Write-Host "sending -- vso[task.setvariable variable=PackagesRoot]$packagesPath" - # Write-Host "##vso[task.setvariable variable=PackagesRoot]$packagesPath" - + - task: PowerShell@2 inputs: targetType: inline + pwsh: true script: | git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' displayName: Clone Internal-Tools repository @@ -193,6 +151,7 @@ stages: - task: PowerShell@2 inputs: targetType: inline + pwsh: true script: | $releaseVersion = '$(ReleaseTag)' -replace '^v','' Write-Verbose -Verbose "Moving packages to staging directory" @@ -212,6 +171,7 @@ stages: - task: PowerShell@2 inputs: targetType: inline + pwsh: true script: | $Path = "$(Pipeline.Workspace)/PSPackagesOfficial" $OutputPath = Join-Path $Path 'hashes.sha256' @@ -233,6 +193,7 @@ stages: - task: PowerShell@2 inputs: targetType: inline + pwsh: true script: | $releaseVersion = '$(ReleaseTag)' -replace '^v','' $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" @@ -243,6 +204,7 @@ stages: - task: PowerShell@2 inputs: targetType: inline + pwsh: true script: | Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName displayName: List all files in the workspace @@ -250,396 +212,43 @@ stages: - task: PowerShell@2 inputs: targetType: inline + pwsh: true script: | - # Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' + Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' $releaseVersion = '$(ReleaseTag)' -replace '^v','' - Write-Verbose -Verbose "Release Version: $releaseVersion" - # $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion - Write-Verbose -Verbose "Added Class SemanticVersion because pwsh 7+ is not available" - class SemanticVersion { - [int]$Major - [int]$Minor - [int]$Patch - [string]$PreReleaseLabel - - SemanticVersion([string]$releaseVersion) { - $versionParts = $releaseVersion -split '[.-]' - for($i = 0; $i -lt $versionParts.Count; $i++) { - if ($i -eq 0) { - $this.Major = [int]$versionParts[$i] - } elseif ($i -eq 1) { - $this.Minor = [int]$versionParts[$i] - } elseif ($i -eq 2) { - $this.Patch = [int]$versionParts[$i] - } elseif ($i -eq 3) { - $this.PreReleaseLabel = $versionParts[$i..($versionParts.Count - 1)] -join '.' - } - } - } - - [string] ToString() { - $version = "$($this.Major).$($this.Minor).$($this.Patch)" - if ($this.PreReleaseLabel) { - $version += "-$($this.PreReleaseLabel)" - } - return $version - } - } - - $semanticVersion = [SemanticVersion]::new($releaseVersion) + $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion + $isPreview = $semanticVersion.PreReleaseLabel -ne $null - + $fileName = if ($isPreview) { "preview.md" } else { $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" } - - $filePath = "$(Pipeline.Workspace)/PowerShell/CHANGELOG/$fileName" + + $filePath = "$env:BUILD_SOURCESDIRECTORY/PowerShell/CHANGELOG/$fileName" Write-Verbose -Verbose "Selected Log file: $filePath" - + if (-not (Test-Path $filePath)) { throw "$filePath not found" } - + $changelog = Get-Content -Path $filePath - + $startPattern = "^## \[" + ([regex]::Escape($releaseVersion)) + "\]" $endPattern = "^## \[{0}\.{1}\.{2}*" -f $semanticVersion.Major, $semanticVersion.Minor, $semanticVersion.Patch - + $clContent = $changelog | ForEach-Object { if ($_ -match $startPattern) { $outputLine = $true } elseif ($_ -match $endPattern) { $outputLine = $false } if ($outputLine) { $_} } | Out-String - + Write-Verbose -Verbose "Selected content: `n$clContent" - function Find-Release { - [CmdletBinding(DefaultParameterSetName="Default")] - param( - [Parameter(Mandatory)] - [string]$Tag - ) - - DynamicParam { - Get-CommonParameters - } - - Begin { - Write-Verbose -Message "Finding release for tag '$Tag'" -Verbose - $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet - } - - End { - $maxRetries = 10 - $retryIntervalSec = 5 - $attempt = 0 - $success = $false - $Result = $null - - while (-not $success -and $attempt -lt $maxRetries) { - try { - Write-Verbose -Message "Retrieving releases for $User/$Repository" -Verbose - $Result = Invoke-RestMethod -Uri "https://api.github.com/repos/$User/$Repository/releases" -Headers $AuthHeader - Write-Verbose -Message "Result from Invoke-RestMethod: $Result" -Verbose - $success = $true - } catch { - $attempt++ - if ($attempt -lt $maxRetries) { - Start-Sleep -Seconds $retryIntervalSec - } else { - throw "Failed to retrieve releases after $maxRetries attempts." - } - } - } - - foreach ($Item in $Result) { - if ($Item.tag_name -eq $Tag) { - Write-Verbose ("Release $Tag is found, upload_url=" + $Item.upload_url) -Verbose - return $Item.id - } - } - } - } - - function New-Release { - [CmdletBinding(DefaultParameterSetName="Default")] - param( - [Parameter(Mandatory)] - [string]$Tag, - - [Parameter(Mandatory)] - [string]$Name, - - [Parameter(Mandatory)] - [string]$Description - ) - - DynamicParam { - Get-CommonParameters - } - - Begin { - Write-Verbose -Message "Creating a new release draft for $Tag" -Verbose - $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet - } - - End { - $maxRetries = 10 - $retryIntervalSec = 5 - $attempt = 0 - $success = $false - - $Body = @{ - tag_name = $Tag - name = $Name - body = $Description - draft = $true - prerelease = $false - } - - $BodyInJson = $Body | ConvertTo-Json - - while (-not $success -and $attempt -lt $maxRetries) { - try { - $Result = Invoke-RestMethod -Headers $AuthHeader -Method Post -Body $BodyInJson -Uri "https://api.github.com/repos/$User/$Repository/releases" - $success = $true - } catch { - $attempt++ - if ($attempt -lt $maxRetries) { - Start-Sleep -Seconds $retryIntervalSec - } else { - throw "Failed to create release after $maxRetries attempts." - } - } - } - - return $Result.id - } - } - - function Find-ReleaseAsset { - [CmdletBinding(DefaultParameterSetName="Default")] - param( - [Parameter(Mandatory)] - [int]$ReleaseId, - - [Parameter(Mandatory)] - [string]$AssetName - ) - - DynamicParam { - Get-CommonParameters - } - - Begin { - Write-Verbose -Message "Searching for asset '$AssetName' in release $ReleaseId" -Verbose - $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet - } - - End { - $maxRetries = 10 - $retryIntervalSec = 5 - $attempt = 0 - $success = $false - - while (-not $success -and $attempt -lt $maxRetries) { - try { - $Result = Invoke-RestMethod -Headers $AuthHeader -Uri "https://api.github.com/repos/$User/$Repository/releases/$ReleaseId/assets" - $success = $true - } catch { - $attempt++ - if ($attempt -lt $maxRetries) { - Start-Sleep -Seconds $retryIntervalSec - } else { - throw "Failed to retrieve assets after $maxRetries attempts." - } - } - } - - foreach ($Item in $Result) { - if ($Item.name -eq $AssetName) { - Write-Verbose "Asset $AssetName was already uploaded, id: $($Item.id)" -Verbose - Write-Verbose (" Asset is ready for download: " + $Item.browser_download_url) -Verbose - return $Item.id - } - } - } - } - - function Push-ReleaseAsset { - [CmdletBinding(DefaultParameterSetName="Default")] - param( - [Parameter(Mandatory)] - [int]$ReleaseId, - - [Parameter(Mandatory)] - [string]$AssetPath, - - [switch]$PassThru - ) - - DynamicParam { - Get-CommonParameters - } - - Begin { - Write-Verbose -Message "Pushing asset '$AssetPath' to release $ReleaseId" -Verbose - $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet - } - - End { - $maxRetries = 10 - $retryIntervalSec = 5 - $attempt = 0 - $success = $false - - $type_7z = "application/octet-stream" - $type_exe = "application/x-msdownload" - - $AssetName = Split-Path $AssetPath -Leaf - - if ([System.IO.Path]::GetExtension($AssetName) -eq ".7z") { - $content_type = $type_7z - } else { - $content_type = $type_exe - } - - $Header = $AuthHeader + @{"Content-Type"=$content_type; "name"=$AssetName;} - $Uri = "https://uploads.github.com/repos/$User/$Repository/releases/$ReleaseId/assets?name=$AssetName" - - $Body = [System.IO.File]::ReadAllBytes($AssetPath) - - while (-not $success -and $attempt -lt $maxRetries) { - try { - $Result = Invoke-RestMethod -Headers $Header -Method Post -Body $Body -Uri $Uri - $success = $true - } catch { - $attempt++ - if ($attempt -lt $maxRetries) { - Start-Sleep -Seconds $retryIntervalSec - } else { - throw "Failed to upload asset after $maxRetries attempts." - } - } - } - - Write-Verbose -Message "$($Result.state)" -Verbose - - if ($Result.state -eq "uploaded") { - Write-Host "'$AssetName' -- Upload succeeded" -ForegroundColor Green - Write-Verbose -Message " Download URL: $($Result.browser_download_url)" -Verbose - } else { - Write-Host "'$AssetName' -- Upload failed with StatusCode $($Result.StatusCode)" -ForegroundColor Red - } - - ## Pass through the HTTP result if '-PassThru' is specified - if ($PassThru) { return $Result } - } - } - - $Script:Auth = $null - - function Get-CommonParameters { - $ParamDict = [System.Management.Automation.RuntimeDefinedParameterDictionary]::new() - - ## Add the '-User' parameter - ## [Parameter(Mandatory)] - ## [string]$User - $UserAtts = [System.Collections.ObjectModel.Collection[System.Attribute]]::new() - $UserAtts.Add([System.Management.Automation.ParameterAttribute]@{Mandatory = $true}) > $null - $ParamDict.Add("User", [System.Management.Automation.RuntimeDefinedParameter]::new("User", [string], $UserAtts)) > $null - - ## Add the '-Repository' parameter - ## [Parameter(Mandatory)] - ## [string]$Repository - $RepositoryAtts = [System.Collections.ObjectModel.Collection[System.Attribute]]::new() - $RepositoryAtts.Add([System.Management.Automation.ParameterAttribute]@{Mandatory = $true}) > $null - $ParamDict.Add("Repository", [System.Management.Automation.RuntimeDefinedParameter]::new("Repository", [string], $RepositoryAtts)) > $null - - ## Add the '-Token' parameter - ## [Parameter(Mandatory, ParameterSetName="UseSpecifiedToken")] - ## [string]$Token - $TokenAtts = [System.Collections.ObjectModel.Collection[System.Attribute]]::new() - $TokenAtts.Add([System.Management.Automation.ParameterAttribute]@{Mandatory = $true; ParameterSetName="UseSpecifiedToken"}) > $null - $ParamDict.Add("Token", [System.Management.Automation.RuntimeDefinedParameter]::new("Token", [string], $TokenAtts)) > $null - - return $ParamDict - } - - function Get-CommonParamValues([System.Management.Automation.PSCmdlet]$Cmdlet) { - $AuthHeader = $Script:Auth - if ($Cmdlet.ParameterSetName -eq "UseSpecifiedToken") { - $Token = $Cmdlet.MyInvocation.BoundParameters["Token"] - $AuthHeader = @{"Authorization"="token $Token"} - } - if (-not $AuthHeader) { - throw "Authentication token is not specified." - } - - $User = $Cmdlet.MyInvocation.BoundParameters["User"] - $Repository = $Cmdlet.MyInvocation.BoundParameters["Repository"] - return $User, $Repository, $AuthHeader - } - - function Publish-ReleaseDraft { - [CmdletBinding(DefaultParameterSetName="Default")] - param( - [Parameter(Mandatory)] - [string]$Tag, - - [Parameter(Mandatory)] - [string]$Name, - - [Parameter(Mandatory)] - [string]$Description, - - [Parameter(Mandatory)] - [string]$PackageFolder - ) - - DynamicParam { Get-CommonParameters } - - Begin { - Write-Verbose -Message "Publishing release draft for $Tag" -Verbose - $User, $Repository, $AuthHeader = Get-CommonParamValues -Cmdlet $PSCmdlet - $Script:Auth = $AuthHeader - $CommonParams = @{User=$User; Repository=$Repository} - } - - End { - $ReleaseId = Find-Release -Tag $Tag @CommonParams - if ($ReleaseId) { - Write-Host "Release for $Tag already created, Release-Id: $ReleaseId" -ForegroundColor Green - } else { - $StringBuilder = [System.Text.StringBuilder]::new($Description, $Description.Length + 2kb) - $StringBuilder.AppendLine().AppendLine() > $null - $StringBuilder.AppendLine("### SHA256 Hashes of the release artifacts").AppendLine() > $null - Get-ChildItem -Path $PackageFolder -File | ForEach-Object { - $PackageName = $_.Name - $SHA256 = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash - $StringBuilder.AppendLine("- $PackageName").AppendLine(" - $SHA256") > $null - } - - $Description = $StringBuilder.ToString() - $ReleaseId = New-Release -Tag $Tag -Name $Name -Description $Description @CommonParams - Write-Host "Release for $Tag created, Release-Id: $ReleaseId" -ForegroundColor Green - } - - Get-ChildItem -Path $PackageFolder -File | ForEach-Object { - $PackageName = $_.Name - $PackageId = Find-ReleaseAsset -ReleaseId $ReleaseId -AssetName $PackageName @CommonParams - if (-not $PackageId) { - Push-ReleaseAsset -ReleaseId $ReleaseId -AssetPath $_.FullName @CommonParams - } - } - } - } - $clContent = "Testing... DeployBox... `n`n" + $clContent - + Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder "$(Pipeline.Workspace)/PSPackagesOfficial" -Token $(GitHubReleasePat) displayName: Publish Release Draft From 55e555f7d690a547bc0149aa7a3814cc409a64c6 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 15:45:32 -0600 Subject: [PATCH 097/316] fixed CL file location --- .pipelines/templates/release-githubtasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 246bd5954e3..776603d3706 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -227,7 +227,7 @@ stages: $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" } - $filePath = "$env:BUILD_SOURCESDIRECTORY/PowerShell/CHANGELOG/$fileName" + $filePath = "$(Pipeline.Workspace)/PowerShell/CHANGELOG/$fileName" Write-Verbose -Verbose "Selected Log file: $filePath" if (-not (Test-Path $filePath)) { From 0bf33c06e765a57f3c13ea4943b7182a0990a813 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 16:10:50 -0600 Subject: [PATCH 098/316] refactoring to original format --- .pipelines/PowerShell-Release-Official.yml | 24 +- .pipelines/templates/release-githubtasks.yml | 495 ++++++++----------- 2 files changed, 236 insertions(+), 283 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index c2380cd0320..14390eca4e8 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -278,8 +278,28 @@ extends: parameters: SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} - - template: /.pipelines/templates/release-githubtasks.yml@self - # dependsOn: BlobPublic + - stage: PublishGitHubRelease + displayName: Publish GitHub Release + dependsOn: BlobPublic + variables: + ob_release_environment: Test + template: /.pipelines/templates/release-githubtasks.yml@self + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Push Git Tag + jobName: PushGitTag + dependsOnJob: GithubReleaseDraft + instructions: | + Push the git tag to upstream + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Make Draft Public + jobName: DraftPublic + dependsOnJob: PushGitTag + instructions: | + Make the GitHub Release Draft Public - stage: PublishNuGet displayName: Publish NuGet diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 776603d3706..9059d662c38 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -1,298 +1,231 @@ -stages: -- stage: PublishGitHubRelease - displayName: Publish GitHub Release +jobs: +- job: GithubReleaseDraft + displayName: Create GitHub Release Draft + condition: succeeded() + pool: + type: release + os: windows + templateContext: + inputs: + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_upload_upload_packages variables: - ob_release_environment: Test - jobs: - - job: GithubReleaseDraft - displayName: Create GitHub Release Draft - condition: succeeded() - pool: - type: release - os: windows - templateContext: - inputs: - - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_upload_upload_packages - variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - value: 1 - - group: 'mscodehub-code-read-akv' - - group: 'Azure Blob variable group' - - group: 'GitHubTokens' - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_sdl_codeSignValidation_enabled - value: false - - name: ob_sdl_binskim_enabled - value: false - - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - - steps: - # - checkout: self - # clean: true - # env: - # ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - # - task: Bash@3 - # inputs: - # targetType: inline - # script: | - # sudo tdnf install -y powershell - # displayName: Install PowerShell - - # Get-ChildItem -Directory -Path 'C:\Program Files\PowerShell\7*' | Select-Object -First 1 -ExpandProperty FullName - # x64 msi installer - - task: PowerShell@2 - inputs: - targetType: inline - script: | - $localInstallerPath = Get-ChildItem -Path $(Pipeline.Workspace) -Filter '*win-x64.msi' | Select-Object -First 1 -ExpandProperty FullName - if (Test-Path -Path $localInstallerPath) { - Write-Verbose -Verbose "Installer found at $localInstallerPath" - } else { - throw "Installer not found" - } - Write-Verbose -Verbose "Installing PowerShell via msiexec" - Start-Process -FilePath msiexec -ArgumentList "/package $localInstallerPath /quiet REGISTER_MANIFEST=1" -Wait -NoNewWindow + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: 'mscodehub-code-read-akv' + - group: 'Azure Blob variable group' + - group: 'GitHubTokens' + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - $pwshPath = Get-ChildItem -Directory -Path 'C:\Program Files\PowerShell\7*' | Select-Object -First 1 -ExpandProperty FullName + steps: + - task: PowerShell@2 + inputs: + targetType: inline + script: | + $localInstallerPath = Get-ChildItem -Path $(Pipeline.Workspace) -Filter '*win-x64.msi' | Select-Object -First 1 -ExpandProperty FullName + if (Test-Path -Path $localInstallerPath) { + Write-Verbose -Verbose "Installer found at $localInstallerPath" + } else { + throw "Installer not found" + } + Write-Verbose -Verbose "Installing PowerShell via msiexec" + Start-Process -FilePath msiexec -ArgumentList "/package $localInstallerPath /quiet REGISTER_MANIFEST=1" -Wait -NoNewWindow + $pwshPath = Get-ChildItem -Directory -Path 'C:\Program Files\PowerShell\7*' | Select-Object -First 1 -ExpandProperty FullName + if (Test-Path -Path $pwshPath) { + Write-Verbose -Verbose "PowerShell installed at $pwshPath" + Write-Verbose -Verbose "Adding pwsh to env:PATH" + Write-Host "##vso[task.prependpath]$pwshPath" + } else { + throw "PowerShell not installed" + } + displayName: Install pwsh 7 - if (Test-Path -Path $pwshPath) { - Write-Verbose -Verbose "PowerShell installed at $pwshPath" - Write-Verbose -Verbose "Adding pwsh to env:PATH" - Write-Host "##vso[task.prependpath]$pwshPath" - } else { - throw "PowerShell not installed" - } - displayName: Install pwsh 7 + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Write-Verbose -Verbose "Pwsh 7 Installed" + Write-Verbose -Verbose "env:Path: " + $env:PATH -split ';' | ForEach-Object { + Write-Verbose -Verbose $_ + } + displayName: Check pwsh 7 installation - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - Write-Verbose -Verbose "Pwsh 7 Installed" - Write-Verbose -Verbose "env:Path: " - $env:PATH -split ';' | ForEach-Object { - Write-Verbose -Verbose $_ - } - displayName: Check pwsh 7 installation + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Write-Verbose -Verbose "Deploy Box Product Pathway Does Not Support the `"checkout`" task" + git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' + cd $(Pipeline.Workspace)/PowerShell + $branchParts = $ENV:BUILD_SOURCEBRANCH -split '/' + $branchName = "$($branchParts[-2])/$($branchParts[-1])" + git checkout $branchName + displayName: Checkout repository - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' - cd $(Pipeline.Workspace)/PowerShell - $branchParts = $ENV:BUILD_SOURCEBRANCH -split '/' - $branchName = "$($branchParts[-2])/$($branchParts[-1])" - git checkout $branchName - displayName: Checkout repository + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + $variable = 'releaseTag' + $branch = $ENV:BUILD_SOURCEBRANCH + if($branch -notmatch '^.*((release/|rebuild/.*rebuild))') + { + throw "Branch name is not in release format: '$branch'" + } - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - $variable = 'releaseTag' - $branch = $ENV:BUILD_SOURCEBRANCH - if($branch -notmatch '^.*((release/|rebuild/.*rebuild))') - { - throw "Branch name is not in release format: '$branch'" - } - - $releaseTag = $Branch -replace '^.*((release|rebuild)/)' - $vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag" - Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose - Write-Host -Object "##$vstsCommandString" - displayName: Set Release Tag - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - $azureVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '\.', '-' - $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - $version = '$(ReleaseTag)'.ToLowerInvariant().Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - displayName: Set container name + $releaseTag = $Branch -replace '^.*((release|rebuild)/)' + $vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag" + Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose + Write-Host -Object "##$vstsCommandString" + displayName: Set Release Tag - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose - displayName: 'Capture Environment Variables' - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' - displayName: Clone Internal-Tools repository + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + $azureVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '\.', '-' + $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + + $version = '$(ReleaseTag)'.ToLowerInvariant().Substring(1) + $vstsCommandString = "vso[task.setvariable variable=Version]$version" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: Set container name - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - Write-Verbose -Verbose "Moving packages to staging directory" - $packagesRoot = "$(Pipeline.Workspace)/PSPackagesOfficial" - New-Item -Path $packagesRoot -ItemType Directory -Force > $null + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + displayName: 'Capture Environment Variables' - Get-ChildItem -Path $(Pipeline.Workspace) | - Where-Object { $_.Name -match "powershell-*" -or $_.Name -match "powershell_*" -and -not $_.PSIsContainer } | + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' + displayName: Clone Internal-Tools repository + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + Write-Verbose -Verbose "Moving packages to staging directory" + $packagesRoot = "$(Pipeline.Workspace)/PSPackagesOfficial" + New-Item -Path $packagesRoot -ItemType Directory -Force > $null + Get-ChildItem -Path $(Pipeline.Workspace) | + Where-Object { $_.Name -match "powershell-*" -or $_.Name -match "powershell_*" -and -not $_.PSIsContainer } | + ForEach-Object { + Move-Item -Path $_.FullName -Destination $packagesRoot + } + Write-Verbose -Verbose "List all packages in the staging directory" + Get-ChildItem -Path $packagesRoot -Recurse | Select-Object -ExpandProperty FullName + displayName: Move packages to staging directory + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + $Path = "$(Pipeline.Workspace)/PSPackagesOfficial" + $OutputPath = Join-Path $Path 'hashes.sha256' + $srcPaths = @($Path) + $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File + $checksums = $packages | ForEach-Object { - Move-Item -Path $_.FullName -Destination $packagesRoot + Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" + $packageName = $_.Name + $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() + # the '*' before the packagename signifies it is a binary + "$hash *$packageName" } + $checksums | Out-File -FilePath $OutputPath -Force + $fileContent = Get-Content -Path $OutputPath -Raw | Out-String + Write-Verbose -Verbose -Message $fileContent + displayName: Add sha256 hashes - Write-Verbose -Verbose "List all packages in the staging directory" - Get-ChildItem -Path $packagesRoot -Recurse | Select-Object -ExpandProperty FullName - displayName: Move packages to staging directory - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - $Path = "$(Pipeline.Workspace)/PSPackagesOfficial" - $OutputPath = Join-Path $Path 'hashes.sha256' - $srcPaths = @($Path) - $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File - $checksums = $packages | - ForEach-Object { - Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" - $packageName = $_.Name - $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() - # the '*' before the packagename signifies it is a binary - "$hash *$packageName" - } - $checksums | Out-File -FilePath $OutputPath -Force - $fileContent = Get-Content -Path $OutputPath -Raw | Out-String - Write-Verbose -Verbose -Message $fileContent - displayName: Add sha256 hashes + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: 'Set release version' - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Set release version' + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName + displayName: List all files in the workspace - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName - displayName: List all files in the workspace + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion - - $isPreview = $semanticVersion.PreReleaseLabel -ne $null - - $fileName = if ($isPreview) { - "preview.md" - } - else { - $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" - } - - $filePath = "$(Pipeline.Workspace)/PowerShell/CHANGELOG/$fileName" - Write-Verbose -Verbose "Selected Log file: $filePath" - - if (-not (Test-Path $filePath)) { - throw "$filePath not found" - } - - $changelog = Get-Content -Path $filePath - - $startPattern = "^## \[" + ([regex]::Escape($releaseVersion)) + "\]" - $endPattern = "^## \[{0}\.{1}\.{2}*" -f $semanticVersion.Major, $semanticVersion.Minor, $semanticVersion.Patch - - $clContent = $changelog | ForEach-Object { - if ($_ -match $startPattern) { $outputLine = $true } - elseif ($_ -match $endPattern) { $outputLine = $false } - if ($outputLine) { $_} - } | Out-String - - Write-Verbose -Verbose "Selected content: `n$clContent" - - $clContent = "Testing... DeployBox... `n`n" + $clContent - - Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder "$(Pipeline.Workspace)/PSPackagesOfficial" -Token $(GitHubReleasePat) - displayName: Publish Release Draft + $isPreview = $semanticVersion.PreReleaseLabel -ne $null - # - job: PushGitTag - # displayName: Push Git Tag - # pool: - # type: server - # timeoutInMinutes: 4320 # job times out in 3 days - # steps: - # - task: ManualValidation@0 - # displayName: Push Git Tag - # timeoutInMinutes: 2880 - # inputs: - # instructions: Push the git tag to upstream - # onTimeout: reject - # notifyUsers: '' -# - # - job: DraftPublic - # displayName: Make Draft Public - # pool: - # type: server - # timeoutInMinutes: 4320 # job times out in 3 days - # steps: - # - task: ManualValidation@0 - # displayName: Make Draft Public - # timeoutInMinutes: 2880 - # inputs: - # instructions: Make the GitHub Release Draft Public - # onTimeout: reject - # notifyUsers: '' - - # - template: /.pipelines/templates/approvalJob.yml@self - # parameters: - # displayName: Push Git Tag - # jobName: PushGitTag - # dependsOnJob: GithubReleaseDraft - # instructions: | - # Push the git tag to upstream + $fileName = if ($isPreview) { + "preview.md" + } + else { + $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" + } + + $filePath = "$(Pipeline.Workspace)/PowerShell/CHANGELOG/$fileName" + Write-Verbose -Verbose "Selected Log file: $filePath" + + if (-not (Test-Path $filePath)) { + throw "$filePath not found" + } + + $changelog = Get-Content -Path $filePath + + $startPattern = "^## \[" + ([regex]::Escape($releaseVersion)) + "\]" + $endPattern = "^## \[{0}\.{1}\.{2}*" -f $semanticVersion.Major, $semanticVersion.Minor, $semanticVersion.Patch + + $clContent = $changelog | ForEach-Object { + if ($_ -match $startPattern) { $outputLine = $true } + elseif ($_ -match $endPattern) { $outputLine = $false } + if ($outputLine) { $_} + } | Out-String + + Write-Verbose -Verbose "Selected content: `n$clContent" + $clContent = "Testing... DeployBox... `n`n" + $clContent - # - template: /.pipelines/templates/approvalJob.yml@self - # parameters: - # displayName: Make Draft Public - # jobName: DraftPublic - # dependsOnJob: PushGitTag - # instructions: | - # Make the GitHub Release Draft Public - \ No newline at end of file + Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder "$(Pipeline.Workspace)/PSPackagesOfficial" -Token $(GitHubReleasePat) + displayName: Publish Release Draft From f9b7bb448ef4c31a129ff5c081e5bf38f7f000df Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 16:13:57 -0600 Subject: [PATCH 099/316] made approval jobs a separate stage and template --- .pipelines/PowerShell-Release-Official.yml | 43 +++++++++++++--------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 14390eca4e8..ffa98864706 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -283,23 +283,32 @@ extends: dependsOn: BlobPublic variables: ob_release_environment: Test - template: /.pipelines/templates/release-githubtasks.yml@self - - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Push Git Tag - jobName: PushGitTag - dependsOnJob: GithubReleaseDraft - instructions: | - Push the git tag to upstream - - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Make Draft Public - jobName: DraftPublic - dependsOnJob: PushGitTag - instructions: | - Make the GitHub Release Draft Public + jobs: + - template: /.pipelines/templates/release-githubtasks.yml@self + + - stage: PushGitTag + displayName: Push Git Tag + dependsOn: PublishGitHubRelease + jobs: + - template: /.pipelines/templates/release-push-git-tag.yml@self + parameters: + displayName: Push Git Tag + jobName: PushGitTag + dependsOnJob: GithubReleaseDraft + instructions: | + Push the git tag to upstream + + - stage: MakeDraftPublic + displayName: Make Draft Public + dependsOn: PushGitTag + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Make Draft Public + jobName: DraftPublic + dependsOnJob: PushGitTag + instructions: | + Make the GitHub Release Draft Public - stage: PublishNuGet displayName: Publish NuGet From 40d93e1b3c0ff317bace2209f850371cdc877df3 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 16:15:00 -0600 Subject: [PATCH 100/316] copilot auto fixed wrong approvalJob --- .pipelines/PowerShell-Release-Official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index ffa98864706..427a4465764 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -290,7 +290,7 @@ extends: displayName: Push Git Tag dependsOn: PublishGitHubRelease jobs: - - template: /.pipelines/templates/release-push-git-tag.yml@self + - template: /.pipelines/templates/approvalJob.yml@self parameters: displayName: Push Git Tag jobName: PushGitTag From 2cf38bbdaed66d9b8643d6d173680ec7644fc2e1 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 12 Nov 2024 16:16:10 -0600 Subject: [PATCH 101/316] removed job dependency because stage dependency exists --- .pipelines/PowerShell-Release-Official.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 427a4465764..7d779cc3f2d 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -294,7 +294,6 @@ extends: parameters: displayName: Push Git Tag jobName: PushGitTag - dependsOnJob: GithubReleaseDraft instructions: | Push the git tag to upstream @@ -306,7 +305,6 @@ extends: parameters: displayName: Make Draft Public jobName: DraftPublic - dependsOnJob: PushGitTag instructions: | Make the GitHub Release Draft Public From 95b4042ccb9b66a82209c73270ffbc9fb77b739a Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 13 Nov 2024 10:48:32 -0600 Subject: [PATCH 102/316] copying packages in separate folders --- .pipelines/PowerShell-Release-Official.yml | 2 ++ .pipelines/templates/release-githubtasks.yml | 2 +- .pipelines/templates/release-publish-nuget.yml | 3 ++- .pipelines/templates/uploadToAzure.yml | 12 +++++++++++- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 7d779cc3f2d..5b6afb84e4c 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -311,6 +311,8 @@ extends: - stage: PublishNuGet displayName: Publish NuGet # dependsOn: PublishGitHubRelease + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/release-publish-nuget.yml@self parameters: diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 9059d662c38..aef2691d730 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -9,7 +9,7 @@ jobs: inputs: - input: pipelineArtifact pipeline: PSPackagesOfficial - artifactName: drop_upload_upload_packages + artifactName: GitHubPackages variables: - name: runCodesignValidationInjection value: false diff --git a/.pipelines/templates/release-publish-nuget.yml b/.pipelines/templates/release-publish-nuget.yml index 9425dc96c64..04c48c2714c 100644 --- a/.pipelines/templates/release-publish-nuget.yml +++ b/.pipelines/templates/release-publish-nuget.yml @@ -8,7 +8,8 @@ jobs: displayName: Publish to NuGet condition: succeeded() pool: - type: windows + type: release + os: windows variables: - name: runCodesignValidationInjection value: false diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index c2de1caebe1..195aea5fca9 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -235,10 +235,20 @@ jobs: - pwsh: | Write-Verbose -Verbose "Copying Github Release files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" + + Write-Verbose -Verbose "Creating output directory for GitHub Release files: $(ob_outputDirectory)/GitHubPackages" + New-Item -Path $(ob_outputDirectory)/GitHubPackages -ItemType Directory -Force Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | Where-Object { $_.Extension -notin '.msix', '.nupkg' } | ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | - Copy-Item -Destination $(ob_outputDirectory) -Recurse + Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse + + Write-Verbose -Verbose "Creating output directory for NuGet packages: $(ob_outputDirectory)/NuGetPackages" + New-Item -Path $(ob_outputDirectory)/NuGetPackages -ItemType Directory -Force + Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | + Where-Object { $_.Extension -eq '.nupkg' } | + ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | + Copy-Item -Destination $(ob_outputDirectory)/NuGetPackages -Recurse displayName: Copy downloads to Artifacts - pwsh: | From 5e107ffcdd06c355829883b10b15c88cf9c31417 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 13 Nov 2024 13:45:55 -0600 Subject: [PATCH 103/316] changed package path, turned on DBP on nuget --- .pipelines/templates/release-githubtasks.yml | 62 ++--------------- .../templates/release-publish-nuget.yml | 68 ++++++++++++++++--- .pipelines/templates/uploadToAzure.yml | 2 + 3 files changed, 66 insertions(+), 66 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index aef2691d730..8c8970afaa4 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -9,7 +9,7 @@ jobs: inputs: - input: pipelineArtifact pipeline: PSPackagesOfficial - artifactName: GitHubPackages + artifactName: drop_upload_upload_packages variables: - name: runCodesignValidationInjection value: false @@ -36,7 +36,7 @@ jobs: inputs: targetType: inline script: | - $localInstallerPath = Get-ChildItem -Path $(Pipeline.Workspace) -Filter '*win-x64.msi' | Select-Object -First 1 -ExpandProperty FullName + $localInstallerPath = Get-ChildItem -Path "$(Pipeline.Workspace)/GitHubPackages" -Filter '*win-x64.msi' | Select-Object -First 1 -ExpandProperty FullName if (Test-Path -Path $localInstallerPath) { Write-Verbose -Verbose "Installer found at $localInstallerPath" } else { @@ -79,39 +79,7 @@ jobs: git checkout $branchName displayName: Checkout repository - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - $variable = 'releaseTag' - $branch = $ENV:BUILD_SOURCEBRANCH - if($branch -notmatch '^.*((release/|rebuild/.*rebuild))') - { - throw "Branch name is not in release format: '$branch'" - } - - $releaseTag = $Branch -replace '^.*((release|rebuild)/)' - $vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag" - Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose - Write-Host -Object "##$vstsCommandString" - displayName: Set Release Tag - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - $azureVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '\.', '-' - $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - $version = '$(ReleaseTag)'.ToLowerInvariant().Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - displayName: Set container name + - template: release-SetReleaseTagAndContainerName.yml - task: PowerShell@2 inputs: @@ -128,31 +96,13 @@ jobs: script: | git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' displayName: Clone Internal-Tools repository - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - Write-Verbose -Verbose "Moving packages to staging directory" - $packagesRoot = "$(Pipeline.Workspace)/PSPackagesOfficial" - New-Item -Path $packagesRoot -ItemType Directory -Force > $null - Get-ChildItem -Path $(Pipeline.Workspace) | - Where-Object { $_.Name -match "powershell-*" -or $_.Name -match "powershell_*" -and -not $_.PSIsContainer } | - ForEach-Object { - Move-Item -Path $_.FullName -Destination $packagesRoot - } - Write-Verbose -Verbose "List all packages in the staging directory" - Get-ChildItem -Path $packagesRoot -Recurse | Select-Object -ExpandProperty FullName - displayName: Move packages to staging directory - + - task: PowerShell@2 inputs: targetType: inline pwsh: true script: | - $Path = "$(Pipeline.Workspace)/PSPackagesOfficial" + $Path = "$(Pipeline.Workspace)/GitHubPackages" $OutputPath = Join-Path $Path 'hashes.sha256' $srcPaths = @($Path) $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File @@ -227,5 +177,5 @@ jobs: Write-Verbose -Verbose "Selected content: `n$clContent" $clContent = "Testing... DeployBox... `n`n" + $clContent - Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder "$(Pipeline.Workspace)/PSPackagesOfficial" -Token $(GitHubReleasePat) + Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder "$(Pipeline.Workspace)/GitHubPackages" -Token $(GitHubReleasePat) displayName: Publish Release Draft diff --git a/.pipelines/templates/release-publish-nuget.yml b/.pipelines/templates/release-publish-nuget.yml index 04c48c2714c..41114183414 100644 --- a/.pipelines/templates/release-publish-nuget.yml +++ b/.pipelines/templates/release-publish-nuget.yml @@ -10,6 +10,11 @@ jobs: pool: type: release os: windows + templateContext: + inputs: + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_upload_upload_packages variables: - name: runCodesignValidationInjection value: false @@ -30,10 +35,52 @@ jobs: value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: - - checkout: self - clean: true - env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + - task: PowerShell@2 + inputs: + targetType: inline + script: | + $localInstallerPath = Get-ChildItem -Path "$(Pipeline.Workspace)/GitHubPackages" -Filter '*win-x64.msi' | Select-Object -First 1 -ExpandProperty FullName + if (Test-Path -Path $localInstallerPath) { + Write-Verbose -Verbose "Installer found at $localInstallerPath" + } else { + throw "Installer not found" + } + Write-Verbose -Verbose "Installing PowerShell via msiexec" + Start-Process -FilePath msiexec -ArgumentList "/package $localInstallerPath /quiet REGISTER_MANIFEST=1" -Wait -NoNewWindow + $pwshPath = Get-ChildItem -Directory -Path 'C:\Program Files\PowerShell\7*' | Select-Object -First 1 -ExpandProperty FullName + if (Test-Path -Path $pwshPath) { + Write-Verbose -Verbose "PowerShell installed at $pwshPath" + Write-Verbose -Verbose "Adding pwsh to env:PATH" + Write-Host "##vso[task.prependpath]$pwshPath" + } else { + throw "PowerShell not installed" + } + displayName: Install pwsh 7 + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Write-Verbose -Verbose "Pwsh 7 Installed" + Write-Verbose -Verbose "env:Path: " + $env:PATH -split ';' | ForEach-Object { + Write-Verbose -Verbose $_ + } + displayName: Check pwsh 7 installation + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Write-Verbose -Verbose "Deploy Box Product Pathway Does Not Support the `"checkout`" task" + git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' + cd $(Pipeline.Workspace)/PowerShell + $branchParts = $ENV:BUILD_SOURCEBRANCH -split '/' + $branchName = "$($branchParts[-2])/$($branchParts[-1])" + git checkout $branchName + displayName: Checkout repository - template: release-SetReleaseTagAndContainerName.yml @@ -41,27 +88,28 @@ jobs: Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' - - download: PSPackagesOfficial - artifact: drop_nupkg_build_nupkg - displayName: Download nuget packages - - pwsh: | #Exclude all global tool packages. Their names start with 'PowerShell.' $null = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/release" - Copy-Item "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/drop_nupkg_build_nupkg/*.nupkg" -Destination "$(Pipeline.Workspace)/release" -Exclude "PowerShell.*.nupkg" -Force -Verbose + Copy-Item "$(Pipeline.Workspace)/NuGetPackages/*.nupkg" -Destination "$(Pipeline.Workspace)/release" -Exclude "PowerShell.*.nupkg" -Force -Verbose $releaseVersion = '$(VERSION)' - $globalToolPath = "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/drop_nupkg_build_nupkg/PowerShell.$releaseVersion.nupkg" + $globalToolPath = "$(Pipeline.Workspace)/NuGetPackages/PowerShell.$releaseVersion.nupkg" if ($releaseVersion -notlike '*-*') { # Copy the global tool package for stable releases Copy-Item $globalToolPath -Destination "$(Pipeline.Workspace)/release" } + Write-Verbose -Verbose "The .nupkgs below will be pushed:" Get-ChildItem "$(Pipeline.Workspace)/release" -recurse displayName: Download and capture nupkgs condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) + - task: ManualValidation@0 + displayName: 'Manual validation' + + - task: NuGetCommand@2 displayName: 'NuGet push' condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index 195aea5fca9..7c7483f8893 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -5,6 +5,8 @@ jobs: pool: type: windows variables: + - name: ob_sdl_sbom_enabled + value: false - name: runCodesignValidationInjection value: false - name: NugetSecurityAnalysisWarningLevel From d8f02d737ab2f594f4d5635a94f941ed2c29d24a Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 13 Nov 2024 13:49:13 -0600 Subject: [PATCH 104/316] removed nugetcommand --- .../templates/release-publish-nuget.yml | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.pipelines/templates/release-publish-nuget.yml b/.pipelines/templates/release-publish-nuget.yml index 41114183414..2e6134e9e10 100644 --- a/.pipelines/templates/release-publish-nuget.yml +++ b/.pipelines/templates/release-publish-nuget.yml @@ -106,15 +106,11 @@ jobs: displayName: Download and capture nupkgs condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) - - task: ManualValidation@0 - displayName: 'Manual validation' - - - - task: NuGetCommand@2 - displayName: 'NuGet push' - condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) - inputs: - command: push - packagesToPush: '$(Pipeline.Workspace)/release/*.nupkg' - nuGetFeedType: external - publishFeedCredentials: PowerShellNuGetOrgPush + # - task: NuGetCommand@2 + # displayName: 'NuGet push' + # condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) + # inputs: + # command: push + # packagesToPush: '$(Pipeline.Workspace)/release/*.nupkg' + # nuGetFeedType: external + # publishFeedCredentials: PowerShellNuGetOrgPush From 78a0abce43383d21a03e31cbb80c53d1eb42cffe Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 13 Nov 2024 14:28:08 -0600 Subject: [PATCH 105/316] Production, turn on dependencies --- .pipelines/PowerShell-Release-Official.yml | 24 +++++++++---------- .../templates/release-publish-nuget.yml | 16 ++++++------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 5b6afb84e4c..c77e6c8bb83 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -77,7 +77,7 @@ resources: - releases/* extends: - template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates + template: v2/OneBranch.Official.CrossPlat.yml@templates parameters: release: category: NonAzure @@ -282,7 +282,7 @@ extends: displayName: Publish GitHub Release dependsOn: BlobPublic variables: - ob_release_environment: Test + ob_release_environment: Production jobs: - template: /.pipelines/templates/release-githubtasks.yml@self @@ -310,9 +310,9 @@ extends: - stage: PublishNuGet displayName: Publish NuGet - # dependsOn: PublishGitHubRelease + dependsOn: PublishGitHubRelease variables: - ob_release_environment: Test + ob_release_environment: Production jobs: - template: /.pipelines/templates/release-publish-nuget.yml@self parameters: @@ -320,14 +320,14 @@ extends: - stage: PublishPMC displayName: Publish PMC - # dependsOn: PublishGitHubRelease + dependsOn: PublishGitHubRelease jobs: - template: /.pipelines/templates/release-publish-pmc.yml@self parameters: skipPublish: ${{ parameters.SkipPMCPublish }} - stage: ReleaseDocker - # dependsOn: PublishGitHubRelease + dependsOn: PublishGitHubRelease displayName: 'Docker Release' jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -346,7 +346,7 @@ extends: Kickoff docker release - stage: UpdateDotnetDocker - # dependsOn: PublishGitHubRelease + dependsOn: PublishGitHubRelease displayName: Update DotNet SDK Docker images jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -361,7 +361,7 @@ extends: 4. create PR targeting nightly branch - stage: UpdateWinGet - # dependsOn: PublishGitHubRelease + dependsOn: PublishGitHubRelease displayName: Add manifest entry to winget jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -372,7 +372,7 @@ extends: This is typically done by the community 1-2 days after the release. - stage: PublishMsix - # dependsOn: PublishGitHubRelease + dependsOn: PublishGitHubRelease displayName: Publish MSIX to store jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -383,7 +383,7 @@ extends: Ask Steve to release MSIX bundle package to Store - stage: PublishVPack - # dependsOn: PublishGitHubRelease + dependsOn: PublishGitHubRelease displayName: Release vPack jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -401,13 +401,13 @@ extends: # - template: templates/release-UpdateDepsJson.yml - stage: UploadBuildInfoJson - # dependsOn: PublishGitHubRelease + dependsOn: PublishGitHubRelease displayName: Upload BuildInfo.json jobs: - template: /.pipelines/templates/release-upload-buildinfo.yml@self - stage: ReleaseSymbols - # dependsOn: PublishGitHubRelease + dependsOn: PublishGitHubRelease displayName: Release Symbols jobs: - template: /.pipelines/templates/release-symbols.yml@self diff --git a/.pipelines/templates/release-publish-nuget.yml b/.pipelines/templates/release-publish-nuget.yml index 2e6134e9e10..35e2fb8999d 100644 --- a/.pipelines/templates/release-publish-nuget.yml +++ b/.pipelines/templates/release-publish-nuget.yml @@ -106,11 +106,11 @@ jobs: displayName: Download and capture nupkgs condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) - # - task: NuGetCommand@2 - # displayName: 'NuGet push' - # condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) - # inputs: - # command: push - # packagesToPush: '$(Pipeline.Workspace)/release/*.nupkg' - # nuGetFeedType: external - # publishFeedCredentials: PowerShellNuGetOrgPush + - task: NuGetCommand@2 + displayName: 'NuGet push' + condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) + inputs: + command: push + packagesToPush: '$(Pipeline.Workspace)/release/*.nupkg' + nuGetFeedType: external + publishFeedCredentials: PowerShellNuGetOrgPush From ee4c0f3011a797b2966b0320dc3f9a665fb36d07 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 28 Oct 2024 18:51:53 +0000 Subject: [PATCH 106/316] Added Deploy Box Product Pathway to Github Release and Nuget Release Pipelines --- .pipelines/PowerShell-Release-Official.yml | 40 ++- .pipelines/templates/release-githubtasks.yml | 262 ++++++++++-------- .../templates/release-publish-nuget.yml | 67 ++++- .pipelines/templates/uploadToAzure.yml | 20 ++ 4 files changed, 252 insertions(+), 137 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 2538d1b5370..c77e6c8bb83 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -79,12 +79,18 @@ resources: extends: template: v2/OneBranch.Official.CrossPlat.yml@templates parameters: + release: + category: NonAzure # still using KS2 because we are not yet using a Box Product Deployment + # featureFlags: + # LinuxHostVersion: + # Network: KS2 + # WindowsHostVersion: + # Network: KS2 featureFlags: - LinuxHostVersion: - Network: KS2 WindowsHostVersion: - Network: KS2 + Version: 2022 + Network: Netlock cloudvault: enabled: false globalSdl: @@ -112,6 +118,8 @@ extends: stages: - stage: msixbundle displayName: 'Create MSIX Bundle' + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/release-create-msix.yml@self @@ -273,12 +281,38 @@ extends: - stage: PublishGitHubRelease displayName: Publish GitHub Release dependsOn: BlobPublic + variables: + ob_release_environment: Production jobs: - template: /.pipelines/templates/release-githubtasks.yml@self + + - stage: PushGitTag + displayName: Push Git Tag + dependsOn: PublishGitHubRelease + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Push Git Tag + jobName: PushGitTag + instructions: | + Push the git tag to upstream + + - stage: MakeDraftPublic + displayName: Make Draft Public + dependsOn: PushGitTag + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Make Draft Public + jobName: DraftPublic + instructions: | + Make the GitHub Release Draft Public - stage: PublishNuGet displayName: Publish NuGet dependsOn: PublishGitHubRelease + variables: + ob_release_environment: Production jobs: - template: /.pipelines/templates/release-publish-nuget.yml@self parameters: diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 039c171290f..3df461d11c7 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -3,7 +3,13 @@ jobs: displayName: Create GitHub Release Draft condition: succeeded() pool: - type: windows + type: release + os: windows + templateContext: + inputs: + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_upload_upload_packages variables: - name: runCodesignValidationInjection value: false @@ -26,139 +32,149 @@ jobs: value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: - - checkout: self - clean: true - env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - - - template: release-SetReleaseTagAndContainerName.yml - - - pwsh: | - Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose - displayName: 'Capture Environment Variables' - - - pwsh: | - # Uninstall Azure RM modules - $azRmModules = Get-Module -Name AzureRM* -ListAvailable - if ($azRmModules) { - $azRmModules | Remove-Module -Force - } - - # Install Az.Storage module if not already installed - if (-not (Get-Module -Name Az.Storage -ListAvailable)) { - Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose - } - displayName: Install Az.Storage module + - task: PowerShell@2 + inputs: + targetType: inline + script: | + $localInstallerPath = Get-ChildItem -Path "$(Pipeline.Workspace)/GitHubPackages" -Filter '*win-x64.msi' | Select-Object -First 1 -ExpandProperty FullName + if (Test-Path -Path $localInstallerPath) { + Write-Verbose -Verbose "Installer found at $localInstallerPath" + } else { + throw "Installer not found" + } + Write-Verbose -Verbose "Installing PowerShell via msiexec" + Start-Process -FilePath msiexec -ArgumentList "/package $localInstallerPath /quiet REGISTER_MANIFEST=1" -Wait -NoNewWindow + $pwshPath = Get-ChildItem -Directory -Path 'C:\Program Files\PowerShell\7*' | Select-Object -First 1 -ExpandProperty FullName + if (Test-Path -Path $pwshPath) { + Write-Verbose -Verbose "PowerShell installed at $pwshPath" + Write-Verbose -Verbose "Adding pwsh to env:PATH" + Write-Host "##vso[task.prependpath]$pwshPath" + } else { + throw "PowerShell not installed" + } + displayName: Install pwsh 7 - - task: AzurePowerShell@5 - displayName: Download packages from Azure Storage + - task: PowerShell@2 inputs: - azureSubscription: az-blob-cicd-infra - scriptType: inlineScript - azurePowerShellVersion: LatestVersion + targetType: inline pwsh: true - inline: | - $storageAccount = "$(StorageAccount)" - $containerName = "$(AzureVersion)" - $destinationPath = "$(System.ArtifactsDirectory)" - - # Get storage account context - $storageContext = New-AzStorageContext -StorageAccountName $storageAccount - - $blobList = Get-AzStorageBlob -Container $containerName -Context $storageContext - foreach ($blob in $blobList) { - $blobName = $blob.Name - $destinationFile = Join-Path -Path $destinationPath -ChildPath $blobName - Get-AzStorageBlobContent -Container $containerName -Blob $blobName -Destination $destinationFile -Context $storageContext -Force - Write-Output "Downloaded $blobName to $destinationFile" + script: | + Write-Verbose -Verbose "Pwsh 7 Installed" + Write-Verbose -Verbose "env:Path: " + $env:PATH -split ';' | ForEach-Object { + Write-Verbose -Verbose $_ } + displayName: Check pwsh 7 installation - $packagesPath = Get-ChildItem -Path $destinationPath\*.deb -Recurse -File | Select-Object -First 1 -ExpandProperty DirectoryName - Write-Host "sending -- vso[task.setvariable variable=PackagesRoot]$packagesPath" - Write-Host "##vso[task.setvariable variable=PackagesRoot]$packagesPath" + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Write-Verbose -Verbose "Deploy Box Product Pathway Does Not Support the `"checkout`" task" + git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' + cd $(Pipeline.Workspace)/PowerShell + $branchParts = $ENV:BUILD_SOURCEBRANCH -split '/' + $branchName = "$($branchParts[-2])/$($branchParts[-1])" + git checkout $branchName + displayName: Checkout repository - - pwsh: | - Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty FullName - displayName: Capture downloaded artifacts + - template: release-SetReleaseTagAndContainerName.yml - - pwsh: | - git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + displayName: 'Capture Environment Variables' + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' displayName: Clone Internal-Tools repository - - pwsh: | - $Path = "$(System.ArtifactsDirectory)" - $OutputPath = Join-Path $Path 'hashes.sha256' - $srcPaths = @($Path) - $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File - $checksums = $packages | - ForEach-Object { - Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" - $packageName = $_.Name - $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() - # the '*' before the packagename signifies it is a binary - "$hash *$packageName" - } - $checksums | Out-File -FilePath $OutputPath -Force - $fileContent = Get-Content -Path $OutputPath -Raw | Out-String - Write-Verbose -Verbose -Message $fileContent + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + $Path = "$(Pipeline.Workspace)/GitHubPackages" + $OutputPath = Join-Path $Path 'hashes.sha256' + $srcPaths = @($Path) + $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File + $checksums = $packages | + ForEach-Object { + Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" + $packageName = $_.Name + $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() + # the '*' before the packagename signifies it is a binary + "$hash *$packageName" + } + $checksums | Out-File -FilePath $OutputPath -Force + $fileContent = Get-Content -Path $OutputPath -Raw | Out-String + Write-Verbose -Verbose -Message $fileContent displayName: Add sha256 hashes - - pwsh: | - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" displayName: 'Set release version' + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Get-ChildItem $(Pipeline.Workspace) -recurse | Select-Object -ExpandProperty FullName + displayName: List all files in the workspace - - pwsh: | - Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion - - $isPreview = $semanticVersion.PreReleaseLabel -ne $null - - $fileName = if ($isPreview) { - "preview.md" - } - else { - $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" - } - - $filePath = "$env:BUILD_SOURCESDIRECTORY/PowerShell/CHANGELOG/$fileName" - Write-Verbose -Verbose "Selected Log file: $filePath" - - if (-not (Test-Path $filePath)) { - throw "$filePath not found" - } - - $changelog = Get-Content -Path $filePath - - $startPattern = "^## \[" + ([regex]::Escape($releaseVersion)) + "\]" - $endPattern = "^## \[{0}\.{1}\.{2}*" -f $semanticVersion.Major, $semanticVersion.Minor, $semanticVersion.Patch - - $clContent = $changelog | ForEach-Object { - if ($_ -match $startPattern) { $outputLine = $true } - elseif ($_ -match $endPattern) { $outputLine = $false } - if ($outputLine) { $_} - } | Out-String - - Write-Verbose -Verbose "Selected content: `n$clContent" - - Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder $(PackagesRoot) -Token $(GitHubReleasePat) + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' + $releaseVersion = '$(ReleaseTag)' -replace '^v','' + $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion + + $isPreview = $semanticVersion.PreReleaseLabel -ne $null + + $fileName = if ($isPreview) { + "preview.md" + } + else { + $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" + } + + $filePath = "$(Pipeline.Workspace)/PowerShell/CHANGELOG/$fileName" + Write-Verbose -Verbose "Selected Log file: $filePath" + + if (-not (Test-Path $filePath)) { + throw "$filePath not found" + } + + $changelog = Get-Content -Path $filePath + + $startPattern = "^## \[" + ([regex]::Escape($releaseVersion)) + "\]" + $endPattern = "^## \[{0}\.{1}\.{2}*" -f $semanticVersion.Major, $semanticVersion.Minor, $semanticVersion.Patch + + $clContent = $changelog | ForEach-Object { + if ($_ -match $startPattern) { $outputLine = $true } + elseif ($_ -match $endPattern) { $outputLine = $false } + if ($outputLine) { $_} + } | Out-String + + Write-Verbose -Verbose "Selected content: `n$clContent" + + Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description $clContent -User PowerShell -Repository PowerShell -PackageFolder "$(Pipeline.Workspace)/GitHubPackages" -Token $(GitHubReleasePat) displayName: Publish Release Draft - -- template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Push Git Tag - jobName: PushGitTag - dependsOnJob: GithubReleaseDraft - instructions: | - Push the git tag to upstream - -- template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Make Draft Public - jobName: DraftPublic - dependsOnJob: PushGitTag - instructions: | - Make the GitHub Release Draft Public diff --git a/.pipelines/templates/release-publish-nuget.yml b/.pipelines/templates/release-publish-nuget.yml index 9425dc96c64..35e2fb8999d 100644 --- a/.pipelines/templates/release-publish-nuget.yml +++ b/.pipelines/templates/release-publish-nuget.yml @@ -8,7 +8,13 @@ jobs: displayName: Publish to NuGet condition: succeeded() pool: - type: windows + type: release + os: windows + templateContext: + inputs: + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_upload_upload_packages variables: - name: runCodesignValidationInjection value: false @@ -29,10 +35,52 @@ jobs: value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: - - checkout: self - clean: true - env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + - task: PowerShell@2 + inputs: + targetType: inline + script: | + $localInstallerPath = Get-ChildItem -Path "$(Pipeline.Workspace)/GitHubPackages" -Filter '*win-x64.msi' | Select-Object -First 1 -ExpandProperty FullName + if (Test-Path -Path $localInstallerPath) { + Write-Verbose -Verbose "Installer found at $localInstallerPath" + } else { + throw "Installer not found" + } + Write-Verbose -Verbose "Installing PowerShell via msiexec" + Start-Process -FilePath msiexec -ArgumentList "/package $localInstallerPath /quiet REGISTER_MANIFEST=1" -Wait -NoNewWindow + $pwshPath = Get-ChildItem -Directory -Path 'C:\Program Files\PowerShell\7*' | Select-Object -First 1 -ExpandProperty FullName + if (Test-Path -Path $pwshPath) { + Write-Verbose -Verbose "PowerShell installed at $pwshPath" + Write-Verbose -Verbose "Adding pwsh to env:PATH" + Write-Host "##vso[task.prependpath]$pwshPath" + } else { + throw "PowerShell not installed" + } + displayName: Install pwsh 7 + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Write-Verbose -Verbose "Pwsh 7 Installed" + Write-Verbose -Verbose "env:Path: " + $env:PATH -split ';' | ForEach-Object { + Write-Verbose -Verbose $_ + } + displayName: Check pwsh 7 installation + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Write-Verbose -Verbose "Deploy Box Product Pathway Does Not Support the `"checkout`" task" + git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' + cd $(Pipeline.Workspace)/PowerShell + $branchParts = $ENV:BUILD_SOURCEBRANCH -split '/' + $branchName = "$($branchParts[-2])/$($branchParts[-1])" + git checkout $branchName + displayName: Checkout repository - template: release-SetReleaseTagAndContainerName.yml @@ -40,23 +88,20 @@ jobs: Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' - - download: PSPackagesOfficial - artifact: drop_nupkg_build_nupkg - displayName: Download nuget packages - - pwsh: | #Exclude all global tool packages. Their names start with 'PowerShell.' $null = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/release" - Copy-Item "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/drop_nupkg_build_nupkg/*.nupkg" -Destination "$(Pipeline.Workspace)/release" -Exclude "PowerShell.*.nupkg" -Force -Verbose + Copy-Item "$(Pipeline.Workspace)/NuGetPackages/*.nupkg" -Destination "$(Pipeline.Workspace)/release" -Exclude "PowerShell.*.nupkg" -Force -Verbose $releaseVersion = '$(VERSION)' - $globalToolPath = "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/drop_nupkg_build_nupkg/PowerShell.$releaseVersion.nupkg" + $globalToolPath = "$(Pipeline.Workspace)/NuGetPackages/PowerShell.$releaseVersion.nupkg" if ($releaseVersion -notlike '*-*') { # Copy the global tool package for stable releases Copy-Item $globalToolPath -Destination "$(Pipeline.Workspace)/release" } + Write-Verbose -Verbose "The .nupkgs below will be pushed:" Get-ChildItem "$(Pipeline.Workspace)/release" -recurse displayName: Download and capture nupkgs condition: and(ne('${{ parameters.skipPublish }}', 'false'), succeeded()) diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index 174b5810de0..7c7483f8893 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -5,6 +5,8 @@ jobs: pool: type: windows variables: + - name: ob_sdl_sbom_enabled + value: false - name: runCodesignValidationInjection value: false - name: NugetSecurityAnalysisWarningLevel @@ -233,6 +235,24 @@ jobs: Get-ChildItem '$(Build.ArtifactStagingDirectory)/downloads' | Select-Object -ExpandProperty FullName displayName: 'Capture downloads' + - pwsh: | + Write-Verbose -Verbose "Copying Github Release files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" + + Write-Verbose -Verbose "Creating output directory for GitHub Release files: $(ob_outputDirectory)/GitHubPackages" + New-Item -Path $(ob_outputDirectory)/GitHubPackages -ItemType Directory -Force + Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | + Where-Object { $_.Extension -notin '.msix', '.nupkg' } | + ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | + Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse + + Write-Verbose -Verbose "Creating output directory for NuGet packages: $(ob_outputDirectory)/NuGetPackages" + New-Item -Path $(ob_outputDirectory)/NuGetPackages -ItemType Directory -Force + Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | + Where-Object { $_.Extension -eq '.nupkg' } | + ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | + Copy-Item -Destination $(ob_outputDirectory)/NuGetPackages -Recurse + displayName: Copy downloads to Artifacts + - pwsh: | # Create output directory for packages which have been uploaded to blob storage New-Item -Path $(Build.ArtifactStagingDirectory)/uploaded -ItemType Directory -Force From ac6def35db4c1060598925720fb43632bac492cb Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 13 Nov 2024 16:24:22 -0600 Subject: [PATCH 107/316] Templatized pwsh install and repo checkout, clean ups --- .pipelines/PowerShell-Release-Official.yml | 6 --- .../templates/release-checkout-pwsh-repo.yml | 9 ++++ .pipelines/templates/release-githubtasks.yml | 49 ++----------------- .pipelines/templates/release-install-pwsh.yml | 34 +++++++++++++ .../templates/release-publish-nuget.yml | 47 +----------------- 5 files changed, 48 insertions(+), 97 deletions(-) create mode 100644 .pipelines/templates/release-checkout-pwsh-repo.yml create mode 100644 .pipelines/templates/release-install-pwsh.yml diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index c77e6c8bb83..871557abe04 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -81,12 +81,6 @@ extends: parameters: release: category: NonAzure - # still using KS2 because we are not yet using a Box Product Deployment - # featureFlags: - # LinuxHostVersion: - # Network: KS2 - # WindowsHostVersion: - # Network: KS2 featureFlags: WindowsHostVersion: Version: 2022 diff --git a/.pipelines/templates/release-checkout-pwsh-repo.yml b/.pipelines/templates/release-checkout-pwsh-repo.yml new file mode 100644 index 00000000000..9cbcde606af --- /dev/null +++ b/.pipelines/templates/release-checkout-pwsh-repo.yml @@ -0,0 +1,9 @@ +steps: + - pwsh: | + Write-Verbose -Verbose "Deploy Box Product Pathway Does Not Support the `"checkout`" task" + git clone --depth 1 https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' + cd $(Pipeline.Workspace)/PowerShell + $branchParts = $ENV:BUILD_SOURCEBRANCH -split '/' + $branchName = "$($branchParts[-2])/$($branchParts[-1])" + git checkout $branchName + displayName: Checkout Powershell Repository diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 3df461d11c7..fd96fc4ee97 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -32,52 +32,9 @@ jobs: value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: - - task: PowerShell@2 - inputs: - targetType: inline - script: | - $localInstallerPath = Get-ChildItem -Path "$(Pipeline.Workspace)/GitHubPackages" -Filter '*win-x64.msi' | Select-Object -First 1 -ExpandProperty FullName - if (Test-Path -Path $localInstallerPath) { - Write-Verbose -Verbose "Installer found at $localInstallerPath" - } else { - throw "Installer not found" - } - Write-Verbose -Verbose "Installing PowerShell via msiexec" - Start-Process -FilePath msiexec -ArgumentList "/package $localInstallerPath /quiet REGISTER_MANIFEST=1" -Wait -NoNewWindow - $pwshPath = Get-ChildItem -Directory -Path 'C:\Program Files\PowerShell\7*' | Select-Object -First 1 -ExpandProperty FullName - if (Test-Path -Path $pwshPath) { - Write-Verbose -Verbose "PowerShell installed at $pwshPath" - Write-Verbose -Verbose "Adding pwsh to env:PATH" - Write-Host "##vso[task.prependpath]$pwshPath" - } else { - throw "PowerShell not installed" - } - displayName: Install pwsh 7 - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - Write-Verbose -Verbose "Pwsh 7 Installed" - Write-Verbose -Verbose "env:Path: " - $env:PATH -split ';' | ForEach-Object { - Write-Verbose -Verbose $_ - } - displayName: Check pwsh 7 installation + - template: release-install-pwsh.yml - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - Write-Verbose -Verbose "Deploy Box Product Pathway Does Not Support the `"checkout`" task" - git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' - cd $(Pipeline.Workspace)/PowerShell - $branchParts = $ENV:BUILD_SOURCEBRANCH -split '/' - $branchName = "$($branchParts[-2])/$($branchParts[-1])" - git checkout $branchName - displayName: Checkout repository + - template: release-checkout-pwsh-repo.yml - template: release-SetReleaseTagAndContainerName.yml @@ -94,7 +51,7 @@ jobs: targetType: inline pwsh: true script: | - git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' + git clone --depth 1 https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' displayName: Clone Internal-Tools repository - task: PowerShell@2 diff --git a/.pipelines/templates/release-install-pwsh.yml b/.pipelines/templates/release-install-pwsh.yml new file mode 100644 index 00000000000..9d7080a7e78 --- /dev/null +++ b/.pipelines/templates/release-install-pwsh.yml @@ -0,0 +1,34 @@ +steps: + - task: PowerShell@2 + inputs: + targetType: inline + script: | + $localInstallerPath = Get-ChildItem -Path "$(Pipeline.Workspace)/GitHubPackages" -Filter '*win-x64.msi' | Select-Object -First 1 -ExpandProperty FullName + if (Test-Path -Path $localInstallerPath) { + Write-Verbose -Verbose "Installer found at $localInstallerPath" + } else { + throw "Installer not found" + } + Write-Verbose -Verbose "Installing PowerShell via msiexec" + Start-Process -FilePath msiexec -ArgumentList "/package $localInstallerPath /quiet REGISTER_MANIFEST=1" -Wait -NoNewWindow + $pwshPath = Get-ChildItem -Directory -Path 'C:\Program Files\PowerShell\7*' | Select-Object -First 1 -ExpandProperty FullName + if (Test-Path -Path $pwshPath) { + Write-Verbose -Verbose "PowerShell installed at $pwshPath" + Write-Verbose -Verbose "Adding pwsh to env:PATH" + Write-Host "##vso[task.prependpath]$pwshPath" + } else { + throw "PowerShell not installed" + } + displayName: Install pwsh 7 + + - task: PowerShell@2 + inputs: + targetType: inline + pwsh: true + script: | + Write-Verbose -Verbose "Pwsh 7 Installed" + Write-Verbose -Verbose "env:Path: " + $env:PATH -split ';' | ForEach-Object { + Write-Verbose -Verbose $_ + } + displayName: Check pwsh 7 installation diff --git a/.pipelines/templates/release-publish-nuget.yml b/.pipelines/templates/release-publish-nuget.yml index 35e2fb8999d..eeff1a894a6 100644 --- a/.pipelines/templates/release-publish-nuget.yml +++ b/.pipelines/templates/release-publish-nuget.yml @@ -35,52 +35,9 @@ jobs: value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: - - task: PowerShell@2 - inputs: - targetType: inline - script: | - $localInstallerPath = Get-ChildItem -Path "$(Pipeline.Workspace)/GitHubPackages" -Filter '*win-x64.msi' | Select-Object -First 1 -ExpandProperty FullName - if (Test-Path -Path $localInstallerPath) { - Write-Verbose -Verbose "Installer found at $localInstallerPath" - } else { - throw "Installer not found" - } - Write-Verbose -Verbose "Installing PowerShell via msiexec" - Start-Process -FilePath msiexec -ArgumentList "/package $localInstallerPath /quiet REGISTER_MANIFEST=1" -Wait -NoNewWindow - $pwshPath = Get-ChildItem -Directory -Path 'C:\Program Files\PowerShell\7*' | Select-Object -First 1 -ExpandProperty FullName - if (Test-Path -Path $pwshPath) { - Write-Verbose -Verbose "PowerShell installed at $pwshPath" - Write-Verbose -Verbose "Adding pwsh to env:PATH" - Write-Host "##vso[task.prependpath]$pwshPath" - } else { - throw "PowerShell not installed" - } - displayName: Install pwsh 7 - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - Write-Verbose -Verbose "Pwsh 7 Installed" - Write-Verbose -Verbose "env:Path: " - $env:PATH -split ';' | ForEach-Object { - Write-Verbose -Verbose $_ - } - displayName: Check pwsh 7 installation + - template: release-install-pwsh.yml - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - Write-Verbose -Verbose "Deploy Box Product Pathway Does Not Support the `"checkout`" task" - git clone https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' - cd $(Pipeline.Workspace)/PowerShell - $branchParts = $ENV:BUILD_SOURCEBRANCH -split '/' - $branchName = "$($branchParts[-2])/$($branchParts[-1])" - git checkout $branchName - displayName: Checkout repository + - template: release-checkout-pwsh-repo.yml - template: release-SetReleaseTagAndContainerName.yml From 3215a02ecd0ffa3cbb352684fc5c32b8b822dabe Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 30 Oct 2024 17:16:30 -0700 Subject: [PATCH 108/316] Fix seed max value for Container Linux CI (#24510) --- tools/ci.psm1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/ci.psm1 b/tools/ci.psm1 index 73c6cb41e83..6628d54e043 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -863,11 +863,17 @@ function Invoke-InitializeContainerStage { # For PRs set the seed to the PR number so that the image is always the same $seed = $env:SYSTEM_PULLREQUEST_PULLREQUESTID + if(!$seed) { # for non-PRs use the integer identifier of the build as the seed. $seed = $fallbackSeed } + # cut down to 32 bits and keep the most varying parts, which are lower bits + if ($seed -ge [Int32]::MaxValue) { + $seed = [int]($seed -band [int]::MaxValue) + } + Write-Verbose "Seed: $seed" -Verbose # Get the latest image matrix JSON for preview From 4637af11114d189870b650a3a5976bad967a15b1 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 30 Oct 2024 18:21:09 -0700 Subject: [PATCH 109/316] Bump .NET to 9.0.100-rc.2.24474.11 (#24509) --- global.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global.json b/global.json index 0519736ab95..2dd360f846e 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "9.0.100-rc.1.24452.12" + "version": "9.0.100-rc.2.24474.11" } } From 6db98a48879da0899c44e0fcd8df31630c09edea Mon Sep 17 00:00:00 2001 From: alerickson <25858831+alerickson@users.noreply.github.com> Date: Thu, 31 Oct 2024 00:27:47 -0700 Subject: [PATCH 110/316] Update PSResourceGet to v1.1.0-RC2 (#24512) * Update PSResourceGet to v1.1.0-RC2 * Fix windows packaging * Update windows bom * Update version string to be lowercase --------- Co-authored-by: Aditya Patwardhan --- src/Modules/PSGalleryModules.csproj | 2 +- tools/packaging/boms/windows.json | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Modules/PSGalleryModules.csproj b/src/Modules/PSGalleryModules.csproj index e105facf8e4..9df1121f38b 100644 --- a/src/Modules/PSGalleryModules.csproj +++ b/src/Modules/PSGalleryModules.csproj @@ -13,7 +13,7 @@ - + diff --git a/tools/packaging/boms/windows.json b/tools/packaging/boms/windows.json index c467adbbfa9..c9fd280930f 100644 --- a/tools/packaging/boms/windows.json +++ b/tools/packaging/boms/windows.json @@ -787,14 +787,6 @@ "Pattern": "Microsoft.WSMan.Runtime.xml", "FileType": "NonProduct" }, - { - "Pattern": "Modules/*.json", - "FileType": "NonProduct" - }, - { - "Pattern": "Modules/*.sha256", - "FileType": "NonProduct" - }, { "Pattern": "Modules/Microsoft.PowerShell.Archive/*.cat", "FileType": "NonProduct" @@ -900,7 +892,11 @@ "FileType": "NonProduct" }, { - "Pattern": "Modules\\PSReadLine\\_manifest\\spdx_2.2\\manifest.cat", + "Pattern": "Modules\\Microsoft.PowerShell.PSResourceGet\\PSResourceRepository.adml", + "FileType": "NonProduct" + }, + { + "Pattern": "Modules\\Microsoft.PowerShell.PSResourceGet\\PSResourceRepository.admx", "FileType": "NonProduct" }, { @@ -3455,6 +3451,10 @@ "Pattern": "Modules/PSDiagnostics/PSDiagnostics.psm1", "FileType": "Product" }, + { + "Pattern": "Modules\\Microsoft.PowerShell.PSResourceGet\\InstallPSResourceGetPolicyDefinitions.ps1", + "FileType": "Product" + }, { "Pattern": "pwsh.dll", "FileType": "Product" From 4ade2d4bd6ef439156a1b3e6610de93fede10a05 Mon Sep 17 00:00:00 2001 From: Patrick Meinecke Date: Thu, 31 Oct 2024 12:19:52 -0400 Subject: [PATCH 111/316] Update changelog for v7.4.6 release (#24496) --- CHANGELOG/7.4.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/CHANGELOG/7.4.md b/CHANGELOG/7.4.md index a117e3a41b6..91ac95caa86 100644 --- a/CHANGELOG/7.4.md +++ b/CHANGELOG/7.4.md @@ -1,5 +1,48 @@ # 7.4 Changelog +## [7.4.6] - 2024-10-22 + +### Build and Packaging Improvements + +
+ + + +

Bump .NET SDK to 8.0.403

+ + + +
    +
  • Copy to static site instead of making blob public (#24269) (#24473)
    • +
  • Add ability to capture MSBuild Binary logs when restore fails (#24128)
    • +
  • Keep the roff file when gzipping it. (#24450)
    • +
  • Update PowerShell-Coordinated_Packages-Official.yml (#24449)
    • +
  • Update and add new NuGet package sources for different environments. (#24440)
    • +
  • Add PMC mapping for Debian 12 (bookworm) (#24413)
    • +
  • Fixes to Azure Public feed usage (#24429)
    • +
  • Delete assets/AppImageThirdPartyNotices.txt (#24256)
    • +
  • Delete demos directory (#24258)
    • +
  • Add specific path for issues in tsaconfig (#24244)
    • +
  • Checkin generated manpage (#24423)
    • +
  • Add updated libicu dependency for Debian packages (#24301)
    • +
  • Add mapping to azurelinux repo (#24290)
    • +
  • Update vpack pipeline (#24281)
    • +
  • Add BaseUrl to buildinfo json file (#24376)
    • +
  • Delete the msix blob if it's already there (#24353)
    • +
  • Make some release tests run in a hosted pools (#24270)
    • +
  • Create new pipeline for compliance (#24252)
    • +
  • Use Managed Identity for APIScan authentication (#24243)
    • +
  • Check Create and Submit in vPack build by default (#24181)
    • +
  • Capture environment better (#24148)
    • +
  • Refactor Nuget package source creation to use New-NugetPackageSource function (#24104)
    • +
  • Make Microsoft feeds the default (#24426)
    • +
  • Bump to .NET 8.0.403 and update dependencies (#24405)
    • +
+ +
+ +[7.4.6]: https://github.com/PowerShell/PowerShell/compare/v7.4.5...v7.4.6 + ## [7.4.5] - 2024-08-20 ### General Cmdlet Updates and Fixes From 43cbbdf6e7d275d484f792f24276719a71a97bf7 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 31 Oct 2024 13:13:04 -0700 Subject: [PATCH 112/316] Add a way to use only NuGet feed sources (#24528) * Update build.psm1 * update default feeds * fix issue with cgmanifest having duplicate existing entries * update pattern --- build.psm1 | 15 +++++++++------ nuget.config | 2 +- src/Modules/nuget.config | 2 +- test/tools/Modules/nuget.config | 2 +- tools/findMissingNotices.ps1 | 6 ++++-- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/build.psm1 b/build.psm1 index 7855e46ac1a..e21194c9af9 100644 --- a/build.psm1 +++ b/build.psm1 @@ -740,7 +740,7 @@ function Switch-PSNugetConfig { param( [Parameter(Mandatory = $true, ParameterSetName = 'user')] [Parameter(Mandatory = $true, ParameterSetName = 'nouser')] - [ValidateSet('Public', 'Private')] + [ValidateSet('Public', 'Private', 'NuGetOnly')] [string] $Source, [Parameter(Mandatory = $true, ParameterSetName = 'user')] @@ -760,16 +760,19 @@ function Switch-PSNugetConfig { } } + $dotnetSdk = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v2'; Name = 'dotnet' } + $gallery = [NugetPackageSource] @{Url = 'https://www.powershellgallery.com/api/v2/'; Name = 'psgallery' } + $nugetorg = [NugetPackageSource] @{Url = 'https://api.nuget.org/v3/index.json'; Name = 'nuget.org' } if ( $Source -eq 'Public') { - $dotnetSdk = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v2'; Name = 'dotnet' } - $gallery = [NugetPackageSource] @{Url = 'https://www.powershellgallery.com/api/v2/'; Name = 'psgallery' } - $nugetorg = [NugetPackageSource] @{Url = 'https://api.nuget.org/v3/index.json'; Name = 'nuget.org' } - New-NugetConfigFile -NugetPackageSource $nugetorg, $dotnetSdk -Destination "$PSScriptRoot/" @extraParams New-NugetConfigFile -NugetPackageSource $gallery -Destination "$PSScriptRoot/src/Modules/" @extraParams New-NugetConfigFile -NugetPackageSource $gallery -Destination "$PSScriptRoot/test/tools/Modules/" @extraParams + } elseif ( $Source -eq 'NuGetOnly') { + New-NugetConfigFile -NugetPackageSource $nugetorg -Destination "$PSScriptRoot/" @extraParams + New-NugetConfigFile -NugetPackageSource $gallery -Destination "$PSScriptRoot/src/Modules/" @extraParams + New-NugetConfigFile -NugetPackageSource $gallery -Destination "$PSScriptRoot/test/tools/Modules/" @extraParams } elseif ( $Source -eq 'Private') { - $powerShellPackages = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/PowerShell-7-5-preview-test-2/nuget/v3/index.json'; Name = 'powershell' } + $powerShellPackages = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/PowerShell/nuget/v3/index.json'; Name = 'powershell' } New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/" @extraParams New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/src/Modules/" @extraParams diff --git a/nuget.config b/nuget.config index 3ca2bee3c18..388a65572dd 100644 --- a/nuget.config +++ b/nuget.config @@ -2,7 +2,7 @@ - + diff --git a/src/Modules/nuget.config b/src/Modules/nuget.config index 3ca2bee3c18..388a65572dd 100644 --- a/src/Modules/nuget.config +++ b/src/Modules/nuget.config @@ -2,7 +2,7 @@ - + diff --git a/test/tools/Modules/nuget.config b/test/tools/Modules/nuget.config index 3ca2bee3c18..388a65572dd 100644 --- a/test/tools/Modules/nuget.config +++ b/test/tools/Modules/nuget.config @@ -2,7 +2,7 @@ - + diff --git a/tools/findMissingNotices.ps1 b/tools/findMissingNotices.ps1 index 2915853ec38..490edebb81b 100644 --- a/tools/findMissingNotices.ps1 +++ b/tools/findMissingNotices.ps1 @@ -26,7 +26,9 @@ $existingRegistrationsJson.Registrations | ForEach-Object { $registration = [Registration]$_ if ($registration.Component) { $name = $registration.Component.Name() - $existingRegistrationTable.Add($name, $registration) + if (!$existingRegistrationTable.ContainsKey($name)) { + $existingRegistrationTable.Add($name, $registration) + } } } @@ -103,7 +105,7 @@ function ConvertTo-SemVer { So, I'm making the logic work for that scenario by thorwing away any part that doesn't match non-pre-release semver portion #> - $null = $Version -match '^(\d+\.\d+\.\d+)).*' + $null = $Version -match '^(\d+\.\d+\.\d+).*' $desiredVersion = $matches[1] } From d7b53bec8390d3a371665eea4c3eb3aa98a07785 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Thu, 31 Oct 2024 13:29:49 -0700 Subject: [PATCH 113/316] Revert "Update package references (#24414)" (#24532) This reverts commit 01ae63263661351c812c8ab5fee3a6cceaa559c0. --- ...oft.PowerShell.Commands.Diagnostics.csproj | 2 +- ...soft.PowerShell.Commands.Management.csproj | 2 +- ...crosoft.PowerShell.Commands.Utility.csproj | 4 ++-- ...crosoft.PowerShell.CoreCLR.Eventing.csproj | 2 +- .../Microsoft.PowerShell.SDK.csproj | 10 +++++----- .../Microsoft.WSMan.Management.csproj | 2 +- .../PSVersionInfoGenerator.csproj | 2 +- .../System.Management.Automation.csproj | 20 ++++++++++--------- test/tools/TestService/TestService.csproj | 4 ++-- 9 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj b/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj index 34502ced25d..9f36b8134d1 100644 --- a/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj +++ b/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj index 4f20755c6e6..695ed6f21e4 100644 --- a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj +++ b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj @@ -47,7 +47,7 @@ - + diff --git a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj index dd2bf636955..113d62231f0 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj +++ b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj @@ -33,8 +33,8 @@ - - + + diff --git a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj index 003999e97b3..1296b1e28ba 100644 --- a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj +++ b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj index 5a1001ddc2c..34c462edbd4 100644 --- a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj +++ b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj @@ -17,13 +17,13 @@ - + - - - + + + - + diff --git a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj index 77aa3cef77c..9a8f4973d1a 100644 --- a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj +++ b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/System.Management.Automation/SourceGenerators/PSVersionInfoGenerator/PSVersionInfoGenerator.csproj b/src/System.Management.Automation/SourceGenerators/PSVersionInfoGenerator/PSVersionInfoGenerator.csproj index 678824a6e8a..a8f91c5f9e5 100644 --- a/src/System.Management.Automation/SourceGenerators/PSVersionInfoGenerator/PSVersionInfoGenerator.csproj +++ b/src/System.Management.Automation/SourceGenerators/PSVersionInfoGenerator/PSVersionInfoGenerator.csproj @@ -15,6 +15,6 @@ - + diff --git a/src/System.Management.Automation/System.Management.Automation.csproj b/src/System.Management.Automation/System.Management.Automation.csproj index cefbbfa31f0..e4b0c5d7e48 100644 --- a/src/System.Management.Automation/System.Management.Automation.csproj +++ b/src/System.Management.Automation/System.Management.Automation.csproj @@ -19,7 +19,9 @@ - + @@ -32,16 +34,16 @@ - - - - + + + + - + - - - + + + diff --git a/test/tools/TestService/TestService.csproj b/test/tools/TestService/TestService.csproj index 56b17f3d7bc..f6ca75e1dae 100644 --- a/test/tools/TestService/TestService.csproj +++ b/test/tools/TestService/TestService.csproj @@ -15,8 +15,8 @@ - - + + From c3ad4383f8a0798cb73293e1f1c81d18ec36751f Mon Sep 17 00:00:00 2001 From: PowerShell Team Bot <69177312+pwshBot@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:15:28 -0700 Subject: [PATCH 114/316] [master] Update branch for release (#24534) * Update package references * Update cgmanifest * Update IsolatedModule.Tests.ps1 --------- Co-authored-by: Travis Plunk --- ...oft.PowerShell.Commands.Diagnostics.csproj | 2 +- ...soft.PowerShell.Commands.Management.csproj | 2 +- ...crosoft.PowerShell.Commands.Utility.csproj | 10 +- ...crosoft.PowerShell.CoreCLR.Eventing.csproj | 2 +- .../Microsoft.PowerShell.SDK.csproj | 10 +- .../Microsoft.WSMan.Management.csproj | 2 +- .../PSVersionInfoGenerator.csproj | 4 +- .../System.Management.Automation.csproj | 20 ++-- .../BenchmarkDotNet.Extensions.csproj | 4 +- .../dotnet-tools/Reporting/Reporting.csproj | 2 +- .../ResultsComparer/ResultsComparer.csproj | 4 +- .../engine/Module/IsolatedModule.Tests.ps1 | 3 + ...soft.PowerShell.NamedPipeConnection.csproj | 2 +- .../TestAlc/init/Test.Isolated.Init.csproj | 4 +- .../nested/Test.Isolated.Nested.csproj | 6 +- .../TestAlc/root/Test.Isolated.Root.csproj | 4 +- test/tools/TestService/TestService.csproj | 6 +- test/tools/WebListener/WebListener.csproj | 4 +- test/xUnit/xUnit.tests.csproj | 11 ++- tools/cgmanifest.json | 94 +++++++------------ 20 files changed, 85 insertions(+), 111 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj b/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj index 9f36b8134d1..34502ced25d 100644 --- a/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj +++ b/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj index 695ed6f21e4..4f20755c6e6 100644 --- a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj +++ b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj @@ -47,7 +47,7 @@ - + diff --git a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj index 113d62231f0..eee0929dada 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj +++ b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj @@ -8,7 +8,7 @@ - + @@ -32,10 +32,10 @@ - - - - + + + + diff --git a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj index 1296b1e28ba..003999e97b3 100644 --- a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj +++ b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj index 34c462edbd4..0f1fef9362b 100644 --- a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj +++ b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj @@ -17,13 +17,13 @@ - + - - - + + + - + diff --git a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj index 9a8f4973d1a..77aa3cef77c 100644 --- a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj +++ b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/System.Management.Automation/SourceGenerators/PSVersionInfoGenerator/PSVersionInfoGenerator.csproj b/src/System.Management.Automation/SourceGenerators/PSVersionInfoGenerator/PSVersionInfoGenerator.csproj index a8f91c5f9e5..5e7287debb6 100644 --- a/src/System.Management.Automation/SourceGenerators/PSVersionInfoGenerator/PSVersionInfoGenerator.csproj +++ b/src/System.Management.Automation/SourceGenerators/PSVersionInfoGenerator/PSVersionInfoGenerator.csproj @@ -14,7 +14,7 @@ - - + + diff --git a/src/System.Management.Automation/System.Management.Automation.csproj b/src/System.Management.Automation/System.Management.Automation.csproj index e4b0c5d7e48..cefbbfa31f0 100644 --- a/src/System.Management.Automation/System.Management.Automation.csproj +++ b/src/System.Management.Automation/System.Management.Automation.csproj @@ -19,9 +19,7 @@ - + @@ -34,16 +32,16 @@ - - - - + + + + - + - - - + + + diff --git a/test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj b/test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj index 2a398dad179..1852caa69c2 100644 --- a/test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj +++ b/test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj @@ -6,8 +6,8 @@ - - + + diff --git a/test/perf/dotnet-tools/Reporting/Reporting.csproj b/test/perf/dotnet-tools/Reporting/Reporting.csproj index 70447cf5d73..72a831b9406 100644 --- a/test/perf/dotnet-tools/Reporting/Reporting.csproj +++ b/test/perf/dotnet-tools/Reporting/Reporting.csproj @@ -6,7 +6,7 @@ - + diff --git a/test/perf/dotnet-tools/ResultsComparer/ResultsComparer.csproj b/test/perf/dotnet-tools/ResultsComparer/ResultsComparer.csproj index a8b48dde151..98a949de641 100644 --- a/test/perf/dotnet-tools/ResultsComparer/ResultsComparer.csproj +++ b/test/perf/dotnet-tools/ResultsComparer/ResultsComparer.csproj @@ -9,7 +9,7 @@ - - + + diff --git a/test/powershell/engine/Module/IsolatedModule.Tests.ps1 b/test/powershell/engine/Module/IsolatedModule.Tests.ps1 index 5d1289271de..00dae01838c 100644 --- a/test/powershell/engine/Module/IsolatedModule.Tests.ps1 +++ b/test/powershell/engine/Module/IsolatedModule.Tests.ps1 @@ -3,6 +3,9 @@ Describe "Isolated module scenario - load the whole module in custom ALC" -Tag 'CI' { It "Loading 'IsolatedModule' should work as expected" { + + Set-ItResult -Pending -Because "The test is failing as we cannot depend on Newtonsoft.Json v10.0.0 as it has security vulnerabilities." + ## The 'IsolatedModule' module can be found at '\test\tools\Modules'. ## The module assemblies are created and deployed by '\test\tools\TestAlc'. ## The module defines its own custom ALC and has its module structure organized in a special way that allows the module to be loaded in that custom ALC. diff --git a/test/tools/NamedPipeConnection/src/code/Microsoft.PowerShell.NamedPipeConnection.csproj b/test/tools/NamedPipeConnection/src/code/Microsoft.PowerShell.NamedPipeConnection.csproj index aa50d6ec179..b3d5b83a79e 100644 --- a/test/tools/NamedPipeConnection/src/code/Microsoft.PowerShell.NamedPipeConnection.csproj +++ b/test/tools/NamedPipeConnection/src/code/Microsoft.PowerShell.NamedPipeConnection.csproj @@ -15,6 +15,6 @@ - + diff --git a/test/tools/TestAlc/init/Test.Isolated.Init.csproj b/test/tools/TestAlc/init/Test.Isolated.Init.csproj index c1a291fa550..c8d37ac959a 100644 --- a/test/tools/TestAlc/init/Test.Isolated.Init.csproj +++ b/test/tools/TestAlc/init/Test.Isolated.Init.csproj @@ -1,6 +1,6 @@ - + @@ -15,7 +15,7 @@ - + diff --git a/test/tools/TestAlc/nested/Test.Isolated.Nested.csproj b/test/tools/TestAlc/nested/Test.Isolated.Nested.csproj index 85ea03a9c4f..cd9e87f5bed 100644 --- a/test/tools/TestAlc/nested/Test.Isolated.Nested.csproj +++ b/test/tools/TestAlc/nested/Test.Isolated.Nested.csproj @@ -1,6 +1,6 @@ - + @@ -16,8 +16,8 @@ - - + + diff --git a/test/tools/TestAlc/root/Test.Isolated.Root.csproj b/test/tools/TestAlc/root/Test.Isolated.Root.csproj index ab333e0668a..2a757064167 100644 --- a/test/tools/TestAlc/root/Test.Isolated.Root.csproj +++ b/test/tools/TestAlc/root/Test.Isolated.Root.csproj @@ -1,6 +1,6 @@ - + @@ -15,7 +15,7 @@ - + diff --git a/test/tools/TestService/TestService.csproj b/test/tools/TestService/TestService.csproj index f6ca75e1dae..94126cd2efa 100644 --- a/test/tools/TestService/TestService.csproj +++ b/test/tools/TestService/TestService.csproj @@ -1,6 +1,6 @@ - + Very tiny windows service to do service testing @@ -15,8 +15,8 @@ - - + + diff --git a/test/tools/WebListener/WebListener.csproj b/test/tools/WebListener/WebListener.csproj index 791be2fd228..22028234824 100644 --- a/test/tools/WebListener/WebListener.csproj +++ b/test/tools/WebListener/WebListener.csproj @@ -7,7 +7,7 @@ - - + + diff --git a/test/xUnit/xUnit.tests.csproj b/test/xUnit/xUnit.tests.csproj index b09abbf7483..f0827571094 100644 --- a/test/xUnit/xUnit.tests.csproj +++ b/test/xUnit/xUnit.tests.csproj @@ -23,11 +23,14 @@ - + - - - + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + diff --git a/tools/cgmanifest.json b/tools/cgmanifest.json index a1b0338c41f..32e9cc0110f 100644 --- a/tools/cgmanifest.json +++ b/tools/cgmanifest.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { @@ -55,7 +56,7 @@ "Type": "nuget", "Nuget": { "Name": "JsonSchema.Net", - "Version": "7.0.1" + "Version": "7.2.3" } }, "DevelopmentDependency": false @@ -65,7 +66,7 @@ "Type": "nuget", "Nuget": { "Name": "Markdig.Signed", - "Version": "0.37.0" + "Version": "0.38.0" } }, "DevelopmentDependency": false @@ -105,7 +106,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.CodeAnalysis.Common", - "Version": "4.9.2" + "Version": "4.11.0" } }, "DevelopmentDependency": false @@ -115,7 +116,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.CodeAnalysis.CSharp", - "Version": "4.9.2" + "Version": "4.11.0" } }, "DevelopmentDependency": false @@ -125,7 +126,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Extensions.ObjectPool", - "Version": "8.0.4" + "Version": "8.0.10" } }, "DevelopmentDependency": false @@ -205,7 +206,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Win32.Registry", - "Version": "4.7.0" + "Version": "5.0.0" } }, "DevelopmentDependency": false @@ -225,7 +226,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Windows.Compatibility", - "Version": "8.0.8" + "Version": "8.0.10" } }, "DevelopmentDependency": false @@ -245,7 +246,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.android-arm.runtime.native.System.IO.Ports", - "Version": "9.0.0-preview.3.24172.9" + "Version": "9.0.0-rc.2.24473.5" } }, "DevelopmentDependency": false @@ -255,7 +256,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.android-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.0-preview.3.24172.9" + "Version": "9.0.0-rc.2.24473.5" } }, "DevelopmentDependency": false @@ -265,7 +266,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.android-x64.runtime.native.System.IO.Ports", - "Version": "9.0.0-preview.3.24172.9" + "Version": "9.0.0-rc.2.24473.5" } }, "DevelopmentDependency": false @@ -275,7 +276,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.android-x86.runtime.native.System.IO.Ports", - "Version": "9.0.0-preview.3.24172.9" + "Version": "9.0.0-rc.2.24473.5" } }, "DevelopmentDependency": false @@ -305,7 +306,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-bionic-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.0-preview.3.24172.9" + "Version": "9.0.0-rc.2.24473.5" } }, "DevelopmentDependency": false @@ -315,7 +316,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-bionic-x64.runtime.native.System.IO.Ports", - "Version": "9.0.0-preview.3.24172.9" + "Version": "9.0.0-rc.2.24473.5" } }, "DevelopmentDependency": false @@ -325,7 +326,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-musl-arm.runtime.native.System.IO.Ports", - "Version": "9.0.0-preview.3.24172.9" + "Version": "9.0.0-rc.2.24473.5" } }, "DevelopmentDependency": false @@ -335,7 +336,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-musl-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.0-preview.3.24172.9" + "Version": "9.0.0-rc.2.24473.5" } }, "DevelopmentDependency": false @@ -345,7 +346,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-musl-x64.runtime.native.System.IO.Ports", - "Version": "9.0.0-preview.3.24172.9" + "Version": "9.0.0-rc.2.24473.5" } }, "DevelopmentDependency": false @@ -365,7 +366,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.maccatalyst-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.0-preview.3.24172.9" + "Version": "9.0.0-rc.2.24473.5" } }, "DevelopmentDependency": false @@ -375,7 +376,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.maccatalyst-x64.runtime.native.System.IO.Ports", - "Version": "9.0.0-preview.3.24172.9" + "Version": "9.0.0-rc.2.24473.5" } }, "DevelopmentDependency": false @@ -515,7 +516,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Configuration.ConfigurationManager", - "Version": "8.0.0" + "Version": "8.0.1" } }, "DevelopmentDependency": false @@ -525,7 +526,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Data.Odbc", - "Version": "8.0.0" + "Version": "8.0.1" } }, "DevelopmentDependency": false @@ -535,7 +536,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Data.OleDb", - "Version": "8.0.0" + "Version": "8.0.1" } }, "DevelopmentDependency": false @@ -565,17 +566,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Diagnostics.EventLog", - "Version": "8.0.0" - } - }, - "DevelopmentDependency": false - }, - { - "Component": { - "Type": "nuget", - "Nuget": { - "Name": "System.Diagnostics.PerformanceCounter", - "Version": "8.0.0" + "Version": "8.0.1" } }, "DevelopmentDependency": false @@ -585,7 +576,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Diagnostics.PerformanceCounter", - "Version": "8.0.0" + "Version": "8.0.1" } }, "DevelopmentDependency": false @@ -595,7 +586,7 @@ "Type": "nuget", "Nuget": { "Name": "System.DirectoryServices.AccountManagement", - "Version": "8.0.0" + "Version": "8.0.1" } }, "DevelopmentDependency": false @@ -625,17 +616,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Drawing.Common", - "Version": "8.0.8" - } - }, - "DevelopmentDependency": false - }, - { - "Component": { - "Type": "nuget", - "Nuget": { - "Name": "System.Formats.Asn1", - "Version": "8.0.1" + "Version": "8.0.10" } }, "DevelopmentDependency": false @@ -645,7 +626,7 @@ "Type": "nuget", "Nuget": { "Name": "System.IO.Packaging", - "Version": "8.0.0" + "Version": "8.0.1" } }, "DevelopmentDependency": false @@ -735,17 +716,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Runtime.Caching", - "Version": "8.0.0" - } - }, - "DevelopmentDependency": false - }, - { - "Component": { - "Type": "nuget", - "Nuget": { - "Name": "System.Runtime.CompilerServices.Unsafe", - "Version": "6.0.0" + "Version": "8.0.1" } }, "DevelopmentDependency": false @@ -765,7 +736,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Cryptography.Pkcs", - "Version": "8.0.0" + "Version": "8.0.1" } }, "DevelopmentDependency": false @@ -785,7 +756,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Cryptography.Xml", - "Version": "8.0.1" + "Version": "8.0.2" } }, "DevelopmentDependency": false @@ -875,7 +846,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ServiceProcess.ServiceController", - "Version": "8.0.0" + "Version": "8.0.1" } }, "DevelopmentDependency": false @@ -940,6 +911,5 @@ }, "DevelopmentDependency": false } - ], - "$schema": "https://json.schemastore.org/component-detection-manifest.json" + ] } From 34ebc924b73a94c70e7677b159592a231bbd5ff0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:32:16 -0800 Subject: [PATCH 115/316] Bump github/codeql-action from 3.26.10 to 3.27.0 (#24483) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.26.10 to 3.27.0. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/e2b3eafc8d227b0241d48be5f425d47c2d750a13...662472033e021d55d94146f66f6058822b0b39fd) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 4 ++-- .github/workflows/scorecards.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3722dc165cf..ae1c0cf479f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 + uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -66,4 +66,4 @@ jobs: name: Build - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 + uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index af857aa665c..ab38f9ef013 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -66,6 +66,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@e2b3eafc8d227b0241d48be5f425d47c2d750a13 # v3.26.10 + uses: github/codeql-action/upload-sarif@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 with: sarif_file: results.sarif From c7dfc18d1c86d275fa59c7fec54de38fe2522ccf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:36:14 -0800 Subject: [PATCH 116/316] Bump actions/upload-artifact from 4.4.0 to 4.4.3 (#24410) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.4.0 to 4.4.3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/50769540e7f4bd5e21e526ee35c689e35e0d6874...b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/scorecards.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index ab38f9ef013..46a2c4571af 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -58,7 +58,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: SARIF file path: results.sarif From 163343450b4d099ad884e64894f1e8f7c8673ffb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:37:41 -0800 Subject: [PATCH 117/316] Bump agrc/reminder-action from 1.0.14 to 1.0.15 (#24384) Bumps [agrc/reminder-action](https://github.com/agrc/reminder-action) from 1.0.14 to 1.0.15. - [Release notes](https://github.com/agrc/reminder-action/releases) - [Changelog](https://github.com/agrc/reminder-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/agrc/reminder-action/compare/b5cc06580b6a711baddf6a947131f85a422fa263...45201302ec0071cce809a483111bda4cdc7d10f2) --- updated-dependencies: - dependency-name: agrc/reminder-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/processReminders.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/processReminders.yml b/.github/workflows/processReminders.yml index c6ac936343c..82734c829d6 100644 --- a/.github/workflows/processReminders.yml +++ b/.github/workflows/processReminders.yml @@ -17,4 +17,4 @@ jobs: steps: - name: check reminders and notify - uses: agrc/reminder-action@b5cc06580b6a711baddf6a947131f85a422fa263 # v1.0.14 + uses: agrc/reminder-action@45201302ec0071cce809a483111bda4cdc7d10f2 # v1.0.15 From 0bdfbe6db5a596866dcf9357baf8de3eb4b47652 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:38:07 -0800 Subject: [PATCH 118/316] Bump actions/checkout from 4.2.1 to 4.2.2 (#24488) Bumps [actions/checkout](https://github.com/actions/checkout) from 4.2.1 to 4.2.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871...11bd71901bbe5b1630ceea73d27597364c9af683) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/markdownLink.yml | 4 ++-- .github/workflows/markdownLinkDaily.yml | 2 +- .github/workflows/rebase.yml | 2 +- .github/workflows/scorecards.yml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ae1c0cf479f..fde3f8c7697 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -37,7 +37,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: '0' diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 448b1bd1447..de65aadcfcf 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -17,6 +17,6 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: 'Dependency Review' uses: actions/dependency-review-action@5a2ce3f5b92ee19cbb1541a4984c76d921601d7c # v4.3.4 diff --git a/.github/workflows/markdownLink.yml b/.github/workflows/markdownLink.yml index 039d62bff2f..02bb496a091 100644 --- a/.github/workflows/markdownLink.yml +++ b/.github/workflows/markdownLink.yml @@ -11,7 +11,7 @@ jobs: markdown-link-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1 with: use-quiet-mode: 'yes' @@ -25,7 +25,7 @@ jobs: statuses: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: # Full git history is needed to get a proper # list of changed files within `super-linter` diff --git a/.github/workflows/markdownLinkDaily.yml b/.github/workflows/markdownLinkDaily.yml index c24edb7ab36..563790e39ca 100644 --- a/.github/workflows/markdownLinkDaily.yml +++ b/.github/workflows/markdownLinkDaily.yml @@ -18,7 +18,7 @@ jobs: if: github.repository == 'PowerShell/PowerShell' steps: - name: Checkout - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Check Links uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1 with: diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml index 2f490224468..b6a45200eb6 100644 --- a/.github/workflows/rebase.yml +++ b/.github/workflows/rebase.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the latest code - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 - name: Post rebase started comment to pull request diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 46a2c4571af..1ac917d3af8 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -31,7 +31,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false From a155b99c868def04a4b44e224cbd899b4ecc0c8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:38:41 -0800 Subject: [PATCH 119/316] Bump actions/dependency-review-action from 4.3.4 to 4.4.0 (#24503) Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 4.3.4 to 4.4.0. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/5a2ce3f5b92ee19cbb1541a4984c76d921601d7c...4081bf99e2866ebe428fc0477b69eb4fcda7220a) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dependency-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index de65aadcfcf..74eee94fba4 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -19,4 +19,4 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: 'Dependency Review' - uses: actions/dependency-review-action@5a2ce3f5b92ee19cbb1541a4984c76d921601d7c # v4.3.4 + uses: actions/dependency-review-action@4081bf99e2866ebe428fc0477b69eb4fcda7220a # v4.4.0 From f8b8e8520ee1fccce6eaaac577574e7a0f3bfac7 Mon Sep 17 00:00:00 2001 From: Jordan Borean Date: Tue, 12 Nov 2024 13:41:22 +1000 Subject: [PATCH 120/316] Support PSObject wrapped values in ArgumentToEncodingTransformationAttribute (#24555) --- src/System.Management.Automation/utils/EncodingUtils.cs | 2 ++ test/powershell/engine/Basic/Encoding.Tests.ps1 | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/System.Management.Automation/utils/EncodingUtils.cs b/src/System.Management.Automation/utils/EncodingUtils.cs index 060b052f744..cdb467d213a 100644 --- a/src/System.Management.Automation/utils/EncodingUtils.cs +++ b/src/System.Management.Automation/utils/EncodingUtils.cs @@ -115,6 +115,8 @@ internal sealed class ArgumentToEncodingTransformationAttribute : ArgumentTransf { public override object Transform(EngineIntrinsics engineIntrinsics, object inputData) { + inputData = PSObject.Base(inputData); + switch (inputData) { case string stringName: diff --git a/test/powershell/engine/Basic/Encoding.Tests.ps1 b/test/powershell/engine/Basic/Encoding.Tests.ps1 index dc1a0b15ba2..455c7e2ddea 100644 --- a/test/powershell/engine/Basic/Encoding.Tests.ps1 +++ b/test/powershell/engine/Basic/Encoding.Tests.ps1 @@ -89,6 +89,8 @@ Describe "File encoding tests" -Tag CI { It "Parameter 'Encoding' should accept ''" -TestCases @( @{ encoding = 1251 } @{ encoding = "windows-1251" } + # Piping the string creates a PSObject boxed value that we are testing. + @{ encoding = ("windows-1251" | Write-Output) } ) { param ( $encoding ) $testFile = "${TESTDRIVE}/fileEncoding-$($encoding).txt" From b658219a7cb5b954cf5ff429f3a872169dc239ea Mon Sep 17 00:00:00 2001 From: PowerShell Team Bot <69177312+pwshBot@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:57:38 -0800 Subject: [PATCH 121/316] Bump .NET 9 and dependencies (#24573) --- DotnetRuntimeMetadata.json | 2 +- global.json | 2 +- ...oft.PowerShell.Commands.Diagnostics.csproj | 2 +- ...soft.PowerShell.Commands.Management.csproj | 2 +- ...crosoft.PowerShell.Commands.Utility.csproj | 4 +- ...crosoft.PowerShell.CoreCLR.Eventing.csproj | 2 +- .../Microsoft.PowerShell.SDK.csproj | 10 +- .../Microsoft.WSMan.Management.csproj | 2 +- .../System.Management.Automation.csproj | 16 +-- test/tools/TestService/TestService.csproj | 4 +- test/tools/WebListener/WebListener.csproj | 4 +- tools/cgmanifest.json | 106 +++++++++--------- 12 files changed, 78 insertions(+), 78 deletions(-) diff --git a/DotnetRuntimeMetadata.json b/DotnetRuntimeMetadata.json index 50884cd0a3c..472b5958a8c 100644 --- a/DotnetRuntimeMetadata.json +++ b/DotnetRuntimeMetadata.json @@ -4,7 +4,7 @@ "quality": "daily", "qualityFallback": "preview", "packageVersionPattern": "9.0.0-preview.6", - "sdkImageVersion": "9.0.0-preview.6.24327.7", + "sdkImageVersion": "9.0.100", "nextChannel": "9.0.0-preview.7", "azureFeed": "", "sdkImageOverride": "" diff --git a/global.json b/global.json index 2dd360f846e..65324522984 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "9.0.100-rc.2.24474.11" + "version": "9.0.100" } } diff --git a/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj b/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj index 34502ced25d..a25ce533fba 100644 --- a/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj +++ b/src/Microsoft.PowerShell.Commands.Diagnostics/Microsoft.PowerShell.Commands.Diagnostics.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj index 4f20755c6e6..9b049c21d51 100644 --- a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj +++ b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj @@ -47,7 +47,7 @@ - + diff --git a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj index eee0929dada..0f012d39604 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj +++ b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj @@ -33,8 +33,8 @@ - - + + diff --git a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj index 003999e97b3..5c28e4fe256 100644 --- a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj +++ b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj index 0f1fef9362b..15b503e3d1a 100644 --- a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj +++ b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj @@ -17,13 +17,13 @@ - + - - - + + + - + diff --git a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj index 77aa3cef77c..89a147cb855 100644 --- a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj +++ b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/System.Management.Automation/System.Management.Automation.csproj b/src/System.Management.Automation/System.Management.Automation.csproj index cefbbfa31f0..c00aed6617b 100644 --- a/src/System.Management.Automation/System.Management.Automation.csproj +++ b/src/System.Management.Automation/System.Management.Automation.csproj @@ -32,16 +32,16 @@ - - - - + + + + - + - - - + + + diff --git a/test/tools/TestService/TestService.csproj b/test/tools/TestService/TestService.csproj index 94126cd2efa..00e4ba3e4b3 100644 --- a/test/tools/TestService/TestService.csproj +++ b/test/tools/TestService/TestService.csproj @@ -15,8 +15,8 @@ - - + + diff --git a/test/tools/WebListener/WebListener.csproj b/test/tools/WebListener/WebListener.csproj index 22028234824..d037627b1ff 100644 --- a/test/tools/WebListener/WebListener.csproj +++ b/test/tools/WebListener/WebListener.csproj @@ -7,7 +7,7 @@ - - + + diff --git a/tools/cgmanifest.json b/tools/cgmanifest.json index 32e9cc0110f..f823d713d55 100644 --- a/tools/cgmanifest.json +++ b/tools/cgmanifest.json @@ -1,5 +1,4 @@ { - "$schema": "https://json.schemastore.org/component-detection-manifest.json", "Registrations": [ { "Component": { @@ -126,7 +125,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Extensions.ObjectPool", - "Version": "8.0.10" + "Version": "8.0.11" } }, "DevelopmentDependency": false @@ -196,7 +195,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Win32.Registry.AccessControl", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -216,7 +215,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Win32.SystemEvents", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -226,7 +225,7 @@ "Type": "nuget", "Nuget": { "Name": "Microsoft.Windows.Compatibility", - "Version": "8.0.10" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -246,7 +245,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.android-arm.runtime.native.System.IO.Ports", - "Version": "9.0.0-rc.2.24473.5" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -256,7 +255,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.android-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.0-rc.2.24473.5" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -266,7 +265,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.android-x64.runtime.native.System.IO.Ports", - "Version": "9.0.0-rc.2.24473.5" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -276,7 +275,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.android-x86.runtime.native.System.IO.Ports", - "Version": "9.0.0-rc.2.24473.5" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -286,7 +285,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-arm.runtime.native.System.IO.Ports", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -296,7 +295,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-arm64.runtime.native.System.IO.Ports", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -306,7 +305,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-bionic-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.0-rc.2.24473.5" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -316,7 +315,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-bionic-x64.runtime.native.System.IO.Ports", - "Version": "9.0.0-rc.2.24473.5" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -326,7 +325,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-musl-arm.runtime.native.System.IO.Ports", - "Version": "9.0.0-rc.2.24473.5" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -336,7 +335,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-musl-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.0-rc.2.24473.5" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -346,7 +345,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-musl-x64.runtime.native.System.IO.Ports", - "Version": "9.0.0-rc.2.24473.5" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -356,7 +355,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.linux-x64.runtime.native.System.IO.Ports", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -366,7 +365,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.maccatalyst-arm64.runtime.native.System.IO.Ports", - "Version": "9.0.0-rc.2.24473.5" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -376,7 +375,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.maccatalyst-x64.runtime.native.System.IO.Ports", - "Version": "9.0.0-rc.2.24473.5" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -396,7 +395,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.native.System.IO.Ports", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -406,7 +405,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.osx-arm64.runtime.native.System.IO.Ports", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -416,7 +415,7 @@ "Type": "nuget", "Nuget": { "Name": "runtime.osx-x64.runtime.native.System.IO.Ports", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -476,7 +475,7 @@ "Type": "nuget", "Nuget": { "Name": "System.CodeDom", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -496,7 +495,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ComponentModel.Composition.Registration", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -506,7 +505,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ComponentModel.Composition", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -516,7 +515,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Configuration.ConfigurationManager", - "Version": "8.0.1" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -526,7 +525,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Data.Odbc", - "Version": "8.0.1" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -536,7 +535,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Data.OleDb", - "Version": "8.0.1" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -556,7 +555,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Diagnostics.DiagnosticSource", - "Version": "8.0.1" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -566,7 +565,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Diagnostics.EventLog", - "Version": "8.0.1" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -576,7 +575,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Diagnostics.PerformanceCounter", - "Version": "8.0.1" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -586,7 +585,7 @@ "Type": "nuget", "Nuget": { "Name": "System.DirectoryServices.AccountManagement", - "Version": "8.0.1" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -596,7 +595,7 @@ "Type": "nuget", "Nuget": { "Name": "System.DirectoryServices.Protocols", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -606,7 +605,7 @@ "Type": "nuget", "Nuget": { "Name": "System.DirectoryServices", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -616,7 +615,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Drawing.Common", - "Version": "8.0.10" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -626,7 +625,7 @@ "Type": "nuget", "Nuget": { "Name": "System.IO.Packaging", - "Version": "8.0.1" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -636,7 +635,7 @@ "Type": "nuget", "Nuget": { "Name": "System.IO.Ports", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -646,7 +645,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Management", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -656,7 +655,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Net.Http.WinHttpHandler", - "Version": "8.0.2" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -686,7 +685,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Reflection.Context", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -716,7 +715,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Runtime.Caching", - "Version": "8.0.1" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -736,7 +735,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Cryptography.Pkcs", - "Version": "8.0.1" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -746,7 +745,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Cryptography.ProtectedData", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -756,7 +755,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Cryptography.Xml", - "Version": "8.0.2" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -766,7 +765,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Security.Permissions", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -836,7 +835,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ServiceModel.Syndication", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -846,7 +845,7 @@ "Type": "nuget", "Nuget": { "Name": "System.ServiceProcess.ServiceController", - "Version": "8.0.1" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -856,7 +855,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Speech", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -866,7 +865,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Text.Encoding.CodePages", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -876,7 +875,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Text.Encodings.Web", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -886,7 +885,7 @@ "Type": "nuget", "Nuget": { "Name": "System.Threading.AccessControl", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false @@ -906,10 +905,11 @@ "Type": "nuget", "Nuget": { "Name": "System.Windows.Extensions", - "Version": "8.0.0" + "Version": "9.0.0" } }, "DevelopmentDependency": false } - ] + ], + "$schema": "https://json.schemastore.org/component-detection-manifest.json" } From fbd5dfed9e9d2a5029e00ccd9744cdcb1655be14 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 14 Nov 2024 11:38:55 -0600 Subject: [PATCH 122/316] added stage uploading cl and tool to artifacts, removed ob_release_env variable on first stage --- .pipelines/PowerShell-Release-Official.yml | 10 +++- .../templates/release-upload-cl-tools.yml | 58 +++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 .pipelines/templates/release-upload-cl-tools.yml diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 871557abe04..bab29a12394 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -112,8 +112,6 @@ extends: stages: - stage: msixbundle displayName: 'Create MSIX Bundle' - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/release-create-msix.yml@self @@ -272,6 +270,14 @@ extends: parameters: SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} + - stage: UploadChangelogAndTools + displayName: Upload Changelog and Tools + dependsOn: BlobPublic + jobs: + - template: /.pipelines/templates/release-upload-cl-tools.yml@self + parameters: + depth: 1 + - stage: PublishGitHubRelease displayName: Publish GitHub Release dependsOn: BlobPublic diff --git a/.pipelines/templates/release-upload-cl-tools.yml b/.pipelines/templates/release-upload-cl-tools.yml new file mode 100644 index 00000000000..992b9a409b9 --- /dev/null +++ b/.pipelines/templates/release-upload-cl-tools.yml @@ -0,0 +1,58 @@ +parameters: + - name: depth + type: number + default: 1 + +jobs: +- job: upload_to_artifacts + displayName: Upload packages + condition: succeeded() + pool: + type: windows + variables: + - name: ob_sdl_sbom_enabled + value: false + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: ob_sdl_codeql_compiled_enabled + value: false + + steps: + - checkout: self + clean: true + fetchDepth: ${{ parameters.depth }} + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + displayName: 'Checkout source code' + + - pwsh: | + git clone --depth ${{ parameters.depth }} https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' + displayName: 'Clone tools repository' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(Pipeline.Workspace)/tools/*' + artifact: 'tools' + publishLocation: 'pipeline' + displayName: 'Publish tools artifact' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(Pipeline.Workspace)/PowerShell/CHANGELOG/*' + artifact: 'changelog' + publishLocation: 'pipeline' + displayName: 'Publish PowerShell artifact' From fa6f23e53e5a78b005dadeadfb60a713785ef535 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 14 Nov 2024 11:54:11 -0600 Subject: [PATCH 123/316] removed publish artifact --- .../templates/release-upload-cl-tools.yml | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/.pipelines/templates/release-upload-cl-tools.yml b/.pipelines/templates/release-upload-cl-tools.yml index 992b9a409b9..e25d2b4e68a 100644 --- a/.pipelines/templates/release-upload-cl-tools.yml +++ b/.pipelines/templates/release-upload-cl-tools.yml @@ -42,17 +42,13 @@ jobs: - pwsh: | git clone --depth ${{ parameters.depth }} https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' displayName: 'Clone tools repository' + + - pwsh: | + New-Item -ItemType Directory -Path "$(ob_outputDirectory)/tools" + Copy-Item "$(Pipeline.Workspace)/tools/*" -Destination "$(ob_outputDirectory)/tools" -Force -Verbose + displayName: 'Upload tools to artifacts' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '$(Pipeline.Workspace)/tools/*' - artifact: 'tools' - publishLocation: 'pipeline' - displayName: 'Publish tools artifact' - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '$(Pipeline.Workspace)/PowerShell/CHANGELOG/*' - artifact: 'changelog' - publishLocation: 'pipeline' - displayName: 'Publish PowerShell artifact' + - pwsh: | + New-Item -ItemType Directory -Path "$(ob_outputDirectory)/CHANGELOG" + Copy-Item "$(Pipeline.Workspace)/PowerShell/CHANGELOG/*" -Destination "$(ob_outputDirectory)/CHANGELOG" -Force -Verbose + displayName: 'Upload changelog to artifacts' From a044b8895b866f4b49dbd4801d68f2541b0dd347 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 14 Nov 2024 14:21:00 -0600 Subject: [PATCH 124/316] removed variables like tsa config template --- .pipelines/PowerShell-Release-Official.yml | 1 - .pipelines/templates/release-upload-cl-tools.yml | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index c8e710d9107..4a1b9a7b889 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -274,7 +274,6 @@ extends: - stage: UploadChangelogAndTools displayName: Upload Changelog and Tools - dependsOn: BlobPublic jobs: - template: /.pipelines/templates/release-upload-cl-tools.yml@self parameters: diff --git a/.pipelines/templates/release-upload-cl-tools.yml b/.pipelines/templates/release-upload-cl-tools.yml index e25d2b4e68a..233028a26b6 100644 --- a/.pipelines/templates/release-upload-cl-tools.yml +++ b/.pipelines/templates/release-upload-cl-tools.yml @@ -24,12 +24,6 @@ jobs: value: false - name: ob_sdl_binskim_enabled value: false - - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - - name: ob_sdl_codeql_compiled_enabled - value: false steps: - checkout: self @@ -37,7 +31,7 @@ jobs: fetchDepth: ${{ parameters.depth }} env: ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - displayName: 'Checkout source code' + displayName: 'Checkout PowerShell Repo' - pwsh: | git clone --depth ${{ parameters.depth }} https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' From d0238b09cfddd36fd5429164ac72ca03e5b0b9e6 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 14 Nov 2024 16:32:22 -0600 Subject: [PATCH 125/316] updated image windows container image --- .pipelines/PowerShell-Release-Official.yml | 2 +- .pipelines/templates/release-upload-cl-tools.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 4a1b9a7b889..ed507904884 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -50,7 +50,7 @@ variables: - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: WindowsContainerImage - value: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' + value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - name: LinuxContainerImage value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 - name: ReleaseTagVar diff --git a/.pipelines/templates/release-upload-cl-tools.yml b/.pipelines/templates/release-upload-cl-tools.yml index 233028a26b6..3645db6fb0d 100644 --- a/.pipelines/templates/release-upload-cl-tools.yml +++ b/.pipelines/templates/release-upload-cl-tools.yml @@ -10,6 +10,7 @@ jobs: pool: type: windows variables: + - group: 'mscodehub-code-read-akv' - name: ob_sdl_sbom_enabled value: false - name: runCodesignValidationInjection From d1ff825992dace91bbedc3b0c6ab1200eaa829c3 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 14 Nov 2024 16:50:10 -0600 Subject: [PATCH 126/316] using REPOROOT, removed pwsh checkout, -recurse tools --- .pipelines/templates/release-upload-cl-tools.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.pipelines/templates/release-upload-cl-tools.yml b/.pipelines/templates/release-upload-cl-tools.yml index 3645db6fb0d..62a5500be4c 100644 --- a/.pipelines/templates/release-upload-cl-tools.yml +++ b/.pipelines/templates/release-upload-cl-tools.yml @@ -27,23 +27,16 @@ jobs: value: false steps: - - checkout: self - clean: true - fetchDepth: ${{ parameters.depth }} - env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - displayName: 'Checkout PowerShell Repo' - - pwsh: | git clone --depth ${{ parameters.depth }} https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' displayName: 'Clone tools repository' - pwsh: | New-Item -ItemType Directory -Path "$(ob_outputDirectory)/tools" - Copy-Item "$(Pipeline.Workspace)/tools/*" -Destination "$(ob_outputDirectory)/tools" -Force -Verbose + Copy-Item "$(Pipeline.Workspace)/tools/*" -Destination "$(ob_outputDirectory)/tools" -Recurse -Force -Verbose displayName: 'Upload tools to artifacts' - pwsh: | New-Item -ItemType Directory -Path "$(ob_outputDirectory)/CHANGELOG" - Copy-Item "$(Pipeline.Workspace)/PowerShell/CHANGELOG/*" -Destination "$(ob_outputDirectory)/CHANGELOG" -Force -Verbose + Copy-Item "$(REPOROOT)/CHANGELOG/*" -Destination "$(ob_outputDirectory)/CHANGELOG" -Force -Verbose displayName: 'Upload changelog to artifacts' From 036e871f40802fdc7d2c890e14f9472b4f7b5f91 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 14 Nov 2024 17:12:28 -0600 Subject: [PATCH 127/316] added turning off tsa config, credscan, and codesignValidation --- .pipelines/templates/release-upload-cl-tools.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pipelines/templates/release-upload-cl-tools.yml b/.pipelines/templates/release-upload-cl-tools.yml index 62a5500be4c..20ea2d4270d 100644 --- a/.pipelines/templates/release-upload-cl-tools.yml +++ b/.pipelines/templates/release-upload-cl-tools.yml @@ -25,6 +25,12 @@ jobs: value: false - name: ob_sdl_binskim_enabled value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: ob_sdl_codeSignValidation_enabled + value: false steps: - pwsh: | From 6f02cb2f6fb39c0c10b6395851568416556ead72 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 12:19:48 -0600 Subject: [PATCH 128/316] removed cl and tool stage --- .pipelines/PowerShell-Release-Official.yml | 8 --- .../templates/release-upload-cl-tools.yml | 58 ------------------- 2 files changed, 66 deletions(-) delete mode 100644 .pipelines/templates/release-upload-cl-tools.yml diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index bab29a12394..ca6e765a5b2 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -270,14 +270,6 @@ extends: parameters: SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} - - stage: UploadChangelogAndTools - displayName: Upload Changelog and Tools - dependsOn: BlobPublic - jobs: - - template: /.pipelines/templates/release-upload-cl-tools.yml@self - parameters: - depth: 1 - - stage: PublishGitHubRelease displayName: Publish GitHub Release dependsOn: BlobPublic diff --git a/.pipelines/templates/release-upload-cl-tools.yml b/.pipelines/templates/release-upload-cl-tools.yml deleted file mode 100644 index 992b9a409b9..00000000000 --- a/.pipelines/templates/release-upload-cl-tools.yml +++ /dev/null @@ -1,58 +0,0 @@ -parameters: - - name: depth - type: number - default: 1 - -jobs: -- job: upload_to_artifacts - displayName: Upload packages - condition: succeeded() - pool: - type: windows - variables: - - name: ob_sdl_sbom_enabled - value: false - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - value: 1 - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_sdl_codeSignValidation_enabled - value: false - - name: ob_sdl_binskim_enabled - value: false - - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - - name: ob_sdl_codeql_compiled_enabled - value: false - - steps: - - checkout: self - clean: true - fetchDepth: ${{ parameters.depth }} - env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - displayName: 'Checkout source code' - - - pwsh: | - git clone --depth ${{ parameters.depth }} https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' - displayName: 'Clone tools repository' - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '$(Pipeline.Workspace)/tools/*' - artifact: 'tools' - publishLocation: 'pipeline' - displayName: 'Publish tools artifact' - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '$(Pipeline.Workspace)/PowerShell/CHANGELOG/*' - artifact: 'changelog' - publishLocation: 'pipeline' - displayName: 'Publish PowerShell artifact' From 249fc4b99dcf155fc9f66b18a274cbf9f8d371ad Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 12:27:01 -0600 Subject: [PATCH 129/316] combined approval stages --- .pipelines/PowerShell-Release-Official.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index ca6e765a5b2..4e3cc3e8915 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -278,8 +278,8 @@ extends: jobs: - template: /.pipelines/templates/release-githubtasks.yml@self - - stage: PushGitTag - displayName: Push Git Tag + - stage: PushGitTagAndMakeDraftPublic + displayName: Push Git Tag and Make Draft Public dependsOn: PublishGitHubRelease jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -289,11 +289,8 @@ extends: instructions: | Push the git tag to upstream - - stage: MakeDraftPublic - displayName: Make Draft Public - dependsOn: PushGitTag - jobs: - template: /.pipelines/templates/approvalJob.yml@self + dependsOnJob: PushGitTag parameters: displayName: Make Draft Public jobName: DraftPublic From 9229cfb4ca65566480672eba682521a17861eb93 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 12:29:33 -0600 Subject: [PATCH 130/316] removed unnecessary variable srcPath --- .pipelines/templates/release-githubtasks.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index fd96fc4ee97..1f5b0511013 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -61,8 +61,7 @@ jobs: script: | $Path = "$(Pipeline.Workspace)/GitHubPackages" $OutputPath = Join-Path $Path 'hashes.sha256' - $srcPaths = @($Path) - $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse -File + $packages = Get-ChildItem -Path $Path -Include * -Recurse -File $checksums = $packages | ForEach-Object { Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" From 3260f12886a5c40d658fe2a232a7939479413403 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 12:38:59 -0600 Subject: [PATCH 131/316] commented out package pipeline changes --- .pipelines/templates/uploadToAzure.yml | 34 +++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index 7c7483f8893..1a5186e5e6a 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -235,23 +235,23 @@ jobs: Get-ChildItem '$(Build.ArtifactStagingDirectory)/downloads' | Select-Object -ExpandProperty FullName displayName: 'Capture downloads' - - pwsh: | - Write-Verbose -Verbose "Copying Github Release files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" - - Write-Verbose -Verbose "Creating output directory for GitHub Release files: $(ob_outputDirectory)/GitHubPackages" - New-Item -Path $(ob_outputDirectory)/GitHubPackages -ItemType Directory -Force - Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | - Where-Object { $_.Extension -notin '.msix', '.nupkg' } | - ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | - Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse - - Write-Verbose -Verbose "Creating output directory for NuGet packages: $(ob_outputDirectory)/NuGetPackages" - New-Item -Path $(ob_outputDirectory)/NuGetPackages -ItemType Directory -Force - Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | - Where-Object { $_.Extension -eq '.nupkg' } | - ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | - Copy-Item -Destination $(ob_outputDirectory)/NuGetPackages -Recurse - displayName: Copy downloads to Artifacts + # - pwsh: | + # Write-Verbose -Verbose "Copying Github Release files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" + # + # Write-Verbose -Verbose "Creating output directory for GitHub Release files: $(ob_outputDirectory)/GitHubPackages" + # New-Item -Path $(ob_outputDirectory)/GitHubPackages -ItemType Directory -Force + # Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | + # Where-Object { $_.Extension -notin '.msix', '.nupkg' } | + # ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | + # Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse + # + # Write-Verbose -Verbose "Creating output directory for NuGet packages: $(ob_outputDirectory)/NuGetPackages" + # New-Item -Path $(ob_outputDirectory)/NuGetPackages -ItemType Directory -Force + # Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | + # Where-Object { $_.Extension -eq '.nupkg' } | + # ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | + # Copy-Item -Destination $(ob_outputDirectory)/NuGetPackages -Recurse + # displayName: Copy downloads to Artifacts - pwsh: | # Create output directory for packages which have been uploaded to blob storage From 28cb85084045f21584def41dc7f259c8841b58e7 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 13:06:39 -0600 Subject: [PATCH 132/316] added package gathering in the release template --- .pipelines/PowerShell-Release-Official.yml | 5 + .../templates/release-download-packages.yml | 139 ++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 .pipelines/templates/release-download-packages.yml diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 4e3cc3e8915..9fb1ae1d183 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -110,6 +110,11 @@ extends: tsaOptionsFile: .config\tsaoptions.json stages: + - stage: DownloadPackages + displayName: 'Download Packages' + jobs: + - template: /.pipelines/templates/release-download-packages.yml@self + - stage: msixbundle displayName: 'Create MSIX Bundle' jobs: diff --git a/.pipelines/templates/release-download-packages.yml b/.pipelines/templates/release-download-packages.yml new file mode 100644 index 00000000000..63ed7d8219f --- /dev/null +++ b/.pipelines/templates/release-download-packages.yml @@ -0,0 +1,139 @@ +jobs: +- job: upload_packages + displayName: Upload packages + condition: succeeded() + pool: + type: windows + variables: + - name: ob_sdl_sbom_enabled + value: false + - name: runCodesignValidationInjection + value: false + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: ob_sdl_codeql_compiled_enabled + value: false + + steps: + - pwsh: | + New-Item -Path '$(Build.ArtifactStagingDirectory)/downloads' -ItemType Directory -Force + displayName: Create downloads directory + + - download: PSPackagesOfficial + artifact: drop_linux_package_deb + displayName: Download linux deb packages + + - download: PSPackagesOfficial + artifact: drop_linux_package_fxdependent + displayName: Download linux fx packages + + - download: PSPackagesOfficial + artifact: drop_linux_package_mariner_arm64 + displayName: Download linux mariner packages + + - download: PSPackagesOfficial + artifact: drop_linux_package_mariner_x64 + displayName: Download linux mariner x64 packages + + - download: PSPackagesOfficial + artifact: drop_linux_package_minSize + displayName: Download linux min packages + + - download: PSPackagesOfficial + artifact: drop_linux_package_rpm + displayName: Download linux rpm packages + + - download: PSPackagesOfficial + artifact: drop_linux_package_tar + displayName: Download linux tar packages + + - download: PSPackagesOfficial + artifact: drop_linux_package_tar_alpine + displayName: Download linux tar alpine packages + + - download: PSPackagesOfficial + artifact: drop_linux_package_tar_alpine_fxd + displayName: Download linux tar alpine fxd packages + + - download: PSPackagesOfficial + artifact: drop_linux_package_tar_arm + displayName: Download linux tar arm packages + + - download: PSPackagesOfficial + artifact: drop_linux_package_tar_arm64 + displayName: Download linux tar arm 64 packages + + - download: PSPackagesOfficial + artifact: drop_nupkg_build_nupkg + displayName: Download nupkg packages + + - download: PSPackagesOfficial + artifact: drop_windows_package_package_win_arm64 + displayName: Download windows arm64 packages + + - download: PSPackagesOfficial + artifact: drop_windows_package_package_win_fxdependent + displayName: Download windows fxdependent packages + + - download: PSPackagesOfficial + artifact: drop_windows_package_package_win_fxdependentWinDesktop + displayName: Download windows fxdependentWinDesktop packages + + - download: PSPackagesOfficial + artifact: drop_windows_package_package_win_minsize + displayName: Download windows minsize packages + + - download: PSPackagesOfficial + artifact: drop_windows_package_package_win_x64 + displayName: Download windows x64 packages + + - download: PSPackagesOfficial + artifact: drop_windows_package_package_win_x86 + displayName: Download windows x86 packages + + - download: PSPackagesOfficial + artifact: macos-pkgs + displayName: Download macos tar packages + + - download: PSPackagesOfficial + artifact: drop_mac_package_sign_package_macos_arm64 + displayName: Download macos arm packages + + - download: PSPackagesOfficial + artifact: drop_mac_package_sign_package_macos_x64 + displayName: Download macos x64 packages + + - pwsh: | + Get-ChildItem '$(Pipeline.Workspace)/PSPackagesOfficial' -Recurse | Select-Object -ExpandProperty FullName + displayName: 'Capture downloads' + + - pwsh: | + $PackagesPath = '$(Pipeline.Workspace)/PSPackagesOfficial' + Write-Verbose -Verbose "Copying Github Release files in $PackagesPath to use in Release Pipeline" + + Write-Verbose -Verbose "Creating output directory for GitHub Release files: $(ob_outputDirectory)/GitHubPackages" + New-Item -Path $(ob_outputDirectory)/GitHubPackages -ItemType Directory -Force + Get-ChildItem -Path "$PackagesPath/*" -Recurse | + Where-Object { $_.Extension -notin '.msix', '.nupkg' } | + ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | + Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse + + Write-Verbose -Verbose "Creating output directory for NuGet packages: $(ob_outputDirectory)/NuGetPackages" + New-Item -Path $(ob_outputDirectory)/NuGetPackages -ItemType Directory -Force + Get-ChildItem -Path "$PackagesPath/*" -Recurse | + Where-Object { $_.Extension -eq '.nupkg' } | + ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | + Copy-Item -Destination $(ob_outputDirectory)/NuGetPackages -Recurse + displayName: Copy downloads to Artifacts From 6b85b99c2cbd58ab9abfb95bbd3fa4161138cd85 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 13:17:16 -0600 Subject: [PATCH 133/316] added variable template --- .../templates/release-download-packages.yml | 21 +---------------- .pipelines/templates/release-githubtasks.yml | 23 ++----------------- .../templates/release-publish-nuget.yml | 18 +-------------- .../templates/variable/release-shared.yml | 22 ++++++++++++++++++ 4 files changed, 26 insertions(+), 58 deletions(-) create mode 100644 .pipelines/templates/variable/release-shared.yml diff --git a/.pipelines/templates/release-download-packages.yml b/.pipelines/templates/release-download-packages.yml index 63ed7d8219f..bce47af409c 100644 --- a/.pipelines/templates/release-download-packages.yml +++ b/.pipelines/templates/release-download-packages.yml @@ -5,26 +5,7 @@ jobs: pool: type: windows variables: - - name: ob_sdl_sbom_enabled - value: false - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - value: 1 - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_sdl_codeSignValidation_enabled - value: false - - name: ob_sdl_binskim_enabled - value: false - - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - - name: ob_sdl_codeql_compiled_enabled - value: false + - template: ./variable/release-shared.yml@self steps: - pwsh: | diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 1f5b0511013..7ea4a671cb8 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -8,28 +8,9 @@ jobs: templateContext: inputs: - input: pipelineArtifact - pipeline: PSPackagesOfficial - artifactName: drop_upload_upload_packages + artifactName: drop_DownloadPackages_upload_packages variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - value: 1 - - group: 'mscodehub-code-read-akv' - - group: 'Azure Blob variable group' - - group: 'GitHubTokens' - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_sdl_codeSignValidation_enabled - value: false - - name: ob_sdl_binskim_enabled - value: false - - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - template: ./variable/release-shared.yml@self steps: - template: release-install-pwsh.yml diff --git a/.pipelines/templates/release-publish-nuget.yml b/.pipelines/templates/release-publish-nuget.yml index eeff1a894a6..7f2bace7547 100644 --- a/.pipelines/templates/release-publish-nuget.yml +++ b/.pipelines/templates/release-publish-nuget.yml @@ -16,23 +16,7 @@ jobs: pipeline: PSPackagesOfficial artifactName: drop_upload_upload_packages variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - value: 1 - - group: 'mscodehub-code-read-akv' - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_sdl_codeSignValidation_enabled - value: false - - name: ob_sdl_binskim_enabled - value: false - - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - template: ./variable/release-shared.yml@self steps: - template: release-install-pwsh.yml diff --git a/.pipelines/templates/variable/release-shared.yml b/.pipelines/templates/variable/release-shared.yml new file mode 100644 index 00000000000..b121c32ebdc --- /dev/null +++ b/.pipelines/templates/variable/release-shared.yml @@ -0,0 +1,22 @@ +variables: + - name: ob_sdl_sbom_enabled + value: false + - name: runCodesignValidationInjection + value: false + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: 'mscodehub-code-read-akv' + - group: 'Azure Blob variable group' + - group: 'GitHubTokens' + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_binskim_enabled + value: false + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: ob_sdl_codeql_compiled_enabled + value: false From 42c8ea663b7bb516ea9c335787eb8921634addaa Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 13:24:37 -0600 Subject: [PATCH 134/316] fixed dependsonjob --- .pipelines/PowerShell-Release-Official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index ac3af37fc8b..dc4761e473c 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -297,9 +297,9 @@ extends: Push the git tag to upstream - template: /.pipelines/templates/approvalJob.yml@self - dependsOnJob: PushGitTag parameters: displayName: Make Draft Public + dependsOnJob: PushGitTag jobName: DraftPublic instructions: | Make the GitHub Release Draft Public From c549ed2775f1790ed9902d4a3f6433a5b8f4454a Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 13:39:04 -0600 Subject: [PATCH 135/316] added depnedency, added filtering file by ext --- .pipelines/PowerShell-Release-Official.yml | 2 +- .pipelines/templates/release-download-packages.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index dc4761e473c..8758d4a8067 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -279,7 +279,7 @@ extends: - stage: PublishGitHubRelease displayName: Publish GitHub Release - dependsOn: BlobPublic + dependsOn: BlobPublic, DownloadPackages variables: ob_release_environment: Production jobs: diff --git a/.pipelines/templates/release-download-packages.yml b/.pipelines/templates/release-download-packages.yml index bce47af409c..afc545d8d05 100644 --- a/.pipelines/templates/release-download-packages.yml +++ b/.pipelines/templates/release-download-packages.yml @@ -107,7 +107,8 @@ jobs: Write-Verbose -Verbose "Creating output directory for GitHub Release files: $(ob_outputDirectory)/GitHubPackages" New-Item -Path $(ob_outputDirectory)/GitHubPackages -ItemType Directory -Force Get-ChildItem -Path "$PackagesPath/*" -Recurse | - Where-Object { $_.Extension -notin '.msix', '.nupkg' } | + Where-Object { $_.Extension -notin '.msix', '.nupkg' } | + Where-Object { $_.Extension -in '.tar.gz', '.pkg', '.msi', '.zip', '.deb', '.rpm', '.zip' } | ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse From a964ca0362f1c94ffb0185eea5aa3d4afd945947 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 13:40:28 -0600 Subject: [PATCH 136/316] removed dependency --- .pipelines/PowerShell-Release-Official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 8758d4a8067..dc4761e473c 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -279,7 +279,7 @@ extends: - stage: PublishGitHubRelease displayName: Publish GitHub Release - dependsOn: BlobPublic, DownloadPackages + dependsOn: BlobPublic variables: ob_release_environment: Production jobs: From e0cc1f25184ce59ba06acc3d81fec45527d5076e Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 13:50:34 -0600 Subject: [PATCH 137/316] .gz --- .pipelines/templates/release-download-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-download-packages.yml b/.pipelines/templates/release-download-packages.yml index afc545d8d05..ab00eb59b0b 100644 --- a/.pipelines/templates/release-download-packages.yml +++ b/.pipelines/templates/release-download-packages.yml @@ -108,7 +108,7 @@ jobs: New-Item -Path $(ob_outputDirectory)/GitHubPackages -ItemType Directory -Force Get-ChildItem -Path "$PackagesPath/*" -Recurse | Where-Object { $_.Extension -notin '.msix', '.nupkg' } | - Where-Object { $_.Extension -in '.tar.gz', '.pkg', '.msi', '.zip', '.deb', '.rpm', '.zip' } | + Where-Object { $_.Extension -in '.gz', '.pkg', '.msi', '.zip', '.deb', '.rpm', '.zip' } | ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse From 96cea2dcf7b1552e8604ae290e17491cd199631b Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 14:04:35 -0600 Subject: [PATCH 138/316] added depends on --- .pipelines/PowerShell-Release-Official.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index dc4761e473c..020784e89df 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -112,6 +112,7 @@ extends: stages: - stage: DownloadPackages displayName: 'Download Packages' + dependsOn: [] jobs: - template: /.pipelines/templates/release-download-packages.yml@self @@ -279,7 +280,7 @@ extends: - stage: PublishGitHubRelease displayName: Publish GitHub Release - dependsOn: BlobPublic + dependsOn: BlobPublic, DownloadPackages variables: ob_release_environment: Production jobs: From 6f72dace30305fc10ccbdf473115faa966055748 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 14:16:50 -0600 Subject: [PATCH 139/316] added verbose, fixed checkout --- .pipelines/templates/release-download-packages.yml | 12 ++++-------- .pipelines/templates/variable/release-shared.yml | 9 +++++++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.pipelines/templates/release-download-packages.yml b/.pipelines/templates/release-download-packages.yml index ab00eb59b0b..099aaff09b1 100644 --- a/.pipelines/templates/release-download-packages.yml +++ b/.pipelines/templates/release-download-packages.yml @@ -6,12 +6,10 @@ jobs: type: windows variables: - template: ./variable/release-shared.yml@self + parameters: + REPOROOT: $(Build.SourcesDirectory) steps: - - pwsh: | - New-Item -Path '$(Build.ArtifactStagingDirectory)/downloads' -ItemType Directory -Force - displayName: Create downloads directory - - download: PSPackagesOfficial artifact: drop_linux_package_deb displayName: Download linux deb packages @@ -109,13 +107,11 @@ jobs: Get-ChildItem -Path "$PackagesPath/*" -Recurse | Where-Object { $_.Extension -notin '.msix', '.nupkg' } | Where-Object { $_.Extension -in '.gz', '.pkg', '.msi', '.zip', '.deb', '.rpm', '.zip' } | - ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | - Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse + Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse -Verbose Write-Verbose -Verbose "Creating output directory for NuGet packages: $(ob_outputDirectory)/NuGetPackages" New-Item -Path $(ob_outputDirectory)/NuGetPackages -ItemType Directory -Force Get-ChildItem -Path "$PackagesPath/*" -Recurse | Where-Object { $_.Extension -eq '.nupkg' } | - ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | - Copy-Item -Destination $(ob_outputDirectory)/NuGetPackages -Recurse + Copy-Item -Destination $(ob_outputDirectory)/NuGetPackages -Recurse -Verbose displayName: Copy downloads to Artifacts diff --git a/.pipelines/templates/variable/release-shared.yml b/.pipelines/templates/variable/release-shared.yml index b121c32ebdc..f1a8c1311d2 100644 --- a/.pipelines/templates/variable/release-shared.yml +++ b/.pipelines/templates/variable/release-shared.yml @@ -1,3 +1,8 @@ +parameters: + - name: REPOROOT + type: string + default: $(Build.SourcesDirectory)\PowerShell + variables: - name: ob_sdl_sbom_enabled value: false @@ -15,8 +20,8 @@ variables: - name: ob_sdl_binskim_enabled value: false - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + value: $(REPOROOT)\.config\tsaoptions.json - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + value: $(REPOROOT)\.config\suppress.json - name: ob_sdl_codeql_compiled_enabled value: false From 7736a6b1c9c21182cdaeffd0311341e26bd0528e Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 14:30:34 -0600 Subject: [PATCH 140/316] fixed dependencies --- .pipelines/PowerShell-Release-Official.yml | 38 ++++++++++++++-------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 020784e89df..3f12117c5fa 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -118,6 +118,7 @@ extends: - stage: msixbundle displayName: 'Create MSIX Bundle' + dependsOn: [] variables: ob_release_environment: Test jobs: @@ -125,6 +126,7 @@ extends: - stage: validateSdk displayName: 'Validate SDK' + dependsOn: [] jobs: - template: /.pipelines/templates/release-validate-sdk.yml@self parameters: @@ -149,6 +151,7 @@ extends: - stage: gbltool displayName: 'Validate Global tools' + dependsOn: [] jobs: - template: /.pipelines/templates/release-validate-globaltools.yml@self parameters: @@ -166,6 +169,7 @@ extends: - stage: fxdpackages displayName: 'Validate FXD Packages' + dependsOn: [] jobs: - template: /.pipelines/templates/release-validate-fxdpackages.yml@self parameters: @@ -202,6 +206,7 @@ extends: - stage: validatePackages displayName: 'Validate Packages' + dependsOn: [] jobs: - template: /.pipelines/templates/release-validate-packagenames.yml@self @@ -280,7 +285,7 @@ extends: - stage: PublishGitHubRelease displayName: Publish GitHub Release - dependsOn: BlobPublic, DownloadPackages + dependsOn: DownloadPackages variables: ob_release_environment: Production jobs: @@ -307,7 +312,7 @@ extends: - stage: PublishNuGet displayName: Publish NuGet - dependsOn: PublishGitHubRelease + dependsOn: PushGitTagAndMakeDraftPublic variables: ob_release_environment: Production jobs: @@ -317,14 +322,14 @@ extends: - stage: PublishPMC displayName: Publish PMC - dependsOn: PublishGitHubRelease + dependsOn: PushGitTagAndMakeDraftPublic jobs: - template: /.pipelines/templates/release-publish-pmc.yml@self parameters: skipPublish: ${{ parameters.SkipPMCPublish }} - stage: ReleaseDocker - dependsOn: PublishGitHubRelease + dependsOn: PushGitTagAndMakeDraftPublic displayName: 'Docker Release' jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -343,7 +348,7 @@ extends: Kickoff docker release - stage: UpdateDotnetDocker - dependsOn: PublishGitHubRelease + dependsOn: PushGitTagAndMakeDraftPublic displayName: Update DotNet SDK Docker images jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -358,7 +363,7 @@ extends: 4. create PR targeting nightly branch - stage: UpdateWinGet - dependsOn: PublishGitHubRelease + dependsOn: PushGitTagAndMakeDraftPublic displayName: Add manifest entry to winget jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -369,7 +374,7 @@ extends: This is typically done by the community 1-2 days after the release. - stage: PublishMsix - dependsOn: PublishGitHubRelease + dependsOn: PushGitTagAndMakeDraftPublic displayName: Publish MSIX to store jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -380,7 +385,7 @@ extends: Ask Steve to release MSIX bundle package to Store - stage: PublishVPack - dependsOn: PublishGitHubRelease + dependsOn: PushGitTagAndMakeDraftPublic displayName: Release vPack jobs: - template: /.pipelines/templates/approvalJob.yml@self @@ -398,20 +403,22 @@ extends: # - template: templates/release-UpdateDepsJson.yml - stage: UploadBuildInfoJson - dependsOn: PublishGitHubRelease + dependsOn: PushGitTagAndMakeDraftPublic displayName: Upload BuildInfo.json jobs: - template: /.pipelines/templates/release-upload-buildinfo.yml@self - stage: ReleaseSymbols - dependsOn: PublishGitHubRelease + dependsOn: PushGitTagAndMakeDraftPublic displayName: Release Symbols jobs: - template: /.pipelines/templates/release-symbols.yml@self - stage: ChangesToMaster displayName: Ensure changes are in GH master - dependsOn: ['PublishNuGet', 'PublishPMC'] + dependsOn: + - PublishNuGet + - PublishPMC jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -422,7 +429,7 @@ extends: - stage: ReleaseSnap displayName: Release Snap - dependsOn: 'ChangesToMaster' + dependsOn: ChangesToMaster jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -433,7 +440,7 @@ extends: - stage: ReleaseToMU displayName: Release to MU - dependsOn: ['PublishNuGet', 'PublishPMC', 'ChangesToMaster'] + dependsOn: PushGitTagAndMakeDraftPublic # This only needs the blob to be available jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: @@ -443,7 +450,10 @@ extends: - stage: ReleaseClose displayName: Finish Release - dependsOn: ['ReleaseToMU', 'ReleaseSymbols', 'ReleaseSnap'] + dependsOn: + - ReleaseToMU + - ReleaseSymbols + - ReleaseSnap jobs: - template: /.pipelines/templates/approvalJob.yml@self parameters: From e17151ae0755874d883c1cf05b5fb78d96ae9b41 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 14:32:55 -0600 Subject: [PATCH 141/316] fixed dependencies 2 --- .pipelines/PowerShell-Release-Official.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index 3f12117c5fa..aba64d4d49d 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -285,7 +285,9 @@ extends: - stage: PublishGitHubRelease displayName: Publish GitHub Release - dependsOn: DownloadPackages + dependsOn: + - DownloadPackages + - UpdateChangeLog variables: ob_release_environment: Production jobs: From 9b73d87fd95b00c8a36adcc46d01c33b9017a3a1 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 14:35:54 -0600 Subject: [PATCH 142/316] fixed dependcies 3 --- .pipelines/PowerShell-Release-Official.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index aba64d4d49d..f025c42f460 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -275,14 +275,6 @@ extends: Update and merge the changelog for the release. This step is required for creating GitHub draft release. - - stage: BlobPublic - displayName: Make Blob Public - dependsOn: UpdateChangeLog - jobs: - - template: /.pipelines/templates/release-MakeBlobPublic.yml@self - parameters: - SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} - - stage: PublishGitHubRelease displayName: Publish GitHub Release dependsOn: @@ -311,7 +303,17 @@ extends: jobName: DraftPublic instructions: | Make the GitHub Release Draft Public - + + - stage: BlobPublic + displayName: Make Blob Public + dependsOn: + - UpdateChangeLog + - PushGitTagAndMakeDraftPublic + jobs: + - template: /.pipelines/templates/release-MakeBlobPublic.yml@self + parameters: + SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} + - stage: PublishNuGet displayName: Publish NuGet dependsOn: PushGitTagAndMakeDraftPublic From 1037cbee898eba537f83731808f4e5539081f68a Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 15:04:00 -0600 Subject: [PATCH 143/316] enabled sbom --- .pipelines/templates/release-download-packages.yml | 5 +++++ .pipelines/templates/variable/release-shared.yml | 9 ++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.pipelines/templates/release-download-packages.yml b/.pipelines/templates/release-download-packages.yml index 099aaff09b1..27a3098d1e1 100644 --- a/.pipelines/templates/release-download-packages.yml +++ b/.pipelines/templates/release-download-packages.yml @@ -8,8 +8,13 @@ jobs: - template: ./variable/release-shared.yml@self parameters: REPOROOT: $(Build.SourcesDirectory) + SBOM: true steps: + - pwsh: | + Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + displayName: Capture environment variables + - download: PSPackagesOfficial artifact: drop_linux_package_deb displayName: Download linux deb packages diff --git a/.pipelines/templates/variable/release-shared.yml b/.pipelines/templates/variable/release-shared.yml index f1a8c1311d2..cbb2816a4e6 100644 --- a/.pipelines/templates/variable/release-shared.yml +++ b/.pipelines/templates/variable/release-shared.yml @@ -2,10 +2,13 @@ parameters: - name: REPOROOT type: string default: $(Build.SourcesDirectory)\PowerShell + - name: SBOM + type: boolean + default: false variables: - name: ob_sdl_sbom_enabled - value: false + value: ${{ parameters.SBOM }} - name: runCodesignValidationInjection value: false - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE @@ -20,8 +23,8 @@ variables: - name: ob_sdl_binskim_enabled value: false - name: ob_sdl_tsa_configFile - value: $(REPOROOT)\.config\tsaoptions.json + value: ${{ parameters.REPOROOT }}\.config\tsaoptions.json - name: ob_sdl_credscan_suppressionsFile - value: $(REPOROOT)\.config\suppress.json + value: ${{ parameters.REPOROOT }}\.config\suppress.json - name: ob_sdl_codeql_compiled_enabled value: false From 79b1e2f30290f1706b783db5417968bb0ff0137f Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 15:12:45 -0600 Subject: [PATCH 144/316] disabled signing setup --- .pipelines/templates/variable/release-shared.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/templates/variable/release-shared.yml b/.pipelines/templates/variable/release-shared.yml index cbb2816a4e6..92ab56199d4 100644 --- a/.pipelines/templates/variable/release-shared.yml +++ b/.pipelines/templates/variable/release-shared.yml @@ -7,6 +7,8 @@ parameters: default: false variables: + - name: ob_signing_setup_enabled + value: false - name: ob_sdl_sbom_enabled value: ${{ parameters.SBOM }} - name: runCodesignValidationInjection From 0b0b8b7b5796b8afb59f1b82ee5e7b3e0812747b Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 15:40:47 -0600 Subject: [PATCH 145/316] fixed checkout --- .pipelines/templates/release-checkout-pwsh-repo.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.pipelines/templates/release-checkout-pwsh-repo.yml b/.pipelines/templates/release-checkout-pwsh-repo.yml index 9cbcde606af..973a938571b 100644 --- a/.pipelines/templates/release-checkout-pwsh-repo.yml +++ b/.pipelines/templates/release-checkout-pwsh-repo.yml @@ -1,9 +1,13 @@ steps: - pwsh: | Write-Verbose -Verbose "Deploy Box Product Pathway Does Not Support the `"checkout`" task" - git clone --depth 1 https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' - cd $(Pipeline.Workspace)/PowerShell - $branchParts = $ENV:BUILD_SOURCEBRANCH -split '/' + if ($ENV:BUILD_REASON -eq 'PullRequest') { + throw 'We don't support PRs' + } + + $branchParts = $ENV:BUILD_SOURCEBRANCH -replace '^refs/head/' $branchName = "$($branchParts[-2])/$($branchParts[-1])" - git checkout $branchName + Write-Verbose -Verbose "Branch Name: $branchName" + git clone --depth 1 --branch $branchName https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' + cd $(Pipeline.Workspace)/PowerShell displayName: Checkout Powershell Repository From 6733fb1676f476fa5c53afbb9756d5052943a009 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 16:02:35 -0600 Subject: [PATCH 146/316] single qute problem --- .pipelines/templates/release-checkout-pwsh-repo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-checkout-pwsh-repo.yml b/.pipelines/templates/release-checkout-pwsh-repo.yml index 973a938571b..712b0d47ea5 100644 --- a/.pipelines/templates/release-checkout-pwsh-repo.yml +++ b/.pipelines/templates/release-checkout-pwsh-repo.yml @@ -2,7 +2,7 @@ steps: - pwsh: | Write-Verbose -Verbose "Deploy Box Product Pathway Does Not Support the `"checkout`" task" if ($ENV:BUILD_REASON -eq 'PullRequest') { - throw 'We don't support PRs' + throw 'We dont support PRs' } $branchParts = $ENV:BUILD_SOURCEBRANCH -replace '^refs/head/' From ca2aa0808c062c4563ec7f26101ba5a3fbfe32fd Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 16:29:43 -0600 Subject: [PATCH 147/316] checking env build source --- .pipelines/templates/release-checkout-pwsh-repo.yml | 1 + .pipelines/templates/release-githubtasks.yml | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.pipelines/templates/release-checkout-pwsh-repo.yml b/.pipelines/templates/release-checkout-pwsh-repo.yml index 712b0d47ea5..fe7e2f88cd5 100644 --- a/.pipelines/templates/release-checkout-pwsh-repo.yml +++ b/.pipelines/templates/release-checkout-pwsh-repo.yml @@ -5,6 +5,7 @@ steps: throw 'We dont support PRs' } + Write-Verbose -Verbose $ENV:BUILD_SOURCEBRANCH $branchParts = $ENV:BUILD_SOURCEBRANCH -replace '^refs/head/' $branchName = "$($branchParts[-2])/$($branchParts[-1])" Write-Verbose -Verbose "Branch Name: $branchName" diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 7ea4a671cb8..efb78290f9f 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -5,10 +5,10 @@ jobs: pool: type: release os: windows - templateContext: - inputs: - - input: pipelineArtifact - artifactName: drop_DownloadPackages_upload_packages + # templateContext: + # inputs: + # - input: pipelineArtifact + # artifactName: drop_DownloadPackages_upload_packages variables: - template: ./variable/release-shared.yml@self From 44f36ccec4323b946a24b7442381b2fd69a1ce1b Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 16:39:02 -0600 Subject: [PATCH 148/316] testing checkout --- .pipelines/templates/release-githubtasks.yml | 24 ++++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index efb78290f9f..193c03f97e8 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -5,28 +5,28 @@ jobs: pool: type: release os: windows - # templateContext: - # inputs: - # - input: pipelineArtifact - # artifactName: drop_DownloadPackages_upload_packages + templateContext: + inputs: + - input: pipelineArtifact + pipeline: PSPackagesOfficial # temporary need to remove later + artifactName: drop_DownloadPackages_upload_packages variables: - template: ./variable/release-shared.yml@self steps: - - template: release-install-pwsh.yml - - - template: release-checkout-pwsh-repo.yml - - - template: release-SetReleaseTagAndContainerName.yml - - task: PowerShell@2 inputs: targetType: inline - pwsh: true script: | - Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' + - template: release-install-pwsh.yml + + - template: release-checkout-pwsh-repo.yml + + - template: release-SetReleaseTagAndContainerName.yml + - task: PowerShell@2 inputs: targetType: inline From 236a1a95af24b34fbd97a298b476efcf87fc56ce Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 16:40:07 -0600 Subject: [PATCH 149/316] using old pipeline --- .pipelines/templates/release-githubtasks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 193c03f97e8..63299dabbd9 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -9,7 +9,8 @@ jobs: inputs: - input: pipelineArtifact pipeline: PSPackagesOfficial # temporary need to remove later - artifactName: drop_DownloadPackages_upload_packages + artifactName: drop_upload_upload_packages + # artifactName: drop_DownloadPackages_upload_packages variables: - template: ./variable/release-shared.yml@self From 98a23ce359459ca6c6191d970878326cc340ede7 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 16:45:12 -0600 Subject: [PATCH 150/316] checkout --- .pipelines/templates/release-checkout-pwsh-repo.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/templates/release-checkout-pwsh-repo.yml b/.pipelines/templates/release-checkout-pwsh-repo.yml index fe7e2f88cd5..6fb2d1e1da2 100644 --- a/.pipelines/templates/release-checkout-pwsh-repo.yml +++ b/.pipelines/templates/release-checkout-pwsh-repo.yml @@ -7,7 +7,6 @@ steps: Write-Verbose -Verbose $ENV:BUILD_SOURCEBRANCH $branchParts = $ENV:BUILD_SOURCEBRANCH -replace '^refs/head/' - $branchName = "$($branchParts[-2])/$($branchParts[-1])" Write-Verbose -Verbose "Branch Name: $branchName" git clone --depth 1 --branch $branchName https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' cd $(Pipeline.Workspace)/PowerShell From 4e9b5c391ba8f1ab68ea7a5a8f567adf393c0a5b Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 16:51:47 -0600 Subject: [PATCH 151/316] branch name --- .pipelines/templates/release-checkout-pwsh-repo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-checkout-pwsh-repo.yml b/.pipelines/templates/release-checkout-pwsh-repo.yml index 6fb2d1e1da2..58645bf43fb 100644 --- a/.pipelines/templates/release-checkout-pwsh-repo.yml +++ b/.pipelines/templates/release-checkout-pwsh-repo.yml @@ -6,7 +6,7 @@ steps: } Write-Verbose -Verbose $ENV:BUILD_SOURCEBRANCH - $branchParts = $ENV:BUILD_SOURCEBRANCH -replace '^refs/head/' + $branchName = $ENV:BUILD_SOURCEBRANCH -replace '^refs/head/' Write-Verbose -Verbose "Branch Name: $branchName" git clone --depth 1 --branch $branchName https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' cd $(Pipeline.Workspace)/PowerShell From 47d187a44082fecfc78701aa2c4b3bf583c68827 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 16:56:49 -0600 Subject: [PATCH 152/316] refs/heads --- .pipelines/templates/release-checkout-pwsh-repo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-checkout-pwsh-repo.yml b/.pipelines/templates/release-checkout-pwsh-repo.yml index 58645bf43fb..9a7486887a6 100644 --- a/.pipelines/templates/release-checkout-pwsh-repo.yml +++ b/.pipelines/templates/release-checkout-pwsh-repo.yml @@ -6,7 +6,7 @@ steps: } Write-Verbose -Verbose $ENV:BUILD_SOURCEBRANCH - $branchName = $ENV:BUILD_SOURCEBRANCH -replace '^refs/head/' + $branchName = $ENV:BUILD_SOURCEBRANCH -replace '^refs/heads/' Write-Verbose -Verbose "Branch Name: $branchName" git clone --depth 1 --branch $branchName https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/PowerShell '$(Pipeline.Workspace)/PowerShell' cd $(Pipeline.Workspace)/PowerShell From 30407236b3b1f2213fda7bd732720ed6495348b0 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 15 Nov 2024 17:03:10 -0600 Subject: [PATCH 153/316] fixed back to original intention --- .pipelines/templates/release-githubtasks.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 63299dabbd9..42db2b20b73 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -8,9 +8,7 @@ jobs: templateContext: inputs: - input: pipelineArtifact - pipeline: PSPackagesOfficial # temporary need to remove later - artifactName: drop_upload_upload_packages - # artifactName: drop_DownloadPackages_upload_packages + artifactName: drop_DownloadPackages_upload_packages variables: - template: ./variable/release-shared.yml@self From f78f364fd785552271b71eb8af045a211b5f2f14 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 18 Nov 2024 13:26:59 -0600 Subject: [PATCH 154/316] separating out ev2 --- .../PowerShell-Release-Official-Azure.yml | 130 ++++++++++++++++++ .pipelines/templates/release-prep-for-ev2.yml | 8 ++ .pipelines/templates/release-publish-pmc.yml | 11 +- 3 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 .pipelines/PowerShell-Release-Official-Azure.yml create mode 100644 .pipelines/templates/release-prep-for-ev2.yml diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml new file mode 100644 index 00000000000..890c15a6bd6 --- /dev/null +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -0,0 +1,130 @@ +trigger: none + +parameters: # parameters are shown up in ADO UI in a build queue time + - name: 'debug' + displayName: 'Enable debug output' + type: boolean + default: false + - name: InternalSDKBlobURL + displayName: URL to the blob having internal .NET SDK + type: string + default: ' ' + - name: ReleaseTagVar + displayName: Release Tag + type: string + default: 'fromBranch' + - name: SKIP_SIGNING + displayName: Skip Signing + type: string + default: 'NO' + +variables: + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] + - name: system.debug + value: ${{ parameters.debug }} + - name: ENABLE_PRS_DELAYSIGN + value: 1 + - name: ROOT + value: $(Build.SourcesDirectory) + - name: REPOROOT + value: $(Build.SourcesDirectory) + - name: OUTPUTROOT + value: $(REPOROOT)\out + - name: NUGET_XMLDOC_MODE + value: none + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: WindowsContainerImage + value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + - name: LinuxContainerImage + value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 + - name: ReleaseTagVar + value: ${{ parameters.ReleaseTagVar }} + - group: PoolNames + +resources: + repositories: + - repository: templates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + + pipelines: + - pipeline: CoOrdinatedBuildPipeline + source: 'PowerShell-Coordinated Packages-Official' + + - pipeline: PSPackagesOfficial + source: 'PowerShell-Packages-Official' + trigger: + branches: + include: + - master + - releases/* + +extends: + template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates + parameters: + featureFlags: + WindowsHostVersion: + Version: 2022 + Network: Netlock + cloudvault: + enabled: false + globalSdl: + disableLegacyManifest: true + # disabled Armory as we dont have any ARM templates to scan. It fails on some sample ARM templates. + armory: + enabled: false + asyncSdl: + enabled: true + tsaOptionsFile: .config/tsaoptions.json + tsa: + enabled: true + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + binskim: + break: false # always break the build on binskim issues in addition to TSA upload + policheck: + break: true # always break the build on policheck issues. You can disable it by setting to 'false' + # suppression: + # suppressionFile: $(Build.SourcesDirectory)\.gdn\global.gdnsuppress + tsaOptionsFile: .config\tsaoptions.json + stages: + - stage: PrepForEV2 + displayName: Prep for EV2 + dependsOn: [] + variables: + ob_release_environment: Test + jobs: + - template: /.pipelines/templates/release-prep-for-ev2.yml@self + + # - stage: BlobPublic + # displayName: Make Blob Public + # dependsOn: PrepForEV2 + # jobs: + # - template: /.pipelines/templates/release-MakeBlobPublic.yml@self + + - stage: PublishPMC + displayName: Publish PMC + dependsOn: + - PrepForEV2 + - BlobPublic + jobs: + - template: /.pipelines/templates/release-publish-pmc.yml@self + + # - stage: UploadBuildInfoJson + # dependsOn: + # - PrepForEV2 + # - BlobPublic + # displayName: Upload BuildInfo.json + # jobs: + # - template: /.pipelines/templates/release-upload-buildinfo.yml@self diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml new file mode 100644 index 00000000000..922b88eb064 --- /dev/null +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -0,0 +1,8 @@ +jobs: +- job: PrepForEV2 + pool: + type: release + steps: + - pwsh: | + Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose + displayName: 'Capture Environment Variables' diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 27311611e61..8a6972575fc 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -1,18 +1,9 @@ -parameters: - - name: skipPublish - default: false - type: boolean - jobs: - job: PMCPublish displayName: Publish to PMC condition: succeeded() pool: - type: linux - isCustom: true - name: PowerShell1ES - demands: - - ImageOverride -equals PSMMSUbuntu20.04-Secure + type: release variables: - name: runCodesignValidationInjection value: false From c7902dae3a6425a93e928050ce17f1829cfd24a4 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 18 Nov 2024 13:28:45 -0600 Subject: [PATCH 155/316] added ob_release to test --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 890c15a6bd6..568bc01f6c3 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -118,6 +118,8 @@ extends: dependsOn: - PrepForEV2 - BlobPublic + variables: + ob_release_environment: Test jobs: - template: /.pipelines/templates/release-publish-pmc.yml@self From 4f18e5ed892071511e2265444390ebfbd3a2dc88 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 18 Nov 2024 15:05:07 -0600 Subject: [PATCH 156/316] removed release and ob for first stage --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 -- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 568bc01f6c3..b83bf6e9b7a 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -102,8 +102,6 @@ extends: - stage: PrepForEV2 displayName: Prep for EV2 dependsOn: [] - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/release-prep-for-ev2.yml@self diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 922b88eb064..3af2c1babc1 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -1,7 +1,7 @@ jobs: - job: PrepForEV2 pool: - type: release + type: linux steps: - pwsh: | Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose From 3a4b0d3e60c3cd895874c410632a19c4dd083dd7 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 10:05:03 -0600 Subject: [PATCH 157/316] correct and to And --- .pipelines/templates/release-publish-pmc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 8a6972575fc..6d02c80d7fc 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -28,7 +28,7 @@ jobs: - checkout: self ## the global setting on lfs didn't work lfs: false - - template: release-SetReleaseTagAndContainerName.yml + - template: release-SetReleaseTagandContainerName.yml - pwsh: | $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' From 881504e3a3d3831833e9d5ecd60c20e2fad6acf1 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 10:09:48 -0600 Subject: [PATCH 158/316] removed ob_outputdirectory --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index b83bf6e9b7a..5c4a2ef0e87 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -39,8 +39,6 @@ variables: value: none - name: skipNugetSecurityAnalysis value: true - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: WindowsContainerImage value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - name: LinuxContainerImage From f6eb8366d7ec5099d4efa93d75ae3685c15e1d2d Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 10:10:19 -0600 Subject: [PATCH 159/316] removed ob_outputdirectory --- .pipelines/templates/release-publish-pmc.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 6d02c80d7fc..1a9ebc7f13a 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -13,8 +13,6 @@ jobs: value: 1 - group: 'mscodehub-code-read-akv' - group: 'packages.microsoft.com' - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: ob_sdl_codeSignValidation_enabled value: false - name: ob_sdl_binskim_enabled From 7a76ef54226a06b7894e3166bb68503303b3d9a7 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 10:20:50 -0600 Subject: [PATCH 160/316] adding ob_outputDirectory --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 ++ .pipelines/templates/release-prep-for-ev2.yml | 3 +++ .pipelines/templates/release-publish-pmc.yml | 2 ++ 3 files changed, 7 insertions(+) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 5c4a2ef0e87..b83bf6e9b7a 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -39,6 +39,8 @@ variables: value: none - name: skipNugetSecurityAnalysis value: true + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: WindowsContainerImage value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - name: LinuxContainerImage diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 3af2c1babc1..d48d609bca7 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -2,6 +2,9 @@ jobs: - job: PrepForEV2 pool: type: linux + variables: + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' steps: - pwsh: | Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 1a9ebc7f13a..6d02c80d7fc 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -13,6 +13,8 @@ jobs: value: 1 - group: 'mscodehub-code-read-akv' - group: 'packages.microsoft.com' + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: ob_sdl_codeSignValidation_enabled value: false - name: ob_sdl_binskim_enabled From f77b34c1bf6eb2f905a52b1aff008519a5dcb1b8 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 10:21:56 -0600 Subject: [PATCH 161/316] added Test_ --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index b83bf6e9b7a..bc285913899 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -111,7 +111,7 @@ extends: # jobs: # - template: /.pipelines/templates/release-MakeBlobPublic.yml@self - - stage: PublishPMC + - stage: Test_PublishPMC displayName: Publish PMC dependsOn: - PrepForEV2 From 6b535a8e8d30d810499080ede2b584f106616b40 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 11:45:37 -0600 Subject: [PATCH 162/316] added ev2 folders --- .../templates/EV2Specs/RollOutSpec.json | 28 ++++ .../templates/EV2Specs/ScopeBindings.json | 23 ++++ .../ServiceGroupRoot/Shell/Run/Run.ps1 | 58 ++++++++ .../templates/EV2Specs/ServiceModel.json | 51 +++++++ .../EV2Specs/UploadLinux.Rollout.json | 61 +++++++++ .pipelines/templates/release-prep-for-ev2.yml | 128 +++++++++++++++++- .pipelines/templates/release-publish-pmc.yml | 95 +++++++------ 7 files changed, 393 insertions(+), 51 deletions(-) create mode 100644 .pipelines/templates/EV2Specs/RollOutSpec.json create mode 100644 .pipelines/templates/EV2Specs/ScopeBindings.json create mode 100644 .pipelines/templates/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 create mode 100644 .pipelines/templates/EV2Specs/ServiceModel.json create mode 100644 .pipelines/templates/EV2Specs/UploadLinux.Rollout.json diff --git a/.pipelines/templates/EV2Specs/RollOutSpec.json b/.pipelines/templates/EV2Specs/RollOutSpec.json new file mode 100644 index 00000000000..9ed971068cc --- /dev/null +++ b/.pipelines/templates/EV2Specs/RollOutSpec.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/rolloutSpecification.json", + "contentVersion": "1.0.0.0", + "rolloutMetadata": { + "serviceModelPath": "ServiceModel.json", + "ScopeBindingsPath": "ScopeBindings.json", + "name": "OneBranch-Demo-Container-Deployment", + "rolloutType": "Major", + "buildSource": { + "parameters": { + "versionFile": "buildver.txt" + } + }, + "Notification": { + "Email": { + "To": "default" + } + } + }, + "orchestratedSteps": [ + { + "name": "UploadLinuxContainer", + "targetType": "ServiceResource", + "targetName": "LinuxContainerUpload", + "actions": ["Shell/Run"] + } + ] +} diff --git a/.pipelines/templates/EV2Specs/ScopeBindings.json b/.pipelines/templates/EV2Specs/ScopeBindings.json new file mode 100644 index 00000000000..c3a98555867 --- /dev/null +++ b/.pipelines/templates/EV2Specs/ScopeBindings.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/scopeBindings.json", + "contentVersion": "0.0.0.1", + "scopeBindings": [ + { + "scopeTagName": "Global", + "bindings": [ + { + "find": "__SUBSCRIPTION_ID__", + "replaceWith": "$azureSubscriptionId()" + }, + { + "find": "__RESOURCE_GROUP__", + "replaceWith": "$azureResourceGroup()" + }, + { + "find": "__BUILD_VERSION__", + "replaceWith": "$buildVersion()" + } + ] + } + ] +} diff --git a/.pipelines/templates/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/templates/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 new file mode 100644 index 00000000000..63a8316b828 --- /dev/null +++ b/.pipelines/templates/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -0,0 +1,58 @@ +# ensure SAS variables were passed in +if ($null -eq $env:RELEASE_TAG) +{ + Write-Verbose -Verbose "RELEASE_TAG variable didn't get passed correctly" + return 1 +} + +if ($null -eq $env:AAD_CLIENT_ID) +{ + Write-Verbose -Verbose "AAD_CLIENT_ID variable didn't get passed correctly" + return 1 +} + +if ($null -eq $env:BLOB_FOLDER_NAME) +{ + Write-Verbose -Verbose "BLOB_FOLDER_NAME variable didn't get passed correctly" + return 1 +} + +if ($null -eq $env:LTS) +{ + Write-Verbose -Verbose "LTS variable didn't get passed correctly" + return 1 +} + +if ($null -eq $env:FOR_PRODUCTION) +{ + Write-Verbose -Verbose "FOR_PRODUCTION variable didn't get parsed properly" + return 1 +} + +if ($null -eq $env:SKIP_PUBLISH) +{ + Write-Verbose -Verbose "SKIP_PUBLISH variable didn't get parsed properly" + return 1 +} + +if ($null -eq $env:MAPPING_FILE_PATH) +{ + Write-Verbose -Verbose "MAPPING_FILE_PATH variable didn't get parsed properly" + return 1 +} + +try { + Write-Verbose -Verbose "ReleaseTag: $env:RELEASE_TAG" + Write-Verbose -Verbose "AAD_Client_ID: $env:AAD_CLIENT_ID" + Write-Verbose -Verbose "Blob_folder_name: $env:BLOB_FOLDER_NAME" + Write-Verbose -Verbose "LTS: $env:LTS" + Write-Verbose -Verbose "for_production: $env:FOR_PRODUCTION" + Write-Verbose -Verbose "Skip_publish: $env:SKIP_PUBLISH" + Write-Verbose -Verbose "mapping file: $env:MAPPING_FILE_PATH" +} +catch { + Write-Error -ErrorAction Stop $_.Exception.Message + return 1 +} + +return 0 diff --git a/.pipelines/templates/EV2Specs/ServiceModel.json b/.pipelines/templates/EV2Specs/ServiceModel.json new file mode 100644 index 00000000000..515a7fb34ab --- /dev/null +++ b/.pipelines/templates/EV2Specs/ServiceModel.json @@ -0,0 +1,51 @@ +{ + "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/serviceModel.json", + "contentVersion": "1.0.0.0", + "serviceMetadata": { + "serviceGroup": "OneBranch-PowerShellDocker", + "environment": "Test" + }, + "serviceResourceGroupDefinitions": [ + { + "name": "OneBranch-PowerShellDocker-RGDef", + "serviceResourceDefinitions": [ + { + "name": "OneBranch-PowerShellDocker.Shell-SRDef", + "composedOf": { + "extension": { + "shell": [ + { + "type": "Run", + "properties": { + "imageName": "adm-mariner-20-l", + "imageVersion": "v5" + } + } + ] + } + } + } + ] + } + ], + "serviceResourceGroups": [ + { + "azureResourceGroupName": "default", + "location": "East US", + "instanceOf": "OneBranch-PowerShellDocker-RGDef", + "azureSubscriptionId": "default", + "scopeTags": [ + { + "name": "Global" + } + ], + "serviceResources": [ + { + "Name": "LinuxContainerUpload", + "InstanceOf": "OneBranch-PowerShellDocker.Shell-SRDef", + "RolloutParametersPath": "UploadLinux.Rollout.json" + } + ] + } + ] +} diff --git a/.pipelines/templates/EV2Specs/UploadLinux.Rollout.json b/.pipelines/templates/EV2Specs/UploadLinux.Rollout.json new file mode 100644 index 00000000000..35901082628 --- /dev/null +++ b/.pipelines/templates/EV2Specs/UploadLinux.Rollout.json @@ -0,0 +1,61 @@ +{ + "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/rolloutParameters.json", + "contentVersion": "1.0.0.0", + "shellExtensions": [ + { + "name": "Run", + "type": "Run", + "properties": { + "maxExecutionTime": "PT2H" + }, + "package": { + "reference": { + "path": "Shell/Run.tar" + } + }, + "launch": { + "command": [ + "/bin/bash", + "-c", + "pwsh ./Run/Run.ps1" + ], + "environmentVariables": [ + { + "name": "RELEASE_TAG", + "value": "default" + }, + { + "name": "AAD_CLIENT_ID", + "value" : "default" + }, + { + "name": "BLOB_FOLDER_NAME", + "value": "default" + }, + { + "name": "LTS", + "value": "default" + }, + { + "name": "FOR_PRODUCTION", + "value": "true" + }, + { + "name": "SKIP_PUBLISH", + "value": "true" + }, + { + "name": "MAPPING_FILE_PATH", + "default": "default" + } + ], + "identity": { + "type": "userAssigned", + "userAssignedIdentities": [ + "default" + ] + } + } + } + ] +} diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index d48d609bca7..298c2ea4f12 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -1,11 +1,135 @@ jobs: -- job: PrepForEV2 +- job: CopyEV2FilesToArtifact + displayName: Copy EV2 Files to Artifact pool: - type: linux + type: windows variables: - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' steps: + - checkout: self ## the global setting on lfs didn't work + lfs: false + + - template: release-SetReleaseTagandContainerName.yml + + - pwsh: | + $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' + $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Set Package version + + - pwsh: | + $branch = 'mirror-target' + $gitArgs = "clone", + "--verbose", + "--branch", + "$branch", + "https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools", + '$(Pipeline.Workspace)/tools' + $gitArgs | Write-Verbose -Verbose + git $gitArgs + displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub + - pwsh: | Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' + + - task: onebranch.pipeline.signing@1 + displayName: Sign 1st party files + inputs: + command: 'sign' + signing_profile: external_distribution + files_to_sign: '**\*.ps1' + search_root: '$(Build.SourcesDirectory)/EV2Specs/ServiceGroupRoot/Shell' + + - pwsh: | + $srcPath = Join-Path '$(Build.SourcesDirectory)/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' + $pathToRunTarFile = Join-Path '$(Build.SourcesDirectory)/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" + tar -cvf $pathToRunTarFile -C $srcPath ./Run + displayName: Compress Run script into tar file as needed for EV2 Shell extension + + - pwsh: | + $pathToEV2Folder = Join-Path -Path '$(Build.SourcesDirectory)' -ChildPath 'EV2Specs' + $pathToServiceGroupRootFolder = Join-Path $pathToEV2Folder -ChildPath 'ServiceGroupRoot' + $pathToParametersFolder = Join-Path -Path $pathToServiceGroupRootFolder -ChildPath 'Parameters' + New-Item -Path $pathToParametersFolder -ItemType Directory + $pathToImageMetadataFile = Join-Path -Path $pathToParametersFolder -ChildPath 'ImageMetadata.json' + $pathToChannelJsonFile = Join-Path -Path $pathToParametersFolder -ChildPath 'ChannelInfo.json' + $currentChannel = '$(channel)' + + $channelHash = @{channel=$currentChannel} + $channelHash | ConvertTo-Json | Out-File $pathToChannelJsonFile + + $toolsFolderPath = Join-Path -Path '$(Build.SourcesDirectory)' -ChildPath 'tools' + $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' + $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' + Write-Verbose -Verbose "ipmo path: $buildHelperModulePath" + Import-Module $buildHelperModulePath + if ($currentChannel -eq "stable") + { + .\build.ps1 -UpdateImageMetadata -Channel $currentChannel -StableVersion '$(version)' -MetadataFilePath $pathToImageMetadataFile + } + elseif ($currentChannel -eq "preview") + { + .\build.ps1 -UpdateImageMetadata -Channel $currentChannel -PreviewVersion '$(version)' -MetadataFilePath $pathToImageMetadataFile + } + elseif ($currentChannel -eq "lts") + { + .\build.ps1 -UpdateImageMetadata -Channel $currentChannel -LtsVersion '$(version)' -MetadataFilePath $pathToImageMetadataFile + } + + $imgMetaJsonPathExists = Test-Path -Path $pathToImageMetadataFile + $channelJsonPathExists = Test-Path -Path $pathToChannelJsonFile + displayName: 'Create image metadata json needed for EV2 Shell Extension script' + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(Build.SourcesDirectory)/EV2Specs/ServiceGroupRoot' -ChildPath 'UploadLinux.Rollout.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + $envVarArrayLen = $content.shellExtensions.launch.environmentVariables.Length + for ($i=0; $i -lt $envVarArrayLen; $i++) + { + $name = $($content.shellExtensions.launch.environmentVariables[$i].name) + if ($name -eq "DESTINATION_ACR_NAME") + { + $content.shellExtensions.launch.environmentVariables[$i].value = '$(acr_name)' + } + elseif ($name -eq "MI_NAME"){ + $content.shellExtensions.launch.environmentVariables[$i].value = '$(managed_identity_name)' + } + elseif ($name -eq "MI_CLIENTID") + { + $content.shellExtensions.launch.environmentVariables[$i].value = '$(managed_identity_clientid)' + } + } + + $identityString = "/subscriptions/$(acr_subscription)/resourcegroups/$(acr_resource_group)/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$(managed_identity_name)" + $content.shellExtensions.launch.identity.userAssignedIdentities[0] = $identityString + + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 6 | Out-File $pathToJsonFile + displayName: 'Replace values in UploadLinux.Rollout.json file' + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(Build.SourcesDirectory)/EV2Specs/ServiceGroupRoot' -ChildPath 'RolloutSpec.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + $content.RolloutMetadata.Notification.Email.To = '$(email_address)' + + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 4 | Out-File $pathToJsonFile + displayName: 'Replace values in RolloutSpecPath.json' + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(Build.SourcesDirectory)/EV2Specs/ServiceGroupRoot' -ChildPath 'ServiceModel.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + $content.ServiceResourceGroups[0].AzureResourceGroupName = '$(acr_resource_group)' + $content.ServiceResourceGroups[0].AzureSubscriptionId = '$(acr_subscription)' + + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile + displayName: 'Replace values in ServiceModel.json' + + - task: CopyFiles@2 + inputs: + Contents: 'EV2Specs/**' + TargetFolder: $(ob_outputDirectory) diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 6d02c80d7fc..37b9daa3c17 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -25,57 +25,54 @@ jobs: value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: - - checkout: self ## the global setting on lfs didn't work - lfs: false + # - task: PipAuthenticate@1 + # inputs: + # artifactFeeds: 'pmc' + # pythonDownloadServiceConnections: pmcDownload - - template: release-SetReleaseTagandContainerName.yml + # - pwsh: | + # pip install pmc-cli==1.12.0 + # + # $newPath = (resolve-path '~/.local/bin').providerpath + # $vstsCommandString = "vso[task.setvariable variable=PATH]${env:PATH}:$newPath" + # Write-Host "sending " + $vstsCommandString + # Write-Host "##$vstsCommandString" + # displayName: Install pmc cli + + # - pwsh: | + # $metadata = Get-Content -Path "$(Build.SourcesDirectory)/tools/metadata.json" -Raw | ConvertFrom-Json + # $params = @{ + # ReleaseTag = "$(ReleaseTag)" + # AadClientId = "$(PmcCliClientID)" + # BlobFolderName = "$(ReleaseTag)" + # LTS = $metadata.LTSRelease.Latest + # ForProduction = $true + # SkipPublish = $${{ parameters.skipPublish }} + # MappingFilePath = '$(System.DefaultWorkingDirectory)/tools/packages.microsoft.com/mapping.json' + # } +# + # $params | Out-String -width 9999 -Stream | write-Verbose -Verbose +# + # & '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/releaseLinuxPackages.ps1' @params + # displayName: Run release script - - pwsh: | - $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' - $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: Set Package version - - - pwsh: | - $branch = 'mirror-target' - $gitArgs = "clone", - "--verbose", - "--branch", - "$branch", - "https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools", - '$(Pipeline.Workspace)/tools' - $gitArgs | Write-Verbose -Verbose - git $gitArgs - displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub - - - task: PipAuthenticate@1 + - task: DownloadPipelineArtifact@2 inputs: - artifactFeeds: 'pmc' - pythonDownloadServiceConnections: pmcDownload - - - pwsh: | - pip install pmc-cli==1.12.0 - - $newPath = (resolve-path '~/.local/bin').providerpath - $vstsCommandString = "vso[task.setvariable variable=PATH]${env:PATH}:$newPath" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: Install pmc cli + targetPath: '$(Pipeline.Workspace)' + artifact: drop_PrepForEV2_CopyEv2FilesToArtifact + displayName: 'Download drop_PrepForEV2_CopyEv2FilesToArtifact artifact that has all files needed' - - pwsh: | - $metadata = Get-Content -Path "$(Build.SourcesDirectory)/tools/metadata.json" -Raw | ConvertFrom-Json - $params = @{ - ReleaseTag = "$(ReleaseTag)" - AadClientId = "$(PmcCliClientID)" - BlobFolderName = "$(ReleaseTag)" - LTS = $metadata.LTSRelease.Latest - ForProduction = $true - SkipPublish = $${{ parameters.skipPublish }} - MappingFilePath = '$(System.DefaultWorkingDirectory)/tools/packages.microsoft.com/mapping.json' - } - - $params | Out-String -width 9999 -Stream | write-Verbose -Verbose + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + targetPath: '$(Pipeline.Workspace)' + displayName: 'Download to get EV2 Files' + - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 - & '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/releaseLinuxPackages.ps1' @params - displayName: Run release script + displayName: 'Ev2: Push to ACR' + inputs: + UseServerMonitorTask: true + EndpointProviderType: ApprovalService + ApprovalServiceEnvironment: Production + ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot' + RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From 69c6d55bcb39fb9be6def157cfaf967d2c5a1481 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 11:47:11 -0600 Subject: [PATCH 163/316] approval service env to Test --- .pipelines/templates/release-publish-pmc.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 37b9daa3c17..d78173427d6 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -67,12 +67,13 @@ jobs: buildType: 'current' targetPath: '$(Pipeline.Workspace)' displayName: 'Download to get EV2 Files' + - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 displayName: 'Ev2: Push to ACR' inputs: UseServerMonitorTask: true EndpointProviderType: ApprovalService - ApprovalServiceEnvironment: Production + ApprovalServiceEnvironment: Test ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot' RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From a1ce30147fcaff57a13e6865d08245d38a79cdd7 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 11:49:07 -0600 Subject: [PATCH 164/316] removed approval service --- .pipelines/templates/release-publish-pmc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index d78173427d6..7fd290917b7 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -73,7 +73,7 @@ jobs: displayName: 'Ev2: Push to ACR' inputs: UseServerMonitorTask: true - EndpointProviderType: ApprovalService + # EndpointProviderType: ApprovalService ApprovalServiceEnvironment: Test ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot' RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From fc79b50a77528952caa1199ce8a05779e87ce75e Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 11:50:48 -0600 Subject: [PATCH 165/316] changed inputs --- .pipelines/templates/release-publish-pmc.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 7fd290917b7..b684bb41183 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -73,7 +73,9 @@ jobs: displayName: 'Ev2: Push to ACR' inputs: UseServerMonitorTask: true + EnableStrictValidation: false + ValidateOnly: false # EndpointProviderType: ApprovalService - ApprovalServiceEnvironment: Test + # ApprovalServiceEnvironment: Test ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot' RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From b38dfb0ea2f6200c6cbb68666ef8125f4c9daa79 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 11:52:19 -0600 Subject: [PATCH 166/316] removed invalid inputs --- .pipelines/templates/release-publish-pmc.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index b684bb41183..4d7ad37477f 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -50,9 +50,9 @@ jobs: # SkipPublish = $${{ parameters.skipPublish }} # MappingFilePath = '$(System.DefaultWorkingDirectory)/tools/packages.microsoft.com/mapping.json' # } -# + # # $params | Out-String -width 9999 -Stream | write-Verbose -Verbose -# + # # & '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/releaseLinuxPackages.ps1' @params # displayName: Run release script @@ -75,7 +75,5 @@ jobs: UseServerMonitorTask: true EnableStrictValidation: false ValidateOnly: false - # EndpointProviderType: ApprovalService - # ApprovalServiceEnvironment: Test ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot' RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From 1eb57c2ecc172bfcde2b8c77c7b71a4ec5eeeb43 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 11:54:01 -0600 Subject: [PATCH 167/316] modified inputs --- .pipelines/templates/release-publish-pmc.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 4d7ad37477f..67716540909 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -69,11 +69,12 @@ jobs: displayName: 'Download to get EV2 Files' - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 - - displayName: 'Ev2: Push to ACR' + displayName: 'Ev2: Push to PMC' inputs: UseServerMonitorTask: true EnableStrictValidation: false ValidateOnly: false + ConnectedServiceName: 'Onebranch Release EV2' + ServiceRootLocation: 'LinkedArtifact' ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot' RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From fdc76844acbfd064268ff1c9f9cc96a314ab6658 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 11:58:18 -0600 Subject: [PATCH 168/316] moved jsons to correct location --- .../templates/EV2Specs/{ => ServiceGroupRoot}/RollOutSpec.json | 0 .../templates/EV2Specs/{ => ServiceGroupRoot}/ScopeBindings.json | 0 .../templates/EV2Specs/{ => ServiceGroupRoot}/ServiceModel.json | 0 .../EV2Specs/{ => ServiceGroupRoot}/UploadLinux.Rollout.json | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename .pipelines/templates/EV2Specs/{ => ServiceGroupRoot}/RollOutSpec.json (100%) rename .pipelines/templates/EV2Specs/{ => ServiceGroupRoot}/ScopeBindings.json (100%) rename .pipelines/templates/EV2Specs/{ => ServiceGroupRoot}/ServiceModel.json (100%) rename .pipelines/templates/EV2Specs/{ => ServiceGroupRoot}/UploadLinux.Rollout.json (100%) diff --git a/.pipelines/templates/EV2Specs/RollOutSpec.json b/.pipelines/templates/EV2Specs/ServiceGroupRoot/RollOutSpec.json similarity index 100% rename from .pipelines/templates/EV2Specs/RollOutSpec.json rename to .pipelines/templates/EV2Specs/ServiceGroupRoot/RollOutSpec.json diff --git a/.pipelines/templates/EV2Specs/ScopeBindings.json b/.pipelines/templates/EV2Specs/ServiceGroupRoot/ScopeBindings.json similarity index 100% rename from .pipelines/templates/EV2Specs/ScopeBindings.json rename to .pipelines/templates/EV2Specs/ServiceGroupRoot/ScopeBindings.json diff --git a/.pipelines/templates/EV2Specs/ServiceModel.json b/.pipelines/templates/EV2Specs/ServiceGroupRoot/ServiceModel.json similarity index 100% rename from .pipelines/templates/EV2Specs/ServiceModel.json rename to .pipelines/templates/EV2Specs/ServiceGroupRoot/ServiceModel.json diff --git a/.pipelines/templates/EV2Specs/UploadLinux.Rollout.json b/.pipelines/templates/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json similarity index 100% rename from .pipelines/templates/EV2Specs/UploadLinux.Rollout.json rename to .pipelines/templates/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json From 189fd2d0db3acc1d2217704bd1bc3c44479f0b1b Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 12:03:05 -0600 Subject: [PATCH 169/316] copied verbatim from eng hub --- .pipelines/templates/release-publish-pmc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 67716540909..2a90f690883 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -76,5 +76,6 @@ jobs: ValidateOnly: false ConnectedServiceName: 'Onebranch Release EV2' ServiceRootLocation: 'LinkedArtifact' + RolloutSpecType: 'RSPath' ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot' RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From 878468427f3b5475b54a769c1e586cdc2cae1e36 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 12:04:13 -0600 Subject: [PATCH 170/316] removed dependency to blob public --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index bc285913899..5eb31daabb5 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -115,7 +115,7 @@ extends: displayName: Publish PMC dependsOn: - PrepForEV2 - - BlobPublic + # - BlobPublic variables: ob_release_environment: Test jobs: From a0ca13a63a58d16c84f488c4db1aab9470827b9e Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 13:58:42 -0600 Subject: [PATCH 171/316] added copy --- .pipelines/templates/release-prep-for-ev2.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 298c2ea4f12..bb84c9e68b0 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -30,13 +30,21 @@ jobs: $gitArgs | Write-Verbose -Verbose git $gitArgs displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub - + - pwsh: | Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' - task: onebranch.pipeline.signing@1 - displayName: Sign 1st party files + displayName: Sign Release Tools + inputs: + command: 'sign' + signing_profile: external_distribution + files_to_sign: '**\*.ps1' + search_root: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4' + + - task: onebranch.pipeline.signing@1 + displayName: Sign Run.ps1 inputs: command: 'sign' signing_profile: external_distribution @@ -129,6 +137,11 @@ jobs: $content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile displayName: 'Replace values in ServiceModel.json' + - task: CopyFiles@2 + inputs: + Contents: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/**' + TargetFolder: 'EV2Specs/ServiceGroupRoot/tools' + - task: CopyFiles@2 inputs: Contents: 'EV2Specs/**' From 54a1c95c6906587d36ba902c801109aaf5f596b0 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 19 Nov 2024 14:00:15 -0600 Subject: [PATCH 172/316] added docker deployment test service connection --- .pipelines/templates/release-publish-pmc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 2a90f690883..e0c26b1a3f4 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -74,7 +74,7 @@ jobs: UseServerMonitorTask: true EnableStrictValidation: false ValidateOnly: false - ConnectedServiceName: 'Onebranch Release EV2' + ConnectedServiceName: 'docker-deployement-test' ServiceRootLocation: 'LinkedArtifact' RolloutSpecType: 'RSPath' ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot' From 31300833b50fd40847298e66c50b1dc0f21351c2 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 20 Nov 2024 09:42:51 -0600 Subject: [PATCH 173/316] added mscodehubPAT variable group --- .pipelines/templates/release-prep-for-ev2.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index bb84c9e68b0..93780050667 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -6,6 +6,7 @@ jobs: variables: - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - group: 'mscodehub-code-read-akv' steps: - checkout: self ## the global setting on lfs didn't work lfs: false From 64a1c950bc1fac766a7219050ffdcb24f2f15210 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 21 Nov 2024 11:15:41 -0600 Subject: [PATCH 174/316] signing profile to internal azure service --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 93780050667..f47e020b63f 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -40,7 +40,7 @@ jobs: displayName: Sign Release Tools inputs: command: 'sign' - signing_profile: external_distribution + signing_profile: internal_azure_service files_to_sign: '**\*.ps1' search_root: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4' From fc557f259ce888aacc0a000e19f00abf9a06c5e8 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 21 Nov 2024 11:19:17 -0600 Subject: [PATCH 175/316] capture workspace --- .pipelines/templates/release-prep-for-ev2.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index f47e020b63f..453367535ea 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -36,6 +36,10 @@ jobs: Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' + - pwsh: | + Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose + displayName: 'Capture Workspace' + - task: onebranch.pipeline.signing@1 displayName: Sign Release Tools inputs: From 84d445cdcb82d80619631ea94e160882bd402b26 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 21 Nov 2024 12:38:44 -0500 Subject: [PATCH 176/316] comment out internal tools signing for now --- .../ServiceGroupRoot/Shell/Run/settings.toml | 4 ++++ .pipelines/templates/release-prep-for-ev2.yml | 14 +++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 .pipelines/templates/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml diff --git a/.pipelines/templates/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml b/.pipelines/templates/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml new file mode 100644 index 00000000000..877529312cf --- /dev/null +++ b/.pipelines/templates/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml @@ -0,0 +1,4 @@ +[default] +base_url = "https://pmc-ingest.trafficmanager.net/api/v4" +auth_type = "msi" +client_id = "..." # Client ID of your User-Assigned Managed Identity diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 453367535ea..cb88d2a81f6 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -40,13 +40,13 @@ jobs: Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose displayName: 'Capture Workspace' - - task: onebranch.pipeline.signing@1 - displayName: Sign Release Tools - inputs: - command: 'sign' - signing_profile: internal_azure_service - files_to_sign: '**\*.ps1' - search_root: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4' + # - task: onebranch.pipeline.signing@1 + # displayName: Sign Release Tools + # inputs: + # command: 'sign' + # signing_profile: internal_azure_service + # files_to_sign: '**\*.ps1' + # search_root: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4' - task: onebranch.pipeline.signing@1 displayName: Sign Run.ps1 From 04407c631cdabcca9af9464d83fc3df7e8db1eaf Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 21 Nov 2024 12:56:32 -0500 Subject: [PATCH 177/316] move EV2Specs folder and fix path reference to it --- .../EV2Specs/ServiceGroupRoot/RollOutSpec.json | 0 .../EV2Specs/ServiceGroupRoot/ScopeBindings.json | 0 .../EV2Specs/ServiceGroupRoot/ServiceModel.json | 0 .../EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 0 .../EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml | 0 .../EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json | 0 .pipelines/templates/release-prep-for-ev2.yml | 8 ++++---- 7 files changed, 4 insertions(+), 4 deletions(-) rename .pipelines/{templates => }/EV2Specs/ServiceGroupRoot/RollOutSpec.json (100%) rename .pipelines/{templates => }/EV2Specs/ServiceGroupRoot/ScopeBindings.json (100%) rename .pipelines/{templates => }/EV2Specs/ServiceGroupRoot/ServiceModel.json (100%) rename .pipelines/{templates => }/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 (100%) rename .pipelines/{templates => }/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml (100%) rename .pipelines/{templates => }/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json (100%) diff --git a/.pipelines/templates/EV2Specs/ServiceGroupRoot/RollOutSpec.json b/.pipelines/EV2Specs/ServiceGroupRoot/RollOutSpec.json similarity index 100% rename from .pipelines/templates/EV2Specs/ServiceGroupRoot/RollOutSpec.json rename to .pipelines/EV2Specs/ServiceGroupRoot/RollOutSpec.json diff --git a/.pipelines/templates/EV2Specs/ServiceGroupRoot/ScopeBindings.json b/.pipelines/EV2Specs/ServiceGroupRoot/ScopeBindings.json similarity index 100% rename from .pipelines/templates/EV2Specs/ServiceGroupRoot/ScopeBindings.json rename to .pipelines/EV2Specs/ServiceGroupRoot/ScopeBindings.json diff --git a/.pipelines/templates/EV2Specs/ServiceGroupRoot/ServiceModel.json b/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json similarity index 100% rename from .pipelines/templates/EV2Specs/ServiceGroupRoot/ServiceModel.json rename to .pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json diff --git a/.pipelines/templates/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 similarity index 100% rename from .pipelines/templates/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 rename to .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 diff --git a/.pipelines/templates/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml similarity index 100% rename from .pipelines/templates/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml rename to .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml diff --git a/.pipelines/templates/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json b/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json similarity index 100% rename from .pipelines/templates/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json rename to .pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index cb88d2a81f6..635322e3645 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -54,16 +54,16 @@ jobs: command: 'sign' signing_profile: external_distribution files_to_sign: '**\*.ps1' - search_root: '$(Build.SourcesDirectory)/EV2Specs/ServiceGroupRoot/Shell' + search_root: '$(Build.SourcesDirectory)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' - pwsh: | - $srcPath = Join-Path '$(Build.SourcesDirectory)/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' - $pathToRunTarFile = Join-Path '$(Build.SourcesDirectory)/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" + $srcPath = Join-Path '$(Build.SourcesDirectory)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' + $pathToRunTarFile = Join-Path '$(Build.SourcesDirectory)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" tar -cvf $pathToRunTarFile -C $srcPath ./Run displayName: Compress Run script into tar file as needed for EV2 Shell extension - pwsh: | - $pathToEV2Folder = Join-Path -Path '$(Build.SourcesDirectory)' -ChildPath 'EV2Specs' + $pathToEV2Folder = Join-Path -Path '$(Build.SourcesDirectory)/.pipelines' -ChildPath 'EV2Specs' $pathToServiceGroupRootFolder = Join-Path $pathToEV2Folder -ChildPath 'ServiceGroupRoot' $pathToParametersFolder = Join-Path -Path $pathToServiceGroupRootFolder -ChildPath 'Parameters' New-Item -Path $pathToParametersFolder -ItemType Directory From a296f2daaa1488b619508866811faa9b397d96a3 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 21 Nov 2024 13:19:44 -0500 Subject: [PATCH 178/316] test path of Run.ps1 and change files to sign pattern --- .pipelines/templates/release-prep-for-ev2.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 635322e3645..c13d5eff5d9 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -6,6 +6,8 @@ jobs: variables: - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: repoRoot + value: $(Build.SourcesDirectory) - group: 'mscodehub-code-read-akv' steps: - checkout: self ## the global setting on lfs didn't work @@ -38,8 +40,15 @@ jobs: - pwsh: | Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose + Get-ChildItem '$(Build.SourcesDirectory)' displayName: 'Capture Workspace' + - pwsh: | + $pathToRunFolder = Join-Path -Path "$(Build.SourcesDirectory)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run" -ChildPath '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/releaseLinuxPackages.ps1' + $pathExists = Test-Path $pathToRunFolder + Write-Verbose -Verbose "path exists: $pathExists" + displayName: 'Test if Run.ps1 path exists/can be found' + # - task: onebranch.pipeline.signing@1 # displayName: Sign Release Tools # inputs: @@ -53,8 +62,8 @@ jobs: inputs: command: 'sign' signing_profile: external_distribution - files_to_sign: '**\*.ps1' - search_root: '$(Build.SourcesDirectory)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' + files_to_sign: '*.ps1' + search_root: '$(Build.SourcesDirectory)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' - pwsh: | $srcPath = Join-Path '$(Build.SourcesDirectory)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' From ae30baf312bc8120ebacbcdb6511c59464b62d3f Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 21 Nov 2024 13:45:52 -0500 Subject: [PATCH 179/316] refer to PowerShell files with project name in path as multiple repo's were checked out --- .pipelines/templates/release-prep-for-ev2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index c13d5eff5d9..e2909e46189 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -7,7 +7,7 @@ jobs: - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: repoRoot - value: $(Build.SourcesDirectory) + value: '$(Build.SourcesDirectory)/PowerShell' - group: 'mscodehub-code-read-akv' steps: - checkout: self ## the global setting on lfs didn't work @@ -63,7 +63,7 @@ jobs: command: 'sign' signing_profile: external_distribution files_to_sign: '*.ps1' - search_root: '$(Build.SourcesDirectory)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' + search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' - pwsh: | $srcPath = Join-Path '$(Build.SourcesDirectory)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' From 9143efa765e4cc6ed13fd07008c8ef1764945896 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 21 Nov 2024 14:00:46 -0500 Subject: [PATCH 180/316] comment out signing for now --- .pipelines/templates/release-prep-for-ev2.yml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index e2909e46189..8bdb8ca4cd6 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -44,7 +44,7 @@ jobs: displayName: 'Capture Workspace' - pwsh: | - $pathToRunFolder = Join-Path -Path "$(Build.SourcesDirectory)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run" -ChildPath '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/releaseLinuxPackages.ps1' + $pathToRunFolder = Join-Path -Path "$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run" -ChildPath '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/releaseLinuxPackages.ps1' $pathExists = Test-Path $pathToRunFolder Write-Verbose -Verbose "path exists: $pathExists" displayName: 'Test if Run.ps1 path exists/can be found' @@ -57,22 +57,22 @@ jobs: # files_to_sign: '**\*.ps1' # search_root: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4' - - task: onebranch.pipeline.signing@1 - displayName: Sign Run.ps1 - inputs: - command: 'sign' - signing_profile: external_distribution - files_to_sign: '*.ps1' - search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' + # - task: onebranch.pipeline.signing@1 + # displayName: Sign Run.ps1 + # inputs: + # command: 'sign' + # signing_profile: external_distribution + # files_to_sign: '*.ps1' + # search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' - pwsh: | - $srcPath = Join-Path '$(Build.SourcesDirectory)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' - $pathToRunTarFile = Join-Path '$(Build.SourcesDirectory)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" + $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' + $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" tar -cvf $pathToRunTarFile -C $srcPath ./Run displayName: Compress Run script into tar file as needed for EV2 Shell extension - pwsh: | - $pathToEV2Folder = Join-Path -Path '$(Build.SourcesDirectory)/.pipelines' -ChildPath 'EV2Specs' + $pathToEV2Folder = Join-Path -Path '$(repoRoot)/.pipelines' -ChildPath 'EV2Specs' $pathToServiceGroupRootFolder = Join-Path $pathToEV2Folder -ChildPath 'ServiceGroupRoot' $pathToParametersFolder = Join-Path -Path $pathToServiceGroupRootFolder -ChildPath 'Parameters' New-Item -Path $pathToParametersFolder -ItemType Directory From 6a25b9ba25a67457e4d10e4e79a39dcc3d174586 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 21 Nov 2024 16:36:46 -0500 Subject: [PATCH 181/316] enable TiP signing --- .pipelines/templates/release-prep-for-ev2.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 8bdb8ca4cd6..eeb3cb406ba 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -9,6 +9,8 @@ jobs: - name: repoRoot value: '$(Build.SourcesDirectory)/PowerShell' - group: 'mscodehub-code-read-akv' + - name: ob_tipsign_enabled + value: true # Enable TiP sign steps: - checkout: self ## the global setting on lfs didn't work lfs: false @@ -44,7 +46,7 @@ jobs: displayName: 'Capture Workspace' - pwsh: | - $pathToRunFolder = Join-Path -Path "$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run" -ChildPath '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/releaseLinuxPackages.ps1' + $pathToRunFolder = Join-Path -Path "$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run" -ChildPath 'Run.ps1' $pathExists = Test-Path $pathToRunFolder Write-Verbose -Verbose "path exists: $pathExists" displayName: 'Test if Run.ps1 path exists/can be found' @@ -57,13 +59,13 @@ jobs: # files_to_sign: '**\*.ps1' # search_root: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4' - # - task: onebranch.pipeline.signing@1 - # displayName: Sign Run.ps1 - # inputs: - # command: 'sign' - # signing_profile: external_distribution - # files_to_sign: '*.ps1' - # search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' + - task: onebranch.pipeline.signing@1 + displayName: Sign Run.ps1 + inputs: + command: 'sign' + signing_profile: 'tipsign' # external_distribution + files_to_sign: '*.ps1' + search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' - pwsh: | $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' From db6cc968d5994bf0b14c55e3f53e6c2ad7e718ad Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 21 Nov 2024 16:52:10 -0500 Subject: [PATCH 182/316] comment out unused tasks for now --- .pipelines/templates/release-prep-for-ev2.yml | 88 +------------------ 1 file changed, 4 insertions(+), 84 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index eeb3cb406ba..18674e486c1 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -73,90 +73,10 @@ jobs: tar -cvf $pathToRunTarFile -C $srcPath ./Run displayName: Compress Run script into tar file as needed for EV2 Shell extension - - pwsh: | - $pathToEV2Folder = Join-Path -Path '$(repoRoot)/.pipelines' -ChildPath 'EV2Specs' - $pathToServiceGroupRootFolder = Join-Path $pathToEV2Folder -ChildPath 'ServiceGroupRoot' - $pathToParametersFolder = Join-Path -Path $pathToServiceGroupRootFolder -ChildPath 'Parameters' - New-Item -Path $pathToParametersFolder -ItemType Directory - $pathToImageMetadataFile = Join-Path -Path $pathToParametersFolder -ChildPath 'ImageMetadata.json' - $pathToChannelJsonFile = Join-Path -Path $pathToParametersFolder -ChildPath 'ChannelInfo.json' - $currentChannel = '$(channel)' - - $channelHash = @{channel=$currentChannel} - $channelHash | ConvertTo-Json | Out-File $pathToChannelJsonFile - - $toolsFolderPath = Join-Path -Path '$(Build.SourcesDirectory)' -ChildPath 'tools' - $buildHelperFolderPath = Join-Path -Path $toolsFolderPath -ChildPath 'buildHelper' - $buildHelperModulePath = Join-Path -Path $buildHelperFolderPath -ChildPath 'buildHelper.psm1' - Write-Verbose -Verbose "ipmo path: $buildHelperModulePath" - Import-Module $buildHelperModulePath - if ($currentChannel -eq "stable") - { - .\build.ps1 -UpdateImageMetadata -Channel $currentChannel -StableVersion '$(version)' -MetadataFilePath $pathToImageMetadataFile - } - elseif ($currentChannel -eq "preview") - { - .\build.ps1 -UpdateImageMetadata -Channel $currentChannel -PreviewVersion '$(version)' -MetadataFilePath $pathToImageMetadataFile - } - elseif ($currentChannel -eq "lts") - { - .\build.ps1 -UpdateImageMetadata -Channel $currentChannel -LtsVersion '$(version)' -MetadataFilePath $pathToImageMetadataFile - } - - $imgMetaJsonPathExists = Test-Path -Path $pathToImageMetadataFile - $channelJsonPathExists = Test-Path -Path $pathToChannelJsonFile - displayName: 'Create image metadata json needed for EV2 Shell Extension script' - - - pwsh: | - $pathToJsonFile = Join-Path -Path '$(Build.SourcesDirectory)/EV2Specs/ServiceGroupRoot' -ChildPath 'UploadLinux.Rollout.json' - $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json - $envVarArrayLen = $content.shellExtensions.launch.environmentVariables.Length - for ($i=0; $i -lt $envVarArrayLen; $i++) - { - $name = $($content.shellExtensions.launch.environmentVariables[$i].name) - if ($name -eq "DESTINATION_ACR_NAME") - { - $content.shellExtensions.launch.environmentVariables[$i].value = '$(acr_name)' - } - elseif ($name -eq "MI_NAME"){ - $content.shellExtensions.launch.environmentVariables[$i].value = '$(managed_identity_name)' - } - elseif ($name -eq "MI_CLIENTID") - { - $content.shellExtensions.launch.environmentVariables[$i].value = '$(managed_identity_clientid)' - } - } - - $identityString = "/subscriptions/$(acr_subscription)/resourcegroups/$(acr_resource_group)/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$(managed_identity_name)" - $content.shellExtensions.launch.identity.userAssignedIdentities[0] = $identityString - - Remove-Item -Path $pathToJsonFile - $content | ConvertTo-Json -Depth 6 | Out-File $pathToJsonFile - displayName: 'Replace values in UploadLinux.Rollout.json file' - - - pwsh: | - $pathToJsonFile = Join-Path -Path '$(Build.SourcesDirectory)/EV2Specs/ServiceGroupRoot' -ChildPath 'RolloutSpec.json' - $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json - $content.RolloutMetadata.Notification.Email.To = '$(email_address)' - - Remove-Item -Path $pathToJsonFile - $content | ConvertTo-Json -Depth 4 | Out-File $pathToJsonFile - displayName: 'Replace values in RolloutSpecPath.json' - - - pwsh: | - $pathToJsonFile = Join-Path -Path '$(Build.SourcesDirectory)/EV2Specs/ServiceGroupRoot' -ChildPath 'ServiceModel.json' - $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json - $content.ServiceResourceGroups[0].AzureResourceGroupName = '$(acr_resource_group)' - $content.ServiceResourceGroups[0].AzureSubscriptionId = '$(acr_subscription)' - - Remove-Item -Path $pathToJsonFile - $content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile - displayName: 'Replace values in ServiceModel.json' - - - task: CopyFiles@2 - inputs: - Contents: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/**' - TargetFolder: 'EV2Specs/ServiceGroupRoot/tools' + # - task: CopyFiles@2 + # inputs: + # Contents: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/**' + # TargetFolder: 'EV2Specs/ServiceGroupRoot/tools' - task: CopyFiles@2 inputs: From 97c7fa12aca9baf55a92667b6d8425aadec69a8a Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 22 Nov 2024 13:38:28 -0500 Subject: [PATCH 183/316] add in prep steps --- .pipelines/templates/release-prep-for-ev2.yml | 182 +++++++++++------- 1 file changed, 115 insertions(+), 67 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 18674e486c1..452eb191342 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -12,73 +12,121 @@ jobs: - name: ob_tipsign_enabled value: true # Enable TiP sign steps: - - checkout: self ## the global setting on lfs didn't work - lfs: false + - checkout: self ## the global setting on lfs didn't work + lfs: false - - template: release-SetReleaseTagandContainerName.yml + - template: release-SetReleaseTagandContainerName.yml - - pwsh: | - $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' - $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: Set Package version + - pwsh: | + $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' + $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Set Package version - - pwsh: | - $branch = 'mirror-target' - $gitArgs = "clone", - "--verbose", - "--branch", - "$branch", - "https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools", - '$(Pipeline.Workspace)/tools' - $gitArgs | Write-Verbose -Verbose - git $gitArgs - displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub - - - pwsh: | - Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose - displayName: 'Capture Environment Variables' - - - pwsh: | - Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose - Get-ChildItem '$(Build.SourcesDirectory)' - displayName: 'Capture Workspace' - - - pwsh: | - $pathToRunFolder = Join-Path -Path "$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run" -ChildPath 'Run.ps1' - $pathExists = Test-Path $pathToRunFolder - Write-Verbose -Verbose "path exists: $pathExists" - displayName: 'Test if Run.ps1 path exists/can be found' - - # - task: onebranch.pipeline.signing@1 - # displayName: Sign Release Tools - # inputs: - # command: 'sign' - # signing_profile: internal_azure_service - # files_to_sign: '**\*.ps1' - # search_root: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4' - - - task: onebranch.pipeline.signing@1 - displayName: Sign Run.ps1 - inputs: - command: 'sign' - signing_profile: 'tipsign' # external_distribution - files_to_sign: '*.ps1' - search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' - - - pwsh: | - $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' - $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" - tar -cvf $pathToRunTarFile -C $srcPath ./Run - displayName: Compress Run script into tar file as needed for EV2 Shell extension - - # - task: CopyFiles@2 - # inputs: - # Contents: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/**' - # TargetFolder: 'EV2Specs/ServiceGroupRoot/tools' - - - task: CopyFiles@2 - inputs: - Contents: 'EV2Specs/**' - TargetFolder: $(ob_outputDirectory) + - pwsh: | + $branch = 'mirror-target' + $gitArgs = "clone", + "--verbose", + "--branch", + "$branch", + "https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools", + '$(Pipeline.Workspace)/tools' + $gitArgs | Write-Verbose -Verbose + git $gitArgs + displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub + + - pwsh: | + Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose + displayName: 'Capture Environment Variables' + + - pwsh: | + Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose + Get-ChildItem '$(Build.SourcesDirectory)' + displayName: 'Capture Workspace' + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.9' + inputs: + versionSpec: 3.9 + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'pmc' + pythonDownloadServiceConnections: pmcDownload + displayName: 'Pip Authenticate' + + - pwsh: | + $ev2SpecsFolder = "$(Build.SourcesDirectory)/.pipelines/EV2Specs" + $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" + + Write-Verbose -Verbose "Download pmc-cli to folder without installing it" + $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" + New-Item -Path $pythonDlFolderPath + pip3 download -d pythonDlFolderPath pmc-cli + displayName: 'Download pmc-cli package' + + - pwsh: | + Write-Verbose -Verbose "Copy built .deb packages" + # assuming ESRP signed packages are here + # TODO: either copy packages to a folder and .tar it all up- i believe there's a size limit + # so option 2: make a .tar.gz file and pass it the way I do for images + displayName: 'Copy signed .deb packages - probably to a .tar.gz to pass as a file var' + + - pwsh: | + $pathToEV2Folder = Join-Path -Path '$(Build.SourcesDirectory)/.pipelines' -ChildPath 'EV2Specs' + $pathToServiceGroupRootFolder = Join-Path $pathToEV2Folder -ChildPath 'ServiceGroupRoot' + $pathToParametersFolder = Join-Path -Path $pathToServiceGroupRootFolder -ChildPath 'Parameters' + New-Item -Path $pathToParametersFolder -ItemType Directory + $pathToPMCMetadataFile = Join-Path -Path $pathToParametersFolder -ChildPath 'PmcMetadata.json' + + $metadata = Get-Content -Path "$(Build.SourcesDirectory)/PowerShell/tools/metadata.json" -Raw | ConvertFrom-Json + $skipPublishVar = '${{ parameters.skipPublish }}' + $metadataHash = @{} + $metadataHash["ReleaseTag"] = "$(ReleaseTag)" + $metadataHash["AadClientId"] = "$(PmcCliClientID)" + $metadataHash["BlobFolderName"] = "$(ReleaseTag)" + $metadataHash["LTS"] = $metadata.LTSRelease.Latest + $metadataHash["ForProduction"] = $true + $metadataHash["SkipPublish"] = $skipPublishVar + + $metadataHash | ConvertTo-Json | Out-File $pathToPMCMetadataFile + + Write-Verbose -Verbose "Copy mapping.json file to EV2Specs folder. file path at path $mappingFilePath exists: $mappingFilePathExists" + $mappingFilePath = Join-Path -Path '$(System.DefaultWorkingDirectory)/tools/packages.microsoft.com' -ChildPath 'mapping.json' + $mappingFilePathExists = Test-Path $mappingFilePath + $mappingFileEV2Path = Join-Path -Path $pathToParametersFolder -ChildPath "mapping.json" + Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path + displayName: 'Create pmcScriptMetadata.json and mapping.json file' + + - pwsh: | + $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' + $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" + tar -cvf $pathToRunTarFile -C $srcPath ./Run + # TODO: ensure this has: Run.ps1, settings.toml, python_dl + # tar cvf pmc_packages.tar start.sh settings.toml packages python_dl + displayName: 'Create archive for the shell extension' + + - task: onebranch.pipeline.signing@1 + displayName: Sign Run.ps1 + inputs: + command: 'sign' + signing_profile: 'tipsign' # external_distribution + files_to_sign: '*.ps1' + search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' + + - pwsh: | + $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' + $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" + tar -cvf $pathToRunTarFile -C $srcPath ./Run + displayName: 'Compress Run script into tar file as needed for EV2 Shell extension' + + # - task: CopyFiles@2 + # inputs: + # Contents: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/**' + # TargetFolder: 'EV2Specs/ServiceGroupRoot/tools' + + - task: CopyFiles@2 + inputs: + Contents: 'EV2Specs/**' + TargetFolder: $(ob_outputDirectory) From f50c304c140a0bf01e540ff5b43b95528f910a9c Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 22 Nov 2024 13:49:58 -0500 Subject: [PATCH 184/316] use higher python version available --- .pipelines/templates/release-prep-for-ev2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 452eb191342..ec5e87d5187 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -46,9 +46,9 @@ jobs: displayName: 'Capture Workspace' - task: UsePythonVersion@0 - displayName: 'Use Python 3.9' + displayName: 'Use Python 3.12.7' inputs: - versionSpec: 3.9 + versionSpec: 3.12.7 - task: PipAuthenticate@1 inputs: From 8aced41787dc155b7b3e4b45a3dccf98ef7e05ea Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 22 Nov 2024 14:23:29 -0500 Subject: [PATCH 185/316] use repoRoot in path --- .pipelines/templates/release-prep-for-ev2.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index ec5e87d5187..ab72c648608 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -57,7 +57,7 @@ jobs: displayName: 'Pip Authenticate' - pwsh: | - $ev2SpecsFolder = "$(Build.SourcesDirectory)/.pipelines/EV2Specs" + $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" Write-Verbose -Verbose "Download pmc-cli to folder without installing it" @@ -74,13 +74,13 @@ jobs: displayName: 'Copy signed .deb packages - probably to a .tar.gz to pass as a file var' - pwsh: | - $pathToEV2Folder = Join-Path -Path '$(Build.SourcesDirectory)/.pipelines' -ChildPath 'EV2Specs' + $pathToEV2Folder = Join-Path -Path '$(repoRoot)/.pipelines' -ChildPath 'EV2Specs' $pathToServiceGroupRootFolder = Join-Path $pathToEV2Folder -ChildPath 'ServiceGroupRoot' $pathToParametersFolder = Join-Path -Path $pathToServiceGroupRootFolder -ChildPath 'Parameters' New-Item -Path $pathToParametersFolder -ItemType Directory $pathToPMCMetadataFile = Join-Path -Path $pathToParametersFolder -ChildPath 'PmcMetadata.json' - $metadata = Get-Content -Path "$(Build.SourcesDirectory)/PowerShell/tools/metadata.json" -Raw | ConvertFrom-Json + $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json $skipPublishVar = '${{ parameters.skipPublish }}' $metadataHash = @{} $metadataHash["ReleaseTag"] = "$(ReleaseTag)" From 3ac9477ae3239b5d983822913925533102e5f31b Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 22 Nov 2024 14:41:27 -0500 Subject: [PATCH 186/316] add verbose and error logging to pip3 download command --- .pipelines/templates/release-prep-for-ev2.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index ab72c648608..3265f507bf6 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -63,7 +63,9 @@ jobs: Write-Verbose -Verbose "Download pmc-cli to folder without installing it" $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" New-Item -Path $pythonDlFolderPath - pip3 download -d pythonDlFolderPath pmc-cli + pip3 download -d pythonDlFolderPath pmc-cli -v 2> error.txt + cat error.txt + Write-Verbose -Verbose "done pip download" displayName: 'Download pmc-cli package' - pwsh: | From dc332914961484b77d257a2fb69a3de7863f179c Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 2 Dec 2024 13:28:19 -0500 Subject: [PATCH 187/316] fix typo in pip3 command --- .pipelines/templates/release-prep-for-ev2.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 3265f507bf6..d2b0f0dd14a 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -63,8 +63,9 @@ jobs: Write-Verbose -Verbose "Download pmc-cli to folder without installing it" $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" New-Item -Path $pythonDlFolderPath - pip3 download -d pythonDlFolderPath pmc-cli -v 2> error.txt + pip3 download -d $pythonDlFolderPath pmc-cli -v 2> error.txt cat error.txt + Get-ChildItem $pythonDlFolderPath Write-Verbose -Verbose "done pip download" displayName: 'Download pmc-cli package' From 4e9ceeac4b24d4f7f84cad137bbd256784a8b47c Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 2 Dec 2024 13:52:54 -0500 Subject: [PATCH 188/316] add other pip3 command --- .pipelines/templates/release-prep-for-ev2.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index d2b0f0dd14a..d16d61814b2 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -63,8 +63,9 @@ jobs: Write-Verbose -Verbose "Download pmc-cli to folder without installing it" $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" New-Item -Path $pythonDlFolderPath - pip3 download -d $pythonDlFolderPath pmc-cli -v 2> error.txt - cat error.txt + pip3 --version + pip3 download -d $pythonDlFolderPath pmc-cli -v + Write-Verbose -Verbose "trying gci" Get-ChildItem $pythonDlFolderPath Write-Verbose -Verbose "done pip download" displayName: 'Download pmc-cli package' From 1830d7d5622ffed15fe9a2a31b5ea857844412e5 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 2 Dec 2024 15:25:59 -0500 Subject: [PATCH 189/316] use full path of pip3 executable --- .pipelines/templates/release-prep-for-ev2.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index d16d61814b2..7278ac6218d 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -65,6 +65,7 @@ jobs: New-Item -Path $pythonDlFolderPath pip3 --version pip3 download -d $pythonDlFolderPath pmc-cli -v + & 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' download -d $pythonDlFolderPath pmc-cli Write-Verbose -Verbose "trying gci" Get-ChildItem $pythonDlFolderPath Write-Verbose -Verbose "done pip download" From 3eeba3052e0e695ee872f327f393e04268794c27 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 2 Dec 2024 15:38:24 -0500 Subject: [PATCH 190/316] check exe path --- .pipelines/templates/release-prep-for-ev2.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 7278ac6218d..fe70a5af0bd 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -65,6 +65,10 @@ jobs: New-Item -Path $pythonDlFolderPath pip3 --version pip3 download -d $pythonDlFolderPath pmc-cli -v + $pipExePath = Join-Path -Path "C:\__t\Python\3.12.7\x64\Scripts" -ChildPath "pip3.exe" + $pipExePathExists = Test-Path $pipExePath + Write-Verbose -Verbose "pip3 path exists: $pipExePathExists" + Get-ChildItem "C:\__t\Python\3.12.7\x64\Scripts" & 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' download -d $pythonDlFolderPath pmc-cli Write-Verbose -Verbose "trying gci" Get-ChildItem $pythonDlFolderPath From 697d0f1fc2d88c2b2600ef708eaeb96c49cff286 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 2 Dec 2024 15:50:52 -0500 Subject: [PATCH 191/316] fix pip3 command syntax and dir location --- .pipelines/templates/release-prep-for-ev2.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index fe70a5af0bd..923e18cef51 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -69,7 +69,8 @@ jobs: $pipExePathExists = Test-Path $pipExePath Write-Verbose -Verbose "pip3 path exists: $pipExePathExists" Get-ChildItem "C:\__t\Python\3.12.7\x64\Scripts" - & 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' download -d $pythonDlFolderPath pmc-cli + New-Item -Path "myPythonPkg" + & 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' download pmc-cli -d myPythonPkg Write-Verbose -Verbose "trying gci" Get-ChildItem $pythonDlFolderPath Write-Verbose -Verbose "done pip download" From a9c16230e1dd8906ca1ea8166f9c3b68d575d6a8 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 2 Dec 2024 16:02:04 -0500 Subject: [PATCH 192/316] try bash step --- .pipelines/templates/release-prep-for-ev2.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 923e18cef51..bb7b176fea4 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -55,6 +55,15 @@ jobs: artifactFeeds: 'pmc' pythonDownloadServiceConnections: pmcDownload displayName: 'Pip Authenticate' + + - bash: | + EV2_DIR="$(Build.SourcesDirectory)/.pipelines/EV2Specs" + SHEXT_BUNDLE_DIR="$EV2_DIR/ServiceGroupRoot/Shell/Run" + + echo "1) Get pmc-cli" + cd "$SHEXT_BUNDLE_DIR" + mkdir python_dl + pip3 download -d python_dl pmc-cli - pwsh: | $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" From 688e489967a95fc25e3901d37efab9f28b5f5ab3 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 2 Dec 2024 17:16:04 -0500 Subject: [PATCH 193/316] Remove bash step --- .pipelines/templates/release-prep-for-ev2.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index bb7b176fea4..cdb908b9c47 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -55,15 +55,6 @@ jobs: artifactFeeds: 'pmc' pythonDownloadServiceConnections: pmcDownload displayName: 'Pip Authenticate' - - - bash: | - EV2_DIR="$(Build.SourcesDirectory)/.pipelines/EV2Specs" - SHEXT_BUNDLE_DIR="$EV2_DIR/ServiceGroupRoot/Shell/Run" - - echo "1) Get pmc-cli" - cd "$SHEXT_BUNDLE_DIR" - mkdir python_dl - pip3 download -d python_dl pmc-cli - pwsh: | $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" @@ -78,6 +69,7 @@ jobs: $pipExePathExists = Test-Path $pipExePath Write-Verbose -Verbose "pip3 path exists: $pipExePathExists" Get-ChildItem "C:\__t\Python\3.12.7\x64\Scripts" + New-Item -Path "myPythonPkg" & 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' download pmc-cli -d myPythonPkg Write-Verbose -Verbose "trying gci" From cb8a564a077d1f61a1e22062cee62a3b8e00d109 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 2 Dec 2024 17:32:53 -0500 Subject: [PATCH 194/316] change artifactFeed name --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index cdb908b9c47..eeeff27719e 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -52,7 +52,7 @@ jobs: - task: PipAuthenticate@1 inputs: - artifactFeeds: 'pmc' + artifactFeeds: 'msazure-Compute-PMC@Release' pythonDownloadServiceConnections: pmcDownload displayName: 'Pip Authenticate' From 60e6e81381d71b95546ba6cde51c8828221799b5 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 2 Dec 2024 17:47:57 -0500 Subject: [PATCH 195/316] set artifactFeedName to project/feed per docs --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index eeeff27719e..290f84b27c5 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -52,7 +52,7 @@ jobs: - task: PipAuthenticate@1 inputs: - artifactFeeds: 'msazure-Compute-PMC@Release' + artifactFeeds: 'PowerShellCore/msazure-Compute-PMC@Release' pythonDownloadServiceConnections: pmcDownload displayName: 'Pip Authenticate' From 607beda9e83a1fb79aa62539059239289f9f5721 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 2 Dec 2024 17:58:18 -0500 Subject: [PATCH 196/316] Remove the service connection --- .pipelines/templates/release-prep-for-ev2.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 290f84b27c5..712f873a08d 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -53,7 +53,6 @@ jobs: - task: PipAuthenticate@1 inputs: artifactFeeds: 'PowerShellCore/msazure-Compute-PMC@Release' - pythonDownloadServiceConnections: pmcDownload displayName: 'Pip Authenticate' - pwsh: | From e90aba78cbedbdebe7ae70cebe59744d95db5c4c Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 2 Dec 2024 18:09:23 -0500 Subject: [PATCH 197/316] add pip verbose command --- .pipelines/templates/release-prep-for-ev2.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 712f873a08d..6eb7cb4ef50 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -64,6 +64,7 @@ jobs: New-Item -Path $pythonDlFolderPath pip3 --version pip3 download -d $pythonDlFolderPath pmc-cli -v + pip3 install -vvv pmc-cli $pipExePath = Join-Path -Path "C:\__t\Python\3.12.7\x64\Scripts" -ChildPath "pip3.exe" $pipExePathExists = Test-Path $pipExePath Write-Verbose -Verbose "pip3 path exists: $pipExePathExists" From f1b50898eaaaa1567897ef8c70dbb1dd53b4da94 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 3 Dec 2024 12:14:00 -0500 Subject: [PATCH 198/316] use correct feed name --- .pipelines/templates/release-prep-for-ev2.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 6eb7cb4ef50..4f68e6af944 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -52,7 +52,9 @@ jobs: - task: PipAuthenticate@1 inputs: - artifactFeeds: 'PowerShellCore/msazure-Compute-PMC@Release' + # artifactFeeds: 'PowerShellCore/msazure-Compute-PMC@Release' + artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' + pythonDownloadServiceConnections: pmcDownload displayName: 'Pip Authenticate' - pwsh: | @@ -64,7 +66,6 @@ jobs: New-Item -Path $pythonDlFolderPath pip3 --version pip3 download -d $pythonDlFolderPath pmc-cli -v - pip3 install -vvv pmc-cli $pipExePath = Join-Path -Path "C:\__t\Python\3.12.7\x64\Scripts" -ChildPath "pip3.exe" $pipExePathExists = Test-Path $pipExePath Write-Verbose -Verbose "pip3 path exists: $pipExePathExists" From 6f2c6fb77ba976ddb0b7780d2d27ae4323c944e7 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 3 Dec 2024 13:03:25 -0500 Subject: [PATCH 199/316] remove the service connection and add verbose logging --- .pipelines/templates/release-prep-for-ev2.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 4f68e6af944..5f2fab23633 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -54,7 +54,7 @@ jobs: inputs: # artifactFeeds: 'PowerShellCore/msazure-Compute-PMC@Release' artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' - pythonDownloadServiceConnections: pmcDownload + # pythonDownloadServiceConnections: pmcDownload displayName: 'Pip Authenticate' - pwsh: | @@ -64,8 +64,8 @@ jobs: Write-Verbose -Verbose "Download pmc-cli to folder without installing it" $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" New-Item -Path $pythonDlFolderPath - pip3 --version - pip3 download -d $pythonDlFolderPath pmc-cli -v + pip3 --version --verbose + pip3 download -d $pythonDlFolderPath pmc-cli --verbose $pipExePath = Join-Path -Path "C:\__t\Python\3.12.7\x64\Scripts" -ChildPath "pip3.exe" $pipExePathExists = Test-Path $pipExePath Write-Verbose -Verbose "pip3 path exists: $pipExePathExists" From 2702743443164aa20f016e06d8b481a4411e85e5 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 3 Dec 2024 13:17:18 -0500 Subject: [PATCH 200/316] get version info only --- .pipelines/templates/release-prep-for-ev2.yml | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 5f2fab23633..aa849126831 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -58,24 +58,31 @@ jobs: displayName: 'Pip Authenticate' - pwsh: | - $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" - $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" - - Write-Verbose -Verbose "Download pmc-cli to folder without installing it" - $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" - New-Item -Path $pythonDlFolderPath pip3 --version --verbose - pip3 download -d $pythonDlFolderPath pmc-cli --verbose - $pipExePath = Join-Path -Path "C:\__t\Python\3.12.7\x64\Scripts" -ChildPath "pip3.exe" - $pipExePathExists = Test-Path $pipExePath - Write-Verbose -Verbose "pip3 path exists: $pipExePathExists" - Get-ChildItem "C:\__t\Python\3.12.7\x64\Scripts" - - New-Item -Path "myPythonPkg" - & 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' download pmc-cli -d myPythonPkg - Write-Verbose -Verbose "trying gci" - Get-ChildItem $pythonDlFolderPath - Write-Verbose -Verbose "done pip download" + & 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' --version --verbose + Push-Location 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' + Write-Verbose -Verbose "trying push pop" + pip3 --version --verbose + Pop-Location + + # $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" + # $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" + + # Write-Verbose -Verbose "Download pmc-cli to folder without installing it" + # $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" + # New-Item -Path $pythonDlFolderPath + # pip3 --version --verbose + # pip3 download -d $pythonDlFolderPath pmc-cli --verbose + # $pipExePath = Join-Path -Path "C:\__t\Python\3.12.7\x64\Scripts" -ChildPath "pip3.exe" + # $pipExePathExists = Test-Path $pipExePath + # Write-Verbose -Verbose "pip3 path exists: $pipExePathExists" + # Get-ChildItem "C:\__t\Python\3.12.7\x64\Scripts" + + # New-Item -Path "myPythonPkg" + # & 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' download pmc-cli -d myPythonPkg + # Write-Verbose -Verbose "trying gci" + # Get-ChildItem $pythonDlFolderPath + # Write-Verbose -Verbose "done pip download" displayName: 'Download pmc-cli package' - pwsh: | From 5aeec6eb4a6ab30c11da21644c9ba550cadc1850 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 3 Dec 2024 13:27:06 -0500 Subject: [PATCH 201/316] fix typo in Push-Location --- .pipelines/templates/release-prep-for-ev2.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index aa849126831..c5f4e379f46 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -58,9 +58,10 @@ jobs: displayName: 'Pip Authenticate' - pwsh: | - pip3 --version --verbose + $versionInfo = pip3 --version --verbose + Write-Verbose -Verbose "version info is: $versionInfo" & 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' --version --verbose - Push-Location 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' + Push-Location 'C:\__t\Python\3.12.7\x64\Scripts' Write-Verbose -Verbose "trying push pop" pip3 --version --verbose Pop-Location From 04cd7acbd95c5ea84411a7d2d1e7d40e04b5eee5 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 3 Dec 2024 13:39:31 -0500 Subject: [PATCH 202/316] use pip instead of pip3 --- .pipelines/templates/release-prep-for-ev2.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index c5f4e379f46..d30924cfbe8 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -58,12 +58,13 @@ jobs: displayName: 'Pip Authenticate' - pwsh: | - $versionInfo = pip3 --version --verbose + $versionInfo = pip --version --verbose Write-Verbose -Verbose "version info is: $versionInfo" - & 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' --version --verbose + & 'C:\__t\Python\3.12.7\x64\Scripts\pip.exe' --version --verbose Push-Location 'C:\__t\Python\3.12.7\x64\Scripts' Write-Verbose -Verbose "trying push pop" - pip3 --version --verbose + $pushedVersionInfo = ./pip.exe --version --verbose + Write-Verbose -Verbose "p version info: $pushedVersionInfo" Pop-Location # $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" From 6982273170df07b5d355e8a9145c60a18add86f0 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 3 Dec 2024 13:48:51 -0500 Subject: [PATCH 203/316] upgrade pip --- .pipelines/templates/release-prep-for-ev2.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index d30924cfbe8..8eea2f15090 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -58,6 +58,7 @@ jobs: displayName: 'Pip Authenticate' - pwsh: | + python -m pip install --upgrade pip $versionInfo = pip --version --verbose Write-Verbose -Verbose "version info is: $versionInfo" & 'C:\__t\Python\3.12.7\x64\Scripts\pip.exe' --version --verbose From c9136d77f7cac1aa6da465e824b5bd221d1e6931 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 3 Dec 2024 14:27:07 -0500 Subject: [PATCH 204/316] add pip download command --- .pipelines/templates/release-prep-for-ev2.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 8eea2f15090..79175dfdeb7 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -61,12 +61,14 @@ jobs: python -m pip install --upgrade pip $versionInfo = pip --version --verbose Write-Verbose -Verbose "version info is: $versionInfo" - & 'C:\__t\Python\3.12.7\x64\Scripts\pip.exe' --version --verbose - Push-Location 'C:\__t\Python\3.12.7\x64\Scripts' - Write-Verbose -Verbose "trying push pop" - $pushedVersionInfo = ./pip.exe --version --verbose - Write-Verbose -Verbose "p version info: $pushedVersionInfo" - Pop-Location + + $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" + $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" + + Write-Verbose -Verbose "Download pmc-cli to folder without installing it" + $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" + pip download -d $pythonDlFolderPath pmc-cli --verbose + # $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" # $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" From 5fff937e8e7a7e26925d74148fad049c77953700 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 3 Dec 2024 16:44:26 -0500 Subject: [PATCH 205/316] add code to download needed artifacts and create .tar.gz uploaded to ob_outputdirectory --- .pipelines/templates/release-prep-for-ev2.yml | 66 +++++++++++-------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 79175dfdeb7..3aa37549372 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -59,43 +59,53 @@ jobs: - pwsh: | python -m pip install --upgrade pip - $versionInfo = pip --version --verbose - Write-Verbose -Verbose "version info is: $versionInfo" + pip --version --verbose + Write-Verbose -Verbose "Download pmc-cli to folder without installing it" $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" - - Write-Verbose -Verbose "Download pmc-cli to folder without installing it" $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" pip download -d $pythonDlFolderPath pmc-cli --verbose + displayName: 'Download pmc-cli package' + - download: PSPackagesOfficial + artifact: 'drop_linux_package_deb' + displayName: 'Download artifact containing .deb_amd64.deb file from PSPackagesOfficial triggering pipeline' - # $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" - # $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" - - # Write-Verbose -Verbose "Download pmc-cli to folder without installing it" - # $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" - # New-Item -Path $pythonDlFolderPath - # pip3 --version --verbose - # pip3 download -d $pythonDlFolderPath pmc-cli --verbose - # $pipExePath = Join-Path -Path "C:\__t\Python\3.12.7\x64\Scripts" -ChildPath "pip3.exe" - # $pipExePathExists = Test-Path $pipExePath - # Write-Verbose -Verbose "pip3 path exists: $pipExePathExists" - # Get-ChildItem "C:\__t\Python\3.12.7\x64\Scripts" - - # New-Item -Path "myPythonPkg" - # & 'C:\__t\Python\3.12.7\x64\Scripts\pip3.exe' download pmc-cli -d myPythonPkg - # Write-Verbose -Verbose "trying gci" - # Get-ChildItem $pythonDlFolderPath - # Write-Verbose -Verbose "done pip download" - displayName: 'Download pmc-cli package' + - download: PSPackagesOfficial + artifact: 'drop_linux_package_rpm' + displayName: 'Download artifact containing .rh.x64_86.rpm file from PSPackagesOfficial triggering pipeline' + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_mariner_x64' + displayName: 'Download artifact containing .cm.x86_64.rpm file from PSPackagesOfficial triggering pipeline' + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_mariner_arm64' + displayName: 'Download artifact containing .cm.aarch64.rpm file from PSPackagesOfficial triggering pipeline' - pwsh: | - Write-Verbose -Verbose "Copy built .deb packages" - # assuming ESRP signed packages are here - # TODO: either copy packages to a folder and .tar it all up- i believe there's a size limit - # so option 2: make a .tar.gz file and pass it the way I do for images - displayName: 'Copy signed .deb packages - probably to a .tar.gz to pass as a file var' + Write-Verbose -Verbose "Copy ESRP signed .deb and .rpm packages" + $downloadedPipelineFolder = Join-Path '$(Pipeline.Workspace)' -ChildPath 'PSPackagesOfficial' + $packagesFolder = Join-Path '$(Pipeline.Workspace)' -ChildPath 'packages' + New-Item -Path $packagesFolder + + $packageFiles = Get-ChildItem -Path $downloadedPipelineFolder -Recurse -Directory -Filter "drop_*" | Get-ChildItem -File -Include *.deb, *.rpm + foreach ($file in $packageFiles) + { + Copy-Item -Path $($file.FullName) -Destination $packagesFolder + } + + $packagesTarGzDestination = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'packages.tar.gz' + tar -czvf $packagesTarGzDestination -C $packagesFolder . + displayName: 'Copy signed .deb and .rpm packages to .tar.gz to pass as a file var to shell extension' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Pipeline.Workspace)' + Contents: '*packages.tar.gz' + TargetFolder: $(ob_outputDirectory) + displayName: Copy packages.tar.gz file to $(ob_outputDirectory) - pwsh: | $pathToEV2Folder = Join-Path -Path '$(repoRoot)/.pipelines' -ChildPath 'EV2Specs' From 443ceaf308d7b027b573e45908bab5c022a11ec1 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 3 Dec 2024 16:56:21 -0500 Subject: [PATCH 206/316] add verbose logging --- .pipelines/templates/release-prep-for-ev2.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 3aa37549372..3b8a9292eaa 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -93,7 +93,8 @@ jobs: $packageFiles = Get-ChildItem -Path $downloadedPipelineFolder -Recurse -Directory -Filter "drop_*" | Get-ChildItem -File -Include *.deb, *.rpm foreach ($file in $packageFiles) { - Copy-Item -Path $($file.FullName) -Destination $packagesFolder + Write-Verbose -Verbose "copying file: $($file.FullName)" + Copy-Item -Path $($file.FullName) -Destination $packagesFolder -Verbose } $packagesTarGzDestination = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'packages.tar.gz' From c18c180baf3e630900d9e23ee29c080755f943a3 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 3 Dec 2024 17:06:56 -0500 Subject: [PATCH 207/316] add one more folder --- .pipelines/templates/release-prep-for-ev2.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 3b8a9292eaa..0c3a5a49c6a 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -87,7 +87,8 @@ jobs: - pwsh: | Write-Verbose -Verbose "Copy ESRP signed .deb and .rpm packages" $downloadedPipelineFolder = Join-Path '$(Pipeline.Workspace)' -ChildPath 'PSPackagesOfficial' - $packagesFolder = Join-Path '$(Pipeline.Workspace)' -ChildPath 'packages' + $srcFilesFolder = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'SourceFiles' + $packagesFolder = Join-Path -Path $srcFilesFolder -ChildPath 'packages' New-Item -Path $packagesFolder $packageFiles = Get-ChildItem -Path $downloadedPipelineFolder -Recurse -Directory -Filter "drop_*" | Get-ChildItem -File -Include *.deb, *.rpm From a0084c21dcdd7be49dda85fbbc4543d55c1e0c99 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 3 Dec 2024 17:37:42 -0500 Subject: [PATCH 208/316] add -ItemType Directory --- .pipelines/templates/release-prep-for-ev2.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 0c3a5a49c6a..eeec1471ab4 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -88,8 +88,9 @@ jobs: Write-Verbose -Verbose "Copy ESRP signed .deb and .rpm packages" $downloadedPipelineFolder = Join-Path '$(Pipeline.Workspace)' -ChildPath 'PSPackagesOfficial' $srcFilesFolder = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'SourceFiles' + New-Item -Path $srcFilesFolder -ItemType Directory $packagesFolder = Join-Path -Path $srcFilesFolder -ChildPath 'packages' - New-Item -Path $packagesFolder + New-Item -Path $packagesFolder -ItemType Directory $packageFiles = Get-ChildItem -Path $downloadedPipelineFolder -Recurse -Directory -Filter "drop_*" | Get-ChildItem -File -Include *.deb, *.rpm foreach ($file in $packageFiles) From 2ae8e4554eb97f6f3163cb50254a456a0cc888f1 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 4 Dec 2024 11:32:19 -0500 Subject: [PATCH 209/316] add variable group reference --- .pipelines/templates/release-prep-for-ev2.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index eeec1471ab4..d1289775289 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -9,6 +9,7 @@ jobs: - name: repoRoot value: '$(Build.SourcesDirectory)/PowerShell' - group: 'mscodehub-code-read-akv' + - group: 'packages.microsoft.com' - name: ob_tipsign_enabled value: true # Enable TiP sign steps: @@ -121,7 +122,7 @@ jobs: $skipPublishVar = '${{ parameters.skipPublish }}' $metadataHash = @{} $metadataHash["ReleaseTag"] = "$(ReleaseTag)" - $metadataHash["AadClientId"] = "$(PmcCliClientID)" + $metadataHash["AadClientId"] = '$(PmcCliClientID)' $metadataHash["BlobFolderName"] = "$(ReleaseTag)" $metadataHash["LTS"] = $metadata.LTSRelease.Latest $metadataHash["ForProduction"] = $true From 3fc655fe98a835271c3b53aa3f10c9ed04119d50 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 4 Dec 2024 11:50:50 -0500 Subject: [PATCH 210/316] remove var group and var for now --- .pipelines/templates/release-prep-for-ev2.yml | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index d1289775289..540a99e9222 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -9,7 +9,6 @@ jobs: - name: repoRoot value: '$(Build.SourcesDirectory)/PowerShell' - group: 'mscodehub-code-read-akv' - - group: 'packages.microsoft.com' - name: ob_tipsign_enabled value: true # Enable TiP sign steps: @@ -53,9 +52,7 @@ jobs: - task: PipAuthenticate@1 inputs: - # artifactFeeds: 'PowerShellCore/msazure-Compute-PMC@Release' artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' - # pythonDownloadServiceConnections: pmcDownload displayName: 'Pip Authenticate' - pwsh: | @@ -122,7 +119,7 @@ jobs: $skipPublishVar = '${{ parameters.skipPublish }}' $metadataHash = @{} $metadataHash["ReleaseTag"] = "$(ReleaseTag)" - $metadataHash["AadClientId"] = '$(PmcCliClientID)' + # $metadataHash["AadClientId"] = '$(PmcCliClientID)' $metadataHash["BlobFolderName"] = "$(ReleaseTag)" $metadataHash["LTS"] = $metadata.LTSRelease.Latest $metadataHash["ForProduction"] = $true @@ -137,32 +134,21 @@ jobs: Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path displayName: 'Create pmcScriptMetadata.json and mapping.json file' - - pwsh: | - $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' - $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" - tar -cvf $pathToRunTarFile -C $srcPath ./Run - # TODO: ensure this has: Run.ps1, settings.toml, python_dl - # tar cvf pmc_packages.tar start.sh settings.toml packages python_dl - displayName: 'Create archive for the shell extension' - - task: onebranch.pipeline.signing@1 - displayName: Sign Run.ps1 inputs: command: 'sign' signing_profile: 'tipsign' # external_distribution files_to_sign: '*.ps1' search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' + displayName: Sign Run.ps1 - pwsh: | $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" tar -cvf $pathToRunTarFile -C $srcPath ./Run - displayName: 'Compress Run script into tar file as needed for EV2 Shell extension' - - # - task: CopyFiles@2 - # inputs: - # Contents: '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/**' - # TargetFolder: 'EV2Specs/ServiceGroupRoot/tools' + # TODO: ensure this has: Run.ps1, settings.toml, python_dl + # tar cvf pmc_packages.tar start.sh settings.toml packages python_dl + displayName: 'Create archive for the shell extension' - task: CopyFiles@2 inputs: From 8555a183d906d039e91000efce6b956a90c31eda Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 4 Dec 2024 12:07:03 -0500 Subject: [PATCH 211/316] fix path to mapping.json --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 540a99e9222..622fe380791 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -128,7 +128,7 @@ jobs: $metadataHash | ConvertTo-Json | Out-File $pathToPMCMetadataFile Write-Verbose -Verbose "Copy mapping.json file to EV2Specs folder. file path at path $mappingFilePath exists: $mappingFilePathExists" - $mappingFilePath = Join-Path -Path '$(System.DefaultWorkingDirectory)/tools/packages.microsoft.com' -ChildPath 'mapping.json' + $mappingFilePath = Join-Path -Path '$(repoRoot)/tools/packages.microsoft.com' -ChildPath 'mapping.json' $mappingFilePathExists = Test-Path $mappingFilePath $mappingFileEV2Path = Join-Path -Path $pathToParametersFolder -ChildPath "mapping.json" Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path From 9b2774587cc81e8036b660c5ab198102b5cbabce Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 4 Dec 2024 14:46:36 -0500 Subject: [PATCH 212/316] add credscan suppression file var --- .pipelines/templates/release-prep-for-ev2.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 622fe380791..3a13290cb0f 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -11,6 +11,8 @@ jobs: - group: 'mscodehub-code-read-akv' - name: ob_tipsign_enabled value: true # Enable TiP sign + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: - checkout: self ## the global setting on lfs didn't work lfs: false From df945f1e15e3af785ed4e6e626aa1a0315981968 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 4 Dec 2024 15:23:33 -0500 Subject: [PATCH 213/316] copy over EV2Specs folder with fixed path --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- .pipelines/templates/release-publish-pmc.yml | 35 ++----------------- 2 files changed, 3 insertions(+), 34 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 3a13290cb0f..616ac56faab 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -154,5 +154,5 @@ jobs: - task: CopyFiles@2 inputs: - Contents: 'EV2Specs/**' + Contents: '$(repoRoot)/.pipelines/EV2Specs/**' TargetFolder: $(ob_outputDirectory) diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index e0c26b1a3f4..1a72ef83236 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -25,37 +25,6 @@ jobs: value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: - # - task: PipAuthenticate@1 - # inputs: - # artifactFeeds: 'pmc' - # pythonDownloadServiceConnections: pmcDownload - - # - pwsh: | - # pip install pmc-cli==1.12.0 - # - # $newPath = (resolve-path '~/.local/bin').providerpath - # $vstsCommandString = "vso[task.setvariable variable=PATH]${env:PATH}:$newPath" - # Write-Host "sending " + $vstsCommandString - # Write-Host "##$vstsCommandString" - # displayName: Install pmc cli - - # - pwsh: | - # $metadata = Get-Content -Path "$(Build.SourcesDirectory)/tools/metadata.json" -Raw | ConvertFrom-Json - # $params = @{ - # ReleaseTag = "$(ReleaseTag)" - # AadClientId = "$(PmcCliClientID)" - # BlobFolderName = "$(ReleaseTag)" - # LTS = $metadata.LTSRelease.Latest - # ForProduction = $true - # SkipPublish = $${{ parameters.skipPublish }} - # MappingFilePath = '$(System.DefaultWorkingDirectory)/tools/packages.microsoft.com/mapping.json' - # } - # - # $params | Out-String -width 9999 -Stream | write-Verbose -Verbose - # - # & '$(Pipeline.Workspace)/tools/packages.microsoft.com-v4/releaseLinuxPackages.ps1' @params - # displayName: Run release script - - task: DownloadPipelineArtifact@2 inputs: targetPath: '$(Pipeline.Workspace)' @@ -77,5 +46,5 @@ jobs: ConnectedServiceName: 'docker-deployement-test' ServiceRootLocation: 'LinkedArtifact' RolloutSpecType: 'RSPath' - ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot' - RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEv2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' + ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' + RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From 17174a687d04b28e84953e853221243150cf86db Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 4 Dec 2024 15:45:51 -0500 Subject: [PATCH 214/316] specify SourceFolder for CopyFiles task --- .pipelines/templates/release-prep-for-ev2.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 616ac56faab..cd8619ac6de 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -154,5 +154,6 @@ jobs: - task: CopyFiles@2 inputs: - Contents: '$(repoRoot)/.pipelines/EV2Specs/**' + SourceFolder: '$(repoRoot)/.pipelines' + Contents: 'EV2Specs/**' TargetFolder: $(ob_outputDirectory) From af2334bf9e4c4cb33989e5362beb6b9b54c28ebc Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 4 Dec 2024 16:27:44 -0500 Subject: [PATCH 215/316] put packages.tar.gz under ServiceGroupRoot/Parameters --- .pipelines/templates/release-prep-for-ev2.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index cd8619ac6de..b1c020aceab 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -101,14 +101,12 @@ jobs: $packagesTarGzDestination = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'packages.tar.gz' tar -czvf $packagesTarGzDestination -C $packagesFolder . - displayName: 'Copy signed .deb and .rpm packages to .tar.gz to pass as a file var to shell extension' - - task: CopyFiles@2 - inputs: - SourceFolder: '$(Pipeline.Workspace)' - Contents: '*packages.tar.gz' - TargetFolder: $(ob_outputDirectory) - displayName: Copy packages.tar.gz file to $(ob_outputDirectory) + $pathToEV2Folder = Join-Path -Path '$(repoRoot)/.pipelines' -ChildPath 'EV2Specs' + $pathToServiceGroupRootFolder = Join-Path $pathToEV2Folder -ChildPath 'ServiceGroupRoot' + $pathToParametersFolder = Join-Path -Path $pathToServiceGroupRootFolder -ChildPath 'Parameters' + Copy-Item $packagesTarGzDestination -Destination $pathToParametersFolder + displayName: 'Copy signed .deb and .rpm packages to .tar.gz to pass as a file var to shell extension' - pwsh: | $pathToEV2Folder = Join-Path -Path '$(repoRoot)/.pipelines' -ChildPath 'EV2Specs' From 4fc6dc65324edd03be71b5b8f74b8dca937fcfa7 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 4 Dec 2024 17:04:51 -0500 Subject: [PATCH 216/316] create Parameters folder once and refer to pmc MI var --- .pipelines/templates/release-prep-for-ev2.yml | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index b1c020aceab..df23941afe7 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -8,6 +8,8 @@ jobs: value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: repoRoot value: '$(Build.SourcesDirectory)/PowerShell' + - name: ev2ParametersFolder + value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Paramters' - group: 'mscodehub-code-read-akv' - name: ob_tipsign_enabled value: true # Enable TiP sign @@ -47,6 +49,10 @@ jobs: Get-ChildItem '$(Build.SourcesDirectory)' displayName: 'Capture Workspace' + - pwsh: | + New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory + displayName: 'Create Parameters folder under EV2Specs folder' + - task: UsePythonVersion@0 displayName: 'Use Python 3.12.7' inputs: @@ -99,21 +105,23 @@ jobs: Copy-Item -Path $($file.FullName) -Destination $packagesFolder -Verbose } - $packagesTarGzDestination = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'packages.tar.gz' + $packagesTarGzDestination = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'packages.tar.gz' tar -czvf $packagesTarGzDestination -C $packagesFolder . - - $pathToEV2Folder = Join-Path -Path '$(repoRoot)/.pipelines' -ChildPath 'EV2Specs' - $pathToServiceGroupRootFolder = Join-Path $pathToEV2Folder -ChildPath 'ServiceGroupRoot' - $pathToParametersFolder = Join-Path -Path $pathToServiceGroupRootFolder -ChildPath 'Parameters' - Copy-Item $packagesTarGzDestination -Destination $pathToParametersFolder displayName: 'Copy signed .deb and .rpm packages to .tar.gz to pass as a file var to shell extension' - pwsh: | - $pathToEV2Folder = Join-Path -Path '$(repoRoot)/.pipelines' -ChildPath 'EV2Specs' - $pathToServiceGroupRootFolder = Join-Path $pathToEV2Folder -ChildPath 'ServiceGroupRoot' - $pathToParametersFolder = Join-Path -Path $pathToServiceGroupRootFolder -ChildPath 'Parameters' - New-Item -Path $pathToParametersFolder -ItemType Directory - $pathToPMCMetadataFile = Join-Path -Path $pathToParametersFolder -ChildPath 'PmcMetadata.json' + $settingFilePath = Join-Path '$(ev2ParametersFolder)' -ChildPath 'settings.toml' + New-Item -Path $settingFilePath -ItemType File + $pmcMIClientID = '$(PmcMIClientID)' + + Add-Content -Path $settingFilePath -Value "[default]" + Add-Content -Path $settingFilePath -Value "base_url = `"https://pmc-ingest.trafficmanager.net/api/v4`"" + Add-Content -Path $settingFilePath -Value "auth_type = `"msi`"" + Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" + displayName: 'Create settings.toml file with MI clientId populated' + + - pwsh: | + $pathToPMCMetadataFile = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'pmcMetadata.json' $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json $skipPublishVar = '${{ parameters.skipPublish }}' @@ -130,7 +138,7 @@ jobs: Write-Verbose -Verbose "Copy mapping.json file to EV2Specs folder. file path at path $mappingFilePath exists: $mappingFilePathExists" $mappingFilePath = Join-Path -Path '$(repoRoot)/tools/packages.microsoft.com' -ChildPath 'mapping.json' $mappingFilePathExists = Test-Path $mappingFilePath - $mappingFileEV2Path = Join-Path -Path $pathToParametersFolder -ChildPath "mapping.json" + $mappingFileEV2Path = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath "mapping.json" Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path displayName: 'Create pmcScriptMetadata.json and mapping.json file' From 03c64ecd85a6d2640020c57053917bd92b2df477 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 4 Dec 2024 17:23:13 -0500 Subject: [PATCH 217/316] refer to variable group --- .pipelines/templates/release-prep-for-ev2.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index df23941afe7..f7a5cb4f7c8 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -11,6 +11,7 @@ jobs: - name: ev2ParametersFolder value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Paramters' - group: 'mscodehub-code-read-akv' + - group: 'packages.microsoft.com' - name: ob_tipsign_enabled value: true # Enable TiP sign - name: ob_sdl_credscan_suppressionsFile From 7580a9c0fc2f27b041d2c4dc59a72a918116950f Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 5 Dec 2024 13:21:36 -0500 Subject: [PATCH 218/316] update EV2 json files and test pmc tool in run script --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 64 +++++++++---------- .../ServiceGroupRoot/Shell/Run/settings.toml | 4 -- .../ServiceGroupRoot/UploadLinux.Rollout.json | 37 +++++------ .pipelines/templates/release-prep-for-ev2.yml | 54 ++++++++++++---- 4 files changed, 86 insertions(+), 73 deletions(-) delete mode 100644 .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 63a8316b828..13620d1db3d 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -1,54 +1,48 @@ -# ensure SAS variables were passed in -if ($null -eq $env:RELEASE_TAG) +if ($null -eq $env:MAPPING_FILE) { - Write-Verbose -Verbose "RELEASE_TAG variable didn't get passed correctly" + Write-Verbose -Verbose "MAPPING_FILE variable didn't get passed correctly" return 1 } -if ($null -eq $env:AAD_CLIENT_ID) +if ($null -eq $env:PWSH_PACKAGES_TARGZIP) { - Write-Verbose -Verbose "AAD_CLIENT_ID variable didn't get passed correctly" + Write-Verbose -Verbose "PWSH_PACKAGES_TARGZIP variable didn't get passed correctly" return 1 } -if ($null -eq $env:BLOB_FOLDER_NAME) +if ($null -eq $env:PMC_METADATA) { - Write-Verbose -Verbose "BLOB_FOLDER_NAME variable didn't get passed correctly" + Write-Verbose -Verbose "PMC_METADATA variable didn't get passed correctly" return 1 } -if ($null -eq $env:LTS) -{ - Write-Verbose -Verbose "LTS variable didn't get passed correctly" - return 1 -} +try { + Write-Verbose -Verbose "mapping.json file: $env:MAPPING_FILE" + Write-Verbose -Verbose "pwsh packages.tar.gz file: $env:PWSH_PACKAGES_TARGZIP" + Write-Verbose -Verbose "pmcMetadata.json file: $env:PMC_METADATA" -if ($null -eq $env:FOR_PRODUCTION) -{ - Write-Verbose -Verbose "FOR_PRODUCTION variable didn't get parsed properly" - return 1 -} + Write-Verbose -Verbose "Downloading files" + Invoke-WebRequest -Uri $env:MAPPING_FILE -OutFile mapping.json + Invoke-WebRequest -Uri $env:PWSH_PACKAGES_TARGZIP -OutFile packages.tar.gz + Invoke-WebRequest -Uri $env:PMC_METADATA -OutFile pmcMetadata.json -if ($null -eq $env:SKIP_PUBLISH) -{ - Write-Verbose -Verbose "SKIP_PUBLISH variable didn't get parsed properly" - return 1 -} + $settingsFile = Join-Path '/package/unarchive' -ChildPath 'settings.toml' + $settingsFileExists = Test-Path -Path $settingsFile + Write-Verbose -Verbose "settings.toml exists: $settingsFileExists" + $pythonDlFolder = Join-Path '/package/unarchive' -ChildPath 'python_dl' + $pyPathExists = Test-Path -Path $pythonDlFolder + Write-Verbose -Verbose "python_dl folder path exists: $pyPathExists" -if ($null -eq $env:MAPPING_FILE_PATH) -{ - Write-Verbose -Verbose "MAPPING_FILE_PATH variable didn't get parsed properly" - return 1 -} + Write-Verbose -Verbose "Installing pmc-cli" + python -m pip install --upgrade pip + pip --version --verbose + pip install /package/unarchive/python_dl/*.whl + + Write-Verbose -Verbose "Test pmc-cli" + + which pmc + pmc -d -c $settingsFile repo list --name "azurelinux-3.0-prod-ms-oss-x86_64-yum" || exit 1 -try { - Write-Verbose -Verbose "ReleaseTag: $env:RELEASE_TAG" - Write-Verbose -Verbose "AAD_Client_ID: $env:AAD_CLIENT_ID" - Write-Verbose -Verbose "Blob_folder_name: $env:BLOB_FOLDER_NAME" - Write-Verbose -Verbose "LTS: $env:LTS" - Write-Verbose -Verbose "for_production: $env:FOR_PRODUCTION" - Write-Verbose -Verbose "Skip_publish: $env:SKIP_PUBLISH" - Write-Verbose -Verbose "mapping file: $env:MAPPING_FILE_PATH" } catch { Write-Error -ErrorAction Stop $_.Exception.Message diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml deleted file mode 100644 index 877529312cf..00000000000 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/settings.toml +++ /dev/null @@ -1,4 +0,0 @@ -[default] -base_url = "https://pmc-ingest.trafficmanager.net/api/v4" -auth_type = "msi" -client_id = "..." # Client ID of your User-Assigned Managed Identity diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json b/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json index 35901082628..d7c75c2e216 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json +++ b/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json @@ -21,32 +21,25 @@ ], "environmentVariables": [ { - "name": "RELEASE_TAG", - "value": "default" + "name": "MAPPING_FILE", + "reference": + { + "path": "Parameters\\mapping.json" + } }, { - "name": "AAD_CLIENT_ID", - "value" : "default" + "name": "PWSH_PACKAGES_TARGZIP", + "reference": + { + "path": "Parameters\\packages.tar.gz" + } }, { - "name": "BLOB_FOLDER_NAME", - "value": "default" - }, - { - "name": "LTS", - "value": "default" - }, - { - "name": "FOR_PRODUCTION", - "value": "true" - }, - { - "name": "SKIP_PUBLISH", - "value": "true" - }, - { - "name": "MAPPING_FILE_PATH", - "default": "default" + "name": "PMC_METADATA", + "reference": + { + "path": "Parameters\\pmcMetadata.json" + } } ], "identity": { diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index f7a5cb4f7c8..71d265789b9 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -8,6 +8,8 @@ jobs: value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: repoRoot value: '$(Build.SourcesDirectory)/PowerShell' + - name: ev2ServiceGroupRootFolder + value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot' - name: ev2ParametersFolder value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Paramters' - group: 'mscodehub-code-read-akv' @@ -110,17 +112,6 @@ jobs: tar -czvf $packagesTarGzDestination -C $packagesFolder . displayName: 'Copy signed .deb and .rpm packages to .tar.gz to pass as a file var to shell extension' - - pwsh: | - $settingFilePath = Join-Path '$(ev2ParametersFolder)' -ChildPath 'settings.toml' - New-Item -Path $settingFilePath -ItemType File - $pmcMIClientID = '$(PmcMIClientID)' - - Add-Content -Path $settingFilePath -Value "[default]" - Add-Content -Path $settingFilePath -Value "base_url = `"https://pmc-ingest.trafficmanager.net/api/v4`"" - Add-Content -Path $settingFilePath -Value "auth_type = `"msi`"" - Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" - displayName: 'Create settings.toml file with MI clientId populated' - - pwsh: | $pathToPMCMetadataFile = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'pmcMetadata.json' @@ -143,6 +134,17 @@ jobs: Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path displayName: 'Create pmcScriptMetadata.json and mapping.json file' + - pwsh: | + $settingFilePath = Join-Path '$(ev2ServiceGroupRootFolder)/Shell/Run' -ChildPath 'settings.toml' + New-Item -Path $settingFilePath -ItemType File + $pmcMIClientID = '$(PmcMIClientID)' + + Add-Content -Path $settingFilePath -Value "[default]" + Add-Content -Path $settingFilePath -Value "base_url = `"https://pmc-ingest.trafficmanager.net/api/v4`"" + Add-Content -Path $settingFilePath -Value "auth_type = `"msi`"" + Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" + displayName: 'Create settings.toml file with MI clientId populated' + - task: onebranch.pipeline.signing@1 inputs: command: 'sign' @@ -156,9 +158,37 @@ jobs: $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" tar -cvf $pathToRunTarFile -C $srcPath ./Run # TODO: ensure this has: Run.ps1, settings.toml, python_dl - # tar cvf pmc_packages.tar start.sh settings.toml packages python_dl displayName: 'Create archive for the shell extension' + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + $content.RolloutMetadata.Notification.Email.To = '$(PmcEV2SupportEmail)' + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 4 | Out-File $pathToJsonFile + displayName: 'Replace values in RolloutSpecPath.json' + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'UploadLinux.Rollout.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + + $identityString = "/subscriptions/$(PmcSubscription)/resourcegroups/$(PmcResourceGroup)/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$(PmcMIName)" + $content.shellExtensions.launch.identity.userAssignedIdentities[0] = $identityString + + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 6 | Out-File $pathToJsonFile + displayName: 'Replace values in UploadLinux.Rollout.json file' + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'ServiceModel.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + $content.ServiceResourceGroups[0].AzureResourceGroupName = '$(PmcResourceGroup)' + $content.ServiceResourceGroups[0].AzureSubscriptionId = '$(PmcSubscription)' + + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile + displayName: 'Replace values in ServiceModel.json' + - task: CopyFiles@2 inputs: SourceFolder: '$(repoRoot)/.pipelines' From 06391806c6831f7c8633627cd34ba58f9b6ea336 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 5 Dec 2024 13:46:06 -0500 Subject: [PATCH 219/316] add tsaconfig file var and fix typo --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 ++ .pipelines/templates/release-prep-for-ev2.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 5eb31daabb5..33f4430e6c3 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -41,6 +41,8 @@ variables: value: true - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\.config\tsaoptions.json - name: WindowsContainerImage value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - name: LinuxContainerImage diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 71d265789b9..80994ccdb47 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -11,7 +11,7 @@ jobs: - name: ev2ServiceGroupRootFolder value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot' - name: ev2ParametersFolder - value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Paramters' + value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Parameters' - group: 'mscodehub-code-read-akv' - group: 'packages.microsoft.com' - name: ob_tipsign_enabled From 5ba52b944026518592f8eb646632020c27440701 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 5 Dec 2024 15:17:00 -0500 Subject: [PATCH 220/316] add buildVer.txt file --- .pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt b/.pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt new file mode 100644 index 00000000000..e69de29bb2d From a5dc54804891305ca59bc99c46788a90d4239d56 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 5 Dec 2024 15:39:12 -0500 Subject: [PATCH 221/316] populate version --- .pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt b/.pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt index e69de29bb2d..7dea76edb3d 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt +++ b/.pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt @@ -0,0 +1 @@ +1.0.1 From 9c9d22e7e78ed4165c4d6386a9434ff1ab0ba3cd Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 5 Dec 2024 17:27:28 -0500 Subject: [PATCH 222/316] have EV2 task be seperated out into its own stage --- .../PowerShell-Release-Official-Azure.yml | 30 +- .pipelines/templates/release-prep-for-ev2.yml | 395 +++++++++--------- .pipelines/templates/release-publish-pmc.yml | 110 +++-- 3 files changed, 279 insertions(+), 256 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 33f4430e6c3..72220b98be7 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -101,11 +101,13 @@ extends: # suppressionFile: $(Build.SourcesDirectory)\.gdn\global.gdnsuppress tsaOptionsFile: .config\tsaoptions.json stages: - - stage: PrepForEV2 - displayName: Prep for EV2 - dependsOn: [] - jobs: - - template: /.pipelines/templates/release-prep-for-ev2.yml@self + - template: /.pipelines/templates/release-prep-for-ev2.yml@self + - template: /.pipelines/templates/release-publish-pmc.yml@self + # - stage: PrepForEV2 + # displayName: Prep for EV2 + # dependsOn: [] + # jobs: + # - template: /.pipelines/templates/release-prep-for-ev2.yml@self # - stage: BlobPublic # displayName: Make Blob Public @@ -113,15 +115,15 @@ extends: # jobs: # - template: /.pipelines/templates/release-MakeBlobPublic.yml@self - - stage: Test_PublishPMC - displayName: Publish PMC - dependsOn: - - PrepForEV2 - # - BlobPublic - variables: - ob_release_environment: Test - jobs: - - template: /.pipelines/templates/release-publish-pmc.yml@self + # - stage: Test_PublishPMC + # displayName: Publish PMC + # dependsOn: + # - PrepForEV2 + # # - BlobPublic + # variables: + # ob_release_environment: Test + # jobs: + # - template: /.pipelines/templates/release-publish-pmc.yml@self # - stage: UploadBuildInfoJson # dependsOn: diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 80994ccdb47..882432292d5 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -1,196 +1,199 @@ -jobs: -- job: CopyEV2FilesToArtifact - displayName: Copy EV2 Files to Artifact - pool: - type: windows - variables: - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: repoRoot - value: '$(Build.SourcesDirectory)/PowerShell' - - name: ev2ServiceGroupRootFolder - value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot' - - name: ev2ParametersFolder - value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Parameters' - - group: 'mscodehub-code-read-akv' - - group: 'packages.microsoft.com' - - name: ob_tipsign_enabled - value: true # Enable TiP sign - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - steps: - - checkout: self ## the global setting on lfs didn't work - lfs: false - - - template: release-SetReleaseTagandContainerName.yml - - - pwsh: | - $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' - $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: Set Package version - - - pwsh: | - $branch = 'mirror-target' - $gitArgs = "clone", - "--verbose", - "--branch", - "$branch", - "https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools", - '$(Pipeline.Workspace)/tools' - $gitArgs | Write-Verbose -Verbose - git $gitArgs - displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub - - - pwsh: | - Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose - displayName: 'Capture Environment Variables' - - - pwsh: | - Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose - Get-ChildItem '$(Build.SourcesDirectory)' - displayName: 'Capture Workspace' - - - pwsh: | - New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory - displayName: 'Create Parameters folder under EV2Specs folder' - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.12.7' - inputs: - versionSpec: 3.12.7 - - - task: PipAuthenticate@1 - inputs: - artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' - displayName: 'Pip Authenticate' - - - pwsh: | - python -m pip install --upgrade pip - pip --version --verbose - - Write-Verbose -Verbose "Download pmc-cli to folder without installing it" - $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" - $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" - $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" - pip download -d $pythonDlFolderPath pmc-cli --verbose - displayName: 'Download pmc-cli package' - - - download: PSPackagesOfficial - artifact: 'drop_linux_package_deb' - displayName: 'Download artifact containing .deb_amd64.deb file from PSPackagesOfficial triggering pipeline' - - - download: PSPackagesOfficial - artifact: 'drop_linux_package_rpm' - displayName: 'Download artifact containing .rh.x64_86.rpm file from PSPackagesOfficial triggering pipeline' - - - download: PSPackagesOfficial - artifact: 'drop_linux_package_mariner_x64' - displayName: 'Download artifact containing .cm.x86_64.rpm file from PSPackagesOfficial triggering pipeline' - - - download: PSPackagesOfficial - artifact: 'drop_linux_package_mariner_arm64' - displayName: 'Download artifact containing .cm.aarch64.rpm file from PSPackagesOfficial triggering pipeline' - - - pwsh: | - Write-Verbose -Verbose "Copy ESRP signed .deb and .rpm packages" - $downloadedPipelineFolder = Join-Path '$(Pipeline.Workspace)' -ChildPath 'PSPackagesOfficial' - $srcFilesFolder = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'SourceFiles' - New-Item -Path $srcFilesFolder -ItemType Directory - $packagesFolder = Join-Path -Path $srcFilesFolder -ChildPath 'packages' - New-Item -Path $packagesFolder -ItemType Directory - - $packageFiles = Get-ChildItem -Path $downloadedPipelineFolder -Recurse -Directory -Filter "drop_*" | Get-ChildItem -File -Include *.deb, *.rpm - foreach ($file in $packageFiles) - { - Write-Verbose -Verbose "copying file: $($file.FullName)" - Copy-Item -Path $($file.FullName) -Destination $packagesFolder -Verbose - } - - $packagesTarGzDestination = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'packages.tar.gz' - tar -czvf $packagesTarGzDestination -C $packagesFolder . - displayName: 'Copy signed .deb and .rpm packages to .tar.gz to pass as a file var to shell extension' - - - pwsh: | - $pathToPMCMetadataFile = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'pmcMetadata.json' - - $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json - $skipPublishVar = '${{ parameters.skipPublish }}' - $metadataHash = @{} - $metadataHash["ReleaseTag"] = "$(ReleaseTag)" - # $metadataHash["AadClientId"] = '$(PmcCliClientID)' - $metadataHash["BlobFolderName"] = "$(ReleaseTag)" - $metadataHash["LTS"] = $metadata.LTSRelease.Latest - $metadataHash["ForProduction"] = $true - $metadataHash["SkipPublish"] = $skipPublishVar - - $metadataHash | ConvertTo-Json | Out-File $pathToPMCMetadataFile - - Write-Verbose -Verbose "Copy mapping.json file to EV2Specs folder. file path at path $mappingFilePath exists: $mappingFilePathExists" - $mappingFilePath = Join-Path -Path '$(repoRoot)/tools/packages.microsoft.com' -ChildPath 'mapping.json' - $mappingFilePathExists = Test-Path $mappingFilePath - $mappingFileEV2Path = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath "mapping.json" - Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path - displayName: 'Create pmcScriptMetadata.json and mapping.json file' - - - pwsh: | - $settingFilePath = Join-Path '$(ev2ServiceGroupRootFolder)/Shell/Run' -ChildPath 'settings.toml' - New-Item -Path $settingFilePath -ItemType File - $pmcMIClientID = '$(PmcMIClientID)' - - Add-Content -Path $settingFilePath -Value "[default]" - Add-Content -Path $settingFilePath -Value "base_url = `"https://pmc-ingest.trafficmanager.net/api/v4`"" - Add-Content -Path $settingFilePath -Value "auth_type = `"msi`"" - Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" - displayName: 'Create settings.toml file with MI clientId populated' - - - task: onebranch.pipeline.signing@1 - inputs: - command: 'sign' - signing_profile: 'tipsign' # external_distribution - files_to_sign: '*.ps1' - search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' - displayName: Sign Run.ps1 - - - pwsh: | - $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' - $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" - tar -cvf $pathToRunTarFile -C $srcPath ./Run - # TODO: ensure this has: Run.ps1, settings.toml, python_dl - displayName: 'Create archive for the shell extension' - - - pwsh: | - $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json' - $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json - $content.RolloutMetadata.Notification.Email.To = '$(PmcEV2SupportEmail)' - Remove-Item -Path $pathToJsonFile - $content | ConvertTo-Json -Depth 4 | Out-File $pathToJsonFile - displayName: 'Replace values in RolloutSpecPath.json' - - - pwsh: | - $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'UploadLinux.Rollout.json' - $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json - - $identityString = "/subscriptions/$(PmcSubscription)/resourcegroups/$(PmcResourceGroup)/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$(PmcMIName)" - $content.shellExtensions.launch.identity.userAssignedIdentities[0] = $identityString - - Remove-Item -Path $pathToJsonFile - $content | ConvertTo-Json -Depth 6 | Out-File $pathToJsonFile - displayName: 'Replace values in UploadLinux.Rollout.json file' - - - pwsh: | - $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'ServiceModel.json' - $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json - $content.ServiceResourceGroups[0].AzureResourceGroupName = '$(PmcResourceGroup)' - $content.ServiceResourceGroups[0].AzureSubscriptionId = '$(PmcSubscription)' - - Remove-Item -Path $pathToJsonFile - $content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile - displayName: 'Replace values in ServiceModel.json' - - - task: CopyFiles@2 - inputs: - SourceFolder: '$(repoRoot)/.pipelines' - Contents: 'EV2Specs/**' - TargetFolder: $(ob_outputDirectory) +stages: +- stage: PrepForEV2 + displayName: 'Copy and prep all files needed for EV2 stage' + jobs: + - job: CopyEV2FilesToArtifact + displayName: 'Copy EV2 Files to Artifact' + pool: + type: windows + variables: + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: repoRoot + value: '$(Build.SourcesDirectory)/PowerShell' + - name: ev2ServiceGroupRootFolder + value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot' + - name: ev2ParametersFolder + value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Parameters' + - group: 'mscodehub-code-read-akv' + - group: 'packages.microsoft.com' + - name: ob_tipsign_enabled + value: true # Enable TiP sign + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + steps: + - checkout: self ## the global setting on lfs didn't work + lfs: false + + - template: release-SetReleaseTagandContainerName.yml + + - pwsh: | + $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' + $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Set Package version + + - pwsh: | + $branch = 'mirror-target' + $gitArgs = "clone", + "--verbose", + "--branch", + "$branch", + "https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools", + '$(Pipeline.Workspace)/tools' + $gitArgs | Write-Verbose -Verbose + git $gitArgs + displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub + + - pwsh: | + Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose + displayName: 'Capture Environment Variables' + + - pwsh: | + Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose + Get-ChildItem '$(Build.SourcesDirectory)' + displayName: 'Capture Workspace' + + - pwsh: | + New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory + displayName: 'Create Parameters folder under EV2Specs folder' + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.12.7' + inputs: + versionSpec: 3.12.7 + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' + displayName: 'Pip Authenticate' + + - pwsh: | + python -m pip install --upgrade pip + pip --version --verbose + + Write-Verbose -Verbose "Download pmc-cli to folder without installing it" + $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" + $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" + $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" + pip download -d $pythonDlFolderPath pmc-cli --verbose + displayName: 'Download pmc-cli package' + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_deb' + displayName: 'Download artifact containing .deb_amd64.deb file from PSPackagesOfficial triggering pipeline' + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_rpm' + displayName: 'Download artifact containing .rh.x64_86.rpm file from PSPackagesOfficial triggering pipeline' + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_mariner_x64' + displayName: 'Download artifact containing .cm.x86_64.rpm file from PSPackagesOfficial triggering pipeline' + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_mariner_arm64' + displayName: 'Download artifact containing .cm.aarch64.rpm file from PSPackagesOfficial triggering pipeline' + + - pwsh: | + Write-Verbose -Verbose "Copy ESRP signed .deb and .rpm packages" + $downloadedPipelineFolder = Join-Path '$(Pipeline.Workspace)' -ChildPath 'PSPackagesOfficial' + $srcFilesFolder = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'SourceFiles' + New-Item -Path $srcFilesFolder -ItemType Directory + $packagesFolder = Join-Path -Path $srcFilesFolder -ChildPath 'packages' + New-Item -Path $packagesFolder -ItemType Directory + + $packageFiles = Get-ChildItem -Path $downloadedPipelineFolder -Recurse -Directory -Filter "drop_*" | Get-ChildItem -File -Include *.deb, *.rpm + foreach ($file in $packageFiles) + { + Write-Verbose -Verbose "copying file: $($file.FullName)" + Copy-Item -Path $($file.FullName) -Destination $packagesFolder -Verbose + } + + $packagesTarGzDestination = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'packages.tar.gz' + tar -czvf $packagesTarGzDestination -C $packagesFolder . + displayName: 'Copy signed .deb and .rpm packages to .tar.gz to pass as a file var to shell extension' + + - pwsh: | + $pathToPMCMetadataFile = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'pmcMetadata.json' + + $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json + $skipPublishVar = '${{ parameters.skipPublish }}' + $metadataHash = @{} + $metadataHash["ReleaseTag"] = "$(ReleaseTag)" + # $metadataHash["AadClientId"] = '$(PmcCliClientID)' + $metadataHash["BlobFolderName"] = "$(ReleaseTag)" + $metadataHash["LTS"] = $metadata.LTSRelease.Latest + $metadataHash["ForProduction"] = $true + $metadataHash["SkipPublish"] = $skipPublishVar + + $metadataHash | ConvertTo-Json | Out-File $pathToPMCMetadataFile + + Write-Verbose -Verbose "Copy mapping.json file to EV2Specs folder. file path at path $mappingFilePath exists: $mappingFilePathExists" + $mappingFilePath = Join-Path -Path '$(repoRoot)/tools/packages.microsoft.com' -ChildPath 'mapping.json' + $mappingFilePathExists = Test-Path $mappingFilePath + $mappingFileEV2Path = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath "mapping.json" + Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path + displayName: 'Create pmcScriptMetadata.json and mapping.json file' + + - pwsh: | + $settingFilePath = Join-Path '$(ev2ServiceGroupRootFolder)/Shell/Run' -ChildPath 'settings.toml' + New-Item -Path $settingFilePath -ItemType File + $pmcMIClientID = '$(PmcMIClientID)' + + Add-Content -Path $settingFilePath -Value "[default]" + Add-Content -Path $settingFilePath -Value "base_url = `"https://pmc-ingest.trafficmanager.net/api/v4`"" + Add-Content -Path $settingFilePath -Value "auth_type = `"msi`"" + Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" + displayName: 'Create settings.toml file with MI clientId populated' + + - task: onebranch.pipeline.signing@1 + inputs: + command: 'sign' + signing_profile: 'tipsign' # external_distribution + files_to_sign: '*.ps1' + search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' + displayName: Sign Run.ps1 + + - pwsh: | + $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' + $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" + tar -cvf $pathToRunTarFile -C $srcPath ./Run + # TODO: ensure this has: Run.ps1, settings.toml, python_dl + displayName: 'Create archive for the shell extension' + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + $content.RolloutMetadata.Notification.Email.To = '$(PmcEV2SupportEmail)' + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 4 | Out-File $pathToJsonFile + displayName: 'Replace values in RolloutSpecPath.json' + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'UploadLinux.Rollout.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + + $identityString = "/subscriptions/$(PmcSubscription)/resourcegroups/$(PmcResourceGroup)/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$(PmcMIName)" + $content.shellExtensions.launch.identity.userAssignedIdentities[0] = $identityString + + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 6 | Out-File $pathToJsonFile + displayName: 'Replace values in UploadLinux.Rollout.json file' + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'ServiceModel.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + $content.ServiceResourceGroups[0].AzureResourceGroupName = '$(PmcResourceGroup)' + $content.ServiceResourceGroups[0].AzureSubscriptionId = '$(PmcSubscription)' + + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile + displayName: 'Replace values in ServiceModel.json' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(repoRoot)/.pipelines' + Contents: 'EV2Specs/**' + TargetFolder: $(ob_outputDirectory) diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 1a72ef83236..b07de94997c 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -1,50 +1,68 @@ -jobs: -- job: PMCPublish - displayName: Publish to PMC - condition: succeeded() - pool: - type: release +stages: +# - stage: 'Prod_Release' +- stage: 'Test_Release' + displayName: 'Deploy packages to PMC with EV2' + dependsOn: + - PrepForEV2 variables: - - name: runCodesignValidationInjection - value: false - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - value: 1 - - group: 'mscodehub-code-read-akv' - - group: 'packages.microsoft.com' - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_sdl_codeSignValidation_enabled - value: false - - name: ob_sdl_binskim_enabled - value: false - - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: ob_release_environment + value: "Test" + - name: repoRoot + value: $(Build.SourcesDirectory) + jobs: + - job: Test_ReleaseJob + displayName: Publish to PMC + pool: + type: release + # variables: + # - name: runCodesignValidationInjection + # value: false + # - name: NugetSecurityAnalysisWarningLevel + # value: none + # - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + # value: 1 + # - group: 'mscodehub-code-read-akv' + # - group: 'packages.microsoft.com' + # - name: ob_outputDirectory + # value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + # - name: ob_sdl_codeSignValidation_enabled + # value: false + # - name: ob_sdl_binskim_enabled + # value: false + # - name: ob_sdl_tsa_configFile + # value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + # - name: ob_sdl_credscan_suppressionsFile + # value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - steps: - - task: DownloadPipelineArtifact@2 - inputs: - targetPath: '$(Pipeline.Workspace)' - artifact: drop_PrepForEV2_CopyEv2FilesToArtifact - displayName: 'Download drop_PrepForEV2_CopyEv2FilesToArtifact artifact that has all files needed' + steps: + - task: DownloadPipelineArtifact@2 + inputs: + targetPath: '$(Pipeline.Workspace)' + artifact: drop_PrepForEV2_CopyEv2FilesToArtifact + displayName: 'Download drop_PrepForEV2_CopyEv2FilesToArtifact artifact that has all files needed' - - task: DownloadPipelineArtifact@2 - inputs: - buildType: 'current' - targetPath: '$(Pipeline.Workspace)' - displayName: 'Download to get EV2 Files' + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + targetPath: '$(Pipeline.Workspace)' + displayName: 'Download to get EV2 Files' - - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 - displayName: 'Ev2: Push to PMC' - inputs: - UseServerMonitorTask: true - EnableStrictValidation: false - ValidateOnly: false - ConnectedServiceName: 'docker-deployement-test' - ServiceRootLocation: 'LinkedArtifact' - RolloutSpecType: 'RSPath' - ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' - RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' + - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 + displayName: 'TEST Ev2: Push to ACR' + inputs: + UseServerMonitorTask: true + ConnectedServiceName: 'docker-deployement-test' + ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' + RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' + + # - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 + # displayName: 'Ev2: Push to PMC' + # inputs: + # UseServerMonitorTask: true + # EnableStrictValidation: false + # ValidateOnly: false + # ConnectedServiceName: 'docker-deployement-test' + # ServiceRootLocation: 'LinkedArtifact' + # RolloutSpecType: 'RSPath' + # ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' + # RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From 25b30a53b0d73f6441967807b6a9f86b40df6128 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 10 Jan 2025 15:22:23 -0500 Subject: [PATCH 223/316] update service connection name in EV2 test task --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 114 +++++++++++++++++- .pipelines/templates/release-publish-pmc.yml | 2 +- 2 files changed, 111 insertions(+), 5 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 13620d1db3d..defd3e36cae 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -4,6 +4,12 @@ if ($null -eq $env:MAPPING_FILE) return 1 } +if ($null -eq $env:REPO_LIST_FILE) +{ + Write-Verbose -Verbose "REPO_LIST_FILE variable didn't get passed correctly" + return 1 +} + if ($null -eq $env:PWSH_PACKAGES_TARGZIP) { Write-Verbose -Verbose "PWSH_PACKAGES_TARGZIP variable didn't get passed correctly" @@ -17,12 +23,9 @@ if ($null -eq $env:PMC_METADATA) } try { - Write-Verbose -Verbose "mapping.json file: $env:MAPPING_FILE" - Write-Verbose -Verbose "pwsh packages.tar.gz file: $env:PWSH_PACKAGES_TARGZIP" - Write-Verbose -Verbose "pmcMetadata.json file: $env:PMC_METADATA" - Write-Verbose -Verbose "Downloading files" Invoke-WebRequest -Uri $env:MAPPING_FILE -OutFile mapping.json + Invoke-WebRequest -Uri $env:REPO_LIST_FILE -OutFile repoList.json Invoke-WebRequest -Uri $env:PWSH_PACKAGES_TARGZIP -OutFile packages.tar.gz Invoke-WebRequest -Uri $env:PMC_METADATA -OutFile pmcMetadata.json @@ -43,6 +46,109 @@ try { which pmc pmc -d -c $settingsFile repo list --name "azurelinux-3.0-prod-ms-oss-x86_64-yum" || exit 1 + $metadataFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "pmcMetadata.json" + $metadataContent = Get-Content -Path $metadataFilePath | ConvertFrom-Json + $releaseVersion = $metadataContent.ReleaseTag + + + + $repoListFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "repoList.json" + $repoListPathExists = Test-Path $repoListFilePath + if (!$repoListPathExists) + { + Write-Verbose -Verbose "repoList.json expected at $repoListFilePath does not exist" + return 1 + } + + $repoList = Get-RepoList -RepoConfigPath $configPath -OutFile $repoListPath #TODO: seems to require repoclient cli tool + + # Get-PackageInfo() + $mappingFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "mapping.json" + $mappingPathExists = Test-Path $mappingFilePath + if (!$mappingPathExists) + { + Write-Verbose -Verbose "mapping.json expected at $mappingFilePath does not exist" + return 1 + } + + $mapping = Get-Content -Raw -LiteralPath $mappingFilePath | ConvertFrom-Json -AsHashtable + foreach ($package in $mapping.Packages) + { + Write-Verbose "package: $package" + $packageChannel = $package.channel + if (!$packageChannel) { + $packageChannel = 'all' + } + + Write-Verbose "package channel: $packageChannel" + if ($packageChannel -eq 'all' -or $packageChannel -eq $Channel) { + $repoIds = [System.Collections.Generic.List[string]]::new() + $packageFormat = $package.PackageFormat + Write-Verbose "package format: $packageFormat" -Verbose + $extension = [System.io.path]::GetExtension($packageFormat) + $packageType = $extension -replace '^\.' + + if ($package.distribution.count -gt 1) { + throw "Package $($package | out-string) has more than one Distribution." + } + + foreach ($distribution in $package.distribution) { + $urlGlob = $package.url + switch ($packageType) { + 'deb' { + $urlGlob = $urlGlob + '-apt' + } + 'rpm' { + $urlGlob = $urlGlob + '-yum' + } + default { + throw "Unknown package type: $packageType" + } + } + + Write-Verbose "---Finding repo id for: $urlGlob---" -Verbose + + $repos = $repoList | Where-Object { $_.name -eq $urlGlob } + + if ($repos.id) { + Write-Verbose "Found repo id: $($repos.id)" -Verbose + $repoIds.AddRange(([string[]]$repos.id)) + } + else { + Write-Failure "Could not find repo for $urlGlob" + } + } + + if ($repoIds.Count -le 0) { + Write-Verbose -Verbose "no repoIds found that match our packages" + # Write-Output ($package + @{ "RepoId" = $repoIds.ToArray() }) + } + + foreach ($pkg in $Package) { + if ($pkg.RepoId.count -gt 1) { + throw "Package $($pkg.name) has more than one repo id." + } + if ($pkg.Distribution.count -gt 1) { + throw "Package $($pkg.name) has more than one Distribution." + } + $repo = $pkg.RepoId | Select-Object -First 1 + $distribution = $pkg.Distribution | Select-Object -First 1 + foreach ($name in $PackageName) { + $pkgName = $pkg.PackageFormat.Replace('PACKAGE_NAME', $name).Replace('POWERSHELL_RELEASE', $ReleaseVersion) + Write-Verbose "Creating info object for package '$pkgName' for repo '$repo'" + $result = [pscustomobject]@{ + PackageName = $pkgName + RepoId = $repo + Distribution = $distribution + } + Write-Verbose $result -Verbose + Write-Output $result + } + } + } + } + + } catch { Write-Error -ErrorAction Stop $_.Exception.Message diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index b07de94997c..4ee755cac97 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -51,7 +51,7 @@ stages: displayName: 'TEST Ev2: Push to ACR' inputs: UseServerMonitorTask: true - ConnectedServiceName: 'docker-deployement-test' + ConnectedServiceName: 'pmc-ev2-deployment' ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From c341841639b47b70013da5baab3affa4d3225260 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 13 Jan 2025 19:51:29 -0500 Subject: [PATCH 224/316] see which versions are available --- .pipelines/templates/release-prep-for-ev2.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 882432292d5..000f03f1e09 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -57,6 +57,7 @@ stages: - pwsh: | New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory + Get-ChildItem '$(Agent.ToolsDirectory)' displayName: 'Create Parameters folder under EV2Specs folder' - task: UsePythonVersion@0 From a433c51a48bb0e12e814ac19d577cb8dbb74f762 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 13 Jan 2025 20:27:01 -0500 Subject: [PATCH 225/316] add -Recurse parameter --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 000f03f1e09..7055cae3aa5 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -57,7 +57,7 @@ stages: - pwsh: | New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory - Get-ChildItem '$(Agent.ToolsDirectory)' + Get-ChildItem '$(Agent.ToolsDirectory)' -Recurse displayName: 'Create Parameters folder under EV2Specs folder' - task: UsePythonVersion@0 From 91ee83da14cd41cc5e9eb0d9ae450dd97ee42ebd Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 13 Jan 2025 20:45:32 -0500 Subject: [PATCH 226/316] specify version as 3 --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 7055cae3aa5..dec13d185e0 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -63,7 +63,7 @@ stages: - task: UsePythonVersion@0 displayName: 'Use Python 3.12.7' inputs: - versionSpec: 3.12.7 + versionSpec: 3 - task: PipAuthenticate@1 inputs: From 0016435eb942df8831d1c8233d784e379457557d Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 13 Jan 2025 21:01:33 -0500 Subject: [PATCH 227/316] test expected path --- .pipelines/templates/release-prep-for-ev2.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index dec13d185e0..730404aa337 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -58,6 +58,8 @@ stages: - pwsh: | New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory Get-ChildItem '$(Agent.ToolsDirectory)' -Recurse + $expectedPythonPathExists = Test-Path "C:\__t\Python\" + Write-Verbose -Verbose "expected python path exists: $expectedPythonPathExists" displayName: 'Create Parameters folder under EV2Specs folder' - task: UsePythonVersion@0 From e9a53ad6fc6cb3a9075ea44b7421785a70edc68e Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Mon, 13 Jan 2025 21:58:33 -0500 Subject: [PATCH 228/316] comment out python package related code for now --- .pipelines/templates/release-prep-for-ev2.yml | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 730404aa337..cdfb8aa78b8 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -62,26 +62,26 @@ stages: Write-Verbose -Verbose "expected python path exists: $expectedPythonPathExists" displayName: 'Create Parameters folder under EV2Specs folder' - - task: UsePythonVersion@0 - displayName: 'Use Python 3.12.7' - inputs: - versionSpec: 3 - - - task: PipAuthenticate@1 - inputs: - artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' - displayName: 'Pip Authenticate' - - - pwsh: | - python -m pip install --upgrade pip - pip --version --verbose - - Write-Verbose -Verbose "Download pmc-cli to folder without installing it" - $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" - $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" - $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" - pip download -d $pythonDlFolderPath pmc-cli --verbose - displayName: 'Download pmc-cli package' + # - task: UsePythonVersion@0 + # displayName: 'Use Python 3.12.7' + # inputs: + # versionSpec: 3.12.7 + + # - task: PipAuthenticate@1 + # inputs: + # artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' + # displayName: 'Pip Authenticate' + + # - pwsh: | + # python -m pip install --upgrade pip + # pip --version --verbose + + # Write-Verbose -Verbose "Download pmc-cli to folder without installing it" + # $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" + # $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" + # $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" + # pip download -d $pythonDlFolderPath pmc-cli --verbose + # displayName: 'Download pmc-cli package' - download: PSPackagesOfficial artifact: 'drop_linux_package_deb' From 0391f4344f91ad4c0647f1bf2abc771d539821d4 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 14 Jan 2025 11:41:05 -0500 Subject: [PATCH 229/316] try recommended version --- .pipelines/templates/release-prep-for-ev2.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index cdfb8aa78b8..c22a730e89c 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -62,10 +62,10 @@ stages: Write-Verbose -Verbose "expected python path exists: $expectedPythonPathExists" displayName: 'Create Parameters folder under EV2Specs folder' - # - task: UsePythonVersion@0 - # displayName: 'Use Python 3.12.7' - # inputs: - # versionSpec: 3.12.7 + - task: UsePythonVersion@0 + displayName: 'Use Python 3.12.7' + inputs: + versionSpec: 3.9 # - task: PipAuthenticate@1 # inputs: From 4c607a5f69517737204ae4b67c68c6d40a4693db Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 14 Jan 2025 16:26:14 -0500 Subject: [PATCH 230/316] try checking onebranch container image path for python --- .pipelines/templates/release-prep-for-ev2.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index c22a730e89c..c84f19ba787 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -60,12 +60,14 @@ stages: Get-ChildItem '$(Agent.ToolsDirectory)' -Recurse $expectedPythonPathExists = Test-Path "C:\__t\Python\" Write-Verbose -Verbose "expected python path exists: $expectedPythonPathExists" + $onebranchPythonPathExists = Test-Path "C:\Program Files\Miniconda3\python.exe" + Write-Verbose -Verbose "expected OB python path exists" displayName: 'Create Parameters folder under EV2Specs folder' - task: UsePythonVersion@0 displayName: 'Use Python 3.12.7' inputs: - versionSpec: 3.9 + versionSpec: 3.10.14 # - task: PipAuthenticate@1 # inputs: From fa5d858a885bf6a598fa63ec16e9063277850eea Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 14 Jan 2025 16:36:34 -0500 Subject: [PATCH 231/316] fix logging statement --- .pipelines/templates/release-prep-for-ev2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index c84f19ba787..8bf8646aad9 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -57,11 +57,11 @@ stages: - pwsh: | New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory - Get-ChildItem '$(Agent.ToolsDirectory)' -Recurse + # Get-ChildItem '$(Agent.ToolsDirectory)' -Recurse $expectedPythonPathExists = Test-Path "C:\__t\Python\" Write-Verbose -Verbose "expected python path exists: $expectedPythonPathExists" $onebranchPythonPathExists = Test-Path "C:\Program Files\Miniconda3\python.exe" - Write-Verbose -Verbose "expected OB python path exists" + Write-Verbose -Verbose "expected OB python path exists: $onebranchPythonPathExists" displayName: 'Create Parameters folder under EV2Specs folder' - task: UsePythonVersion@0 From 9c92480b6e90bdb98d6cab2d47e92c9a2b52f9ee Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 14 Jan 2025 16:50:53 -0500 Subject: [PATCH 232/316] add back rest of python related tasks to see if OB installed python will work --- .pipelines/templates/release-prep-for-ev2.yml | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 8bf8646aad9..c17c5769996 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -64,26 +64,26 @@ stages: Write-Verbose -Verbose "expected OB python path exists: $onebranchPythonPathExists" displayName: 'Create Parameters folder under EV2Specs folder' - - task: UsePythonVersion@0 - displayName: 'Use Python 3.12.7' + # - task: UsePythonVersion@0 + # displayName: 'Use Python 3.12.7' + # inputs: + # versionSpec: 3.10.14 + + - task: PipAuthenticate@1 inputs: - versionSpec: 3.10.14 + artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' + displayName: 'Pip Authenticate' - # - task: PipAuthenticate@1 - # inputs: - # artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' - # displayName: 'Pip Authenticate' - - # - pwsh: | - # python -m pip install --upgrade pip - # pip --version --verbose - - # Write-Verbose -Verbose "Download pmc-cli to folder without installing it" - # $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" - # $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" - # $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" - # pip download -d $pythonDlFolderPath pmc-cli --verbose - # displayName: 'Download pmc-cli package' + - pwsh: | + python -m pip install --upgrade pip + pip --version --verbose + + Write-Verbose -Verbose "Download pmc-cli to folder without installing it" + $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" + $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" + $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" + pip download -d $pythonDlFolderPath pmc-cli --verbose + displayName: 'Download pmc-cli package' - download: PSPackagesOfficial artifact: 'drop_linux_package_deb' From e31e430175c8e417e2baf904ac1d7c9ed740dcee Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 15 Jan 2025 15:31:21 -0500 Subject: [PATCH 233/316] get repolist.json --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 190 +++++++++--------- .../ServiceGroupRoot/UploadLinux.Rollout.json | 7 + .pipelines/templates/release-prep-for-ev2.yml | 15 +- 3 files changed, 107 insertions(+), 105 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index defd3e36cae..c0a9d7d3c83 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -52,101 +52,101 @@ try { - $repoListFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "repoList.json" - $repoListPathExists = Test-Path $repoListFilePath - if (!$repoListPathExists) - { - Write-Verbose -Verbose "repoList.json expected at $repoListFilePath does not exist" - return 1 - } - - $repoList = Get-RepoList -RepoConfigPath $configPath -OutFile $repoListPath #TODO: seems to require repoclient cli tool - - # Get-PackageInfo() - $mappingFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "mapping.json" - $mappingPathExists = Test-Path $mappingFilePath - if (!$mappingPathExists) - { - Write-Verbose -Verbose "mapping.json expected at $mappingFilePath does not exist" - return 1 - } - - $mapping = Get-Content -Raw -LiteralPath $mappingFilePath | ConvertFrom-Json -AsHashtable - foreach ($package in $mapping.Packages) - { - Write-Verbose "package: $package" - $packageChannel = $package.channel - if (!$packageChannel) { - $packageChannel = 'all' - } - - Write-Verbose "package channel: $packageChannel" - if ($packageChannel -eq 'all' -or $packageChannel -eq $Channel) { - $repoIds = [System.Collections.Generic.List[string]]::new() - $packageFormat = $package.PackageFormat - Write-Verbose "package format: $packageFormat" -Verbose - $extension = [System.io.path]::GetExtension($packageFormat) - $packageType = $extension -replace '^\.' - - if ($package.distribution.count -gt 1) { - throw "Package $($package | out-string) has more than one Distribution." - } - - foreach ($distribution in $package.distribution) { - $urlGlob = $package.url - switch ($packageType) { - 'deb' { - $urlGlob = $urlGlob + '-apt' - } - 'rpm' { - $urlGlob = $urlGlob + '-yum' - } - default { - throw "Unknown package type: $packageType" - } - } - - Write-Verbose "---Finding repo id for: $urlGlob---" -Verbose - - $repos = $repoList | Where-Object { $_.name -eq $urlGlob } - - if ($repos.id) { - Write-Verbose "Found repo id: $($repos.id)" -Verbose - $repoIds.AddRange(([string[]]$repos.id)) - } - else { - Write-Failure "Could not find repo for $urlGlob" - } - } - - if ($repoIds.Count -le 0) { - Write-Verbose -Verbose "no repoIds found that match our packages" - # Write-Output ($package + @{ "RepoId" = $repoIds.ToArray() }) - } - - foreach ($pkg in $Package) { - if ($pkg.RepoId.count -gt 1) { - throw "Package $($pkg.name) has more than one repo id." - } - if ($pkg.Distribution.count -gt 1) { - throw "Package $($pkg.name) has more than one Distribution." - } - $repo = $pkg.RepoId | Select-Object -First 1 - $distribution = $pkg.Distribution | Select-Object -First 1 - foreach ($name in $PackageName) { - $pkgName = $pkg.PackageFormat.Replace('PACKAGE_NAME', $name).Replace('POWERSHELL_RELEASE', $ReleaseVersion) - Write-Verbose "Creating info object for package '$pkgName' for repo '$repo'" - $result = [pscustomobject]@{ - PackageName = $pkgName - RepoId = $repo - Distribution = $distribution - } - Write-Verbose $result -Verbose - Write-Output $result - } - } - } - } + # $repoListFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "repoList.json" + # $repoListPathExists = Test-Path $repoListFilePath + # if (!$repoListPathExists) + # { + # Write-Verbose -Verbose "repoList.json expected at $repoListFilePath does not exist" + # return 1 + # } + + # $repoList = Get-RepoList -RepoConfigPath $configPath -OutFile $repoListPath #TODO: seems to require repoclient cli tool + + # # Get-PackageInfo() + # $mappingFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "mapping.json" + # $mappingPathExists = Test-Path $mappingFilePath + # if (!$mappingPathExists) + # { + # Write-Verbose -Verbose "mapping.json expected at $mappingFilePath does not exist" + # return 1 + # } + + # $mapping = Get-Content -Raw -LiteralPath $mappingFilePath | ConvertFrom-Json -AsHashtable + # foreach ($package in $mapping.Packages) + # { + # Write-Verbose "package: $package" + # $packageChannel = $package.channel + # if (!$packageChannel) { + # $packageChannel = 'all' + # } + + # Write-Verbose "package channel: $packageChannel" + # if ($packageChannel -eq 'all' -or $packageChannel -eq $Channel) { + # $repoIds = [System.Collections.Generic.List[string]]::new() + # $packageFormat = $package.PackageFormat + # Write-Verbose "package format: $packageFormat" -Verbose + # $extension = [System.io.path]::GetExtension($packageFormat) + # $packageType = $extension -replace '^\.' + + # if ($package.distribution.count -gt 1) { + # throw "Package $($package | out-string) has more than one Distribution." + # } + + # foreach ($distribution in $package.distribution) { + # $urlGlob = $package.url + # switch ($packageType) { + # 'deb' { + # $urlGlob = $urlGlob + '-apt' + # } + # 'rpm' { + # $urlGlob = $urlGlob + '-yum' + # } + # default { + # throw "Unknown package type: $packageType" + # } + # } + + # Write-Verbose "---Finding repo id for: $urlGlob---" -Verbose + + # $repos = $repoList | Where-Object { $_.name -eq $urlGlob } + + # if ($repos.id) { + # Write-Verbose "Found repo id: $($repos.id)" -Verbose + # $repoIds.AddRange(([string[]]$repos.id)) + # } + # else { + # Write-Failure "Could not find repo for $urlGlob" + # } + # } + + # if ($repoIds.Count -le 0) { + # Write-Verbose -Verbose "no repoIds found that match our packages" + # # Write-Output ($package + @{ "RepoId" = $repoIds.ToArray() }) + # } + + # foreach ($pkg in $Package) { + # if ($pkg.RepoId.count -gt 1) { + # throw "Package $($pkg.name) has more than one repo id." + # } + # if ($pkg.Distribution.count -gt 1) { + # throw "Package $($pkg.name) has more than one Distribution." + # } + # $repo = $pkg.RepoId | Select-Object -First 1 + # $distribution = $pkg.Distribution | Select-Object -First 1 + # foreach ($name in $PackageName) { + # $pkgName = $pkg.PackageFormat.Replace('PACKAGE_NAME', $name).Replace('POWERSHELL_RELEASE', $ReleaseVersion) + # Write-Verbose "Creating info object for package '$pkgName' for repo '$repo'" + # $result = [pscustomobject]@{ + # PackageName = $pkgName + # RepoId = $repo + # Distribution = $distribution + # } + # Write-Verbose $result -Verbose + # Write-Output $result + # } + # } + # } + # } } diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json b/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json index d7c75c2e216..bc97d04a33e 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json +++ b/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json @@ -27,6 +27,13 @@ "path": "Parameters\\mapping.json" } }, + { + "name": "REPO_LIST_FILE", + "reference": + { + "path": "Parameters\\repolist.json" + } + }, { "name": "PWSH_PACKAGES_TARGZIP", "reference": diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index c17c5769996..c6cef86ae9c 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -57,18 +57,8 @@ stages: - pwsh: | New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory - # Get-ChildItem '$(Agent.ToolsDirectory)' -Recurse - $expectedPythonPathExists = Test-Path "C:\__t\Python\" - Write-Verbose -Verbose "expected python path exists: $expectedPythonPathExists" - $onebranchPythonPathExists = Test-Path "C:\Program Files\Miniconda3\python.exe" - Write-Verbose -Verbose "expected OB python path exists: $onebranchPythonPathExists" displayName: 'Create Parameters folder under EV2Specs folder' - # - task: UsePythonVersion@0 - # displayName: 'Use Python 3.12.7' - # inputs: - # versionSpec: 3.10.14 - - task: PipAuthenticate@1 inputs: artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' @@ -85,6 +75,11 @@ stages: pip download -d $pythonDlFolderPath pmc-cli --verbose displayName: 'Download pmc-cli package' + - pwsh: | + $repolistFile = Join-Path "$(Build.SourcesDirectory)/Internal-PowerShellTeam-Tools/packages.microsoft.com-v4" -ChildPath "repolist.json" + Copy-Item -Path $repolistFile -Destination '$(ev2ParametersFolder)' -Verbose + displayName: 'Copy repolist.json file from Internal-PowerShellTeam-Tools to EV2 folder' + - download: PSPackagesOfficial artifact: 'drop_linux_package_deb' displayName: 'Download artifact containing .deb_amd64.deb file from PSPackagesOfficial triggering pipeline' From 07a0d009f55bc600ec533ba8090ec22ba4a8663f Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 15 Jan 2025 15:44:20 -0500 Subject: [PATCH 234/316] debug why file doesn't exist --- .pipelines/templates/release-prep-for-ev2.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index c6cef86ae9c..7f52e1e047b 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -76,6 +76,7 @@ stages: displayName: 'Download pmc-cli package' - pwsh: | + Get-ChildItem '$(Build.SourcesDirectory)' $repolistFile = Join-Path "$(Build.SourcesDirectory)/Internal-PowerShellTeam-Tools/packages.microsoft.com-v4" -ChildPath "repolist.json" Copy-Item -Path $repolistFile -Destination '$(ev2ParametersFolder)' -Verbose displayName: 'Copy repolist.json file from Internal-PowerShellTeam-Tools to EV2 folder' From 8d9fb65a98da1666961fafe1b06424bbeb547ecf Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 15 Jan 2025 16:10:31 -0500 Subject: [PATCH 235/316] fix path --- .pipelines/templates/release-prep-for-ev2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 7f52e1e047b..136fde74ad5 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -76,8 +76,8 @@ stages: displayName: 'Download pmc-cli package' - pwsh: | - Get-ChildItem '$(Build.SourcesDirectory)' - $repolistFile = Join-Path "$(Build.SourcesDirectory)/Internal-PowerShellTeam-Tools/packages.microsoft.com-v4" -ChildPath "repolist.json" + Get-ChildItem '$(Pipeline.Workspace)/tools' + $repolistFile = Join-Path "$(Pipeline.Workspace)/tools/packages.microsoft.com-v4" -ChildPath "repolist.json" Copy-Item -Path $repolistFile -Destination '$(ev2ParametersFolder)' -Verbose displayName: 'Copy repolist.json file from Internal-PowerShellTeam-Tools to EV2 folder' From cd352bca9620de1f7666324376a46e59aa0b0989 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 00:49:23 -0500 Subject: [PATCH 236/316] fix path in Run.ps1 --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index c0a9d7d3c83..e79b43e3aa4 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -29,17 +29,17 @@ try { Invoke-WebRequest -Uri $env:PWSH_PACKAGES_TARGZIP -OutFile packages.tar.gz Invoke-WebRequest -Uri $env:PMC_METADATA -OutFile pmcMetadata.json - $settingsFile = Join-Path '/package/unarchive' -ChildPath 'settings.toml' + $settingsFile = Join-Path '/package/unarchive/Run' -ChildPath 'settings.toml' $settingsFileExists = Test-Path -Path $settingsFile Write-Verbose -Verbose "settings.toml exists: $settingsFileExists" - $pythonDlFolder = Join-Path '/package/unarchive' -ChildPath 'python_dl' + $pythonDlFolder = Join-Path '/package/unarchive/Run' -ChildPath 'python_dl' $pyPathExists = Test-Path -Path $pythonDlFolder Write-Verbose -Verbose "python_dl folder path exists: $pyPathExists" Write-Verbose -Verbose "Installing pmc-cli" python -m pip install --upgrade pip pip --version --verbose - pip install /package/unarchive/python_dl/*.whl + pip install /package/unarchive/Run/python_dl/*.whl Write-Verbose -Verbose "Test pmc-cli" From 83145e0a477e1ff8e660d54a88e6d41162eecf02 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 10:00:00 -0500 Subject: [PATCH 237/316] update container version --- .pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json | 2 +- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json b/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json index 515a7fb34ab..2f14ba34bea 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json +++ b/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json @@ -18,7 +18,7 @@ "type": "Run", "properties": { "imageName": "adm-mariner-20-l", - "imageVersion": "v5" + "imageVersion": "v11" } } ] diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index e79b43e3aa4..e4d1913e9e8 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -36,6 +36,8 @@ try { $pyPathExists = Test-Path -Path $pythonDlFolder Write-Verbose -Verbose "python_dl folder path exists: $pyPathExists" + Write-Verbose -Verbose "doing gci" + Get-ChildItem "/package" -Recurse Write-Verbose -Verbose "Installing pmc-cli" python -m pip install --upgrade pip pip --version --verbose From a959922669a57e865d0af34ab4249d832cbaff1e Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 10:30:18 -0500 Subject: [PATCH 238/316] invoke crane tool which should come installed like python --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index e4d1913e9e8..b39f6b1daba 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -39,6 +39,7 @@ try { Write-Verbose -Verbose "doing gci" Get-ChildItem "/package" -Recurse Write-Verbose -Verbose "Installing pmc-cli" + crane version python -m pip install --upgrade pip pip --version --verbose pip install /package/unarchive/Run/python_dl/*.whl From 93aaca8eeee781e143c70bd63c1cc5d21dc66254 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 11:18:37 -0500 Subject: [PATCH 239/316] refer to python exe --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index b39f6b1daba..4a60a1fc549 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -40,7 +40,7 @@ try { Get-ChildItem "/package" -Recurse Write-Verbose -Verbose "Installing pmc-cli" crane version - python -m pip install --upgrade pip + python.exe -m pip install --upgrade pip pip --version --verbose pip install /package/unarchive/Run/python_dl/*.whl From aa5137644340c916476ce8192f7abf6ead68d419 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 11:38:55 -0500 Subject: [PATCH 240/316] just use pip --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 4a60a1fc549..5dc649cb7cf 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -40,7 +40,7 @@ try { Get-ChildItem "/package" -Recurse Write-Verbose -Verbose "Installing pmc-cli" crane version - python.exe -m pip install --upgrade pip + # python.exe -m pip install --upgrade pip pip --version --verbose pip install /package/unarchive/Run/python_dl/*.whl From c56dae1c1d6c18e4943336734e31714c7f474fef Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 12:07:54 -0500 Subject: [PATCH 241/316] upgrade pip --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 5dc649cb7cf..3159596dd5b 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -41,6 +41,7 @@ try { Write-Verbose -Verbose "Installing pmc-cli" crane version # python.exe -m pip install --upgrade pip + pip install --upgrade pip pip --version --verbose pip install /package/unarchive/Run/python_dl/*.whl From 62c972f0eabf04918672196554803e5a0d8fb5ad Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 13:13:46 -0500 Subject: [PATCH 242/316] use linux pool image when installing pip package --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 136fde74ad5..c79fe1630b0 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -5,7 +5,7 @@ stages: - job: CopyEV2FilesToArtifact displayName: 'Copy EV2 Files to Artifact' pool: - type: windows + type: linux variables: - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' From 2f69294c729858f10d6390bdd4a7a8fe70fa3410 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 13:23:53 -0500 Subject: [PATCH 243/316] use python3 on linux --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index c79fe1630b0..31ed40c85c5 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -65,7 +65,7 @@ stages: displayName: 'Pip Authenticate' - pwsh: | - python -m pip install --upgrade pip + python3 -m pip install --upgrade pip pip --version --verbose Write-Verbose -Verbose "Download pmc-cli to folder without installing it" From 7bd46197ed5ec5a4ec8714a18590d927cdd391f7 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 13:49:50 -0500 Subject: [PATCH 244/316] revert use of python 3 --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 31ed40c85c5..c79fe1630b0 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -65,7 +65,7 @@ stages: displayName: 'Pip Authenticate' - pwsh: | - python3 -m pip install --upgrade pip + python -m pip install --upgrade pip pip --version --verbose Write-Verbose -Verbose "Download pmc-cli to folder without installing it" From 0a06213574ad1b5de3e771a86555f1cdcd1b5abd Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 13:51:11 -0500 Subject: [PATCH 245/316] comment out python code for now --- .pipelines/templates/release-prep-for-ev2.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index c79fe1630b0..e0f0ef7a892 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -64,16 +64,16 @@ stages: artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' displayName: 'Pip Authenticate' - - pwsh: | - python -m pip install --upgrade pip - pip --version --verbose - - Write-Verbose -Verbose "Download pmc-cli to folder without installing it" - $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" - $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" - $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" - pip download -d $pythonDlFolderPath pmc-cli --verbose - displayName: 'Download pmc-cli package' + # - pwsh: | + # python -m pip install --upgrade pip + # pip --version --verbose + + # Write-Verbose -Verbose "Download pmc-cli to folder without installing it" + # $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" + # $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" + # $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" + # pip download -d $pythonDlFolderPath pmc-cli --verbose + # displayName: 'Download pmc-cli package' - pwsh: | Get-ChildItem '$(Pipeline.Workspace)/tools' From a2296cbac230a8030f4288903e65460fdfffe0b1 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 14:02:51 -0500 Subject: [PATCH 246/316] mitigate file watcher issue --- .pipelines/templates/release-prep-for-ev2.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index e0f0ef7a892..e348add7691 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -45,6 +45,8 @@ stages: $gitArgs | Write-Verbose -Verbose git $gitArgs displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub + env: + ob_restore_phase: true # Set ob_restore_phase to true to mitigate file watcher issue: https://eng.ms/docs/products/onebranch/signing/troubleshooting/filewatcher - pwsh: | Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose From 94d9718b6ae035ef6b3554df339d83dd85b742b6 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 14:13:05 -0500 Subject: [PATCH 247/316] set linuxEsrpSigning: true --- .pipelines/PowerShell-Release-Official-Azure.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 72220b98be7..88fa3b97ac8 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -77,6 +77,7 @@ extends: WindowsHostVersion: Version: 2022 Network: Netlock + linuxEsrpSigning: true cloudvault: enabled: false globalSdl: From 3a9249514a97df27ccf15fa5da2a675fded855c2 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 14:27:14 -0500 Subject: [PATCH 248/316] try moving set up signing task to right before signing --- .pipelines/templates/release-prep-for-ev2.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index e348add7691..3b94b30f32d 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -150,6 +150,8 @@ stages: Add-Content -Path $settingFilePath -Value "auth_type = `"msi`"" Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" displayName: 'Create settings.toml file with MI clientId populated' + env: + ob_restore_phase: true # Set ob_restore_phase to true to mitigate file watcher issue: https://eng.ms/docs/products/onebranch/signing/troubleshooting/filewatcher - task: onebranch.pipeline.signing@1 inputs: From d910d9d826d4592b16c4966d082f258122d05606 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 14:35:05 -0500 Subject: [PATCH 249/316] remove other ob_restore_phase --- .pipelines/templates/release-prep-for-ev2.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 3b94b30f32d..21652425e26 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -45,8 +45,6 @@ stages: $gitArgs | Write-Verbose -Verbose git $gitArgs displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub - env: - ob_restore_phase: true # Set ob_restore_phase to true to mitigate file watcher issue: https://eng.ms/docs/products/onebranch/signing/troubleshooting/filewatcher - pwsh: | Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose From 0c6750af94cbab3d0a4880b1df3cd9d58b8a091a Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 14:45:07 -0500 Subject: [PATCH 250/316] keep using windows image, but specify platform for pip download --- .../PowerShell-Release-Official-Azure.yml | 1 - .pipelines/templates/release-prep-for-ev2.yml | 22 +++++++++---------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 88fa3b97ac8..72220b98be7 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -77,7 +77,6 @@ extends: WindowsHostVersion: Version: 2022 Network: Netlock - linuxEsrpSigning: true cloudvault: enabled: false globalSdl: diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 21652425e26..cef52b64c24 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -5,7 +5,7 @@ stages: - job: CopyEV2FilesToArtifact displayName: 'Copy EV2 Files to Artifact' pool: - type: linux + type: windows variables: - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' @@ -64,16 +64,16 @@ stages: artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' displayName: 'Pip Authenticate' - # - pwsh: | - # python -m pip install --upgrade pip - # pip --version --verbose + - pwsh: | + python -m pip install --upgrade pip + pip --version --verbose - # Write-Verbose -Verbose "Download pmc-cli to folder without installing it" - # $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" - # $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" - # $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" - # pip download -d $pythonDlFolderPath pmc-cli --verbose - # displayName: 'Download pmc-cli package' + Write-Verbose -Verbose "Download pmc-cli to folder without installing it" + $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" + $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" + $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" + pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux1_x86_64 --verbose + displayName: 'Download pmc-cli package' - pwsh: | Get-ChildItem '$(Pipeline.Workspace)/tools' @@ -148,8 +148,6 @@ stages: Add-Content -Path $settingFilePath -Value "auth_type = `"msi`"" Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" displayName: 'Create settings.toml file with MI clientId populated' - env: - ob_restore_phase: true # Set ob_restore_phase to true to mitigate file watcher issue: https://eng.ms/docs/products/onebranch/signing/troubleshooting/filewatcher - task: onebranch.pipeline.signing@1 inputs: From 7a2f213e7484922df6fd64e4b0b65b13f98d775c Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 14:56:35 -0500 Subject: [PATCH 251/316] add binaries only flag needed by pip --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index cef52b64c24..be60d87373c 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -72,7 +72,7 @@ stages: $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" - pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux1_x86_64 --verbose + pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux1_x86_64 --only-binary=:all: --verbose displayName: 'Download pmc-cli package' - pwsh: | From f21602e9a1cb4e684c7de7de93bfe2c0fcfdf225 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 15:13:38 -0500 Subject: [PATCH 252/316] try latest manylinux platform --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index be60d87373c..7a26192a467 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -72,7 +72,7 @@ stages: $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" - pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux1_x86_64 --only-binary=:all: --verbose + pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux_2_28_x86_64 --only-binary=:all: --verbose displayName: 'Download pmc-cli package' - pwsh: | From be92d298f54f6ceb0ef6ee5ce92cf50494350698 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 15:24:59 -0500 Subject: [PATCH 253/316] use linux pool image for job --- .pipelines/templates/release-prep-for-ev2.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 7a26192a467..adf0fd52b48 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -5,7 +5,7 @@ stages: - job: CopyEV2FilesToArtifact displayName: 'Copy EV2 Files to Artifact' pool: - type: windows + type: linux variables: - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' @@ -72,7 +72,7 @@ stages: $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" - pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux_2_28_x86_64 --only-binary=:all: --verbose + pip download -d $pythonDlFolderPath pmc-cli --verbose displayName: 'Download pmc-cli package' - pwsh: | @@ -149,13 +149,13 @@ stages: Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" displayName: 'Create settings.toml file with MI clientId populated' - - task: onebranch.pipeline.signing@1 - inputs: - command: 'sign' - signing_profile: 'tipsign' # external_distribution - files_to_sign: '*.ps1' - search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' - displayName: Sign Run.ps1 + # - task: onebranch.pipeline.signing@1 + # inputs: + # command: 'sign' + # signing_profile: 'tipsign' # external_distribution + # files_to_sign: '*.ps1' + # search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' + # displayName: Sign Run.ps1 - pwsh: | $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' From efbc85c08333d28a102a8a18e2dbb3dfc1aa2dac Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 15:32:43 -0500 Subject: [PATCH 254/316] use manylinux platform from run that only failed at signing --- .pipelines/templates/release-prep-for-ev2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index adf0fd52b48..70aefd4f67a 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -5,7 +5,7 @@ stages: - job: CopyEV2FilesToArtifact displayName: 'Copy EV2 Files to Artifact' pool: - type: linux + type: windows variables: - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' @@ -72,7 +72,7 @@ stages: $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" - pip download -d $pythonDlFolderPath pmc-cli --verbose + pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux_2_17_x86_64 --only-binary=:all: --verbose displayName: 'Download pmc-cli package' - pwsh: | From 02ab150de7572088e25554eaaee6b1aadb238575 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 16:05:24 -0500 Subject: [PATCH 255/316] use linux pool image for job again with ob_restore_phase --- .pipelines/templates/release-prep-for-ev2.yml | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 70aefd4f67a..1c88c255e74 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -5,7 +5,7 @@ stages: - job: CopyEV2FilesToArtifact displayName: 'Copy EV2 Files to Artifact' pool: - type: windows + type: linux variables: - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' @@ -24,6 +24,8 @@ stages: steps: - checkout: self ## the global setting on lfs didn't work lfs: false + env: + ob_restore_phase: true - template: release-SetReleaseTagandContainerName.yml @@ -45,6 +47,8 @@ stages: $gitArgs | Write-Verbose -Verbose git $gitArgs displayName: Clone Internal-PowerShellTeam-Tools from MSCodeHub + env: + ob_restore_phase: true - pwsh: | Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose @@ -148,14 +152,16 @@ stages: Add-Content -Path $settingFilePath -Value "auth_type = `"msi`"" Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" displayName: 'Create settings.toml file with MI clientId populated' + env: + ob_restore_phase: true - # - task: onebranch.pipeline.signing@1 - # inputs: - # command: 'sign' - # signing_profile: 'tipsign' # external_distribution - # files_to_sign: '*.ps1' - # search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' - # displayName: Sign Run.ps1 + - task: onebranch.pipeline.signing@1 + inputs: + command: 'sign' + signing_profile: 'tipsign' # external_distribution + files_to_sign: '*.ps1' + search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' + displayName: Sign Run.ps1 - pwsh: | $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' From a1fa02d3401fd2727f4dc4caca66e4ec9033b05f Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 16:14:00 -0500 Subject: [PATCH 256/316] use python3 on linux --- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 1c88c255e74..496e475ffab 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -69,7 +69,7 @@ stages: displayName: 'Pip Authenticate' - pwsh: | - python -m pip install --upgrade pip + python3 -m pip install --upgrade pip pip --version --verbose Write-Verbose -Verbose "Download pmc-cli to folder without installing it" From 169258b7e55b597e04934356a22fef35c71467f9 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 16:26:07 -0500 Subject: [PATCH 257/316] add ob_restore_phase: true to more tasks --- .pipelines/templates/release-prep-for-ev2.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 496e475ffab..a38d8b0685e 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -177,6 +177,8 @@ stages: Remove-Item -Path $pathToJsonFile $content | ConvertTo-Json -Depth 4 | Out-File $pathToJsonFile displayName: 'Replace values in RolloutSpecPath.json' + env: + ob_restore_phase: true - pwsh: | $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'UploadLinux.Rollout.json' @@ -188,6 +190,8 @@ stages: Remove-Item -Path $pathToJsonFile $content | ConvertTo-Json -Depth 6 | Out-File $pathToJsonFile displayName: 'Replace values in UploadLinux.Rollout.json file' + env: + ob_restore_phase: true - pwsh: | $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'ServiceModel.json' @@ -198,6 +202,8 @@ stages: Remove-Item -Path $pathToJsonFile $content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile displayName: 'Replace values in ServiceModel.json' + env: + ob_restore_phase: true - task: CopyFiles@2 inputs: From 8c21397457d1810f5d0eeb35a77ababea1cafb71 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 17:05:42 -0500 Subject: [PATCH 258/316] move the file tasks up --- .pipelines/templates/release-prep-for-ev2.yml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index a38d8b0685e..e381dff0890 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -155,21 +155,6 @@ stages: env: ob_restore_phase: true - - task: onebranch.pipeline.signing@1 - inputs: - command: 'sign' - signing_profile: 'tipsign' # external_distribution - files_to_sign: '*.ps1' - search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' - displayName: Sign Run.ps1 - - - pwsh: | - $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' - $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" - tar -cvf $pathToRunTarFile -C $srcPath ./Run - # TODO: ensure this has: Run.ps1, settings.toml, python_dl - displayName: 'Create archive for the shell extension' - - pwsh: | $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json' $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json @@ -205,6 +190,21 @@ stages: env: ob_restore_phase: true + - task: onebranch.pipeline.signing@1 + inputs: + command: 'sign' + signing_profile: 'tipsign' # external_distribution + files_to_sign: '*.ps1' + search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' + displayName: Sign Run.ps1 + + - pwsh: | + $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' + $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" + tar -cvf $pathToRunTarFile -C $srcPath ./Run + # TODO: ensure this has: Run.ps1, settings.toml, python_dl + displayName: 'Create archive for the shell extension' + - task: CopyFiles@2 inputs: SourceFolder: '$(repoRoot)/.pipelines' From 76ad9d30fbb254e1069cb14820ab08c9558a6127 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 17:14:01 -0500 Subject: [PATCH 259/316] move the file tasks up again --- .pipelines/templates/release-prep-for-ev2.yml | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index e381dff0890..a3e32af0772 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -142,19 +142,6 @@ stages: Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path displayName: 'Create pmcScriptMetadata.json and mapping.json file' - - pwsh: | - $settingFilePath = Join-Path '$(ev2ServiceGroupRootFolder)/Shell/Run' -ChildPath 'settings.toml' - New-Item -Path $settingFilePath -ItemType File - $pmcMIClientID = '$(PmcMIClientID)' - - Add-Content -Path $settingFilePath -Value "[default]" - Add-Content -Path $settingFilePath -Value "base_url = `"https://pmc-ingest.trafficmanager.net/api/v4`"" - Add-Content -Path $settingFilePath -Value "auth_type = `"msi`"" - Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" - displayName: 'Create settings.toml file with MI clientId populated' - env: - ob_restore_phase: true - - pwsh: | $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json' $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json @@ -162,8 +149,6 @@ stages: Remove-Item -Path $pathToJsonFile $content | ConvertTo-Json -Depth 4 | Out-File $pathToJsonFile displayName: 'Replace values in RolloutSpecPath.json' - env: - ob_restore_phase: true - pwsh: | $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'UploadLinux.Rollout.json' @@ -175,8 +160,6 @@ stages: Remove-Item -Path $pathToJsonFile $content | ConvertTo-Json -Depth 6 | Out-File $pathToJsonFile displayName: 'Replace values in UploadLinux.Rollout.json file' - env: - ob_restore_phase: true - pwsh: | $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'ServiceModel.json' @@ -187,6 +170,17 @@ stages: Remove-Item -Path $pathToJsonFile $content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile displayName: 'Replace values in ServiceModel.json' + + - pwsh: | + $settingFilePath = Join-Path '$(ev2ServiceGroupRootFolder)/Shell/Run' -ChildPath 'settings.toml' + New-Item -Path $settingFilePath -ItemType File + $pmcMIClientID = '$(PmcMIClientID)' + + Add-Content -Path $settingFilePath -Value "[default]" + Add-Content -Path $settingFilePath -Value "base_url = `"https://pmc-ingest.trafficmanager.net/api/v4`"" + Add-Content -Path $settingFilePath -Value "auth_type = `"msi`"" + Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" + displayName: 'Create settings.toml file with MI clientId populated' env: ob_restore_phase: true From deda847e98609b033a47f04b8b382a76a0504404 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 17:42:02 -0500 Subject: [PATCH 260/316] add ob_restore_phase: true to all tasks --- .pipelines/templates/release-prep-for-ev2.yml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index a3e32af0772..2a383972370 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -53,20 +53,28 @@ stages: - pwsh: | Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' + env: + ob_restore_phase: true - pwsh: | Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose Get-ChildItem '$(Build.SourcesDirectory)' displayName: 'Capture Workspace' + env: + ob_restore_phase: true - pwsh: | New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory displayName: 'Create Parameters folder under EV2Specs folder' + env: + ob_restore_phase: true - task: PipAuthenticate@1 inputs: artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' displayName: 'Pip Authenticate' + env: + ob_restore_phase: true - pwsh: | python3 -m pip install --upgrade pip @@ -78,28 +86,40 @@ stages: $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux_2_17_x86_64 --only-binary=:all: --verbose displayName: 'Download pmc-cli package' + env: + ob_restore_phase: true - pwsh: | Get-ChildItem '$(Pipeline.Workspace)/tools' $repolistFile = Join-Path "$(Pipeline.Workspace)/tools/packages.microsoft.com-v4" -ChildPath "repolist.json" Copy-Item -Path $repolistFile -Destination '$(ev2ParametersFolder)' -Verbose displayName: 'Copy repolist.json file from Internal-PowerShellTeam-Tools to EV2 folder' + env: + ob_restore_phase: true - download: PSPackagesOfficial artifact: 'drop_linux_package_deb' displayName: 'Download artifact containing .deb_amd64.deb file from PSPackagesOfficial triggering pipeline' + env: + ob_restore_phase: true - download: PSPackagesOfficial artifact: 'drop_linux_package_rpm' displayName: 'Download artifact containing .rh.x64_86.rpm file from PSPackagesOfficial triggering pipeline' + env: + ob_restore_phase: true - download: PSPackagesOfficial artifact: 'drop_linux_package_mariner_x64' displayName: 'Download artifact containing .cm.x86_64.rpm file from PSPackagesOfficial triggering pipeline' + env: + ob_restore_phase: true - download: PSPackagesOfficial artifact: 'drop_linux_package_mariner_arm64' displayName: 'Download artifact containing .cm.aarch64.rpm file from PSPackagesOfficial triggering pipeline' + env: + ob_restore_phase: true - pwsh: | Write-Verbose -Verbose "Copy ESRP signed .deb and .rpm packages" @@ -119,6 +139,8 @@ stages: $packagesTarGzDestination = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'packages.tar.gz' tar -czvf $packagesTarGzDestination -C $packagesFolder . displayName: 'Copy signed .deb and .rpm packages to .tar.gz to pass as a file var to shell extension' + env: + ob_restore_phase: true - pwsh: | $pathToPMCMetadataFile = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'pmcMetadata.json' @@ -141,6 +163,8 @@ stages: $mappingFileEV2Path = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath "mapping.json" Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path displayName: 'Create pmcScriptMetadata.json and mapping.json file' + env: + ob_restore_phase: true - pwsh: | $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json' @@ -149,6 +173,8 @@ stages: Remove-Item -Path $pathToJsonFile $content | ConvertTo-Json -Depth 4 | Out-File $pathToJsonFile displayName: 'Replace values in RolloutSpecPath.json' + env: + ob_restore_phase: true - pwsh: | $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'UploadLinux.Rollout.json' @@ -160,6 +186,8 @@ stages: Remove-Item -Path $pathToJsonFile $content | ConvertTo-Json -Depth 6 | Out-File $pathToJsonFile displayName: 'Replace values in UploadLinux.Rollout.json file' + env: + ob_restore_phase: true - pwsh: | $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'ServiceModel.json' @@ -170,6 +198,8 @@ stages: Remove-Item -Path $pathToJsonFile $content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile displayName: 'Replace values in ServiceModel.json' + env: + ob_restore_phase: true - pwsh: | $settingFilePath = Join-Path '$(ev2ServiceGroupRootFolder)/Shell/Run' -ChildPath 'settings.toml' From cdfcae2b7648c865dc0256000619ef3801070394 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 17:55:01 -0500 Subject: [PATCH 261/316] move template call before checkout step --- .pipelines/templates/release-prep-for-ev2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 2a383972370..e44a1e9d594 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -22,12 +22,12 @@ stages: - name: ob_sdl_credscan_suppressionsFile value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: + - template: release-SetReleaseTagandContainerName.yml + - checkout: self ## the global setting on lfs didn't work lfs: false env: ob_restore_phase: true - - - template: release-SetReleaseTagandContainerName.yml - pwsh: | $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' From 9e3e26b6d64440d1d30bb5e27e5280968428c0f5 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 18:15:17 -0500 Subject: [PATCH 262/316] add ob_restore_phase based parameter to template that sets tag and container name --- .../templates/release-SetReleaseTagandContainerName.yml | 7 +++++++ .pipelines/templates/release-prep-for-ev2.yml | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-SetReleaseTagandContainerName.yml b/.pipelines/templates/release-SetReleaseTagandContainerName.yml index 7e88624b45c..7b795de5965 100644 --- a/.pipelines/templates/release-SetReleaseTagandContainerName.yml +++ b/.pipelines/templates/release-SetReleaseTagandContainerName.yml @@ -1,3 +1,6 @@ +parameters: + restorePhase: false + steps: - pwsh: | $variable = 'releaseTag' @@ -12,6 +15,8 @@ steps: Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose Write-Host -Object "##$vstsCommandString" displayName: Set Release Tag + env: + ob_restore_phase: ${{ parameters.restorePhase }} - pwsh: | $azureVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '\.', '-' @@ -24,3 +29,5 @@ steps: Write-Host ("sending " + $vstsCommandString) Write-Host "##$vstsCommandString" displayName: Set container name + env: + ob_restore_phase: ${{ parameters.restorePhase }} diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index e44a1e9d594..bb867c6ac38 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -22,12 +22,14 @@ stages: - name: ob_sdl_credscan_suppressionsFile value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: - - template: release-SetReleaseTagandContainerName.yml - - checkout: self ## the global setting on lfs didn't work lfs: false env: ob_restore_phase: true + + - template: release-SetReleaseTagandContainerName.yml + parameters: + restorePhase: true - pwsh: | $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' From e636a58f2154421aba5bba2c979dda553addf320 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 18:31:40 -0500 Subject: [PATCH 263/316] remove ob_restore_phase from initial checkout --- .pipelines/templates/release-prep-for-ev2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index bb867c6ac38..46fcf67657a 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -24,8 +24,6 @@ stages: steps: - checkout: self ## the global setting on lfs didn't work lfs: false - env: - ob_restore_phase: true - template: release-SetReleaseTagandContainerName.yml parameters: @@ -37,6 +35,8 @@ stages: Write-Host "sending " + $vstsCommandString Write-Host "##$vstsCommandString" displayName: Set Package version + env: + ob_restore_phase: true - pwsh: | $branch = 'mirror-target' From 1cc27a84517d724381bfd36290548c7c61767912 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 18:41:55 -0500 Subject: [PATCH 264/316] check pipeline workspace --- .pipelines/templates/release-prep-for-ev2.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 46fcf67657a..842fcf0bcd6 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -24,6 +24,8 @@ stages: steps: - checkout: self ## the global setting on lfs didn't work lfs: false + env: + ob_restore_phase: true - template: release-SetReleaseTagandContainerName.yml parameters: @@ -59,8 +61,15 @@ stages: ob_restore_phase: true - pwsh: | - Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose - Get-ChildItem '$(Build.SourcesDirectory)' + $attemptPath = Test-Path '$(Build.SourcesDirectory)/PowerShell/.pipelines' + Write-Verbose -Verbose "attempt build src dir: $attemptPath" + displayName: 'Capture BuildDirectory' + env: + ob_restore_phase: true + + - pwsh: | + $attemptPath = Test-Path '$(Pipeline.Workspace)/PowerShell/.pipelines' + Write-Verbose -Verbose "attempt pipeline workspace: $attemptPath" displayName: 'Capture Workspace' env: ob_restore_phase: true From 838e398bde04ec5297c2d2d21d79be3ac0282ad7 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 18:53:17 -0500 Subject: [PATCH 265/316] more verbose statements for debugging --- .pipelines/templates/release-prep-for-ev2.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 842fcf0bcd6..ca89570398c 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -63,6 +63,7 @@ stages: - pwsh: | $attemptPath = Test-Path '$(Build.SourcesDirectory)/PowerShell/.pipelines' Write-Verbose -Verbose "attempt build src dir: $attemptPath" + Get-ChildItem '$(Build.SourcesDirectory)' displayName: 'Capture BuildDirectory' env: ob_restore_phase: true @@ -70,6 +71,7 @@ stages: - pwsh: | $attemptPath = Test-Path '$(Pipeline.Workspace)/PowerShell/.pipelines' Write-Verbose -Verbose "attempt pipeline workspace: $attemptPath" + Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose displayName: 'Capture Workspace' env: ob_restore_phase: true @@ -178,6 +180,14 @@ stages: ob_restore_phase: true - pwsh: | + $path = '$(Build.SourcesDirectory)/PowerShell/.pipelines' + $attemptPath = Test-Path $path + $path2 = '$(ev2ServiceGroupRootFolder)' + $attemptPath2 = Test-Path $path2 + Write-Verbose -Verbose "attempt build src dir $path exists : $attemptPath" + Write-Verbose -Verbose "attempt path 2 $path2 exists: $attemptPath2" + Get-ChildItem '$(Build.SourcesDirectory)' + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json' $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json $content.RolloutMetadata.Notification.Email.To = '$(PmcEV2SupportEmail)' From 2146453201fc18d076c4d36ee7154bf39af5fb54 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 19:01:45 -0500 Subject: [PATCH 266/316] more verbose statements for debugging --- .pipelines/templates/release-prep-for-ev2.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index ca89570398c..fbd6cd352bb 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -77,6 +77,9 @@ stages: ob_restore_phase: true - pwsh: | + $path = '$(Build.SourcesDirectory)/PowerShell/.pipelines' + $attemptPath = Test-Path $path + Write-Verbose -Verbose "path exists pre: $path : $attemptPath" New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory displayName: 'Create Parameters folder under EV2Specs folder' env: @@ -186,7 +189,7 @@ stages: $attemptPath2 = Test-Path $path2 Write-Verbose -Verbose "attempt build src dir $path exists : $attemptPath" Write-Verbose -Verbose "attempt path 2 $path2 exists: $attemptPath2" - Get-ChildItem '$(Build.SourcesDirectory)' + Get-ChildItem '$(Build.SourcesDirectory)' -Recurse $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json' $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json From 5f1e3228d8a2930b4567b6caa1347fd1b44d595b Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 19:14:55 -0500 Subject: [PATCH 267/316] move Ev2Specs folder out of .pipelines as ob_restore_phase doesn't copy folders beggining with period --- .pipelines/templates/release-prep-for-ev2.yml | 4 ++-- .../EV2Specs => EV2Specs}/ServiceGroupRoot/RollOutSpec.json | 0 .../EV2Specs => EV2Specs}/ServiceGroupRoot/ScopeBindings.json | 0 .../EV2Specs => EV2Specs}/ServiceGroupRoot/ServiceModel.json | 0 .../EV2Specs => EV2Specs}/ServiceGroupRoot/Shell/Run/Run.ps1 | 0 .../ServiceGroupRoot/UploadLinux.Rollout.json | 0 .../EV2Specs => EV2Specs}/ServiceGroupRoot/buildVer.txt | 0 7 files changed, 2 insertions(+), 2 deletions(-) rename {.pipelines/EV2Specs => EV2Specs}/ServiceGroupRoot/RollOutSpec.json (100%) rename {.pipelines/EV2Specs => EV2Specs}/ServiceGroupRoot/ScopeBindings.json (100%) rename {.pipelines/EV2Specs => EV2Specs}/ServiceGroupRoot/ServiceModel.json (100%) rename {.pipelines/EV2Specs => EV2Specs}/ServiceGroupRoot/Shell/Run/Run.ps1 (100%) rename {.pipelines/EV2Specs => EV2Specs}/ServiceGroupRoot/UploadLinux.Rollout.json (100%) rename {.pipelines/EV2Specs => EV2Specs}/ServiceGroupRoot/buildVer.txt (100%) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index fbd6cd352bb..d7d01b8aee0 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -12,9 +12,9 @@ stages: - name: repoRoot value: '$(Build.SourcesDirectory)/PowerShell' - name: ev2ServiceGroupRootFolder - value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot' + value: '$(Build.SourcesDirectory)/PowerShell/EV2Specs/ServiceGroupRoot' - name: ev2ParametersFolder - value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Parameters' + value: '$(Build.SourcesDirectory)/PowerShell/EV2Specs/ServiceGroupRoot/Parameters' - group: 'mscodehub-code-read-akv' - group: 'packages.microsoft.com' - name: ob_tipsign_enabled diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/RollOutSpec.json b/EV2Specs/ServiceGroupRoot/RollOutSpec.json similarity index 100% rename from .pipelines/EV2Specs/ServiceGroupRoot/RollOutSpec.json rename to EV2Specs/ServiceGroupRoot/RollOutSpec.json diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/ScopeBindings.json b/EV2Specs/ServiceGroupRoot/ScopeBindings.json similarity index 100% rename from .pipelines/EV2Specs/ServiceGroupRoot/ScopeBindings.json rename to EV2Specs/ServiceGroupRoot/ScopeBindings.json diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json b/EV2Specs/ServiceGroupRoot/ServiceModel.json similarity index 100% rename from .pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json rename to EV2Specs/ServiceGroupRoot/ServiceModel.json diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 similarity index 100% rename from .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 rename to EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json b/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json similarity index 100% rename from .pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json rename to EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt b/EV2Specs/ServiceGroupRoot/buildVer.txt similarity index 100% rename from .pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt rename to EV2Specs/ServiceGroupRoot/buildVer.txt From 03fceae6392e80110ada3099a3ea4554c38464c6 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 19:27:37 -0500 Subject: [PATCH 268/316] change casing --- EV2Specs/ServiceGroupRoot/RolloutSpec.json | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 EV2Specs/ServiceGroupRoot/RolloutSpec.json diff --git a/EV2Specs/ServiceGroupRoot/RolloutSpec.json b/EV2Specs/ServiceGroupRoot/RolloutSpec.json new file mode 100644 index 00000000000..9ed971068cc --- /dev/null +++ b/EV2Specs/ServiceGroupRoot/RolloutSpec.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/rolloutSpecification.json", + "contentVersion": "1.0.0.0", + "rolloutMetadata": { + "serviceModelPath": "ServiceModel.json", + "ScopeBindingsPath": "ScopeBindings.json", + "name": "OneBranch-Demo-Container-Deployment", + "rolloutType": "Major", + "buildSource": { + "parameters": { + "versionFile": "buildver.txt" + } + }, + "Notification": { + "Email": { + "To": "default" + } + } + }, + "orchestratedSteps": [ + { + "name": "UploadLinuxContainer", + "targetType": "ServiceResource", + "targetName": "LinuxContainerUpload", + "actions": ["Shell/Run"] + } + ] +} From 60c9a509b602038b9d4a38bb6ef706f7fdb870fc Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 19:43:02 -0500 Subject: [PATCH 269/316] move EV2 files back to .pipelines folder --- .../ServiceGroupRoot/RolloutSpec.json | 0 .../ServiceGroupRoot/ScopeBindings.json | 0 .../ServiceGroupRoot/ServiceModel.json | 0 .../ServiceGroupRoot/Shell/Run/Run.ps1 | 0 .../ServiceGroupRoot/UploadLinux.Rollout.json | 0 .../EV2Specs}/ServiceGroupRoot/buildVer.txt | 0 .pipelines/templates/release-prep-for-ev2.yml | 4 +-- EV2Specs/ServiceGroupRoot/RollOutSpec.json | 28 ------------------- 8 files changed, 2 insertions(+), 30 deletions(-) rename {EV2Specs => .pipelines/EV2Specs}/ServiceGroupRoot/RolloutSpec.json (100%) rename {EV2Specs => .pipelines/EV2Specs}/ServiceGroupRoot/ScopeBindings.json (100%) rename {EV2Specs => .pipelines/EV2Specs}/ServiceGroupRoot/ServiceModel.json (100%) rename {EV2Specs => .pipelines/EV2Specs}/ServiceGroupRoot/Shell/Run/Run.ps1 (100%) rename {EV2Specs => .pipelines/EV2Specs}/ServiceGroupRoot/UploadLinux.Rollout.json (100%) rename {EV2Specs => .pipelines/EV2Specs}/ServiceGroupRoot/buildVer.txt (100%) delete mode 100644 EV2Specs/ServiceGroupRoot/RollOutSpec.json diff --git a/EV2Specs/ServiceGroupRoot/RolloutSpec.json b/.pipelines/EV2Specs/ServiceGroupRoot/RolloutSpec.json similarity index 100% rename from EV2Specs/ServiceGroupRoot/RolloutSpec.json rename to .pipelines/EV2Specs/ServiceGroupRoot/RolloutSpec.json diff --git a/EV2Specs/ServiceGroupRoot/ScopeBindings.json b/.pipelines/EV2Specs/ServiceGroupRoot/ScopeBindings.json similarity index 100% rename from EV2Specs/ServiceGroupRoot/ScopeBindings.json rename to .pipelines/EV2Specs/ServiceGroupRoot/ScopeBindings.json diff --git a/EV2Specs/ServiceGroupRoot/ServiceModel.json b/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json similarity index 100% rename from EV2Specs/ServiceGroupRoot/ServiceModel.json rename to .pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json diff --git a/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 similarity index 100% rename from EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 rename to .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 diff --git a/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json b/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json similarity index 100% rename from EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json rename to .pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json diff --git a/EV2Specs/ServiceGroupRoot/buildVer.txt b/.pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt similarity index 100% rename from EV2Specs/ServiceGroupRoot/buildVer.txt rename to .pipelines/EV2Specs/ServiceGroupRoot/buildVer.txt diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index d7d01b8aee0..fbd6cd352bb 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -12,9 +12,9 @@ stages: - name: repoRoot value: '$(Build.SourcesDirectory)/PowerShell' - name: ev2ServiceGroupRootFolder - value: '$(Build.SourcesDirectory)/PowerShell/EV2Specs/ServiceGroupRoot' + value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot' - name: ev2ParametersFolder - value: '$(Build.SourcesDirectory)/PowerShell/EV2Specs/ServiceGroupRoot/Parameters' + value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Parameters' - group: 'mscodehub-code-read-akv' - group: 'packages.microsoft.com' - name: ob_tipsign_enabled diff --git a/EV2Specs/ServiceGroupRoot/RollOutSpec.json b/EV2Specs/ServiceGroupRoot/RollOutSpec.json deleted file mode 100644 index 9ed971068cc..00000000000 --- a/EV2Specs/ServiceGroupRoot/RollOutSpec.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/rolloutSpecification.json", - "contentVersion": "1.0.0.0", - "rolloutMetadata": { - "serviceModelPath": "ServiceModel.json", - "ScopeBindingsPath": "ScopeBindings.json", - "name": "OneBranch-Demo-Container-Deployment", - "rolloutType": "Major", - "buildSource": { - "parameters": { - "versionFile": "buildver.txt" - } - }, - "Notification": { - "Email": { - "To": "default" - } - } - }, - "orchestratedSteps": [ - { - "name": "UploadLinuxContainer", - "targetType": "ServiceResource", - "targetName": "LinuxContainerUpload", - "actions": ["Shell/Run"] - } - ] -} From 7451530dd3327f59b18b667c264901cc1aa59b95 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 20:18:00 -0500 Subject: [PATCH 270/316] update location of rg --- .pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json b/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json index 2f14ba34bea..00555349c35 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json +++ b/.pipelines/EV2Specs/ServiceGroupRoot/ServiceModel.json @@ -31,7 +31,7 @@ "serviceResourceGroups": [ { "azureResourceGroupName": "default", - "location": "East US", + "location": "West US 3", "instanceOf": "OneBranch-PowerShellDocker-RGDef", "azureSubscriptionId": "default", "scopeTags": [ From 6e8f7c83ccad8477292b58918a610f698add5eaa Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 20:57:02 -0500 Subject: [PATCH 271/316] remove which command --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 3159596dd5b..073c7daa9ac 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -47,7 +47,6 @@ try { Write-Verbose -Verbose "Test pmc-cli" - which pmc pmc -d -c $settingsFile repo list --name "azurelinux-3.0-prod-ms-oss-x86_64-yum" || exit 1 $metadataFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "pmcMetadata.json" From 02b3093fa2732edc47f5b265ce98a82fc388afba Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 22:49:00 -0500 Subject: [PATCH 272/316] see accessible pmc repos --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 073c7daa9ac..87124543455 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -47,7 +47,7 @@ try { Write-Verbose -Verbose "Test pmc-cli" - pmc -d -c $settingsFile repo list --name "azurelinux-3.0-prod-ms-oss-x86_64-yum" || exit 1 + pmc -d -c $settingsFile repo list --accessible || exit 1 $metadataFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "pmcMetadata.json" $metadataContent = Get-Content -Path $metadataFilePath | ConvertFrom-Json From 63584ba0195dd3096268c2aba2c2057eec924a02 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Thu, 16 Jan 2025 23:18:35 -0500 Subject: [PATCH 273/316] fix var names --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 84 ++++++++++++++----- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 87124543455..1eb5e7a7809 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -29,41 +29,81 @@ try { Invoke-WebRequest -Uri $env:PWSH_PACKAGES_TARGZIP -OutFile packages.tar.gz Invoke-WebRequest -Uri $env:PMC_METADATA -OutFile pmcMetadata.json - $settingsFile = Join-Path '/package/unarchive/Run' -ChildPath 'settings.toml' - $settingsFileExists = Test-Path -Path $settingsFile - Write-Verbose -Verbose "settings.toml exists: $settingsFileExists" + # create variables to those paths and test them + $repoListFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "repoList.json" + $repoListPathExists = Test-Path $repoListFilePath + if (!$repoListPathExists) + { + Write-Verbose -Verbose "repoList.json expected at $repoListFilePath does not exist" + return 1 + } + + $mappingFilePath = Join-Path "/package/unarchive/" -ChildPath "mapping.json" + $mappingFilePathExists = Test-Path $mappingFilePath + if (!$mappingFilePathExists) + { + Write-Verbose -Verbose "mapping.json expected at $mappingFilePath does not exist" + return 1 + } + + $packagesTarPath = Join-Path -Path "/package/unarchive/" -ChildPath "packages.tar.gz" + $packagesTarPathExists = Test-Path $packagesTarPath + if (!$packagesTarPathExists) + { + Write-Verbose -Verbose "packages.tar.gz expected at $packagesTarPath does not exist" + return 1 + } + + $metadataFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "pmcMetadata.json" + $metadataFilePathExists = Test-Path $metadataFilePath + if (!$metadataFilePathExists) + { + Write-Verbose -Verbose "pmcMetadata.json expected at $metadataFilePath does not exist" + return 1 + } + + # files in the extracted Run dir + $configPath = Join-Path '/package/unarchive/Run' -ChildPath 'settings.toml' + $configPathExists = Test-Path -Path $configPath + if (!$configPathExists) + { + Write-Verbose -Verbose "settings.toml expected at $configPath does not exist" + return 1 + } + $pythonDlFolder = Join-Path '/package/unarchive/Run' -ChildPath 'python_dl' $pyPathExists = Test-Path -Path $pythonDlFolder - Write-Verbose -Verbose "python_dl folder path exists: $pyPathExists" + if (!$pyPathExists) + { + Write-Verbose -Verbose "python_dl expected at $pythonDlFolder does not exist" + return 1 + } - Write-Verbose -Verbose "doing gci" - Get-ChildItem "/package" -Recurse Write-Verbose -Verbose "Installing pmc-cli" - crane version - # python.exe -m pip install --upgrade pip pip install --upgrade pip pip --version --verbose pip install /package/unarchive/Run/python_dl/*.whl Write-Verbose -Verbose "Test pmc-cli" + pmc -d -c $configPath repo list --accessible || exit 1 - pmc -d -c $settingsFile repo list --accessible || exit 1 - - $metadataFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "pmcMetadata.json" + # Get metadata + $channel = "" + $packageNames = @() $metadataContent = Get-Content -Path $metadataFilePath | ConvertFrom-Json $releaseVersion = $metadataContent.ReleaseTag - - - # $repoListFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "repoList.json" - # $repoListPathExists = Test-Path $repoListFilePath - # if (!$repoListPathExists) - # { - # Write-Verbose -Verbose "repoList.json expected at $repoListFilePath does not exist" - # return 1 - # } - - # $repoList = Get-RepoList -RepoConfigPath $configPath -OutFile $repoListPath #TODO: seems to require repoclient cli tool + if ($releaseVersion.Contains('-')) { + $channel = 'preview' + $packageNames = @('powershell-preview') + } + else { + $channel = 'stable' + $packageNames = @('powershell') + } + + Write-Verbose -Verbose "---Getting repository list---" + #$repoList = Get-RepoList -RepoConfigPath $configPath -OutFile $repoListPath #TODO: seems to require repoclient cli tool # # Get-PackageInfo() # $mappingFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "mapping.json" From e5e2d9a0cc94fde4e7d21e3be2b9c6e644c75cce Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 00:05:18 -0500 Subject: [PATCH 274/316] limit pmc repo list and add Get-PackageInfo() implementation --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 167 +++++++++--------- 1 file changed, 81 insertions(+), 86 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 1eb5e7a7809..ba522c6f6c2 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -102,94 +102,89 @@ try { $packageNames = @('powershell') } - Write-Verbose -Verbose "---Getting repository list---" - #$repoList = Get-RepoList -RepoConfigPath $configPath -OutFile $repoListPath #TODO: seems to require repoclient cli tool - - # # Get-PackageInfo() - # $mappingFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "mapping.json" - # $mappingPathExists = Test-Path $mappingFilePath - # if (!$mappingPathExists) - # { - # Write-Verbose -Verbose "mapping.json expected at $mappingFilePath does not exist" - # return 1 + # TODO: figure out where this comes in + # if ($LTS) { + # $packageNames += @('powershell-lts') # } - # $mapping = Get-Content -Raw -LiteralPath $mappingFilePath | ConvertFrom-Json -AsHashtable - # foreach ($package in $mapping.Packages) - # { - # Write-Verbose "package: $package" - # $packageChannel = $package.channel - # if (!$packageChannel) { - # $packageChannel = 'all' - # } - - # Write-Verbose "package channel: $packageChannel" - # if ($packageChannel -eq 'all' -or $packageChannel -eq $Channel) { - # $repoIds = [System.Collections.Generic.List[string]]::new() - # $packageFormat = $package.PackageFormat - # Write-Verbose "package format: $packageFormat" -Verbose - # $extension = [System.io.path]::GetExtension($packageFormat) - # $packageType = $extension -replace '^\.' - - # if ($package.distribution.count -gt 1) { - # throw "Package $($package | out-string) has more than one Distribution." - # } - - # foreach ($distribution in $package.distribution) { - # $urlGlob = $package.url - # switch ($packageType) { - # 'deb' { - # $urlGlob = $urlGlob + '-apt' - # } - # 'rpm' { - # $urlGlob = $urlGlob + '-yum' - # } - # default { - # throw "Unknown package type: $packageType" - # } - # } - - # Write-Verbose "---Finding repo id for: $urlGlob---" -Verbose - - # $repos = $repoList | Where-Object { $_.name -eq $urlGlob } - - # if ($repos.id) { - # Write-Verbose "Found repo id: $($repos.id)" -Verbose - # $repoIds.AddRange(([string[]]$repos.id)) - # } - # else { - # Write-Failure "Could not find repo for $urlGlob" - # } - # } - - # if ($repoIds.Count -le 0) { - # Write-Verbose -Verbose "no repoIds found that match our packages" - # # Write-Output ($package + @{ "RepoId" = $repoIds.ToArray() }) - # } - - # foreach ($pkg in $Package) { - # if ($pkg.RepoId.count -gt 1) { - # throw "Package $($pkg.name) has more than one repo id." - # } - # if ($pkg.Distribution.count -gt 1) { - # throw "Package $($pkg.name) has more than one Distribution." - # } - # $repo = $pkg.RepoId | Select-Object -First 1 - # $distribution = $pkg.Distribution | Select-Object -First 1 - # foreach ($name in $PackageName) { - # $pkgName = $pkg.PackageFormat.Replace('PACKAGE_NAME', $name).Replace('POWERSHELL_RELEASE', $ReleaseVersion) - # Write-Verbose "Creating info object for package '$pkgName' for repo '$repo'" - # $result = [pscustomobject]@{ - # PackageName = $pkgName - # RepoId = $repo - # Distribution = $distribution - # } - # Write-Verbose $result -Verbose - # Write-Output $result - # } - # } - # } - # } + Write-Verbose -Verbose "---Getting repository list---" + # tbh, I don't know if repoList.json is really needed- confused what the purpose of that is + $rawResponse = pmc --config $configPath repo list --limit 8 #TODO: actually 800 + $response = $rawResponse | ConvertFrom-Json + $limit = $($response.limit) + $count = $($response.count) + Write-Verbose -Verbose "limit is: $limit and count is: $count" + $repoList = $response.results + + Write-Verbose -Verbose "---Getting package info---" #TODO rename once I get what it really does + # start of method + Write-Verbose "Reading mapping file from '$mappingFilePath'" -Verbose + $mapping = Get-Content -Raw -LiteralPath $mappingFilePath | ConvertFrom-Json -AsHashtable + $mappedReposUsedByPwsh = @() + foreach ($package in $mapping.Packages) + { + Write-Verbose "package: $package" + $packageChannel = $package.channel + if (!$packageChannel) { + $packageChannel = 'all' + } + + Write-Verbose "package channel: $packageChannel" + if ($packageChannel -eq 'all' -or $packageChannel -eq $channel) + { + $repoIds = [System.Collections.Generic.List[string]]::new() + $packageFormat = $package.PackageFormat + Write-Verbose "package format: $packageFormat" -Verbose + $extension = [System.io.path]::GetExtension($packageFormat) + $packageType = $extension -replace '^\.' + + if ($package.distribution.count -gt 1) { # TODO: do we want to remove distribution stuff rn? ask. + throw "Package $($package | out-string) has more than one Distribution." + } + + foreach ($distribution in package.distribution) + { + $urlGlob = $package.url + switch ($packageType) + { + 'deb' { + $urlGlob = $urlGlob + '-apt' + } + 'rpm' { + $urlGlob = $urlGlob + '-yum' + } + default { + throw "Unknown package type: $packageType" + } + } + + Write-Verbose "---Finding repo id for: $urlGlob---" -Verbose + $repos = $RepoList | Where-Object { $_.name -eq $urlGlob } + + if ($repos.id) { + Write-Verbose "Found repo id: $($repos.id)" -Verbose + $repoIds.AddRange(([string[]]$repos.id)) + } + else { + Write-Failure "Could not find repo for $urlGlob" + } + + if ($repoIds.Count -gt 0) { + $mappedReposUsedByPwsh += ($package + @{ "RepoId" = $repoIds.ToArray() }) #TODO: adjust output type for this as needed + } + } + } + } + + Write-Verbose -Verbose "mapped repos length: $($mappedReposUsedByPwsh.Length)" + # END of Get-PackageInfo() + + #TODO: Anam: have some sort of object to contain output of the for loop + + # # Publish the packages based on the mapping file + # $publishedPackages = Get-PackageInfo -MappingFile $MappingFilePath -RepoList $repoList -Channel $channel | + # New-RepoPackageObject -ReleaseVersion $releaseVersion -PackageName $packageNames | + # Publish-PackageFromBlob -PwshVersion $releaseVersion -ConfigPath $configPath -BlobUriPrefix "$BlobBaseUri/$BlobFolderName" -WhatIf:$WhatIfPreference } From 7e687b836b414faf7965671f907b9d902e8988c3 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 00:24:12 -0500 Subject: [PATCH 275/316] remove repo list command that was writing and fix typo --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index ba522c6f6c2..8de3d5589e9 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -85,7 +85,7 @@ try { pip install /package/unarchive/Run/python_dl/*.whl Write-Verbose -Verbose "Test pmc-cli" - pmc -d -c $configPath repo list --accessible || exit 1 + # pmc -d -c $configPath repo list --accessible || exit 1 # Get metadata $channel = "" @@ -109,7 +109,7 @@ try { Write-Verbose -Verbose "---Getting repository list---" # tbh, I don't know if repoList.json is really needed- confused what the purpose of that is - $rawResponse = pmc --config $configPath repo list --limit 8 #TODO: actually 800 + $rawResponse = pmc --config $configPath repo list --limit 800 $response = $rawResponse | ConvertFrom-Json $limit = $($response.limit) $count = $($response.count) @@ -142,7 +142,7 @@ try { throw "Package $($package | out-string) has more than one Distribution." } - foreach ($distribution in package.distribution) + foreach ($distribution in $package.distribution) { $urlGlob = $package.url switch ($packageType) @@ -179,7 +179,9 @@ try { Write-Verbose -Verbose "mapped repos length: $($mappedReposUsedByPwsh.Length)" # END of Get-PackageInfo() - #TODO: Anam: have some sort of object to contain output of the for loop + # BEGIN New-RepoPackageObject() + + # # Publish the packages based on the mapping file # $publishedPackages = Get-PackageInfo -MappingFile $MappingFilePath -RepoList $repoList -Channel $channel | From 2966dce83f1c2f5d60193d3e66c73672f229e46d Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 00:41:17 -0500 Subject: [PATCH 276/316] add New-RepoPackageObject() implementation --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 8de3d5589e9..4f79af00f30 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -117,7 +117,8 @@ try { $repoList = $response.results Write-Verbose -Verbose "---Getting package info---" #TODO rename once I get what it really does - # start of method + # BEGIN Get-PackageInfo() + # for the packages powershell publishes to, resolve it's mapping with the directory of repos and basically map our packages to their actual repo Ids Write-Verbose "Reading mapping file from '$mappingFilePath'" -Verbose $mapping = Get-Content -Raw -LiteralPath $mappingFilePath | ConvertFrom-Json -AsHashtable $mappedReposUsedByPwsh = @() @@ -163,7 +164,7 @@ try { if ($repos.id) { Write-Verbose "Found repo id: $($repos.id)" -Verbose - $repoIds.AddRange(([string[]]$repos.id)) + $repoIds.AddRange(([string[]]$repos.id)) #tbh seems like a package should only have 1 repo Id } else { Write-Failure "Could not find repo for $urlGlob" @@ -180,7 +181,38 @@ try { # END of Get-PackageInfo() # BEGIN New-RepoPackageObject() + $repoPackageObjects = @() + foreach ($pkg in $mappedReposUsedByPwsh) + { + if ($pkg.RepoId.count -gt 1) { + throw "Package $($pkg.name) has more than one repo id." + } + + if ($pkg.Distribution.count -gt 1) { + throw "Package $($pkg.name) has more than one Distribution." + } + + $pkgRepo = $pkg.RepoId | Select-Object -First 1 + $pkgDistribution = $pkg.Distribution | Select-Object -First 1 + + foreach ($name in $packageNames) { + $pkgName = $pkg.PackageFormat.Replace('PACKAGE_NAME', $name).Replace('POWERSHELL_RELEASE', $releaseVersion) + Write-Verbose "Creating info object for package '$pkgName' for repo '$pkgRepo'" + $result = [pscustomobject]@{ + PackageName = $pkgName + RepoId = $repo + Distribution = $pkgDistribution + } + + Write-Verbose $result -Verbose + $repoPackageObjects += $result + } + } + # END of New-RepoPackageObject() - I think this and the method before can be combined + + Write-Verbose -Verbose "count of repoPackageObjects: $($repoPackageObjects.Length)" + # BEGIN Publish-PackageFromBlob() # # Publish the packages based on the mapping file From 742a6a014fa915765433e490bec74b4c7c409f8b Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 10:14:27 -0500 Subject: [PATCH 277/316] fix verbose statement and add commented out implementation of Publish-PackageFromBlob() --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 128 +++++++++++++++++- 1 file changed, 126 insertions(+), 2 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 4f79af00f30..480423f2f29 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -200,7 +200,7 @@ try { Write-Verbose "Creating info object for package '$pkgName' for repo '$pkgRepo'" $result = [pscustomobject]@{ PackageName = $pkgName - RepoId = $repo + RepoId = $pkgRepo Distribution = $pkgDistribution } @@ -212,9 +212,133 @@ try { Write-Verbose -Verbose "count of repoPackageObjects: $($repoPackageObjects.Length)" - # BEGIN Publish-PackageFromBlob() + # # BEGIN Publish-PackageFromBlob() + # $packagePaths = @() #TODO: why is this needed/collected?? + # $packages = @() + # foreach ($pkgObj in $repoPackageObjects) + # { + # # RHEL and CentOS packages have a tweak in the name... + # if ($pkgObj.PackageName.EndsWith('.rpm')) { #TODO: can I do this in the 1 condensed method + # $pkgObjName = $PackageName.Replace($releaseVersion, $releaseVersion.Replace('-', '_')) + # } + # $packagePath = "$TempDirPath/$pkgObjName" #TODO now: resolve correctly! + + # # For blob downloads they download each package once, not sure why line304 is happening-> $packagePaths += $packagePath + + # $packages += @{ + # PackagePath = $packagePath + # PackageName = $pkgObjName + # RepoId = $pkgObj.RepoId + # Distribution = $pkgObj.Distribution #TODO remove later + # } + # } + + # # end block of Publish-PackageFromBlob() + + # # Don't fail outright when an error occurs, but instead pool them until + # # after attempting to publish every package. That way we can choose to + # # proceed for a partial failure. + # $errorMessage = [System.Collections.Generic.List[string]]::new() + # foreach ($finalPackage in $packages) + # { + # Write-Verbose "---Staging package: $($finalPackage.PackageName)---" -Verbose + # $packagePath = $finalPackage.PackagePath + # $pkgRepo = $finalPackage.RepoId + + # #TODO: should process if/else here- needed or nah? + # $extension = [System.io.path]::GetExtension($packagePath) + # $packageType = $extension -replace '^\.' + # Write-Verbose "packageType: $packageType" -Verbose + + # $packageListJson = pmc --config $configPath package $packageType list --file $packagePath + # $list = $packageListJson | ConvertFrom-Json + + # $packageId = @() #TODO: powershell doesn't need it but for my sanity + # if ($list.count -ne 0) + # { + # Write-Verbose "Package '$packagePath' already exists, skipping upload" -Verbose + # $packageId = $list.results.id | Select-Object -First 1 + # } + # else { + # # PMC UPDATE COMMAND + # Write-Verbose -Verbose "Uploading package, config: '$configPath' package: '$packagePath'" + # $uploadResult = $null + # try { + # $uploadResult = pmc --config $configPath package upload $packagePath --type $packageType + # } + # catch { + # $errorMessage.Add("Uploading package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") + # continue + # } + + # $packageId = ($uploadResult | ConvertFrom-Json).id + # } + + # Write-Verbose "Got package ID: '$packageId'" -Verbose + # $distribution = $finalPackage.Distribution | select-object -First 1 + # Write-Verbose "distribution: $distribution" -Verbose + + # if (!$SkipPublish) + # { + # # TODO: important, do we even have this? need this? + # Write-Verbose "---Publishing package: $($finalPackage.PackageName) to $pkgRepo---" -Verbose + + # if (($packageType -ne 'rpm') -and ($packageType -ne 'deb')) + # { + # throw "Unsupported package type: $packageType" + # return 1 + # } + # else { + # # UPDATE PMC COMMAND + # $rawUpdateResponse = $null + # try { + # if ($packageType -eq 'rpm') { + # $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo --add-packages $packageId + # } elseif ($packageType -eq 'deb') { + # $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo $distribution --add-packages $packageId #TODO: if the distribution is not needed, probs can be simplified + # } + # } + # catch { + # $errorMessage.Add("Invoking update for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") + # continue + # } + + # $state = $rawUpdateResponse.state + # if ($state -ne 'Completed') { + # $errorMessage.Add("Publishing package $($finalPackage.PackageName) to $pkgRepo failed: $rawUpdateResponse") + # continue + # } + # } + + # # PUBLISH PMC COMMAND + # # The CLI outputs messages and JSON in the same stream, so we must sift through it for now + # # This is planned to be fixed with a switch in a later release + # # TODO: Anam, figure out if this was fixed, and if so lets use this switch + # Write-Output ([pscustomobject]($package + @{ + # PackageId = $packageId + # })) + + # # At this point, the changes are staged and will eventually be publish. + # # Running publish, causes them to go live "immediately" + # try { + # pmc --config $configPath repo publish $pkgRepo + # } + # catch { + # $errorMessage.Add("Running final publish for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") + # continue + # } + # } else { + # Write-Verbose -Verbose "Skipping Uploading package --config-file '$configPath' package add '$packagePath' --repoID '$pkgRepo'" + # } + # } + + # if ($errorMessage) { + # throw $errorMessage -join [Environment]::NewLine + # } + + # END Publish-PackageFromBlob() # # Publish the packages based on the mapping file # $publishedPackages = Get-PackageInfo -MappingFile $MappingFilePath -RepoList $repoList -Channel $channel | # New-RepoPackageObject -ReleaseVersion $releaseVersion -PackageName $packageNames | From a03ed3b96613dfe412be9eeacbaecf6766b00a9d Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 10:49:14 -0500 Subject: [PATCH 278/316] add extraction of packages from packages.tar.gz --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 480423f2f29..463b6cfa9fd 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -54,6 +54,13 @@ try { return 1 } + # Extract files from 'packages.tar.gz' + Write-Verbose -Verbose "---Extracting files from packages.tar.gz---" + $pwshPackagesFolder = Join-Path -Path "/package/unarchive/" -ChildPath "packages" + New-Item -Path $pwshPackagesFolder -ItemType Directory + tar -xzvf $packagesTarPath -C $pwshPackagesFolder --force-local + Get-ChildItem $pwshPackagesFolder -Recurse + $metadataFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "pmcMetadata.json" $metadataFilePathExists = Test-Path $metadataFilePath if (!$metadataFilePathExists) From 11b70c90ea5c5bfd909798252b0816a819b2ceef Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 12:39:24 -0500 Subject: [PATCH 279/316] change release tag version --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 18 ++---------------- .pipelines/templates/release-prep-for-ev2.yml | 5 ++--- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 463b6cfa9fd..543e92a167f 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -91,9 +91,6 @@ try { pip --version --verbose pip install /package/unarchive/Run/python_dl/*.whl - Write-Verbose -Verbose "Test pmc-cli" - # pmc -d -c $configPath repo list --accessible || exit 1 - # Get metadata $channel = "" $packageNames = @() @@ -220,7 +217,6 @@ try { Write-Verbose -Verbose "count of repoPackageObjects: $($repoPackageObjects.Length)" # # BEGIN Publish-PackageFromBlob() - # $packagePaths = @() #TODO: why is this needed/collected?? # $packages = @() # foreach ($pkgObj in $repoPackageObjects) @@ -230,9 +226,7 @@ try { # $pkgObjName = $PackageName.Replace($releaseVersion, $releaseVersion.Replace('-', '_')) # } - # $packagePath = "$TempDirPath/$pkgObjName" #TODO now: resolve correctly! - - # # For blob downloads they download each package once, not sure why line304 is happening-> $packagePaths += $packagePath + # $packagePath = "$pwshPackagesFolder/$pkgObjName" # $packages += @{ # PackagePath = $packagePath @@ -289,7 +283,7 @@ try { # if (!$SkipPublish) # { - # # TODO: important, do we even have this? need this? + # # TODO: important, do we even have SkipPublish? need this? # Write-Verbose "---Publishing package: $($finalPackage.PackageName) to $pkgRepo---" -Verbose # if (($packageType -ne 'rpm') -and ($packageType -ne 'deb')) @@ -344,14 +338,6 @@ try { # if ($errorMessage) { # throw $errorMessage -join [Environment]::NewLine # } - - # END Publish-PackageFromBlob() - # # Publish the packages based on the mapping file - # $publishedPackages = Get-PackageInfo -MappingFile $MappingFilePath -RepoList $repoList -Channel $channel | - # New-RepoPackageObject -ReleaseVersion $releaseVersion -PackageName $packageNames | - # Publish-PackageFromBlob -PwshVersion $releaseVersion -ConfigPath $configPath -BlobUriPrefix "$BlobBaseUri/$BlobFolderName" -WhatIf:$WhatIfPreference - - } catch { Write-Error -ErrorAction Stop $_.Exception.Message diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index fbd6cd352bb..ae226840352 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -164,9 +164,8 @@ stages: $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json $skipPublishVar = '${{ parameters.skipPublish }}' $metadataHash = @{} - $metadataHash["ReleaseTag"] = "$(ReleaseTag)" - # $metadataHash["AadClientId"] = '$(PmcCliClientID)' - $metadataHash["BlobFolderName"] = "$(ReleaseTag)" + # $metadataHash["ReleaseTag"] = "$(ReleaseTag)" # TODO: UNCOMMENT + $metadataHash["ReleaseTag"] = "v7.5.0.1" $metadataHash["LTS"] = $metadata.LTSRelease.Latest $metadataHash["ForProduction"] = $true $metadataHash["SkipPublish"] = $skipPublishVar From 723b76116bb0d2df624224079e1b0ea8e15d5c3e Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 14:05:48 -0500 Subject: [PATCH 280/316] add Skip publish CI param --- .../PowerShell-Release-Official-Azure.yml | 36 +++---------------- .pipelines/templates/release-prep-for-ev2.yml | 7 ++-- 2 files changed, 10 insertions(+), 33 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 72220b98be7..3425b449211 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -5,14 +5,10 @@ parameters: # parameters are shown up in ADO UI in a build queue time displayName: 'Enable debug output' type: boolean default: false - - name: InternalSDKBlobURL - displayName: URL to the blob having internal .NET SDK - type: string - default: ' ' - - name: ReleaseTagVar - displayName: Release Tag - type: string - default: 'fromBranch' + - name: skipPublish + displayName: Skip PMC Publish + type: boolean + default: false - name: SKIP_SIGNING displayName: Skip Signing type: string @@ -47,8 +43,6 @@ variables: value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - name: LinuxContainerImage value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0 - - name: ReleaseTagVar - value: ${{ parameters.ReleaseTagVar }} - group: PoolNames resources: @@ -102,28 +96,8 @@ extends: tsaOptionsFile: .config\tsaoptions.json stages: - template: /.pipelines/templates/release-prep-for-ev2.yml@self - - template: /.pipelines/templates/release-publish-pmc.yml@self - # - stage: PrepForEV2 - # displayName: Prep for EV2 - # dependsOn: [] - # jobs: - # - template: /.pipelines/templates/release-prep-for-ev2.yml@self - - # - stage: BlobPublic - # displayName: Make Blob Public - # dependsOn: PrepForEV2 - # jobs: - # - template: /.pipelines/templates/release-MakeBlobPublic.yml@self - # - stage: Test_PublishPMC - # displayName: Publish PMC - # dependsOn: - # - PrepForEV2 - # # - BlobPublic - # variables: - # ob_release_environment: Test - # jobs: - # - template: /.pipelines/templates/release-publish-pmc.yml@self + - template: /.pipelines/templates/release-publish-pmc.yml@self # - stage: UploadBuildInfoJson # dependsOn: diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index ae226840352..4e4a9cd5e99 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -32,6 +32,8 @@ stages: restorePhase: true - pwsh: | + $skipPubValue = '${{ parameters.skipPublish }}' + Write-Verbose -Verbose "skip publish value: $skipPubValue" $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" Write-Host "sending " + $vstsCommandString @@ -162,13 +164,14 @@ stages: $pathToPMCMetadataFile = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'pmcMetadata.json' $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json - $skipPublishVar = '${{ parameters.skipPublish }}' + $skipPublish = '${{ parameters.skipPublish }}' + Write-Verbose -Verbose "skip publish value: $skipPublish" $metadataHash = @{} # $metadataHash["ReleaseTag"] = "$(ReleaseTag)" # TODO: UNCOMMENT $metadataHash["ReleaseTag"] = "v7.5.0.1" $metadataHash["LTS"] = $metadata.LTSRelease.Latest $metadataHash["ForProduction"] = $true - $metadataHash["SkipPublish"] = $skipPublishVar + $metadataHash["SkipPublish"] = $skipPublish $metadataHash | ConvertTo-Json | Out-File $pathToPMCMetadataFile From bb844388ef87f090afeb0f32e272982612655558 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 14:33:16 -0500 Subject: [PATCH 281/316] add Skip publish as template param as well --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 45 +++++++------------ .../PowerShell-Release-Official-Azure.yml | 2 + .../release-SetReleaseTagandContainerName.yml | 3 +- .pipelines/templates/release-prep-for-ev2.yml | 4 ++ 4 files changed, 23 insertions(+), 31 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 543e92a167f..a1e58bdadfa 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -4,12 +4,6 @@ if ($null -eq $env:MAPPING_FILE) return 1 } -if ($null -eq $env:REPO_LIST_FILE) -{ - Write-Verbose -Verbose "REPO_LIST_FILE variable didn't get passed correctly" - return 1 -} - if ($null -eq $env:PWSH_PACKAGES_TARGZIP) { Write-Verbose -Verbose "PWSH_PACKAGES_TARGZIP variable didn't get passed correctly" @@ -25,19 +19,10 @@ if ($null -eq $env:PMC_METADATA) try { Write-Verbose -Verbose "Downloading files" Invoke-WebRequest -Uri $env:MAPPING_FILE -OutFile mapping.json - Invoke-WebRequest -Uri $env:REPO_LIST_FILE -OutFile repoList.json Invoke-WebRequest -Uri $env:PWSH_PACKAGES_TARGZIP -OutFile packages.tar.gz Invoke-WebRequest -Uri $env:PMC_METADATA -OutFile pmcMetadata.json # create variables to those paths and test them - $repoListFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "repoList.json" - $repoListPathExists = Test-Path $repoListFilePath - if (!$repoListPathExists) - { - Write-Verbose -Verbose "repoList.json expected at $repoListFilePath does not exist" - return 1 - } - $mappingFilePath = Join-Path "/package/unarchive/" -ChildPath "mapping.json" $mappingFilePathExists = Test-Path $mappingFilePath if (!$mappingFilePathExists) @@ -96,6 +81,9 @@ try { $packageNames = @() $metadataContent = Get-Content -Path $metadataFilePath | ConvertFrom-Json $releaseVersion = $metadataContent.ReleaseTag + $skipPublish = $metadataContent.SkipPublish + $lts = $metadataContent.LTS + Write-Verbose -Verbose "skip publish: $skipPublish" #TODO: remove if ($releaseVersion.Contains('-')) { $channel = 'preview' @@ -106,21 +94,19 @@ try { $packageNames = @('powershell') } - # TODO: figure out where this comes in - # if ($LTS) { - # $packageNames += @('powershell-lts') - # } + if ($lts) { + $packageNames += @('powershell-lts') + } Write-Verbose -Verbose "---Getting repository list---" - # tbh, I don't know if repoList.json is really needed- confused what the purpose of that is $rawResponse = pmc --config $configPath repo list --limit 800 $response = $rawResponse | ConvertFrom-Json $limit = $($response.limit) $count = $($response.count) - Write-Verbose -Verbose "limit is: $limit and count is: $count" + Write-Verbose -Verbose "'pmc repo list' limit is: $limit and count is: $count" $repoList = $response.results - Write-Verbose -Verbose "---Getting package info---" #TODO rename once I get what it really does + Write-Verbose -Verbose "---Getting package info---" # BEGIN Get-PackageInfo() # for the packages powershell publishes to, resolve it's mapping with the directory of repos and basically map our packages to their actual repo Ids Write-Verbose "Reading mapping file from '$mappingFilePath'" -Verbose @@ -143,7 +129,7 @@ try { $extension = [System.io.path]::GetExtension($packageFormat) $packageType = $extension -replace '^\.' - if ($package.distribution.count -gt 1) { # TODO: do we want to remove distribution stuff rn? ask. + if ($package.distribution.count -gt 1) { throw "Package $($package | out-string) has more than one Distribution." } @@ -164,7 +150,7 @@ try { } Write-Verbose "---Finding repo id for: $urlGlob---" -Verbose - $repos = $RepoList | Where-Object { $_.name -eq $urlGlob } + $repos = $repoList | Where-Object { $_.name -eq $urlGlob } if ($repos.id) { Write-Verbose "Found repo id: $($repos.id)" -Verbose @@ -175,7 +161,7 @@ try { } if ($repoIds.Count -gt 0) { - $mappedReposUsedByPwsh += ($package + @{ "RepoId" = $repoIds.ToArray() }) #TODO: adjust output type for this as needed + $mappedReposUsedByPwsh += ($package + @{ "RepoId" = $repoIds.ToArray() }) } } } @@ -232,7 +218,7 @@ try { # PackagePath = $packagePath # PackageName = $pkgObjName # RepoId = $pkgObj.RepoId - # Distribution = $pkgObj.Distribution #TODO remove later + # Distribution = $pkgObj.Distribution # } # } @@ -256,7 +242,7 @@ try { # $packageListJson = pmc --config $configPath package $packageType list --file $packagePath # $list = $packageListJson | ConvertFrom-Json - # $packageId = @() #TODO: powershell doesn't need it but for my sanity + # $packageId = @() # if ($list.count -ne 0) # { # Write-Verbose "Package '$packagePath' already exists, skipping upload" -Verbose @@ -281,9 +267,8 @@ try { # $distribution = $finalPackage.Distribution | select-object -First 1 # Write-Verbose "distribution: $distribution" -Verbose - # if (!$SkipPublish) + # if (!$skipPublish) # { - # # TODO: important, do we even have SkipPublish? need this? # Write-Verbose "---Publishing package: $($finalPackage.PackageName) to $pkgRepo---" -Verbose # if (($packageType -ne 'rpm') -and ($packageType -ne 'deb')) @@ -298,7 +283,7 @@ try { # if ($packageType -eq 'rpm') { # $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo --add-packages $packageId # } elseif ($packageType -eq 'deb') { - # $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo $distribution --add-packages $packageId #TODO: if the distribution is not needed, probs can be simplified + # $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo $distribution --add-packages $packageId # } # } # catch { diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 3425b449211..add2e2b8db6 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -96,6 +96,8 @@ extends: tsaOptionsFile: .config\tsaoptions.json stages: - template: /.pipelines/templates/release-prep-for-ev2.yml@self + parameters: + skipPublish: ${{ parameters.skipPublish }} - template: /.pipelines/templates/release-publish-pmc.yml@self diff --git a/.pipelines/templates/release-SetReleaseTagandContainerName.yml b/.pipelines/templates/release-SetReleaseTagandContainerName.yml index 7b795de5965..99188351fdf 100644 --- a/.pipelines/templates/release-SetReleaseTagandContainerName.yml +++ b/.pipelines/templates/release-SetReleaseTagandContainerName.yml @@ -1,5 +1,6 @@ parameters: - restorePhase: false +- name: restorePhase + default: false steps: - pwsh: | diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 4e4a9cd5e99..6c6272f851a 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -1,3 +1,7 @@ +parameters: +- name: skipPublish + default: false + stages: - stage: PrepForEV2 displayName: 'Copy and prep all files needed for EV2 stage' From fd0d7921113e2ee45b50193b94aa959849c4c39c Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 14:51:45 -0500 Subject: [PATCH 282/316] fix boolean written to pmcMetadata.json --- .pipelines/templates/release-prep-for-ev2.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 6c6272f851a..149153625a1 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -25,6 +25,8 @@ stages: value: true # Enable TiP sign - name: ob_sdl_credscan_suppressionsFile value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: skipPublish + value: ${{ parameters.skipPublish }} steps: - checkout: self ## the global setting on lfs didn't work lfs: false @@ -36,8 +38,7 @@ stages: restorePhase: true - pwsh: | - $skipPubValue = '${{ parameters.skipPublish }}' - Write-Verbose -Verbose "skip publish value: $skipPubValue" + Write-Verbose -Verbose "skip publish value: $skipPublish" $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" Write-Host "sending " + $vstsCommandString @@ -168,14 +169,13 @@ stages: $pathToPMCMetadataFile = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'pmcMetadata.json' $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json - $skipPublish = '${{ parameters.skipPublish }}' - Write-Verbose -Verbose "skip publish value: $skipPublish" $metadataHash = @{} + $skipPublishBoolean = '$(skipPublish)' -eq "false" ? false : true # $metadataHash["ReleaseTag"] = "$(ReleaseTag)" # TODO: UNCOMMENT $metadataHash["ReleaseTag"] = "v7.5.0.1" $metadataHash["LTS"] = $metadata.LTSRelease.Latest $metadataHash["ForProduction"] = $true - $metadataHash["SkipPublish"] = $skipPublish + $metadataHash["SkipPublish"] = $skipPublishBoolean $metadataHash | ConvertTo-Json | Out-File $pathToPMCMetadataFile From f592cba3689de8c1d62cf06820fb061d625e3e36 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 15:10:49 -0500 Subject: [PATCH 283/316] correctly convert boolean written to pmcMetadata.json --- .pipelines/templates/release-prep-for-ev2.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 149153625a1..3420c2987a5 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -1,5 +1,6 @@ parameters: - name: skipPublish + type: boolean default: false stages: @@ -25,8 +26,6 @@ stages: value: true # Enable TiP sign - name: ob_sdl_credscan_suppressionsFile value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - - name: skipPublish - value: ${{ parameters.skipPublish }} steps: - checkout: self ## the global setting on lfs didn't work lfs: false @@ -38,7 +37,8 @@ stages: restorePhase: true - pwsh: | - Write-Verbose -Verbose "skip publish value: $skipPublish" + $sPubValue = '${{ parameters.skipPublish }}' + Write-Verbose -Verbose "skip publish value: $sPubValue" $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" Write-Host "sending " + $vstsCommandString @@ -170,12 +170,12 @@ stages: $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json $metadataHash = @{} - $skipPublishBoolean = '$(skipPublish)' -eq "false" ? false : true + $skipPublishValue = '${{ parameters.skipPublish }}' # $metadataHash["ReleaseTag"] = "$(ReleaseTag)" # TODO: UNCOMMENT $metadataHash["ReleaseTag"] = "v7.5.0.1" $metadataHash["LTS"] = $metadata.LTSRelease.Latest $metadataHash["ForProduction"] = $true - $metadataHash["SkipPublish"] = $skipPublishBoolean + $metadataHash["SkipPublish"] = [System.Convert]::ToBoolean($skipPublishValue) $metadataHash | ConvertTo-Json | Out-File $pathToPMCMetadataFile From 38b5da46d78fbf1bfebe0823a2126e3b5f36bef9 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 15:31:28 -0500 Subject: [PATCH 284/316] test the function --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index a1e58bdadfa..33d188657ad 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -1,3 +1,12 @@ +function Get-Info { + param( + [string] + $PkgName + ) + + Write-Verbose -Verbose "pkg name: $pkgName" +} + if ($null -eq $env:MAPPING_FILE) { Write-Verbose -Verbose "MAPPING_FILE variable didn't get passed correctly" @@ -84,6 +93,7 @@ try { $skipPublish = $metadataContent.SkipPublish $lts = $metadataContent.LTS Write-Verbose -Verbose "skip publish: $skipPublish" #TODO: remove + Get-Info -PkgName "testing" if ($releaseVersion.Contains('-')) { $channel = 'preview' From 8174446580784c245f4a1c6ee688a3ba3f8a3f67 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 15:52:54 -0500 Subject: [PATCH 285/316] add final publishing code in --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 242 +++++++++--------- 1 file changed, 121 insertions(+), 121 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 33d188657ad..b524b72eb02 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -212,127 +212,127 @@ try { Write-Verbose -Verbose "count of repoPackageObjects: $($repoPackageObjects.Length)" - # # BEGIN Publish-PackageFromBlob() - # $packages = @() - - # foreach ($pkgObj in $repoPackageObjects) - # { - # # RHEL and CentOS packages have a tweak in the name... - # if ($pkgObj.PackageName.EndsWith('.rpm')) { #TODO: can I do this in the 1 condensed method - # $pkgObjName = $PackageName.Replace($releaseVersion, $releaseVersion.Replace('-', '_')) - # } - - # $packagePath = "$pwshPackagesFolder/$pkgObjName" - - # $packages += @{ - # PackagePath = $packagePath - # PackageName = $pkgObjName - # RepoId = $pkgObj.RepoId - # Distribution = $pkgObj.Distribution - # } - # } - - # # end block of Publish-PackageFromBlob() - - # # Don't fail outright when an error occurs, but instead pool them until - # # after attempting to publish every package. That way we can choose to - # # proceed for a partial failure. - # $errorMessage = [System.Collections.Generic.List[string]]::new() - # foreach ($finalPackage in $packages) - # { - # Write-Verbose "---Staging package: $($finalPackage.PackageName)---" -Verbose - # $packagePath = $finalPackage.PackagePath - # $pkgRepo = $finalPackage.RepoId - - # #TODO: should process if/else here- needed or nah? - # $extension = [System.io.path]::GetExtension($packagePath) - # $packageType = $extension -replace '^\.' - # Write-Verbose "packageType: $packageType" -Verbose - - # $packageListJson = pmc --config $configPath package $packageType list --file $packagePath - # $list = $packageListJson | ConvertFrom-Json - - # $packageId = @() - # if ($list.count -ne 0) - # { - # Write-Verbose "Package '$packagePath' already exists, skipping upload" -Verbose - # $packageId = $list.results.id | Select-Object -First 1 - # } - # else { - # # PMC UPDATE COMMAND - # Write-Verbose -Verbose "Uploading package, config: '$configPath' package: '$packagePath'" - # $uploadResult = $null - # try { - # $uploadResult = pmc --config $configPath package upload $packagePath --type $packageType - # } - # catch { - # $errorMessage.Add("Uploading package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") - # continue - # } - - # $packageId = ($uploadResult | ConvertFrom-Json).id - # } - - # Write-Verbose "Got package ID: '$packageId'" -Verbose - # $distribution = $finalPackage.Distribution | select-object -First 1 - # Write-Verbose "distribution: $distribution" -Verbose - - # if (!$skipPublish) - # { - # Write-Verbose "---Publishing package: $($finalPackage.PackageName) to $pkgRepo---" -Verbose - - # if (($packageType -ne 'rpm') -and ($packageType -ne 'deb')) - # { - # throw "Unsupported package type: $packageType" - # return 1 - # } - # else { - # # UPDATE PMC COMMAND - # $rawUpdateResponse = $null - # try { - # if ($packageType -eq 'rpm') { - # $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo --add-packages $packageId - # } elseif ($packageType -eq 'deb') { - # $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo $distribution --add-packages $packageId - # } - # } - # catch { - # $errorMessage.Add("Invoking update for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") - # continue - # } - - # $state = $rawUpdateResponse.state - # if ($state -ne 'Completed') { - # $errorMessage.Add("Publishing package $($finalPackage.PackageName) to $pkgRepo failed: $rawUpdateResponse") - # continue - # } - # } - - # # PUBLISH PMC COMMAND - # # The CLI outputs messages and JSON in the same stream, so we must sift through it for now - # # This is planned to be fixed with a switch in a later release - # # TODO: Anam, figure out if this was fixed, and if so lets use this switch - # Write-Output ([pscustomobject]($package + @{ - # PackageId = $packageId - # })) - - # # At this point, the changes are staged and will eventually be publish. - # # Running publish, causes them to go live "immediately" - # try { - # pmc --config $configPath repo publish $pkgRepo - # } - # catch { - # $errorMessage.Add("Running final publish for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") - # continue - # } - # } else { - # Write-Verbose -Verbose "Skipping Uploading package --config-file '$configPath' package add '$packagePath' --repoID '$pkgRepo'" - # } - # } - - # if ($errorMessage) { - # throw $errorMessage -join [Environment]::NewLine - # } + # BEGIN Publish-PackageFromBlob() + $packages = @() + + foreach ($pkgObj in $repoPackageObjects) + { + # RHEL and CentOS packages have a tweak in the name... + if ($pkgObj.PackageName.EndsWith('.rpm')) { #TODO: can I do this in the 1 condensed method + $pkgObjName = $PackageName.Replace($releaseVersion, $releaseVersion.Replace('-', '_')) + } + + $packagePath = "$pwshPackagesFolder/$pkgObjName" + + $packages += @{ + PackagePath = $packagePath + PackageName = $pkgObjName + RepoId = $pkgObj.RepoId + Distribution = $pkgObj.Distribution + } + } + + # end block of Publish-PackageFromBlob() + + # Don't fail outright when an error occurs, but instead pool them until + # after attempting to publish every package. That way we can choose to + # proceed for a partial failure. + $errorMessage = [System.Collections.Generic.List[string]]::new() + foreach ($finalPackage in $packages) + { + Write-Verbose "---Staging package: $($finalPackage.PackageName)---" -Verbose + $packagePath = $finalPackage.PackagePath + $pkgRepo = $finalPackage.RepoId + + #TODO: should process if/else here- needed or nah? + $extension = [System.io.path]::GetExtension($packagePath) + $packageType = $extension -replace '^\.' + Write-Verbose "packageType: $packageType" -Verbose + + $packageListJson = pmc --config $configPath package $packageType list --file $packagePath + $list = $packageListJson | ConvertFrom-Json + + $packageId = @() + if ($list.count -ne 0) + { + Write-Verbose "Package '$packagePath' already exists, skipping upload" -Verbose + $packageId = $list.results.id | Select-Object -First 1 + } + else { + # PMC UPDATE COMMAND + Write-Verbose -Verbose "Uploading package, config: '$configPath' package: '$packagePath'" + $uploadResult = $null + try { + $uploadResult = pmc --config $configPath package upload $packagePath --type $packageType + } + catch { + $errorMessage.Add("Uploading package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") + continue + } + + $packageId = ($uploadResult | ConvertFrom-Json).id + } + + Write-Verbose "Got package ID: '$packageId'" -Verbose + $distribution = $finalPackage.Distribution | select-object -First 1 + Write-Verbose "distribution: $distribution" -Verbose + + if (!$skipPublish) + { + Write-Verbose "---Publishing package: $($finalPackage.PackageName) to $pkgRepo---" -Verbose + + if (($packageType -ne 'rpm') -and ($packageType -ne 'deb')) + { + throw "Unsupported package type: $packageType" + return 1 + } + else { + # UPDATE PMC COMMAND + $rawUpdateResponse = $null + try { + if ($packageType -eq 'rpm') { + $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo --add-packages $packageId + } elseif ($packageType -eq 'deb') { + $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo $distribution --add-packages $packageId + } + } + catch { + $errorMessage.Add("Invoking update for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") + continue + } + + $state = $rawUpdateResponse.state + if ($state -ne 'Completed') { + $errorMessage.Add("Publishing package $($finalPackage.PackageName) to $pkgRepo failed: $rawUpdateResponse") + continue + } + } + + # PUBLISH PMC COMMAND + # The CLI outputs messages and JSON in the same stream, so we must sift through it for now + # This is planned to be fixed with a switch in a later release + # TODO: Anam, figure out if this was fixed, and if so lets use this switch + Write-Verbose -Verbose ([pscustomobject]($package + @{ + PackageId = $packageId + })) + + # At this point, the changes are staged and will eventually be publish. + # Running publish, causes them to go live "immediately" + try { + pmc --config $configPath repo publish $pkgRepo + } + catch { + $errorMessage.Add("Running final publish for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") + continue + } + } else { + Write-Verbose -Verbose "Skipping Uploading package --config-file '$configPath' package add '$packagePath' --repoID '$pkgRepo'" + } + } + + if ($errorMessage) { + throw $errorMessage -join [Environment]::NewLine + } } catch { Write-Error -ErrorAction Stop $_.Exception.Message From 6c8a4746db3c39ac85264bbda399d65a88abb474 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 16:12:08 -0500 Subject: [PATCH 286/316] fix typo --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index b524b72eb02..e43b254169a 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -219,7 +219,7 @@ try { { # RHEL and CentOS packages have a tweak in the name... if ($pkgObj.PackageName.EndsWith('.rpm')) { #TODO: can I do this in the 1 condensed method - $pkgObjName = $PackageName.Replace($releaseVersion, $releaseVersion.Replace('-', '_')) + $pkgObjName = $pkgObj.PackageName.Replace($releaseVersion, $releaseVersion.Replace('-', '_')) } $packagePath = "$pwshPackagesFolder/$pkgObjName" From d63d4449a2a296f0dbf197411fd0369d57f6501d Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 16:40:35 -0500 Subject: [PATCH 287/316] trim 'v' off release version --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index e43b254169a..cac1402a2b2 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -89,7 +89,7 @@ try { $channel = "" $packageNames = @() $metadataContent = Get-Content -Path $metadataFilePath | ConvertFrom-Json - $releaseVersion = $metadataContent.ReleaseTag + $releaseVersion = $metadataContent.ReleaseTag.TrimStart('v') $skipPublish = $metadataContent.SkipPublish $lts = $metadataContent.LTS Write-Verbose -Verbose "skip publish: $skipPublish" #TODO: remove From ea4071a7eafcbc0fb14a36c02344e0bc89eeb5b8 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 17:10:18 -0500 Subject: [PATCH 288/316] fix the testing version --- .pipelines/templates/release-prep-for-ev2.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 3420c2987a5..12134702516 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -37,8 +37,6 @@ stages: restorePhase: true - pwsh: | - $sPubValue = '${{ parameters.skipPublish }}' - Write-Verbose -Verbose "skip publish value: $sPubValue" $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" Write-Host "sending " + $vstsCommandString @@ -172,7 +170,7 @@ stages: $metadataHash = @{} $skipPublishValue = '${{ parameters.skipPublish }}' # $metadataHash["ReleaseTag"] = "$(ReleaseTag)" # TODO: UNCOMMENT - $metadataHash["ReleaseTag"] = "v7.5.0.1" + $metadataHash["ReleaseTag"] = "v7.5.0" $metadataHash["LTS"] = $metadata.LTSRelease.Latest $metadataHash["ForProduction"] = $true $metadataHash["SkipPublish"] = [System.Convert]::ToBoolean($skipPublishValue) From 45df76f28708438440c5f22d23f8e86609be568f Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Fri, 17 Jan 2025 18:31:25 -0500 Subject: [PATCH 289/316] clean up code into methods --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 552 +++++++++++++----- 1 file changed, 407 insertions(+), 145 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index cac1402a2b2..3ab67c69d71 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -1,128 +1,25 @@ -function Get-Info { +<# +This function gets info from pmc's derived list of all repositories and from mapping.json (which contains info on just the repositories powershell publishes packages to, their package formats, etc) +to create a list of repositories PowerShell cares about along with repository Ids, repository full Urls and associated package that will be published to it. +#> +function Get-MappedRepositoryIds { param( - [string] - $PkgName + [Parameter(Mandatory)] + [hashtable] + $Mapping, + + [Parameter(Mandatory)] + $RepoList, + + # LTS is not consider a package in this context. + # LTS is just another package name. + [Parameter(Mandatory)] + [ValidateSet('stable', 'preview')] + $Channel ) - Write-Verbose -Verbose "pkg name: $pkgName" -} - -if ($null -eq $env:MAPPING_FILE) -{ - Write-Verbose -Verbose "MAPPING_FILE variable didn't get passed correctly" - return 1 -} - -if ($null -eq $env:PWSH_PACKAGES_TARGZIP) -{ - Write-Verbose -Verbose "PWSH_PACKAGES_TARGZIP variable didn't get passed correctly" - return 1 -} - -if ($null -eq $env:PMC_METADATA) -{ - Write-Verbose -Verbose "PMC_METADATA variable didn't get passed correctly" - return 1 -} - -try { - Write-Verbose -Verbose "Downloading files" - Invoke-WebRequest -Uri $env:MAPPING_FILE -OutFile mapping.json - Invoke-WebRequest -Uri $env:PWSH_PACKAGES_TARGZIP -OutFile packages.tar.gz - Invoke-WebRequest -Uri $env:PMC_METADATA -OutFile pmcMetadata.json - - # create variables to those paths and test them - $mappingFilePath = Join-Path "/package/unarchive/" -ChildPath "mapping.json" - $mappingFilePathExists = Test-Path $mappingFilePath - if (!$mappingFilePathExists) - { - Write-Verbose -Verbose "mapping.json expected at $mappingFilePath does not exist" - return 1 - } - - $packagesTarPath = Join-Path -Path "/package/unarchive/" -ChildPath "packages.tar.gz" - $packagesTarPathExists = Test-Path $packagesTarPath - if (!$packagesTarPathExists) - { - Write-Verbose -Verbose "packages.tar.gz expected at $packagesTarPath does not exist" - return 1 - } - - # Extract files from 'packages.tar.gz' - Write-Verbose -Verbose "---Extracting files from packages.tar.gz---" - $pwshPackagesFolder = Join-Path -Path "/package/unarchive/" -ChildPath "packages" - New-Item -Path $pwshPackagesFolder -ItemType Directory - tar -xzvf $packagesTarPath -C $pwshPackagesFolder --force-local - Get-ChildItem $pwshPackagesFolder -Recurse - - $metadataFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "pmcMetadata.json" - $metadataFilePathExists = Test-Path $metadataFilePath - if (!$metadataFilePathExists) - { - Write-Verbose -Verbose "pmcMetadata.json expected at $metadataFilePath does not exist" - return 1 - } - - # files in the extracted Run dir - $configPath = Join-Path '/package/unarchive/Run' -ChildPath 'settings.toml' - $configPathExists = Test-Path -Path $configPath - if (!$configPathExists) - { - Write-Verbose -Verbose "settings.toml expected at $configPath does not exist" - return 1 - } - - $pythonDlFolder = Join-Path '/package/unarchive/Run' -ChildPath 'python_dl' - $pyPathExists = Test-Path -Path $pythonDlFolder - if (!$pyPathExists) - { - Write-Verbose -Verbose "python_dl expected at $pythonDlFolder does not exist" - return 1 - } - - Write-Verbose -Verbose "Installing pmc-cli" - pip install --upgrade pip - pip --version --verbose - pip install /package/unarchive/Run/python_dl/*.whl - - # Get metadata - $channel = "" - $packageNames = @() - $metadataContent = Get-Content -Path $metadataFilePath | ConvertFrom-Json - $releaseVersion = $metadataContent.ReleaseTag.TrimStart('v') - $skipPublish = $metadataContent.SkipPublish - $lts = $metadataContent.LTS - Write-Verbose -Verbose "skip publish: $skipPublish" #TODO: remove - Get-Info -PkgName "testing" - - if ($releaseVersion.Contains('-')) { - $channel = 'preview' - $packageNames = @('powershell-preview') - } - else { - $channel = 'stable' - $packageNames = @('powershell') - } - - if ($lts) { - $packageNames += @('powershell-lts') - } - - Write-Verbose -Verbose "---Getting repository list---" - $rawResponse = pmc --config $configPath repo list --limit 800 - $response = $rawResponse | ConvertFrom-Json - $limit = $($response.limit) - $count = $($response.count) - Write-Verbose -Verbose "'pmc repo list' limit is: $limit and count is: $count" - $repoList = $response.results - - Write-Verbose -Verbose "---Getting package info---" - # BEGIN Get-PackageInfo() - # for the packages powershell publishes to, resolve it's mapping with the directory of repos and basically map our packages to their actual repo Ids - Write-Verbose "Reading mapping file from '$mappingFilePath'" -Verbose - $mapping = Get-Content -Raw -LiteralPath $mappingFilePath | ConvertFrom-Json -AsHashtable $mappedReposUsedByPwsh = @() - foreach ($package in $mapping.Packages) + foreach ($package in $Mapping.Packages) { Write-Verbose "package: $package" $packageChannel = $package.channel @@ -131,7 +28,7 @@ try { } Write-Verbose "package channel: $packageChannel" - if ($packageChannel -eq 'all' -or $packageChannel -eq $channel) + if ($packageChannel -eq 'all' -or $packageChannel -eq $Channel) { $repoIds = [System.Collections.Generic.List[string]]::new() $packageFormat = $package.PackageFormat @@ -160,11 +57,11 @@ try { } Write-Verbose "---Finding repo id for: $urlGlob---" -Verbose - $repos = $repoList | Where-Object { $_.name -eq $urlGlob } + $repos = $RepoList | Where-Object { $_.name -eq $urlGlob } if ($repos.id) { Write-Verbose "Found repo id: $($repos.id)" -Verbose - $repoIds.AddRange(([string[]]$repos.id)) #tbh seems like a package should only have 1 repo Id + $repoIds.AddRange(([string[]]$repos.id)) #TODO seems like a package should only have 1 repo Id } else { Write-Failure "Could not find repo for $urlGlob" @@ -178,11 +75,29 @@ try { } Write-Verbose -Verbose "mapped repos length: $($mappedReposUsedByPwsh.Length)" - # END of Get-PackageInfo() + return $mappedReposUsedByPwsh +} - # BEGIN New-RepoPackageObject() - $repoPackageObjects = @() - foreach ($pkg in $mappedReposUsedByPwsh) +<# +This function resolves the package name and powershell version in the package name format. +#> +function Get-RepositoryPackageObject() { + param( + [Parameter(Mandatory)] + [psobject[]] + $RepoObjects, + + [Parameter(Mandatory)] + [string] + $ReleaseVersion, + + [Parameter(Mandatory)] + [string[]] + $PackageName + ) + + $packageObjects = @() + foreach ($pkg in $RepoObjects) { if ($pkg.RepoId.count -gt 1) { throw "Package $($pkg.name) has more than one repo id." @@ -195,8 +110,8 @@ try { $pkgRepo = $pkg.RepoId | Select-Object -First 1 $pkgDistribution = $pkg.Distribution | Select-Object -First 1 - foreach ($name in $packageNames) { - $pkgName = $pkg.PackageFormat.Replace('PACKAGE_NAME', $name).Replace('POWERSHELL_RELEASE', $releaseVersion) + foreach ($name in $PackageName) { + $pkgName = $pkg.PackageFormat.Replace('PACKAGE_NAME', $name).Replace('POWERSHELL_RELEASE', $ReleaseVersion) Write-Verbose "Creating info object for package '$pkgName' for repo '$pkgRepo'" $result = [pscustomobject]@{ PackageName = $pkgName @@ -205,21 +120,38 @@ try { } Write-Verbose $result -Verbose - $repoPackageObjects += $result + $packageObjects += $result } } - # END of New-RepoPackageObject() - I think this and the method before can be combined - Write-Verbose -Verbose "count of repoPackageObjects: $($repoPackageObjects.Length)" + Write-Verbose -Verbose "count of repoPackageObjects: $($packageObjects.Length)" + +} + +<# +This function stages, uploads and publishes the powershell packages to their associated repositories in PMC. +#> +function Publish-PackageToPMC() { + param( + [Parameter(Mandatory)] + [pscustomobject[]] + $PackageObject, + + [Parameter(Mandatory)] + [string] + $ReleaseVersion, + + [bool] + $SkipPublish + ) - # BEGIN Publish-PackageFromBlob() $packages = @() - foreach ($pkgObj in $repoPackageObjects) + foreach ($pkgObj in $PackageObject) { # RHEL and CentOS packages have a tweak in the name... if ($pkgObj.PackageName.EndsWith('.rpm')) { #TODO: can I do this in the 1 condensed method - $pkgObjName = $pkgObj.PackageName.Replace($releaseVersion, $releaseVersion.Replace('-', '_')) + $pkgObjName = $pkgObj.PackageName.Replace($ReleaseVersion, $ReleaseVersion.Replace('-', '_')) } $packagePath = "$pwshPackagesFolder/$pkgObjName" @@ -249,7 +181,7 @@ try { $packageType = $extension -replace '^\.' Write-Verbose "packageType: $packageType" -Verbose - $packageListJson = pmc --config $configPath package $packageType list --file $packagePath + $packageListJson = pmc --config $ConfigPath package $packageType list --file $packagePath $list = $packageListJson | ConvertFrom-Json $packageId = @() @@ -260,10 +192,10 @@ try { } else { # PMC UPDATE COMMAND - Write-Verbose -Verbose "Uploading package, config: '$configPath' package: '$packagePath'" + Write-Verbose -Verbose "Uploading package, config: '$ConfigPath' package: '$packagePath'" $uploadResult = $null try { - $uploadResult = pmc --config $configPath package upload $packagePath --type $packageType + $uploadResult = pmc --config $ConfigPath package upload $packagePath --type $packageType } catch { $errorMessage.Add("Uploading package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") @@ -277,7 +209,7 @@ try { $distribution = $finalPackage.Distribution | select-object -First 1 Write-Verbose "distribution: $distribution" -Verbose - if (!$skipPublish) + if (!$SkipPublish) { Write-Verbose "---Publishing package: $($finalPackage.PackageName) to $pkgRepo---" -Verbose @@ -291,9 +223,9 @@ try { $rawUpdateResponse = $null try { if ($packageType -eq 'rpm') { - $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo --add-packages $packageId + $rawUpdateResponse = pmc --config $ConfigPath repo package update $pkgRepo --add-packages $packageId } elseif ($packageType -eq 'deb') { - $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo $distribution --add-packages $packageId + $rawUpdateResponse = pmc --config $ConfigPath repo package update $pkgRepo $distribution --add-packages $packageId } } catch { @@ -311,7 +243,6 @@ try { # PUBLISH PMC COMMAND # The CLI outputs messages and JSON in the same stream, so we must sift through it for now # This is planned to be fixed with a switch in a later release - # TODO: Anam, figure out if this was fixed, and if so lets use this switch Write-Verbose -Verbose ([pscustomobject]($package + @{ PackageId = $packageId })) @@ -319,14 +250,14 @@ try { # At this point, the changes are staged and will eventually be publish. # Running publish, causes them to go live "immediately" try { - pmc --config $configPath repo publish $pkgRepo + pmc --config $ConfigPath repo publish $pkgRepo } catch { $errorMessage.Add("Running final publish for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") continue } } else { - Write-Verbose -Verbose "Skipping Uploading package --config-file '$configPath' package add '$packagePath' --repoID '$pkgRepo'" + Write-Verbose -Verbose "Skipping Uploading package --config-file '$ConfigPath' package add '$packagePath' --repoID '$pkgRepo'" } } @@ -334,6 +265,337 @@ try { throw $errorMessage -join [Environment]::NewLine } } + +if ($null -eq $env:MAPPING_FILE) +{ + Write-Verbose -Verbose "MAPPING_FILE variable didn't get passed correctly" + return 1 +} + +if ($null -eq $env:PWSH_PACKAGES_TARGZIP) +{ + Write-Verbose -Verbose "PWSH_PACKAGES_TARGZIP variable didn't get passed correctly" + return 1 +} + +if ($null -eq $env:PMC_METADATA) +{ + Write-Verbose -Verbose "PMC_METADATA variable didn't get passed correctly" + return 1 +} + +try { + Write-Verbose -Verbose "Downloading files" + Invoke-WebRequest -Uri $env:MAPPING_FILE -OutFile mapping.json + Invoke-WebRequest -Uri $env:PWSH_PACKAGES_TARGZIP -OutFile packages.tar.gz + Invoke-WebRequest -Uri $env:PMC_METADATA -OutFile pmcMetadata.json + + # create variables to those paths and test them + $mappingFilePath = Join-Path "/package/unarchive/" -ChildPath "mapping.json" + $mappingFilePathExists = Test-Path $mappingFilePath + if (!$mappingFilePathExists) + { + Write-Verbose -Verbose "mapping.json expected at $mappingFilePath does not exist" + return 1 + } + + $packagesTarPath = Join-Path -Path "/package/unarchive/" -ChildPath "packages.tar.gz" + $packagesTarPathExists = Test-Path $packagesTarPath + if (!$packagesTarPathExists) + { + Write-Verbose -Verbose "packages.tar.gz expected at $packagesTarPath does not exist" + return 1 + } + + # Extract files from 'packages.tar.gz' + Write-Verbose -Verbose "---Extracting files from packages.tar.gz---" + $pwshPackagesFolder = Join-Path -Path "/package/unarchive/" -ChildPath "packages" + New-Item -Path $pwshPackagesFolder -ItemType Directory + tar -xzvf $packagesTarPath -C $pwshPackagesFolder --force-local + Get-ChildItem $pwshPackagesFolder -Recurse + + $metadataFilePath = Join-Path -Path "/package/unarchive/" -ChildPath "pmcMetadata.json" + $metadataFilePathExists = Test-Path $metadataFilePath + if (!$metadataFilePathExists) + { + Write-Verbose -Verbose "pmcMetadata.json expected at $metadataFilePath does not exist" + return 1 + } + + # files in the extracted Run dir + $configPath = Join-Path '/package/unarchive/Run' -ChildPath 'settings.toml' + $configPathExists = Test-Path -Path $configPath + if (!$configPathExists) + { + Write-Verbose -Verbose "settings.toml expected at $configPath does not exist" + return 1 + } + + $pythonDlFolder = Join-Path '/package/unarchive/Run' -ChildPath 'python_dl' + $pyPathExists = Test-Path -Path $pythonDlFolder + if (!$pyPathExists) + { + Write-Verbose -Verbose "python_dl expected at $pythonDlFolder does not exist" + return 1 + } + + Write-Verbose -Verbose "Installing pmc-cli" + pip install --upgrade pip + pip --version --verbose + pip install /package/unarchive/Run/python_dl/*.whl + + # Get metadata + $channel = "" + $packageNames = @() + $metadataContent = Get-Content -Path $metadataFilePath | ConvertFrom-Json + $releaseVersion = $metadataContent.ReleaseTag.TrimStart('v') + $skipPublish = $metadataContent.SkipPublish + $lts = $metadataContent.LTS + Write-Verbose -Verbose "skip publish: $skipPublish" #TODO: remove + Get-Info -PkgName "testing" + + if ($releaseVersion.Contains('-')) { + $channel = 'preview' + $packageNames = @('powershell-preview') + } + else { + $channel = 'stable' + $packageNames = @('powershell') + } + + if ($lts) { + $packageNames += @('powershell-lts') + } + + Write-Verbose -Verbose "---Getting repository list---" + $rawResponse = pmc --config $configPath repo list --limit 800 + $response = $rawResponse | ConvertFrom-Json + $limit = $($response.limit) + $count = $($response.count) + Write-Verbose -Verbose "'pmc repo list' limit is: $limit and count is: $count" + $repoList = $response.results + + Write-Verbose -Verbose "---Getting package info---" + + + Write-Verbose "Reading mapping file from '$mappingFilePath'" -Verbose + $mapping = Get-Content -Raw -LiteralPath $mappingFilePath | ConvertFrom-Json -AsHashtable + $mappedReposUsedByPwsh = Get-MappedRepositoryIds -Mapping $mapping -RepoList $repoList -Channel $channel + $repoPackageObjects = Get-RepositoryPackageObject -RepoObjects $mappedReposUsedByPwsh -PackageName $packageNames -ReleaseVersion $releaseVersion + Write-Verbose -Verbose "skip publish $skipPublish" + # Publish-PackageToPMC -PackageObject $repoPackageObjects -ReleaseVersion $releaseVersion -ConfigPath $configPath -SkipPublish $skipPublish + # foreach ($package in $mapping.Packages) + # { + # Write-Verbose "package: $package" + # $packageChannel = $package.channel + # if (!$packageChannel) { + # $packageChannel = 'all' + # } + + # Write-Verbose "package channel: $packageChannel" + # if ($packageChannel -eq 'all' -or $packageChannel -eq $channel) + # { + # $repoIds = [System.Collections.Generic.List[string]]::new() + # $packageFormat = $package.PackageFormat + # Write-Verbose "package format: $packageFormat" -Verbose + # $extension = [System.io.path]::GetExtension($packageFormat) + # $packageType = $extension -replace '^\.' + + # if ($package.distribution.count -gt 1) { + # throw "Package $($package | out-string) has more than one Distribution." + # } + + # foreach ($distribution in $package.distribution) + # { + # $urlGlob = $package.url + # switch ($packageType) + # { + # 'deb' { + # $urlGlob = $urlGlob + '-apt' + # } + # 'rpm' { + # $urlGlob = $urlGlob + '-yum' + # } + # default { + # throw "Unknown package type: $packageType" + # } + # } + + # Write-Verbose "---Finding repo id for: $urlGlob---" -Verbose + # $repos = $repoList | Where-Object { $_.name -eq $urlGlob } + + # if ($repos.id) { + # Write-Verbose "Found repo id: $($repos.id)" -Verbose + # $repoIds.AddRange(([string[]]$repos.id)) #tbh seems like a package should only have 1 repo Id + # } + # else { + # Write-Failure "Could not find repo for $urlGlob" + # } + + # if ($repoIds.Count -gt 0) { + # $mappedReposUsedByPwsh += ($package + @{ "RepoId" = $repoIds.ToArray() }) + # } + # } + # } + # } + + # Write-Verbose -Verbose "mapped repos length: $($mappedReposUsedByPwsh.Length)" + # # END of Get-PackageInfo() + + # BEGIN New-RepoPackageObject() + # $repoPackageObjects = @() + # foreach ($pkg in $mappedReposUsedByPwsh) + # { + # if ($pkg.RepoId.count -gt 1) { + # throw "Package $($pkg.name) has more than one repo id." + # } + + # if ($pkg.Distribution.count -gt 1) { + # throw "Package $($pkg.name) has more than one Distribution." + # } + + # $pkgRepo = $pkg.RepoId | Select-Object -First 1 + # $pkgDistribution = $pkg.Distribution | Select-Object -First 1 + + # foreach ($name in $packageNames) { + # $pkgName = $pkg.PackageFormat.Replace('PACKAGE_NAME', $name).Replace('POWERSHELL_RELEASE', $releaseVersion) + # Write-Verbose "Creating info object for package '$pkgName' for repo '$pkgRepo'" + # $result = [pscustomobject]@{ + # PackageName = $pkgName + # RepoId = $pkgRepo + # Distribution = $pkgDistribution + # } + + # Write-Verbose $result -Verbose + # $repoPackageObjects += $result + # } + # } + # # END of New-RepoPackageObject() - I think this and the method before can be combined + + # Write-Verbose -Verbose "count of repoPackageObjects: $($repoPackageObjects.Length)" + + # # BEGIN Publish-PackageFromBlob() + # $packages = @() + + # foreach ($pkgObj in $repoPackageObjects) + # { + # # RHEL and CentOS packages have a tweak in the name... + # if ($pkgObj.PackageName.EndsWith('.rpm')) { #TODO: can I do this in the 1 condensed method + # $pkgObjName = $pkgObj.PackageName.Replace($releaseVersion, $releaseVersion.Replace('-', '_')) + # } + + # $packagePath = "$pwshPackagesFolder/$pkgObjName" + + # $packages += @{ + # PackagePath = $packagePath + # PackageName = $pkgObjName + # RepoId = $pkgObj.RepoId + # Distribution = $pkgObj.Distribution + # } + # } + + # # end block of Publish-PackageFromBlob() + + # # Don't fail outright when an error occurs, but instead pool them until + # # after attempting to publish every package. That way we can choose to + # # proceed for a partial failure. + # $errorMessage = [System.Collections.Generic.List[string]]::new() + # foreach ($finalPackage in $packages) + # { + # Write-Verbose "---Staging package: $($finalPackage.PackageName)---" -Verbose + # $packagePath = $finalPackage.PackagePath + # $pkgRepo = $finalPackage.RepoId + + # #TODO: should process if/else here- needed or nah? + # $extension = [System.io.path]::GetExtension($packagePath) + # $packageType = $extension -replace '^\.' + # Write-Verbose "packageType: $packageType" -Verbose + + # $packageListJson = pmc --config $configPath package $packageType list --file $packagePath + # $list = $packageListJson | ConvertFrom-Json + + # $packageId = @() + # if ($list.count -ne 0) + # { + # Write-Verbose "Package '$packagePath' already exists, skipping upload" -Verbose + # $packageId = $list.results.id | Select-Object -First 1 + # } + # else { + # # PMC UPDATE COMMAND + # Write-Verbose -Verbose "Uploading package, config: '$configPath' package: '$packagePath'" + # $uploadResult = $null + # try { + # $uploadResult = pmc --config $configPath package upload $packagePath --type $packageType + # } + # catch { + # $errorMessage.Add("Uploading package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") + # continue + # } + + # $packageId = ($uploadResult | ConvertFrom-Json).id + # } + + # Write-Verbose "Got package ID: '$packageId'" -Verbose + # $distribution = $finalPackage.Distribution | select-object -First 1 + # Write-Verbose "distribution: $distribution" -Verbose + + # if (!$skipPublish) + # { + # Write-Verbose "---Publishing package: $($finalPackage.PackageName) to $pkgRepo---" -Verbose + + # if (($packageType -ne 'rpm') -and ($packageType -ne 'deb')) + # { + # throw "Unsupported package type: $packageType" + # return 1 + # } + # else { + # # UPDATE PMC COMMAND + # $rawUpdateResponse = $null + # try { + # if ($packageType -eq 'rpm') { + # $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo --add-packages $packageId + # } elseif ($packageType -eq 'deb') { + # $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo $distribution --add-packages $packageId + # } + # } + # catch { + # $errorMessage.Add("Invoking update for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") + # continue + # } + + # $state = $rawUpdateResponse.state + # if ($state -ne 'Completed') { + # $errorMessage.Add("Publishing package $($finalPackage.PackageName) to $pkgRepo failed: $rawUpdateResponse") + # continue + # } + # } + + # # PUBLISH PMC COMMAND + # # The CLI outputs messages and JSON in the same stream, so we must sift through it for now + # # This is planned to be fixed with a switch in a later release + # # TODO: Anam, figure out if this was fixed, and if so lets use this switch + # Write-Verbose -Verbose ([pscustomobject]($package + @{ + # PackageId = $packageId + # })) + + # # At this point, the changes are staged and will eventually be publish. + # # Running publish, causes them to go live "immediately" + # try { + # pmc --config $configPath repo publish $pkgRepo + # } + # catch { + # $errorMessage.Add("Running final publish for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") + # continue + # } + # } else { + # Write-Verbose -Verbose "Skipping Uploading package --config-file '$configPath' package add '$packagePath' --repoID '$pkgRepo'" + # } + # } + + # if ($errorMessage) { + # throw $errorMessage -join [Environment]::NewLine + # } +} catch { Write-Error -ErrorAction Stop $_.Exception.Message return 1 From 36eb67915361d3f9879237444008ffde1086c79b Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Sat, 18 Jan 2025 02:25:07 -0500 Subject: [PATCH 290/316] remove call to dead method --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 3ab67c69d71..86b8620651a 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -352,7 +352,6 @@ try { $skipPublish = $metadataContent.SkipPublish $lts = $metadataContent.LTS Write-Verbose -Verbose "skip publish: $skipPublish" #TODO: remove - Get-Info -PkgName "testing" if ($releaseVersion.Contains('-')) { $channel = 'preview' From d1645182627d48f9b26e0663f79c51c9162059c5 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Sat, 18 Jan 2025 11:44:30 -0500 Subject: [PATCH 291/316] add final publishing code in --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 86b8620651a..8a796ecaff6 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -382,7 +382,7 @@ try { $mappedReposUsedByPwsh = Get-MappedRepositoryIds -Mapping $mapping -RepoList $repoList -Channel $channel $repoPackageObjects = Get-RepositoryPackageObject -RepoObjects $mappedReposUsedByPwsh -PackageName $packageNames -ReleaseVersion $releaseVersion Write-Verbose -Verbose "skip publish $skipPublish" - # Publish-PackageToPMC -PackageObject $repoPackageObjects -ReleaseVersion $releaseVersion -ConfigPath $configPath -SkipPublish $skipPublish + Publish-PackageToPMC -PackageObject $repoPackageObjects -ReleaseVersion $releaseVersion -ConfigPath $configPath -SkipPublish $skipPublish # foreach ($package in $mapping.Packages) # { # Write-Verbose "package: $package" From 727f6a034b5bc67afecdecc87ffa51974350bb71 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Sat, 18 Jan 2025 12:07:45 -0500 Subject: [PATCH 292/316] add return --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 8a796ecaff6..f9e423a3a21 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -125,6 +125,7 @@ function Get-RepositoryPackageObject() { } Write-Verbose -Verbose "count of repoPackageObjects: $($packageObjects.Length)" + return $packageObjects } From aec1f621acfed8531a07a34131f5006d47550379 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Sat, 18 Jan 2025 12:24:16 -0500 Subject: [PATCH 293/316] add missing param --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index f9e423a3a21..ee62d186a56 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -138,10 +138,15 @@ function Publish-PackageToPMC() { [pscustomobject[]] $PackageObject, + [Parameter(Mandatory)] + [string] + $ConfigPath, + [Parameter(Mandatory)] [string] $ReleaseVersion, + [Parameter(Mandatory)] [bool] $SkipPublish ) From 7a7ae1911887ec209efc29ebdeffd6c8dfb3f3ef Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 11:17:32 -0500 Subject: [PATCH 294/316] combine code that creates package objects into one method --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 288 ++---------------- 1 file changed, 32 insertions(+), 256 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index ee62d186a56..a35b45e5fda 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -61,7 +61,7 @@ function Get-MappedRepositoryIds { if ($repos.id) { Write-Verbose "Found repo id: $($repos.id)" -Verbose - $repoIds.AddRange(([string[]]$repos.id)) #TODO seems like a package should only have 1 repo Id + $repoIds.AddRange(([string[]]$repos.id)) } else { Write-Failure "Could not find repo for $urlGlob" @@ -79,9 +79,10 @@ function Get-MappedRepositoryIds { } <# -This function resolves the package name and powershell version in the package name format. +This function creates package objects for the packages to be published, +with the package name (ie package name format resolve with channel based PackageName and pwsh version), repoId, distribution and package path. #> -function Get-RepositoryPackageObject() { +function Get-PackageObjects() { param( [Parameter(Mandatory)] [psobject[]] @@ -96,7 +97,8 @@ function Get-RepositoryPackageObject() { $PackageName ) - $packageObjects = @() + $packages = @() + foreach ($pkg in $RepoObjects) { if ($pkg.RepoId.count -gt 1) { @@ -112,21 +114,32 @@ function Get-RepositoryPackageObject() { foreach ($name in $PackageName) { $pkgName = $pkg.PackageFormat.Replace('PACKAGE_NAME', $name).Replace('POWERSHELL_RELEASE', $ReleaseVersion) - Write-Verbose "Creating info object for package '$pkgName' for repo '$pkgRepo'" - $result = [pscustomobject]@{ - PackageName = $pkgName - RepoId = $pkgRepo + + if ($pkgName.EndsWith('.rpm')) { + $pkgName = $pkgName.Replace($ReleaseVersion, $ReleaseVersion.Replace('-', '_')) + } + + $packagePath = "$pwshPackagesFolder/$pkgName" + $packagePathExists = Test-Path -Path $packagePath + if (!$packagePathExists) + { + throw "package path $packagePath does not exist" + } + + Write-Verbose "Creating package info object for package '$pkgName' for repo '$pkgRepo'" + $packages += @{ + PackagePath = $packagePath + PackageName = $pkgName + RepoId = $pkgRepo Distribution = $pkgDistribution } - Write-Verbose $result -Verbose - $packageObjects += $result + Write-Verbose -Verbose "package info obj: Name: $pkgName RepoId: $pkgRepo Distribution: $pkgDistribution PackagePath: $packagePath" } } - Write-Verbose -Verbose "count of repoPackageObjects: $($packageObjects.Length)" - return $packageObjects - + Write-Verbose -Verbose "count of packages objects: $($packages.Length)" + return $packages } <# @@ -142,47 +155,21 @@ function Publish-PackageToPMC() { [string] $ConfigPath, - [Parameter(Mandatory)] - [string] - $ReleaseVersion, - [Parameter(Mandatory)] [bool] $SkipPublish ) - $packages = @() - - foreach ($pkgObj in $PackageObject) - { - # RHEL and CentOS packages have a tweak in the name... - if ($pkgObj.PackageName.EndsWith('.rpm')) { #TODO: can I do this in the 1 condensed method - $pkgObjName = $pkgObj.PackageName.Replace($ReleaseVersion, $ReleaseVersion.Replace('-', '_')) - } - - $packagePath = "$pwshPackagesFolder/$pkgObjName" - - $packages += @{ - PackagePath = $packagePath - PackageName = $pkgObjName - RepoId = $pkgObj.RepoId - Distribution = $pkgObj.Distribution - } - } - - # end block of Publish-PackageFromBlob() - # Don't fail outright when an error occurs, but instead pool them until # after attempting to publish every package. That way we can choose to # proceed for a partial failure. $errorMessage = [System.Collections.Generic.List[string]]::new() - foreach ($finalPackage in $packages) + foreach ($finalPackage in $PackageObject) { Write-Verbose "---Staging package: $($finalPackage.PackageName)---" -Verbose $packagePath = $finalPackage.PackagePath $pkgRepo = $finalPackage.RepoId - #TODO: should process if/else here- needed or nah? $extension = [System.io.path]::GetExtension($packagePath) $packageType = $extension -replace '^\.' Write-Verbose "packageType: $packageType" -Verbose @@ -197,7 +184,7 @@ function Publish-PackageToPMC() { $packageId = $list.results.id | Select-Object -First 1 } else { - # PMC UPDATE COMMAND + # PMC UPLOAD COMMAND Write-Verbose -Verbose "Uploading package, config: '$ConfigPath' package: '$packagePath'" $uploadResult = $null try { @@ -225,7 +212,7 @@ function Publish-PackageToPMC() { return 1 } else { - # UPDATE PMC COMMAND + # PMC UPDATE COMMAND $rawUpdateResponse = $null try { if ($packageType -eq 'rpm') { @@ -246,7 +233,7 @@ function Publish-PackageToPMC() { } } - # PUBLISH PMC COMMAND + # PMC PUBLISH COMMAND # The CLI outputs messages and JSON in the same stream, so we must sift through it for now # This is planned to be fixed with a switch in a later release Write-Verbose -Verbose ([pscustomobject]($package + @{ @@ -386,220 +373,9 @@ try { Write-Verbose "Reading mapping file from '$mappingFilePath'" -Verbose $mapping = Get-Content -Raw -LiteralPath $mappingFilePath | ConvertFrom-Json -AsHashtable $mappedReposUsedByPwsh = Get-MappedRepositoryIds -Mapping $mapping -RepoList $repoList -Channel $channel - $repoPackageObjects = Get-RepositoryPackageObject -RepoObjects $mappedReposUsedByPwsh -PackageName $packageNames -ReleaseVersion $releaseVersion + $packageObjects = Get-PackageObjects -RepoObjects $mappedReposUsedByPwsh -PackageName $packageNames -ReleaseVersion $releaseVersion Write-Verbose -Verbose "skip publish $skipPublish" - Publish-PackageToPMC -PackageObject $repoPackageObjects -ReleaseVersion $releaseVersion -ConfigPath $configPath -SkipPublish $skipPublish - # foreach ($package in $mapping.Packages) - # { - # Write-Verbose "package: $package" - # $packageChannel = $package.channel - # if (!$packageChannel) { - # $packageChannel = 'all' - # } - - # Write-Verbose "package channel: $packageChannel" - # if ($packageChannel -eq 'all' -or $packageChannel -eq $channel) - # { - # $repoIds = [System.Collections.Generic.List[string]]::new() - # $packageFormat = $package.PackageFormat - # Write-Verbose "package format: $packageFormat" -Verbose - # $extension = [System.io.path]::GetExtension($packageFormat) - # $packageType = $extension -replace '^\.' - - # if ($package.distribution.count -gt 1) { - # throw "Package $($package | out-string) has more than one Distribution." - # } - - # foreach ($distribution in $package.distribution) - # { - # $urlGlob = $package.url - # switch ($packageType) - # { - # 'deb' { - # $urlGlob = $urlGlob + '-apt' - # } - # 'rpm' { - # $urlGlob = $urlGlob + '-yum' - # } - # default { - # throw "Unknown package type: $packageType" - # } - # } - - # Write-Verbose "---Finding repo id for: $urlGlob---" -Verbose - # $repos = $repoList | Where-Object { $_.name -eq $urlGlob } - - # if ($repos.id) { - # Write-Verbose "Found repo id: $($repos.id)" -Verbose - # $repoIds.AddRange(([string[]]$repos.id)) #tbh seems like a package should only have 1 repo Id - # } - # else { - # Write-Failure "Could not find repo for $urlGlob" - # } - - # if ($repoIds.Count -gt 0) { - # $mappedReposUsedByPwsh += ($package + @{ "RepoId" = $repoIds.ToArray() }) - # } - # } - # } - # } - - # Write-Verbose -Verbose "mapped repos length: $($mappedReposUsedByPwsh.Length)" - # # END of Get-PackageInfo() - - # BEGIN New-RepoPackageObject() - # $repoPackageObjects = @() - # foreach ($pkg in $mappedReposUsedByPwsh) - # { - # if ($pkg.RepoId.count -gt 1) { - # throw "Package $($pkg.name) has more than one repo id." - # } - - # if ($pkg.Distribution.count -gt 1) { - # throw "Package $($pkg.name) has more than one Distribution." - # } - - # $pkgRepo = $pkg.RepoId | Select-Object -First 1 - # $pkgDistribution = $pkg.Distribution | Select-Object -First 1 - - # foreach ($name in $packageNames) { - # $pkgName = $pkg.PackageFormat.Replace('PACKAGE_NAME', $name).Replace('POWERSHELL_RELEASE', $releaseVersion) - # Write-Verbose "Creating info object for package '$pkgName' for repo '$pkgRepo'" - # $result = [pscustomobject]@{ - # PackageName = $pkgName - # RepoId = $pkgRepo - # Distribution = $pkgDistribution - # } - - # Write-Verbose $result -Verbose - # $repoPackageObjects += $result - # } - # } - # # END of New-RepoPackageObject() - I think this and the method before can be combined - - # Write-Verbose -Verbose "count of repoPackageObjects: $($repoPackageObjects.Length)" - - # # BEGIN Publish-PackageFromBlob() - # $packages = @() - - # foreach ($pkgObj in $repoPackageObjects) - # { - # # RHEL and CentOS packages have a tweak in the name... - # if ($pkgObj.PackageName.EndsWith('.rpm')) { #TODO: can I do this in the 1 condensed method - # $pkgObjName = $pkgObj.PackageName.Replace($releaseVersion, $releaseVersion.Replace('-', '_')) - # } - - # $packagePath = "$pwshPackagesFolder/$pkgObjName" - - # $packages += @{ - # PackagePath = $packagePath - # PackageName = $pkgObjName - # RepoId = $pkgObj.RepoId - # Distribution = $pkgObj.Distribution - # } - # } - - # # end block of Publish-PackageFromBlob() - - # # Don't fail outright when an error occurs, but instead pool them until - # # after attempting to publish every package. That way we can choose to - # # proceed for a partial failure. - # $errorMessage = [System.Collections.Generic.List[string]]::new() - # foreach ($finalPackage in $packages) - # { - # Write-Verbose "---Staging package: $($finalPackage.PackageName)---" -Verbose - # $packagePath = $finalPackage.PackagePath - # $pkgRepo = $finalPackage.RepoId - - # #TODO: should process if/else here- needed or nah? - # $extension = [System.io.path]::GetExtension($packagePath) - # $packageType = $extension -replace '^\.' - # Write-Verbose "packageType: $packageType" -Verbose - - # $packageListJson = pmc --config $configPath package $packageType list --file $packagePath - # $list = $packageListJson | ConvertFrom-Json - - # $packageId = @() - # if ($list.count -ne 0) - # { - # Write-Verbose "Package '$packagePath' already exists, skipping upload" -Verbose - # $packageId = $list.results.id | Select-Object -First 1 - # } - # else { - # # PMC UPDATE COMMAND - # Write-Verbose -Verbose "Uploading package, config: '$configPath' package: '$packagePath'" - # $uploadResult = $null - # try { - # $uploadResult = pmc --config $configPath package upload $packagePath --type $packageType - # } - # catch { - # $errorMessage.Add("Uploading package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") - # continue - # } - - # $packageId = ($uploadResult | ConvertFrom-Json).id - # } - - # Write-Verbose "Got package ID: '$packageId'" -Verbose - # $distribution = $finalPackage.Distribution | select-object -First 1 - # Write-Verbose "distribution: $distribution" -Verbose - - # if (!$skipPublish) - # { - # Write-Verbose "---Publishing package: $($finalPackage.PackageName) to $pkgRepo---" -Verbose - - # if (($packageType -ne 'rpm') -and ($packageType -ne 'deb')) - # { - # throw "Unsupported package type: $packageType" - # return 1 - # } - # else { - # # UPDATE PMC COMMAND - # $rawUpdateResponse = $null - # try { - # if ($packageType -eq 'rpm') { - # $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo --add-packages $packageId - # } elseif ($packageType -eq 'deb') { - # $rawUpdateResponse = pmc --config $configPath repo package update $pkgRepo $distribution --add-packages $packageId - # } - # } - # catch { - # $errorMessage.Add("Invoking update for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") - # continue - # } - - # $state = $rawUpdateResponse.state - # if ($state -ne 'Completed') { - # $errorMessage.Add("Publishing package $($finalPackage.PackageName) to $pkgRepo failed: $rawUpdateResponse") - # continue - # } - # } - - # # PUBLISH PMC COMMAND - # # The CLI outputs messages and JSON in the same stream, so we must sift through it for now - # # This is planned to be fixed with a switch in a later release - # # TODO: Anam, figure out if this was fixed, and if so lets use this switch - # Write-Verbose -Verbose ([pscustomobject]($package + @{ - # PackageId = $packageId - # })) - - # # At this point, the changes are staged and will eventually be publish. - # # Running publish, causes them to go live "immediately" - # try { - # pmc --config $configPath repo publish $pkgRepo - # } - # catch { - # $errorMessage.Add("Running final publish for package $($finalPackage.PackageName) to $pkgRepo failed. See errors above for details.") - # continue - # } - # } else { - # Write-Verbose -Verbose "Skipping Uploading package --config-file '$configPath' package add '$packagePath' --repoID '$pkgRepo'" - # } - # } - - # if ($errorMessage) { - # throw $errorMessage -join [Environment]::NewLine - # } + Publish-PackageToPMC -PackageObject $packageObjects -ReleaseVersion $releaseVersion -ConfigPath $configPath -SkipPublish $skipPublish } catch { Write-Error -ErrorAction Stop $_.Exception.Message From ea55e85c2047489077abd8251b5d7a5dd679abd2 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 11:36:43 -0500 Subject: [PATCH 295/316] remove unused param --- .pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index a35b45e5fda..04642923355 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -375,7 +375,7 @@ try { $mappedReposUsedByPwsh = Get-MappedRepositoryIds -Mapping $mapping -RepoList $repoList -Channel $channel $packageObjects = Get-PackageObjects -RepoObjects $mappedReposUsedByPwsh -PackageName $packageNames -ReleaseVersion $releaseVersion Write-Verbose -Verbose "skip publish $skipPublish" - Publish-PackageToPMC -PackageObject $packageObjects -ReleaseVersion $releaseVersion -ConfigPath $configPath -SkipPublish $skipPublish + Publish-PackageToPMC -PackageObject $packageObjects -ConfigPath $configPath -SkipPublish $skipPublish } catch { Write-Error -ErrorAction Stop $_.Exception.Message From 153501f085f87cedd501f80025370fbd116c6ec2 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 12:30:06 -0500 Subject: [PATCH 296/316] clean up unused vars --- .../ServiceGroupRoot/Shell/Run/Run.ps1 | 1 - .pipelines/templates/release-prep-for-ev2.yml | 2 +- .pipelines/templates/release-publish-pmc.yml | 28 ++----------------- 3 files changed, 4 insertions(+), 27 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 index 04642923355..fc00c871c94 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 +++ b/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run/Run.ps1 @@ -344,7 +344,6 @@ try { $releaseVersion = $metadataContent.ReleaseTag.TrimStart('v') $skipPublish = $metadataContent.SkipPublish $lts = $metadataContent.LTS - Write-Verbose -Verbose "skip publish: $skipPublish" #TODO: remove if ($releaseVersion.Contains('-')) { $channel = 'preview' diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 12134702516..1a0889c470f 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -251,10 +251,10 @@ stages: displayName: Sign Run.ps1 - pwsh: | + # folder to tar must have: Run.ps1, settings.toml, python_dl $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" tar -cvf $pathToRunTarFile -C $srcPath ./Run - # TODO: ensure this has: Run.ps1, settings.toml, python_dl displayName: 'Create archive for the shell extension' - task: CopyFiles@2 diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 4ee755cac97..61d3296d815 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -14,25 +14,6 @@ stages: displayName: Publish to PMC pool: type: release - # variables: - # - name: runCodesignValidationInjection - # value: false - # - name: NugetSecurityAnalysisWarningLevel - # value: none - # - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE - # value: 1 - # - group: 'mscodehub-code-read-akv' - # - group: 'packages.microsoft.com' - # - name: ob_outputDirectory - # value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - # - name: ob_sdl_codeSignValidation_enabled - # value: false - # - name: ob_sdl_binskim_enabled - # value: false - # - name: ob_sdl_tsa_configFile - # value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - # - name: ob_sdl_credscan_suppressionsFile - # value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: - task: DownloadPipelineArtifact@2 @@ -48,7 +29,7 @@ stages: displayName: 'Download to get EV2 Files' - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 - displayName: 'TEST Ev2: Push to ACR' + displayName: 'TEST Ev2: Push to PMC' inputs: UseServerMonitorTask: true ConnectedServiceName: 'pmc-ev2-deployment' @@ -59,10 +40,7 @@ stages: # displayName: 'Ev2: Push to PMC' # inputs: # UseServerMonitorTask: true - # EnableStrictValidation: false - # ValidateOnly: false - # ConnectedServiceName: 'docker-deployement-test' - # ServiceRootLocation: 'LinkedArtifact' - # RolloutSpecType: 'RSPath' + # EndpointProviderType: ApprovalService + # ApprovalServiceEnvironment: Production # ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' # RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From d82a812dc46d5bb4e35d42194815089bfe87f4fa Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 13:22:08 -0500 Subject: [PATCH 297/316] clean up unused code --- .../PowerShell-Release-Official-Azure.yml | 8 ------- .pipelines/templates/release-publish-pmc.yml | 21 ++++++------------- 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index add2e2b8db6..0b6e800039d 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -100,11 +100,3 @@ extends: skipPublish: ${{ parameters.skipPublish }} - template: /.pipelines/templates/release-publish-pmc.yml@self - - # - stage: UploadBuildInfoJson - # dependsOn: - # - PrepForEV2 - # - BlobPublic - # displayName: Upload BuildInfo.json - # jobs: - # - template: /.pipelines/templates/release-upload-buildinfo.yml@self diff --git a/.pipelines/templates/release-publish-pmc.yml b/.pipelines/templates/release-publish-pmc.yml index 61d3296d815..d5454845211 100644 --- a/.pipelines/templates/release-publish-pmc.yml +++ b/.pipelines/templates/release-publish-pmc.yml @@ -1,16 +1,15 @@ stages: -# - stage: 'Prod_Release' -- stage: 'Test_Release' +- stage: 'Prod_Release' displayName: 'Deploy packages to PMC with EV2' dependsOn: - PrepForEV2 variables: - name: ob_release_environment - value: "Test" + value: "Production" - name: repoRoot value: $(Build.SourcesDirectory) jobs: - - job: Test_ReleaseJob + - job: Prod_ReleaseJob displayName: Publish to PMC pool: type: release @@ -29,18 +28,10 @@ stages: displayName: 'Download to get EV2 Files' - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 - displayName: 'TEST Ev2: Push to PMC' + displayName: 'Ev2: Push to PMC' inputs: UseServerMonitorTask: true - ConnectedServiceName: 'pmc-ev2-deployment' + EndpointProviderType: ApprovalService + ApprovalServiceEnvironment: Production ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' - - # - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 - # displayName: 'Ev2: Push to PMC' - # inputs: - # UseServerMonitorTask: true - # EndpointProviderType: ApprovalService - # ApprovalServiceEnvironment: Production - # ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' - # RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From 8fbaeaa2785f278b4f67c40df331bced7107d51f Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 13:22:58 -0500 Subject: [PATCH 298/316] use actual tag --- .pipelines/templates/release-prep-for-ev2.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 1a0889c470f..4c5aeedbdc9 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -169,8 +169,7 @@ stages: $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json $metadataHash = @{} $skipPublishValue = '${{ parameters.skipPublish }}' - # $metadataHash["ReleaseTag"] = "$(ReleaseTag)" # TODO: UNCOMMENT - $metadataHash["ReleaseTag"] = "v7.5.0" + $metadataHash["ReleaseTag"] = "$(ReleaseTag)" $metadataHash["LTS"] = $metadata.LTSRelease.Latest $metadataHash["ForProduction"] = $true $metadataHash["SkipPublish"] = [System.Convert]::ToBoolean($skipPublishValue) From 3b0dc1e07fa9d1718d8b81d5c431539204bdc5f5 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 14:14:03 -0500 Subject: [PATCH 299/316] update conflicting files --- .pipelines/PowerShell-Release-Official.yml | 21 +++--- .pipelines/templates/release-githubtasks.yml | 66 ++++++------------- .../templates/release-publish-nuget.yml | 11 ++-- .../templates/variable/release-shared.yml | 10 +++ 4 files changed, 47 insertions(+), 61 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index f025c42f460..2acdbe4d7ac 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -63,6 +63,10 @@ resources: type: git name: OneBranch.Pipelines/GovernedTemplates ref: refs/heads/main + - repository: PSInternalTools + type: git + name: PowerShellCore/Internal-PowerShellTeam-Tools + ref: refs/heads/master pipelines: - pipeline: CoOrdinatedBuildPipeline @@ -84,7 +88,7 @@ extends: featureFlags: WindowsHostVersion: Version: 2022 - Network: Netlock + Network: KS3 cloudvault: enabled: false globalSdl: @@ -110,17 +114,14 @@ extends: tsaOptionsFile: .config\tsaoptions.json stages: - - stage: DownloadPackages - displayName: 'Download Packages' - dependsOn: [] + - stage: setReleaseTagAndUploadTools + displayName: 'Set Release Tag and Upload Tools' jobs: - - template: /.pipelines/templates/release-download-packages.yml@self + - template: /.pipelines/templates/release-SetTagAndTools.yml@self - stage: msixbundle displayName: 'Create MSIX Bundle' dependsOn: [] - variables: - ob_release_environment: Test jobs: - template: /.pipelines/templates/release-create-msix.yml@self @@ -278,7 +279,7 @@ extends: - stage: PublishGitHubRelease displayName: Publish GitHub Release dependsOn: - - DownloadPackages + - setReleaseTagAndUploadTools - UpdateChangeLog variables: ob_release_environment: Production @@ -316,7 +317,9 @@ extends: - stage: PublishNuGet displayName: Publish NuGet - dependsOn: PushGitTagAndMakeDraftPublic + dependsOn: + - setReleaseTagAndUploadTools + - PushGitTagAndMakeDraftPublic variables: ob_release_environment: Production jobs: diff --git a/.pipelines/templates/release-githubtasks.yml b/.pipelines/templates/release-githubtasks.yml index 42db2b20b73..acdb45e18fb 100644 --- a/.pipelines/templates/release-githubtasks.yml +++ b/.pipelines/templates/release-githubtasks.yml @@ -8,32 +8,26 @@ jobs: templateContext: inputs: - input: pipelineArtifact - artifactName: drop_DownloadPackages_upload_packages + artifactName: drop_setReleaseTagAndUploadTools_SetTagAndTools + - input: pipelineArtifact + pipeline: PSPackagesOfficial + artifactName: drop_upload_upload_packages variables: - - template: ./variable/release-shared.yml@self + - template: ./variable/release-shared.yml@self + parameters: + RELEASETAG: $[ stageDependencies.setReleaseTagAndUploadTools.SetTagAndTools.outputs['OutputReleaseTag.releaseTag'] ] steps: - task: PowerShell@2 inputs: targetType: inline script: | - Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose + Write-Verbose -Verbose "Release Tag: $(ReleaseTag)" + Get-ChildItem Env: | Out-String -Stream | Write-Verbose -Verbose displayName: 'Capture Environment Variables' - template: release-install-pwsh.yml - - template: release-checkout-pwsh-repo.yml - - - template: release-SetReleaseTagAndContainerName.yml - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - git clone --depth 1 https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' - displayName: Clone Internal-Tools repository - - task: PowerShell@2 inputs: targetType: inline @@ -54,17 +48,6 @@ jobs: $fileContent = Get-Content -Path $OutputPath -Raw | Out-String Write-Verbose -Verbose -Message $fileContent displayName: Add sha256 hashes - - - task: PowerShell@2 - inputs: - targetType: inline - pwsh: true - script: | - $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $vstsCommandString = "vso[task.setvariable variable=ReleaseVersion]$releaseVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Set release version' - task: PowerShell@2 inputs: @@ -79,21 +62,12 @@ jobs: targetType: inline pwsh: true script: | - Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' + Import-module '$(Pipeline.Workspace)/ToolArtifact/GitHubRelease.psm1' $releaseVersion = '$(ReleaseTag)' -replace '^v','' - $semanticVersion = [System.Management.Automation.SemanticVersion]$releaseVersion - - $isPreview = $semanticVersion.PreReleaseLabel -ne $null - - $fileName = if ($isPreview) { - "preview.md" - } - else { - $semanticVersion.Major.ToString() + "." + $semanticVersion.Minor.ToString() + ".md" - } - - $filePath = "$(Pipeline.Workspace)/PowerShell/CHANGELOG/$fileName" - Write-Verbose -Verbose "Selected Log file: $filePath" + Write-Verbose -Verbose "Available modules: " + Get-Module | Write-Verbose -Verbose + + $filePath = Get-ChildItem -Path "$(Pipeline.Workspace)/CHANGELOG" -Filter '*.md' | Select-Object -First 1 -ExpandProperty FullName if (-not (Test-Path $filePath)) { throw "$filePath not found" @@ -101,14 +75,12 @@ jobs: $changelog = Get-Content -Path $filePath - $startPattern = "^## \[" + ([regex]::Escape($releaseVersion)) + "\]" - $endPattern = "^## \[{0}\.{1}\.{2}*" -f $semanticVersion.Major, $semanticVersion.Minor, $semanticVersion.Patch + $headingPattern = "^## \[\d+\.\d+\.\d+" + $headingStartLines = $changelog | Select-String -Pattern $headingPattern | Select-Object -ExpandProperty LineNumber + $startLine = $headingStartLines[0] + $endLine = $headingStartLines[1] - 1 - $clContent = $changelog | ForEach-Object { - if ($_ -match $startPattern) { $outputLine = $true } - elseif ($_ -match $endPattern) { $outputLine = $false } - if ($outputLine) { $_} - } | Out-String + $clContent = $changelog | Select-Object -Skip ($startLine-1) -First ($endLine - $startLine) | Out-String Write-Verbose -Verbose "Selected content: `n$clContent" diff --git a/.pipelines/templates/release-publish-nuget.yml b/.pipelines/templates/release-publish-nuget.yml index 7f2bace7547..98249844d4c 100644 --- a/.pipelines/templates/release-publish-nuget.yml +++ b/.pipelines/templates/release-publish-nuget.yml @@ -12,20 +12,21 @@ jobs: os: windows templateContext: inputs: + - input: pipelineArtifact + artifactName: drop_setReleaseTagAndUploadTools_SetTagAndTools - input: pipelineArtifact pipeline: PSPackagesOfficial artifactName: drop_upload_upload_packages variables: - template: ./variable/release-shared.yml@self + parameters: + VERSION: $[ stageDependencies.setReleaseTagAndUploadTools.SetTagAndTools.outputs['OutputVersion.Version'] ] steps: - template: release-install-pwsh.yml - - template: release-checkout-pwsh-repo.yml - - - template: release-SetReleaseTagAndContainerName.yml - - pwsh: | + Write-Verbose -Verbose "Version: $(Version)" Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose displayName: 'Capture Environment Variables' @@ -34,7 +35,7 @@ jobs: $null = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/release" Copy-Item "$(Pipeline.Workspace)/NuGetPackages/*.nupkg" -Destination "$(Pipeline.Workspace)/release" -Exclude "PowerShell.*.nupkg" -Force -Verbose - $releaseVersion = '$(VERSION)' + $releaseVersion = '$(Version)' $globalToolPath = "$(Pipeline.Workspace)/NuGetPackages/PowerShell.$releaseVersion.nupkg" if ($releaseVersion -notlike '*-*') { diff --git a/.pipelines/templates/variable/release-shared.yml b/.pipelines/templates/variable/release-shared.yml index 92ab56199d4..f944639a908 100644 --- a/.pipelines/templates/variable/release-shared.yml +++ b/.pipelines/templates/variable/release-shared.yml @@ -5,6 +5,12 @@ parameters: - name: SBOM type: boolean default: false + - name: RELEASETAG + type: string + default: 'Not Initialized' + - name: VERSION + type: string + default: 'Not Initialized' variables: - name: ob_signing_setup_enabled @@ -30,3 +36,7 @@ variables: value: ${{ parameters.REPOROOT }}\.config\suppress.json - name: ob_sdl_codeql_compiled_enabled value: false + - name: ReleaseTag + value: ${{ parameters.RELEASETAG }} + - name: Version + value: ${{ parameters.VERSION }} From cbdf319ee0487529a9a660d5c202b8d5e2172e9e Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 15:08:16 -0500 Subject: [PATCH 300/316] remove code changes to uploadToAzure.yml --- .pipelines/templates/uploadToAzure.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.pipelines/templates/uploadToAzure.yml b/.pipelines/templates/uploadToAzure.yml index 1ae85259d06..7156dedff7a 100644 --- a/.pipelines/templates/uploadToAzure.yml +++ b/.pipelines/templates/uploadToAzure.yml @@ -235,24 +235,6 @@ jobs: Get-ChildItem '$(Build.ArtifactStagingDirectory)/downloads' | Select-Object -ExpandProperty FullName displayName: 'Capture downloads' - # - pwsh: | - # Write-Verbose -Verbose "Copying Github Release files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" - # - # Write-Verbose -Verbose "Creating output directory for GitHub Release files: $(ob_outputDirectory)/GitHubPackages" - # New-Item -Path $(ob_outputDirectory)/GitHubPackages -ItemType Directory -Force - # Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | - # Where-Object { $_.Extension -notin '.msix', '.nupkg' } | - # ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | - # Copy-Item -Destination $(ob_outputDirectory)/GitHubPackages -Recurse - # - # Write-Verbose -Verbose "Creating output directory for NuGet packages: $(ob_outputDirectory)/NuGetPackages" - # New-Item -Path $(ob_outputDirectory)/NuGetPackages -ItemType Directory -Force - # Get-ChildItem -Path "$(Build.ArtifactStagingDirectory)/downloads/*" -Recurse | - # Where-Object { $_.Extension -eq '.nupkg' } | - # ForEach-Object { Write-Verbose -Verbose $_.FullName ; $_ } | - # Copy-Item -Destination $(ob_outputDirectory)/NuGetPackages -Recurse - # displayName: Copy downloads to Artifacts - - pwsh: | Write-Verbose -Verbose "Copying Github Release files in $(Build.ArtifactStagingDirectory)/downloads to use in Release Pipeline" From 134c76becb2ad6dac9e1e6eff5f38b78dc2c8506 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 15:12:24 -0500 Subject: [PATCH 301/316] use official template --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 0b6e800039d..f4dc2ea3491 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -65,7 +65,7 @@ resources: - releases/* extends: - template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates + template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates parameters: featureFlags: WindowsHostVersion: From 61a927162a0c4c36162f3848ea58b285bd7dddc3 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 15:18:49 -0500 Subject: [PATCH 302/316] use correct official template --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index f4dc2ea3491..deb6c6c3054 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -65,7 +65,7 @@ resources: - releases/* extends: - template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates + template: v2/OneBranch.Official.CrossPlat.yml@templates parameters: featureFlags: WindowsHostVersion: From 69093a5b35129a5f9bc0b083a8a0450afef82a65 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 15:30:27 -0500 Subject: [PATCH 303/316] remove generic guardian suppression and tip signing enabling --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 -- .pipelines/templates/release-prep-for-ev2.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index deb6c6c3054..b46625b6b85 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -91,8 +91,6 @@ extends: break: false # always break the build on binskim issues in addition to TSA upload policheck: break: true # always break the build on policheck issues. You can disable it by setting to 'false' - # suppression: - # suppressionFile: $(Build.SourcesDirectory)\.gdn\global.gdnsuppress tsaOptionsFile: .config\tsaoptions.json stages: - template: /.pipelines/templates/release-prep-for-ev2.yml@self diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 4c5aeedbdc9..0260ed5b460 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -22,8 +22,6 @@ stages: value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Parameters' - group: 'mscodehub-code-read-akv' - group: 'packages.microsoft.com' - - name: ob_tipsign_enabled - value: true # Enable TiP sign - name: ob_sdl_credscan_suppressionsFile value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json steps: From f27672be4bdc665367a1707dd2e1415d71f2964e Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 16:23:13 -0500 Subject: [PATCH 304/316] temporarily use non official template and fix how pipeline env var is called --- .pipelines/PowerShell-Release-Official-Azure.yml | 3 ++- .pipelines/templates/release-prep-for-ev2.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index b46625b6b85..b9f4ad7cc44 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -65,7 +65,8 @@ resources: - releases/* extends: - template: v2/OneBranch.Official.CrossPlat.yml@templates + # template: v2/OneBranch.Official.CrossPlat.yml@templates + template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates parameters: featureFlags: WindowsHostVersion: diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 0260ed5b460..31942e78e01 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -167,7 +167,7 @@ stages: $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json $metadataHash = @{} $skipPublishValue = '${{ parameters.skipPublish }}' - $metadataHash["ReleaseTag"] = "$(ReleaseTag)" + $metadataHash["ReleaseTag"] = '$(ReleaseTag)' $metadataHash["LTS"] = $metadata.LTSRelease.Latest $metadataHash["ForProduction"] = $true $metadataHash["SkipPublish"] = [System.Convert]::ToBoolean($skipPublishValue) From da9dbcbcce74608ff24ac40b81b23ec8474aeb63 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 16:24:11 -0500 Subject: [PATCH 305/316] fix indentation --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index b9f4ad7cc44..6f6dcbc30e8 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -66,7 +66,7 @@ resources: extends: # template: v2/OneBranch.Official.CrossPlat.yml@templates - template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates + template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates parameters: featureFlags: WindowsHostVersion: From a6dcef840629f83d0386658c2441335cae0ee3b1 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 16:25:11 -0500 Subject: [PATCH 306/316] skip calling pmc code for now --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 6f6dcbc30e8..c709ee2c4c3 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -98,4 +98,4 @@ extends: parameters: skipPublish: ${{ parameters.skipPublish }} - - template: /.pipelines/templates/release-publish-pmc.yml@self + # - template: /.pipelines/templates/release-publish-pmc.yml@self From 5c7e56ebe87af099fe3088990f87f1ea83ea335c Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 16:34:11 -0500 Subject: [PATCH 307/316] log version --- .pipelines/templates/release-prep-for-ev2.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 31942e78e01..61df62c2ce6 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -162,6 +162,8 @@ stages: ob_restore_phase: true - pwsh: | + $ver = '$(ReleaseTag)' + Write-Verbose -Verbose "ver: $ver" $pathToPMCMetadataFile = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'pmcMetadata.json' $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json From 747560b882b0fdbdd68615e74e9c7a0fb291c99b Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 16:40:49 -0500 Subject: [PATCH 308/316] log version again --- .pipelines/templates/release-prep-for-ev2.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 61df62c2ce6..57b9e2a201f 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -36,6 +36,8 @@ stages: - pwsh: | $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' + $pVersion2 = '$(releaseTag)' + Write-Verbose -Verbose "pVer2: $pVersion2" $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" Write-Host "sending " + $vstsCommandString Write-Host "##$vstsCommandString" From 484ba72d18f2f7348d9d8b37f5e85c2ac4bd9ade Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 16:52:50 -0500 Subject: [PATCH 309/316] refer to releaseTag output var correctly --- .pipelines/templates/release-prep-for-ev2.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 57b9e2a201f..3f2d196f820 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -35,8 +35,8 @@ stages: restorePhase: true - pwsh: | - $packageVersion = '$(ReleaseTag)'.ToLowerInvariant() -replace '^v','' - $pVersion2 = '$(releaseTag)' + $packageVersion = '$(OutputReleaseTag.ReleaseTag)'.ToLowerInvariant() -replace '^v','' + $pVersion2 = '$(OutputReleaseTag.ReleaseTag)' Write-Verbose -Verbose "pVer2: $pVersion2" $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" Write-Host "sending " + $vstsCommandString @@ -164,14 +164,14 @@ stages: ob_restore_phase: true - pwsh: | - $ver = '$(ReleaseTag)' + $ver = '$(OutputReleaseTag.ReleaseTag)' Write-Verbose -Verbose "ver: $ver" $pathToPMCMetadataFile = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'pmcMetadata.json' $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json $metadataHash = @{} $skipPublishValue = '${{ parameters.skipPublish }}' - $metadataHash["ReleaseTag"] = '$(ReleaseTag)' + $metadataHash["ReleaseTag"] = '$(OutputReleaseTag.ReleaseTag)' $metadataHash["LTS"] = $metadata.LTSRelease.Latest $metadataHash["ForProduction"] = $true $metadataHash["SkipPublish"] = [System.Convert]::ToBoolean($skipPublishValue) From d8bf20fb07f5a8493efc01411a15405ff4268576 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 17:00:33 -0500 Subject: [PATCH 310/316] clean up --- .pipelines/templates/release-prep-for-ev2.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 3f2d196f820..ff84cdd822a 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -36,8 +36,6 @@ stages: - pwsh: | $packageVersion = '$(OutputReleaseTag.ReleaseTag)'.ToLowerInvariant() -replace '^v','' - $pVersion2 = '$(OutputReleaseTag.ReleaseTag)' - Write-Verbose -Verbose "pVer2: $pVersion2" $vstsCommandString = "vso[task.setvariable variable=packageVersion]$packageVersion" Write-Host "sending " + $vstsCommandString Write-Host "##$vstsCommandString" @@ -178,9 +176,9 @@ stages: $metadataHash | ConvertTo-Json | Out-File $pathToPMCMetadataFile - Write-Verbose -Verbose "Copy mapping.json file to EV2Specs folder. file path at path $mappingFilePath exists: $mappingFilePathExists" $mappingFilePath = Join-Path -Path '$(repoRoot)/tools/packages.microsoft.com' -ChildPath 'mapping.json' $mappingFilePathExists = Test-Path $mappingFilePath + Write-Verbose -Verbose "Copy mapping.json file to EV2Specs folder. file path at path $mappingFilePath exists: $mappingFilePathExists" $mappingFileEV2Path = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath "mapping.json" Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path displayName: 'Create pmcScriptMetadata.json and mapping.json file' @@ -188,14 +186,6 @@ stages: ob_restore_phase: true - pwsh: | - $path = '$(Build.SourcesDirectory)/PowerShell/.pipelines' - $attemptPath = Test-Path $path - $path2 = '$(ev2ServiceGroupRootFolder)' - $attemptPath2 = Test-Path $path2 - Write-Verbose -Verbose "attempt build src dir $path exists : $attemptPath" - Write-Verbose -Verbose "attempt path 2 $path2 exists: $attemptPath2" - Get-ChildItem '$(Build.SourcesDirectory)' -Recurse - $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json' $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json $content.RolloutMetadata.Notification.Email.To = '$(PmcEV2SupportEmail)' From b5c92c475a329c0e6f84720e825dc89695079d5b Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 17:01:18 -0500 Subject: [PATCH 311/316] use official template --- .pipelines/PowerShell-Release-Official-Azure.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index c709ee2c4c3..b46625b6b85 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -65,8 +65,7 @@ resources: - releases/* extends: - # template: v2/OneBranch.Official.CrossPlat.yml@templates - template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates + template: v2/OneBranch.Official.CrossPlat.yml@templates parameters: featureFlags: WindowsHostVersion: @@ -98,4 +97,4 @@ extends: parameters: skipPublish: ${{ parameters.skipPublish }} - # - template: /.pipelines/templates/release-publish-pmc.yml@self + - template: /.pipelines/templates/release-publish-pmc.yml@self From f5e6188127842389a263fe0361f48f27fcb72615 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 18:05:21 -0500 Subject: [PATCH 312/316] fix signing errors- set linuxEsrpSigning: true --- .pipelines/PowerShell-Release-Official-Azure.yml | 1 + .pipelines/templates/release-prep-for-ev2.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index b46625b6b85..266681946d5 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -71,6 +71,7 @@ extends: WindowsHostVersion: Version: 2022 Network: Netlock + linuxEsrpSigning: true cloudvault: enabled: false globalSdl: diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index ff84cdd822a..8fb8599bb67 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -236,7 +236,7 @@ stages: - task: onebranch.pipeline.signing@1 inputs: command: 'sign' - signing_profile: 'tipsign' # external_distribution + signing_profile: external_distribution files_to_sign: '*.ps1' search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' displayName: Sign Run.ps1 From b364e45a71087f6bb93c2f706e02d4449a868419 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Tue, 21 Jan 2025 18:09:18 -0500 Subject: [PATCH 313/316] linuxEsrpSigning is a featureFlag --- .pipelines/PowerShell-Release-Official-Azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 266681946d5..db6b114d901 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -71,7 +71,7 @@ extends: WindowsHostVersion: Version: 2022 Network: Netlock - linuxEsrpSigning: true + linuxEsrpSigning: true cloudvault: enabled: false globalSdl: From 6064f323b7c6d43300cf767821bfb5723652a8b5 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 22 Jan 2025 12:46:11 -0500 Subject: [PATCH 314/316] remove reference to repoList.json which is no longer used and use created vars --- .../ServiceGroupRoot/UploadLinux.Rollout.json | 7 ----- .pipelines/templates/release-prep-for-ev2.yml | 27 +++---------------- 2 files changed, 4 insertions(+), 30 deletions(-) diff --git a/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json b/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json index bc97d04a33e..d7c75c2e216 100644 --- a/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json +++ b/.pipelines/EV2Specs/ServiceGroupRoot/UploadLinux.Rollout.json @@ -27,13 +27,6 @@ "path": "Parameters\\mapping.json" } }, - { - "name": "REPO_LIST_FILE", - "reference": - { - "path": "Parameters\\repolist.json" - } - }, { "name": "PWSH_PACKAGES_TARGZIP", "reference": diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index 8fb8599bb67..dc1208c2e47 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -64,25 +64,18 @@ stages: ob_restore_phase: true - pwsh: | - $attemptPath = Test-Path '$(Build.SourcesDirectory)/PowerShell/.pipelines' - Write-Verbose -Verbose "attempt build src dir: $attemptPath" Get-ChildItem '$(Build.SourcesDirectory)' displayName: 'Capture BuildDirectory' env: ob_restore_phase: true - pwsh: | - $attemptPath = Test-Path '$(Pipeline.Workspace)/PowerShell/.pipelines' - Write-Verbose -Verbose "attempt pipeline workspace: $attemptPath" Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose displayName: 'Capture Workspace' env: ob_restore_phase: true - pwsh: | - $path = '$(Build.SourcesDirectory)/PowerShell/.pipelines' - $attemptPath = Test-Path $path - Write-Verbose -Verbose "path exists pre: $path : $attemptPath" New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory displayName: 'Create Parameters folder under EV2Specs folder' env: @@ -100,22 +93,12 @@ stages: pip --version --verbose Write-Verbose -Verbose "Download pmc-cli to folder without installing it" - $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" - $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" - $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" + $pythonDlFolderPath = Join-Path '$(ev2ServiceGroupRootFolder)/Shell/Run' -ChildPath "python_dl" pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux_2_17_x86_64 --only-binary=:all: --verbose displayName: 'Download pmc-cli package' env: ob_restore_phase: true - - pwsh: | - Get-ChildItem '$(Pipeline.Workspace)/tools' - $repolistFile = Join-Path "$(Pipeline.Workspace)/tools/packages.microsoft.com-v4" -ChildPath "repolist.json" - Copy-Item -Path $repolistFile -Destination '$(ev2ParametersFolder)' -Verbose - displayName: 'Copy repolist.json file from Internal-PowerShellTeam-Tools to EV2 folder' - env: - ob_restore_phase: true - - download: PSPackagesOfficial artifact: 'drop_linux_package_deb' displayName: 'Download artifact containing .deb_amd64.deb file from PSPackagesOfficial triggering pipeline' @@ -162,8 +145,6 @@ stages: ob_restore_phase: true - pwsh: | - $ver = '$(OutputReleaseTag.ReleaseTag)' - Write-Verbose -Verbose "ver: $ver" $pathToPMCMetadataFile = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'pmcMetadata.json' $metadata = Get-Content -Path "$(repoRoot)/tools/metadata.json" -Raw | ConvertFrom-Json @@ -178,8 +159,8 @@ stages: $mappingFilePath = Join-Path -Path '$(repoRoot)/tools/packages.microsoft.com' -ChildPath 'mapping.json' $mappingFilePathExists = Test-Path $mappingFilePath - Write-Verbose -Verbose "Copy mapping.json file to EV2Specs folder. file path at path $mappingFilePath exists: $mappingFilePathExists" $mappingFileEV2Path = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath "mapping.json" + Write-Verbose -Verbose "Copy mapping.json file at: $mappingFilePath which exists: $mappingFilePathExists to: $mappingFileEV2Path" Copy-Item -Path $mappingFilePath -Destination $mappingFileEV2Path displayName: 'Create pmcScriptMetadata.json and mapping.json file' env: @@ -243,8 +224,8 @@ stages: - pwsh: | # folder to tar must have: Run.ps1, settings.toml, python_dl - $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' - $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" + $srcPath = Join-Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'Shell' + $pathToRunTarFile = Join-Path $srcPath -ChildPath "Run.tar" tar -cvf $pathToRunTarFile -C $srcPath ./Run displayName: 'Create archive for the shell extension' From caa8041e525cf42999e77519b8652e1cca7f71be Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 22 Jan 2025 14:33:48 -0500 Subject: [PATCH 315/316] clean up code --- .pipelines/templates/release-prep-for-ev2.yml | 3 +- .pipelines/templates/release-template-ev2.yml | 254 ++++++++++++++++++ 2 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 .pipelines/templates/release-template-ev2.yml diff --git a/.pipelines/templates/release-prep-for-ev2.yml b/.pipelines/templates/release-prep-for-ev2.yml index dc1208c2e47..cf7982cd5e1 100644 --- a/.pipelines/templates/release-prep-for-ev2.yml +++ b/.pipelines/templates/release-prep-for-ev2.yml @@ -205,9 +205,10 @@ stages: $settingFilePath = Join-Path '$(ev2ServiceGroupRootFolder)/Shell/Run' -ChildPath 'settings.toml' New-Item -Path $settingFilePath -ItemType File $pmcMIClientID = '$(PmcMIClientID)' + $pmcEndpoint = '$(PmcEndpointUrl)' Add-Content -Path $settingFilePath -Value "[default]" - Add-Content -Path $settingFilePath -Value "base_url = `"https://pmc-ingest.trafficmanager.net/api/v4`"" + Add-Content -Path $settingFilePath -Value "base_url = `"$pmcEndpoint`"" Add-Content -Path $settingFilePath -Value "auth_type = `"msi`"" Add-Content -Path $settingFilePath -Value "client_id = `"$pmcMIClientID`"" displayName: 'Create settings.toml file with MI clientId populated' diff --git a/.pipelines/templates/release-template-ev2.yml b/.pipelines/templates/release-template-ev2.yml new file mode 100644 index 00000000000..553d1c7b1ba --- /dev/null +++ b/.pipelines/templates/release-template-ev2.yml @@ -0,0 +1,254 @@ +stages: +- stage: PrepForEV2 + displayName: 'Copy and prep all files needed for EV2 stage' + jobs: + - job: CopyEV2FilesToArtifact + displayName: 'Copy EV2 Files to Artifact' + pool: + type: linux + variables: + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: repoRoot + value: '$(Build.SourcesDirectory)/PowerShell' + - name: ev2ServiceGroupRootFolder + value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot' + - name: ev2ParametersFolder + value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Parameters' + - group: 'mscodehub-code-read-akv' + - group: 'packages.microsoft.com' + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + steps: + - checkout: self ## the global setting on lfs didn't work + lfs: false + env: + ob_restore_phase: true + + - template: release-SetReleaseTagandContainerName.yml + parameters: + restorePhase: true + + - pwsh: | + Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose + displayName: 'Capture Environment Variables' + env: + ob_restore_phase: true + + - pwsh: | + Get-ChildItem '$(Build.SourcesDirectory)' + displayName: 'Capture BuildDirectory' + env: + ob_restore_phase: true + + - pwsh: | + $attemptPath = Test-Path '$(Pipeline.Workspace)/PowerShell/.pipelines' + Write-Verbose -Verbose "attempt pipeline workspace: $attemptPath" + Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose + displayName: 'Capture Workspace' + env: + ob_restore_phase: true + + - pwsh: | + $path = '$(Build.SourcesDirectory)/PowerShell/.pipelines' + $attemptPath = Test-Path $path + Write-Verbose -Verbose "path exists pre: $path : $attemptPath" + New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory + displayName: 'Create Parameters folder under EV2Specs folder' + env: + ob_restore_phase: true + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' + displayName: 'Pip Authenticate' + env: + ob_restore_phase: true + + - pwsh: | + python3 -m pip install --upgrade pip + pip --version --verbose + + Write-Verbose -Verbose "Download pmc-cli to folder without installing it" + $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" + $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" + $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" + pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux_2_17_x86_64 --only-binary=:all: --verbose + displayName: 'Download pmc-cli package' + env: + ob_restore_phase: true + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_deb' + displayName: 'Download artifact containing .deb_amd64.deb file from PSPackagesOfficial triggering pipeline' + env: + ob_restore_phase: true + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_rpm' + displayName: 'Download artifact containing .rh.x64_86.rpm file from PSPackagesOfficial triggering pipeline' + env: + ob_restore_phase: true + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_mariner_x64' + displayName: 'Download artifact containing .cm.x86_64.rpm file from PSPackagesOfficial triggering pipeline' + env: + ob_restore_phase: true + + - download: PSPackagesOfficial + artifact: 'drop_linux_package_mariner_arm64' + displayName: 'Download artifact containing .cm.aarch64.rpm file from PSPackagesOfficial triggering pipeline' + env: + ob_restore_phase: true + + - pwsh: | + Write-Verbose -Verbose "Copy ESRP signed .deb and .rpm packages" + $downloadedPipelineFolder = Join-Path '$(Pipeline.Workspace)' -ChildPath 'PSPackagesOfficial' + $srcFilesFolder = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'SourceFiles' + New-Item -Path $srcFilesFolder -ItemType Directory + $packagesFolder = Join-Path -Path $srcFilesFolder -ChildPath 'packages' + New-Item -Path $packagesFolder -ItemType Directory + + $packageFiles = Get-ChildItem -Path $downloadedPipelineFolder -Recurse -Directory -Filter "drop_*" | Get-ChildItem -File -Include *.deb, *.rpm + foreach ($file in $packageFiles) + { + Write-Verbose -Verbose "copying file: $($file.FullName)" + Copy-Item -Path $($file.FullName) -Destination $packagesFolder -Verbose + } + + $packagesTarGzDestination = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'packages.tar.gz' + tar -czvf $packagesTarGzDestination -C $packagesFolder . + displayName: 'Copy signed .deb and .rpm packages to .tar.gz to pass as a file var to shell extension' + env: + ob_restore_phase: true + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + $content.RolloutMetadata.Notification.Email.To = '$(myEV2SupportEmail)' + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 4 | Out-File $pathToJsonFile + displayName: 'Replace values in RolloutSpecPath.json' + env: + ob_restore_phase: true + + - pwsh: | + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'UploadLinux.Rollout.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + + $identityString = "/subscriptions/$(mySubscription)/resourcegroups/$(myResourceGroup)/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$(myMIName)" + $content.shellExtensions.launch.identity.userAssignedIdentities[0] = $identityString + + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 6 | Out-File $pathToJsonFile + displayName: 'Replace values in UploadLinux.Rollout.json file' + env: + ob_restore_phase: true + + - pwsh: | + # TODO: ensure myResourceGroup and mySubscription are variables in library variable group + $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'ServiceModel.json' + $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json + $content.ServiceResourceGroups[0].AzureResourceGroupName = '$(myResourceGroup)' + $content.ServiceResourceGroups[0].AzureSubscriptionId = '$(mySubscription)' + + Remove-Item -Path $pathToJsonFile + $content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile + displayName: 'Replace values in ServiceModel.json' + env: + ob_restore_phase: true + + - task: onebranch.pipeline.signing@1 + inputs: + command: 'sign' + signing_profile: external_distribution + files_to_sign: '*.ps1' + search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' + displayName: Sign Run.ps1 + + - pwsh: | + # folder to tar (ie /Run) must have: Run.ps1 and any other folders/files you want extracted + $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' + $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" + tar -cvf $pathToRunTarFile -C $srcPath ./Run + displayName: 'Create archive for the shell extension' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(repoRoot)/.pipelines' + Contents: 'EV2Specs/**' + TargetFolder: $(ob_outputDirectory) + +- stage: 'Test_Release' + displayName: 'Test deploy packages to PMC with EV2' + dependsOn: + - PrepForEV2 + variables: + - name: ob_release_environment + value: "Test" + - name: repoRoot + value: $(Build.SourcesDirectory) + jobs: + - job: Test_ReleaseJob + displayName: Publish to PMC + pool: + type: release + + steps: + - task: DownloadPipelineArtifact@2 + inputs: + targetPath: '$(Pipeline.Workspace)' + artifact: drop_PrepForEV2_CopyEv2FilesToArtifact + displayName: 'Download drop_PrepForEV2_CopyEv2FilesToArtifact artifact that has all files needed' + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + targetPath: '$(Pipeline.Workspace)' + displayName: 'Download to get EV2 Files' + + - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 + displayName: 'Ev2: Push to PMC' + inputs: + UseServerMonitorTask: true + ConnectedServiceName: 'pmc-ev2-deployment' + ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' + RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' + +- stage: 'Prod_Release' + displayName: 'Deploy packages to PMC with EV2' + dependsOn: + - PrepForEV2 + variables: + - name: ob_release_environment + value: "Production" + - name: repoRoot + value: $(Build.SourcesDirectory) + jobs: + - job: Prod_ReleaseJob + displayName: Publish to PMC + pool: + type: release + + steps: + - task: DownloadPipelineArtifact@2 + inputs: + targetPath: '$(Pipeline.Workspace)' + artifact: drop_PrepForEV2_CopyEv2FilesToArtifact + displayName: 'Download drop_PrepForEV2_CopyEv2FilesToArtifact artifact that has all files needed' + + - task: DownloadPipelineArtifact@2 + inputs: + buildType: 'current' + targetPath: '$(Pipeline.Workspace)' + displayName: 'Download to get EV2 Files' + + - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 + displayName: 'Ev2: Push to PMC' + inputs: + UseServerMonitorTask: true + EndpointProviderType: ApprovalService + ApprovalServiceEnvironment: Production + ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' + RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' From 4779f30e6d53beaf5fef4253b013b095db585538 Mon Sep 17 00:00:00 2001 From: Anam Navied Date: Wed, 22 Jan 2025 14:41:58 -0500 Subject: [PATCH 316/316] clean up code --- .pipelines/templates/release-template-ev2.yml | 254 ------------------ 1 file changed, 254 deletions(-) delete mode 100644 .pipelines/templates/release-template-ev2.yml diff --git a/.pipelines/templates/release-template-ev2.yml b/.pipelines/templates/release-template-ev2.yml deleted file mode 100644 index 553d1c7b1ba..00000000000 --- a/.pipelines/templates/release-template-ev2.yml +++ /dev/null @@ -1,254 +0,0 @@ -stages: -- stage: PrepForEV2 - displayName: 'Copy and prep all files needed for EV2 stage' - jobs: - - job: CopyEV2FilesToArtifact - displayName: 'Copy EV2 Files to Artifact' - pool: - type: linux - variables: - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: repoRoot - value: '$(Build.SourcesDirectory)/PowerShell' - - name: ev2ServiceGroupRootFolder - value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot' - - name: ev2ParametersFolder - value: '$(Build.SourcesDirectory)/PowerShell/.pipelines/EV2Specs/ServiceGroupRoot/Parameters' - - group: 'mscodehub-code-read-akv' - - group: 'packages.microsoft.com' - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - steps: - - checkout: self ## the global setting on lfs didn't work - lfs: false - env: - ob_restore_phase: true - - - template: release-SetReleaseTagandContainerName.yml - parameters: - restorePhase: true - - - pwsh: | - Get-ChildItem Env: | Out-String -Stream | write-Verbose -Verbose - displayName: 'Capture Environment Variables' - env: - ob_restore_phase: true - - - pwsh: | - Get-ChildItem '$(Build.SourcesDirectory)' - displayName: 'Capture BuildDirectory' - env: - ob_restore_phase: true - - - pwsh: | - $attemptPath = Test-Path '$(Pipeline.Workspace)/PowerShell/.pipelines' - Write-Verbose -Verbose "attempt pipeline workspace: $attemptPath" - Get-ChildItem '$(Pipeline.Workspace)' -Recurse | Out-String -Stream | write-Verbose -Verbose - displayName: 'Capture Workspace' - env: - ob_restore_phase: true - - - pwsh: | - $path = '$(Build.SourcesDirectory)/PowerShell/.pipelines' - $attemptPath = Test-Path $path - Write-Verbose -Verbose "path exists pre: $path : $attemptPath" - New-Item -Path '$(ev2ParametersFolder)' -ItemType Directory - displayName: 'Create Parameters folder under EV2Specs folder' - env: - ob_restore_phase: true - - - task: PipAuthenticate@1 - inputs: - artifactFeeds: 'PowerShellCore/PowerShellCore_PublicPackages' - displayName: 'Pip Authenticate' - env: - ob_restore_phase: true - - - pwsh: | - python3 -m pip install --upgrade pip - pip --version --verbose - - Write-Verbose -Verbose "Download pmc-cli to folder without installing it" - $ev2SpecsFolder = "$(repoRoot)/.pipelines/EV2Specs" - $shellExtensionBundleDir = "$ev2SpecsFolder/ServiceGroupRoot/Shell/Run" - $pythonDlFolderPath = Join-Path -Path $shellExtensionBundleDir -ChildPath "python_dl" - pip download -d $pythonDlFolderPath pmc-cli --platform=manylinux_2_17_x86_64 --only-binary=:all: --verbose - displayName: 'Download pmc-cli package' - env: - ob_restore_phase: true - - - download: PSPackagesOfficial - artifact: 'drop_linux_package_deb' - displayName: 'Download artifact containing .deb_amd64.deb file from PSPackagesOfficial triggering pipeline' - env: - ob_restore_phase: true - - - download: PSPackagesOfficial - artifact: 'drop_linux_package_rpm' - displayName: 'Download artifact containing .rh.x64_86.rpm file from PSPackagesOfficial triggering pipeline' - env: - ob_restore_phase: true - - - download: PSPackagesOfficial - artifact: 'drop_linux_package_mariner_x64' - displayName: 'Download artifact containing .cm.x86_64.rpm file from PSPackagesOfficial triggering pipeline' - env: - ob_restore_phase: true - - - download: PSPackagesOfficial - artifact: 'drop_linux_package_mariner_arm64' - displayName: 'Download artifact containing .cm.aarch64.rpm file from PSPackagesOfficial triggering pipeline' - env: - ob_restore_phase: true - - - pwsh: | - Write-Verbose -Verbose "Copy ESRP signed .deb and .rpm packages" - $downloadedPipelineFolder = Join-Path '$(Pipeline.Workspace)' -ChildPath 'PSPackagesOfficial' - $srcFilesFolder = Join-Path -Path '$(Pipeline.Workspace)' -ChildPath 'SourceFiles' - New-Item -Path $srcFilesFolder -ItemType Directory - $packagesFolder = Join-Path -Path $srcFilesFolder -ChildPath 'packages' - New-Item -Path $packagesFolder -ItemType Directory - - $packageFiles = Get-ChildItem -Path $downloadedPipelineFolder -Recurse -Directory -Filter "drop_*" | Get-ChildItem -File -Include *.deb, *.rpm - foreach ($file in $packageFiles) - { - Write-Verbose -Verbose "copying file: $($file.FullName)" - Copy-Item -Path $($file.FullName) -Destination $packagesFolder -Verbose - } - - $packagesTarGzDestination = Join-Path -Path '$(ev2ParametersFolder)' -ChildPath 'packages.tar.gz' - tar -czvf $packagesTarGzDestination -C $packagesFolder . - displayName: 'Copy signed .deb and .rpm packages to .tar.gz to pass as a file var to shell extension' - env: - ob_restore_phase: true - - - pwsh: | - $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'RolloutSpec.json' - $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json - $content.RolloutMetadata.Notification.Email.To = '$(myEV2SupportEmail)' - Remove-Item -Path $pathToJsonFile - $content | ConvertTo-Json -Depth 4 | Out-File $pathToJsonFile - displayName: 'Replace values in RolloutSpecPath.json' - env: - ob_restore_phase: true - - - pwsh: | - $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'UploadLinux.Rollout.json' - $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json - - $identityString = "/subscriptions/$(mySubscription)/resourcegroups/$(myResourceGroup)/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$(myMIName)" - $content.shellExtensions.launch.identity.userAssignedIdentities[0] = $identityString - - Remove-Item -Path $pathToJsonFile - $content | ConvertTo-Json -Depth 6 | Out-File $pathToJsonFile - displayName: 'Replace values in UploadLinux.Rollout.json file' - env: - ob_restore_phase: true - - - pwsh: | - # TODO: ensure myResourceGroup and mySubscription are variables in library variable group - $pathToJsonFile = Join-Path -Path '$(ev2ServiceGroupRootFolder)' -ChildPath 'ServiceModel.json' - $content = Get-Content -Path $pathToJsonFile | ConvertFrom-Json - $content.ServiceResourceGroups[0].AzureResourceGroupName = '$(myResourceGroup)' - $content.ServiceResourceGroups[0].AzureSubscriptionId = '$(mySubscription)' - - Remove-Item -Path $pathToJsonFile - $content | ConvertTo-Json -Depth 9 | Out-File $pathToJsonFile - displayName: 'Replace values in ServiceModel.json' - env: - ob_restore_phase: true - - - task: onebranch.pipeline.signing@1 - inputs: - command: 'sign' - signing_profile: external_distribution - files_to_sign: '*.ps1' - search_root: '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell/Run' - displayName: Sign Run.ps1 - - - pwsh: | - # folder to tar (ie /Run) must have: Run.ps1 and any other folders/files you want extracted - $srcPath = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot' -ChildPath 'Shell' - $pathToRunTarFile = Join-Path '$(repoRoot)/.pipelines/EV2Specs/ServiceGroupRoot/Shell' -ChildPath "Run.tar" - tar -cvf $pathToRunTarFile -C $srcPath ./Run - displayName: 'Create archive for the shell extension' - - - task: CopyFiles@2 - inputs: - SourceFolder: '$(repoRoot)/.pipelines' - Contents: 'EV2Specs/**' - TargetFolder: $(ob_outputDirectory) - -- stage: 'Test_Release' - displayName: 'Test deploy packages to PMC with EV2' - dependsOn: - - PrepForEV2 - variables: - - name: ob_release_environment - value: "Test" - - name: repoRoot - value: $(Build.SourcesDirectory) - jobs: - - job: Test_ReleaseJob - displayName: Publish to PMC - pool: - type: release - - steps: - - task: DownloadPipelineArtifact@2 - inputs: - targetPath: '$(Pipeline.Workspace)' - artifact: drop_PrepForEV2_CopyEv2FilesToArtifact - displayName: 'Download drop_PrepForEV2_CopyEv2FilesToArtifact artifact that has all files needed' - - - task: DownloadPipelineArtifact@2 - inputs: - buildType: 'current' - targetPath: '$(Pipeline.Workspace)' - displayName: 'Download to get EV2 Files' - - - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 - displayName: 'Ev2: Push to PMC' - inputs: - UseServerMonitorTask: true - ConnectedServiceName: 'pmc-ev2-deployment' - ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' - RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json' - -- stage: 'Prod_Release' - displayName: 'Deploy packages to PMC with EV2' - dependsOn: - - PrepForEV2 - variables: - - name: ob_release_environment - value: "Production" - - name: repoRoot - value: $(Build.SourcesDirectory) - jobs: - - job: Prod_ReleaseJob - displayName: Publish to PMC - pool: - type: release - - steps: - - task: DownloadPipelineArtifact@2 - inputs: - targetPath: '$(Pipeline.Workspace)' - artifact: drop_PrepForEV2_CopyEv2FilesToArtifact - displayName: 'Download drop_PrepForEV2_CopyEv2FilesToArtifact artifact that has all files needed' - - - task: DownloadPipelineArtifact@2 - inputs: - buildType: 'current' - targetPath: '$(Pipeline.Workspace)' - displayName: 'Download to get EV2 Files' - - - task: vsrm-ev2.vss-services-ev2.adm-release-task.ExpressV2Internal@1 - displayName: 'Ev2: Push to PMC' - inputs: - UseServerMonitorTask: true - EndpointProviderType: ApprovalService - ApprovalServiceEnvironment: Production - ServiceRootPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot' - RolloutSpecPath: '$(Pipeline.Workspace)/drop_PrepForEV2_CopyEV2FilesToArtifact/EV2Specs/ServiceGroupRoot/RolloutSpec.json'