From 5d9795dedaf0d3c9f687537af70d3bf715eea619 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 3 Jun 2026 16:46:54 -0500 Subject: [PATCH 01/19] initial apiscan 1ES migration --- .pipelines/1ES/apiscan-gen-notice.yml | 144 ++++++++++ .../1ES/templates/compliance/apiscan.yml | 246 ++++++++++++++++++ .../templates/compliance/generateNotice.yml | 142 ++++++++++ 3 files changed, 532 insertions(+) create mode 100644 .pipelines/1ES/apiscan-gen-notice.yml create mode 100644 .pipelines/1ES/templates/compliance/apiscan.yml create mode 100644 .pipelines/1ES/templates/compliance/generateNotice.yml diff --git a/.pipelines/1ES/apiscan-gen-notice.yml b/.pipelines/1ES/apiscan-gen-notice.yml new file mode 100644 index 0000000000..17a5b0ca77 --- /dev/null +++ b/.pipelines/1ES/apiscan-gen-notice.yml @@ -0,0 +1,144 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# 1ES PIPELINE TEMPLATES (1ES PT) MIGRATION — POC +# ----------------------------------------------- +# This is a parallel migration of `.pipelines/apiscan-gen-notice.yml` +# from OneBranch templates to 1ES Pipeline Templates. The OneBranch +# original is intentionally left untouched so the two pipelines can run +# side-by-side in ADO for validation. +# +# Translation summary (full notes captured in stored memories): +# - resources.repositories: OneBranch.Pipelines/GovernedTemplates @ refs/heads/main +# -> 1ESPipelineTemplates/1ESPipelineTemplates @ refs/tags/release +# - extends.template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates +# -> v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates +# - featureFlags: removed (Windows host version now comes from the pool image) +# - globalSdl: renamed to sdl (sub-keys nearly identical) +# - tsaOptionsFile: moved under sdl.tsa.configFile +# - pool: declared at the extends level (PowerShell1ES + PSMMS2019-Secure), +# inherited by jobs unless overridden +# - stages/templates: unchanged paths (helpers still live under /.pipelines/templates/...) +# +# Open verification items (track during POC run): +# - `PowerShell1ES` + `PSMMS2019-Secure` is Windows Server 2019, not 2022. +# If the build truly needs Win 2022, ask the pool admins to register a +# 2022 image and swap the `image:` field below. +# - Confirm `Unofficial` is the right template (vs `Official`). TSA bug +# filing is gated on CODEQL_ENABLED (master-only) so Unofficial is fine +# for non-release branches. + +name: apiscan-genNotice-1ES-$(BUILD.SOURCEBRANCHNAME)-$(Build.BuildId) +trigger: none + +parameters: + - name: FORCE_CODEQL + displayName: Debugging - Enable CodeQL and set cadence to 1 hour + type: boolean + default: false + +variables: + # PAT permissions NOTE: Declare a SymbolServerPAT variable in this group with a 'microsoft' organizanization scoped PAT with 'Symbols' Read permission. + # A PAT in the wrong org will give a single Error 203. No PAT will give a single Error 401, and individual pdbs may be missing even if permissions are correct. + - group: symbols + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] + # Defines the variables AzureFileCopySubscription, StorageAccount, StorageAccountKey, StorageResourceGroup, StorageSubscriptionName + - group: 'Azure Blob variable group' + # Defines the variables CgPat, CgOrganization, and CgProject + - group: 'ComponentGovernance' + - group: 'PoolNames' + - name: LinuxContainerImage + value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 + - name: WindowsContainerImage + value: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + - ${{ if eq(parameters['FORCE_CODEQL'],'true') }}: + # Cadence is hours before CodeQL will allow a re-upload of the database + - name: CodeQL.Cadence + value: 0 + - name: CODEQL_ENABLED + ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), eq(parameters['FORCE_CODEQL'],'true')) }}: + value: true + ${{ else }}: + value: false + - name: Codeql.TSAEnabled + value: $(CODEQL_ENABLED) + # AnalyzeInPipeline: false = upload results + # AnalyzeInPipeline: true = do not upload results + - name: Codeql.AnalyzeInPipeline + ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), eq(parameters['FORCE_CODEQL'],'true')) }}: + value: false + ${{ else }}: + value: true + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + # Pool default for all jobs in this pipeline. Mirrors the pattern used by + # `release-MakeBlobPublic.yml` and `release-upload-buildinfo.yml` already + # in this repo. Jobs may override this per-job if a different image is needed. + pool: + name: PowerShell1ES + image: PSMMS2019-Secure + os: windows + + sdl: + codeql: + compiled: + enabled: $(CODEQL_ENABLED) + tsaEnabled: $(CODEQL_ENABLED) # This enables TSA bug filing only for CodeQL 3000 + # `armory` was OneBranch-only and is not part of the 1ES PT SDL schema — + # removed during migration to avoid template validation errors. + sbom: + enabled: false + cg: + enabled: true + ignoreDirectories: '.devcontainer,demos,docker,docs,src,test,tools/packaging' + tsa: + enabled: true # 1ES publishes all SDL results to TSA. If TSA is disabled all SDL tools will be forced into 'break' build mode. + configFile: .config\tsaoptions.json # Was `tsaOptionsFile` (top-level) in OneBranch. + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + binskim: + break: true # always break the build on binskim issues in addition to TSA upload + policheck: + break: true # always break the build on policheck issues. You can disable it by setting to 'false' + # APIScan requires a non-Ready-To-Run build + apiscan: + enabled: true + softwareName: "PowerShell" # Default is repo name + versionNumber: "7.6" # Default is build number + isLargeApp: false # Default: false. + symbolsFolder: $(SymbolsServerUrl);$(ob_outputDirectory) +#softwareFolder - relative path to a folder to be scanned. Default value is root of artifacts folder + psscriptanalyzer: + enabled: true + policyName: Microsoft + break: false + + stages: + - stage: APIScan + displayName: 'ApiScan' + dependsOn: [] + jobs: + - template: /.pipelines/1ES/templates/compliance/apiscan.yml@self + parameters: + parentJobs: [] + - stage: notice + displayName: Generate Notice File + dependsOn: [] + jobs: + - template: /.pipelines/1ES/templates/compliance/generateNotice.yml@self + parameters: + parentJobs: [] diff --git a/.pipelines/1ES/templates/compliance/apiscan.yml b/.pipelines/1ES/templates/compliance/apiscan.yml new file mode 100644 index 0000000000..b238a4464a --- /dev/null +++ b/.pipelines/1ES/templates/compliance/apiscan.yml @@ -0,0 +1,246 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# 1ES PIPELINE TEMPLATES (1ES PT) MIGRATION — POC +# ----------------------------------------------- +# Parallel migration of `.pipelines/templates/compliance/apiscan.yml`. +# OneBranch original is unchanged; this copy is consumed only by the 1ES +# entry pipeline at `.pipelines/1ES/apiscan-gen-notice.yml`. +# +# Translation summary applied to this file: +# - pool: { type: windows } +# -> pool: { name: PowerShell1ES, image: PSMMS2019-Secure, os: windows } +# (job-level override; redundant with the entry pipeline default but kept +# for clarity. Drop later if we centralize the pool definition.) +# - container: VS 2022 toolchain container moved to a job-level `container:` +# declaration. NEEDS VERIFICATION that PSMMS2019-Secure agents support +# hosting Windows containers. +# - per-job `ob_sdl_*` variables moved into `templateContext.sdl` block. +# - `ob_outputDirectory` variable preserved (build steps reference it) and +# the folder is also published as a pipeline artifact via +# `templateContext.outputs`. OneBranch did this implicitly; 1ES PT requires +# it explicitly. +# - Helper template references switched from relative (`../...`) to absolute +# `@self` paths so this file can live in a different directory without +# duplicating the helpers. The helpers themselves are 1ES-safe (they only +# reference `ob_restore_phase` as a no-op env var). +# - `ob_restore_phase: true` env var on `checkout: self` dropped — 1ES PT +# runs checkout first by default and the variable has no meaning here. + +parameters: + # The OneBranch original has no `parameters:` block but the entry pipeline + # still passes `parentJobs: []` to it. Azure Pipelines silently accepts + # unknown parameters, but declaring it explicitly is cleaner and matches + # the sibling `generateNotice.yml` template. + - name: parentJobs + type: jobList + default: [] + +jobs: + - job: APIScan + dependsOn: + ${{ parameters.parentJobs }} + variables: + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: ReleaseTagVar + value: fromBranch + # Defines the variables APIScanClient, APIScanTenant and APIScanSecret + - group: PS-PS-APIScan + - name: branchCounterKey + value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])] + - name: branchCounter + value: $[counter(variables['branchCounterKey'], 1)] + - group: DotNetPrivateBuildAccess + - group: ReleasePipelineSecrets + - group: mscodehub-feed-read-general + - group: mscodehub-feed-read-akv + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + # PATH NOTE (1ES PT migration): + # In OneBranch, `checkout: self` placed the repo at + # `$(Build.SourcesDirectory)\PowerShell`. In 1ES PT, the default + # checkout lands directly at `$(Build.SourcesDirectory)`. We adjust + # `repoRoot` accordingly. `set-reporoot.yml` will respect this because + # the env var `REPOROOT` is already set from this YAML variable. + - name: repoRoot + value: '$(Build.SourcesDirectory)' + - name: Codeql.SourceRoot + value: $(repoRoot) + + pool: + name: PowerShell1ES + image: PSMMS2019-Secure + os: windows + + # CONTAINER NOTE (1ES PT migration): + # OneBranch ran this job inside `onebranch.azurecr.io/windows/ltsc2022/vse2022:latest` + # to provide the VS 2022 toolchain that `Start-PSBuild` needs. We do NOT + # reuse that container here because: + # (a) `PSMMS2019-Secure` is a Windows Server 2019 host, which cannot run + # LTSC 2022 Windows containers (host/container Windows versions must match). + # (b) Secure 1ES pools typically don't have Windows container hosting enabled. + # TODO (verify with team): either + # (a) ensure `PSMMS2019-Secure` already has the VS 2022 build toolchain installed, or + # (b) register a Windows Server 2022 image in the `PowerShell1ES` pool and swap + # the `image:` value above to it, then re-add `container:` here. + # If `Start-PSBuild` fails with "msbuild not found" or similar on the first POC run, + # option (b) is the fix. + + # APIScan can take a long time + timeoutInMinutes: 180 + + templateContext: + # Per-job SDL configuration — replaces OneBranch's `ob_sdl_*` variables. + # Paths use `$(Build.SourcesDirectory)` (no `\PowerShell` suffix) because + # 1ES PT's default checkout lands the repo directly at SourcesDirectory. + sdl: + tsa: + configFile: $(Build.SourcesDirectory)\.config\tsaoptions.json + credscan: + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + # 1ES PT requires explicit artifact declarations. The OneBranch pipeline + # implicitly published `$(ob_outputDirectory)` as an artifact named + # `drop__`; we replicate that convention here. + outputs: + - output: pipelineArtifact + displayName: 'Publish APIScan output' + targetPath: $(ob_outputDirectory) + artifactName: drop_APIScan_APIScan + condition: succeededOrFailed() + + steps: + - checkout: self + clean: true + fetchTags: true + fetchDepth: 1000 + displayName: Checkout PowerShell + retryCountOnTaskFailure: 1 + + # `ob_restore_phase` is a no-op in 1ES PT (no restore-phase concept). + # Pass `false` explicitly so the helpers don't emit confusing env vars. + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: no + ob_restore_phase: false + + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: '$(repoRoot)' + ob_restore_phase: false + + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + useGlobalJson: true + packageType: 'sdk' + workingDirectory: $(Build.SourcesDirectory)" + + - pwsh: | + Import-Module .\build.psm1 -force + Find-DotNet + dotnet tool install dotnet-symbol --tool-path $(Agent.ToolsDirectory)\tools\dotnet-symbol + $symbolToolPath = Get-ChildItem -Path $(Agent.ToolsDirectory)\tools\dotnet-symbol\dotnet-symbol.exe | Select-Object -First 1 -ExpandProperty FullName + Write-Host "##vso[task.setvariable variable=symbolToolPath]$symbolToolPath" + displayName: Install dotnet-symbol + workingDirectory: '$(repoRoot)' + retryCountOnTaskFailure: 2 + + - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. + displayName: 🔏 CodeQL 3000 Init + condition: eq(variables['CODEQL_ENABLED'], 'true') + inputs: + Language: csharp + + - pwsh: | + Import-Module .\build.psm1 -force + Find-DotNet + Start-PSBuild -Configuration StaticAnalysis -PSModuleRestore -Clean -Runtime fxdependent-win-desktop + + $OutputFolder = Split-Path (Get-PSOutput) + + Write-Verbose -Verbose -Message "Deleting ref folder from output folder" + if (Test-Path $OutputFolder/ref) { + Remove-Item -Recurse -Force $OutputFolder/ref + } + + $Destination = '$(ob_outputDirectory)' + if (-not (Test-Path $Destination)) { + Write-Verbose -Verbose -Message "Creating destination folder '$Destination'" + $null = mkdir $Destination + } + + Copy-Item -Path "$OutputFolder\*" -Destination $Destination -Recurse -Verbose + workingDirectory: '$(repoRoot)' + displayName: 'Build PowerShell Source' + + - pwsh: | + # Only keep windows runtimes + Write-Verbose -Verbose -Message "Deleting non-win-x64 runtimes ..." + Get-ChildItem -Path '$(ob_outputDirectory)\runtimes\*' | Where-Object {$_.FullName -notmatch '.*\\runtimes\\win'} | Foreach-Object { + Write-Verbose -Verbose -Message "Deleting $($_.FullName)" + Remove-Item -Path $_.FullName -Recurse -Force + } + + # Remove win-x86/arm/arm64 runtimes due to issues with those runtimes + Write-Verbose -Verbose -Message "Temporarily deleting win-x86/arm/arm64 runtimes ..." + Get-ChildItem -Path '$(ob_outputDirectory)\runtimes\*' | Where-Object {$_.FullName -match '.*\\runtimes\\win-(x86|arm)'} | Foreach-Object { + Write-Verbose -Verbose -Message "Deleting $($_.FullName)" + Remove-Item -Path $_.FullName -Recurse -Force + } + + Write-Host + Write-Verbose -Verbose -Message "Show content in 'runtimes' folder:" + Get-ChildItem -Path '$(ob_outputDirectory)\runtimes' + Write-Host + workingDirectory: '$(repoRoot)' + displayName: 'Remove unused runtimes' + + - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. + displayName: 🔏 CodeQL 3000 Finalize + condition: eq(variables['CODEQL_ENABLED'], 'true') + + - pwsh: | + Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + workingDirectory: '$(repoRoot)' + displayName: Capture Environment + condition: succeededOrFailed() + + # Explicitly download symbols for the drop since the SDL image doesn't have http://SymWeb access and APIScan cannot handle https yet. + - pwsh: | + Import-Module .\build.psm1 -force + Find-DotNet + $pat = '$(SymbolServerPAT)' + if ($pat -like '*PAT*' -or $pat -eq '') + { + throw 'No PAT defined' + } + $url = 'https://microsoft.artifacts.visualstudio.com/defaultcollection/_apis/symbol/symsrv' + $(symbolToolPath) --authenticated-server-path $(SymbolServerPAT) $url --symbols -d "$env:ob_outputDirectory\*" --recurse-subdirectories + displayName: 'Download Symbols for binaries' + retryCountOnTaskFailure: 2 + workingDirectory: '$(repoRoot)' + + - pwsh: | + Get-ChildItem '$(ob_outputDirectory)' -File -Recurse | + Foreach-Object { + [pscustomobject]@{ + Path = $_.FullName + Version = $_.VersionInfo.FileVersion + Md5Hash = (Get-FileHash -Algorithm MD5 -Path $_.FullName).Hash + Sha512Hash = (Get-FileHash -Algorithm SHA512 -Path $_.FullName).Hash + } + } | Export-Csv -Path '$(Build.SourcesDirectory)/ReleaseFileHash.csv' + workingDirectory: '$(repoRoot)' + displayName: 'Create release file hash artifact' + + - pwsh: | + Copy-Item -Path '$(Build.SourcesDirectory)/ReleaseFileHash.csv' -Destination '$(ob_outputDirectory)' -Verbose + displayName: 'Publish Build File Hash artifact' + + - pwsh: | + Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + displayName: Capture Environment + condition: succeededOrFailed() + workingDirectory: '$(repoRoot)' diff --git a/.pipelines/1ES/templates/compliance/generateNotice.yml b/.pipelines/1ES/templates/compliance/generateNotice.yml new file mode 100644 index 0000000000..bdfd9955cd --- /dev/null +++ b/.pipelines/1ES/templates/compliance/generateNotice.yml @@ -0,0 +1,142 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# 1ES PIPELINE TEMPLATES (1ES PT) MIGRATION — POC +# ----------------------------------------------- +# Parallel migration of `.pipelines/templates/compliance/generateNotice.yml`. +# OneBranch original is unchanged; this copy is consumed only by the 1ES +# entry pipeline at `.pipelines/1ES/apiscan-gen-notice.yml`. +# +# Translation summary applied to this file: +# - pool: { type: windows } +# -> pool: { name: PowerShell1ES, image: PSMMS2019-Secure, os: windows } +# - per-job `ob_sdl_*` variables moved into `templateContext.sdl` block. +# `ob_sdl_apiscan_enabled: false` becomes `templateContext.sdl.apiscan.enabled: false`. +# - `ob_outputDirectory` preserved (build steps reference it); also published +# as a pipeline artifact via `templateContext.outputs`. + +parameters: + - name: parentJobs + type: jobList + +jobs: + - job: generateNotice + displayName: Generate Notice + dependsOn: + ${{ parameters.parentJobs }} + + variables: + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT/notice' + # PATH NOTE (1ES PT migration): see header comment. 1ES default checkout + # lands the repo at `$(Build.SourcesDirectory)`, not at a `\PowerShell` subdir. + - name: repoRoot + value: '$(Build.SourcesDirectory)' + + pool: + name: PowerShell1ES + image: PSMMS2019-Secure + os: windows + + timeoutInMinutes: 15 + + templateContext: + sdl: + apiscan: + enabled: false + # Paths use `$(Build.SourcesDirectory)` (no `\PowerShell` suffix) because + # 1ES PT's default checkout lands the repo directly at SourcesDirectory. + tsa: + configFile: $(Build.SourcesDirectory)\.config\tsaoptions.json + credscan: + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + outputs: + - output: pipelineArtifact + displayName: 'Publish Notice output' + targetPath: $(ob_outputDirectory) + artifactName: drop_notice_generateNotice + condition: succeededOrFailed() + + steps: + - checkout: self + clean: true + + - pwsh: | + [string]$Branch=$env:BUILD_SOURCEBRANCH + $branchOnly = $Branch -replace '^refs/heads/'; + $branchOnly = $branchOnly -replace '[_\-]' + + if ($branchOnly -eq 'master') { + $container = 'tpn' + } else { + $branchOnly = $branchOnly -replace '[\./]', '-' + $container = "tpn-$branchOnly" + } + + $vstsCommandString = "vso[task.setvariable variable=tpnContainer]$container" + Write-Verbose -Message $vstsCommandString -Verbose + Write-Host -Object "##$vstsCommandString" + displayName: Set ContainerName + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(repoRoot)\tools\cgmanifest\tpn' + + - task: msospo.ospo-extension.8d7f9abb-6896-461d-9e25-4f74ed65ddb2.notice@0 + displayName: 'NOTICE File Generator' + inputs: + outputfile: '$(ob_outputDirectory)\ThirdPartyNotices.txt' + # output format can be html or text + outputformat: text + # this isn't working + # additionaldata: $(Build.SourcesDirectory)\assets\additionalAttributions.txt + + - pwsh: | + Get-Content -Raw -Path $(repoRoot)\assets\additionalAttributions.txt | Out-File '$(ob_outputDirectory)\ThirdPartyNotices.txt' -Encoding utf8NoBOM -Force -Append + Get-Content -Raw -Path $(repoRoot)\assets\additionalAttributions.txt + displayName: Append Additional Attributions + continueOnError: true + + - pwsh: | + Get-Content -Raw -Path '$(ob_outputDirectory)\ThirdPartyNotices.txt' + displayName: Capture Notice + continueOnError: true + + - task: AzurePowerShell@5 + displayName: Upload Notice + inputs: + azureSubscription: az-blob-cicd-infra + scriptType: inlineScript + azurePowerShellVersion: LatestVersion + workingDirectory: '$(repoRoot)' + pwsh: true + inline: | + try { + $downloadsDirectory = '$(Build.ArtifactStagingDirectory)/downloads' + $uploadedDirectory = '$(Build.ArtifactStagingDirectory)/uploaded' + $storageAccountName = "pscoretestdata" + $containerName = '$(tpnContainer)' + $blobName = 'ThirdPartyNotices.txt' + $noticePath = "$(ob_outputDirectory)\$blobName" + + Write-Verbose -Verbose "creating context ($storageAccountName) ..." + $context = New-AzStorageContext -StorageAccountName $storageAccountName -UseConnectedAccount + + Write-Verbose -Verbose "checking if container ($containerName) exists ..." + $containerExists = Get-AzStorageContainer -Name $containerName -Context $context -ErrorAction SilentlyContinue + if (-not $containerExists) { + Write-Verbose -Verbose "Creating container ..." + $null = New-AzStorageContainer -Name $containerName -Context $context + Write-Verbose -Verbose "Blob container $containerName created successfully." + } + + Write-Verbose -Verbose "Setting blob ($blobName) content ($noticePath) ..." + $null = Set-AzStorageBlobContent -File $noticePath -Container $containerName -Blob $blobName -Context $context -confirm:$false -force + Write-Verbose -Verbose "Done" + } catch { + Get-Error + throw + } From 7e96a1b77f9e2d93f15ff00d2eb05a2f4e262d5b Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 4 Jun 2026 10:09:23 -0500 Subject: [PATCH 02/19] Fix 1ES POC: switch pool image to 1ES PT-compliant MMSWindows2022-g2-Secure Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .pipelines/1ES/apiscan-gen-notice.yml | 18 ++++++++---- .../1ES/templates/compliance/apiscan.yml | 28 +++++++++---------- .../templates/compliance/generateNotice.yml | 5 ++-- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/.pipelines/1ES/apiscan-gen-notice.yml b/.pipelines/1ES/apiscan-gen-notice.yml index 17a5b0ca77..2068aeb190 100644 --- a/.pipelines/1ES/apiscan-gen-notice.yml +++ b/.pipelines/1ES/apiscan-gen-notice.yml @@ -16,14 +16,16 @@ # - featureFlags: removed (Windows host version now comes from the pool image) # - globalSdl: renamed to sdl (sub-keys nearly identical) # - tsaOptionsFile: moved under sdl.tsa.configFile -# - pool: declared at the extends level (PowerShell1ES + PSMMS2019-Secure), +# - pool: declared at the extends level (PowerShell1ES + MMSWindows2022-g2-Secure), # inherited by jobs unless overridden # - stages/templates: unchanged paths (helpers still live under /.pipelines/templates/...) # # Open verification items (track during POC run): -# - `PowerShell1ES` + `PSMMS2019-Secure` is Windows Server 2019, not 2022. -# If the build truly needs Win 2022, ask the pool admins to register a -# 2022 image and swap the `image:` field below. +# - Image was switched from `PSMMS2019-Secure` (Win 2019, not 1ES PT-compliant) +# to `MMSWindows2022-g2-Secure` (Win 2022, 1ES PT-compliant) after build 684120 +# hit `1ES PT Error: Using an image without 1es-pt-prerequisites artifact is not +# allowed`. Confirm Win 2022 toolchain on this image satisfies Start-PSBuild +# (may need a `container:` for VS 2022 toolchain — re-add at job level if needed). # - Confirm `Unofficial` is the right template (vs `Official`). TSA bug # filing is gated on CODEQL_ENABLED (master-only) so Unofficial is fine # for non-release branches. @@ -88,7 +90,13 @@ extends: # in this repo. Jobs may override this per-job if a different image is needed. pool: name: PowerShell1ES - image: PSMMS2019-Secure + # 1ES PT requires images that include the `windows-1es-pt-prerequisites-v2` + # artifact. `PSMMS2019-Secure` (used by OneBranch jobs in this repo) does + # NOT include it and causes `1ES PT Pre-Job` to hard-fail. We use + # `MMSWindows2022-g2-Secure` from the same CloudTestImages/CloudTestGallery — + # it is 1ES PT compliant and also matches the original pipeline's + # `WindowsHostVersion: 2022` intent. + image: MMSWindows2022-g2-Secure os: windows sdl: diff --git a/.pipelines/1ES/templates/compliance/apiscan.yml b/.pipelines/1ES/templates/compliance/apiscan.yml index b238a4464a..2e053968be 100644 --- a/.pipelines/1ES/templates/compliance/apiscan.yml +++ b/.pipelines/1ES/templates/compliance/apiscan.yml @@ -9,12 +9,11 @@ # # Translation summary applied to this file: # - pool: { type: windows } -# -> pool: { name: PowerShell1ES, image: PSMMS2019-Secure, os: windows } +# -> pool: { name: PowerShell1ES, image: MMSWindows2022-g2-Secure, os: windows } # (job-level override; redundant with the entry pipeline default but kept # for clarity. Drop later if we centralize the pool definition.) -# - container: VS 2022 toolchain container moved to a job-level `container:` -# declaration. NEEDS VERIFICATION that PSMMS2019-Secure agents support -# hosting Windows containers. +# - container: VS 2022 toolchain container removed in initial POC. Re-add if +# Start-PSBuild fails with "msbuild not found" (now safe — host is Win 2022). # - per-job `ob_sdl_*` variables moved into `templateContext.sdl` block. # - `ob_outputDirectory` variable preserved (build steps reference it) and # the folder is also published as a pipeline artifact via @@ -70,22 +69,21 @@ jobs: pool: name: PowerShell1ES - image: PSMMS2019-Secure + # 1ES PT compliant image (see entry pipeline for rationale). Must match + # the host OS of any container declared below. + image: MMSWindows2022-g2-Secure os: windows # CONTAINER NOTE (1ES PT migration): # OneBranch ran this job inside `onebranch.azurecr.io/windows/ltsc2022/vse2022:latest` # to provide the VS 2022 toolchain that `Start-PSBuild` needs. We do NOT - # reuse that container here because: - # (a) `PSMMS2019-Secure` is a Windows Server 2019 host, which cannot run - # LTSC 2022 Windows containers (host/container Windows versions must match). - # (b) Secure 1ES pools typically don't have Windows container hosting enabled. - # TODO (verify with team): either - # (a) ensure `PSMMS2019-Secure` already has the VS 2022 build toolchain installed, or - # (b) register a Windows Server 2022 image in the `PowerShell1ES` pool and swap - # the `image:` value above to it, then re-add `container:` here. - # If `Start-PSBuild` fails with "msbuild not found" or similar on the first POC run, - # option (b) is the fix. + # reuse that container in the initial POC because: + # (a) Secure 1ES pools typically don't have Windows container hosting enabled. + # (b) The new image `MMSWindows2022-g2-Secure` may already include the + # VS 2022 build toolchain (Windows Server 2022 MMS images commonly do). + # TODO (verify with team after first green POC run): if `Start-PSBuild` fails + # with "msbuild not found" or similar, either install the toolchain via an + # image artifact or re-add `container: vse2022` here (now safe — host is Win 2022). # APIScan can take a long time timeoutInMinutes: 180 diff --git a/.pipelines/1ES/templates/compliance/generateNotice.yml b/.pipelines/1ES/templates/compliance/generateNotice.yml index bdfd9955cd..99730e4e38 100644 --- a/.pipelines/1ES/templates/compliance/generateNotice.yml +++ b/.pipelines/1ES/templates/compliance/generateNotice.yml @@ -9,7 +9,7 @@ # # Translation summary applied to this file: # - pool: { type: windows } -# -> pool: { name: PowerShell1ES, image: PSMMS2019-Secure, os: windows } +# -> pool: { name: PowerShell1ES, image: MMSWindows2022-g2-Secure, os: windows } # - per-job `ob_sdl_*` variables moved into `templateContext.sdl` block. # `ob_sdl_apiscan_enabled: false` becomes `templateContext.sdl.apiscan.enabled: false`. # - `ob_outputDirectory` preserved (build steps reference it); also published @@ -37,7 +37,8 @@ jobs: pool: name: PowerShell1ES - image: PSMMS2019-Secure + # 1ES PT compliant image (see entry pipeline for rationale). + image: MMSWindows2022-g2-Secure os: windows timeoutInMinutes: 15 From 80650c5385596e0af5cf46994009c1cf9269417b Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 4 Jun 2026 10:17:45 -0500 Subject: [PATCH 03/19] Switch 1ES POC pool to Azure-Pipelines-1ESPT-ExDShared (windows-latest) so Pre-Job passes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .pipelines/1ES/apiscan-gen-notice.yml | 42 ++++++++++++------- .../1ES/templates/compliance/apiscan.yml | 25 ++++++----- .../templates/compliance/generateNotice.yml | 9 ++-- 3 files changed, 46 insertions(+), 30 deletions(-) diff --git a/.pipelines/1ES/apiscan-gen-notice.yml b/.pipelines/1ES/apiscan-gen-notice.yml index 2068aeb190..cea032f52d 100644 --- a/.pipelines/1ES/apiscan-gen-notice.yml +++ b/.pipelines/1ES/apiscan-gen-notice.yml @@ -16,16 +16,32 @@ # - featureFlags: removed (Windows host version now comes from the pool image) # - globalSdl: renamed to sdl (sub-keys nearly identical) # - tsaOptionsFile: moved under sdl.tsa.configFile -# - pool: declared at the extends level (PowerShell1ES + MMSWindows2022-g2-Secure), -# inherited by jobs unless overridden +# - pool: declared at the extends level (Azure-Pipelines-1ESPT-ExDShared +# + windows-latest), inherited by jobs unless overridden # - stages/templates: unchanged paths (helpers still live under /.pipelines/templates/...) # +# POOL/IMAGE DECISION (POC SCOPE): +# The original OneBranch pipeline runs on `PowerShell1ES` (team-owned curated +# pool). For the POC we deliberately switched to the Microsoft-managed +# `Azure-Pipelines-1ESPT-ExDShared` pool because: +# 1. PowerShell1ES has NO 1ES PT-compliant images registered (build 684120 +# failed with "1ES PT Error: image lacks 1es-pt-prerequisites-v2 artifact"). +# 2. Switching to a 1ES PT-compliant CloudTest image (`MMSWindows2022-g2-Secure`) +# failed in build 684335 with "Image ... doesn't exist in pool PowerShell1ES" +# — pool admin must explicitly register images; the image gallery isn't +# searched implicitly. +# 3. `Azure-Pipelines-1ESPT-ExDShared` is the standard Microsoft-managed +# 1ES PT pool (24 Win Server 2022 agents, image alias `windows-latest`, +# compliant out of the box, no team admin required). +# For production rollout, the PowerShell1ES pool admin should register a +# 1ES PT-compliant Win 2022 image (e.g., `MMSWindows2022-g2-Secure` from +# CloudTestImages/CloudTestGallery, sub 723b64f0-884d-4994-b6de-8960d049cb7e) +# so the POC can be retargeted at PowerShell1ES for full network/credential parity. +# # Open verification items (track during POC run): -# - Image was switched from `PSMMS2019-Secure` (Win 2019, not 1ES PT-compliant) -# to `MMSWindows2022-g2-Secure` (Win 2022, 1ES PT-compliant) after build 684120 -# hit `1ES PT Error: Using an image without 1es-pt-prerequisites artifact is not -# allowed`. Confirm Win 2022 toolchain on this image satisfies Start-PSBuild -# (may need a `container:` for VS 2022 toolchain — re-add at job level if needed). +# - Confirm `windows-latest` on ExDShared has the VS 2022 build toolchain +# `Start-PSBuild` needs. If not, either re-add a `container: vse2022` +# at job level OR install the toolchain inline. # - Confirm `Unofficial` is the right template (vs `Official`). TSA bug # filing is gated on CODEQL_ENABLED (master-only) so Unofficial is fine # for non-release branches. @@ -89,14 +105,10 @@ extends: # `release-MakeBlobPublic.yml` and `release-upload-buildinfo.yml` already # in this repo. Jobs may override this per-job if a different image is needed. pool: - name: PowerShell1ES - # 1ES PT requires images that include the `windows-1es-pt-prerequisites-v2` - # artifact. `PSMMS2019-Secure` (used by OneBranch jobs in this repo) does - # NOT include it and causes `1ES PT Pre-Job` to hard-fail. We use - # `MMSWindows2022-g2-Secure` from the same CloudTestImages/CloudTestGallery — - # it is 1ES PT compliant and also matches the original pipeline's - # `WindowsHostVersion: 2022` intent. - image: MMSWindows2022-g2-Secure + # See "POOL/IMAGE DECISION" in the header comment for why this POC runs on + # the Microsoft-managed ExDShared pool instead of team-owned PowerShell1ES. + name: Azure-Pipelines-1ESPT-ExDShared + image: windows-latest os: windows sdl: diff --git a/.pipelines/1ES/templates/compliance/apiscan.yml b/.pipelines/1ES/templates/compliance/apiscan.yml index 2e053968be..073cdec794 100644 --- a/.pipelines/1ES/templates/compliance/apiscan.yml +++ b/.pipelines/1ES/templates/compliance/apiscan.yml @@ -9,9 +9,10 @@ # # Translation summary applied to this file: # - pool: { type: windows } -# -> pool: { name: PowerShell1ES, image: MMSWindows2022-g2-Secure, os: windows } -# (job-level override; redundant with the entry pipeline default but kept -# for clarity. Drop later if we centralize the pool definition.) +# -> pool: { name: Azure-Pipelines-1ESPT-ExDShared, image: windows-latest, os: windows } +# (POC uses Microsoft-managed 1ES PT pool — see entry pipeline header for +# rationale. Job-level override is redundant with the entry pipeline default +# but kept for clarity. Drop later if we centralize the pool definition.) # - container: VS 2022 toolchain container removed in initial POC. Re-add if # Start-PSBuild fails with "msbuild not found" (now safe — host is Win 2022). # - per-job `ob_sdl_*` variables moved into `templateContext.sdl` block. @@ -68,22 +69,24 @@ jobs: value: $(repoRoot) pool: - name: PowerShell1ES - # 1ES PT compliant image (see entry pipeline for rationale). Must match - # the host OS of any container declared below. - image: MMSWindows2022-g2-Secure + # Microsoft-managed 1ES PT pool — see entry pipeline header for rationale. + # `windows-latest` is the Microsoft-managed alias for a 1ES PT-compliant + # Win Server 2022 image (currently build 20348). + name: Azure-Pipelines-1ESPT-ExDShared + image: windows-latest os: windows # CONTAINER NOTE (1ES PT migration): # OneBranch ran this job inside `onebranch.azurecr.io/windows/ltsc2022/vse2022:latest` # to provide the VS 2022 toolchain that `Start-PSBuild` needs. We do NOT # reuse that container in the initial POC because: - # (a) Secure 1ES pools typically don't have Windows container hosting enabled. - # (b) The new image `MMSWindows2022-g2-Secure` may already include the - # VS 2022 build toolchain (Windows Server 2022 MMS images commonly do). + # (a) Microsoft-managed 1ES PT pool images are intended to be used directly; + # container nesting adds complexity and may not be supported. + # (b) The `windows-latest` image typically includes a modern VS Build Tools + # toolchain sufficient for Start-PSBuild. # TODO (verify with team after first green POC run): if `Start-PSBuild` fails # with "msbuild not found" or similar, either install the toolchain via an - # image artifact or re-add `container: vse2022` here (now safe — host is Win 2022). + # inline step or re-add `container: vse2022` here. # APIScan can take a long time timeoutInMinutes: 180 diff --git a/.pipelines/1ES/templates/compliance/generateNotice.yml b/.pipelines/1ES/templates/compliance/generateNotice.yml index 99730e4e38..85025974b0 100644 --- a/.pipelines/1ES/templates/compliance/generateNotice.yml +++ b/.pipelines/1ES/templates/compliance/generateNotice.yml @@ -9,7 +9,8 @@ # # Translation summary applied to this file: # - pool: { type: windows } -# -> pool: { name: PowerShell1ES, image: MMSWindows2022-g2-Secure, os: windows } +# -> pool: { name: Azure-Pipelines-1ESPT-ExDShared, image: windows-latest, os: windows } +# (POC uses Microsoft-managed 1ES PT pool — see entry pipeline header for rationale.) # - per-job `ob_sdl_*` variables moved into `templateContext.sdl` block. # `ob_sdl_apiscan_enabled: false` becomes `templateContext.sdl.apiscan.enabled: false`. # - `ob_outputDirectory` preserved (build steps reference it); also published @@ -36,9 +37,9 @@ jobs: value: '$(Build.SourcesDirectory)' pool: - name: PowerShell1ES - # 1ES PT compliant image (see entry pipeline for rationale). - image: MMSWindows2022-g2-Secure + # Microsoft-managed 1ES PT pool — see entry pipeline header for rationale. + name: Azure-Pipelines-1ESPT-ExDShared + image: windows-latest os: windows timeoutInMinutes: 15 From 9552921d1e717c299e24bbc12ee46674b455ff74 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 4 Jun 2026 11:22:32 -0500 Subject: [PATCH 04/19] Retarget 1ES POC at team-owned PowerShell1ES pool with MMS2022 image The PowerShell1ES pool was migrated to the 1ES-maintained MMS2022 image (full name MMSWindows2022-1ESPT-v2) by an unmerged-but-deployed branch in PsImageFactory (PR #38413, deployed ~4 months ago). It is 1ES PT-compliant out of the box, so the previous ExDShared fallback is no longer needed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .pipelines/1ES/apiscan-gen-notice.yml | 52 +++++++++++-------- .../1ES/templates/compliance/apiscan.yml | 27 +++++----- .../templates/compliance/generateNotice.yml | 12 +++-- 3 files changed, 48 insertions(+), 43 deletions(-) diff --git a/.pipelines/1ES/apiscan-gen-notice.yml b/.pipelines/1ES/apiscan-gen-notice.yml index cea032f52d..ac98c381f7 100644 --- a/.pipelines/1ES/apiscan-gen-notice.yml +++ b/.pipelines/1ES/apiscan-gen-notice.yml @@ -16,30 +16,35 @@ # - featureFlags: removed (Windows host version now comes from the pool image) # - globalSdl: renamed to sdl (sub-keys nearly identical) # - tsaOptionsFile: moved under sdl.tsa.configFile -# - pool: declared at the extends level (Azure-Pipelines-1ESPT-ExDShared -# + windows-latest), inherited by jobs unless overridden +# - pool: declared at the extends level (PowerShell1ES +# + MMS2022), inherited by jobs unless overridden # - stages/templates: unchanged paths (helpers still live under /.pipelines/templates/...) # # POOL/IMAGE DECISION (POC SCOPE): -# The original OneBranch pipeline runs on `PowerShell1ES` (team-owned curated -# pool). For the POC we deliberately switched to the Microsoft-managed -# `Azure-Pipelines-1ESPT-ExDShared` pool because: -# 1. PowerShell1ES has NO 1ES PT-compliant images registered (build 684120 -# failed with "1ES PT Error: image lacks 1es-pt-prerequisites-v2 artifact"). -# 2. Switching to a 1ES PT-compliant CloudTest image (`MMSWindows2022-g2-Secure`) -# failed in build 684335 with "Image ... doesn't exist in pool PowerShell1ES" -# — pool admin must explicitly register images; the image gallery isn't -# searched implicitly. -# 3. `Azure-Pipelines-1ESPT-ExDShared` is the standard Microsoft-managed -# 1ES PT pool (24 Win Server 2022 agents, image alias `windows-latest`, -# compliant out of the box, no team admin required). -# For production rollout, the PowerShell1ES pool admin should register a -# 1ES PT-compliant Win 2022 image (e.g., `MMSWindows2022-g2-Secure` from -# CloudTestImages/CloudTestGallery, sub 723b64f0-884d-4994-b6de-8960d049cb7e) -# so the POC can be retargeted at PowerShell1ES for full network/credential parity. +# Targets team-owned `PowerShell1ES` pool with the 1ES-maintained image `MMS2022` +# (full name `MMSWindows2022-1ESPT-v2`, an `existing` resource managed by the 1ES +# team and registered in our CloudTest RG). It is 1ES PT-compliant out of the +# box (the `-1ESPT-v2` suffix denotes the required prerequisite artifact is baked +# in). Use is proven in production by the team's GitHub-mirror runner, which +# uses the same pool + image combination. +# +# History (kept for context): +# - Build 684120 against `PowerShell1ES + PSMMS2019-Secure` failed because that +# image lacks the 1ES PT prereq artifact. +# - Build 684335 against `PowerShell1ES + MMSWindows2022-g2-Secure` failed +# because the gallery image name does not match the name registered in the +# pool. The pool uses the local alias `MMS2022` for that image family. +# - Build 684337 ran on the Microsoft-managed `Azure-Pipelines-1ESPT-ExDShared` +# pool as a temporary workaround while the team-pool image situation was +# being sorted out. That fallback is no longer needed. +# +# NB: `main` of the team's PsImageFactory repo still shows `PowerShell1ES` mapping +# to `PSMMS2022-AzDO`. The deployed pool actually maps to `MMS2022`, set by +# PR #38413 (`addAzureCLI` branch) which was deployed but never merged. Until +# that PR is merged, `main` will misrepresent the deployed pool state. # # Open verification items (track during POC run): -# - Confirm `windows-latest` on ExDShared has the VS 2022 build toolchain +# - Confirm `MMS2022` (PowerShell1ES) ships with a VS 2022 build toolchain # `Start-PSBuild` needs. If not, either re-add a `container: vse2022` # at job level OR install the toolchain inline. # - Confirm `Unofficial` is the right template (vs `Official`). TSA bug @@ -105,10 +110,11 @@ extends: # `release-MakeBlobPublic.yml` and `release-upload-buildinfo.yml` already # in this repo. Jobs may override this per-job if a different image is needed. pool: - # See "POOL/IMAGE DECISION" in the header comment for why this POC runs on - # the Microsoft-managed ExDShared pool instead of team-owned PowerShell1ES. - name: Azure-Pipelines-1ESPT-ExDShared - image: windows-latest + # See "POOL/IMAGE DECISION" in the header comment for rationale. + # `MMS2022` is the 1ES-maintained `MMSWindows2022-1ESPT-v2` image, registered + # at 100% buffer in the PowerShell1ES pool (per PsImageFactory PR #38413). + name: PowerShell1ES + image: MMS2022 os: windows sdl: diff --git a/.pipelines/1ES/templates/compliance/apiscan.yml b/.pipelines/1ES/templates/compliance/apiscan.yml index 073cdec794..8fdb177e3c 100644 --- a/.pipelines/1ES/templates/compliance/apiscan.yml +++ b/.pipelines/1ES/templates/compliance/apiscan.yml @@ -9,8 +9,8 @@ # # Translation summary applied to this file: # - pool: { type: windows } -# -> pool: { name: Azure-Pipelines-1ESPT-ExDShared, image: windows-latest, os: windows } -# (POC uses Microsoft-managed 1ES PT pool — see entry pipeline header for +# -> pool: { name: PowerShell1ES, image: MMS2022, os: windows } +# (Team-owned pool + 1ES-maintained image — see entry pipeline header for # rationale. Job-level override is redundant with the entry pipeline default # but kept for clarity. Drop later if we centralize the pool definition.) # - container: VS 2022 toolchain container removed in initial POC. Re-add if @@ -69,24 +69,21 @@ jobs: value: $(repoRoot) pool: - # Microsoft-managed 1ES PT pool — see entry pipeline header for rationale. - # `windows-latest` is the Microsoft-managed alias for a 1ES PT-compliant - # Win Server 2022 image (currently build 20348). - name: Azure-Pipelines-1ESPT-ExDShared - image: windows-latest + # Team-owned PowerShell1ES pool with the 1ES-maintained `MMS2022` image + # (full name `MMSWindows2022-1ESPT-v2`). See entry pipeline header for + # rationale and history. + name: PowerShell1ES + image: MMS2022 os: windows # CONTAINER NOTE (1ES PT migration): # OneBranch ran this job inside `onebranch.azurecr.io/windows/ltsc2022/vse2022:latest` # to provide the VS 2022 toolchain that `Start-PSBuild` needs. We do NOT - # reuse that container in the initial POC because: - # (a) Microsoft-managed 1ES PT pool images are intended to be used directly; - # container nesting adds complexity and may not be supported. - # (b) The `windows-latest` image typically includes a modern VS Build Tools - # toolchain sufficient for Start-PSBuild. - # TODO (verify with team after first green POC run): if `Start-PSBuild` fails - # with "msbuild not found" or similar, either install the toolchain via an - # inline step or re-add `container: vse2022` here. + # reuse that container in the initial POC because the `MMS2022` image already + # ships with a modern VS Build Tools toolchain. TODO (verify with team after + # first green POC run): if `Start-PSBuild` fails with "msbuild not found" or + # similar, either install the toolchain via an inline step or re-add + # `container: vse2022` here. # APIScan can take a long time timeoutInMinutes: 180 diff --git a/.pipelines/1ES/templates/compliance/generateNotice.yml b/.pipelines/1ES/templates/compliance/generateNotice.yml index 85025974b0..8ed8d81706 100644 --- a/.pipelines/1ES/templates/compliance/generateNotice.yml +++ b/.pipelines/1ES/templates/compliance/generateNotice.yml @@ -9,8 +9,8 @@ # # Translation summary applied to this file: # - pool: { type: windows } -# -> pool: { name: Azure-Pipelines-1ESPT-ExDShared, image: windows-latest, os: windows } -# (POC uses Microsoft-managed 1ES PT pool — see entry pipeline header for rationale.) +# -> pool: { name: PowerShell1ES, image: MMS2022, os: windows } +# (Team-owned pool + 1ES-maintained image — see entry pipeline header for rationale.) # - per-job `ob_sdl_*` variables moved into `templateContext.sdl` block. # `ob_sdl_apiscan_enabled: false` becomes `templateContext.sdl.apiscan.enabled: false`. # - `ob_outputDirectory` preserved (build steps reference it); also published @@ -37,9 +37,11 @@ jobs: value: '$(Build.SourcesDirectory)' pool: - # Microsoft-managed 1ES PT pool — see entry pipeline header for rationale. - name: Azure-Pipelines-1ESPT-ExDShared - image: windows-latest + # Team-owned PowerShell1ES pool with the 1ES-maintained `MMS2022` image + # (full name `MMSWindows2022-1ESPT-v2`). See entry pipeline header for + # rationale and history. + name: PowerShell1ES + image: MMS2022 os: windows timeoutInMinutes: 15 From 1fa69605b1d33b70fe423c380a6d4e2cdb330402 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 4 Jun 2026 11:35:49 -0500 Subject: [PATCH 05/19] Fix Guardian TSAUpload by using absolute path for sdl.tsa.configFile Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .pipelines/1ES/apiscan-gen-notice.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.pipelines/1ES/apiscan-gen-notice.yml b/.pipelines/1ES/apiscan-gen-notice.yml index ac98c381f7..e1b9bfa783 100644 --- a/.pipelines/1ES/apiscan-gen-notice.yml +++ b/.pipelines/1ES/apiscan-gen-notice.yml @@ -131,7 +131,13 @@ extends: ignoreDirectories: '.devcontainer,demos,docker,docs,src,test,tools/packaging' tsa: enabled: true # 1ES publishes all SDL results to TSA. If TSA is disabled all SDL tools will be forced into 'break' build mode. - configFile: .config\tsaoptions.json # Was `tsaOptionsFile` (top-level) in OneBranch. + # 1ES PT requires an absolute path for `sdl.tsa.configFile` — the Guardian CLI's + # working dir for the auto-injected SDL Sources Analysis job is NOT $(Build.SourcesDirectory), + # so a relative path like `.config\tsaoptions.json` resolves to the wrong location and + # fails with `GuardianFileNotFoundException`. (OneBranch's `globalSdl.tsaOptionsFile` + # auto-resolved relative paths against the sources dir; 1ES PT does not.) + # Pattern confirmed in microsoft/PowerToys, microsoft/vscode, dotnet/macios, and others. + configFile: $(Build.SourcesDirectory)\.config\tsaoptions.json credscan: enabled: true scanFolder: $(Build.SourcesDirectory) From db357f4dfda4cf0e726063710841c8c13e205dc8 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Thu, 4 Jun 2026 12:03:02 -0500 Subject: [PATCH 06/19] 1ES POC cleanup: drop dead pool/container vars, derive APIScan version from branch - Remove unreferenced PoolNames variable group + LinuxContainerImage / WindowsContainerImage (zero references in the 1ES tree). - Replace hardcoded sdl.apiscan.versionNumber "7.6" with a value derived from Build.SourceBranchName at template-expansion time. Fixes silent TSA mislabelling on 7.9+ branches. - Keep ob_outputDirectory name intact (build.psm1 reads $env:OB_OUTPUTDIRECTORY). - Document each decision inline with NB comments. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .pipelines/1ES/apiscan-gen-notice.yml | 35 ++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/.pipelines/1ES/apiscan-gen-notice.yml b/.pipelines/1ES/apiscan-gen-notice.yml index e1b9bfa783..9d16213739 100644 --- a/.pipelines/1ES/apiscan-gen-notice.yml +++ b/.pipelines/1ES/apiscan-gen-notice.yml @@ -64,6 +64,13 @@ variables: # PAT permissions NOTE: Declare a SymbolServerPAT variable in this group with a 'microsoft' organizanization scoped PAT with 'Symbols' Read permission. # A PAT in the wrong org will give a single Error 203. No PAT will give a single Error 401, and individual pdbs may be missing even if permissions are correct. - group: symbols + # NB (1ES PT migration): variable name kept as `ob_outputDirectory` because + # `build.psm1` reads the env var `$env:OB_OUTPUTDIRECTORY` literally (see + # lines ~1085 of build.psm1). Renaming requires a coordinated PR across + # build.psm1 + every YAML reference + the OneBranch originals that are + # still alive during parallel validation. Defer until OneBranch retirement. + # The `/ONEBRANCH_ARTIFACT` suffix in the path is just a folder name and is + # safe to rename at any time, but is kept for parity with the OneBranch original. - name: ob_outputDirectory value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - name: CDP_DEFINITION_BUILD_COUNT @@ -72,11 +79,14 @@ variables: - group: 'Azure Blob variable group' # Defines the variables CgPat, CgOrganization, and CgProject - group: 'ComponentGovernance' - - group: 'PoolNames' - - name: LinuxContainerImage - value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 - - name: WindowsContainerImage - value: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest + # NB (1ES PT migration): the `PoolNames` variable group and the + # `LinuxContainerImage` / `WindowsContainerImage` variables were dropped here. + # The OneBranch original declared them but never referenced them in its + # `extends.parameters` block (the pool was hard-coded in the OneBranch + # template, and there was no container reference). In 1ES PT we declare + # pool + image inline on `extends.parameters.pool` and we don't use a job + # container, so all three are unreferenced. Removed to reduce noise and to + # avoid the pipeline-authorization prompt for `PoolNames`. - ${{ if eq(parameters['FORCE_CODEQL'],'true') }}: # Cadence is hours before CodeQL will allow a re-upload of the database - name: CodeQL.Cadence @@ -95,6 +105,19 @@ variables: value: false ${{ else }}: value: true + # APIScan version label — derived from the release/rebuild branch name so that + # TSA bug filing groups results under the correct PowerShell version bucket. + # Hardcoding (the prior `"7.6"`) caused 7.9 branch builds to file bugs under + # the 7.6 bucket. Evaluated at template-expansion time so the value is baked + # into the `sdl.apiscan.versionNumber` field below. + # refs/heads/release/v7.6.0 -> Build.SourceBranchName='v7.6.0' -> '7.6.0' + # refs/heads/rebuild/v7.9.99-rebuild.9 -> Build.SourceBranchName='v7.9.99-rebuild.9' -> '7.9.99' (split on '-', take [0], strip 'v') + # refs/heads/master -> fallback '7.9' (update at PowerShell major.minor bump) + - name: APIScanVersionNumber + ${{ if startsWith(variables['Build.SourceBranchName'], 'v') }}: + value: ${{ replace(split(variables['Build.SourceBranchName'], '-')[0], 'v', '') }} + ${{ else }}: + value: '7.9' resources: repositories: @@ -150,7 +173,7 @@ extends: apiscan: enabled: true softwareName: "PowerShell" # Default is repo name - versionNumber: "7.6" # Default is build number + versionNumber: ${{ variables.APIScanVersionNumber }} # Derived from branch — see APIScanVersionNumber definition above isLargeApp: false # Default: false. symbolsFolder: $(SymbolsServerUrl);$(ob_outputDirectory) #softwareFolder - relative path to a folder to be scanned. Default value is root of artifacts folder From 0da97e8b93ba3a23ace9e7744f7e17e775e60d35 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Fri, 5 Jun 2026 11:03:05 -0500 Subject: [PATCH 07/19] Add 1ES PT Unofficial scaffold for PowerShell-Coordinated_Packages Phase A of the OneBranch -> 1ES Pipeline Templates migration for the Coordinated_Packages pipeline. Builds and stubs signing end-to-end so the scaffold can be validated in ADO before ESRP signing onboarding completes. Files: .pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml Entry pipeline. Extends v1/1ES.Unofficial.PipelineTemplate.yml. CodeQL enabled unconditionally (compiled + enabledOnNonDefaultBranches + tsaEnabled) so every branch -- including rebuild/* shipping branches and migration test branches -- gets coverage. Branch-gated CodeQL would miss rebuild/* releases and block validating the SDL injection plumbing without cutting a real release. Global SDL paths use $(Build.SourcesDirectory)\.config\... (NO \PowerShell\ segment) because the auto-injected SDL Sources Analysis job runs without the stage-repo helper. .pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml 5-stage layout: prep / macos / linux / windows / test_and_release_artifacts. .pipelines/1ES/templates/stage-repo-under-PowerShell.yml NEW helper. Copy-Items repo into a \PowerShell\ subfolder so existing scripts that hardcode $(Build.SourcesDirectory)\PowerShell\... keep working. Replaces OneBranch cloneToOfficialPath.yml (which used `git clone` -- not viable in 1ES PT where source IS the parent of dest). Hardened: ErrorAction=Stop, excludes .git, post-copy sanity check for build.psm1 / .config/tsaoptions.json / .config/suppress.json. .pipelines/1ES/templates/mac.yml macOS build (Azure Pipelines + macOS-latest + isCustom:true) plus Windows sign job (PowerShell1ES + MMS2022). Apple sign + verify stubbed. codeSignValidation disabled on sign job while signing is stubbed -- 1ES PT's separate codeSignValidation SDL check fails regardless of Update-PSSignedBuildFolder's tolerance for unsigned files in Unofficial mode. Belt-and-suspenders ##vso[artifact.upload] fallback added because templateContext.outputs on custom pools is untested in this codebase. .pipelines/1ES/templates/linux.yml Linux build (PowerShell1ES + PSMMSAzureLinux3.0-Secure) plus Windows sign job. Per-job CodeQL3000Init/Finalize tasks dropped -- 1ES PT injects them automatically from sdl.codeql config. .pipelines/1ES/templates/windows-hosted-build.yml Single Windows job (PowerShell1ES + MMS2022) that builds and signs in place. 3 signing stubs (obj 1P, nupkg, obp-file-signing dispatch). Per-job CodeQL3000 tasks dropped. .pipelines/1ES/templates/testartifacts.yml Win + nonwin test package staging. Win job passes $(PowerShellRoot) -- NOT $(RepoRoot) -- to insert-nuget-config-azfeed.yml so the private-feed nuget.config lands in the staged repo, not the un-staged root that $(RepoRoot) still points to after the stage-repo helper runs. .pipelines/1ES/templates/obp-file-signing.yml 1P + 3P signing stubbed. Folder-prep and Update-PSSignedBuildFolder invocation preserved. Header lists the 4 ESRP KeyCodes needed for unstub (CP-230012, CP-231522, CP-401405, CP-401337-Apple). Reuses these OneBranch templates unchanged: .pipelines/templates/variables/PowerShell-Coordinated_Packages-Variables.yml .pipelines/templates/SetVersionVariables.yml .pipelines/templates/set-reporoot.yml .pipelines/templates/insert-nuget-config-azfeed.yml .pipelines/templates/install-dotnet.yml .pipelines/templates/rebuild-branch-check.yml .pipelines/templates/step/finalize.yml Phase B (real ESRP signing) starts once ESRP keycode onboarding lands. Phase C forks this Unofficial scaffold to an Official sibling. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...Shell-Coordinated_Packages-NonOfficial.yml | 214 +++++++++++ .pipelines/1ES/templates/linux.yml | 215 +++++++++++ .pipelines/1ES/templates/mac.yml | 235 ++++++++++++ .pipelines/1ES/templates/obp-file-signing.yml | 217 ++++++++++++ .../templates/stage-repo-under-PowerShell.yml | 71 ++++ ...PowerShell-Coordinated_Packages-Stages.yml | 221 ++++++++++++ .pipelines/1ES/templates/testartifacts.yml | 163 +++++++++ .../1ES/templates/windows-hosted-build.yml | 333 ++++++++++++++++++ 8 files changed, 1669 insertions(+) create mode 100644 .pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml create mode 100644 .pipelines/1ES/templates/linux.yml create mode 100644 .pipelines/1ES/templates/mac.yml create mode 100644 .pipelines/1ES/templates/obp-file-signing.yml create mode 100644 .pipelines/1ES/templates/stage-repo-under-PowerShell.yml create mode 100644 .pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml create mode 100644 .pipelines/1ES/templates/testartifacts.yml create mode 100644 .pipelines/1ES/templates/windows-hosted-build.yml diff --git a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml new file mode 100644 index 0000000000..e9241b5677 --- /dev/null +++ b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml @@ -0,0 +1,214 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# 1ES PIPELINE TEMPLATES (1ES PT) MIGRATION — UNOFFICIAL +# ------------------------------------------------------ +# This is a parallel migration of `.pipelines/NonOfficial/PowerShell-Coordinated_Packages-NonOfficial.yml` +# from OneBranch templates to 1ES Pipeline Templates. The OneBranch original +# is intentionally left untouched so the two pipelines can run side-by-side +# in ADO for validation. +# +# WHY UNOFFICIAL FIRST: +# Migrating to Unofficial first lets us validate the YAML structure, pool +# selection, SDL injection, and artifact production WITHOUT needing the +# production ESRP service connections (which are still being onboarded). +# Once this Unofficial pipeline is green end-to-end, we fork it to an +# Official sibling that swaps in real ESRP signing. +# +# Translation summary (full notes in `files/onebranch-to-1es-migration.md`): +# - resources.repositories: OneBranch.Pipelines/GovernedTemplates @ refs/heads/main +# -> 1ESPipelineTemplates/1ESPipelineTemplates @ refs/tags/release +# - extends.template: v2/OneBranch.NonOfficial.CrossPlat.yml@onebranchTemplates +# -> v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates +# - featureFlags: removed (Windows/Linux host version come from the pool image) +# - globalSdl: renamed to sdl (sub-keys nearly identical) +# - tsaOptionsFile: moved under sdl.tsa.configFile, made ABSOLUTE +# ($(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json) +# See migration playbook gotcha #3a for why. +# - armory: removed (OneBranch-only; not part of 1ES PT SDL schema). +# - pool: declared at the extends level (Windows: PowerShell1ES +# + MMS2022). Per-job overrides for Linux/macOS jobs. +# - stages/templates: point at `.pipelines/1ES/templates/...` siblings, NOT +# the OneBranch originals (because per-job templates +# need their own pool/templateContext translations). +# +# POOL/IMAGE STRATEGY: +# - Windows jobs -> PowerShell1ES + MMS2022 (1ES-maintained, proven in apiscan POC) +# - Linux jobs -> PowerShell1ES + PSMMSAzureLinux3.0-Secure (closest match to +# the OneBranch container `mcr.microsoft.com/onebranch/azurelinux/build:3.0`). +# ⚠ Verify with team that PowerShell1ES pool carries this image. +# If not, fall back to `1ES-Linux-Ubuntu-2204` Microsoft-hosted pool. +# - macOS jobs -> Azure Pipelines + vmImage macOS-latest (no team-owned macOS image +# exists in the CloudTest inventory; macOS jobs MUST use the +# Microsoft-hosted Azure Pipelines shared pool). +# +# CODEQL POLICY: +# - 1ES PT auto-runs CodeQL3000 on the ADO repo default branch only by default. +# - PowerShell's ADO default is `internal-master`, NOT release branches, so without +# intervention CodeQL would never run on the branches that ship to customers. +# - Shipping branches are `release/v7.x.y` AND `rebuild/v7.x.y-rebuild.N` — both +# ship code to customers, neither is the default branch. +# - We set `sdl.codeql.enabled: true` + `enabledOnNonDefaultBranches: true` +# UNCONDITIONALLY so every branch (default, release, rebuild, PR, dev, test) +# gets CodeQL coverage. Running on every branch also lets us validate the +# SDL injection plumbing without waiting to cut a release. +# - The per-job CodeQL3000Init/Finalize tasks are dropped (1ES PT injects them +# automatically when `sdl.codeql` is enabled). +# - `FORCE_CODEQL=true` is still useful for debugging: it switches +# CodeQL.Cadence from 24h to 1h so back-to-back uploads don't get throttled. +# +# STILL TODO (will be addressed in subsequent commits / Official sibling): +# - Stub signing: every onebranch.pipeline.signing@1 call replaced with a +# `pwsh: Write-Host "TODO: ESRP sign ..."` placeholder. Pipeline runs end-to-end +# but produces UNSIGNED artifacts — strictly for validating the migration scaffold. +# - Confirm Linux pool/image mapping (PowerShell1ES + PSMMSAzureLinux3.0-Secure). +# - Once Unofficial green, build Official sibling with real ESRP signing. + +trigger: none + +parameters: + - name: InternalSDKBlobURL + displayName: URL to the blob having internal .NET SDK + type: string + default: ' ' + - name: ReleaseTagVar + displayName: Release Tag + type: string + default: 'fromBranch' + - name: SKIP_SIGNING + displayName: Debugging - Skip Signing + type: string + default: 'NO' + - name: RUN_TEST_AND_RELEASE + displayName: Debugging - Run Test and Release Artifacts Stage + type: boolean + default: true + - name: RUN_WINDOWS + displayName: Debugging - Enable Windows Stage + type: boolean + default: true + - name: ENABLE_MSBUILD_BINLOGS + displayName: Debugging - Enable MSBuild Binary Logs + type: boolean + default: false + - name: FORCE_CODEQL + displayName: Debugging - Enable CodeQL and set cadence to 1 hour + type: boolean + default: false + +name: bins-$(BUILD.SOURCEBRANCHNAME)-1ES-nonofficial-$(Build.BuildId) + +variables: + # Inherit the same variable scaffold as the OneBranch original. The shared + # variables file is 1ES-safe (it only declares string variables + groups — + # no ob_* SDL knobs at the global level except `ob_sdl_binskim_enabled`, + # which 1ES PT will silently ignore at the variables block scope). + - template: /.pipelines/templates/variables/PowerShell-Coordinated_Packages-Variables.yml@self + parameters: + InternalSDKBlobURL: ${{ parameters.InternalSDKBlobURL }} + ReleaseTagVar: ${{ parameters.ReleaseTagVar }} + SKIP_SIGNING: ${{ parameters.SKIP_SIGNING }} + ENABLE_MSBUILD_BINLOGS: ${{ parameters.ENABLE_MSBUILD_BINLOGS }} + FORCE_CODEQL: ${{ parameters.FORCE_CODEQL }} + +resources: + repositories: + # ComplianceRepo is preserved verbatim (used by helper templates that + # invoke PoliCheck / CredScan via the GH-hosted PowerShell/compliance repo). + - repository: ComplianceRepo + type: github + endpoint: ComplianceGHRepo + name: PowerShell/compliance + ref: master + # NEW: 1ES Pipeline Templates resource (replaces `onebranchTemplates`). + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + # Pool default for all jobs. Windows jobs inherit; macOS/Linux jobs override + # at the per-job level inside the templates. + pool: + name: PowerShell1ES + image: MMS2022 + os: windows + + sdl: + # CodeQL: enabled unconditionally on ALL branches. + # + # Rationale: + # - 1ES PT's default behaviour runs CodeQL only on the ADO repo's + # `default` branch. PowerShell's ADO default branch is `internal-master`, + # which is NEVER used for releases. Shipping branches are: + # * `release/v7.x.y` — standard releases + # * `rebuild/v7.x.y-rebuild.N` — rebuild releases (see rebuild-branch-check.yml) + # Both ship to customers; neither is the default branch. Default + # behaviour would therefore skip CodeQL on EVERY shipping branch. + # + # - Forcing CodeQL on every branch (including PR / dev / test branches) + # also lets us validate the migration scaffold itself — TSA bug + # filing, SDL injection, CodeQL database upload cadence — without + # waiting to cut a release. + # + # - The `$(CODEQL_ENABLED)` variable from upstream Variables.yml + # remains defined (still consumed by the apiscan-gen-notice POC) but + # is intentionally NOT referenced here. The FORCE_CODEQL parameter + # still does useful work: it pushes CodeQL.Cadence from 24h → 1h for + # debugging, so back-to-back uploads don't get throttled. + codeql: + compiled: + enabled: true + enabledOnNonDefaultBranches: true + tsaEnabled: true # File CodeQL findings to TSA on every run; TSA dedupes + + # `armory` was OneBranch-only. Removed. + + sbom: + enabled: true + + cg: + enabled: true + ignoreDirectories: '.devcontainer,demos,docker,docs,src,test,tools/packaging' + + tsa: + enabled: true # 1ES publishes all SDL results to TSA. Disabling forces all SDL tools into 'break' mode. + # 1ES PT requires an absolute path for `sdl.tsa.configFile` (validated via + # apiscan-gen-notice POC — relative paths fail with `GuardianFileNotFoundException`). + # + # IMPORTANT: this path is consumed by the auto-injected SDL Sources Analysis + # job, which runs WITHOUT the `stage-repo-under-PowerShell.yml` helper. That + # job sees the repo at $(Build.SourcesDirectory) directly. DO NOT add a + # `\PowerShell\` segment here — only the per-job `templateContext.sdl.tsa` + # overrides (inside user jobs that DO run the staging helper) use the + # `\PowerShell\.config\...` form. + configFile: $(Build.SourcesDirectory)\.config\tsaoptions.json + + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + # Same rationale as tsa.configFile above — global SDL paths must NOT include + # `\PowerShell\` because the SDL Sources Analysis job does not stage the repo. + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + + # BinSkim runs at the user-job level in 1ES PT. Disabled at the global + # level to mirror the OneBranch original's behaviour. Re-enabled per-job + # for binary-producing jobs in the job templates (windows/linux build). + binskim: + enabled: false + exactToolVersion: 4.4.2 + + # APIScan: not part of the build pipeline. Lives in + # `.pipelines/1ES/apiscan-gen-notice.yml`. Explicit off here matches + # the OneBranch original. + apiscan: + enabled: false + + stages: + - template: /.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml@self + parameters: + RUN_WINDOWS: ${{ parameters.RUN_WINDOWS }} + RUN_TEST_AND_RELEASE: ${{ parameters.RUN_TEST_AND_RELEASE }} + OfficialBuild: false diff --git a/.pipelines/1ES/templates/linux.yml b/.pipelines/1ES/templates/linux.yml new file mode 100644 index 0000000000..dc6ea7f508 --- /dev/null +++ b/.pipelines/1ES/templates/linux.yml @@ -0,0 +1,215 @@ +parameters: + Runtime: 'linux-x64' + BuildConfiguration: 'release' + JobName: 'build_linux' + +# 1ES PT translation of /.pipelines/templates/linux.yml. +# Two jobs: +# 1. build_ — Linux pool (PowerShell1ES + PSMMSAzureLinux3.0-Secure) +# 2. sign_ — Windows pool (PowerShell1ES + MMS2022), STUBBED ESRP signing. +# +# Notes +# ----- +# - CodeQL3000Init / CodeQL3000Finalize tasks DROPPED. 1ES PT auto-injects CodeQL +# based on `sdl.codeql` settings (configured at the pipeline-extends level in the +# entry .yml). Manual init/finalize is no longer needed. +# - cloneToOfficialPath.yml replaced with stage-repo-under-PowerShell.yml. +# - Artifact naming preserved: `drop_linux_build_${{ parameters.JobName }}` matches the +# OneBranch auto-generated name that the sign job downloads. + +jobs: +- job: build_${{ parameters.JobName }} + displayName: Build_Linux_${{ parameters.Runtime }}_${{ parameters.BuildConfiguration }} + condition: succeeded() + pool: + name: PowerShell1ES + image: PSMMSAzureLinux3.0-Secure + os: linux + variables: + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_NOLOGO + value: 1 + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: BUILDCONFIGURATION + value: ${{ parameters.BuildConfiguration }} + - name: Runtime + value: ${{ parameters.Runtime }} + - name: PowerShellRoot + value: '$(Build.SourcesDirectory)/PowerShell' + + templateContext: + sdl: + codeSignValidation: + enabled: false + binskim: + enabled: true + tsa: + configFile: $(Build.SourcesDirectory)/PowerShell/.config/tsaoptions.json + credscan: + suppressionsFile: $(Build.SourcesDirectory)/PowerShell/.config/suppress.json + sbom: + packageName: 'Microsoft.Powershell.Linux.${{ parameters.Runtime }}' + codeql: + compiled: + enabled: false + outputs: + - output: pipelineArtifact + targetPath: $(ob_outputDirectory) + artifactName: drop_linux_build_${{ parameters.JobName }} + + steps: + - checkout: self + clean: true + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + ob_restore_phase: false + + - template: /.pipelines/1ES/templates/stage-repo-under-PowerShell.yml@self + + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: $(PowerShellRoot) + + - template: /.pipelines/templates/install-dotnet.yml@self + + - pwsh: | + $runtime = $env:RUNTIME + + $params = @{} + if ($env:BUILDCONFIGURATION -eq 'minSize') { + Write-Verbose -Message "Building for minimal size" + $params['ForMinimalSize'] = $true + } + + Write-Verbose -Message "Building PowerShell with Runtime: $runtime" + Import-Module -Name $(PowerShellRoot)/build.psm1 -Force + $buildWithSymbolsPath = New-Item -ItemType Directory -Path $(Pipeline.Workspace)/Symbols_$(Runtime) -Force + + $null = New-Item -ItemType Directory -Path $buildWithSymbolsPath -Force -Verbose + + $ReleaseTagParam = @{} + + if ($env:RELEASETAGVAR) { + $ReleaseTagParam['ReleaseTag'] = $env:RELEASETAGVAR + } + + Start-PSBuild -Runtime $runtime -Configuration Release -Output $buildWithSymbolsPath @params -Clean -PSModuleRestore @ReleaseTagParam + + $outputPath = Join-Path '$(ob_outputDirectory)' 'psoptions' + $null = New-Item -ItemType Directory -Path $outputPath -Force + $psOptPath = "$outputPath/psoptions.json" + Save-PSOptions -PSOptionsPath $psOptPath + + Write-Verbose -Verbose "Verifying pdbs exist in build folder" + $pdbs = Get-ChildItem -Path $buildWithSymbolsPath -Recurse -Filter *.pdb + if ($pdbs.Count -eq 0) { + Write-Error -Message "No pdbs found in build folder" + } + else { + Write-Verbose -Verbose "Found $($pdbs.Count) pdbs in build folder" + $pdbs | ForEach-Object { + Write-Verbose -Verbose "Pdb: $($_.FullName)" + } + } + + Write-Verbose -Verbose "Completed building PowerShell for '$env:BUILDCONFIGURATION' configuration" + displayName: 'Build Linux - $(Runtime)' + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + + - pwsh: | + $platform = 'linux' + $vstsCommandString = "vso[task.setvariable variable=ArtifactPlatform]$platform" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: Set artifact platform + + - pwsh: | + $pathForUpload = New-Item -ItemType Directory -Path '$(ob_outputDirectory)/Unsigned-$(Runtime)' -Force + Write-Verbose -Verbose -Message "pathForUpload: $pathForUpload" + Copy-Item -Path '$(Pipeline.Workspace)/Symbols_$(Runtime)/*' -Destination $pathForUpload -Recurse -Force -Verbose + displayName: Copy unsigned files for upload + + - template: /.pipelines/templates/step/finalize.yml@self + +- job: sign_${{ parameters.JobName }} + displayName: Sign_Linux_${{ parameters.Runtime }}_${{ parameters.BuildConfiguration }} + condition: succeeded() + dependsOn: build_${{ parameters.JobName }} + pool: + name: PowerShell1ES + image: MMS2022 + os: windows + variables: + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_NOLOGO + value: 1 + - group: DotNetPrivateBuildAccess + - group: certificate_logical_to_actual + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: BuildConfiguration + value: ${{ parameters.BuildConfiguration }} + - name: Runtime + value: ${{ parameters.Runtime }} + - name: PowerShellRoot + value: '$(Build.SourcesDirectory)\PowerShell' + + templateContext: + sdl: + codeSignValidation: + enabled: false + binskim: + enabled: false + tsa: + configFile: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + credscan: + suppressionsFile: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + codeql: + compiled: + enabled: false + outputs: + - output: pipelineArtifact + targetPath: $(ob_outputDirectory) + artifactName: drop_linux_sign_${{ parameters.JobName }} + + steps: + - checkout: self + clean: true + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + ob_restore_phase: false + + - template: /.pipelines/1ES/templates/stage-repo-under-PowerShell.yml@self + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: drop_linux_build_${{ parameters.JobName }} + path: $(Pipeline.Workspace)/drop_linux_build + displayName: Download build + + - pwsh: | + Get-ChildItem -Path $(Pipeline.Workspace)/drop_linux_build -Recurse + displayName: Capture downloaded files + + - pwsh: | + $pwshPath = Get-ChildItem -Path $(Pipeline.Workspace)/drop_linux_build -File -Recurse | Where-Object { $_.Name -eq 'pwsh' } + $rootPath = Split-Path -Path $pwshPath.FullName -Parent + Write-Verbose -Verbose "Setting vso[task.setvariable variable=DropRootPath]$rootPath" + Write-Host "##vso[task.setvariable variable=DropRootPath]$rootPath" + displayName: Set drop root path + + - template: /.pipelines/1ES/templates/obp-file-signing.yml@self + parameters: + binPath: $(DropRootPath) + OfficialBuild: $(ps_official_build) + + - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/1ES/templates/mac.yml b/.pipelines/1ES/templates/mac.yml new file mode 100644 index 0000000000..1e09e8f627 --- /dev/null +++ b/.pipelines/1ES/templates/mac.yml @@ -0,0 +1,235 @@ +parameters: + buildArchitecture: 'x64' + +# 1ES PT translation of /.pipelines/templates/mac.yml. +# Two jobs: +# 1. build_macOS_ — runs on Microsoft-hosted macOS pool (1ES PT custom pool). +# 2. sign_ — runs on PowerShell1ES Windows pool; STUBBED Apple signing. +# +# Open questions / risks +# ---------------------- +# - macOS pool: there is no 1ES-managed macOS image in the CloudTest inventory. We fall +# back to Microsoft-hosted `Azure Pipelines` + `vmImage: macOS-latest` with the +# `isCustom: true` escape hatch. This works in 1ES PT extends scenarios but skips most +# SDL injection (codeql etc. don't run on custom pools). For Unofficial that's fine; for +# Official we'd need to either keep this approach with explicit SDL waivers or pursue a +# managed macOS image. +# - Apple ESRP signing (apple_cert_id KeyCode CP-401337-Apple) is stubbed. The verification +# step is also gated off because there's no real Developer ID signature to verify. + +jobs: +- job: build_macOS_${{ parameters.buildArchitecture }} + displayName: Build macOS ${{ parameters.buildArchitecture }} + condition: succeeded() + pool: + name: Azure Pipelines + vmImage: 'macOS-latest' + os: macOS + isCustom: true # 1ES PT escape hatch for Microsoft-hosted shared pools + + variables: + - name: HOMEBREW_NO_ANALYTICS + value: 1 + - name: NugetSecurityAnalysisWarningLevel + value: none + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory # kept verbatim: build.psm1 reads $env:OB_OUTPUTDIRECTORY literally + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: PowerShellRoot + value: $(Build.SourcesDirectory) + + templateContext: + sdl: + codeSignValidation: + enabled: false + outputs: + - output: pipelineArtifact + targetPath: $(ob_outputDirectory) + artifactName: macosBinResults-${{ parameters.buildArchitecture }} + + steps: + - checkout: self + clean: true + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + ob_restore_phase: false + + - pwsh: | + # create folder + sudo mkdir "$(Agent.TempDirectory)/PowerShell" + # make the current user the owner + sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell" + displayName: 'Create $(Agent.TempDirectory)/PowerShell' + + ## We cross compile for arm64, so the arch is always x64 + - template: /.pipelines/templates/install-dotnet.yml@self + + - pwsh: | + Import-Module $(PowerShellRoot)/build.psm1 -Force + Start-PSBootstrap -Scenario Package + displayName: 'Bootstrap VM' + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: $(PowerShellRoot) + - pwsh: | + $env:AzDevOpsFeedPAT2 = '$(powershellPackageReadPat)' + # Add -SkipReleaseChecks as a mitigation to unblock release. + # macos-10.15 does not allow creating a folder under root. Hence, moving the folder. + + Import-Module ./build.psm1 -Force + + $ReleaseTagParam = @{} + + if ($env:RELEASETAGVAR) { + $ReleaseTagParam['ReleaseTag'] = $env:RELEASETAGVAR + } + + Start-PSBuild -Runtime 'osx-${{ parameters.buildArchitecture }}' -Configuration Release -PSModuleRestore -Clean -Output $(OB_OUTPUTDIRECTORY) @ReleaseTagParam + + $psOptPath = "$(OB_OUTPUTDIRECTORY)/psoptions.json" + Save-PSOptions -PSOptionsPath $psOptPath + + $entitlements = "$(PowerShellRoot)/assets/macos-entitlements.plist" + $pwshBin = "$(OB_OUTPUTDIRECTORY)/pwsh" + Write-Verbose -Verbose "Applying entitlements to $pwshBin" + codesign --sign - --force --options runtime --entitlements $entitlements $pwshBin + if ($LASTEXITCODE -ne 0) { + throw "codesign failed with exit code $LASTEXITCODE" + } + + $env:AzDevOpsFeedPAT2 = $null + displayName: 'Build' + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + + # Belt-and-suspenders artifact publish for the macOS custom pool. + # `templateContext.outputs` SHOULD publish via PublishPipelineArtifact@1, but on + # `isCustom: true` (Microsoft-hosted) pools this path is not as well-trodden as on + # 1ES-managed pools. The original OneBranch mac.yml relied on an explicit + # `##vso[artifact.upload]` here, so we keep that as a fallback. If the + # templateContext.outputs publish succeeds, this is a no-op duplicate (ADO + # tolerates this); if it fails silently, the sign job's DownloadPipelineArtifact + # call still has an artifact to fetch. + - pwsh: | + Write-Host "##vso[artifact.upload containerfolder=macosBinResults-$(BuildArchitecture);artifactname=macosBinResults-$(BuildArchitecture)]$(OB_OUTPUTDIRECTORY)" + displayName: 'Upload macOS build artifact (fallback for custom pool)' + + - template: /.pipelines/templates/step/finalize.yml@self + +- job: sign_${{ parameters.buildArchitecture }} + displayName: Sign_macOS_${{ parameters.buildArchitecture }} + condition: succeeded() + dependsOn: build_macOS_${{ parameters.buildArchitecture }} + pool: + name: PowerShell1ES + image: MMS2022 + os: windows + variables: + - name: NugetSecurityAnalysisWarningLevel + value: none + - group: DotNetPrivateBuildAccess + - group: certificate_logical_to_actual + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: BuildArchitecture + value: ${{ parameters.buildArchitecture }} + - name: PowerShellRoot + value: '$(Build.SourcesDirectory)\PowerShell' + + templateContext: + sdl: + # Stub signing in effect — Apple signatures are NOT applied. Disable + # codeSignValidation while stubbed; re-enable when ESRP Apple signing + # (CP-401337-Apple) is wired up in the Official sibling pipeline. + codeSignValidation: + enabled: false + tsa: + configFile: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + credscan: + suppressionsFile: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + codeql: + compiled: + enabled: false + sbom: + packageName: 'Microsoft.Powershell.MacOS.${{ parameters.buildArchitecture }}' + outputs: + - output: pipelineArtifact + targetPath: $(ob_outputDirectory) + artifactName: drop_macos_sign_${{ parameters.buildArchitecture }} + + steps: + - checkout: self + clean: true + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + ob_restore_phase: false + + - template: /.pipelines/1ES/templates/stage-repo-under-PowerShell.yml@self + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: 'macosBinResults-$(BuildArchitecture)' + path: '$(Pipeline.Workspace)\Symbols' + displayName: Download build + + - pwsh: | + Get-ChildItem "$(Pipeline.Workspace)\*" -Recurse + displayName: 'Capture Downloaded Artifacts' + # Diagnostics is not critical it passes every time it runs + continueOnError: true + + - pwsh: | + $runtime = '$(BuildArchitecture)' + Write-Host "sending.. vso[task.setvariable variable=Runtime]$runtime" + Write-Host "##vso[task.setvariable variable=Runtime]$runtime" + + $rootPath = "$(Pipeline.Workspace)\Symbols" + Write-Verbose -Verbose "Setting vso[task.setvariable variable=DropRootPath]$rootPath" + Write-Host "##vso[task.setvariable variable=DropRootPath]$rootPath" + displayName: Expand symbols zip + + - template: /.pipelines/1ES/templates/obp-file-signing.yml@self + parameters: + binPath: $(DropRootPath) + OfficialBuild: $(ps_official_build) + + # STUB SECTION — Apple Developer ID codesign of Mach-O binaries. + # Once ESRP onboarding is complete and apple_cert_id (KeyCode CP-401337-Apple, + # MacAppDeveloperSign) is wired up, replace the stub below with an EsrpCodeSigning@5 + # task call. The Compress/Expand framing stays the same. + - pwsh: | + $signedDir = "$(ob_outputDirectory)/Signed-$(Runtime)" + $zipFile = "$(Pipeline.Workspace)/macho-$(BuildArchitecture).zip" + Compress-Archive -Path "$signedDir/*" -DestinationPath $zipFile -Force + displayName: Compress signed folder for Apple signing + + - pwsh: | + Write-Host '##[warning]STUB: would invoke ESRP Apple CodeSign Mach-O binaries with KeyCode "$(apple_cert_id)" (MacAppDeveloperSign, Hardening=--options=runtime)' + Write-Host ' files_to_sign: macho-$(BuildArchitecture).zip' + Write-Host ' search_root: $(Pipeline.Workspace)' + displayName: 'Apple CodeSign Mach-O binaries (STUB — ESRP onboarding pending)' + + - pwsh: | + $signedDir = "$(ob_outputDirectory)/Signed-$(Runtime)" + $zipFile = "$(Pipeline.Workspace)/macho-$(BuildArchitecture).zip" + Expand-Archive -Path $zipFile -DestinationPath $signedDir -Force -Verbose + displayName: Expand Apple-signed Mach-O binaries into signed output + + # STUB: verification gated off because real signatures are not applied. + # Re-enable when real ESRP Apple signing is wired up. + - pwsh: | + Write-Host '##[warning]STUB: Developer ID signature verification skipped (real Apple signing not yet enabled)' + $signedDir = "$(ob_outputDirectory)/Signed-$(Runtime)" + Get-ChildItem $signedDir -Recurse -Include 'pwsh', '*.dylib' | ForEach-Object { + Write-Host "Would verify: $($_.FullName)" + } + displayName: 'Verify Developer ID signature on Mach-O binaries (STUB)' + + - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/1ES/templates/obp-file-signing.yml b/.pipelines/1ES/templates/obp-file-signing.yml new file mode 100644 index 0000000000..3d1468dd16 --- /dev/null +++ b/.pipelines/1ES/templates/obp-file-signing.yml @@ -0,0 +1,217 @@ +parameters: + binPath: '$(ob_outputDirectory)' + globalTool: 'false' + SigningProfile: 'external_distribution' + OfficialBuild: true + vPackScenario: false + +# 1ES PT translation of /.pipelines/templates/obp-file-signing.yml — STUBBED VERSION. +# +# Why stubs? +# ---------- +# The OneBranch original calls `task: onebranch.pipeline.signing@1` against ESRP +# KeyCodes that PowerShell is still in the process of onboarding for the new 1ES service +# connection (external_distribution / msft_3rd_party_cert_id / nuget_cert_id / apple_cert_id). +# Until that onboarding is complete, the Unofficial 1ES pipeline cannot do real signing. +# +# This stub keeps the SAME FILE LAYOUT WORK as the original (copy files to be signed +# into staging folders, prepare third-party folder, run Update-PSSignedBuildFolder) so the +# pipeline still produces a recognisable artifact tree. The actual `signing@1` task calls +# are replaced with `pwsh: Write-Host "TODO: ESRP sign ..."` placeholders so the pipeline +# runs end-to-end and emits UNSIGNED artifacts. +# +# Migration path back to real signing (in the Official 1ES sibling, post-ESRP onboarding): +# Replace each "TODO: ESRP sign" pwsh step with the appropriate 1ES PT signing pattern, +# e.g. EsrpCodeSigning@5 with the new 1ES service connection, mapping: +# SigningProfile -> KeyCode CP-230012 (external_distribution) +# $(msft_3rd_party_cert_id) -> KeyCode CP-231522 (3rd party Authenticode) +# $(nuget_cert_id) -> KeyCode CP-401405 (NuGet package signing) +# $(apple_cert_id) -> KeyCode CP-401337-Apple (MacAppDeveloperSign) +# See files/onebranch-to-1es-migration.md "ESRP KeyCode mapping" section for the full table. + +steps: +- pwsh: | + $fullSymbolsFolder = '${{ parameters.binPath }}' + Write-Verbose -Verbose "fullSymbolsFolder == $fullSymbolsFolder" + Get-ChildItem -Recurse $fullSymbolsFolder | Select-Object -ExpandProperty FullName | Write-Verbose -Verbose + $filesToSignDirectory = "$(Pipeline.Workspace)/toBeSigned" + if ((Test-Path -Path $filesToSignDirectory)) { + Remove-Item -Path $filesToSignDirectory -Recurse -Force + } + $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force + + $itemsToCopyWithRecurse = @( + "$($fullSymbolsFolder)/*.ps1" + "$($fullSymbolsFolder)/Microsoft.PowerShell*.dll" + ) + $itemsToCopy = @{ + "$($fullSymbolsFolder)/*.ps1" = "" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Host/Microsoft.PowerShell.Host.psd1" = "Modules/Microsoft.PowerShell.Host" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Management/Microsoft.PowerShell.Management.psd1" = "Modules/Microsoft.PowerShell.Management" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Security/Microsoft.PowerShell.Security.psd1" = "Modules/Microsoft.PowerShell.Security" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Utility/Microsoft.PowerShell.Utility.psd1" = "Modules/Microsoft.PowerShell.Utility" + "$($fullSymbolsFolder)/pwsh.dll" = "" + "$($fullSymbolsFolder)/System.Management.Automation.dll" = "" + } + ## Windows only modules + if('$(ArtifactPlatform)' -eq 'windows') { + $itemsToCopy += @{ + "$($fullSymbolsFolder)/pwsh.exe" = "" + "$($fullSymbolsFolder)/Microsoft.Management.Infrastructure.CimCmdlets.dll" = "" + "$($fullSymbolsFolder)/Microsoft.WSMan.*.dll" = "" + "$($fullSymbolsFolder)/Modules/CimCmdlets/CimCmdlets.psd1" = "Modules/CimCmdlets" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Diagnostics/Diagnostics.format.ps1xml" = "Modules/Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Diagnostics/Event.format.ps1xml" = "Modules/Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Diagnostics/GetEvent.types.ps1xml" = "Modules/Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Security/Security.types.ps1xml" = "Modules/Microsoft.PowerShell.Security" + "$($fullSymbolsFolder)/Modules/Microsoft.PowerShell.Diagnostics/Microsoft.PowerShell.Diagnostics.psd1" = "Modules/Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)/Modules/Microsoft.WSMan.Management/Microsoft.WSMan.Management.psd1" = "Modules/Microsoft.WSMan.Management" + "$($fullSymbolsFolder)/Modules/Microsoft.WSMan.Management/WSMan.format.ps1xml" = "Modules/Microsoft.WSMan.Management" + "$($fullSymbolsFolder)/Modules/PSDiagnostics/PSDiagnostics.ps?1" = "Modules/PSDiagnostics" + } + } + + $itemsToExclude = @( + # This package is retrieved from https://www.github.com/powershell/MarkdownRender + "$($fullSymbolsFolder)/Microsoft.PowerShell.MarkdownRender.dll" + ) + + if('$(ArtifactPlatform)' -eq 'linux' -or '$(ArtifactPlatform)' -eq 'macos') { + $itemsToExclude += "$($fullSymbolsFolder)/pwsh" + } + + Write-Verbose -verbose "recursively copying $($itemsToCopyWithRecurse | out-string) to $filesToSignDirectory" + Copy-Item -Path $itemsToCopyWithRecurse -Destination $filesToSignDirectory -Recurse -verbose -exclude $itemsToExclude + Write-Verbose -verbose "recursive copy done." + + foreach($pattern in $itemsToCopy.Keys) { + $destinationFolder = Join-Path $filesToSignDirectory -ChildPath $itemsToCopy.$pattern + $null = New-Item -ItemType Directory -Path $destinationFolder -Force + Write-Verbose -verbose "copying $pattern to $destinationFolder" + + if (-not (Test-Path -Path $pattern)) { + Write-Verbose -verbose "No files found for pattern $pattern" + continue + } + + Copy-Item -Path $pattern -Destination $destinationFolder -Recurse -verbose + } + + Write-Verbose -verbose "copying done." + Write-Verbose -verbose "Files to be signed at: $filesToSignDirectory" + + Get-ChildItem -Recurse -File $filesToSignDirectory | Select-Object -Property FullName + displayName: 'Prepare files to be signed' + +# STUB: Original was `task: onebranch.pipeline.signing@1` with `signing_profile: ${{ parameters.SigningProfile }}` +# (default `external_distribution`, KeyCode CP-230012). Replace this stub once ESRP onboarding is complete. +- pwsh: | + Write-Host '##[warning]STUB: would invoke ESRP signing with profile "${{ parameters.SigningProfile }}"' + Write-Host ' files_to_sign: **\*.psd1;**\*.psm1;**\*.ps1xml;**\*.ps1;**\*.dll;**\*.exe;**\pwsh' + Write-Host ' search_root: $(Pipeline.Workspace)/toBeSigned' + Write-Host '' + Write-Host 'Files in search_root (would be signed):' + if (Test-Path '$(Pipeline.Workspace)/toBeSigned') { + Get-ChildItem -Path '$(Pipeline.Workspace)/toBeSigned' -Recurse -File | + Select-Object -ExpandProperty FullName | + ForEach-Object { Write-Host " $_" } + } else { + Write-Host ' (toBeSigned folder does not exist)' + } + displayName: 'Sign 1st party files (STUB — ESRP onboarding pending)' + +- pwsh : | + Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + displayName: Capture environment + +- pwsh: | + Import-Module $(PowerShellRoot)/build.psm1 -Force + Import-Module $(PowerShellRoot)/tools/packaging -Force + + $BuildPath = (Get-Item '${{ parameters.binPath }}').FullName + Write-Verbose -Verbose -Message "BuildPath: $BuildPath" + + $officialBuild = [System.Convert]::ToBoolean('${{ parameters.OfficialBuild }}') + ## copy all files to be signed to build folder + Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath '$(Pipeline.Workspace)/toBeSigned' -OfficialBuild $officialBuild + + $dlls = Get-ChildItem $BuildPath/*.dll, $BuildPath/*.exe -Recurse + $signatures = $dlls | Get-AuthenticodeSignature + $officialIssuerPattern = '^CN=(Microsoft Code Signing PCA|Microsoft Root Certificate Authority|Microsoft Corporation).*' + $testCert = '^CN=(Microsoft|TestAzureEngBuildCodeSign).*' + $missingSignatures = $signatures | Where-Object { $_.status -eq 'notsigned' -or $_.SignerCertificate.Issuer -notmatch $testCert -or $_.SignerCertificate.Issuer -notmatch $officialIssuerPattern} | select-object -ExpandProperty Path + + Write-Verbose -verbose "to be signed:`r`n $($missingSignatures | Out-String)" + + $filesToSignDirectory = "$(Pipeline.Workspace)/thirdPartyToBeSigned" + if (Test-Path $filesToSignDirectory) { + Remove-Item -Path $filesToSignDirectory -Recurse -Force + } + $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force -Verbose + + $missingSignatures | ForEach-Object { + $pathWithoutLeaf = Split-Path $_ + $relativePath = $pathWithoutLeaf.replace($BuildPath,'') + Write-Verbose -Verbose -Message "relativePath: $relativePath" + $targetDirectory = Join-Path -Path $filesToSignDirectory -ChildPath $relativePath + Write-Verbose -Verbose -Message "targetDirectory: $targetDirectory" + if(!(Test-Path $targetDirectory)) + { + $null = New-Item -ItemType Directory -Path $targetDirectory -Force -Verbose + } + Copy-Item -Path $_ -Destination $targetDirectory + } + displayName: Create ThirdParty Signing Folder + +# STUB: Original was `task: onebranch.pipeline.signing@1` with `signing_profile: $(msft_3rd_party_cert_id)` +# (KeyCode CP-231522, 3rd-party Authenticode). Replace this stub once ESRP onboarding is complete. +- pwsh: | + Write-Host '##[warning]STUB: would invoke ESRP signing with profile "$(msft_3rd_party_cert_id)" (3rd-party Authenticode)' + Write-Host ' files_to_sign: **\*.dll;**\*.exe' + Write-Host ' search_root: $(Pipeline.Workspace)/thirdPartyToBeSigned' + Write-Host '' + Write-Host 'Files in search_root (would be signed):' + if (Test-Path '$(Pipeline.Workspace)/thirdPartyToBeSigned') { + Get-ChildItem -Path '$(Pipeline.Workspace)/thirdPartyToBeSigned' -Recurse -File | + Select-Object -ExpandProperty FullName | + ForEach-Object { Write-Host " $_" } + } else { + Write-Host ' (thirdPartyToBeSigned folder does not exist)' + } + displayName: 'Sign 3rd Party files (STUB — ESRP onboarding pending)' + +- pwsh: | + Get-ChildItem '$(Pipeline.Workspace)/thirdPartyToBeSigned/*' + displayName: Capture ThirdParty Signed files + +- pwsh: | + $officialBuild = [System.Convert]::ToBoolean('${{ parameters.OfficialBuild }}') + $vPackScenario = [System.Convert]::ToBoolean('${{ parameters.vPackScenario }}') + Import-Module '$(PowerShellRoot)/build.psm1' -Force + Import-Module '$(PowerShellRoot)/tools/packaging' -Force + $isGlobalTool = '${{ parameters.globalTool }}' -eq 'true' + + if ($vPackScenario) { + Write-Verbose -Verbose -Message "vPackScenario is true, copying to $(ob_outputDirectory)" + $pathForUpload = New-Item -ItemType Directory -Path '$(ob_outputDirectory)' -Force + Write-Verbose -Verbose -Message "pathForUpload: $pathForUpload" + Copy-Item -Path '${{ parameters.binPath }}\*' -Destination $pathForUpload -Recurse -Force -Verbose + Write-Verbose -Verbose -Message "Files copied to $pathForUpload" + } + elseif (-not $isGlobalTool) { + $pathForUpload = New-Item -ItemType Directory -Path '$(ob_outputDirectory)/Signed-$(Runtime)' -Force + Write-Verbose -Verbose -Message "pathForUpload: $pathForUpload" + Copy-Item -Path '${{ parameters.binPath }}\*' -Destination $pathForUpload -Recurse -Force -Verbose + Write-Verbose -Verbose -Message "Files copied to $pathForUpload" + } + else { + $pathForUpload = '${{ parameters.binPath }}' + } + + Write-Verbose "Copying third party signed files to the build folder" + $thirdPartySignedFilesPath = (Get-Item '$(Pipeline.Workspace)/thirdPartyToBeSigned').FullName + Update-PSSignedBuildFolder -BuildPath $pathForUpload -SignedFilesPath $thirdPartySignedFilesPath -OfficialBuild $officialBuild + + displayName: 'Copy signed files for upload' + +- template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml b/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml new file mode 100644 index 0000000000..5076fb4dd1 --- /dev/null +++ b/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml @@ -0,0 +1,71 @@ +parameters: + - name: cleanFirst + type: boolean + default: true + +# Stages the checked-out repository contents into a `PowerShell\` subfolder so that +# all script paths inherited from the OneBranch templates (which reference +# `$(Build.SourcesDirectory)\PowerShell\...` and `$env:REPOROOT/PowerShell/...`) keep +# resolving correctly without rewriting every callsite. +# +# Why this helper exists +# ---------------------- +# OneBranch's default checkout puts the repo at `\PowerShell` and exposes the +# parent as `$(Build.SourcesDirectory)`. 1ES PT's default checkout puts the repo at +# `$(Build.SourcesDirectory)` directly. Two ways to bridge the gap: +# (a) Rewrite every `\PowerShell\` path in every script (large surface). +# (b) Materialise a `\PowerShell\` subfolder at job start (small surface). +# We pick (b) for the initial migration to keep the diff reviewable. A follow-up cleanup +# pass can collapse the indirection once Official is also green. +# +# Why Copy-Item, not `git clone ` +# ---------------------------------------------- +# The OneBranch helper `cloneToOfficialPath.yml` does a real `git clone $env:REPOROOT +# $nativePath`. In OneBranch, source and dest are SIBLING folders so it works fine. +# In 1ES PT, source ($(Build.SourcesDirectory)) is the PARENT of dest. A +# `git clone \PowerShell` would try to copy the source repo INCLUDING the +# destination subfolder we're trying to create — fragile and likely to fail on Windows +# file-locking. A plain Copy-Item with a single-folder exclude is simpler and reliable. +# +# Also sets $env:REPOROOT for downstream consumers (SetVersionVariables.yml / +# set-reporoot.yml / insert-nuget-config-azfeed.yml all probe $env:REPOROOT or fall back +# to `./PowerShell/build.psm1`). + +steps: +- pwsh: | + $ErrorActionPreference = 'Stop' + $src = '$(Build.SourcesDirectory)' + $dst = Join-Path $src 'PowerShell' + + if (${{ parameters.cleanFirst }} -and (Test-Path $dst)) { + Remove-Item -Path $dst -Recurse -Force + } + New-Item -ItemType Directory -Path $dst -Force | Out-Null + + # Copy everything except: + # - the destination folder itself (would self-recurse) + # - the `.git` folder (large, locked .pack files often fail to copy on Windows, + # and the staged copy is only used for path-parity script consumption — Git + # operations stay against $(Build.SourcesDirectory) where checkout: self put them) + Get-ChildItem -Path $src -Force -Exclude 'PowerShell', '.git' | ForEach-Object { + Copy-Item -Path $_.FullName -Destination $dst -Recurse -Force + } + + # Verify critical files made it across — surface partial-copy failures NOW + # rather than letting downstream scripts fail with confusing "file not found" + # errors at build time. + $required = @( + Join-Path $dst 'build.psm1' + Join-Path $dst '.config/tsaoptions.json' + Join-Path $dst '.config/suppress.json' + ) + foreach ($p in $required) { + if (-not (Test-Path $p)) { + throw "Staged repo is missing required file: $p" + } + } + + # Expose REPOROOT for downstream helper templates (SetVersionVariables, etc.). + Write-Host "##vso[task.setvariable variable=REPOROOT]$dst" + Write-Verbose -Verbose "Repository staged at $dst" + displayName: 'Stage repo into $(Build.SourcesDirectory)\PowerShell (OneBranch path parity)' diff --git a/.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml b/.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml new file mode 100644 index 0000000000..5bef70f093 --- /dev/null +++ b/.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml @@ -0,0 +1,221 @@ +parameters: + - name: RUN_WINDOWS + type: boolean + default: true + - name: RUN_TEST_AND_RELEASE + type: boolean + default: true + - name: OfficialBuild + type: boolean + +# 1ES PT translation of /.pipelines/templates/stages/PowerShell-Coordinated_Packages-Stages.yml. +# Stage structure mirrors the OneBranch original verbatim; only the per-job pool blocks and +# template paths are rewritten. The 5 stages are: prep, macos, linux, windows, test_and_release_artifacts. + +stages: +- stage: prep + jobs: + - job: SetVars + displayName: Set Variables + # OneBranch: pool: { type: linux } (alias for an internal Linux container pool). + # 1ES PT requires explicit name/image/os. We use PowerShell1ES + PSMMSAzureLinux3.0-Secure + # because the original SetVars only needs a shell — no special Linux tooling. + pool: + name: PowerShell1ES + image: PSMMSAzureLinux3.0-Secure + os: linux + + variables: + - name: ob_outputDirectory # kept verbatim: build.psm1 reads $env:OB_OUTPUTDIRECTORY literally + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT/BuildJson' + - name: repoRoot + value: '$(Build.SourcesDirectory)' + + templateContext: + sdl: + codeSignValidation: + enabled: false + outputs: + - output: pipelineArtifact + targetPath: $(ob_outputDirectory) + artifactName: drop_prep_SetVars + + steps: + - checkout: self + clean: true + + - pwsh: | + Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + displayName: Capture environment variables + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + ob_restore_phase: false # `ob_restore_phase` is a no-op env var in 1ES PT — set to false to suppress + +- stage: macos + displayName: macOS - build and sign + dependsOn: ['prep'] + variables: + - name: ps_official_build + value: ${{ parameters.OfficialBuild }} + jobs: + - template: /.pipelines/1ES/templates/mac.yml@self + parameters: + buildArchitecture: x64 + - template: /.pipelines/1ES/templates/mac.yml@self + parameters: + buildArchitecture: arm64 + +- stage: linux + displayName: linux - build and sign + dependsOn: ['prep'] + variables: + - name: ps_official_build + value: ${{ parameters.OfficialBuild }} + jobs: + - template: /.pipelines/1ES/templates/linux.yml@self + parameters: + Runtime: 'linux-x64' + JobName: 'linux_x64' + + - template: /.pipelines/1ES/templates/linux.yml@self + parameters: + Runtime: 'linux-x64' + JobName: 'linux_x64_minSize' + BuildConfiguration: 'minSize' + + - template: /.pipelines/1ES/templates/linux.yml@self + parameters: + Runtime: 'linux-arm' + JobName: 'linux_arm' + + - template: /.pipelines/1ES/templates/linux.yml@self + parameters: + Runtime: 'linux-arm64' + JobName: 'linux_arm64' + + - template: /.pipelines/1ES/templates/linux.yml@self + parameters: + Runtime: 'fxdependent-linux-x64' + JobName: 'linux_fxd_x64_mariner' + + - template: /.pipelines/1ES/templates/linux.yml@self + parameters: + Runtime: 'fxdependent-linux-arm64' + JobName: 'linux_fxd_arm64_mariner' + + - template: /.pipelines/1ES/templates/linux.yml@self + parameters: + Runtime: 'fxdependent-noopt-linux-musl-x64' + JobName: 'linux_fxd_x64_alpine' + + - template: /.pipelines/1ES/templates/linux.yml@self + parameters: + Runtime: 'fxdependent' + JobName: 'linux_fxd' + + - template: /.pipelines/1ES/templates/linux.yml@self + parameters: + Runtime: 'linux-musl-x64' + JobName: 'linux_x64_alpine' + +- stage: windows + displayName: windows - build and sign + dependsOn: ['prep'] + condition: and(succeeded(),eq('${{ parameters.RUN_WINDOWS }}','true')) + variables: + - name: ps_official_build + value: ${{ parameters.OfficialBuild }} + jobs: + - template: /.pipelines/1ES/templates/windows-hosted-build.yml@self + parameters: + Architecture: x64 + BuildConfiguration: release + JobName: build_windows_x64_release + - template: /.pipelines/1ES/templates/windows-hosted-build.yml@self + parameters: + Architecture: x64 + BuildConfiguration: minSize + JobName: build_windows_x64_minSize_release + - template: /.pipelines/1ES/templates/windows-hosted-build.yml@self + parameters: + Architecture: x86 + JobName: build_windows_x86_release + - template: /.pipelines/1ES/templates/windows-hosted-build.yml@self + parameters: + Architecture: arm64 + JobName: build_windows_arm64_release + - template: /.pipelines/1ES/templates/windows-hosted-build.yml@self + parameters: + Architecture: fxdependent + JobName: build_windows_fxdependent_release + - template: /.pipelines/1ES/templates/windows-hosted-build.yml@self + parameters: + Architecture: fxdependentWinDesktop + JobName: build_windows_fxdependentWinDesktop_release + +- stage: test_and_release_artifacts + displayName: Test and Release Artifacts + dependsOn: ['prep'] + condition: and(succeeded(),eq('${{ parameters.RUN_TEST_AND_RELEASE }}','true')) + jobs: + - template: /.pipelines/1ES/templates/testartifacts.yml@self + + - job: release_json + displayName: Create and Upload release.json + pool: + name: PowerShell1ES + image: MMS2022 + os: windows + variables: + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: repoRoot + value: '$(Build.SourcesDirectory)' + - name: PowerShellRoot + value: '$(Build.SourcesDirectory)\PowerShell' + templateContext: + sdl: + codeSignValidation: + enabled: false + outputs: + - output: pipelineArtifact + targetPath: $(ob_outputDirectory) + artifactName: drop_test_and_release_artifacts_release_json + steps: + - checkout: self + clean: true + # Replicate OneBranch's $(Build.SourcesDirectory)\PowerShell convention so all the + # script paths below remain unchanged. See per-job templates for full explanation. + - template: /.pipelines/1ES/templates/stage-repo-under-PowerShell.yml@self + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + ob_restore_phase: false + - template: /.pipelines/templates/rebuild-branch-check.yml@self + - powershell: | + $metadata = Get-Content '$(Build.SourcesDirectory)/PowerShell/tools/metadata.json' -Raw | ConvertFrom-Json + + # Use the rebuild branch check from the template + $isRebuildBranch = '$(RebuildBranchCheck.IsRebuildBranch)' -eq 'true' + + # Don't mark as LTS release for rebuild branches + $LTS = $metadata.LTSRelease.Package -and -not $isRebuildBranch + + if ($isRebuildBranch) { + Write-Verbose -Message "Rebuild branch detected, not marking as LTS release" -Verbose + } + + @{ ReleaseVersion = "$(Version)"; LTSRelease = $LTS } | ConvertTo-Json | Out-File "$(Build.StagingDirectory)\release.json" + Get-Content "$(Build.StagingDirectory)\release.json" + + if (-not (Test-Path "$(ob_outputDirectory)\metadata")) { + New-Item -ItemType Directory -Path "$(ob_outputDirectory)\metadata" + } + + Copy-Item -Path "$(Build.StagingDirectory)\release.json" -Destination "$(ob_outputDirectory)\metadata" -Force + displayName: Create and upload release.json file to build artifact + retryCountOnTaskFailure: 2 + - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/1ES/templates/testartifacts.yml b/.pipelines/1ES/templates/testartifacts.yml new file mode 100644 index 0000000000..f07c98b9a0 --- /dev/null +++ b/.pipelines/1ES/templates/testartifacts.yml @@ -0,0 +1,163 @@ +jobs: +# 1ES PT translation of /.pipelines/templates/testartifacts.yml. +# Two jobs (win + nonwin). No signing required so this is a near-mechanical translation. + +- job: build_testartifacts_win + variables: + - name: NugetSecurityAnalysisWarningLevel + value: none + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: PowerShellRoot + value: '$(Build.SourcesDirectory)\PowerShell' + + displayName: Build windows test artifacts + condition: succeeded() + pool: + name: PowerShell1ES + image: MMS2022 + os: windows + + templateContext: + sdl: + tsa: + configFile: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + credscan: + suppressionsFile: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + codeSignValidation: + # Preserve OneBranch exclusions exactly; broad exclude pattern stays the same. + excludes: '-|**\*.ps1;-|**\*.psm1;-|**\*.ps1xml;-|**\*.psd1;-|**\*.exe;-|**\*.dll;-|**\*.cdxml' + outputs: + - output: pipelineArtifact + targetPath: $(ob_outputDirectory) + artifactName: drop_test_and_release_artifacts_build_testartifacts_win + + steps: + - checkout: self + clean: true + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + ob_restore_phase: false + + - template: /.pipelines/1ES/templates/stage-repo-under-PowerShell.yml@self + + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + # IMPORTANT: must be $(PowerShellRoot) (staged repo), NOT $(RepoRoot) (set + # earlier by set-reporoot.yml to `.` before staging ran). Writing + # nuget.config into the un-staged root would leave the staged repo's + # restore unauthorised against the private feed. + repoRoot: $(PowerShellRoot) + ob_restore_phase: false + + - template: /.pipelines/templates/install-dotnet.yml@self + + - pwsh: | + New-Item -Path '$(ob_outputDirectory)' -ItemType Directory -Force + Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1 + function BuildTestPackage([string] $runtime) + { + Write-Verbose -Verbose "Starting to build package for $runtime" + New-TestPackage -Destination $(System.ArtifactsDirectory) -Runtime $runtime + if (-not (Test-Path $(System.ArtifactsDirectory)/TestPackage.zip)) + { + throw "Test Package was not found at: $(System.ArtifactsDirectory)" + } + switch ($runtime) + { + win7-x64 { $packageName = "TestPackage-win-x64.zip" } + win7-x86 { $packageName = "TestPackage-win-x86.zip" } + win-arm64 { $packageName = "TestPackage-win-arm64.zip" } + } + Rename-Item $(System.ArtifactsDirectory)/TestPackage.zip $packageName + Copy-Item -Path $(System.ArtifactsDirectory)/$packageName -Destination $(ob_outputDirectory) -Force -Verbose + } + BuildTestPackage -runtime win7-x64 + BuildTestPackage -runtime win7-x86 + BuildTestPackage -runtime win-arm64 + displayName: Build test package and upload + retryCountOnTaskFailure: 1 + + - pwsh: | + Write-Host "This doesn't do anything but make the build phase run." + displayName: Dummy build task + +- job: build_testartifacts_nonwin + variables: + - name: NugetSecurityAnalysisWarningLevel + value: none + - group: DotNetPrivateBuildAccess + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: PowerShellRoot + value: '$(Build.SourcesDirectory)/PowerShell' + displayName: Build non-windows test artifacts + condition: succeeded() + pool: + name: PowerShell1ES + image: PSMMSAzureLinux3.0-Secure + os: linux + + templateContext: + sdl: + codeSignValidation: + enabled: false + outputs: + - output: pipelineArtifact + targetPath: $(ob_outputDirectory) + artifactName: drop_test_and_release_artifacts_build_testartifacts_nonwin + + steps: + - checkout: self + clean: true + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + ob_restore_phase: false + + - template: /.pipelines/1ES/templates/stage-repo-under-PowerShell.yml@self + + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: $(Build.SourcesDirectory)/PowerShell + ob_restore_phase: false + + - template: /.pipelines/templates/install-dotnet.yml@self + + - pwsh: | + New-Item -Path '$(ob_outputDirectory)' -ItemType Directory -Force + Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1 + function BuildTestPackage([string] $runtime) + { + Write-Verbose -Verbose "Starting to build package for $runtime" + New-TestPackage -Destination $(System.ArtifactsDirectory) -Runtime $runtime + if (-not (Test-Path $(System.ArtifactsDirectory)/TestPackage.zip)) + { + throw "Test Package was not found at: $(System.ArtifactsDirectory)" + } + switch ($runtime) + { + linux-x64 { $packageName = "TestPackage-linux-x64.zip" } + linux-arm { $packageName = "TestPackage-linux-arm.zip" } + linux-arm64 { $packageName = "TestPackage-linux-arm64.zip" } + osx-x64 { $packageName = "TestPackage-macOS.zip" } + linux-musl-x64 { $packageName = "TestPackage-alpine-x64.zip"} + } + Rename-Item $(System.ArtifactsDirectory)/TestPackage.zip $packageName + Copy-Item -Path $(System.ArtifactsDirectory)/$packageName -Destination $(ob_outputDirectory) -Force -Verbose + } + BuildTestPackage -runtime linux-x64 + BuildTestPackage -runtime linux-arm + BuildTestPackage -runtime linux-arm64 + BuildTestPackage -runtime osx-x64 + BuildTestPackage -runtime linux-musl-x64 + displayName: Build test package and upload + retryCountOnTaskFailure: 1 + + - pwsh: | + Write-Host "This doesn't do anything but make the build phase run." + displayName: Dummy build task diff --git a/.pipelines/1ES/templates/windows-hosted-build.yml b/.pipelines/1ES/templates/windows-hosted-build.yml new file mode 100644 index 0000000000..507e521bdf --- /dev/null +++ b/.pipelines/1ES/templates/windows-hosted-build.yml @@ -0,0 +1,333 @@ +parameters: + Architecture: 'x64' + BuildConfiguration: 'release' + JobName: 'build_windows' + +# 1ES PT translation of /.pipelines/templates/windows-hosted-build.yml. +# Single Windows job that BOTH builds and signs (the OneBranch original is the same shape). +# Three signing tasks STUBBED: +# 1. obp-file-signing.yml call for the main bin folder (Authenticode). +# 2. obp-file-signing.yml call for the global tool publish folder (fxdependent only). +# 3. Inline `Sign obj files` (external_distribution) and `Sign nupkg files` (nuget_cert_id). +# CodeQL3000Init/Finalize tasks DROPPED (1ES PT auto-injects via sdl.codeql config). +# cloneToOfficialPath.yml replaced with stage-repo-under-PowerShell.yml. + +jobs: +- job: build_windows_${{ parameters.Architecture }}_${{ parameters.BuildConfiguration }} + displayName: Build_Windows_${{ parameters.Architecture }}_${{ parameters.BuildConfiguration }} + condition: succeeded() + pool: + name: PowerShell1ES + image: MMS2022 + os: windows + variables: + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: DOTNET_NOLOGO + value: 1 + - group: DotNetPrivateBuildAccess + - group: certificate_logical_to_actual + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: Architecture + value: ${{ parameters.Architecture }} + - name: BuildConfiguration + value: ${{ parameters.BuildConfiguration }} + - name: PowerShellRoot + value: '$(Build.SourcesDirectory)\PowerShell' + + templateContext: + sdl: + codeSignValidation: + enabled: false + binskim: + enabled: true + tsa: + configFile: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + credscan: + suppressionsFile: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + sbom: + packageName: 'Microsoft.Powershell.Windows.${{ parameters.Architecture }}' + codeql: + compiled: + enabled: false + outputs: + - output: pipelineArtifact + targetPath: $(ob_outputDirectory) + artifactName: drop_windows_${{ parameters.JobName }} + + steps: + - checkout: self + clean: true + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + ob_restore_phase: false + + - template: /.pipelines/1ES/templates/stage-repo-under-PowerShell.yml@self + + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: $(PowerShellRoot) + + - template: /.pipelines/templates/install-dotnet.yml@self + + - pwsh: | + $runtime = switch ($env:Architecture) + { + "x64" { "win7-x64" } + "x86" { "win7-x86" } + "arm64" { "win-arm64" } + "fxdependent" { "fxdependent" } + "fxdependentWinDesktop" { "fxdependent-win-desktop" } + } + + $params = @{} + if ($env:BuildConfiguration -eq 'minSize') { + $params['ForMinimalSize'] = $true + } + + $vstsCommandString = "vso[task.setvariable variable=Runtime]$runtime" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + + Write-Verbose -Message "Building PowerShell with Runtime: $runtime for '$env:BuildConfiguration' configuration" + Import-Module -Name $(PowerShellRoot)/build.psm1 -Force + $buildWithSymbolsPath = New-Item -ItemType Directory -Path $(Pipeline.Workspace)/Symbols_$(Architecture) -Force + + Start-PSBootstrap -Scenario Package + $null = New-Item -ItemType Directory -Path $buildWithSymbolsPath -Force -Verbose + + $ReleaseTagParam = @{} + + if ($env:RELEASETAGVAR) { + $ReleaseTagParam['ReleaseTag'] = $env:RELEASETAGVAR + } + + Start-PSBuild -Runtime $runtime -Configuration Release -Output $buildWithSymbolsPath -Clean -PSModuleRestore @params @ReleaseTagParam + + $refFolderPath = Join-Path $buildWithSymbolsPath 'ref' + Write-Verbose -Verbose "refFolderPath: $refFolderPath" + $outputPath = Join-Path '$(ob_outputDirectory)' 'psoptions' + $null = New-Item -ItemType Directory -Path $outputPath -Force + $psOptPath = "$outputPath/psoptions.json" + Save-PSOptions -PSOptionsPath $psOptPath + + Write-Verbose -Verbose "Verifying pdbs exist in build folder" + $pdbs = Get-ChildItem -Path $buildWithSymbolsPath -Recurse -Filter *.pdb + if ($pdbs.Count -eq 0) { + Write-Error -Message "No pdbs found in build folder" + } + else { + Write-Verbose -Verbose "Found $($pdbs.Count) pdbs in build folder" + $pdbs | ForEach-Object { + Write-Verbose -Verbose "Pdb: $($_.FullName)" + } + + $pdbs | Compress-Archive -DestinationPath "$(ob_outputDirectory)/symbols.zip" -Update + } + + Write-Verbose -Verbose "Completed building PowerShell for '$env:BuildConfiguration' configuration" + displayName: 'Build Windows Universal - $(Architecture)-$(BuildConfiguration) Symbols folder' + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + + - pwsh: | + $runtime = switch ($env:Architecture) + { + "x64" { "win7-x64" } + "x86" { "win7-x86" } + "arm64" { "win-arm64" } + "fxdependent" { "fxdependent" } + "fxdependentWinDesktop" { "fxdependent-win-desktop" } + } + + Import-Module -Name $(PowerShellRoot)/build.psm1 -Force + Find-Dotnet + + ## Build global tool + Write-Verbose -Message "Building PowerShell global tool for Windows.x64" -Verbose + $globalToolCsProjDir = Join-Path $(PowerShellRoot) 'src' 'GlobalTools' 'PowerShell.Windows.x64' + Push-Location -Path $globalToolCsProjDir -Verbose + + $globalToolArtifactPath = Join-Path $(Build.SourcesDirectory) 'GlobalTool' + $vstsCommandString = "vso[task.setvariable variable=GlobalToolArtifactPath]${globalToolArtifactPath}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + + if ($env:RELEASETAGVAR) { + $ReleaseTagToUse = $env:RELEASETAGVAR -Replace '^v' + } + + Write-Verbose -Verbose "Building PowerShell global tool for Windows.x64 with cmdline: dotnet publish --no-self-contained --artifacts-path $globalToolArtifactPath /property:PackageVersion=$(Version) --configuration 'Release' /property:ReleaseTag=$ReleaseTagToUse" + dotnet publish --no-self-contained --artifacts-path $globalToolArtifactPath /property:PackageVersion=$(Version) --configuration 'Release' /property:ReleaseTag=$ReleaseTagToUse + $globalToolBuildModulePath = Join-Path $globalToolArtifactPath 'publish' 'PowerShell.Windows.x64' 'release' + Pop-Location + # do this to ensure everything gets signed. + Restore-PSModuleToBuild -PublishPath $globalToolBuildModulePath + + $buildWithSymbolsPath = Get-Item -Path "$(Pipeline.Workspace)/Symbols_$(Architecture)" + $refFolderPath = Join-Path $buildWithSymbolsPath 'ref' + Write-Verbose -Verbose "refFolderPath: $refFolderPath" + + # Copy reference assemblies + Copy-Item -Path $refFolderPath -Destination $globalToolBuildModulePath -Recurse -Force + + Write-Verbose -Verbose "clean unnecessary files in obj directory" + $objDir = Join-Path $globalToolArtifactPath 'obj' 'PowerShell.Windows.x64' 'release' + + $filesToKeep = @("apphost.exe", "PowerShell.Windows.x64.pdb", "PowerShell.Windows.x64.dll", "project.assets.json") + + # only four files are needed in obj folder for global tool packaging + Get-ChildItem -Path $objDir -File -Recurse | + Where-Object { -not $_.PSIsContainer } | + Where-Object { $_.name -notin $filesToKeep } | + Remove-Item -Verbose + + + displayName: 'Build Winx64 Global tool' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + + - pwsh: | + $platform = 'windows' + $vstsCommandString = "vso[task.setvariable variable=ArtifactPlatform]$platform" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + displayName: Set artifact platform + + - template: /.pipelines/1ES/templates/obp-file-signing.yml@self + parameters: + binPath: '$(Pipeline.Workspace)/Symbols_$(Architecture)' + OfficialBuild: $(ps_official_build) + + ## first we sign all the files in the bin folder + - ${{ if eq(variables['Architecture'], 'fxdependent') }}: + - template: /.pipelines/1ES/templates/obp-file-signing.yml@self + parameters: + binPath: '$(GlobalToolArtifactPath)/publish/PowerShell.Windows.x64/release' + globalTool: 'true' + OfficialBuild: $(ps_official_build) + + - pwsh: | + Get-ChildItem '$(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release' + displayName: Capture obj files + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + + ## STUB: original was `task: onebranch.pipeline.signing@1` with `signing_profile: external_distribution` + ## (KeyCode CP-230012). Sign obj files needed for the global tool packaging. + - pwsh: | + Write-Host '##[warning]STUB: would invoke ESRP signing with profile "external_distribution" (KeyCode CP-230012)' + Write-Host ' files_to_sign: **\*.dll;**\*.exe' + Write-Host ' search_root: $(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release' + if (Test-Path '$(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release') { + Get-ChildItem -Path '$(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release' -Recurse -Include '*.dll','*.exe' | + Select-Object -ExpandProperty FullName | + ForEach-Object { Write-Host " Would sign: $_" } + } + displayName: 'Sign obj files (STUB — ESRP onboarding pending)' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + + - pwsh: | + <# The way the packaging works is a bit tricky as when it is built, we cannot add the modules that come from gallery. + We have to use dotnet pack to build the nupkg and then expand it as a zip. + After expanding we restore the signed files for the modules from the gallery. + We also delete pdbs, content and contentFiles folder which are not necessary. + After that, we repack using Compress-Archive and rename it back to a nupkg. + #> + + Import-Module -Name $(PowerShellRoot)/build.psm1 -Force + Find-Dotnet + + $packagingStrings = Import-PowerShellDataFile "$(PowerShellRoot)\tools\packaging\packaging.strings.psd1" + + $outputPath = Join-Path '$(ob_outputDirectory)' 'globaltool' + $null = New-Item -ItemType Directory -Path $outputPath -Force + $globalToolCsProjDir = Join-Path $(PowerShellRoot) 'src' 'GlobalTools' 'PowerShell.Windows.x64' + Push-Location -Path $globalToolCsProjDir -Verbose + + if ($env:RELASETAGVAR) { + $ReleaseTagToUse = $env:RELASETAGVAR -Replace '^v' + } + + Write-Verbose -Verbose "Packing PowerShell global tool for Windows.x64 with cmdline: dotnet pack --output $outputPath --no-build --artifacts-path '$(GlobalToolArtifactPath)' /property:PackageVersion=$(Version) /property:PackageIcon=Powershell_64.png /property:Version=$(Version) /property:ReleaseTag=$ReleaseTagToUse" + + dotnet pack --output $outputPath --no-build --artifacts-path '$(GlobalToolArtifactPath)' /property:PackageVersion=$(Version) /property:PackageIcon=Powershell_64.png /property:Version=$(Version) /property:ReleaseTag=$ReleaseTagToUse + + Write-Verbose -Verbose "Deleting content and contentFiles folders from the nupkg" + + $nupkgs = Get-ChildItem -Path $outputPath -Filter powershell*.nupkg + + $nupkgName = $nupkgs.Name + $newName = $nupkgName -replace '(\.nupkg)$', '.zip' + Rename-Item -Path $nupkgs.FullName -NewName $newName + + $zipPath = Get-ChildItem -Path $outputPath -Filter powershell*.zip + + # Expand zip and remove content and contentFiles folders + Expand-Archive -Path $zipPath -DestinationPath "$outputPath\temp" -Force + + $modulesToCopy = @( + 'PowerShellGet' + 'PackageManagement' + 'Microsoft.PowerShell.PSResourceGet' + 'Microsoft.PowerShell.Archive' + 'PSReadLine' + 'Microsoft.PowerShell.ThreadJob' + ) + + $sourceModulePath = Join-Path '$(GlobalToolArtifactPath)' 'publish' 'PowerShell.Windows.x64' 'release' 'Modules' + $destModulesPath = Join-Path "$outputPath" 'temp' 'tools' 'net11.0' 'any' 'modules' + + $modulesToCopy | ForEach-Object { + $modulePath = Join-Path $sourceModulePath $_ + Copy-Item -Path $modulePath -Destination $destModulesPath -Recurse -Force + } + + # Copy ref assemblies + Copy-Item '$(Pipeline.Workspace)/Symbols_$(Architecture)/ref' "$outputPath\temp\tools\net11.0\any\ref" -Recurse -Force + + $contentPath = Join-Path "$outputPath\temp" 'content' + $contentFilesPath = Join-Path "$outputPath\temp" 'contentFiles' + + Remove-Item -Path $contentPath,$contentFilesPath -Recurse -Force + + # remove PDBs to reduce the size of the nupkg + Remove-Item -Path "$outputPath\temp\tools\net11.0\any\*.pdb" -Recurse -Force + + # create powershell.config.json + $config = [ordered]@{} + $config.Add("Microsoft.PowerShell:ExecutionPolicy", "RemoteSigned") + $config.Add("WindowsPowerShellCompatibilityModuleDenyList", @("PSScheduledJob", "BestPractices", "UpdateServices")) + + $configPublishPath = Join-Path "$outputPath" 'temp' 'tools' 'net11.0' 'any' "powershell.config.json" + Set-Content -Path $configPublishPath -Value ($config | ConvertTo-Json) -Force -ErrorAction Stop + + Compress-Archive -Path "$outputPath\temp\*" -DestinationPath "$outputPath\$nupkgName" -Force + + Remove-Item -Path "$outputPath\temp" -Recurse -Force + Remove-Item -Path $zipPath -Force + + if (-not (Test-Path "$outputPath\powershell.windows.x64.*.nupkg")) { + throw "Global tool package not found at $outputPath" + } + displayName: 'Pack Windows.x64 global tool' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + + ## STUB: original was `task: onebranch.pipeline.signing@1` with `cp_code: $(nuget_cert_id)` + ## (KeyCode CP-401405, NuGet package signing). + - pwsh: | + Write-Host '##[warning]STUB: would invoke ESRP signing with cp_code "$(nuget_cert_id)" (KeyCode CP-401405, NuGet)' + Write-Host ' files_to_sign: **\*.nupkg' + Write-Host ' search_root: $(ob_outputDirectory)\globaltool' + if (Test-Path '$(ob_outputDirectory)\globaltool') { + Get-ChildItem -Path '$(ob_outputDirectory)\globaltool' -Recurse -Filter '*.nupkg' | + Select-Object -ExpandProperty FullName | + ForEach-Object { Write-Host " Would sign: $_" } + } + displayName: 'Sign nupkg files (STUB — ESRP onboarding pending)' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + + - template: /.pipelines/templates/step/finalize.yml@self From ed8737db07c476c8189b59b4dfb75cdbaaffaf84 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 8 Jun 2026 16:16:25 -0500 Subject: [PATCH 08/19] Fix Linux pool/image pairing in 1ES NonOfficial scaffold Build #684757 (the first ADO build of the new 1ES scaffold) was cancelled at the prep stage with: Remote machine provider issue: Failed to request agent. Exception Image PSMMSAzureLinux3.0-Secure doesn't exist in pool PowerShell1ES Root cause: PowerShell1ES (the team-owned 1ES managed pool, id 108, 35 agents) only carries MMS2022 (Windows Server 2022). It does NOT have any Linux image deployed. The original scaffold tentatively assumed PSMMSAzureLinux3.0-Secure was present based on a partial read of the CloudTest.png inventory; build #684757 disproved that assumption. Fix: * prep/SetVars (Stages.yml) -> PowerShell1ES + MMS2022 + os: windows SetVars only runs pwsh to compute version variables; no Linux-specific tooling is required. Using the proven Windows pairing from the apiscan POC also avoids gating prep on a second pool authorization on first run. * linux.yml build job -> Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest + os: linux * testartifacts.yml nonwin job -> same Azure-Pipelines-1ESPT-ExDShared is the E+D org's shared 1ES PT-managed Linux pool (24 D2ads_v5 agents, 8 GiB RAM, 75 GiB temp storage, Ubuntu 22.04 LTS with the full Microsoft-hosted dev tooling). It is the documented standard fallback per the 1ES PT migration guide. If/when a team-owned Linux image lands on PowerShell1ES (would need a PsImageFactory PR similar to the unmerged addAzureCLI PR that put MMS2022 there), swap the pool: block in 2 places: templates/linux.yml build job and templates/testartifacts.yml nonwin job. Also updates the POOL/IMAGE STRATEGY comment in the entry pipeline and drops the now-resolved "Confirm Linux pool/image mapping" TODO. All 4 modified files validated with ConvertFrom-Yaml; no stale pairing references remain in code (only in comments documenting the lesson). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...Shell-Coordinated_Packages-NonOfficial.yml | 19 +++++++++++-------- .pipelines/1ES/templates/linux.yml | 13 ++++++++++--- ...PowerShell-Coordinated_Packages-Stages.yml | 10 ++++++---- .pipelines/1ES/templates/testartifacts.yml | 7 +++++-- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml index e9241b5677..e64bd2ed8e 100644 --- a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml +++ b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml @@ -33,13 +33,17 @@ # need their own pool/templateContext translations). # # POOL/IMAGE STRATEGY: -# - Windows jobs -> PowerShell1ES + MMS2022 (1ES-maintained, proven in apiscan POC) -# - Linux jobs -> PowerShell1ES + PSMMSAzureLinux3.0-Secure (closest match to -# the OneBranch container `mcr.microsoft.com/onebranch/azurelinux/build:3.0`). -# ⚠ Verify with team that PowerShell1ES pool carries this image. -# If not, fall back to `1ES-Linux-Ubuntu-2204` Microsoft-hosted pool. -# - macOS jobs -> Azure Pipelines + vmImage macOS-latest (no team-owned macOS image -# exists in the CloudTest inventory; macOS jobs MUST use the +# - Windows jobs -> PowerShell1ES + MMS2022 (team-owned 1ES managed pool; +# confirmed working in apiscan-gen-notice POC and SDL +# Sources Analysis job on build #684757). +# - Linux jobs -> Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest +# (E+D shared 1ES PT-managed pool, Ubuntu 22.04 with +# Microsoft-hosted standard tooling). Build #684757 +# proved PowerShell1ES does NOT carry a Linux image — +# `PSMMSAzureLinux3.0-Secure` is not deployed there. +# Swap to a team-owned Linux pool if/when one lands. +# - macOS jobs -> Azure Pipelines + vmImage macOS-latest (no team-owned +# macOS image exists; macOS jobs MUST use the # Microsoft-hosted Azure Pipelines shared pool). # # CODEQL POLICY: @@ -61,7 +65,6 @@ # - Stub signing: every onebranch.pipeline.signing@1 call replaced with a # `pwsh: Write-Host "TODO: ESRP sign ..."` placeholder. Pipeline runs end-to-end # but produces UNSIGNED artifacts — strictly for validating the migration scaffold. -# - Confirm Linux pool/image mapping (PowerShell1ES + PSMMSAzureLinux3.0-Secure). # - Once Unofficial green, build Official sibling with real ESRP signing. trigger: none diff --git a/.pipelines/1ES/templates/linux.yml b/.pipelines/1ES/templates/linux.yml index dc6ea7f508..a82d1f58a7 100644 --- a/.pipelines/1ES/templates/linux.yml +++ b/.pipelines/1ES/templates/linux.yml @@ -5,9 +5,16 @@ parameters: # 1ES PT translation of /.pipelines/templates/linux.yml. # Two jobs: -# 1. build_ — Linux pool (PowerShell1ES + PSMMSAzureLinux3.0-Secure) +# 1. build_ — Linux pool (Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest) # 2. sign_ — Windows pool (PowerShell1ES + MMS2022), STUBBED ESRP signing. # +# POOL CHOICE (Linux): Azure-Pipelines-1ESPT-ExDShared is the E+D org's shared +# 1ES PT-managed Linux pool (24 D2ads_v5 agents, Ubuntu 22.04 with full +# Microsoft-hosted tooling). The team-owned PowerShell1ES pool does NOT currently +# carry a Linux image (build #684757 confirmed `PSMMSAzureLinux3.0-Secure` is +# not deployed there). If/when a team-owned Linux image lands on PowerShell1ES, +# swap the `pool:` block below. +# # Notes # ----- # - CodeQL3000Init / CodeQL3000Finalize tasks DROPPED. 1ES PT auto-injects CodeQL @@ -22,8 +29,8 @@ jobs: displayName: Build_Linux_${{ parameters.Runtime }}_${{ parameters.BuildConfiguration }} condition: succeeded() pool: - name: PowerShell1ES - image: PSMMSAzureLinux3.0-Secure + name: Azure-Pipelines-1ESPT-ExDShared + image: ubuntu-latest os: linux variables: - name: NugetSecurityAnalysisWarningLevel diff --git a/.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml b/.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml index 5bef70f093..e9cc50e617 100644 --- a/.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml +++ b/.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml @@ -18,12 +18,14 @@ stages: - job: SetVars displayName: Set Variables # OneBranch: pool: { type: linux } (alias for an internal Linux container pool). - # 1ES PT requires explicit name/image/os. We use PowerShell1ES + PSMMSAzureLinux3.0-Secure - # because the original SetVars only needs a shell — no special Linux tooling. + # 1ES PT requires explicit name/image/os. SetVars only runs pwsh to compute + # version variables — no Linux-specific tooling required — so we use the + # proven Windows pairing (PowerShell1ES + MMS2022) from the apiscan POC. + # This also avoids gating prep on a second pool authorization on first run. pool: name: PowerShell1ES - image: PSMMSAzureLinux3.0-Secure - os: linux + image: MMS2022 + os: windows variables: - name: ob_outputDirectory # kept verbatim: build.psm1 reads $env:OB_OUTPUTDIRECTORY literally diff --git a/.pipelines/1ES/templates/testartifacts.yml b/.pipelines/1ES/templates/testartifacts.yml index f07c98b9a0..d8bc32278c 100644 --- a/.pipelines/1ES/templates/testartifacts.yml +++ b/.pipelines/1ES/templates/testartifacts.yml @@ -96,9 +96,12 @@ jobs: value: '$(Build.SourcesDirectory)/PowerShell' displayName: Build non-windows test artifacts condition: succeeded() + # POOL CHOICE: Azure-Pipelines-1ESPT-ExDShared is the E+D shared 1ES PT + # Linux pool (Ubuntu 22.04). PowerShell1ES has no Linux image deployed (see + # linux.yml header). Swap to a team-owned Linux pool here if one lands. pool: - name: PowerShell1ES - image: PSMMSAzureLinux3.0-Secure + name: Azure-Pipelines-1ESPT-ExDShared + image: ubuntu-latest os: linux templateContext: From f331e586870c88859bea357f61c4e0a02532861c Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Mon, 8 Jun 2026 16:27:54 -0500 Subject: [PATCH 09/19] Switch 1ES PT Linux jobs to PowerShell1ES + PSMMSUbuntu22.04-Secure After inspecting the PsImageFactory repo, PowerShell1ES.bicep already declares PSMMSUbuntu22.04-Secure (and PSMMSUbuntu20.04-Secure) in its images[] array. Build #684757 failed for PSMMSAzureLinux3.0-Secure specifically because that image has its own bicep but was never plumbed into the pool resource. Swap Linux jobs back to the team-owned PowerShell1ES pool using Ubuntu 22.04 so we're not dependent on the external E+D shared pool. If the next build fails the same way (Ubuntu 22 declared in bicep but deploy.ps1 was never run to push it to Azure), fall back to Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest, or open a #38412-style PsImageFactory PR to plumb in PSMMSAzureLinux3.0-Secure. Affected: - .pipelines/1ES/templates/linux.yml (build job) - .pipelines/1ES/templates/testartifacts.yml (nonwin job) - .pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml (POOL/IMAGE STRATEGY comment block) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...Shell-Coordinated_Packages-NonOfficial.yml | 15 ++++++++------ .pipelines/1ES/templates/linux.yml | 20 ++++++++++--------- .pipelines/1ES/templates/testartifacts.yml | 12 ++++++----- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml index e64bd2ed8e..3a092981be 100644 --- a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml +++ b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml @@ -36,12 +36,15 @@ # - Windows jobs -> PowerShell1ES + MMS2022 (team-owned 1ES managed pool; # confirmed working in apiscan-gen-notice POC and SDL # Sources Analysis job on build #684757). -# - Linux jobs -> Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest -# (E+D shared 1ES PT-managed pool, Ubuntu 22.04 with -# Microsoft-hosted standard tooling). Build #684757 -# proved PowerShell1ES does NOT carry a Linux image — -# `PSMMSAzureLinux3.0-Secure` is not deployed there. -# Swap to a team-owned Linux pool if/when one lands. +# - Linux jobs -> PowerShell1ES + PSMMSUbuntu22.04-Secure (team-owned 1ES +# managed pool). PowerShell1ES.bicep in the PsImageFactory +# repo declares PSMMSUbuntu22.04-Secure in its images[] +# array. Build #684757 confirmed `PSMMSAzureLinux3.0-Secure` +# is NOT plumbed into PowerShell1ES (separate image bicep +# exists but no entry in the pool resource), so we pick +# Ubuntu 22 instead. Fallback if Ubuntu 22 isn't actually +# deployed yet: Azure-Pipelines-1ESPT-ExDShared + +# ubuntu-latest + os: linux. # - macOS jobs -> Azure Pipelines + vmImage macOS-latest (no team-owned # macOS image exists; macOS jobs MUST use the # Microsoft-hosted Azure Pipelines shared pool). diff --git a/.pipelines/1ES/templates/linux.yml b/.pipelines/1ES/templates/linux.yml index a82d1f58a7..22e56ae7e8 100644 --- a/.pipelines/1ES/templates/linux.yml +++ b/.pipelines/1ES/templates/linux.yml @@ -5,15 +5,17 @@ parameters: # 1ES PT translation of /.pipelines/templates/linux.yml. # Two jobs: -# 1. build_ — Linux pool (Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest) +# 1. build_ — Linux pool (PowerShell1ES + PSMMSUbuntu22.04-Secure) # 2. sign_ — Windows pool (PowerShell1ES + MMS2022), STUBBED ESRP signing. # -# POOL CHOICE (Linux): Azure-Pipelines-1ESPT-ExDShared is the E+D org's shared -# 1ES PT-managed Linux pool (24 D2ads_v5 agents, Ubuntu 22.04 with full -# Microsoft-hosted tooling). The team-owned PowerShell1ES pool does NOT currently -# carry a Linux image (build #684757 confirmed `PSMMSAzureLinux3.0-Secure` is -# not deployed there). If/when a team-owned Linux image lands on PowerShell1ES, -# swap the `pool:` block below. +# POOL CHOICE (Linux): PowerShell1ES.bicep in the PsImageFactory repo declares +# `PSMMSUbuntu22.04-Secure` in its images[] array, so the team-owned 1ES managed +# pool can serve Linux jobs without an external pool dependency. (The earlier +# build #684757 failure was for `PSMMSAzureLinux3.0-Secure` specifically, which +# has its own image bicep but was never plumbed into PowerShell1ES.bicep.) +# If Ubuntu 22 isn't actually deployed to the pool (PsImageFactory has no +# auto-deploy pipeline; deploy.ps1 is run manually), fall back to +# `Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest + os: linux`. # # Notes # ----- @@ -29,8 +31,8 @@ jobs: displayName: Build_Linux_${{ parameters.Runtime }}_${{ parameters.BuildConfiguration }} condition: succeeded() pool: - name: Azure-Pipelines-1ESPT-ExDShared - image: ubuntu-latest + name: PowerShell1ES + image: PSMMSUbuntu22.04-Secure os: linux variables: - name: NugetSecurityAnalysisWarningLevel diff --git a/.pipelines/1ES/templates/testartifacts.yml b/.pipelines/1ES/templates/testartifacts.yml index d8bc32278c..a6df017aed 100644 --- a/.pipelines/1ES/templates/testartifacts.yml +++ b/.pipelines/1ES/templates/testartifacts.yml @@ -96,12 +96,14 @@ jobs: value: '$(Build.SourcesDirectory)/PowerShell' displayName: Build non-windows test artifacts condition: succeeded() - # POOL CHOICE: Azure-Pipelines-1ESPT-ExDShared is the E+D shared 1ES PT - # Linux pool (Ubuntu 22.04). PowerShell1ES has no Linux image deployed (see - # linux.yml header). Swap to a team-owned Linux pool here if one lands. + # POOL CHOICE: PowerShell1ES + PSMMSUbuntu22.04-Secure (team-owned 1ES managed + # pool with Ubuntu 22 declared in PowerShell1ES.bicep). If the image isn't + # actually deployed in CloudTest, fall back to + # `Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest + os: linux`. See linux.yml + # header comment for the full pool-choice rationale. pool: - name: Azure-Pipelines-1ESPT-ExDShared - image: ubuntu-latest + name: PowerShell1ES + image: PSMMSUbuntu22.04-Secure os: linux templateContext: From a1922a6d5bfa761287a892208dee2701ac9754aa Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 9 Jun 2026 09:58:17 -0500 Subject: [PATCH 10/19] Fix 4 root causes from 1ES NonOfficial build #685762 Cluster 1 (Linux pool): revert linux.yml + testartifacts.yml to Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest. PSMMSUbuntu22.04-Secure is declared in PowerShell1ES.bicep and deployed, but the image lacks the 1es-pt-prerequisites artifact required by 1ES PT (https://aka.ms/1espt/image-prerequisites). E+D shared images ship the artifact by default. Long-term: PsImageFactory PR to add the artifact to PSMMS* images. Updated header comments to capture the lesson. Cluster 2 (Switch-PSNugetConfig): add 1ES-specific .pipelines/1ES/templates/insert-nuget-config-azfeed.yml that resets `$global:LASTEXITCODE = 0` after the call. Switch-PSNugetConfig -> New-NugetConfigFile runs `git update-index --skip-worktree` against the regenerated nuget.config. The staged repo (Copy-Item without .git) isn't a git workspace, so git fails benignly and leaves $LASTEXITCODE non-zero. pwsh 7.4+ propagates that to the process exit code -> ADO task fails despite Switch-PSNugetConfig actually completing. The `skip-worktree` is purely defensive against accidental commits, which is moot on a Copy-Item'd staging folder. Repointed 5 callsites (linux.yml, mac.yml, windows-hosted-build.yml, testartifacts.yml win + nonwin) to the 1ES wrapper. apiscan POC left on the shared template because its staging path differs. Cluster 3 (Linux nuget capture path): downstream of cluster 1; auto- resolved by the pool revert. Cluster 4 (mac.yml BuildArchitecture): line 119 referenced $(BuildArchitecture) on the BUILD job, but only the SIGN job declared the variable. Changed to template parameter substitution (${{ parameters.buildArchitecture }}) at compile time. Cleaner than mirroring the variable on the build job. All 6 modified YAML files validated with ConvertFrom-Yaml. Playbook updated with Gotchas #4, #5, #6. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...Shell-Coordinated_Packages-NonOfficial.yml | 23 +++-- .../templates/insert-nuget-config-azfeed.yml | 86 +++++++++++++++++++ .pipelines/1ES/templates/linux.yml | 32 ++++--- .pipelines/1ES/templates/mac.yml | 4 +- .pipelines/1ES/templates/testartifacts.yml | 17 ++-- .../1ES/templates/windows-hosted-build.yml | 2 +- 6 files changed, 131 insertions(+), 33 deletions(-) create mode 100644 .pipelines/1ES/templates/insert-nuget-config-azfeed.yml diff --git a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml index 3a092981be..12e28183c0 100644 --- a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml +++ b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml @@ -36,15 +36,20 @@ # - Windows jobs -> PowerShell1ES + MMS2022 (team-owned 1ES managed pool; # confirmed working in apiscan-gen-notice POC and SDL # Sources Analysis job on build #684757). -# - Linux jobs -> PowerShell1ES + PSMMSUbuntu22.04-Secure (team-owned 1ES -# managed pool). PowerShell1ES.bicep in the PsImageFactory -# repo declares PSMMSUbuntu22.04-Secure in its images[] -# array. Build #684757 confirmed `PSMMSAzureLinux3.0-Secure` -# is NOT plumbed into PowerShell1ES (separate image bicep -# exists but no entry in the pool resource), so we pick -# Ubuntu 22 instead. Fallback if Ubuntu 22 isn't actually -# deployed yet: Azure-Pipelines-1ESPT-ExDShared + -# ubuntu-latest + os: linux. +# - Linux jobs -> Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest. E+D +# shared images are guaranteed to carry the +# `1es-pt-prerequisites` artifact required by 1ES PT. +# We tried `PowerShell1ES + PSMMSUbuntu22.04-Secure` in +# build #685762 — the image IS declared in +# PowerShell1ES.bicep AND was deployed (pre-job log +# confirmed imageVersion=23.0.0), but the 1ES PT validator +# rejected it with "Using an image without +# 1es-pt-prerequisites artifact is not allowed" +# (https://aka.ms/1espt/image-prerequisites). Declaring +# an image in a team-owned pool is necessary but not +# sufficient — the image build itself must bake in the +# `1es-pt-prerequisites` artifact. Long-term we need a +# PsImageFactory PR to add that artifact to PSMMS* images. # - macOS jobs -> Azure Pipelines + vmImage macOS-latest (no team-owned # macOS image exists; macOS jobs MUST use the # Microsoft-hosted Azure Pipelines shared pool). diff --git a/.pipelines/1ES/templates/insert-nuget-config-azfeed.yml b/.pipelines/1ES/templates/insert-nuget-config-azfeed.yml new file mode 100644 index 0000000000..ee994eaf5f --- /dev/null +++ b/.pipelines/1ES/templates/insert-nuget-config-azfeed.yml @@ -0,0 +1,86 @@ +# 1ES-specific translation of /.pipelines/templates/insert-nuget-config-azfeed.yml. +# +# WHY A 1ES-LOCAL COPY EXISTS: +# The shared template calls `Switch-PSNugetConfig` (from build.psm1), which calls +# `New-NugetConfigFile`, which calls `git update-index --skip-worktree` against +# each regenerated nuget.config to prevent accidental commits of feed credentials. +# +# In OneBranch the staged repo IS a git workspace (cloneToOfficialPath.yml does a +# real `git clone`), so skip-worktree succeeds. In our 1ES translation, +# `stage-repo-under-PowerShell.yml` deliberately Copy-Items the tree WITHOUT `.git` +# (the staged copy is throwaway, no risk of accidental commit). git then walks +# upward to `$(Build.SourcesDirectory)\.git`, sees `PowerShell/nuget.config` as +# the relative path, and exits non-zero with `fatal: Unable to mark file +# PowerShell/nuget.config` because that path isn't in the original index. +# +# The git failure is semantically a no-op for our staging design: there's no git +# workspace to mark, and no risk of accidental commit from a Copy-Item'd folder. +# But pwsh 7.4+ propagates `$LASTEXITCODE` to the process exit code, so the ADO +# pwsh task fails. This wrapper resets `$LASTEXITCODE = 0` immediately after +# `Switch-PSNugetConfig` so the benign git failure doesn't fail the build. +# +# Stays out of build.psm1 (shared with OneBranch) — the fix is local to the 1ES +# scaffold. If/when build.psm1 is hardened to tolerate non-git destinations +# itself, this wrapper can be replaced with a direct reference to the shared +# template. + +parameters: +- name: "repoRoot" + default: $(REPOROOT) +- name: "ob_restore_phase" + type: boolean + default: true + +steps: +- task: NuGetAuthenticate@1 + displayName: Install Azure Artifacts Credential Provider + inputs: + forceReinstallCredentialProvider: true + +- pwsh: | + try { + $configPath = "${env:NugetConfigDir}/nuget.config" + Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force + + Write-Verbose -Verbose "Running: Switch-PSNugetConfig -Source Private -UserName '$(AzDevopsFeedUserNameKVPAT)' -ClearTextPAT ''" + Switch-PSNugetConfig -Source Private -UserName '$(AzDevopsFeedUserNameKVPAT)' -ClearTextPAT '$(powershellPackageReadPat)' + + # New-NugetConfigFile (inside Switch-PSNugetConfig) runs `git update-index + # --skip-worktree` against the regenerated nuget.config. The staged repo + # is not a git workspace (Copy-Item without .git), so git fails benignly + # and leaves $LASTEXITCODE non-zero. Reset so pwsh exits 0 when the rest + # of the script (Test-Path check below) succeeds. + $global:LASTEXITCODE = 0 + + if(-not (Test-Path $configPath)) + { + throw "nuget.config is not created" + } + } + catch { + Get-Error + throw + } + displayName: 'Switch to production Azure DevOps feed for all nuget.configs' + condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], '')) + env: + NugetConfigDir: ${{ parameters.repoRoot }}/src/Modules + ob_restore_phase: ${{ parameters.ob_restore_phase }} + +- pwsh: | + Get-ChildItem ${{ parameters.repoRoot }}/nuget.config -Recurse | Foreach-Object { + Write-Verbose -Verbose "--- START $($_.fullname) ---" + get-content $_.fullname | Out-String -width 9999 -Stream | write-Verbose -Verbose + Write-Verbose -Verbose "--- END $($_.fullname) ---" + } + displayName: 'Capture all nuget.config files' + condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], '')) + env: + ob_restore_phase: ${{ parameters.ob_restore_phase }} + +- pwsh: | + Get-ChildItem -Path env:VSS* | Out-String -width 9999 -Stream | write-Verbose -Verbose + displayName: Capture VSS* Environment + condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], '')) + env: + ob_restore_phase: ${{ parameters.ob_restore_phase }} diff --git a/.pipelines/1ES/templates/linux.yml b/.pipelines/1ES/templates/linux.yml index 22e56ae7e8..338f3012c9 100644 --- a/.pipelines/1ES/templates/linux.yml +++ b/.pipelines/1ES/templates/linux.yml @@ -5,17 +5,25 @@ parameters: # 1ES PT translation of /.pipelines/templates/linux.yml. # Two jobs: -# 1. build_ — Linux pool (PowerShell1ES + PSMMSUbuntu22.04-Secure) +# 1. build_ — Linux pool (Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest) # 2. sign_ — Windows pool (PowerShell1ES + MMS2022), STUBBED ESRP signing. # -# POOL CHOICE (Linux): PowerShell1ES.bicep in the PsImageFactory repo declares -# `PSMMSUbuntu22.04-Secure` in its images[] array, so the team-owned 1ES managed -# pool can serve Linux jobs without an external pool dependency. (The earlier -# build #684757 failure was for `PSMMSAzureLinux3.0-Secure` specifically, which -# has its own image bicep but was never plumbed into PowerShell1ES.bicep.) -# If Ubuntu 22 isn't actually deployed to the pool (PsImageFactory has no -# auto-deploy pipeline; deploy.ps1 is run manually), fall back to -# `Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest + os: linux`. +# POOL CHOICE (Linux): Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest + os: linux +# is the E+D shared 1ES managed pool whose images are guaranteed to ship with the +# `1es-pt-prerequisites` artifact (the tooling 1ES PT injects into every job). +# +# We initially tried `PowerShell1ES + PSMMSUbuntu22.04-Secure` (v3 swap, commit +# 1b9775add). PSMMSUbuntu22.04-Secure IS declared in PowerShell1ES.bicep AND was +# deployed (build #685762's pre-job log confirmed imageVersion=23.0.0). But the +# 1ES PT validator rejected it with: +# 1ES PT Error: Using an image without 1es-pt-prerequisites artifact is not allowed +# See https://aka.ms/1espt/image-prerequisites — a team-owned image bicep that +# DECLARES an image is necessary but not sufficient; the image build itself must +# bake in the `1es-pt-prerequisites` artifact (which the team-owned PSMMS* images +# currently don't). +# Long-term: open a PsImageFactory PR to add the artifact to PSMMSUbuntu22.04-Secure +# so we can move Linux back to the team-owned pool. In the meantime, E+D shared +# is the path of least resistance. # # Notes # ----- @@ -31,8 +39,8 @@ jobs: displayName: Build_Linux_${{ parameters.Runtime }}_${{ parameters.BuildConfiguration }} condition: succeeded() pool: - name: PowerShell1ES - image: PSMMSUbuntu22.04-Secure + name: Azure-Pipelines-1ESPT-ExDShared + image: ubuntu-latest os: linux variables: - name: NugetSecurityAnalysisWarningLevel @@ -80,7 +88,7 @@ jobs: - template: /.pipelines/1ES/templates/stage-repo-under-PowerShell.yml@self - - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + - template: /.pipelines/1ES/templates/insert-nuget-config-azfeed.yml@self parameters: repoRoot: $(PowerShellRoot) diff --git a/.pipelines/1ES/templates/mac.yml b/.pipelines/1ES/templates/mac.yml index 1e09e8f627..d42788ab77 100644 --- a/.pipelines/1ES/templates/mac.yml +++ b/.pipelines/1ES/templates/mac.yml @@ -73,7 +73,7 @@ jobs: env: __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + - template: /.pipelines/1ES/templates/insert-nuget-config-azfeed.yml@self parameters: repoRoot: $(PowerShellRoot) - pwsh: | @@ -116,7 +116,7 @@ jobs: # tolerates this); if it fails silently, the sign job's DownloadPipelineArtifact # call still has an artifact to fetch. - pwsh: | - Write-Host "##vso[artifact.upload containerfolder=macosBinResults-$(BuildArchitecture);artifactname=macosBinResults-$(BuildArchitecture)]$(OB_OUTPUTDIRECTORY)" + Write-Host "##vso[artifact.upload containerfolder=macosBinResults-${{ parameters.buildArchitecture }};artifactname=macosBinResults-${{ parameters.buildArchitecture }}]$(OB_OUTPUTDIRECTORY)" displayName: 'Upload macOS build artifact (fallback for custom pool)' - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/1ES/templates/testartifacts.yml b/.pipelines/1ES/templates/testartifacts.yml index a6df017aed..24e1dead12 100644 --- a/.pipelines/1ES/templates/testartifacts.yml +++ b/.pipelines/1ES/templates/testartifacts.yml @@ -44,7 +44,7 @@ jobs: - template: /.pipelines/1ES/templates/stage-repo-under-PowerShell.yml@self - - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + - template: /.pipelines/1ES/templates/insert-nuget-config-azfeed.yml@self parameters: # IMPORTANT: must be $(PowerShellRoot) (staged repo), NOT $(RepoRoot) (set # earlier by set-reporoot.yml to `.` before staging ran). Writing @@ -96,14 +96,13 @@ jobs: value: '$(Build.SourcesDirectory)/PowerShell' displayName: Build non-windows test artifacts condition: succeeded() - # POOL CHOICE: PowerShell1ES + PSMMSUbuntu22.04-Secure (team-owned 1ES managed - # pool with Ubuntu 22 declared in PowerShell1ES.bicep). If the image isn't - # actually deployed in CloudTest, fall back to - # `Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest + os: linux`. See linux.yml - # header comment for the full pool-choice rationale. + # POOL CHOICE: Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest. Team-owned + # PSMMS* images don't ship with the `1es-pt-prerequisites` artifact baked in + # (build #685762 was rejected by the 1ES PT validator on this exact basis). + # See linux.yml header for the full rationale. pool: - name: PowerShell1ES - image: PSMMSUbuntu22.04-Secure + name: Azure-Pipelines-1ESPT-ExDShared + image: ubuntu-latest os: linux templateContext: @@ -126,7 +125,7 @@ jobs: - template: /.pipelines/1ES/templates/stage-repo-under-PowerShell.yml@self - - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + - template: /.pipelines/1ES/templates/insert-nuget-config-azfeed.yml@self parameters: repoRoot: $(Build.SourcesDirectory)/PowerShell ob_restore_phase: false diff --git a/.pipelines/1ES/templates/windows-hosted-build.yml b/.pipelines/1ES/templates/windows-hosted-build.yml index 507e521bdf..9f50ee6766 100644 --- a/.pipelines/1ES/templates/windows-hosted-build.yml +++ b/.pipelines/1ES/templates/windows-hosted-build.yml @@ -67,7 +67,7 @@ jobs: - template: /.pipelines/1ES/templates/stage-repo-under-PowerShell.yml@self - - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + - template: /.pipelines/1ES/templates/insert-nuget-config-azfeed.yml@self parameters: repoRoot: $(PowerShellRoot) From aa24b18c4da87d37d06b4b1631bb693d694c0e36 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 9 Jun 2026 10:02:38 -0500 Subject: [PATCH 11/19] Revert Linux jobs to PowerShell1ES pool (C+AI org governance) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per team policy: PowerShell is in the C+AI org and must NOT use pools owned by other orgs (e.g., E+D's Azure-Pipelines-1ESPT-ExDShared). The previous commit used that shared pool as a workaround for the missing 1es-pt-prerequisites artifact on PSMMSUbuntu22.04-Secure — that was a governance violation regardless of the convenience. Reverted linux.yml + testartifacts.yml + entry-pipeline comment block back to PowerShell1ES + PSMMSUbuntu22.04-Secure + os: linux. Updated all comments to: (a) State the governance rule explicitly (team-owned pools only). (b) Document the BLOCKER: PSMMSUbuntu22.04-Secure image build lacks the 1es-pt-prerequisites artifact required by 1ES PT (https://aka.ms/1espt/image-prerequisites). Build #685762 was rejected on this basis. (c) State the RESOLUTION path: PsImageFactory PR to bake the artifact into the PSMMSUbuntu22.04-Secure image build, then redeploy. Linux jobs of this pipeline cannot pass end-to-end until that work lands. Other fixes from the previous commit (nuget LASTEXITCODE wrapper for Windows + mac BuildArchitecture parameter substitution) are RETAINED. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...Shell-Coordinated_Packages-NonOfficial.yml | 36 ++++++++++--------- .pipelines/1ES/templates/linux.yml | 33 ++++++++--------- .pipelines/1ES/templates/testartifacts.yml | 14 ++++---- 3 files changed, 43 insertions(+), 40 deletions(-) diff --git a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml index 12e28183c0..477b03637c 100644 --- a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml +++ b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml @@ -33,26 +33,30 @@ # need their own pool/templateContext translations). # # POOL/IMAGE STRATEGY: +# PowerShell is in the C+AI org. We use ONLY team-owned pools — never +# pools owned by other orgs (e.g., E+D's Azure-Pipelines-1ESPT-ExDShared). +# # - Windows jobs -> PowerShell1ES + MMS2022 (team-owned 1ES managed pool; # confirmed working in apiscan-gen-notice POC and SDL # Sources Analysis job on build #684757). -# - Linux jobs -> Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest. E+D -# shared images are guaranteed to carry the -# `1es-pt-prerequisites` artifact required by 1ES PT. -# We tried `PowerShell1ES + PSMMSUbuntu22.04-Secure` in -# build #685762 — the image IS declared in -# PowerShell1ES.bicep AND was deployed (pre-job log -# confirmed imageVersion=23.0.0), but the 1ES PT validator -# rejected it with "Using an image without -# 1es-pt-prerequisites artifact is not allowed" -# (https://aka.ms/1espt/image-prerequisites). Declaring -# an image in a team-owned pool is necessary but not -# sufficient — the image build itself must bake in the -# `1es-pt-prerequisites` artifact. Long-term we need a -# PsImageFactory PR to add that artifact to PSMMS* images. -# - macOS jobs -> Azure Pipelines + vmImage macOS-latest (no team-owned +# - Linux jobs -> PowerShell1ES + PSMMSUbuntu22.04-Secure. +# BLOCKER: the image build lacks the +# `1es-pt-prerequisites` artifact required by 1ES PT +# (https://aka.ms/1espt/image-prerequisites). Build +# #685762 was rejected with "Using an image without +# 1es-pt-prerequisites artifact is not allowed". +# Declaring the image in PowerShell1ES.bicep and +# deploying it is necessary but not sufficient — the +# image build itself must bake in the artifact. +# PsImageFactory PR is required to add the artifact to +# the PSMMSUbuntu22.04-Secure image build, then +# redeploy. This pipeline cannot pass end-to-end on +# Linux until that's done. +# - macOS jobs -> Azure Pipelines + vmImage macOS-latest. No team-owned # macOS image exists; macOS jobs MUST use the -# Microsoft-hosted Azure Pipelines shared pool). +# Microsoft-hosted Azure Pipelines pool (this is the +# default 1ES PT-allowed Microsoft pool, not an +# external org's shared pool). # # CODEQL POLICY: # - 1ES PT auto-runs CodeQL3000 on the ADO repo default branch only by default. diff --git a/.pipelines/1ES/templates/linux.yml b/.pipelines/1ES/templates/linux.yml index 338f3012c9..75a0e1c207 100644 --- a/.pipelines/1ES/templates/linux.yml +++ b/.pipelines/1ES/templates/linux.yml @@ -5,25 +5,22 @@ parameters: # 1ES PT translation of /.pipelines/templates/linux.yml. # Two jobs: -# 1. build_ — Linux pool (Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest) +# 1. build_ — Linux pool (PowerShell1ES + PSMMSUbuntu22.04-Secure) # 2. sign_ — Windows pool (PowerShell1ES + MMS2022), STUBBED ESRP signing. # -# POOL CHOICE (Linux): Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest + os: linux -# is the E+D shared 1ES managed pool whose images are guaranteed to ship with the -# `1es-pt-prerequisites` artifact (the tooling 1ES PT injects into every job). +# POOL CHOICE (Linux): PowerShell1ES + PSMMSUbuntu22.04-Secure + os: linux. +# PowerShell is in the C+AI org; we use ONLY team-owned pools. We do NOT use +# shared pools from other orgs (e.g., E+D's Azure-Pipelines-1ESPT-ExDShared) +# even when they're convenient — that's a governance violation. # -# We initially tried `PowerShell1ES + PSMMSUbuntu22.04-Secure` (v3 swap, commit -# 1b9775add). PSMMSUbuntu22.04-Secure IS declared in PowerShell1ES.bicep AND was -# deployed (build #685762's pre-job log confirmed imageVersion=23.0.0). But the -# 1ES PT validator rejected it with: -# 1ES PT Error: Using an image without 1es-pt-prerequisites artifact is not allowed -# See https://aka.ms/1espt/image-prerequisites — a team-owned image bicep that -# DECLARES an image is necessary but not sufficient; the image build itself must -# bake in the `1es-pt-prerequisites` artifact (which the team-owned PSMMS* images -# currently don't). -# Long-term: open a PsImageFactory PR to add the artifact to PSMMSUbuntu22.04-Secure -# so we can move Linux back to the team-owned pool. In the meantime, E+D shared -# is the path of least resistance. +# KNOWN BLOCKER: PSMMSUbuntu22.04-Secure as currently built lacks the +# `1es-pt-prerequisites` artifact. The bicep declaration and image deployment +# succeed, but the 1ES PT pre-job validator rejects every job that lands on it +# with: "Using an image without 1es-pt-prerequisites artifact is not allowed" +# (https://aka.ms/1espt/image-prerequisites). Build #685762 hit this. +# RESOLUTION: PsImageFactory PR is required to bake the `1es-pt-prerequisites` +# artifact into the PSMMSUbuntu22.04-Secure image build, then redeploy to +# PowerShell1ES. This file should not be merged/run until that's done. # # Notes # ----- @@ -39,8 +36,8 @@ jobs: displayName: Build_Linux_${{ parameters.Runtime }}_${{ parameters.BuildConfiguration }} condition: succeeded() pool: - name: Azure-Pipelines-1ESPT-ExDShared - image: ubuntu-latest + name: PowerShell1ES + image: PSMMSUbuntu22.04-Secure os: linux variables: - name: NugetSecurityAnalysisWarningLevel diff --git a/.pipelines/1ES/templates/testartifacts.yml b/.pipelines/1ES/templates/testartifacts.yml index 24e1dead12..2f0b6a30d0 100644 --- a/.pipelines/1ES/templates/testartifacts.yml +++ b/.pipelines/1ES/templates/testartifacts.yml @@ -96,13 +96,15 @@ jobs: value: '$(Build.SourcesDirectory)/PowerShell' displayName: Build non-windows test artifacts condition: succeeded() - # POOL CHOICE: Azure-Pipelines-1ESPT-ExDShared + ubuntu-latest. Team-owned - # PSMMS* images don't ship with the `1es-pt-prerequisites` artifact baked in - # (build #685762 was rejected by the 1ES PT validator on this exact basis). - # See linux.yml header for the full rationale. + # POOL CHOICE: PowerShell1ES + PSMMSUbuntu22.04-Secure. We are C+AI org and + # use only team-owned pools (never E+D's or other orgs' shared pools). + # BLOCKER: the image build currently lacks the `1es-pt-prerequisites` + # artifact (build #685762 was rejected by the 1ES PT validator on this exact + # basis). PsImageFactory PR required to bake in the artifact before this + # job will pass pre-job validation. See linux.yml header for full rationale. pool: - name: Azure-Pipelines-1ESPT-ExDShared - image: ubuntu-latest + name: PowerShell1ES + image: PSMMSUbuntu22.04-Secure os: linux templateContext: From 281ee7aaa7d9413f704c2f3415e3f0e769a7fe9d Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 9 Jun 2026 13:03:59 -0500 Subject: [PATCH 12/19] Update Linux pool comment: 1es-pt-prerequisites-v2 artifact baked in PsImageFactory rebuilt PSMMSUbuntu22.04-Secure with the linux-1es-pt-prerequisites-v2 artifact on 2026-06-09, unblocking the Linux jobs that were failing build #685762 with Using an image without 1es-pt-prerequisites artifact is not allowed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...Shell-Coordinated_Packages-NonOfficial.yml | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml index 477b03637c..10913be50e 100644 --- a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml +++ b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml @@ -40,18 +40,15 @@ # confirmed working in apiscan-gen-notice POC and SDL # Sources Analysis job on build #684757). # - Linux jobs -> PowerShell1ES + PSMMSUbuntu22.04-Secure. -# BLOCKER: the image build lacks the -# `1es-pt-prerequisites` artifact required by 1ES PT -# (https://aka.ms/1espt/image-prerequisites). Build -# #685762 was rejected with "Using an image without -# 1es-pt-prerequisites artifact is not allowed". -# Declaring the image in PowerShell1ES.bicep and -# deploying it is necessary but not sufficient — the -# image build itself must bake in the artifact. -# PsImageFactory PR is required to add the artifact to -# the PSMMSUbuntu22.04-Secure image build, then -# redeploy. This pipeline cannot pass end-to-end on -# Linux until that's done. +# Image was rebuilt 2026-06-09 with the +# `linux-1es-pt-prerequisites-v2` artifact baked in +# (PsImageFactory) to satisfy the 1ES PT image-prereq +# gate (https://aka.ms/1espt/image-prerequisites). +# History: build #685762 was rejected with "Using an +# image without 1es-pt-prerequisites artifact is not +# allowed". Note: v2 is the current artifact; never +# mix with v1 (`linux-1es-gpt-prerequisites`) in the +# same image build. # - macOS jobs -> Azure Pipelines + vmImage macOS-latest. No team-owned # macOS image exists; macOS jobs MUST use the # Microsoft-hosted Azure Pipelines pool (this is the From 359b4a0075909d715ca4f5f6d1f5d30c54ad129c Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 9 Jun 2026 13:57:34 -0500 Subject: [PATCH 13/19] Fix 3 cross-cutting bugs from build #686007 (1ES Coordinated_Packages NonOfficial) Bug #1 (Linux Stage-repo): YAML boolean `cleanFirst` parameter rendered as literal `True` in the pwsh inline script, causing `if (True -and ...)` which fails on Linux. Switched to quoted string comparison `if ('${{ parameters.cleanFirst }}' -ieq 'True')` so the value is unambiguously a string regardless of platform pwsh parser. Bug #2 (Windows wix nuget restore): the staged repo at `$(Build.SourcesDirectory)\PowerShell` was created via `Copy-Item` without `.git`. Downstream `build.psm1` calls `git clean -fdX` which treats every file as untracked-and-ignored and wipes `tools/wix/nuget.config` (matches `.gitignore:110` pattern `nuget.config`). That file declares the `dotnet-eng` feed that hosts `Microsoft.Signed.Wix`, so dotnet restore fails with NU1101. Switched the staging helper to `git worktree add --detach $dst HEAD` so the staged path is a real git workspace; tracked files are preserved by `git clean -fdX` as designed. Added the lifecycle dance (`worktree remove --force` -> conditional `Remove-Item` -> unconditional `worktree prune` -> `worktree add`) so prior aborted runs do not leave orphaned worktree registrations blocking the next run. Also added `tools/wix/nuget.config` to the required-files post-check as a canary for this whole class of bug. Bug #3 (macOS Capture VSS* flake): hosted `macOS-latest` pwsh occasionally throws `Call to 'procargs' failed with errno 5` during startup - a libproc race independent of our scripts. Added `retryCountOnTaskFailure: 2` AND `continueOnError: true` to the diagnostic Capture VSS* task: retry to maximise odds of getting useful output, continueOnError as a safety net so a hosted flake never fails an otherwise-green build on a non-load-bearing diagnostic step. Also removed the obsolete `$global:LASTEXITCODE = 0` workaround from `insert-nuget-config-azfeed.yml` - it was masking the Bug #2 root cause. With the staged path now being a real git workspace, `git update-index --skip-worktree` succeeds normally and no reset is needed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/insert-nuget-config-azfeed.yml | 47 ++++------ .../templates/stage-repo-under-PowerShell.yml | 90 ++++++++++++++----- 2 files changed, 87 insertions(+), 50 deletions(-) diff --git a/.pipelines/1ES/templates/insert-nuget-config-azfeed.yml b/.pipelines/1ES/templates/insert-nuget-config-azfeed.yml index ee994eaf5f..23903a9607 100644 --- a/.pipelines/1ES/templates/insert-nuget-config-azfeed.yml +++ b/.pipelines/1ES/templates/insert-nuget-config-azfeed.yml @@ -1,28 +1,21 @@ # 1ES-specific translation of /.pipelines/templates/insert-nuget-config-azfeed.yml. # # WHY A 1ES-LOCAL COPY EXISTS: -# The shared template calls `Switch-PSNugetConfig` (from build.psm1), which calls -# `New-NugetConfigFile`, which calls `git update-index --skip-worktree` against -# each regenerated nuget.config to prevent accidental commits of feed credentials. +# The shared template's only adjustment we need is the `Capture VSS* Environment` +# task being marked `continueOnError: true` — see Gotcha #10 below. Beyond that +# this template is intentionally a near-verbatim port. Originally it also worked +# around a `git update-index --skip-worktree` failure that came from the staged +# repo not being a real git workspace, but that root cause is now fixed in +# `stage-repo-under-PowerShell.yml` (uses `git worktree add` instead of Copy-Item), +# so the `$global:LASTEXITCODE = 0` workaround was removed. # -# In OneBranch the staged repo IS a git workspace (cloneToOfficialPath.yml does a -# real `git clone`), so skip-worktree succeeds. In our 1ES translation, -# `stage-repo-under-PowerShell.yml` deliberately Copy-Items the tree WITHOUT `.git` -# (the staged copy is throwaway, no risk of accidental commit). git then walks -# upward to `$(Build.SourcesDirectory)\.git`, sees `PowerShell/nuget.config` as -# the relative path, and exits non-zero with `fatal: Unable to mark file -# PowerShell/nuget.config` because that path isn't in the original index. -# -# The git failure is semantically a no-op for our staging design: there's no git -# workspace to mark, and no risk of accidental commit from a Copy-Item'd folder. -# But pwsh 7.4+ propagates `$LASTEXITCODE` to the process exit code, so the ADO -# pwsh task fails. This wrapper resets `$LASTEXITCODE = 0` immediately after -# `Switch-PSNugetConfig` so the benign git failure doesn't fail the build. -# -# Stays out of build.psm1 (shared with OneBranch) — the fix is local to the 1ES -# scaffold. If/when build.psm1 is hardened to tolerate non-git destinations -# itself, this wrapper can be replaced with a direct reference to the shared -# template. +# Gotcha #10: macOS hosted pwsh occasionally fails to start with +# `Call to 'procargs' failed with errno 5` thrown from +# `Microsoft.PowerShell.ManagedPSEntry.AttemptExecPwshLogin` +# This is an intermittent libproc race on the macOS-latest hosted image and has +# nothing to do with our script. Because the `Capture VSS* Environment` task is +# purely diagnostic (`Get-ChildItem env:VSS*`), losing it on a rare failure is +# strictly better than failing the whole macOS BUILD job. Hence `continueOnError`. parameters: - name: "repoRoot" @@ -45,13 +38,6 @@ steps: Write-Verbose -Verbose "Running: Switch-PSNugetConfig -Source Private -UserName '$(AzDevopsFeedUserNameKVPAT)' -ClearTextPAT ''" Switch-PSNugetConfig -Source Private -UserName '$(AzDevopsFeedUserNameKVPAT)' -ClearTextPAT '$(powershellPackageReadPat)' - # New-NugetConfigFile (inside Switch-PSNugetConfig) runs `git update-index - # --skip-worktree` against the regenerated nuget.config. The staged repo - # is not a git workspace (Copy-Item without .git), so git fails benignly - # and leaves $LASTEXITCODE non-zero. Reset so pwsh exits 0 when the rest - # of the script (Test-Path check below) succeeds. - $global:LASTEXITCODE = 0 - if(-not (Test-Path $configPath)) { throw "nuget.config is not created" @@ -82,5 +68,10 @@ steps: Get-ChildItem -Path env:VSS* | Out-String -width 9999 -Stream | write-Verbose -Verbose displayName: Capture VSS* Environment condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], '')) + # Diagnostic-only task; never fail the build if hosted pwsh flakes (see Gotcha #10 above). + # Retry first to maximise the odds of actually getting the diagnostic output before + # falling back to continueOnError when the procargs race persists. + retryCountOnTaskFailure: 2 + continueOnError: true env: ob_restore_phase: ${{ parameters.ob_restore_phase }} diff --git a/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml b/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml index 5076fb4dd1..a40c2fde42 100644 --- a/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml +++ b/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml @@ -18,14 +18,41 @@ parameters: # We pick (b) for the initial migration to keep the diff reviewable. A follow-up cleanup # pass can collapse the indirection once Official is also green. # -# Why Copy-Item, not `git clone ` -# ---------------------------------------------- -# The OneBranch helper `cloneToOfficialPath.yml` does a real `git clone $env:REPOROOT -# $nativePath`. In OneBranch, source and dest are SIBLING folders so it works fine. -# In 1ES PT, source ($(Build.SourcesDirectory)) is the PARENT of dest. A -# `git clone \PowerShell` would try to copy the source repo INCLUDING the -# destination subfolder we're trying to create — fragile and likely to fail on Windows -# file-locking. A plain Copy-Item with a single-folder exclude is simpler and reliable. +# Why `git worktree add`, not Copy-Item or `git clone` +# --------------------------------------------------- +# OneBranch's `cloneToOfficialPath.yml` does a real `git clone $env:REPOROOT $nativePath` +# because source and dest are SIBLING folders there. In 1ES PT, source +# ($(Build.SourcesDirectory)) is the PARENT of dest — `git clone /PowerShell` +# would try to copy the source repo INCLUDING the destination subfolder we're creating. +# +# Our v1 used Copy-Item with `-Exclude '.git'`, but that left the staged path WITHOUT a +# .git directory. Two real problems came from that (build #686007): +# +# 1. `Start-PSBuild` runs `git clean -fdX --exclude src/Modules/nuget.config --exclude +# nuget.config` in the staged path. `git clean` walks up to find the nearest .git +# (which becomes `$(Build.SourcesDirectory)/.git` — a DIFFERENT repo's index that +# knows nothing about files under `PowerShell/`). Every staged file therefore looks +# "untracked" to git, and `git clean -fdX` removes anything matching .gitignore. +# The killer case: `tools/wix/nuget.config` is TRACKED in HEAD but ALSO matches the +# `nuget.config` pattern in .gitignore (line 110). In a real git workspace, tracked +# files are NEVER removed by `git clean -fdX`. In our staged-without-.git copy, +# they WERE — wiping the `dotnet-eng` feed declaration that `Microsoft.Signed.Wix` +# restore depends on. (Failure was: `error NU1101: Unable to find package +# Microsoft.Signed.Wix. No packages exist with this id in source(s): powershell`.) +# +# 2. `Switch-PSNugetConfig` -> `New-NugetConfigFile` runs `git update-index +# --skip-worktree` against regenerated nuget.config files. Without .git in the +# staged path, git fails benignly and we had to add a `$global:LASTEXITCODE = 0` +# workaround in `insert-nuget-config-azfeed.yml`. With a real worktree, that +# workaround becomes unnecessary. +# +# `git worktree add --detach $dst HEAD` solves both: +# - Creates $dst as a REAL git workspace sharing the parent's .git database +# (via a tiny `.git` POINTER file, not a copy of the object store — very fast). +# - `--detach` avoids the "branch already checked out in parent worktree" error +# that would otherwise occur because $src is on the same branch. +# - Downstream `git` commands (clean, update-index, describe, rev-parse) all behave +# identically to OneBranch's real-clone semantics. # # Also sets $env:REPOROOT for downstream consumers (SetVersionVariables.yml / # set-reporoot.yml / insert-nuget-config-azfeed.yml all probe $env:REPOROOT or fall back @@ -37,27 +64,46 @@ steps: $src = '$(Build.SourcesDirectory)' $dst = Join-Path $src 'PowerShell' - if (${{ parameters.cleanFirst }} -and (Test-Path $dst)) { - Remove-Item -Path $dst -Recurse -Force + # Compile-time template expansion: `${{ parameters.cleanFirst }}` renders the + # YAML boolean as the literal string `True` or `False` (capitalised, no sigil). + # Wrapping in single quotes turns it into a comparable string — DO NOT leave + # it bare (`if (${{ parameters.cleanFirst }} ...)` would try to invoke a + # cmdlet named `True` and fail with "term not recognized"; this was the + # cause of every Linux Stage-repo failure in build #686007). + if ('${{ parameters.cleanFirst }}' -ieq 'True') { + if (Test-Path $dst) { + # If $dst is a registered worktree, unregister it cleanly first; otherwise + # `git worktree add` below will refuse to recreate it. + & git -C $src worktree remove --force $dst 2>$null + if (Test-Path $dst) { + Remove-Item -Path $dst -Recurse -Force + } + } + # Always prune (even when $dst was already gone) — an orphaned + # $src/.git/worktrees/PowerShell registration from a prior aborted run + # would otherwise block `git worktree add`. + & git -C $src worktree prune 2>$null } - New-Item -ItemType Directory -Path $dst -Force | Out-Null - # Copy everything except: - # - the destination folder itself (would self-recurse) - # - the `.git` folder (large, locked .pack files often fail to copy on Windows, - # and the staged copy is only used for path-parity script consumption — Git - # operations stay against $(Build.SourcesDirectory) where checkout: self put them) - Get-ChildItem -Path $src -Force -Exclude 'PowerShell', '.git' | ForEach-Object { - Copy-Item -Path $_.FullName -Destination $dst -Recurse -Force + # Materialise /PowerShell as a REAL git worktree sharing the parent's + # .git database. See the file header comment for the full rationale; the short + # version is: downstream scripts (Start-PSBuild's `git clean -fdX`, + # Switch-PSNugetConfig's `git update-index --skip-worktree`) assume the staged + # path is a git workspace, and silently corrupt the build when it isn't. + & git -C $src worktree add --detach $dst HEAD + if ($LASTEXITCODE -ne 0) { + throw "git worktree add failed with exit code $LASTEXITCODE" } - # Verify critical files made it across — surface partial-copy failures NOW + # Verify critical files made it across — surface partial-checkout failures NOW # rather than letting downstream scripts fail with confusing "file not found" - # errors at build time. + # errors at build time. `tools/wix/nuget.config` is explicitly guarded because + # it's the canary for the "staged repo isn't a git workspace" regression. $required = @( Join-Path $dst 'build.psm1' Join-Path $dst '.config/tsaoptions.json' Join-Path $dst '.config/suppress.json' + Join-Path $dst 'tools/wix/nuget.config' ) foreach ($p in $required) { if (-not (Test-Path $p)) { @@ -67,5 +113,5 @@ steps: # Expose REPOROOT for downstream helper templates (SetVersionVariables, etc.). Write-Host "##vso[task.setvariable variable=REPOROOT]$dst" - Write-Verbose -Verbose "Repository staged at $dst" - displayName: 'Stage repo into $(Build.SourcesDirectory)\PowerShell (OneBranch path parity)' + Write-Verbose -Verbose "Repository staged at $dst (git worktree from $src)" + displayName: 'Stage repo into $(Build.SourcesDirectory)\PowerShell (git worktree, OneBranch path parity)' From 1323c7ef285bad8a7487c2a022b8bab88ba0c0f6 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 9 Jun 2026 14:52:13 -0500 Subject: [PATCH 14/19] Fix 2 cross-cutting bugs from build #686027 (1ES Coordinated_Packages NonOfficial) Build #686027 ran end-to-end for the first time after the Linux image and git-worktree fixes from b58362c8. It surfaced two new failure clusters, both unrelated to the prior fixes: Cluster A/B/D: ~100K StyleCop/CodeAnalysis errors across Linux build, Windows BUILD, Windows Symbols, and Linux TestArtifacts. All four failures were the same root cause: Roslyn was loading TWO .globalconfig files (parent at $(Build.SourcesDirectory)/.globalconfig from 1ES PT checkout: self, plus child at $(Build.SourcesDirectory)/PowerShell/.globalconfig from our git-worktree staging step). Per Roslyn semantics, dual global-analyzer configs with conflicting keys UNSET those keys and emit a MultipleGlobalAnalyzerKeys warning. With every SA*/CA* suppression unset, all rules fired and TreatWarningsAsErrors escalated them. Fix: in stage-repo-under-PowerShell.yml, after git worktree add succeeds, rename the parent .globalconfig to .globalconfig.disabled-by-stage-repo so Roslyn no longer auto-discovers it. Add a defensive walk-up check from a sample project to assert exactly one .globalconfig remains visible and that it is the worktree copy. Add .globalconfig to the required-files canary list. This bug does NOT manifest under OneBranch because OneBranch's checkout layout puts source and staged repo as siblings, not parent/child. It is specific to the 1ES PT "checkout: self at $(Build.SourcesDirectory) + stage into a subdirectory" pattern. Other auto-discovered files were audited: .editorconfig has root=true (walk-up stops), no Directory.Build/Packages.* exist in the repo, nuget.config uses at the worktree root. Cluster C: macOS BUILD published `macosBinResults-${arch}` artifact twice (once via mid-job `##vso[artifact.upload]`, once via end-of-job PublishPipelineArtifact@1 auto-injected from templateContext.outputs). The second registration failed with `Artifact macosBinResults-x64 already exists for build 686027`, marking the whole job as failed. The mid-job upload was added as a "belt-and-suspenders fallback" under the (incorrect) assumption that ADO tolerates duplicate artifact names; it does not. Fix: remove the mid-job `##vso[artifact.upload]` from mac.yml. Keep templateContext.outputs.pipelineArtifact as the single publish mechanism. Playbook updated with Gotchas #11 (dual .globalconfig conflict) and #12 (duplicate artifact registration). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .pipelines/1ES/templates/mac.yml | 23 +++--- .../templates/stage-repo-under-PowerShell.yml | 78 +++++++++++++++++++ 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/.pipelines/1ES/templates/mac.yml b/.pipelines/1ES/templates/mac.yml index d42788ab77..bdac3e407c 100644 --- a/.pipelines/1ES/templates/mac.yml +++ b/.pipelines/1ES/templates/mac.yml @@ -107,17 +107,18 @@ jobs: env: __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - # Belt-and-suspenders artifact publish for the macOS custom pool. - # `templateContext.outputs` SHOULD publish via PublishPipelineArtifact@1, but on - # `isCustom: true` (Microsoft-hosted) pools this path is not as well-trodden as on - # 1ES-managed pools. The original OneBranch mac.yml relied on an explicit - # `##vso[artifact.upload]` here, so we keep that as a fallback. If the - # templateContext.outputs publish succeeds, this is a no-op duplicate (ADO - # tolerates this); if it fails silently, the sign job's DownloadPipelineArtifact - # call still has an artifact to fetch. - - pwsh: | - Write-Host "##vso[artifact.upload containerfolder=macosBinResults-${{ parameters.buildArchitecture }};artifactname=macosBinResults-${{ parameters.buildArchitecture }}]$(OB_OUTPUTDIRECTORY)" - displayName: 'Upload macOS build artifact (fallback for custom pool)' + # Note: artifact publish is handled by `templateContext.outputs.pipelineArtifact` + # at the top of this job (lines ~46-48). An earlier version of this template kept + # an explicit `##vso[artifact.upload]` here as a belt-and-suspenders fallback, + # under the (incorrect) assumption that ADO tolerates two registrations of the + # same artifact name. Build #686027 disproved that: the mid-job `##vso` upload + # registered `macosBinResults-x64` first, then the end-of-job + # PublishPipelineArtifact@1 (auto-injected by 1ES PT from `templateContext.outputs`) + # failed with `Artifact macosBinResults-x64 already exists for build 686027`. + # Removed in favour of the single 1ES PT mechanism. If the 1ES PT publish ever + # turns out to be unreliable on this `isCustom: true` macOS pool, reinstate a + # SINGLE mechanism (either the templateContext.outputs OR an explicit + # PublishPipelineArtifact@1, never both). - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml b/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml index a40c2fde42..ad1e24581c 100644 --- a/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml +++ b/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml @@ -104,6 +104,7 @@ steps: Join-Path $dst '.config/tsaoptions.json' Join-Path $dst '.config/suppress.json' Join-Path $dst 'tools/wix/nuget.config' + Join-Path $dst '.globalconfig' ) foreach ($p in $required) { if (-not (Test-Path $p)) { @@ -111,6 +112,83 @@ steps: } } + # Neutralise the PARENT `.globalconfig` to prevent a Roslyn + # `MultipleGlobalAnalyzerKeys` conflict with the WORKTREE `.globalconfig`. + # + # Why + # --- + # In OneBranch, the repo is at //PowerShell — the parent + # `/` does NOT contain a copy of the repo, so only ONE + # `.globalconfig` ever exists in the ancestor chain of a project file. + # + # In 1ES PT, `checkout: self` puts the repo at $(Build.SourcesDirectory) directly. + # The `git worktree add` above then materialises a SECOND working copy at + # $(Build.SourcesDirectory)/PowerShell. Both now contain `.globalconfig` at their + # root. When Roslyn auto-discovers config files for a project at + # `$(Build.SourcesDirectory)/PowerShell/src//.csproj`, it walks + # UP the directory tree and picks up BOTH files. Per Roslyn's + # `EditorConfigParser` rules, when two global-analyzer-config files set the + # SAME key for a project, Roslyn UNSETS that key and emits a + # `MultipleGlobalAnalyzerKeys` warning. With every `dotnet_diagnostic.SA*.severity = none` + # suppression unset, every StyleCop rule fires at default severity (warning), + # and `TreatWarningsAsErrors=true` in PowerShell.Common.props escalates them + # to errors. Build #686027 saw ~100K SA* errors across Linux, Windows symbols, + # and TestArtifacts jobs as a direct result. + # + # Why rename instead of delete + # ---------------------------- + # `.globalconfig.disabled-by-stage-repo` is not auto-discovered (Roslyn only + # matches the literal name `.globalconfig`), but it remains on disk for any + # post-mortem inspection. Renaming is reversible; deletion is not. + # + # `.editorconfig` is NOT renamed — the worktree copy has `root = true` (see + # `.editorconfig:10`), so the ancestor walk stops at the worktree root and the + # parent copy is never read. + # + # `nuget.config` is NOT renamed. NuGet does layer/merge package-source + # config from machine -> user -> root-to-leaf project configs, but the + # repo's `nuget.config` at the worktree root has `` inside + # `` and ``, which neutralises any + # inherited sources (including the parent checkout's identical copy). So + # the merge is harmless in practice. + $parentGlobalConfig = Join-Path $src '.globalconfig' + if (Test-Path $parentGlobalConfig) { + $disabledName = '.globalconfig.disabled-by-stage-repo' + $disabledPath = Join-Path $src $disabledName + if (Test-Path $disabledPath) { + Remove-Item -Path $disabledPath -Force + } + Rename-Item -Path $parentGlobalConfig -NewName $disabledName -Force + Write-Verbose -Verbose "Renamed parent .globalconfig -> $disabledName to prevent Roslyn MultipleGlobalAnalyzerKeys conflict with worktree copy at $dst/.globalconfig" + } else { + Write-Verbose -Verbose "Parent $src/.globalconfig not present — nothing to neutralise (unexpected, since `checkout: self` should have produced it; continuing anyway)." + } + + # Defensive sanity check: confirm exactly ONE .globalconfig is now visible to + # the worktree's project ancestors AND it is the worktree copy at $dst/.globalconfig. + # Walks up from a sample project directory; all projects under $dst/src share the + # same upper ancestor chain, so a single project is representative. + $sampleProjectDir = Join-Path $dst 'src/System.Management.Automation' + if (Test-Path $sampleProjectDir) { + $found = @() + $cur = (Get-Item $sampleProjectDir).FullName + while ($cur) { + $candidate = Join-Path $cur '.globalconfig' + if (Test-Path $candidate) { $found += $candidate } + $parent = Split-Path -Path $cur -Parent + if (-not $parent -or $parent -eq $cur) { break } + $cur = $parent + } + Write-Verbose -Verbose ("Ancestor .globalconfig walk from {0} found {1} file(s): {2}" -f $sampleProjectDir, $found.Count, ($found -join '; ')) + $expectedGlobalConfig = (Get-Item (Join-Path $dst '.globalconfig')).FullName + if ($found.Count -ne 1) { + throw ("MultipleGlobalAnalyzerKeys risk: expected exactly 1 .globalconfig in worktree project ancestors, found {0}: {1}" -f $found.Count, ($found -join '; ')) + } + if ($found[0] -ne $expectedGlobalConfig) { + throw ("Unexpected .globalconfig location: expected '{0}', found '{1}'" -f $expectedGlobalConfig, $found[0]) + } + } + # Expose REPOROOT for downstream helper templates (SetVersionVariables, etc.). Write-Host "##vso[task.setvariable variable=REPOROOT]$dst" Write-Verbose -Verbose "Repository staged at $dst (git worktree from $src)" From 2b246866ccf4a3bfe9fbdbd1ee99b2ee1aadfab6 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 9 Jun 2026 15:37:09 -0500 Subject: [PATCH 15/19] Fix Linux dotfile regression from build #686043 (stage-repo assertion) Build #686043 demonstrated both prior fixes (parent .globalconfig rename and mac.yml artifact de-duplication) worked: macOS and Windows jobs all passed, SDL succeeded, and the SA*/CA* analyzer firehose was gone. But all 9 Linux build jobs (and Linux test artifacts) failed in the new defensive walk-up assertion at stage-repo-under-PowerShell.yml:183 with: Get-Item: Could not find item /mnt/vss/_work/1/s/PowerShell/.globalconfig even though the previous line's Test-Path returned True for the same path. Root cause: on Linux, .NET marks any file with a leading "." as Hidden in FileSystemInfo.Attributes (POSIX convention). PowerShell's FileSystemProvider filters those out of Get-Item/Get-ChildItem by default; -Force disables the filter. Test-Path doesn't consult Attributes, so it always saw the dotfile. macOS .NET does NOT flag dotfiles as Hidden, which is why macOS passed the same code path. Fix: - Get-Item (Join-Path $dst '.globalconfig') -> Get-Item -LiteralPath (Join-Path $dst '.globalconfig') -Force (Adopts rubber-duck recommendation: -LiteralPath also avoids wildcard ambiguity for paths containing [ ] *.) - Cosmetic: displayName changed from "$(Build.SourcesDirectory)\PowerShell" to "$(Build.SourcesDirectory)/PowerShell" so the Linux log no longer reads "Stage repo into /mnt/vss/_work/1/s\PowerShell" (mixed separators). Playbook updated with Gotcha #13 (Linux dotfile Get-Item behaviour) and a new decision-tree row for "Get-Item: Could not find item /...". Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../1ES/templates/stage-repo-under-PowerShell.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml b/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml index ad1e24581c..f4d9e811ba 100644 --- a/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml +++ b/.pipelines/1ES/templates/stage-repo-under-PowerShell.yml @@ -180,7 +180,13 @@ steps: $cur = $parent } Write-Verbose -Verbose ("Ancestor .globalconfig walk from {0} found {1} file(s): {2}" -f $sampleProjectDir, $found.Count, ($found -join '; ')) - $expectedGlobalConfig = (Get-Item (Join-Path $dst '.globalconfig')).FullName + # Use -LiteralPath + -Force so .NET on Linux (which marks dotfiles as + # Hidden in FileSystemInfo.Attributes by POSIX convention) still resolves + # the worktree's `.globalconfig`. Without -Force, `Get-Item` returns + # "Could not find item" on Linux even when Test-Path returns True for the + # same path. Empirically: build #686043's Linux jobs all failed here, while + # macOS jobs (where .NET does NOT flag dotfiles as Hidden) succeeded. + $expectedGlobalConfig = (Get-Item -LiteralPath (Join-Path $dst '.globalconfig') -Force).FullName if ($found.Count -ne 1) { throw ("MultipleGlobalAnalyzerKeys risk: expected exactly 1 .globalconfig in worktree project ancestors, found {0}: {1}" -f $found.Count, ($found -join '; ')) } @@ -192,4 +198,4 @@ steps: # Expose REPOROOT for downstream helper templates (SetVersionVariables, etc.). Write-Host "##vso[task.setvariable variable=REPOROOT]$dst" Write-Verbose -Verbose "Repository staged at $dst (git worktree from $src)" - displayName: 'Stage repo into $(Build.SourcesDirectory)\PowerShell (git worktree, OneBranch path parity)' + displayName: 'Stage repo into $(Build.SourcesDirectory)/PowerShell (git worktree, OneBranch path parity)' From 314f48ab542bf2bd78d2371811e617612bc1bf8e Mon Sep 17 00:00:00 2001 From: chungjustin Date: Tue, 9 Jun 2026 16:17:07 -0500 Subject: [PATCH 16/19] 1ES PT: SDL path overrides use forward slashes on Linux/macOS jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build #686056 ran 24 of 25 jobs successfully — the dotfile Get-Item fix from cdb732234 worked perfectly. The only failure was the auto-injected "Guardian: CredScan (Binary)" SDL task in the Linux "Build non-windows test artifacts" job, which failed with: [ERROR] [Invalid Argument] SuppressionsPath: Invalid file path list. File not found - /mnt/vss/_work/1/s\.config\suppress.csk GuardianErrorExitCodeException: credscan completed with exit code 24 Root cause: the entry pipeline's PIPELINE-LEVEL sdl block uses Windows backslash paths ("$(Build.SourcesDirectory)\.config\suppress.json"). On Windows .NET, backslash AND forward slash both work as path separators, so the auto-injected SDL Sources Analysis (self) job — running on a Windows-default agent — passed. On Linux .NET, backslash is a LITERAL character, not a separator. The path resolves to "/mnt/vss/_work/1/s\.config\suppress.json" which the Guardian CLI's file-existence check rejects. The Linux BUILD jobs passed because linux.yml:64-66 already overrides sdl.tsa.configFile + sdl.credscan.suppressionsFile with forward-slash paths under PowerShellRoot. The Linux SIGN jobs and macOS SIGN jobs also had backslash overrides BUT passed by luck — 1ES PT does not inject Binary Analysis CredScan into sign-only jobs (they get CodeSignValidation only). The non-windows testartifacts job is the one job that (a) runs on os: linux AND (b) gets BinaryAnalysis CredScan injected AND (c) did not override the inherited backslash paths. Fix: - testartifacts.yml non-win job: add per-job sdl.tsa.configFile + sdl.credscan.suppressionsFile overrides with forward-slash paths under $(Build.SourcesDirectory)/PowerShell/.config/ (REQUIRED fix — unblocks the failing job). - linux.yml sign job (lines 185, 187): convert backslashes to forward slashes (DEFENSIVE — passing today only because CredScan isn't injected into sign jobs, would break if 1ES PT injection rules change). - mac.yml sign job (lines 153, 155): same defensive cleanup. NOT changed: - Pipeline-level (entry pipeline) sdl paths — still backslash, working fine for SDL Sources Analysis (self) which runs on Windows. Switching them to forward slashes is also safe (Windows accepts both), but out of scope; not changing what's working. - Windows-only job sdl paths (windows-hosted-build.yml, testartifacts.yml win job) — backslash is fine on Windows. Validated all 3 files via ConvertFrom-Yaml. Playbook (/files/onebranch-to-1es-migration.md) — added Gotcha #14 (SDL paths on Linux/macOS) with watch-list of all templateContext.sdl.* overrides and a decision-tree row. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .pipelines/1ES/templates/linux.yml | 4 ++-- .pipelines/1ES/templates/mac.yml | 4 ++-- .pipelines/1ES/templates/testartifacts.yml | 13 +++++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.pipelines/1ES/templates/linux.yml b/.pipelines/1ES/templates/linux.yml index 75a0e1c207..ae496b516e 100644 --- a/.pipelines/1ES/templates/linux.yml +++ b/.pipelines/1ES/templates/linux.yml @@ -182,9 +182,9 @@ jobs: binskim: enabled: false tsa: - configFile: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + configFile: $(Build.SourcesDirectory)/PowerShell/.config/tsaoptions.json credscan: - suppressionsFile: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + suppressionsFile: $(Build.SourcesDirectory)/PowerShell/.config/suppress.json codeql: compiled: enabled: false diff --git a/.pipelines/1ES/templates/mac.yml b/.pipelines/1ES/templates/mac.yml index bdac3e407c..cf4a7fba4a 100644 --- a/.pipelines/1ES/templates/mac.yml +++ b/.pipelines/1ES/templates/mac.yml @@ -150,9 +150,9 @@ jobs: codeSignValidation: enabled: false tsa: - configFile: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + configFile: $(Build.SourcesDirectory)/PowerShell/.config/tsaoptions.json credscan: - suppressionsFile: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + suppressionsFile: $(Build.SourcesDirectory)/PowerShell/.config/suppress.json codeql: compiled: enabled: false diff --git a/.pipelines/1ES/templates/testartifacts.yml b/.pipelines/1ES/templates/testartifacts.yml index 2f0b6a30d0..7d19d29cd5 100644 --- a/.pipelines/1ES/templates/testartifacts.yml +++ b/.pipelines/1ES/templates/testartifacts.yml @@ -111,6 +111,19 @@ jobs: sdl: codeSignValidation: enabled: false + # Per-job SDL path overrides REQUIRED on Linux jobs. The pipeline-level + # `sdl` block in the entry pipeline uses Windows backslash paths (those + # work for the auto-injected SDL Sources Analysis (self) job which runs + # on Windows-default). On Linux .NET, '\' is a literal char, not a path + # separator, so '$(Build.SourcesDirectory)\.config\suppress.json' resolves + # to '/mnt/vss/_work/1/s\.config\suppress.json' which the CredScan tool + # cannot find. See build #686056 'Guardian: CredScan (Binary)' failure + # (https://aka.ms/credscan, GuardianErrorExitCodeException exit code 24). + # Mirror linux.yml build job's per-job override pattern with /-paths. + tsa: + configFile: $(Build.SourcesDirectory)/PowerShell/.config/tsaoptions.json + credscan: + suppressionsFile: $(Build.SourcesDirectory)/PowerShell/.config/suppress.json outputs: - output: pipelineArtifact targetPath: $(ob_outputDirectory) From 4fae5253cd183bea7985d4ca86609a4bccef3788 Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Wed, 10 Jun 2026 09:46:45 -0500 Subject: [PATCH 17/19] Add ESRP test-signing siblings to 3 of 5 signing sites (hybrid stub/real-task gate) Wire MS-Sign Test KeyCodes (CP-466277 for 1P, CP-466279 for 3P) into the 1ES Unofficial pipeline so the EsrpCodeSigning@5 task path can be exercised before full AME-onboarded PROD signing is available. Design: sibling-condition gate using the runtime variable EsrpTestConnectionName. Each modified signing site has two sibling steps: - Existing Write-Host stub, condition: eq(variables['EsrpTestConnectionName'], '') - New EsrpCodeSigning@5 task, condition: ne(variables['EsrpTestConnectionName'], '') In ADO expression syntax, variables['UndefinedVar'] evaluates to empty string, so the default behavior (no variable group attached) preserves the green baseline -- only stubs run. Attaching a variable group named EsrpSigningTest-PowerShell with 6 keys (EsrpTestConnectionName, EsrpTestAppRegClientId, EsrpTestAppRegTenantId, EsrpTestAuthAkvName, EsrpTestAuthCertName, EsrpTestAuthSignCertName) flips every modified site to real signing. Sites modified (3): - .pipelines/1ES/templates/obp-file-signing.yml site 1: 1P Authenticode, CP-466277 - .pipelines/1ES/templates/obp-file-signing.yml site 2: 3P Authenticode, CP-466279 - .pipelines/1ES/templates/windows-hosted-build.yml obj files: 1P Authenticode, CP-466277 Sites that STAY STUBBED (2): - windows-hosted-build.yml nupkg files: no NuGet-compatible test KeyCode available (NuGetSign operation cannot reuse Authenticode codes) - mac.yml Apple Mach-O CodeSign: no Apple test KeyCode available (MacAppDeveloperSign operation, entirely different cert chain) Header comment updates: - PowerShell-Coordinated_Packages-NonOfficial.yml entry pipeline: documents the 6 variable-group keys, KeyCode mapping table, PROD swap path, and the predicate caveat (test-signed files round-trip through both 1P and 3P paths in obp-file-signing.yml -- by design, do not fix) - obp-file-signing.yml: hybrid stub/test-signing mode documentation - windows-hosted-build.yml: per-site mode documentation All 3 modified YAML files validated via ConvertFrom-Yaml. Safe to push to the green rebuild/v7.9.99-rebuild.9 branch: default condition runs stubs (current green baseline preserved); test signing is opt-in by attaching the variable group at queue time. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...Shell-Coordinated_Packages-NonOfficial.yml | 51 ++++++ .pipelines/1ES/templates/obp-file-signing.yml | 166 +++++++++++++++--- .../1ES/templates/windows-hosted-build.yml | 87 +++++++-- 3 files changed, 270 insertions(+), 34 deletions(-) diff --git a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml index 10913be50e..2b9ff01bc6 100644 --- a/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml +++ b/.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml @@ -75,6 +75,57 @@ # `pwsh: Write-Host "TODO: ESRP sign ..."` placeholder. Pipeline runs end-to-end # but produces UNSIGNED artifacts — strictly for validating the migration scaffold. # - Once Unofficial green, build Official sibling with real ESRP signing. +# +# ESRP TEST SIGNING (Phase B preview — opt-in, runtime-gated): +# Three of the five signing sites have a hybrid stub/real-task implementation +# gated by the runtime variable `EsrpTestConnectionName`. When this variable +# is empty (default), every signing site runs the original Write-Host stub +# and the pipeline stays UNSIGNED — preserving the green baseline. When the +# variable is non-empty, three sites switch to real `EsrpCodeSigning@5` tasks +# that exercise the ESRP signing path against MS-Sign Test KeyCodes. +# +# TO ENABLE TEST SIGNING: +# 1. Create a variable group named `EsrpSigningTest-PowerShell` in this +# ADO project with the following 6 variables (values come from the +# ESRP onboarding ticket): +# - EsrpTestConnectionName (federated MI service connection) +# - EsrpTestAppRegClientId (AAD app registration client ID) +# - EsrpTestAppRegTenantId (AAD tenant ID) +# - EsrpTestAuthAkvName (Azure Key Vault name) +# - EsrpTestAuthCertName (auth cert name in the KV) +# - EsrpTestAuthSignCertName (signing cert name in the KV) +# 2. Confirm the AAD app registration is onboarded to the ESRP test ring +# AND authorised for KeyCodes CP-466277 (1P) and CP-466279 (3P). +# 3. Attach the variable group to pipeline 4229 at queue time. +# +# SIGNING-SITE MAPPING (when test signing enabled): +# ───────────────────────────────────────────────────────────────────────── +# Site | TEST KeyCode | Mode +# ───────────────────────────────────────────────────────────────────────── +# obp-file-signing.yml — 1P Authenticode | CP-466277 | HYBRID +# obp-file-signing.yml — 3P Authenticode | CP-466279 | HYBRID +# windows-hosted-build.yml — Sign obj files | CP-466277 | HYBRID +# windows-hosted-build.yml — Sign nupkg files | (none) | STUB ONLY +# mac.yml — Apple CodeSign Mach-O binaries | (none) | STUB ONLY +# ───────────────────────────────────────────────────────────────────────── +# "STUB ONLY" sites stay as Write-Host placeholders even when test signing +# is enabled — no compatible test KeyCode currently exists for NuGet or +# Apple operations. +# +# PROD KeyCode swap path (Official sibling, post-AME-onboarding): +# CP-466277 -> CP-230012 (1P Authenticode prod) +# CP-466279 -> CP-231522 (3P Authenticode prod) +# CP-466277 -> CP-230012 (obj files / 1P prod) +# (nupkg) -> CP-401405 (NuGet prod, needs new EsrpCodeSigning@5 task) +# (Apple) -> CP-401337-Apple (MacAppDeveloperSign prod) +# +# PREDICATE CAVEAT (test signing mode): +# In `obp-file-signing.yml`, the post-1P-signing predicate that flags +# files for 3P re-signing matches test cert issuers as "needs 3P signing". +# This means test-signed files will round-trip through BOTH the 1P and 3P +# paths — by design, exercising both code paths. Not a bug; documented in +# the playbook (`/files/onebranch-to-1es-migration.md` gotcha +# #15) so future readers don't try to "fix" it. trigger: none diff --git a/.pipelines/1ES/templates/obp-file-signing.yml b/.pipelines/1ES/templates/obp-file-signing.yml index 3d1468dd16..125027aa9b 100644 --- a/.pipelines/1ES/templates/obp-file-signing.yml +++ b/.pipelines/1ES/templates/obp-file-signing.yml @@ -5,29 +5,54 @@ parameters: OfficialBuild: true vPackScenario: false -# 1ES PT translation of /.pipelines/templates/obp-file-signing.yml — STUBBED VERSION. +# 1ES PT translation of /.pipelines/templates/obp-file-signing.yml — HYBRID STUB/TEST-SIGNING. # -# Why stubs? -# ---------- -# The OneBranch original calls `task: onebranch.pipeline.signing@1` against ESRP -# KeyCodes that PowerShell is still in the process of onboarding for the new 1ES service -# connection (external_distribution / msft_3rd_party_cert_id / nuget_cert_id / apple_cert_id). -# Until that onboarding is complete, the Unofficial 1ES pipeline cannot do real signing. +# TWO RUN MODES, selected at queue time by whether the runtime variable +# `EsrpTestConnectionName` is defined (typically via attaching the +# `EsrpSigningTest-PowerShell` variable group at the pipeline level): # -# This stub keeps the SAME FILE LAYOUT WORK as the original (copy files to be signed -# into staging folders, prepare third-party folder, run Update-PSSignedBuildFolder) so the -# pipeline still produces a recognisable artifact tree. The actual `signing@1` task calls -# are replaced with `pwsh: Write-Host "TODO: ESRP sign ..."` placeholders so the pipeline -# runs end-to-end and emits UNSIGNED artifacts. +# * STUB MODE (default — variable undefined): the original +# `task: onebranch.pipeline.signing@1` calls are replaced with +# `pwsh: Write-Host` placeholders. Pipeline runs end-to-end and emits +# UNSIGNED artifacts. This is the safe default for the rebuild branch +# and for any contributor without ESRP onboarding. # -# Migration path back to real signing (in the Official 1ES sibling, post-ESRP onboarding): -# Replace each "TODO: ESRP sign" pwsh step with the appropriate 1ES PT signing pattern, -# e.g. EsrpCodeSigning@5 with the new 1ES service connection, mapping: -# SigningProfile -> KeyCode CP-230012 (external_distribution) -# $(msft_3rd_party_cert_id) -> KeyCode CP-231522 (3rd party Authenticode) -# $(nuget_cert_id) -> KeyCode CP-401405 (NuGet package signing) -# $(apple_cert_id) -> KeyCode CP-401337-Apple (MacAppDeveloperSign) -# See files/onebranch-to-1es-migration.md "ESRP KeyCode mapping" section for the full table. +# * TEST SIGNING MODE (variable group attached): real `EsrpCodeSigning@5` +# tasks fire against the MS-Sign Test KeyCodes (https://aka.ms/keycodes — +# auth-gated). Auth is via federated managed identity (no PFX cert in +# pipeline). Requires the linked AAD app registration to be onboarded to +# the ESRP TEST ring AND authorised for these specific KeyCodes. +# +# Variable group `EsrpSigningTest-PowerShell` (to be created by repo admin and +# linked to the pipeline definition before queueing in TEST SIGNING MODE): +# EsrpTestConnectionName - ADO federated MI service-connection name +# EsrpTestAppRegClientId - AAD app reg client ID for the connection +# EsrpTestAppRegTenantId - AAD tenant ID +# EsrpTestAuthAkvName - KeyVault holding auth cert +# EsrpTestAuthCertName - Auth cert name in KeyVault +# EsrpTestAuthSignCertName - Signing cert name in KeyVault +# +# Test KeyCode mapping (this file): +# 1st-party Authenticode (was external_distribution / PROD CP-230012) +# -> TEST CP-466277 (RSA 2048 SHA256 Test 2; same across regions; Lifetime EKU) +# 3rd-party Authenticode (was $(msft_3rd_party_cert_id) / PROD CP-231522) +# -> TEST CP-466279 (RSA 3072 SHA256 Test 2; same across regions; Lifetime EKU) +# +# Final PROD swap when AME onboarding completes (will be Phase B): +# 1. Grep "CP-466277" -> replace with "CP-230012" +# 2. Grep "CP-466279" -> replace with "CP-231522" +# 3. Switch variable group from EsrpSigningTest-PowerShell to the prod equivalent +# 4. Delete the STUB branches if you want to require real signing for every run +# +# NuGet (CP-401405) and Apple (CP-401337-Apple) signing remain STUB-ONLY — no +# confirmed NuGet/Apple test KeyCodes available in the test inventory. See +# windows-hosted-build.yml and mac.yml for those sites. +# +# Predicate note: the 3rd-party detection block below (~line 142) uses regex +# patterns tuned for Microsoft PROD issuers. Test signatures from +# "Microsoft RSA Testing PCA 2020" still satisfy the `testCert` regex so the +# round-trip will exercise both 1st- and 3rd-party signing paths in TEST mode. +# See files/onebranch-to-1es-migration.md Gotcha #15 for full details. steps: - pwsh: | @@ -103,8 +128,9 @@ steps: Get-ChildItem -Recurse -File $filesToSignDirectory | Select-Object -Property FullName displayName: 'Prepare files to be signed' -# STUB: Original was `task: onebranch.pipeline.signing@1` with `signing_profile: ${{ parameters.SigningProfile }}` -# (default `external_distribution`, KeyCode CP-230012). Replace this stub once ESRP onboarding is complete. +# STUB (1st-party): runs when no ESRP test variable group is attached. +# Mirrors original `task: onebranch.pipeline.signing@1` with profile +# `${{ parameters.SigningProfile }}` (default `external_distribution`, PROD CP-230012). - pwsh: | Write-Host '##[warning]STUB: would invoke ESRP signing with profile "${{ parameters.SigningProfile }}"' Write-Host ' files_to_sign: **\*.psd1;**\*.psm1;**\*.ps1xml;**\*.ps1;**\*.dll;**\*.exe;**\pwsh' @@ -119,6 +145,51 @@ steps: Write-Host ' (toBeSigned folder does not exist)' } displayName: 'Sign 1st party files (STUB — ESRP onboarding pending)' + condition: and(succeeded(), eq(variables['EsrpTestConnectionName'], '')) + +# REAL TEST SIGNING (1st-party): runs only when EsrpSigningTest-PowerShell +# variable group is attached. Uses MS-Sign Test KeyCode CP-466277. +- task: EsrpCodeSigning@5 + displayName: 'Sign 1st party files (ESRP test signing)' + condition: and(succeeded(), ne(variables['EsrpTestConnectionName'], '')) + inputs: + ConnectedServiceName: $(EsrpTestConnectionName) + AppRegistrationClientId: $(EsrpTestAppRegClientId) + AppRegistrationTenantId: $(EsrpTestAppRegTenantId) + AuthAKVName: $(EsrpTestAuthAkvName) + AuthCertName: $(EsrpTestAuthCertName) + AuthSignCertName: $(EsrpTestAuthSignCertName) + FolderPath: $(Pipeline.Workspace)/toBeSigned + Pattern: '**/*.psd1,**/*.psm1,**/*.ps1xml,**/*.ps1,**/*.dll,**/*.exe,**/pwsh' + UseMinimatch: true + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-466277", + "operationSetCode": "SigntoolSign", + "parameters": [ + { "parameterName": "OpusName", "parameterValue": "Microsoft" }, + { "parameterName": "OpusInfo", "parameterValue": "http://www.microsoft.com" }, + { "parameterName": "PageHash", "parameterValue": "/NPH" }, + { "parameterName": "FileDigest", "parameterValue": "/fd \"SHA256\"" }, + { "parameterName": "TimeStamp", "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-466277", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 60 + MaxConcurrency: 50 + MaxRetryAttempts: 5 + PendingAnalysisWaitTimeoutMinutes: 5 - pwsh : | Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose @@ -163,8 +234,9 @@ steps: } displayName: Create ThirdParty Signing Folder -# STUB: Original was `task: onebranch.pipeline.signing@1` with `signing_profile: $(msft_3rd_party_cert_id)` -# (KeyCode CP-231522, 3rd-party Authenticode). Replace this stub once ESRP onboarding is complete. +# STUB (3rd-party): runs when no ESRP test variable group is attached. +# Mirrors original `task: onebranch.pipeline.signing@1` with profile +# `$(msft_3rd_party_cert_id)` (PROD CP-231522, 3rd-party Authenticode). - pwsh: | Write-Host '##[warning]STUB: would invoke ESRP signing with profile "$(msft_3rd_party_cert_id)" (3rd-party Authenticode)' Write-Host ' files_to_sign: **\*.dll;**\*.exe' @@ -179,6 +251,52 @@ steps: Write-Host ' (thirdPartyToBeSigned folder does not exist)' } displayName: 'Sign 3rd Party files (STUB — ESRP onboarding pending)' + condition: and(succeeded(), eq(variables['EsrpTestConnectionName'], '')) + +# REAL TEST SIGNING (3rd-party): runs only when EsrpSigningTest-PowerShell +# variable group is attached. Uses MS-Sign Test KeyCode CP-466279 (different +# cert chain from 1P so we exercise both signing paths in TEST mode). +- task: EsrpCodeSigning@5 + displayName: 'Sign 3rd Party files (ESRP test signing)' + condition: and(succeeded(), ne(variables['EsrpTestConnectionName'], '')) + inputs: + ConnectedServiceName: $(EsrpTestConnectionName) + AppRegistrationClientId: $(EsrpTestAppRegClientId) + AppRegistrationTenantId: $(EsrpTestAppRegTenantId) + AuthAKVName: $(EsrpTestAuthAkvName) + AuthCertName: $(EsrpTestAuthCertName) + AuthSignCertName: $(EsrpTestAuthSignCertName) + FolderPath: $(Pipeline.Workspace)/thirdPartyToBeSigned + Pattern: '**/*.dll,**/*.exe' + UseMinimatch: true + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-466279", + "operationSetCode": "SigntoolSign", + "parameters": [ + { "parameterName": "OpusName", "parameterValue": "Microsoft" }, + { "parameterName": "OpusInfo", "parameterValue": "http://www.microsoft.com" }, + { "parameterName": "PageHash", "parameterValue": "/NPH" }, + { "parameterName": "FileDigest", "parameterValue": "/fd \"SHA256\"" }, + { "parameterName": "TimeStamp", "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-466279", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 60 + MaxConcurrency: 50 + MaxRetryAttempts: 5 + PendingAnalysisWaitTimeoutMinutes: 5 - pwsh: | Get-ChildItem '$(Pipeline.Workspace)/thirdPartyToBeSigned/*' diff --git a/.pipelines/1ES/templates/windows-hosted-build.yml b/.pipelines/1ES/templates/windows-hosted-build.yml index 9f50ee6766..9a9d738cd4 100644 --- a/.pipelines/1ES/templates/windows-hosted-build.yml +++ b/.pipelines/1ES/templates/windows-hosted-build.yml @@ -5,10 +5,22 @@ parameters: # 1ES PT translation of /.pipelines/templates/windows-hosted-build.yml. # Single Windows job that BOTH builds and signs (the OneBranch original is the same shape). -# Three signing tasks STUBBED: -# 1. obp-file-signing.yml call for the main bin folder (Authenticode). -# 2. obp-file-signing.yml call for the global tool publish folder (fxdependent only). -# 3. Inline `Sign obj files` (external_distribution) and `Sign nupkg files` (nuget_cert_id). +# Three signing sites, two execution modes per site (selected at queue time by whether +# the runtime variable `EsrpTestConnectionName` is defined — see obp-file-signing.yml +# header for the full variable group story): +# 1. obp-file-signing.yml call for the main bin folder (1P Authenticode) +# - STUB MODE: Write-Host placeholder +# - TEST MODE: EsrpCodeSigning@5 against test code CP-466277 +# 2. obp-file-signing.yml call for the global tool publish folder (fxdependent only) +# - same modes as #1 +# 3. Inline `Sign obj files` (1P Authenticode) +# - STUB MODE: Write-Host placeholder +# - TEST MODE: EsrpCodeSigning@5 against test code CP-466277 +# 4. Inline `Sign nupkg files` ($(nuget_cert_id), PROD CP-401405) +# - STUB MODE ONLY: no NuGet test KeyCode is available in the MS test inventory, +# so this site stays as a Write-Host placeholder even in TEST MODE. Future +# work: identify a NuGet-compatible test code and wire EsrpCodeSigning@5 with +# operationSetCode "NuGetSign" / "NuGetVerify". # CodeQL3000Init/Finalize tasks DROPPED (1ES PT auto-injects via sdl.codeql config). # cloneToOfficialPath.yml replaced with stage-repo-under-PowerShell.yml. @@ -216,8 +228,10 @@ jobs: displayName: Capture obj files condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) - ## STUB: original was `task: onebranch.pipeline.signing@1` with `signing_profile: external_distribution` - ## (KeyCode CP-230012). Sign obj files needed for the global tool packaging. + ## STUB (obj files): runs when no ESRP test variable group is attached. + ## Mirrors original `task: onebranch.pipeline.signing@1` with profile + ## `external_distribution` (PROD CP-230012). Signs obj files needed for the + ## global tool packaging step. - pwsh: | Write-Host '##[warning]STUB: would invoke ESRP signing with profile "external_distribution" (KeyCode CP-230012)' Write-Host ' files_to_sign: **\*.dll;**\*.exe' @@ -228,7 +242,52 @@ jobs: ForEach-Object { Write-Host " Would sign: $_" } } displayName: 'Sign obj files (STUB — ESRP onboarding pending)' - condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent'), eq(variables['EsrpTestConnectionName'], '')) + + ## REAL TEST SIGNING (obj files): runs only when EsrpSigningTest-PowerShell + ## variable group is attached. Uses MS-Sign Test KeyCode CP-466277 (same as + ## 1st-party path in obp-file-signing.yml — both are 1P Authenticode). + - task: EsrpCodeSigning@5 + displayName: 'Sign obj files (ESRP test signing)' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent'), ne(variables['EsrpTestConnectionName'], '')) + inputs: + ConnectedServiceName: $(EsrpTestConnectionName) + AppRegistrationClientId: $(EsrpTestAppRegClientId) + AppRegistrationTenantId: $(EsrpTestAppRegTenantId) + AuthAKVName: $(EsrpTestAuthAkvName) + AuthCertName: $(EsrpTestAuthCertName) + AuthSignCertName: $(EsrpTestAuthSignCertName) + FolderPath: $(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release + Pattern: '**/*.dll,**/*.exe' + UseMinimatch: true + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-466277", + "operationSetCode": "SigntoolSign", + "parameters": [ + { "parameterName": "OpusName", "parameterValue": "Microsoft" }, + { "parameterName": "OpusInfo", "parameterValue": "http://www.microsoft.com" }, + { "parameterName": "PageHash", "parameterValue": "/NPH" }, + { "parameterName": "FileDigest", "parameterValue": "/fd \"SHA256\"" }, + { "parameterName": "TimeStamp", "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-466277", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 60 + MaxConcurrency: 50 + MaxRetryAttempts: 5 + PendingAnalysisWaitTimeoutMinutes: 5 - pwsh: | <# The way the packaging works is a bit tricky as when it is built, we cannot add the modules that come from gallery. @@ -316,8 +375,16 @@ jobs: displayName: 'Pack Windows.x64 global tool' condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) - ## STUB: original was `task: onebranch.pipeline.signing@1` with `cp_code: $(nuget_cert_id)` - ## (KeyCode CP-401405, NuGet package signing). + ## STUB-ONLY (nupkg): original was `task: onebranch.pipeline.signing@1` with + ## `cp_code: $(nuget_cert_id)` (PROD CP-401405, NuGet package signing). + ## This site stays STUB even in TEST MODE — no NuGet-compatible test KeyCode + ## is currently available in the MS test inventory. NuGet signing requires a + ## NuGet-specific cert (operationSetCode "NuGetSign"), and reusing the + ## Authenticode test codes (CP-466277 / CP-466279) would be rejected by ESRP + ## as "KeyCode not authorised for this operation". + ## To unblock: ask ESRP team for a NuGet test code, then add a sibling + ## EsrpCodeSigning@5 task here with `condition: ne(variables['EsrpTestConnectionName'], '')` + ## using operationSetCode "NuGetSign" / "NuGetVerify". - pwsh: | Write-Host '##[warning]STUB: would invoke ESRP signing with cp_code "$(nuget_cert_id)" (KeyCode CP-401405, NuGet)' Write-Host ' files_to_sign: **\*.nupkg' @@ -327,7 +394,7 @@ jobs: Select-Object -ExpandProperty FullName | ForEach-Object { Write-Host " Would sign: $_" } } - displayName: 'Sign nupkg files (STUB — ESRP onboarding pending)' + displayName: 'Sign nupkg files (STUB — no NuGet test KeyCode available)' condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) - template: /.pipelines/templates/step/finalize.yml@self From b8f6ec6b0752032dcb52a31b7de5073c40980bbb Mon Sep 17 00:00:00 2001 From: jshigetomi Date: Wed, 1 Jul 2026 13:55:56 -0700 Subject: [PATCH 18/19] Add Official 1ES pipeline for Coordinated_Packages with real ESRP prod signing --- ...werShell-Coordinated_Packages-Official.yml | 175 +++++++++ .pipelines/1ES/templates/linux.yml | 3 +- .pipelines/1ES/templates/mac.yml | 60 ++- .pipelines/1ES/templates/obp-file-signing.yml | 352 ++++++++++++------ ...PowerShell-Coordinated_Packages-Stages.yml | 17 + .../1ES/templates/windows-hosted-build.yml | 251 ++++++++----- 6 files changed, 639 insertions(+), 219 deletions(-) create mode 100644 .pipelines/1ES/PowerShell-Coordinated_Packages-Official.yml diff --git a/.pipelines/1ES/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/1ES/PowerShell-Coordinated_Packages-Official.yml new file mode 100644 index 0000000000..5863d24185 --- /dev/null +++ b/.pipelines/1ES/PowerShell-Coordinated_Packages-Official.yml @@ -0,0 +1,175 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# 1ES PIPELINE TEMPLATES (1ES PT) MIGRATION — OFFICIAL +# ---------------------------------------------------- +# Official sibling of `.pipelines/1ES/PowerShell-Coordinated_Packages-NonOfficial.yml`. +# Same stages/templates, but it extends the 1ES PT *Official* template and runs REAL +# production ESRP signing (OfficialBuild: true). The OneBranch original +# (`.pipelines/PowerShell-Coordinated_Packages-Official.yml`) is left untouched so the +# two can run side-by-side during migration validation. +# +# WHY A SEPARATE OFFICIAL PIPELINE: +# Only an Official (1ES.Official) pipeline can consume the production ESRP service +# connection `PowerShell-ESRP-Release`: that connection is gated by a 1ES +# "Production Readiness Check" that a 1ES.Unofficial pipeline can never satisfy +# (the check stays pending and blocks the run). Real ESRP signing must therefore be +# validated here, not in the NonOfficial pipeline. +# +# HOW OFFICIAL vs NONOFFICIAL SIGNING IS SELECTED: +# The shared signing templates fork at COMPILE TIME on the `OfficialBuild` parameter +# threaded from the stages template: +# * OfficialBuild: true (this pipeline) -> real `EsrpCodeSigning@5` tasks against +# PROD KeyCodes using `PowerShell-ESRP-Release`. The stub/test tasks are compile- +# time excluded, so `$(EsrpTestConnectionName)` is never referenced here. +# * OfficialBuild: false (NonOfficial) -> the existing stub / ESRP-test hybrid. +# The prod connection is compile-time excluded there, so the NonOfficial pipeline +# never triggers the Production Readiness Check. +# +# PROD SIGNING-SITE MAPPING (this pipeline, OfficialBuild: true): +# ───────────────────────────────────────────────────────────────────────────── +# Site | PROD KeyCode | Operation +# ───────────────────────────────────────────────────────────────────────────── +# obp-file-signing.yml — 1P Authenticode | CP-230012 | SigntoolSign +# obp-file-signing.yml — 3P Authenticode | CP-231522 | SigntoolSign +# windows-hosted-build.yml — Sign obj files (1P)| CP-230012 | SigntoolSign +# windows-hosted-build.yml — Sign nupkg files | CP-401405 | NuGetSign +# mac.yml — Apple CodeSign Mach-O binaries | CP-401337-Apple | MacAppDeveloperSign +# ───────────────────────────────────────────────────────────────────────────── +# All use ConnectedServiceName: PowerShell-ESRP-Release (hardcoded — ADO validates +# service-connection references at compile time, so it cannot come from a variable). +# +# REQUIRED VARIABLE GROUP — `EsrpSigning-PowerShell`: +# Create this variable group in mscodehub/PowerShellCore and link it to this pipeline +# definition. It supplies the ESRP request-signing auth inputs (non-secret identifiers) +# consumed by every prod EsrpCodeSigning@5 task: +# - EsrpProdAppRegClientId (client ID of the ESRP-Signing-Identity managed identity) +# - EsrpProdAppRegTenantId (AME tenant ID the MI lives in) +# - EsrpProdAuthAkvName (Key Vault holding the ESRP request-signing cert, e.g. pscoresignprod-kv) +# - EsrpProdAuthCertName (auth cert name in that Key Vault) +# - EsrpProdAuthSignCertName (request-signing cert name in that Key Vault) +# The MI must have `Key Vault Certificate User` on the vault and be onboarded to ESRP +# with the KeyCodes above authorised. + +trigger: none + +parameters: + - name: InternalSDKBlobURL + displayName: URL to the blob having internal .NET SDK + type: string + default: ' ' + - name: ReleaseTagVar + displayName: Release Tag + type: string + default: 'fromBranch' + - name: SKIP_SIGNING + displayName: Debugging - Skip Signing + type: string + default: 'NO' + - name: RUN_TEST_AND_RELEASE + displayName: Debugging - Run Test and Release Artifacts Stage + type: boolean + default: true + - name: RUN_WINDOWS + displayName: Debugging - Enable Windows Stage + type: boolean + default: true + - name: ENABLE_MSBUILD_BINLOGS + displayName: Debugging - Enable MSBuild Binary Logs + type: boolean + default: false + - name: FORCE_CODEQL + displayName: Debugging - Enable CodeQL and set cadence to 1 hour + type: boolean + default: false + +name: bins-$(BUILD.SOURCEBRANCHNAME)-1ES-official-$(Build.BuildId) + +variables: + # Inherit the same variable scaffold as the NonOfficial pipeline / OneBranch original. + - template: /.pipelines/templates/variables/PowerShell-Coordinated_Packages-Variables.yml@self + parameters: + InternalSDKBlobURL: ${{ parameters.InternalSDKBlobURL }} + ReleaseTagVar: ${{ parameters.ReleaseTagVar }} + SKIP_SIGNING: ${{ parameters.SKIP_SIGNING }} + ENABLE_MSBUILD_BINLOGS: ${{ parameters.ENABLE_MSBUILD_BINLOGS }} + FORCE_CODEQL: ${{ parameters.FORCE_CODEQL }} + # PROD ESRP request-signing auth inputs (see header). Only the Official pipeline + # references these — they are consumed exclusively inside the compile-time + # OfficialBuild: true branches of the shared signing templates. + - group: EsrpSigning-PowerShell + +resources: + repositories: + # ComplianceRepo is preserved verbatim (used by helper templates that + # invoke PoliCheck / CredScan via the GH-hosted PowerShell/compliance repo). + - repository: ComplianceRepo + type: github + endpoint: ComplianceGHRepo + name: PowerShell/compliance + ref: master + # 1ES Pipeline Templates resource (replaces `onebranchTemplates`). + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + # Pool default for all jobs. Windows jobs inherit; macOS/Linux jobs override + # at the per-job level inside the templates. + pool: + name: PowerShell1ES + image: MMS2022 + os: windows + + sdl: + # CodeQL: enabled unconditionally on ALL branches (same rationale as the + # NonOfficial pipeline — PowerShell's ADO default branch is `internal-master`, + # never used for releases; shipping branches are release/* and rebuild/*). + codeql: + compiled: + enabled: true + enabledOnNonDefaultBranches: true + tsaEnabled: true # File CodeQL findings to TSA on every run; TSA dedupes + + # `armory` was OneBranch-only. Removed. + + sbom: + enabled: true + + cg: + enabled: true + ignoreDirectories: '.devcontainer,demos,docker,docs,src,test,tools/packaging' + + tsa: + enabled: true # 1ES publishes all SDL results to TSA. Disabling forces all SDL tools into 'break' mode. + # 1ES PT requires an absolute path for `sdl.tsa.configFile`. This path is + # consumed by the auto-injected SDL Sources Analysis job, which runs WITHOUT + # the `stage-repo-under-PowerShell.yml` helper (repo at $(Build.SourcesDirectory)). + # DO NOT add a `\PowerShell\` segment here. + configFile: $(Build.SourcesDirectory)\.config\tsaoptions.json + + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + + # BinSkim runs at the user-job level in 1ES PT. Disabled at the global level + # to mirror the OneBranch original; re-enabled per-job for binary-producing jobs. + binskim: + enabled: false + exactToolVersion: 4.4.2 + + # APIScan: not part of the build pipeline. Lives in + # `.pipelines/1ES/apiscan-gen-notice.yml`. + apiscan: + enabled: false + + stages: + - template: /.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml@self + parameters: + RUN_WINDOWS: ${{ parameters.RUN_WINDOWS }} + RUN_TEST_AND_RELEASE: ${{ parameters.RUN_TEST_AND_RELEASE }} + OfficialBuild: true diff --git a/.pipelines/1ES/templates/linux.yml b/.pipelines/1ES/templates/linux.yml index ae496b516e..5300c6a509 100644 --- a/.pipelines/1ES/templates/linux.yml +++ b/.pipelines/1ES/templates/linux.yml @@ -2,6 +2,7 @@ parameters: Runtime: 'linux-x64' BuildConfiguration: 'release' JobName: 'build_linux' + OfficialBuild: false # 1ES PT translation of /.pipelines/templates/linux.yml. # Two jobs: @@ -224,6 +225,6 @@ jobs: - template: /.pipelines/1ES/templates/obp-file-signing.yml@self parameters: binPath: $(DropRootPath) - OfficialBuild: $(ps_official_build) + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/1ES/templates/mac.yml b/.pipelines/1ES/templates/mac.yml index cf4a7fba4a..df73c1bce6 100644 --- a/.pipelines/1ES/templates/mac.yml +++ b/.pipelines/1ES/templates/mac.yml @@ -1,5 +1,6 @@ parameters: buildArchitecture: 'x64' + OfficialBuild: false # 1ES PT translation of /.pipelines/templates/mac.yml. # Two jobs: @@ -199,23 +200,60 @@ jobs: - template: /.pipelines/1ES/templates/obp-file-signing.yml@self parameters: binPath: $(DropRootPath) - OfficialBuild: $(ps_official_build) - - # STUB SECTION — Apple Developer ID codesign of Mach-O binaries. - # Once ESRP onboarding is complete and apple_cert_id (KeyCode CP-401337-Apple, - # MacAppDeveloperSign) is wired up, replace the stub below with an EsrpCodeSigning@5 - # task call. The Compress/Expand framing stays the same. + OfficialBuild: ${{ parameters.OfficialBuild }} + + # APPLE SIGNING — Apple Developer ID codesign of Mach-O binaries (packed into + # a zip for signing). The Compress/Expand framing is COMMON to both pipelines; + # only the sign step forks on OfficialBuild: + # * Official (OfficialBuild: true) -> real EsrpCodeSigning@5 MacAppDeveloperSign + # (KeyCode CP-401337-Apple) via PowerShell-ESRP-Release. + # * NonOfficial (OfficialBuild: false) -> Write-Host stub (unsigned). - pwsh: | $signedDir = "$(ob_outputDirectory)/Signed-$(Runtime)" $zipFile = "$(Pipeline.Workspace)/macho-$(BuildArchitecture).zip" Compress-Archive -Path "$signedDir/*" -DestinationPath $zipFile -Force displayName: Compress signed folder for Apple signing - - pwsh: | - Write-Host '##[warning]STUB: would invoke ESRP Apple CodeSign Mach-O binaries with KeyCode "$(apple_cert_id)" (MacAppDeveloperSign, Hardening=--options=runtime)' - Write-Host ' files_to_sign: macho-$(BuildArchitecture).zip' - Write-Host ' search_root: $(Pipeline.Workspace)' - displayName: 'Apple CodeSign Mach-O binaries (STUB — ESRP onboarding pending)' + # PROD Apple signing — emitted ONLY for the Official pipeline. + - ${{ if eq(parameters.OfficialBuild, true) }}: + - task: EsrpCodeSigning@5 + displayName: 'Apple CodeSign Mach-O binaries (ESRP prod signing)' + condition: succeeded() + inputs: + ConnectedServiceName: PowerShell-ESRP-Release + AppRegistrationClientId: $(EsrpProdAppRegClientId) + AppRegistrationTenantId: $(EsrpProdAppRegTenantId) + AuthAKVName: $(EsrpProdAuthAkvName) + AuthCertName: $(EsrpProdAuthCertName) + AuthSignCertName: $(EsrpProdAuthSignCertName) + FolderPath: $(Pipeline.Workspace) + Pattern: 'macho-$(BuildArchitecture).zip' + UseMinimatch: true + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-401337-Apple", + "operationSetCode": "MacAppDeveloperSign", + "parameters": [ + { "parameterName": "Hardening", "parameterValue": "--options=runtime" } + ], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 60 + MaxConcurrency: 50 + MaxRetryAttempts: 5 + PendingAnalysisWaitTimeoutMinutes: 5 + + # STUB Apple signing — emitted ONLY for the NonOfficial pipeline. + - ${{ if ne(parameters.OfficialBuild, true) }}: + - pwsh: | + Write-Host '##[warning]STUB: would invoke ESRP Apple CodeSign Mach-O binaries with KeyCode "$(apple_cert_id)" (MacAppDeveloperSign, Hardening=--options=runtime)' + Write-Host ' files_to_sign: macho-$(BuildArchitecture).zip' + Write-Host ' search_root: $(Pipeline.Workspace)' + displayName: 'Apple CodeSign Mach-O binaries (STUB — ESRP onboarding pending)' - pwsh: | $signedDir = "$(ob_outputDirectory)/Signed-$(Runtime)" diff --git a/.pipelines/1ES/templates/obp-file-signing.yml b/.pipelines/1ES/templates/obp-file-signing.yml index 125027aa9b..37f5b078da 100644 --- a/.pipelines/1ES/templates/obp-file-signing.yml +++ b/.pipelines/1ES/templates/obp-file-signing.yml @@ -2,7 +2,7 @@ parameters: binPath: '$(ob_outputDirectory)' globalTool: 'false' SigningProfile: 'external_distribution' - OfficialBuild: true + OfficialBuild: false vPackScenario: false # 1ES PT translation of /.pipelines/templates/obp-file-signing.yml — HYBRID STUB/TEST-SIGNING. @@ -128,68 +128,122 @@ steps: Get-ChildItem -Recurse -File $filesToSignDirectory | Select-Object -Property FullName displayName: 'Prepare files to be signed' -# STUB (1st-party): runs when no ESRP test variable group is attached. -# Mirrors original `task: onebranch.pipeline.signing@1` with profile -# `${{ parameters.SigningProfile }}` (default `external_distribution`, PROD CP-230012). -- pwsh: | - Write-Host '##[warning]STUB: would invoke ESRP signing with profile "${{ parameters.SigningProfile }}"' - Write-Host ' files_to_sign: **\*.psd1;**\*.psm1;**\*.ps1xml;**\*.ps1;**\*.dll;**\*.exe;**\pwsh' - Write-Host ' search_root: $(Pipeline.Workspace)/toBeSigned' - Write-Host '' - Write-Host 'Files in search_root (would be signed):' - if (Test-Path '$(Pipeline.Workspace)/toBeSigned') { - Get-ChildItem -Path '$(Pipeline.Workspace)/toBeSigned' -Recurse -File | - Select-Object -ExpandProperty FullName | - ForEach-Object { Write-Host " $_" } - } else { - Write-Host ' (toBeSigned folder does not exist)' - } - displayName: 'Sign 1st party files (STUB — ESRP onboarding pending)' - condition: and(succeeded(), eq(variables['EsrpTestConnectionName'], '')) +# PROD SIGNING (1st-party): emitted ONLY for the Official pipeline +# (OfficialBuild: true). Real ESRP Authenticode signing via the production +# service connection PowerShell-ESRP-Release against PROD KeyCode CP-230012 +# (was profile `external_distribution`). Auth inputs (AppReg/AKV/cert) come from +# the `EsrpSigning-PowerShell` variable group declared in the Official entry. +# This block is compile-time excluded from the NonOfficial pipeline so the prod +# connection is never referenced there (avoids the Production Readiness Check). +- ${{ if eq(parameters.OfficialBuild, true) }}: + - task: EsrpCodeSigning@5 + displayName: 'Sign 1st party files (ESRP prod signing)' + condition: succeeded() + inputs: + ConnectedServiceName: PowerShell-ESRP-Release + AppRegistrationClientId: $(EsrpProdAppRegClientId) + AppRegistrationTenantId: $(EsrpProdAppRegTenantId) + AuthAKVName: $(EsrpProdAuthAkvName) + AuthCertName: $(EsrpProdAuthCertName) + AuthSignCertName: $(EsrpProdAuthSignCertName) + FolderPath: $(Pipeline.Workspace)/toBeSigned + Pattern: '**/*.psd1,**/*.psm1,**/*.ps1xml,**/*.ps1,**/*.dll,**/*.exe,**/pwsh' + UseMinimatch: true + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { "parameterName": "OpusName", "parameterValue": "Microsoft" }, + { "parameterName": "OpusInfo", "parameterValue": "http://www.microsoft.com" }, + { "parameterName": "PageHash", "parameterValue": "/NPH" }, + { "parameterName": "FileDigest", "parameterValue": "/fd \"SHA256\"" }, + { "parameterName": "TimeStamp", "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 60 + MaxConcurrency: 50 + MaxRetryAttempts: 5 + PendingAnalysisWaitTimeoutMinutes: 5 + +# STUB / TEST SIGNING (1st-party): emitted ONLY for the NonOfficial pipeline +# (OfficialBuild: false). Preserves the existing hybrid unchanged — STUB when no +# ESRP test variable group is attached, real TEST signing (CP-466277) when it is. +- ${{ if ne(parameters.OfficialBuild, true) }}: + # STUB (1st-party): runs when no ESRP test variable group is attached. + # Mirrors original `task: onebranch.pipeline.signing@1` with profile + # `${{ parameters.SigningProfile }}` (default `external_distribution`, PROD CP-230012). + - pwsh: | + Write-Host '##[warning]STUB: would invoke ESRP signing with profile "${{ parameters.SigningProfile }}"' + Write-Host ' files_to_sign: **\*.psd1;**\*.psm1;**\*.ps1xml;**\*.ps1;**\*.dll;**\*.exe;**\pwsh' + Write-Host ' search_root: $(Pipeline.Workspace)/toBeSigned' + Write-Host '' + Write-Host 'Files in search_root (would be signed):' + if (Test-Path '$(Pipeline.Workspace)/toBeSigned') { + Get-ChildItem -Path '$(Pipeline.Workspace)/toBeSigned' -Recurse -File | + Select-Object -ExpandProperty FullName | + ForEach-Object { Write-Host " $_" } + } else { + Write-Host ' (toBeSigned folder does not exist)' + } + displayName: 'Sign 1st party files (STUB — ESRP onboarding pending)' + condition: and(succeeded(), eq(variables['EsrpTestConnectionName'], '')) -# REAL TEST SIGNING (1st-party): runs only when EsrpSigningTest-PowerShell -# variable group is attached. Uses MS-Sign Test KeyCode CP-466277. -- task: EsrpCodeSigning@5 - displayName: 'Sign 1st party files (ESRP test signing)' - condition: and(succeeded(), ne(variables['EsrpTestConnectionName'], '')) - inputs: - ConnectedServiceName: $(EsrpTestConnectionName) - AppRegistrationClientId: $(EsrpTestAppRegClientId) - AppRegistrationTenantId: $(EsrpTestAppRegTenantId) - AuthAKVName: $(EsrpTestAuthAkvName) - AuthCertName: $(EsrpTestAuthCertName) - AuthSignCertName: $(EsrpTestAuthSignCertName) - FolderPath: $(Pipeline.Workspace)/toBeSigned - Pattern: '**/*.psd1,**/*.psm1,**/*.ps1xml,**/*.ps1,**/*.dll,**/*.exe,**/pwsh' - UseMinimatch: true - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-466277", - "operationSetCode": "SigntoolSign", - "parameters": [ - { "parameterName": "OpusName", "parameterValue": "Microsoft" }, - { "parameterName": "OpusInfo", "parameterValue": "http://www.microsoft.com" }, - { "parameterName": "PageHash", "parameterValue": "/NPH" }, - { "parameterName": "FileDigest", "parameterValue": "/fd \"SHA256\"" }, - { "parameterName": "TimeStamp", "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" } - ], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-466277", - "operationSetCode": "SigntoolVerify", - "parameters": [], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: 60 - MaxConcurrency: 50 - MaxRetryAttempts: 5 - PendingAnalysisWaitTimeoutMinutes: 5 + # REAL TEST SIGNING (1st-party): runs only when EsrpSigningTest-PowerShell + # variable group is attached. Uses MS-Sign Test KeyCode CP-466277. + - task: EsrpCodeSigning@5 + displayName: 'Sign 1st party files (ESRP test signing)' + condition: and(succeeded(), ne(variables['EsrpTestConnectionName'], '')) + inputs: + ConnectedServiceName: $(EsrpTestConnectionName) + AppRegistrationClientId: $(EsrpTestAppRegClientId) + AppRegistrationTenantId: $(EsrpTestAppRegTenantId) + AuthAKVName: $(EsrpTestAuthAkvName) + AuthCertName: $(EsrpTestAuthCertName) + AuthSignCertName: $(EsrpTestAuthSignCertName) + FolderPath: $(Pipeline.Workspace)/toBeSigned + Pattern: '**/*.psd1,**/*.psm1,**/*.ps1xml,**/*.ps1,**/*.dll,**/*.exe,**/pwsh' + UseMinimatch: true + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-466277", + "operationSetCode": "SigntoolSign", + "parameters": [ + { "parameterName": "OpusName", "parameterValue": "Microsoft" }, + { "parameterName": "OpusInfo", "parameterValue": "http://www.microsoft.com" }, + { "parameterName": "PageHash", "parameterValue": "/NPH" }, + { "parameterName": "FileDigest", "parameterValue": "/fd \"SHA256\"" }, + { "parameterName": "TimeStamp", "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-466277", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 60 + MaxConcurrency: 50 + MaxRetryAttempts: 5 + PendingAnalysisWaitTimeoutMinutes: 5 - pwsh : | Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose @@ -234,69 +288,119 @@ steps: } displayName: Create ThirdParty Signing Folder -# STUB (3rd-party): runs when no ESRP test variable group is attached. -# Mirrors original `task: onebranch.pipeline.signing@1` with profile -# `$(msft_3rd_party_cert_id)` (PROD CP-231522, 3rd-party Authenticode). -- pwsh: | - Write-Host '##[warning]STUB: would invoke ESRP signing with profile "$(msft_3rd_party_cert_id)" (3rd-party Authenticode)' - Write-Host ' files_to_sign: **\*.dll;**\*.exe' - Write-Host ' search_root: $(Pipeline.Workspace)/thirdPartyToBeSigned' - Write-Host '' - Write-Host 'Files in search_root (would be signed):' - if (Test-Path '$(Pipeline.Workspace)/thirdPartyToBeSigned') { - Get-ChildItem -Path '$(Pipeline.Workspace)/thirdPartyToBeSigned' -Recurse -File | - Select-Object -ExpandProperty FullName | - ForEach-Object { Write-Host " $_" } - } else { - Write-Host ' (thirdPartyToBeSigned folder does not exist)' - } - displayName: 'Sign 3rd Party files (STUB — ESRP onboarding pending)' - condition: and(succeeded(), eq(variables['EsrpTestConnectionName'], '')) +# PROD SIGNING (3rd-party): emitted ONLY for the Official pipeline +# (OfficialBuild: true). Real ESRP 3rd-party Authenticode signing via the +# production connection PowerShell-ESRP-Release against PROD KeyCode CP-231522 +# (was $(msft_3rd_party_cert_id)). Compile-time excluded from NonOfficial. +- ${{ if eq(parameters.OfficialBuild, true) }}: + - task: EsrpCodeSigning@5 + displayName: 'Sign 3rd Party files (ESRP prod signing)' + condition: succeeded() + inputs: + ConnectedServiceName: PowerShell-ESRP-Release + AppRegistrationClientId: $(EsrpProdAppRegClientId) + AppRegistrationTenantId: $(EsrpProdAppRegTenantId) + AuthAKVName: $(EsrpProdAuthAkvName) + AuthCertName: $(EsrpProdAuthCertName) + AuthSignCertName: $(EsrpProdAuthSignCertName) + FolderPath: $(Pipeline.Workspace)/thirdPartyToBeSigned + Pattern: '**/*.dll,**/*.exe' + UseMinimatch: true + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-231522", + "operationSetCode": "SigntoolSign", + "parameters": [ + { "parameterName": "OpusName", "parameterValue": "Microsoft" }, + { "parameterName": "OpusInfo", "parameterValue": "http://www.microsoft.com" }, + { "parameterName": "PageHash", "parameterValue": "/NPH" }, + { "parameterName": "FileDigest", "parameterValue": "/fd \"SHA256\"" }, + { "parameterName": "TimeStamp", "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-231522", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 60 + MaxConcurrency: 50 + MaxRetryAttempts: 5 + PendingAnalysisWaitTimeoutMinutes: 5 + +# STUB / TEST SIGNING (3rd-party): emitted ONLY for the NonOfficial pipeline +# (OfficialBuild: false). Preserves the existing hybrid unchanged. +- ${{ if ne(parameters.OfficialBuild, true) }}: + # STUB (3rd-party): runs when no ESRP test variable group is attached. + # Mirrors original `task: onebranch.pipeline.signing@1` with profile + # `$(msft_3rd_party_cert_id)` (PROD CP-231522, 3rd-party Authenticode). + - pwsh: | + Write-Host '##[warning]STUB: would invoke ESRP signing with profile "$(msft_3rd_party_cert_id)" (3rd-party Authenticode)' + Write-Host ' files_to_sign: **\*.dll;**\*.exe' + Write-Host ' search_root: $(Pipeline.Workspace)/thirdPartyToBeSigned' + Write-Host '' + Write-Host 'Files in search_root (would be signed):' + if (Test-Path '$(Pipeline.Workspace)/thirdPartyToBeSigned') { + Get-ChildItem -Path '$(Pipeline.Workspace)/thirdPartyToBeSigned' -Recurse -File | + Select-Object -ExpandProperty FullName | + ForEach-Object { Write-Host " $_" } + } else { + Write-Host ' (thirdPartyToBeSigned folder does not exist)' + } + displayName: 'Sign 3rd Party files (STUB — ESRP onboarding pending)' + condition: and(succeeded(), eq(variables['EsrpTestConnectionName'], '')) -# REAL TEST SIGNING (3rd-party): runs only when EsrpSigningTest-PowerShell -# variable group is attached. Uses MS-Sign Test KeyCode CP-466279 (different -# cert chain from 1P so we exercise both signing paths in TEST mode). -- task: EsrpCodeSigning@5 - displayName: 'Sign 3rd Party files (ESRP test signing)' - condition: and(succeeded(), ne(variables['EsrpTestConnectionName'], '')) - inputs: - ConnectedServiceName: $(EsrpTestConnectionName) - AppRegistrationClientId: $(EsrpTestAppRegClientId) - AppRegistrationTenantId: $(EsrpTestAppRegTenantId) - AuthAKVName: $(EsrpTestAuthAkvName) - AuthCertName: $(EsrpTestAuthCertName) - AuthSignCertName: $(EsrpTestAuthSignCertName) - FolderPath: $(Pipeline.Workspace)/thirdPartyToBeSigned - Pattern: '**/*.dll,**/*.exe' - UseMinimatch: true - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-466279", - "operationSetCode": "SigntoolSign", - "parameters": [ - { "parameterName": "OpusName", "parameterValue": "Microsoft" }, - { "parameterName": "OpusInfo", "parameterValue": "http://www.microsoft.com" }, - { "parameterName": "PageHash", "parameterValue": "/NPH" }, - { "parameterName": "FileDigest", "parameterValue": "/fd \"SHA256\"" }, - { "parameterName": "TimeStamp", "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" } - ], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-466279", - "operationSetCode": "SigntoolVerify", - "parameters": [], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: 60 - MaxConcurrency: 50 - MaxRetryAttempts: 5 - PendingAnalysisWaitTimeoutMinutes: 5 + # REAL TEST SIGNING (3rd-party): runs only when EsrpSigningTest-PowerShell + # variable group is attached. Uses MS-Sign Test KeyCode CP-466279 (different + # cert chain from 1P so we exercise both signing paths in TEST mode). + - task: EsrpCodeSigning@5 + displayName: 'Sign 3rd Party files (ESRP test signing)' + condition: and(succeeded(), ne(variables['EsrpTestConnectionName'], '')) + inputs: + ConnectedServiceName: $(EsrpTestConnectionName) + AppRegistrationClientId: $(EsrpTestAppRegClientId) + AppRegistrationTenantId: $(EsrpTestAppRegTenantId) + AuthAKVName: $(EsrpTestAuthAkvName) + AuthCertName: $(EsrpTestAuthCertName) + AuthSignCertName: $(EsrpTestAuthSignCertName) + FolderPath: $(Pipeline.Workspace)/thirdPartyToBeSigned + Pattern: '**/*.dll,**/*.exe' + UseMinimatch: true + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-466279", + "operationSetCode": "SigntoolSign", + "parameters": [ + { "parameterName": "OpusName", "parameterValue": "Microsoft" }, + { "parameterName": "OpusInfo", "parameterValue": "http://www.microsoft.com" }, + { "parameterName": "PageHash", "parameterValue": "/NPH" }, + { "parameterName": "FileDigest", "parameterValue": "/fd \"SHA256\"" }, + { "parameterName": "TimeStamp", "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-466279", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 60 + MaxConcurrency: 50 + MaxRetryAttempts: 5 + PendingAnalysisWaitTimeoutMinutes: 5 - pwsh: | Get-ChildItem '$(Pipeline.Workspace)/thirdPartyToBeSigned/*' diff --git a/.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml b/.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml index e9cc50e617..18c373d8d2 100644 --- a/.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml +++ b/.pipelines/1ES/templates/stages/PowerShell-Coordinated_Packages-Stages.yml @@ -66,9 +66,11 @@ stages: - template: /.pipelines/1ES/templates/mac.yml@self parameters: buildArchitecture: x64 + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/mac.yml@self parameters: buildArchitecture: arm64 + OfficialBuild: ${{ parameters.OfficialBuild }} - stage: linux displayName: linux - build and sign @@ -81,47 +83,56 @@ stages: parameters: Runtime: 'linux-x64' JobName: 'linux_x64' + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/linux.yml@self parameters: Runtime: 'linux-x64' JobName: 'linux_x64_minSize' BuildConfiguration: 'minSize' + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/linux.yml@self parameters: Runtime: 'linux-arm' JobName: 'linux_arm' + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/linux.yml@self parameters: Runtime: 'linux-arm64' JobName: 'linux_arm64' + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/linux.yml@self parameters: Runtime: 'fxdependent-linux-x64' JobName: 'linux_fxd_x64_mariner' + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/linux.yml@self parameters: Runtime: 'fxdependent-linux-arm64' JobName: 'linux_fxd_arm64_mariner' + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/linux.yml@self parameters: Runtime: 'fxdependent-noopt-linux-musl-x64' JobName: 'linux_fxd_x64_alpine' + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/linux.yml@self parameters: Runtime: 'fxdependent' JobName: 'linux_fxd' + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/linux.yml@self parameters: Runtime: 'linux-musl-x64' JobName: 'linux_x64_alpine' + OfficialBuild: ${{ parameters.OfficialBuild }} - stage: windows displayName: windows - build and sign @@ -136,27 +147,33 @@ stages: Architecture: x64 BuildConfiguration: release JobName: build_windows_x64_release + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/windows-hosted-build.yml@self parameters: Architecture: x64 BuildConfiguration: minSize JobName: build_windows_x64_minSize_release + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/windows-hosted-build.yml@self parameters: Architecture: x86 JobName: build_windows_x86_release + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/windows-hosted-build.yml@self parameters: Architecture: arm64 JobName: build_windows_arm64_release + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/windows-hosted-build.yml@self parameters: Architecture: fxdependent JobName: build_windows_fxdependent_release + OfficialBuild: ${{ parameters.OfficialBuild }} - template: /.pipelines/1ES/templates/windows-hosted-build.yml@self parameters: Architecture: fxdependentWinDesktop JobName: build_windows_fxdependentWinDesktop_release + OfficialBuild: ${{ parameters.OfficialBuild }} - stage: test_and_release_artifacts displayName: Test and Release Artifacts diff --git a/.pipelines/1ES/templates/windows-hosted-build.yml b/.pipelines/1ES/templates/windows-hosted-build.yml index 9a9d738cd4..6a58e4853f 100644 --- a/.pipelines/1ES/templates/windows-hosted-build.yml +++ b/.pipelines/1ES/templates/windows-hosted-build.yml @@ -2,6 +2,7 @@ parameters: Architecture: 'x64' BuildConfiguration: 'release' JobName: 'build_windows' + OfficialBuild: false # 1ES PT translation of /.pipelines/templates/windows-hosted-build.yml. # Single Windows job that BOTH builds and signs (the OneBranch original is the same shape). @@ -213,7 +214,7 @@ jobs: - template: /.pipelines/1ES/templates/obp-file-signing.yml@self parameters: binPath: '$(Pipeline.Workspace)/Symbols_$(Architecture)' - OfficialBuild: $(ps_official_build) + OfficialBuild: ${{ parameters.OfficialBuild }} ## first we sign all the files in the bin folder - ${{ if eq(variables['Architecture'], 'fxdependent') }}: @@ -221,73 +222,122 @@ jobs: parameters: binPath: '$(GlobalToolArtifactPath)/publish/PowerShell.Windows.x64/release' globalTool: 'true' - OfficialBuild: $(ps_official_build) + OfficialBuild: ${{ parameters.OfficialBuild }} - pwsh: | Get-ChildItem '$(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release' displayName: Capture obj files condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) - ## STUB (obj files): runs when no ESRP test variable group is attached. - ## Mirrors original `task: onebranch.pipeline.signing@1` with profile - ## `external_distribution` (PROD CP-230012). Signs obj files needed for the - ## global tool packaging step. - - pwsh: | - Write-Host '##[warning]STUB: would invoke ESRP signing with profile "external_distribution" (KeyCode CP-230012)' - Write-Host ' files_to_sign: **\*.dll;**\*.exe' - Write-Host ' search_root: $(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release' - if (Test-Path '$(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release') { - Get-ChildItem -Path '$(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release' -Recurse -Include '*.dll','*.exe' | - Select-Object -ExpandProperty FullName | - ForEach-Object { Write-Host " Would sign: $_" } - } - displayName: 'Sign obj files (STUB — ESRP onboarding pending)' - condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent'), eq(variables['EsrpTestConnectionName'], '')) - - ## REAL TEST SIGNING (obj files): runs only when EsrpSigningTest-PowerShell - ## variable group is attached. Uses MS-Sign Test KeyCode CP-466277 (same as - ## 1st-party path in obp-file-signing.yml — both are 1P Authenticode). - - task: EsrpCodeSigning@5 - displayName: 'Sign obj files (ESRP test signing)' - condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent'), ne(variables['EsrpTestConnectionName'], '')) - inputs: - ConnectedServiceName: $(EsrpTestConnectionName) - AppRegistrationClientId: $(EsrpTestAppRegClientId) - AppRegistrationTenantId: $(EsrpTestAppRegTenantId) - AuthAKVName: $(EsrpTestAuthAkvName) - AuthCertName: $(EsrpTestAuthCertName) - AuthSignCertName: $(EsrpTestAuthSignCertName) - FolderPath: $(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release - Pattern: '**/*.dll,**/*.exe' - UseMinimatch: true - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-466277", - "operationSetCode": "SigntoolSign", - "parameters": [ - { "parameterName": "OpusName", "parameterValue": "Microsoft" }, - { "parameterName": "OpusInfo", "parameterValue": "http://www.microsoft.com" }, - { "parameterName": "PageHash", "parameterValue": "/NPH" }, - { "parameterName": "FileDigest", "parameterValue": "/fd \"SHA256\"" }, - { "parameterName": "TimeStamp", "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" } - ], - "toolName": "sign", - "toolVersion": "1.0" - }, - { - "keyCode": "CP-466277", - "operationSetCode": "SigntoolVerify", - "parameters": [], - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: 60 - MaxConcurrency: 50 - MaxRetryAttempts: 5 - PendingAnalysisWaitTimeoutMinutes: 5 + ## PROD SIGNING (obj files): emitted ONLY for the Official pipeline + ## (OfficialBuild: true). Real ESRP 1P Authenticode of the global-tool obj + ## files via PowerShell-ESRP-Release against PROD KeyCode CP-230012. + - ${{ if eq(parameters.OfficialBuild, true) }}: + - task: EsrpCodeSigning@5 + displayName: 'Sign obj files (ESRP prod signing)' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + inputs: + ConnectedServiceName: PowerShell-ESRP-Release + AppRegistrationClientId: $(EsrpProdAppRegClientId) + AppRegistrationTenantId: $(EsrpProdAppRegTenantId) + AuthAKVName: $(EsrpProdAuthAkvName) + AuthCertName: $(EsrpProdAuthCertName) + AuthSignCertName: $(EsrpProdAuthSignCertName) + FolderPath: $(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release + Pattern: '**/*.dll,**/*.exe' + UseMinimatch: true + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolSign", + "parameters": [ + { "parameterName": "OpusName", "parameterValue": "Microsoft" }, + { "parameterName": "OpusInfo", "parameterValue": "http://www.microsoft.com" }, + { "parameterName": "PageHash", "parameterValue": "/NPH" }, + { "parameterName": "FileDigest", "parameterValue": "/fd \"SHA256\"" }, + { "parameterName": "TimeStamp", "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-230012", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 60 + MaxConcurrency: 50 + MaxRetryAttempts: 5 + PendingAnalysisWaitTimeoutMinutes: 5 + + ## STUB / TEST SIGNING (obj files): emitted ONLY for the NonOfficial pipeline + ## (OfficialBuild: false). Preserves the existing hybrid unchanged. + - ${{ if ne(parameters.OfficialBuild, true) }}: + ## STUB (obj files): runs when no ESRP test variable group is attached. + ## Mirrors original `task: onebranch.pipeline.signing@1` with profile + ## `external_distribution` (PROD CP-230012). Signs obj files needed for the + ## global tool packaging step. + - pwsh: | + Write-Host '##[warning]STUB: would invoke ESRP signing with profile "external_distribution" (KeyCode CP-230012)' + Write-Host ' files_to_sign: **\*.dll;**\*.exe' + Write-Host ' search_root: $(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release' + if (Test-Path '$(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release') { + Get-ChildItem -Path '$(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release' -Recurse -Include '*.dll','*.exe' | + Select-Object -ExpandProperty FullName | + ForEach-Object { Write-Host " Would sign: $_" } + } + displayName: 'Sign obj files (STUB — ESRP onboarding pending)' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent'), eq(variables['EsrpTestConnectionName'], '')) + + ## REAL TEST SIGNING (obj files): runs only when EsrpSigningTest-PowerShell + ## variable group is attached. Uses MS-Sign Test KeyCode CP-466277 (same as + ## 1st-party path in obp-file-signing.yml — both are 1P Authenticode). + - task: EsrpCodeSigning@5 + displayName: 'Sign obj files (ESRP test signing)' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent'), ne(variables['EsrpTestConnectionName'], '')) + inputs: + ConnectedServiceName: $(EsrpTestConnectionName) + AppRegistrationClientId: $(EsrpTestAppRegClientId) + AppRegistrationTenantId: $(EsrpTestAppRegTenantId) + AuthAKVName: $(EsrpTestAuthAkvName) + AuthCertName: $(EsrpTestAuthCertName) + AuthSignCertName: $(EsrpTestAuthSignCertName) + FolderPath: $(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release + Pattern: '**/*.dll,**/*.exe' + UseMinimatch: true + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-466277", + "operationSetCode": "SigntoolSign", + "parameters": [ + { "parameterName": "OpusName", "parameterValue": "Microsoft" }, + { "parameterName": "OpusInfo", "parameterValue": "http://www.microsoft.com" }, + { "parameterName": "PageHash", "parameterValue": "/NPH" }, + { "parameterName": "FileDigest", "parameterValue": "/fd \"SHA256\"" }, + { "parameterName": "TimeStamp", "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" } + ], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-466277", + "operationSetCode": "SigntoolVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 60 + MaxConcurrency: 50 + MaxRetryAttempts: 5 + PendingAnalysisWaitTimeoutMinutes: 5 - pwsh: | <# The way the packaging works is a bit tricky as when it is built, we cannot add the modules that come from gallery. @@ -375,26 +425,61 @@ jobs: displayName: 'Pack Windows.x64 global tool' condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) - ## STUB-ONLY (nupkg): original was `task: onebranch.pipeline.signing@1` with - ## `cp_code: $(nuget_cert_id)` (PROD CP-401405, NuGet package signing). - ## This site stays STUB even in TEST MODE — no NuGet-compatible test KeyCode - ## is currently available in the MS test inventory. NuGet signing requires a - ## NuGet-specific cert (operationSetCode "NuGetSign"), and reusing the - ## Authenticode test codes (CP-466277 / CP-466279) would be rejected by ESRP - ## as "KeyCode not authorised for this operation". - ## To unblock: ask ESRP team for a NuGet test code, then add a sibling - ## EsrpCodeSigning@5 task here with `condition: ne(variables['EsrpTestConnectionName'], '')` - ## using operationSetCode "NuGetSign" / "NuGetVerify". - - pwsh: | - Write-Host '##[warning]STUB: would invoke ESRP signing with cp_code "$(nuget_cert_id)" (KeyCode CP-401405, NuGet)' - Write-Host ' files_to_sign: **\*.nupkg' - Write-Host ' search_root: $(ob_outputDirectory)\globaltool' - if (Test-Path '$(ob_outputDirectory)\globaltool') { - Get-ChildItem -Path '$(ob_outputDirectory)\globaltool' -Recurse -Filter '*.nupkg' | - Select-Object -ExpandProperty FullName | - ForEach-Object { Write-Host " Would sign: $_" } - } - displayName: 'Sign nupkg files (STUB — no NuGet test KeyCode available)' - condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + ## PROD SIGNING (nupkg): emitted ONLY for the Official pipeline + ## (OfficialBuild: true). Real ESRP NuGet signing via PowerShell-ESRP-Release + ## against PROD KeyCode CP-401405 (operationSetCode NuGetSign/NuGetVerify). + - ${{ if eq(parameters.OfficialBuild, true) }}: + - task: EsrpCodeSigning@5 + displayName: 'Sign nupkg files (ESRP prod signing)' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) + inputs: + ConnectedServiceName: PowerShell-ESRP-Release + AppRegistrationClientId: $(EsrpProdAppRegClientId) + AppRegistrationTenantId: $(EsrpProdAppRegTenantId) + AuthAKVName: $(EsrpProdAuthAkvName) + AuthCertName: $(EsrpProdAuthCertName) + AuthSignCertName: $(EsrpProdAuthSignCertName) + FolderPath: $(ob_outputDirectory)\globaltool + Pattern: '**/*.nupkg' + UseMinimatch: true + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetSign", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + }, + { + "keyCode": "CP-401405", + "operationSetCode": "NuGetVerify", + "parameters": [], + "toolName": "sign", + "toolVersion": "1.0" + } + ] + SessionTimeout: 60 + MaxConcurrency: 50 + MaxRetryAttempts: 5 + PendingAnalysisWaitTimeoutMinutes: 5 + + ## STUB (nupkg): emitted ONLY for the NonOfficial pipeline (OfficialBuild: + ## false). NuGet signing has no test KeyCode in the MS test inventory (the + ## Authenticode test codes CP-466277/CP-466279 would be rejected for the + ## NuGetSign operation), so this stays a Write-Host placeholder in NonOfficial. + - ${{ if ne(parameters.OfficialBuild, true) }}: + - pwsh: | + Write-Host '##[warning]STUB: would invoke ESRP signing with cp_code "$(nuget_cert_id)" (KeyCode CP-401405, NuGet)' + Write-Host ' files_to_sign: **\*.nupkg' + Write-Host ' search_root: $(ob_outputDirectory)\globaltool' + if (Test-Path '$(ob_outputDirectory)\globaltool') { + Get-ChildItem -Path '$(ob_outputDirectory)\globaltool' -Recurse -Filter '*.nupkg' | + Select-Object -ExpandProperty FullName | + ForEach-Object { Write-Host " Would sign: $_" } + } + displayName: 'Sign nupkg files (STUB — no NuGet test KeyCode available)' + condition: and(succeeded(), eq(variables['Architecture'], 'fxdependent')) - template: /.pipelines/templates/step/finalize.yml@self From ecfb4f0e8db56551c67b012471e0b22788a945ff Mon Sep 17 00:00:00 2001 From: Justin Chung Date: Tue, 7 Jul 2026 15:38:16 -0500 Subject: [PATCH 19/19] Wire Official 1ES ESRP signing to MI/WIF auth (UseMSIAuthentication) The 5 production EsrpCodeSigning@5 blocks referenced AuthCertName: $(EsrpProdAuthCertName) - a separate authentication cert that was never provisioned. Our ESRP setup uses the ESRP-Signing-Identity managed identity over the PowerShell-ESRP-Release WIF connection, with only the request-signing cert (esrp-auth-sign) in pscoresignprod-kv. Replace AuthCertName with UseMSIAuthentication: true across all Official signing sites (1P CP-230012, 3P CP-231522, NuGet CP-401405, Apple CP-401337-Apple) so the MI's AAD token proves identity to ESRP. Drop EsrpProdAuthCertName from the Official entry-file variable-group doc. Test/stub blocks (EsrpTest*) unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../1ES/PowerShell-Coordinated_Packages-Official.yml | 12 +++++++----- .pipelines/1ES/templates/mac.yml | 2 +- .pipelines/1ES/templates/obp-file-signing.yml | 4 ++-- .pipelines/1ES/templates/windows-hosted-build.yml | 4 ++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.pipelines/1ES/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/1ES/PowerShell-Coordinated_Packages-Official.yml index 5863d24185..63178a99fe 100644 --- a/.pipelines/1ES/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/1ES/PowerShell-Coordinated_Packages-Official.yml @@ -41,13 +41,15 @@ # # REQUIRED VARIABLE GROUP — `EsrpSigning-PowerShell`: # Create this variable group in mscodehub/PowerShellCore and link it to this pipeline -# definition. It supplies the ESRP request-signing auth inputs (non-secret identifiers) -# consumed by every prod EsrpCodeSigning@5 task: +# definition. It supplies the ESRP auth inputs (non-secret identifiers) consumed by every +# prod EsrpCodeSigning@5 task. Auth is via the ESRP-Signing-Identity managed identity over +# the PowerShell-ESRP-Release WIF connection (UseMSIAuthentication: true) — the MI's AAD +# token proves identity to ESRP, so NO separate authentication cert is needed; only the +# request-signing cert (AuthSignCertName) is required: # - EsrpProdAppRegClientId (client ID of the ESRP-Signing-Identity managed identity) # - EsrpProdAppRegTenantId (AME tenant ID the MI lives in) -# - EsrpProdAuthAkvName (Key Vault holding the ESRP request-signing cert, e.g. pscoresignprod-kv) -# - EsrpProdAuthCertName (auth cert name in that Key Vault) -# - EsrpProdAuthSignCertName (request-signing cert name in that Key Vault) +# - EsrpProdAuthAkvName (Key Vault holding the request-signing cert: pscoresignprod-kv) +# - EsrpProdAuthSignCertName (request-signing cert name in that Key Vault: esrp-auth-sign) # The MI must have `Key Vault Certificate User` on the vault and be onboarded to ESRP # with the KeyCodes above authorised. diff --git a/.pipelines/1ES/templates/mac.yml b/.pipelines/1ES/templates/mac.yml index df73c1bce6..649456f4db 100644 --- a/.pipelines/1ES/templates/mac.yml +++ b/.pipelines/1ES/templates/mac.yml @@ -224,8 +224,8 @@ jobs: AppRegistrationClientId: $(EsrpProdAppRegClientId) AppRegistrationTenantId: $(EsrpProdAppRegTenantId) AuthAKVName: $(EsrpProdAuthAkvName) - AuthCertName: $(EsrpProdAuthCertName) AuthSignCertName: $(EsrpProdAuthSignCertName) + UseMSIAuthentication: true FolderPath: $(Pipeline.Workspace) Pattern: 'macho-$(BuildArchitecture).zip' UseMinimatch: true diff --git a/.pipelines/1ES/templates/obp-file-signing.yml b/.pipelines/1ES/templates/obp-file-signing.yml index 37f5b078da..d5f8d84fa6 100644 --- a/.pipelines/1ES/templates/obp-file-signing.yml +++ b/.pipelines/1ES/templates/obp-file-signing.yml @@ -144,8 +144,8 @@ steps: AppRegistrationClientId: $(EsrpProdAppRegClientId) AppRegistrationTenantId: $(EsrpProdAppRegTenantId) AuthAKVName: $(EsrpProdAuthAkvName) - AuthCertName: $(EsrpProdAuthCertName) AuthSignCertName: $(EsrpProdAuthSignCertName) + UseMSIAuthentication: true FolderPath: $(Pipeline.Workspace)/toBeSigned Pattern: '**/*.psd1,**/*.psm1,**/*.ps1xml,**/*.ps1,**/*.dll,**/*.exe,**/pwsh' UseMinimatch: true @@ -301,8 +301,8 @@ steps: AppRegistrationClientId: $(EsrpProdAppRegClientId) AppRegistrationTenantId: $(EsrpProdAppRegTenantId) AuthAKVName: $(EsrpProdAuthAkvName) - AuthCertName: $(EsrpProdAuthCertName) AuthSignCertName: $(EsrpProdAuthSignCertName) + UseMSIAuthentication: true FolderPath: $(Pipeline.Workspace)/thirdPartyToBeSigned Pattern: '**/*.dll,**/*.exe' UseMinimatch: true diff --git a/.pipelines/1ES/templates/windows-hosted-build.yml b/.pipelines/1ES/templates/windows-hosted-build.yml index 6a58e4853f..2ef1697b17 100644 --- a/.pipelines/1ES/templates/windows-hosted-build.yml +++ b/.pipelines/1ES/templates/windows-hosted-build.yml @@ -241,8 +241,8 @@ jobs: AppRegistrationClientId: $(EsrpProdAppRegClientId) AppRegistrationTenantId: $(EsrpProdAppRegTenantId) AuthAKVName: $(EsrpProdAuthAkvName) - AuthCertName: $(EsrpProdAuthCertName) AuthSignCertName: $(EsrpProdAuthSignCertName) + UseMSIAuthentication: true FolderPath: $(GlobalToolArtifactPath)/obj/PowerShell.Windows.x64/release Pattern: '**/*.dll,**/*.exe' UseMinimatch: true @@ -437,8 +437,8 @@ jobs: AppRegistrationClientId: $(EsrpProdAppRegClientId) AppRegistrationTenantId: $(EsrpProdAppRegTenantId) AuthAKVName: $(EsrpProdAuthAkvName) - AuthCertName: $(EsrpProdAuthCertName) AuthSignCertName: $(EsrpProdAuthSignCertName) + UseMSIAuthentication: true FolderPath: $(ob_outputDirectory)\globaltool Pattern: '**/*.nupkg' UseMinimatch: true