From 51000a426905473085397855dfec1268e7ab5658 Mon Sep 17 00:00:00 2001 From: Justin Chung <124807742+jshigetomi@users.noreply.github.com> Date: Tue, 24 Mar 2026 15:37:51 -0500 Subject: [PATCH] Separate Official and NonOfficial templates for ADO pipelines (#26897) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/agents/SplitADOPipelines.agent.md | 164 ++++++++ ...Shell-Coordinated_Packages-NonOfficial.yml | 97 +++++ .../PowerShell-Packages-NonOfficial.yml | 97 +++++ .../PowerShell-Release-Azure-NonOfficial.yml | 76 ++++ .../PowerShell-Release-NonOfficial.yml | 106 ++++++ .../PowerShell-vPack-NonOfficial.yml | 88 +++++ ...werShell-Coordinated_Packages-Official.yml | 247 +----------- .pipelines/PowerShell-Packages-Official.yml | 227 +---------- .../PowerShell-Release-Official-Azure.yml | 34 +- .pipelines/PowerShell-Release-Official.yml | 358 +----------------- .pipelines/PowerShell-vPack-Official.yml | 268 +------------ .pipelines/templates/release-MSIX-Publish.yml | 2 +- .pipelines/templates/release-githubNuget.yml | 4 +- ...PowerShell-Coordinated_Packages-Stages.yml | 202 ++++++++++ .../stages/PowerShell-Packages-Stages.yml | 186 +++++++++ .../stages/PowerShell-Release-Stages.yml | 323 ++++++++++++++++ .../stages/PowerShell-vPack-Stages.yml | 236 ++++++++++++ ...erShell-Coordinated_Packages-Variables.yml | 67 ++++ .../PowerShell-Packages-Variables.yml | 50 +++ .../PowerShell-Release-Azure-Variables.yml | 35 ++ .../PowerShell-Release-Variables.yml | 41 ++ .../variables/PowerShell-vPack-Variables.yml | 39 ++ .../release-shared.yml | 0 23 files changed, 1858 insertions(+), 1089 deletions(-) create mode 100644 .github/agents/SplitADOPipelines.agent.md create mode 100644 .pipelines/NonOfficial/PowerShell-Coordinated_Packages-NonOfficial.yml create mode 100644 .pipelines/NonOfficial/PowerShell-Packages-NonOfficial.yml create mode 100644 .pipelines/NonOfficial/PowerShell-Release-Azure-NonOfficial.yml create mode 100644 .pipelines/NonOfficial/PowerShell-Release-NonOfficial.yml create mode 100644 .pipelines/NonOfficial/PowerShell-vPack-NonOfficial.yml create mode 100644 .pipelines/templates/stages/PowerShell-Coordinated_Packages-Stages.yml create mode 100644 .pipelines/templates/stages/PowerShell-Packages-Stages.yml create mode 100644 .pipelines/templates/stages/PowerShell-Release-Stages.yml create mode 100644 .pipelines/templates/stages/PowerShell-vPack-Stages.yml create mode 100644 .pipelines/templates/variables/PowerShell-Coordinated_Packages-Variables.yml create mode 100644 .pipelines/templates/variables/PowerShell-Packages-Variables.yml create mode 100644 .pipelines/templates/variables/PowerShell-Release-Azure-Variables.yml create mode 100644 .pipelines/templates/variables/PowerShell-Release-Variables.yml create mode 100644 .pipelines/templates/variables/PowerShell-vPack-Variables.yml rename .pipelines/templates/{variable => variables}/release-shared.yml (100%) diff --git a/.github/agents/SplitADOPipelines.agent.md b/.github/agents/SplitADOPipelines.agent.md new file mode 100644 index 00000000000..8322f473e7b --- /dev/null +++ b/.github/agents/SplitADOPipelines.agent.md @@ -0,0 +1,164 @@ +--- +name: SplitADOPipelines +description: This agent will implement and restructure the repository's existing ADO pipelines into Official and NonOfficial pipelines. +tools: ['vscode', 'execute', 'read', 'agent', 'edit', 'search', 'todo'] +--- + +This agent will implement and restructure the repository's existing ADO pipelines into Official and NonOfficial pipelines. + +A repository will have under the ./pipelines directory a series of yaml files that define the ADO pipelines for the repository. + +First confirm if the pipelines are using a toggle switch for Official and NonOfficial. This will look something like this + +```yaml +parameters: + - name: templateFile + value: ${{ iif ( parameters.OfficialBuild, 'v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates', 'v2/OneBranch.NonOfficial.CrossPlat.yml@onebranchTemplates' ) }} +``` + +Followed by: + +```yaml +extends: + template: ${{ variables.templateFile }} +``` + +This is an indicator that this work needs to be done. This toggle switch is no longer allowed and the templates need to be hard coded. + +## Refactoring Steps + +### Step 1: Extract Shared Templates + +For each pipeline file that uses the toggle switch pattern (e.g., `PowerShell-Packages.yml`): + +1. Create a `./pipelines/templates` directory if it doesn't exist +2. Extract the **variables section** into `./pipelines/templates/PowerShell-Packages-Variables.yml` +3. Extract the **stages section** into `./pipelines/templates/PowerShell-Packages-Stages.yml` + +**IMPORTANT**: Only extract the `variables:` and `stages:` sections. All other sections (parameters, resources, extends, etc.) remain in the pipeline files. + +### Step 2: Create Official Pipeline (In-Place Refactoring) + +The original toggle-based file becomes the Official pipeline: + +1. **Keep the file in its original location** (e.g., `./pipelines/PowerShell-Packages.yml` stays where it is) +2. Remove the toggle switch parameter (`templateFile` parameter) +3. Hard-code the Official template reference: + ```yaml + extends: + template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates + ``` +4. Replace the `variables:` section with a template reference: + ```yaml + variables: + - template: templates/PowerShell-Packages-Variables.yml + ``` +5. Replace the `stages:` section with a template reference: + ```yaml + stages: + - template: templates/PowerShell-Packages-Stages.yml + ``` + +### Step 3: Create NonOfficial Pipeline + +1. Create `./pipelines/NonOfficial` directory if it doesn't exist +2. Create the NonOfficial pipeline file (e.g., `./pipelines/NonOfficial/PowerShell-Packages-NonOfficial.yml`) +3. Copy the structure from the refactored Official pipeline +4. Hard-code the NonOfficial template reference: + ```yaml + extends: + template: v2/OneBranch.NonOfficial.CrossPlat.yml@onebranchTemplates + ``` +5. Reference the same shared templates: + ```yaml + variables: + - template: ../templates/PowerShell-Packages-Variables.yml + + stages: + - template: ../templates/PowerShell-Packages-Stages.yml + ``` + +**Note**: The NonOfficial pipeline uses `../templates/` because it's one directory deeper than the Official pipeline. + +### Step 4: Link NonOfficial Pipelines to NonOfficial Dependencies + +After creating NonOfficial pipelines, ensure they consume artifacts from other **NonOfficial** pipelines, not Official ones. + +1. **Check the `resources:` section** in each NonOfficial pipeline for `pipelines:` dependencies +2. **Identify Official pipeline references** that need to be changed to NonOfficial +3. **Update the `source:` field** to point to the NonOfficial version + +**Example Problem:** NonOfficial pipeline pointing to Official dependency +```yaml +resources: + pipelines: + - pipeline: CoOrdinatedBuildPipeline + source: 'PowerShell-Coordinated Binaries-Official' # ❌ Wrong - Official! +``` + +**Solution:** Update to NonOfficial dependency +```yaml +resources: + pipelines: + - pipeline: CoOrdinatedBuildPipeline + source: 'PowerShell-Coordinated Binaries-NonOfficial' # ✅ Correct - NonOfficial! +``` + +**IMPORTANT**: The `source:` field must match the **exact ADO pipeline definition name** as it appears in Azure DevOps, not necessarily the file name. + +### Step 5: Configure Release Environment Parameters (NonAzure Only) + +**This step only applies if the pipeline uses `category: NonAzure` in the release configuration.** + +If you detect this pattern in the original pipeline: + +```yaml +extends: + template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates # or NonOfficial + parameters: + release: + category: NonAzure +``` + +Then you must configure the `ob_release_environment` parameter when referencing the stages template. + +#### Official Pipeline Configuration + +In the Official pipeline (e.g., `./pipelines/PowerShell-Packages.yml`): + +```yaml +stages: + - template: templates/PowerShell-Packages-Stages.yml + parameters: + ob_release_environment: Production +``` + +#### NonOfficial Pipeline Configuration + +In the NonOfficial pipeline (e.g., `./pipelines/NonOfficial/PowerShell-Packages-NonOfficial.yml`): + +```yaml +stages: + - template: ../templates/PowerShell-Packages-Stages.yml + parameters: + ob_release_environment: Test +``` + +#### Update Stages Template to Accept Parameter + +The extracted stages template (e.g., `./pipelines/templates/PowerShell-Packages-Stages.yml`) must declare the parameter at the top: + +```yaml +parameters: + - name: ob_release_environment + type: string + +stages: + # ... rest of stages configuration using ${{ parameters.ob_release_environment }} +``` + +**IMPORTANT**: +- Only configure this for pipelines with `category: NonAzure` +- Official pipelines always use `ob_release_environment: Production` +- NonOfficial pipelines always use `ob_release_environment: Test` +- The stages template must accept this parameter and use it in the appropriate stage configurations diff --git a/.pipelines/NonOfficial/PowerShell-Coordinated_Packages-NonOfficial.yml b/.pipelines/NonOfficial/PowerShell-Coordinated_Packages-NonOfficial.yml new file mode 100644 index 00000000000..55d4c4557d8 --- /dev/null +++ b/.pipelines/NonOfficial/PowerShell-Coordinated_Packages-NonOfficial.yml @@ -0,0 +1,97 @@ +trigger: none + +parameters: + - name: InternalSDKBlobURL + displayName: URL to the blob having internal .NET SDK + type: string + default: ' ' + - name: ReleaseTagVar + displayName: Release Tag + type: string + default: 'fromBranch' + - name: SKIP_SIGNING + displayName: Debugging - Skip Signing + type: string + default: 'NO' + - name: RUN_TEST_AND_RELEASE + displayName: Debugging - Run Test and Release Artifacts Stage + type: boolean + default: true + - name: RUN_WINDOWS + displayName: Debugging - Enable Windows Stage + type: boolean + default: true + - name: ENABLE_MSBUILD_BINLOGS + displayName: Debugging - Enable MSBuild Binary Logs + type: boolean + default: false + - name: FORCE_CODEQL + displayName: Debugging - Enable CodeQL and set cadence to 1 hour + type: boolean + default: false + +name: bins-$(BUILD.SOURCEBRANCHNAME)-nonofficial-$(Build.BuildId) + +resources: + repositories: + - repository: ComplianceRepo + type: github + endpoint: ComplianceGHRepo + name: PowerShell/compliance + ref: master + - repository: onebranchTemplates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +variables: + - template: ../templates/variables/PowerShell-Coordinated_Packages-Variables.yml + parameters: + InternalSDKBlobURL: ${{ parameters.InternalSDKBlobURL }} + ReleaseTagVar: ${{ parameters.ReleaseTagVar }} + SKIP_SIGNING: ${{ parameters.SKIP_SIGNING }} + ENABLE_MSBUILD_BINLOGS: ${{ parameters.ENABLE_MSBUILD_BINLOGS }} + FORCE_CODEQL: ${{ parameters.FORCE_CODEQL }} + +extends: + template: v2/OneBranch.NonOfficial.CrossPlat.yml@onebranchTemplates + parameters: + customTags: 'ES365AIMigrationTooling' + featureFlags: + LinuxHostVersion: + Network: KS3 + WindowsHostVersion: + Network: KS3 + incrementalSDLBinaryAnalysis: true + globalSdl: + disableLegacyManifest: true + # disabled Armorty as we dont have any ARM templates to scan. It fails on some sample ARM templates. + armory: + enabled: false + sbom: + enabled: true + codeql: + compiled: + enabled: $(CODEQL_ENABLED) + tsaEnabled: true # This enables TSA bug filing only for CodeQL 3000 + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + cg: + enabled: true + ignoreDirectories: '.devcontainer,demos,docker,docs,src,test,tools/packaging' + binskim: + enabled: false + exactToolVersion: 4.4.2 + # APIScan requires a non-Ready-To-Run build + apiscan: + enabled: false + tsaOptionsFile: .config\tsaoptions.json + + stages: + - template: ../templates/stages/PowerShell-Coordinated_Packages-Stages.yml + parameters: + RUN_WINDOWS: ${{ parameters.RUN_WINDOWS }} + RUN_TEST_AND_RELEASE: ${{ parameters.RUN_TEST_AND_RELEASE }} + OfficialBuild: false diff --git a/.pipelines/NonOfficial/PowerShell-Packages-NonOfficial.yml b/.pipelines/NonOfficial/PowerShell-Packages-NonOfficial.yml new file mode 100644 index 00000000000..81f343a04a0 --- /dev/null +++ b/.pipelines/NonOfficial/PowerShell-Packages-NonOfficial.yml @@ -0,0 +1,97 @@ +trigger: none + +parameters: # parameters are shown up in ADO UI in a build queue time + - name: ForceAzureBlobDelete + displayName: Delete Azure Blob + type: string + values: + - true + - false + default: false + - name: 'debug' + displayName: 'Enable debug output' + type: boolean + default: false + - name: InternalSDKBlobURL + displayName: URL to the blob having internal .NET SDK + type: string + default: ' ' + - name: ReleaseTagVar + displayName: Release Tag + type: string + default: 'fromBranch' + - name: SKIP_SIGNING + displayName: Skip Signing + type: string + default: 'NO' + - name: disableNetworkIsolation + type: boolean + default: false + +name: pkgs-$(BUILD.SOURCEBRANCHNAME)-nonofficial-$(Build.BuildId) + +variables: + - template: ../templates/variables/PowerShell-Packages-Variables.yml + parameters: + debug: ${{ parameters.debug }} + ForceAzureBlobDelete: ${{ parameters.ForceAzureBlobDelete }} + ReleaseTagVar: ${{ parameters.ReleaseTagVar }} + disableNetworkIsolation: ${{ parameters.disableNetworkIsolation }} + +resources: + pipelines: + - pipeline: CoOrdinatedBuildPipeline + source: 'PowerShell-Coordinated_Packages-NonOfficial' + trigger: + branches: + include: + - master + - releases/* + + repositories: + - repository: onebranchTemplates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +extends: + template: v2/OneBranch.NonOfficial.CrossPlat.yml@onebranchTemplates + parameters: + cloudvault: + enabled: false + featureFlags: + WindowsHostVersion: + Version: 2022 + Network: KS3 + LinuxHostVersion: + Network: KS3 + linuxEsrpSigning: true + incrementalSDLBinaryAnalysis: true + disableNetworkIsolation: ${{ variables.disableNetworkIsolation }} + globalSdl: + disableLegacyManifest: true + # disabled Armorty as we dont have any ARM templates to scan. It fails on some sample ARM templates. + armory: + enabled: false + sbom: + enabled: true + compiled: + enabled: false + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + cg: + enabled: true + ignoreDirectories: '.devcontainer,demos,docker,docs,src,test,tools/packaging' + binskim: + enabled: false + exactToolVersion: 4.4.2 + # APIScan requires a non-Ready-To-Run build + apiscan: + enabled: false + tsaOptionsFile: .config\tsaoptions.json + stages: + - template: ../templates/stages/PowerShell-Packages-Stages.yml + parameters: + OfficialBuild: false diff --git a/.pipelines/NonOfficial/PowerShell-Release-Azure-NonOfficial.yml b/.pipelines/NonOfficial/PowerShell-Release-Azure-NonOfficial.yml new file mode 100644 index 00000000000..681babb2220 --- /dev/null +++ b/.pipelines/NonOfficial/PowerShell-Release-Azure-NonOfficial.yml @@ -0,0 +1,76 @@ +trigger: none + +parameters: # parameters are shown up in ADO UI in a build queue time + - name: 'debug' + displayName: 'Enable debug output' + type: boolean + default: false + - name: skipPublish + displayName: Skip PMC Publish + type: boolean + default: false + - name: SKIP_SIGNING + displayName: Skip Signing + type: string + default: 'NO' + +name: ev2-$(BUILD.SOURCEBRANCHNAME)-nonofficial-$(Build.BuildId) + +variables: + - template: ../templates/variables/PowerShell-Release-Azure-Variables.yml + parameters: + debug: ${{ parameters.debug }} + +resources: + repositories: + - repository: onebranchTemplates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + + pipelines: + - pipeline: CoOrdinatedBuildPipeline + source: 'PowerShell-Coordinated_Packages-NonOfficial' + + - pipeline: PSPackagesOfficial + source: 'PowerShell-Packages-NonOfficial' + trigger: + branches: + include: + - master + - releases/* + +extends: + template: v2/OneBranch.NonOfficial.CrossPlat.yml@onebranchTemplates + parameters: + featureFlags: + WindowsHostVersion: + Version: 2022 + Network: Netlock + linuxEsrpSigning: true + incrementalSDLBinaryAnalysis: true + cloudvault: + enabled: false + globalSdl: + disableLegacyManifest: true + # disabled Armory as we dont have any ARM templates to scan. It fails on some sample ARM templates. + armory: + enabled: false + tsa: + enabled: true + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + binskim: + break: false # always break the build on binskim issues in addition to TSA upload + exactToolVersion: 4.4.2 + policheck: + break: true # always break the build on policheck issues. You can disable it by setting to 'false' + tsaOptionsFile: .config\tsaoptions.json + stages: + - template: /.pipelines/templates/release-prep-for-ev2.yml@self + parameters: + skipPublish: ${{ parameters.skipPublish }} + + - template: /.pipelines/templates/release-publish-pmc.yml@self diff --git a/.pipelines/NonOfficial/PowerShell-Release-NonOfficial.yml b/.pipelines/NonOfficial/PowerShell-Release-NonOfficial.yml new file mode 100644 index 00000000000..ca5a6383f33 --- /dev/null +++ b/.pipelines/NonOfficial/PowerShell-Release-NonOfficial.yml @@ -0,0 +1,106 @@ +trigger: none + +parameters: # parameters are shown up in ADO UI in a build queue time + - name: 'debug' + displayName: 'Enable debug output' + type: boolean + default: false + - name: InternalSDKBlobURL + displayName: URL to the blob having internal .NET SDK + type: string + default: ' ' + - name: ReleaseTagVar + displayName: Release Tag + type: string + default: 'fromBranch' + - name: SKIP_SIGNING + displayName: Skip Signing + type: string + default: 'NO' + - name: SkipPublish + displayName: Skip Publishing to Nuget + type: boolean + default: false + - name: SkipPSInfraInstallers + displayName: Skip Copying Archives and Installers to PSInfrastructure Public Location + type: boolean + default: false + - name: skipMSIXPublish + displayName: Skip MSIX Publish + type: boolean + default: false + +name: release-$(BUILD.SOURCEBRANCHNAME)-nonofficial-$(Build.BuildId) + +variables: + - template: ../templates/variables/PowerShell-Release-Variables.yml + parameters: + debug: ${{ parameters.debug }} + ReleaseTagVar: ${{ parameters.ReleaseTagVar }} + +resources: + repositories: + - repository: onebranchTemplates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + - repository: PSInternalTools + type: git + name: PowerShellCore/Internal-PowerShellTeam-Tools + ref: refs/heads/master + + pipelines: + - pipeline: CoOrdinatedBuildPipeline + source: 'PowerShell-Coordinated_Packages-NonOfficial' + + # NOTE: The alias name "PSPackagesOfficial" is intentionally reused here even + # for the NonOfficial pipeline source. Downstream shared templates (for example, + # release-validate-sdk.yml and release-upload-buildinfo.yml) reference artifacts + # using `download: PSPackagesOfficial`, so changing this alias would break them. + - pipeline: PSPackagesOfficial + source: 'PowerShell-Packages-NonOfficial' + trigger: + branches: + include: + - master + - releases/* + +extends: + template: v2/OneBranch.NonOfficial.CrossPlat.yml@onebranchTemplates + parameters: + release: + category: NonAzure + featureFlags: + WindowsHostVersion: + Version: 2022 + Network: KS3 + incrementalSDLBinaryAnalysis: true + cloudvault: + enabled: false + globalSdl: + disableLegacyManifest: true + # disabled Armory as we dont have any ARM templates to scan. It fails on some sample ARM templates. + armory: + enabled: false + tsa: + enabled: true + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + binskim: + break: false # always break the build on binskim issues in addition to TSA upload + exactToolVersion: 4.4.2 + policheck: + break: true # always break the build on policheck issues. You can disable it by setting to 'false' + # suppression: + # suppressionFile: $(Build.SourcesDirectory)\.gdn\global.gdnsuppress + tsaOptionsFile: .config\tsaoptions.json + + stages: + - template: ../templates/stages/PowerShell-Release-Stages.yml + parameters: + releaseEnvironment: Test + SkipPublish: ${{ parameters.SkipPublish }} + SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} + skipMSIXPublish: ${{ parameters.skipMSIXPublish }} diff --git a/.pipelines/NonOfficial/PowerShell-vPack-NonOfficial.yml b/.pipelines/NonOfficial/PowerShell-vPack-NonOfficial.yml new file mode 100644 index 00000000000..642b169adaf --- /dev/null +++ b/.pipelines/NonOfficial/PowerShell-vPack-NonOfficial.yml @@ -0,0 +1,88 @@ +trigger: none + +parameters: # parameters are shown up in ADO UI in a build queue time +- name: 'createVPack' + displayName: 'Create and Submit VPack' + type: boolean + default: true +- name: vPackName + type: string + displayName: 'VPack Name:' + default: 'PowerShell.BuildTool' + values: + - PowerShell.BuildTool + - PowerShell + - PowerShellDoNotUse +- name: 'ReleaseTagVar' + type: string + displayName: 'Release Tag Var:' + default: 'fromBranch' +- name: 'debug' + displayName: 'Enable debug output' + type: boolean + default: false +- name: netiso + displayName: "Network Isolation Policy" + type: string + values: + - KS4 + - R1 + - Netlock + default: "R1" + +name: vPack_$(Build.SourceBranchName)_NonOfficial_Create.${{ parameters.createVPack }}_Name.${{ parameters.vPackName}}_$(date:yyyyMMdd).$(rev:rr) + +variables: + - template: ../templates/variables/PowerShell-vPack-Variables.yml + parameters: + debug: ${{ parameters.debug }} + ReleaseTagVar: ${{ parameters.ReleaseTagVar }} + netiso: ${{ parameters.netiso }} + +resources: + repositories: + - repository: onebranchTemplates + type: git + name: OneBranch.Pipelines/GovernedTemplates + ref: refs/heads/main + +extends: + template: v2/Microsoft.NonOfficial.yml@onebranchTemplates + parameters: + platform: + name: 'windows_undocked' # windows undocked + + featureFlags: + WindowsHostVersion: + Version: 2022 + Network: ${{ variables.netiso }} + + cloudvault: + enabled: false + + globalSdl: + useCustomPolicy: true # for signing code + disableLegacyManifest: true + # disabled Armory as we dont have any ARM templates to scan. It fails on some sample ARM templates. + armory: + enabled: false + sbom: + enabled: true + compiled: + enabled: false + credscan: + enabled: true + scanFolder: $(Build.SourcesDirectory) + suppressionsFile: $(Build.SourcesDirectory)\.config\suppress.json + binskim: + enabled: false + exactToolVersion: 4.4.2 + # APIScan requires a non-Ready-To-Run build + apiscan: + enabled: false + tsaOptionsFile: .config/tsaoptions.json + stages: + - template: ../templates/stages/PowerShell-vPack-Stages.yml + parameters: + createVPack: ${{ parameters.createVPack }} + vPackName: ${{ parameters.vPackName }} diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index 12460e2861c..82f129a0a5e 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -30,7 +30,7 @@ parameters: type: boolean default: false -name: bins-$(BUILD.SOURCEBRANCHNAME)-prod.true-$(Build.BuildId) +name: bins-$(BUILD.SOURCEBRANCHNAME)-prod-$(Build.BuildId) resources: repositories: @@ -45,57 +45,13 @@ resources: ref: refs/heads/main variables: - - name: PS_RELEASE_BUILD - value: 1 - - name: DOTNET_CLI_TELEMETRY_OPTOUT - value: 1 - - name: POWERSHELL_TELEMETRY_OPTOUT - value: 1 - - name: nugetMultiFeedWarnLevel - value: none - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: skipNugetSecurityAnalysis - value: true - - name: branchCounterKey - value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])] - - name: branchCounter - value: $[counter(variables['branchCounterKey'], 1)] - - name: BUILDSECMON_OPT_IN - value: true - - name: __DOTNET_RUNTIME_FEED - value: ${{ parameters.InternalSDKBlobURL }} - - name: LinuxContainerImage - value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 - - name: WindowsContainerImage - value: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest - - name: CDP_DEFINITION_BUILD_COUNT - value: $[counter('', 0)] - - name: ReleaseTagVar - value: ${{ parameters.ReleaseTagVar }} - - name: SKIP_SIGNING - value: ${{ parameters.SKIP_SIGNING }} - - group: mscodehub-feed-read-general - - group: mscodehub-feed-read-akv - - name: ENABLE_MSBUILD_BINLOGS - value: ${{ parameters.ENABLE_MSBUILD_BINLOGS }} - - ${{ if eq(parameters['FORCE_CODEQL'],'true') }}: - # Cadence is hours before CodeQL will allow a re-upload of the database - - name: CodeQL.Cadence - value: 1 - - name: CODEQL_ENABLED - ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), eq(parameters['FORCE_CODEQL'],'true')) }}: - value: true - ${{ else }}: - value: false - # Fix for BinSkim ICU package error in Linux containers - - name: DOTNET_SYSTEM_GLOBALIZATION_INVARIANT - value: true - # Disable BinSkim at job level to override NonOfficial template defaults - - name: ob_sdl_binskim_enabled - value: false - - name: ps_official_build - value: true + - template: templates/variables/PowerShell-Coordinated_Packages-Variables.yml + parameters: + InternalSDKBlobURL: ${{ parameters.InternalSDKBlobURL }} + ReleaseTagVar: ${{ parameters.ReleaseTagVar }} + SKIP_SIGNING: ${{ parameters.SKIP_SIGNING }} + ENABLE_MSBUILD_BINLOGS: ${{ parameters.ENABLE_MSBUILD_BINLOGS }} + FORCE_CODEQL: ${{ parameters.FORCE_CODEQL }} extends: template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates @@ -135,185 +91,8 @@ extends: tsaOptionsFile: .config\tsaoptions.json stages: - - stage: prep - jobs: - - job: SetVars - displayName: Set Variables - pool: - type: linux - - variables: - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT/BuildJson' - - name: ob_sdl_codeSignValidation_enabled - value: false - - name: ob_sdl_codeql_compiled_enabled - value: false - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - - name: ob_signing_setup_enabled - value: false - - name: ob_sdl_sbom_enabled - value: false - - steps: - - checkout: self - clean: true - env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - - - pwsh: | - Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose - displayName: Capture environment variables - env: - ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase - - - template: /.pipelines/templates/SetVersionVariables.yml@self - parameters: - ReleaseTagVar: $(ReleaseTagVar) - CreateJson: yes - - - stage: macos - displayName: macOS - build and sign - dependsOn: ['prep'] - jobs: - - template: /.pipelines/templates/mac.yml@self - parameters: - buildArchitecture: x64 - - template: /.pipelines/templates/mac.yml@self - parameters: - buildArchitecture: arm64 - - - stage: linux - displayName: linux - build and sign - dependsOn: ['prep'] - jobs: - - template: /.pipelines/templates/linux.yml@self - parameters: - Runtime: 'linux-x64' - JobName: 'linux_x64' - - - template: /.pipelines/templates/linux.yml@self - parameters: - Runtime: 'linux-x64' - JobName: 'linux_x64_minSize' - BuildConfiguration: 'minSize' - - - template: /.pipelines/templates/linux.yml@self - parameters: - Runtime: 'linux-arm' - JobName: 'linux_arm' - - - template: /.pipelines/templates/linux.yml@self - parameters: - Runtime: 'linux-arm64' - JobName: 'linux_arm64' - - - template: /.pipelines/templates/linux.yml@self - parameters: - Runtime: 'fxdependent-linux-x64' - JobName: 'linux_fxd_x64_mariner' - - - template: /.pipelines/templates/linux.yml@self - parameters: - Runtime: 'fxdependent-linux-arm64' - JobName: 'linux_fxd_arm64_mariner' - - - template: /.pipelines/templates/linux.yml@self - parameters: - Runtime: 'fxdependent-noopt-linux-musl-x64' - JobName: 'linux_fxd_x64_alpine' - - - template: /.pipelines/templates/linux.yml@self - parameters: - Runtime: 'fxdependent' - JobName: 'linux_fxd' - - - template: /.pipelines/templates/linux.yml@self - parameters: - Runtime: 'linux-musl-x64' - JobName: 'linux_x64_alpine' - - - stage: windows - displayName: windows - build and sign - dependsOn: ['prep'] - condition: and(succeeded(),eq('${{ parameters.RUN_WINDOWS }}','true')) - jobs: - - template: /.pipelines/templates/windows-hosted-build.yml@self - parameters: - Architecture: x64 - BuildConfiguration: release - JobName: build_windows_x64_release - - template: /.pipelines/templates/windows-hosted-build.yml@self - parameters: - Architecture: x64 - BuildConfiguration: minSize - JobName: build_windows_x64_minSize_release - - template: /.pipelines/templates/windows-hosted-build.yml@self - parameters: - Architecture: x86 - JobName: build_windows_x86_release - - template: /.pipelines/templates/windows-hosted-build.yml@self - parameters: - Architecture: arm64 - JobName: build_windows_arm64_release - - template: /.pipelines/templates/windows-hosted-build.yml@self - parameters: - Architecture: fxdependent - JobName: build_windows_fxdependent_release - - template: /.pipelines/templates/windows-hosted-build.yml@self - parameters: - Architecture: fxdependentWinDesktop - JobName: build_windows_fxdependentWinDesktop_release - - - stage: test_and_release_artifacts - displayName: Test and Release Artifacts - dependsOn: ['prep'] - condition: and(succeeded(),eq('${{ parameters.RUN_TEST_AND_RELEASE }}','true')) - jobs: - - template: /.pipelines/templates/testartifacts.yml@self - - - job: release_json - displayName: Create and Upload release.json - pool: - type: windows - variables: - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json - - name: ob_sdl_credscan_suppressionsFile - value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json - steps: - - checkout: self - clean: true - - template: /.pipelines/templates/SetVersionVariables.yml@self - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - template: /.pipelines/templates/rebuild-branch-check.yml@self - - powershell: | - $metadata = Get-Content '$(Build.SourcesDirectory)/PowerShell/tools/metadata.json' -Raw | ConvertFrom-Json - - # Use the rebuild branch check from the template - $isRebuildBranch = '$(RebuildBranchCheck.IsRebuildBranch)' -eq 'true' - - # Don't mark as LTS release for rebuild branches - $LTS = $metadata.LTSRelease.Package -and -not $isRebuildBranch - - if ($isRebuildBranch) { - Write-Verbose -Message "Rebuild branch detected, not marking as LTS release" -Verbose - } - - @{ ReleaseVersion = "$(Version)"; LTSRelease = $LTS } | ConvertTo-Json | Out-File "$(Build.StagingDirectory)\release.json" - Get-Content "$(Build.StagingDirectory)\release.json" - - if (-not (Test-Path "$(ob_outputDirectory)\metadata")) { - New-Item -ItemType Directory -Path "$(ob_outputDirectory)\metadata" - } - - Copy-Item -Path "$(Build.StagingDirectory)\release.json" -Destination "$(ob_outputDirectory)\metadata" -Force - displayName: Create and upload release.json file to build artifact - retryCountOnTaskFailure: 2 - - template: /.pipelines/templates/step/finalize.yml@self + - template: templates/stages/PowerShell-Coordinated_Packages-Stages.yml + parameters: + RUN_WINDOWS: ${{ parameters.RUN_WINDOWS }} + RUN_TEST_AND_RELEASE: ${{ parameters.RUN_TEST_AND_RELEASE }} + OfficialBuild: true diff --git a/.pipelines/PowerShell-Packages-Official.yml b/.pipelines/PowerShell-Packages-Official.yml index a13ef12378a..8afce29ede7 100644 --- a/.pipelines/PowerShell-Packages-Official.yml +++ b/.pipelines/PowerShell-Packages-Official.yml @@ -28,44 +28,15 @@ parameters: # parameters are shown up in ADO UI in a build queue time type: boolean default: false -name: pkgs-$(BUILD.SOURCEBRANCHNAME)-prod.true-$(Build.BuildId) +name: pkgs-$(BUILD.SOURCEBRANCHNAME)-prod-$(Build.BuildId) variables: - - name: CDP_DEFINITION_BUILD_COUNT - value: $[counter('', 0)] # needed for onebranch.pipeline.version task - - name: system.debug - value: ${{ parameters.debug }} - - name: ENABLE_PRS_DELAYSIGN - value: 1 - - name: ROOT - value: $(Build.SourcesDirectory) - - name: ForceAzureBlobDelete - value: ${{ parameters.ForceAzureBlobDelete }} - - name: NUGET_XMLDOC_MODE - value: none - - name: nugetMultiFeedWarnLevel - value: none - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: skipNugetSecurityAnalysis - value: true - - name: ReleaseTagVar - value: ${{ parameters.ReleaseTagVar }} - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: WindowsContainerImage - value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' # Docker image which is used to build the project - - name: LinuxContainerImage - value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 - - group: mscodehub-feed-read-general - - group: mscodehub-feed-read-akv - - name: branchCounterKey - value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])] - - name: branchCounter - value: $[counter(variables['branchCounterKey'], 1)] - - group: MSIXSigningProfile - - name: disableNetworkIsolation - value: ${{ parameters.disableNetworkIsolation }} + - template: templates/variables/PowerShell-Packages-Variables.yml + parameters: + debug: ${{ parameters.debug }} + ForceAzureBlobDelete: ${{ parameters.ForceAzureBlobDelete }} + ReleaseTagVar: ${{ parameters.ReleaseTagVar }} + disableNetworkIsolation: ${{ parameters.disableNetworkIsolation }} resources: pipelines: @@ -121,184 +92,6 @@ extends: enabled: false tsaOptionsFile: .config\tsaoptions.json stages: - - stage: prep - displayName: 'Prep BuildInfo+Az' - jobs: - - template: /.pipelines/templates/checkAzureContainer.yml@self - - - stage: mac_package - displayName: 'macOS Pkg+Sign' - dependsOn: [] - jobs: - - template: /.pipelines/templates/mac-package-build.yml@self - parameters: - buildArchitecture: x64 - - - template: /.pipelines/templates/mac-package-build.yml@self - parameters: - buildArchitecture: arm64 - - - stage: windows_package_build - displayName: 'Win Pkg (unsigned)' - dependsOn: [] - jobs: - - template: /.pipelines/templates/packaging/windows/package.yml@self - parameters: - runtime: x64 - - - template: /.pipelines/templates/packaging/windows/package.yml@self - parameters: - runtime: arm64 - - - template: /.pipelines/templates/packaging/windows/package.yml@self - parameters: - runtime: x86 - - - template: /.pipelines/templates/packaging/windows/package.yml@self - parameters: - runtime: fxdependent - - - template: /.pipelines/templates/packaging/windows/package.yml@self - parameters: - runtime: fxdependentWinDesktop - - - template: /.pipelines/templates/packaging/windows/package.yml@self - parameters: - runtime: minsize - - - stage: windows_package_sign - displayName: 'Win Pkg Sign' - dependsOn: [windows_package_build] - jobs: - - template: /.pipelines/templates/packaging/windows/sign.yml@self - parameters: - runtime: x64 - - - template: /.pipelines/templates/packaging/windows/sign.yml@self - parameters: - runtime: arm64 - - - template: /.pipelines/templates/packaging/windows/sign.yml@self - parameters: - runtime: x86 - - - template: /.pipelines/templates/packaging/windows/sign.yml@self - parameters: - runtime: fxdependent - - - template: /.pipelines/templates/packaging/windows/sign.yml@self - parameters: - runtime: fxdependentWinDesktop - - - template: /.pipelines/templates/packaging/windows/sign.yml@self - parameters: - runtime: minsize - - - stage: linux_package - displayName: 'Linux Pkg+Sign' - dependsOn: [] - jobs: - - template: /.pipelines/templates/linux-package-build.yml@self - parameters: - unsignedDrop: 'drop_linux_build_linux_x64' - signedDrop: 'drop_linux_sign_linux_x64' - packageType: deb - jobName: deb - - - template: /.pipelines/templates/linux-package-build.yml@self - parameters: - unsignedDrop: 'drop_linux_build_linux_fxd_x64_mariner' - signedDrop: 'drop_linux_sign_linux_fxd_x64_mariner' - packageType: rpm-fxdependent #mariner-x64 - jobName: mariner_x64 - signingProfile: 'CP-459159-pgpdetached' - - - template: /.pipelines/templates/linux-package-build.yml@self - parameters: - unsignedDrop: 'drop_linux_build_linux_fxd_arm64_mariner' - signedDrop: 'drop_linux_sign_linux_fxd_arm64_mariner' - packageType: rpm-fxdependent-arm64 #mariner-arm64 - jobName: mariner_arm64 - signingProfile: 'CP-459159-pgpdetached' - - - template: /.pipelines/templates/linux-package-build.yml@self - parameters: - unsignedDrop: 'drop_linux_build_linux_x64' - signedDrop: 'drop_linux_sign_linux_x64' - packageType: rpm - jobName: rpm - - - template: /.pipelines/templates/linux-package-build.yml@self - parameters: - unsignedDrop: 'drop_linux_build_linux_arm' - signedDrop: 'drop_linux_sign_linux_arm' - packageType: tar-arm - jobName: tar_arm - - - template: /.pipelines/templates/linux-package-build.yml@self - parameters: - unsignedDrop: 'drop_linux_build_linux_arm64' - signedDrop: 'drop_linux_sign_linux_arm64' - packageType: tar-arm64 - jobName: tar_arm64 - - - template: /.pipelines/templates/linux-package-build.yml@self - parameters: - unsignedDrop: 'drop_linux_build_linux_x64_alpine' - signedDrop: 'drop_linux_sign_linux_x64_alpine' - packageType: tar-alpine - jobName: tar_alpine - - - template: /.pipelines/templates/linux-package-build.yml@self - parameters: - unsignedDrop: 'drop_linux_build_linux_fxd' - signedDrop: 'drop_linux_sign_linux_fxd' - packageType: fxdependent - jobName: fxdependent - - - template: /.pipelines/templates/linux-package-build.yml@self - parameters: - unsignedDrop: 'drop_linux_build_linux_x64' - signedDrop: 'drop_linux_sign_linux_x64' - packageType: tar - jobName: tar - - - template: /.pipelines/templates/linux-package-build.yml@self - parameters: - unsignedDrop: 'drop_linux_build_linux_fxd_x64_alpine' - signedDrop: 'drop_linux_sign_linux_fxd_x64_alpine' - packageType: tar-alpine-fxdependent - jobName: tar_alpine_fxd - - - template: /.pipelines/templates/linux-package-build.yml@self - parameters: - unsignedDrop: 'drop_linux_build_linux_x64_minSize' - signedDrop: 'drop_linux_sign_linux_x64_minSize' - packageType: min-size - jobName: minSize - - - stage: nupkg - displayName: 'NuGet Pkg+Sign' - dependsOn: [] - jobs: - - template: /.pipelines/templates/nupkg.yml@self - - - stage: msixbundle - displayName: 'MSIX Bundle+Sign' - dependsOn: [windows_package_build] # Only depends on unsigned packages - jobs: - - template: /.pipelines/templates/package-create-msix.yml@self - parameters: - OfficialBuild: true - - - stage: upload - displayName: 'Upload' - dependsOn: [prep, mac_package, windows_package_sign, linux_package, nupkg, msixbundle] # prep needed for BuildInfo JSON - jobs: - - template: /.pipelines/templates/uploadToAzure.yml@self - - - stage: validatePackages - displayName: 'Validate Packages' - dependsOn: [upload] - jobs: - - template: /.pipelines/templates/release-validate-packagenames.yml@self + - template: templates/stages/PowerShell-Packages-Stages.yml + parameters: + OfficialBuild: true diff --git a/.pipelines/PowerShell-Release-Official-Azure.yml b/.pipelines/PowerShell-Release-Official-Azure.yml index 81543420460..24040a2463d 100644 --- a/.pipelines/PowerShell-Release-Official-Azure.yml +++ b/.pipelines/PowerShell-Release-Official-Azure.yml @@ -14,38 +14,12 @@ parameters: # parameters are shown up in ADO UI in a build queue time type: string default: 'NO' -name: ev2-$(BUILD.SOURCEBRANCHNAME)-prod.true-$(Build.BuildId) +name: ev2-$(BUILD.SOURCEBRANCHNAME)-prod-$(Build.BuildId) variables: - - name: CDP_DEFINITION_BUILD_COUNT - value: $[counter('', 0)] - - name: system.debug - value: ${{ parameters.debug }} - - name: ENABLE_PRS_DELAYSIGN - value: 1 - - name: ROOT - value: $(Build.SourcesDirectory) - - name: REPOROOT - value: $(Build.SourcesDirectory) - - name: OUTPUTROOT - value: $(REPOROOT)\out - - name: NUGET_XMLDOC_MODE - value: none - - name: nugetMultiFeedWarnLevel - value: none - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: skipNugetSecurityAnalysis - value: true - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: ob_sdl_tsa_configFile - value: $(Build.SourcesDirectory)\.config\tsaoptions.json - - name: WindowsContainerImage - value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - - name: LinuxContainerImage - value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 - - group: PoolNames + - template: templates/variables/PowerShell-Release-Azure-Variables.yml + parameters: + debug: ${{ parameters.debug }} resources: repositories: diff --git a/.pipelines/PowerShell-Release-Official.yml b/.pipelines/PowerShell-Release-Official.yml index fa14b9b0acb..3528e6b1471 100644 --- a/.pipelines/PowerShell-Release-Official.yml +++ b/.pipelines/PowerShell-Release-Official.yml @@ -30,43 +30,13 @@ parameters: # parameters are shown up in ADO UI in a build queue time type: boolean default: false -name: release-$(BUILD.SOURCEBRANCHNAME)-prod.true-$(Build.BuildId) +name: release-$(BUILD.SOURCEBRANCHNAME)-prod-$(Build.BuildId) variables: - - name: CDP_DEFINITION_BUILD_COUNT - value: $[counter('', 0)] - - name: system.debug - value: ${{ parameters.debug }} - - name: ENABLE_PRS_DELAYSIGN - value: 1 - - name: ROOT - value: $(Build.SourcesDirectory) - - name: REPOROOT - value: $(Build.SourcesDirectory) - - name: OUTPUTROOT - value: $(REPOROOT)\out - - name: NUGET_XMLDOC_MODE - value: none - - name: nugetMultiFeedWarnLevel - value: none - - name: NugetSecurityAnalysisWarningLevel - value: none - - name: skipNugetSecurityAnalysis - value: true - - name: ob_outputDirectory - value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' - - name: WindowsContainerImage - value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - - name: LinuxContainerImage - value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 - - name: ReleaseTagVar - value: ${{ parameters.ReleaseTagVar }} - - group: PoolNames - - name: releaseEnvironment - value: 'Production' - # Fix for BinSkim ICU package error in Linux containers - - name: DOTNET_SYSTEM_GLOBALIZATION_INVARIANT - value: true + - template: templates/variables/PowerShell-Release-Variables.yml + parameters: + debug: ${{ parameters.debug }} + ReleaseTagVar: ${{ parameters.ReleaseTagVar }} resources: repositories: @@ -124,315 +94,9 @@ extends: tsaOptionsFile: .config\tsaoptions.json stages: - - stage: setReleaseTagAndChangelog - displayName: 'Set Release Tag and Upload Changelog' - jobs: - - template: /.pipelines/templates/release-SetTagAndChangelog.yml@self - - - stage: validateSdk - displayName: 'Validate SDK' - dependsOn: [] - jobs: - - template: /.pipelines/templates/release-validate-sdk.yml@self - parameters: - jobName: "windowsSDK" - displayName: "Windows SDK Validation" - imageName: PSMMS2019-Secure - poolName: $(windowsPool) - - - template: /.pipelines/templates/release-validate-sdk.yml@self - parameters: - jobName: "MacOSSDK" - displayName: "MacOS SDK Validation" - imageName: macOS-latest - poolName: Azure Pipelines - - - template: /.pipelines/templates/release-validate-sdk.yml@self - parameters: - jobName: "LinuxSDK" - displayName: "Linux SDK Validation" - imageName: PSMMSUbuntu22.04-Secure - poolName: $(ubuntuPool) - - - stage: gbltool - displayName: 'Validate Global tools' - dependsOn: [] - jobs: - - template: /.pipelines/templates/release-validate-globaltools.yml@self - parameters: - jobName: "WindowsGlobalTools" - displayName: "Windows Global Tools Validation" - jobtype: windows - - - template: /.pipelines/templates/release-validate-globaltools.yml@self - parameters: - jobName: "LinuxGlobalTools" - displayName: "Linux Global Tools Validation" - jobtype: linux - globalToolExeName: 'pwsh' - globalToolPackageName: 'PowerShell.Linux.x64' - - - stage: fxdpackages - displayName: 'Validate FXD Packages' - dependsOn: [] - jobs: - - template: /.pipelines/templates/release-validate-fxdpackages.yml@self - parameters: - jobName: 'winfxd' - displayName: 'Validate Win Fxd Packages' - jobtype: 'windows' - artifactName: 'drop_windows_package_package_win_fxdependent' - packageNamePattern: '**/*win-fxdependent.zip' - - - template: /.pipelines/templates/release-validate-fxdpackages.yml@self - parameters: - jobName: 'winfxdDesktop' - displayName: 'Validate WinDesktop Fxd Packages' - jobtype: 'windows' - artifactName: 'drop_windows_package_package_win_fxdependentWinDesktop' - packageNamePattern: '**/*win-fxdependentwinDesktop.zip' - - - template: /.pipelines/templates/release-validate-fxdpackages.yml@self - parameters: - jobName: 'linuxfxd' - displayName: 'Validate Linux Fxd Packages' - jobtype: 'linux' - artifactName: 'drop_linux_package_fxdependent' - packageNamePattern: '**/*linux-x64-fxdependent.tar.gz' - - - template: /.pipelines/templates/release-validate-fxdpackages.yml@self - parameters: - jobName: 'linuxArm64fxd' - displayName: 'Validate Linux ARM64 Fxd Packages' - jobtype: 'linux' - artifactName: 'drop_linux_package_fxdependent' - # this is really an architecture independent package - packageNamePattern: '**/*linux-x64-fxdependent.tar.gz' - arm64: 'yes' - enableCredScan: false - - - stage: ManualValidation - dependsOn: [] - displayName: Manual Validation - jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Validate Windows Packages - jobName: ValidateWinPkg - instructions: | - Validate zip package on windows - - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Validate OSX Packages - jobName: ValidateOsxPkg - instructions: | - Validate tar.gz package on osx-arm64 - - - stage: ReleaseAutomation - dependsOn: [] - displayName: 'Release Automation' - jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Start Release Automation - jobName: StartRA - instructions: | - Kick off Release automation build at: https://dev.azure.com/powershell-rel/Release-Automation/_build?definitionId=10&_a=summary - - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Triage results - jobName: TriageRA - dependsOnJob: StartRA - instructions: | - Triage ReleaseAutomation results - - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Signoff Tests - dependsOnJob: TriageRA - jobName: SignoffTests - instructions: | - Signoff ReleaseAutomation results - - - stage: UpdateChangeLog - displayName: Update the changelog - dependsOn: - - ManualValidation - - ReleaseAutomation - - fxdpackages - - gbltool - - validateSdk - - jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Make sure the changelog is updated - jobName: MergeChangeLog - instructions: | - Update and merge the changelog for the release. - This step is required for creating GitHub draft release. - - - stage: PublishGitHubReleaseAndNuget - displayName: Publish GitHub and Nuget Release - dependsOn: - - setReleaseTagAndChangelog - - UpdateChangeLog - variables: - ob_release_environment: ${{ variables.releaseEnvironment }} - jobs: - - template: /.pipelines/templates/release-githubNuget.yml@self - parameters: - skipPublish: ${{ parameters.SkipPublish }} - - - stage: PushGitTagAndMakeDraftPublic - displayName: Push Git Tag and Make Draft Public - dependsOn: PublishGitHubReleaseAndNuget - jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Push Git Tag - jobName: PushGitTag - instructions: | - Push the git tag to upstream - - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Make Draft Public - dependsOnJob: PushGitTag - jobName: DraftPublic - instructions: | - Make the GitHub Release Draft Public - - - stage: BlobPublic - displayName: Make Blob Public - dependsOn: - - UpdateChangeLog - - PushGitTagAndMakeDraftPublic - jobs: - - template: /.pipelines/templates/release-MakeBlobPublic.yml@self - parameters: - SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} - - - stage: PublishPMC - displayName: Publish PMC - dependsOn: PushGitTagAndMakeDraftPublic - jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Publish to PMC - jobName: ReleaseToPMC - instructions: | - Run PowerShell-Release-Official-Azure.yml pipeline to publish to PMC - - - stage: UpdateDotnetDocker - dependsOn: PushGitTagAndMakeDraftPublic - displayName: Update DotNet SDK Docker images - jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Update .NET SDK docker images - jobName: DotnetDocker - instructions: | - Create PR for updating dotnet-docker images to use latest PowerShell version. - 1. Fork and clone https://github.com/dotnet/dotnet-docker.git - 2. git checkout upstream/nightly -b updatePS - 3. dotnet run --project .\eng\update-dependencies\ specific --product-version powershell= --compute-shas - 4. create PR targeting nightly branch - - - stage: UpdateWinGet - dependsOn: PushGitTagAndMakeDraftPublic - displayName: Add manifest entry to winget - jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Add manifest entry to winget - jobName: UpdateWinGet - instructions: | - This is typically done by the community 1-2 days after the release. - - - stage: PublishMsix - dependsOn: - - setReleaseTagAndChangelog - - PushGitTagAndMakeDraftPublic - displayName: Publish MSIX to store - variables: - ob_release_environment: ${{ variables.releaseEnvironment }} - jobs: - - template: /.pipelines/templates/release-MSIX-Publish.yml@self - parameters: - skipMSIXPublish: ${{ parameters.skipMSIXPublish }} - - - stage: PublishVPack - dependsOn: PushGitTagAndMakeDraftPublic - displayName: Release vPack - jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Start 2 vPack Release pipelines - jobName: PublishVPack - instructions: | - 1. Kick off PowerShell-vPack-Official pipeline - 2. Kick off PowerShell-MSIXBundle-VPack pipeline - - # Need to verify if the Az PS / CLI team still uses this. Skippinng for this release. - # - stage: ReleaseDeps - # dependsOn: GitHubTasks - # displayName: Update pwsh.deps.json links - # jobs: - # - template: templates/release-UpdateDepsJson.yml - - - stage: UploadBuildInfoJson - dependsOn: PushGitTagAndMakeDraftPublic - displayName: Upload BuildInfo.json - jobs: - - template: /.pipelines/templates/release-upload-buildinfo.yml@self - - - stage: ReleaseSymbols - dependsOn: PushGitTagAndMakeDraftPublic - displayName: Release Symbols - jobs: - - template: /.pipelines/templates/release-symbols.yml@self - - - stage: ChangesToMaster - displayName: Ensure changes are in GH master - dependsOn: - - PublishPMC - jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Make sure changes are in master - jobName: MergeToMaster - instructions: | - Make sure that changes README.md and metadata.json are merged into master on GitHub. - - - stage: ReleaseToMU - displayName: Release to MU - dependsOn: PushGitTagAndMakeDraftPublic # This only needs the blob to be available - jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Release to MU - instructions: | - Notify the PM team to start the process of releasing to MU. - - - stage: ReleaseClose - displayName: Finish Release - dependsOn: - - ReleaseToMU - - ReleaseSymbols - jobs: - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Retain Build - jobName: RetainBuild - instructions: | - Retain the build - - - template: /.pipelines/templates/approvalJob.yml@self - parameters: - displayName: Delete release branch - jobName: DeleteBranch - instructions: | - Delete release + - template: templates/stages/PowerShell-Release-Stages.yml + parameters: + releaseEnvironment: Production + SkipPublish: ${{ parameters.SkipPublish }} + SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} + skipMSIXPublish: ${{ parameters.skipMSIXPublish }} diff --git a/.pipelines/PowerShell-vPack-Official.yml b/.pipelines/PowerShell-vPack-Official.yml index f7faeb30656..9a7ff3b86e6 100644 --- a/.pipelines/PowerShell-vPack-Official.yml +++ b/.pipelines/PowerShell-vPack-Official.yml @@ -29,37 +29,14 @@ parameters: # parameters are shown up in ADO UI in a build queue time - Netlock default: "R1" -name: vPack_$(Build.SourceBranchName)_Prod.true_Create.${{ parameters.createVPack }}_Name.${{ parameters.vPackName}}_$(date:yyyyMMdd).$(rev:rr) +name: vPack_$(Build.SourceBranchName)_Prod_Create.${{ parameters.createVPack }}_Name.${{ parameters.vPackName}}_$(date:yyyyMMdd).$(rev:rr) variables: - - name: CDP_DEFINITION_BUILD_COUNT - value: $[counter('', 0)] - - name: system.debug - value: ${{ parameters.debug }} - - name: BuildSolution - value: $(Build.SourcesDirectory)\dirs.proj - - name: BuildConfiguration - value: Release - - name: WindowsContainerImage - value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - - name: Codeql.Enabled - value: false # pipeline is not building artifacts; it repackages existing artifacts into a vpack - - name: DOTNET_CLI_TELEMETRY_OPTOUT - value: 1 - - name: POWERSHELL_TELEMETRY_OPTOUT - value: 1 - - name: nugetMultiFeedWarnLevel - value: none - - name: ReleaseTagVar - value: ${{ parameters.ReleaseTagVar }} - - group: Azure Blob variable group - - group: certificate_logical_to_actual # used within signing task - - group: DotNetPrivateBuildAccess - - group: certificate_logical_to_actual - - name: netiso - value: ${{ parameters.netiso }} -# We shouldn't be using PATs anymore -# - group: mscodehub-feed-read-general + - template: templates/variables/PowerShell-vPack-Variables.yml + parameters: + debug: ${{ parameters.debug }} + ReleaseTagVar: ${{ parameters.ReleaseTagVar }} + netiso: ${{ parameters.netiso }} resources: repositories: @@ -104,232 +81,7 @@ extends: enabled: false tsaOptionsFile: .config/tsaoptions.json stages: - - stage: BuildStage - jobs: - - job: BuildJob - pool: - type: windows - - strategy: - matrix: - x86: - architecture: x86 - - x64: - architecture: x64 - - arm64: - architecture: arm64 - - variables: - ArtifactPlatform: 'windows' - ob_artifactBaseName: drop_build_$(architecture) - ob_outputDirectory: '$(BUILD.SOURCESDIRECTORY)\out' - ob_createvpack_enabled: ${{ parameters.createVPack }} - ob_createvpack_owneralias: tplunk - ob_createvpack_versionAs: parts - ob_createvpack_propsFile: true - ob_createvpack_verbose: true - ob_createvpack_packagename: '${{ parameters.vPackName }}.$(architecture)' - ob_createvpack_description: PowerShell $(architecture) $(version) - # I think the variables reload after we transition back to the host so this works. 🤷‍♂️ - ob_createvpack_majorVer: $(pwshMajorVersion) - ob_createvpack_minorVer: $(pwshMinorVersion) - ob_createvpack_patchVer: $(pwshPatchVersion) - ${{ if ne(variables['pwshPrereleaseVersion'], '') }}: - ob_createvpack_prereleaseVer: $(pwshPrereleaseVersion) - ${{ else }}: - ob_createvpack_prereleaseVer: $(Build.SourceVersion) - - steps: - - checkout: self - displayName: Checkout source code - during restore - clean: true - path: s - env: - ob_restore_phase: true - - - template: .pipelines/templates/SetVersionVariables.yml@self - parameters: - ReleaseTagVar: $(ReleaseTagVar) - CreateJson: yes - - - pwsh: | - $version = '$(Version)' - Write-Verbose -Verbose "Version: $version" - if(!$version) { - throw "Version is not set." - } - - $mainVersionParts = $version -split '-' - - Write-Verbose -Verbose "mainVersionParts: $($mainVersionParts[0]) ; $($mainVersionParts[1])" - $versionParts = $mainVersionParts[0] -split '[.]'; - $major = $versionParts[0] - $minor = $versionParts[1] - $patch = $versionParts[2] - - $previewPart = $mainVersionParts[1] - Write-Verbose -Verbose "previewPart: $previewPart" - - Write-Host "major: $major; minor: $minor; patch: $patch;" - - $vstsCommandString = "vso[task.setvariable variable=pwshMajorVersion]$major" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - - $vstsCommandString = "vso[task.setvariable variable=pwshMinorVersion]$minor" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - - $vstsCommandString = "vso[task.setvariable variable=pwshPatchVersion]$patch" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - if($previewPart) { - $vstsCommandString = "vso[task.setvariable variable=pwshPrereleaseVersion]$previewPart" - } else { - Write-Verbose -Verbose "No prerelease part found in version string." - } - displayName: Set ob_createvpack_*Ver - env: - ob_restore_phase: true - - # Validate pwsh*Version variables - - pwsh: | - $variables = @("pwshMajorVersion", "pwshMinorVersion", "pwshPatchVersion") - foreach ($var in $variables) { - if (-not (get-item "Env:\$var" -ErrorAction SilentlyContinue).value) { - throw "Required variable '`$env:$var' is not set." - } - } - displayName: Validate pwsh*Version variables - env: - ob_restore_phase: true - - - pwsh: | - if($env:RELEASETAGVAR -match '-') { - throw "Don't release a preview build without coordinating with Windows Engineering Build Tools Team" - } - displayName: Stop any preview release - env: - ob_restore_phase: true - - - task: UseDotNet@2 - displayName: 'Use .NET Core sdk' - inputs: - packageType: sdk - version: 3.1.x - installationPath: $(Agent.ToolsDirectory)/dotnet - - ### BUILD ### - - - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self - parameters: - repoRoot: $(repoRoot) - - - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. - env: - ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. - inputs: - Enabled: true - AnalyzeInPipeline: false # Do not upload results - Language: csharp - - - task: UseDotNet@2 - displayName: 'Install .NET based on global.json' - inputs: - useGlobalJson: true - workingDirectory: $(repoRoot) - env: - ob_restore_phase: true - - - pwsh: | - # Need to set PowerShellRoot variable for obp-file-signing template - $vstsCommandString = "vso[task.setvariable variable=PowerShellRoot]$(repoRoot)" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - - $Architecture = '$(Architecture)' - $runtime = switch ($Architecture) - { - "x64" { "win7-x64" } - "x86" { "win7-x86" } - "arm64" { "win-arm64" } - } - - $params = @{} - if ($env:BuildConfiguration -eq 'minSize') { - $params['ForMinimalSize'] = $true - } - - $vstsCommandString = "vso[task.setvariable variable=Runtime]$runtime" - Write-Host ("sending " + $vstsCommandString) - Write-Host "##$vstsCommandString" - - Write-Verbose -Message "Building PowerShell with Runtime: $runtime for '$env:BuildConfiguration' configuration" - Import-Module -Name $(repoRoot)/build.psm1 -Force - $buildWithSymbolsPath = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/Symbols_$Architecture" -Force - - Start-PSBootstrap -Scenario Package - $null = New-Item -ItemType Directory -Path $buildWithSymbolsPath -Force -Verbose - - $ReleaseTagParam = @{} - - if ($env:RELEASETAGVAR) { - $ReleaseTagParam['ReleaseTag'] = $env:RELEASETAGVAR - } - - Start-PSBuild -Runtime $runtime -Configuration Release -Output $buildWithSymbolsPath -Clean -PSModuleRestore @params @ReleaseTagParam - - $refFolderPath = Join-Path $buildWithSymbolsPath 'ref' - Write-Verbose -Verbose "refFolderPath: $refFolderPath" - $outputPath = Join-Path '$(ob_outputDirectory)' 'psoptions' - $null = New-Item -ItemType Directory -Path $outputPath -Force - $psOptPath = "$outputPath/psoptions.json" - Save-PSOptions -PSOptionsPath $psOptPath - - Write-Verbose -Verbose "Completed building PowerShell for '$env:BuildConfiguration' configuration" - displayName: Build Windows Universal - $(Architecture) -$(BuildConfiguration) Symbols folder - env: - __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) - ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. - - - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. - env: - ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(repoRoot)\src' - ob_restore_phase: true - - - template: /.pipelines/templates/obp-file-signing.yml@self - parameters: - binPath: '$(Pipeline.Workspace)/Symbols_$(Architecture)' - SigningProfile: $(windows_build_tools_cert_id) - OfficialBuild: false - vPackScenario: true - - ### END OF BUILD ### - - - pwsh: | - Get-ChildItem env:/ob_createvpack_*Ver - Get-ChildItem -Path "$(Pipeline.Workspace)\Symbols_$(Architecture)\*" -Recurse - Get-Content "$(Pipeline.Workspace)\PowerShell\preview.json" -ErrorAction SilentlyContinue | Write-Host - displayName: Debug Output Directory and Version - condition: succeededOrFailed() - - - pwsh: | - Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose - displayName: Capture Environment - condition: succeededOrFailed() - - - pwsh: | - $vpackFiles = Get-ChildItem -Path "$(Pipeline.Workspace)\Symbols_$(Architecture)\*" -Recurse - if($vpackFiles.Count -eq 0) { - throw "No files found in $(Pipeline.Workspace)\Symbols_$(Architecture)" - } - $vpackFiles - displayName: Debug Output Directory and Version - condition: succeededOrFailed() + - template: templates/stages/PowerShell-vPack-Stages.yml + parameters: + createVPack: ${{ parameters.createVPack }} + vPackName: ${{ parameters.vPackName }} diff --git a/.pipelines/templates/release-MSIX-Publish.yml b/.pipelines/templates/release-MSIX-Publish.yml index a92c71f826b..aaef3c6f269 100644 --- a/.pipelines/templates/release-MSIX-Publish.yml +++ b/.pipelines/templates/release-MSIX-Publish.yml @@ -21,7 +21,7 @@ jobs: value: $[ stageDependencies.setReleaseTagAndChangelog.setTagAndChangelog.outputs['ChannelSelection.IsStable'] ] - name: PREVIEW value: $[ stageDependencies.setReleaseTagAndChangelog.setTagAndChangelog.outputs['ChannelSelection.IsPreview'] ] - - template: ./variable/release-shared.yml@self + - template: ./variables/release-shared.yml@self parameters: RELEASETAG: $[ stageDependencies.setReleaseTagAndChangelog.setTagAndChangelog.outputs['OutputReleaseTag.releaseTag'] ] steps: diff --git a/.pipelines/templates/release-githubNuget.yml b/.pipelines/templates/release-githubNuget.yml index 206079c555f..95698554c40 100644 --- a/.pipelines/templates/release-githubNuget.yml +++ b/.pipelines/templates/release-githubNuget.yml @@ -17,7 +17,7 @@ jobs: pipeline: PSPackagesOfficial artifactName: drop_upload_upload_packages variables: - - template: ./variable/release-shared.yml@self + - template: ./variables/release-shared.yml@self parameters: RELEASETAG: $[ stageDependencies.setReleaseTagAndChangelog.setTagAndChangelog.outputs['OutputReleaseTag.releaseTag'] ] @@ -167,7 +167,7 @@ jobs: pipeline: PSPackagesOfficial artifactName: drop_upload_upload_packages variables: - - template: ./variable/release-shared.yml@self + - template: ./variables/release-shared.yml@self parameters: VERSION: $[ stageDependencies.setReleaseTagAndChangelog.SetTagAndChangelog.outputs['OutputVersion.Version'] ] diff --git a/.pipelines/templates/stages/PowerShell-Coordinated_Packages-Stages.yml b/.pipelines/templates/stages/PowerShell-Coordinated_Packages-Stages.yml new file mode 100644 index 00000000000..cd0a4ebc065 --- /dev/null +++ b/.pipelines/templates/stages/PowerShell-Coordinated_Packages-Stages.yml @@ -0,0 +1,202 @@ +parameters: + - name: RUN_WINDOWS + type: boolean + default: true + - name: RUN_TEST_AND_RELEASE + type: boolean + default: true + - name: OfficialBuild + type: boolean + +stages: +- stage: prep + jobs: + - job: SetVars + displayName: Set Variables + pool: + type: linux + + variables: + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT/BuildJson' + - name: ob_sdl_codeSignValidation_enabled + value: false + - name: ob_sdl_codeql_compiled_enabled + value: false + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_signing_setup_enabled + value: false + - name: ob_sdl_sbom_enabled + value: false + + steps: + - checkout: self + clean: true + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - pwsh: | + Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + displayName: Capture environment variables + env: + ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase + + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + +- stage: macos + displayName: macOS - build and sign + dependsOn: ['prep'] + variables: + - name: ps_official_build + value: ${{ parameters.OfficialBuild }} + jobs: + - template: /.pipelines/templates/mac.yml@self + parameters: + buildArchitecture: x64 + - template: /.pipelines/templates/mac.yml@self + parameters: + buildArchitecture: arm64 + +- stage: linux + displayName: linux - build and sign + dependsOn: ['prep'] + variables: + - name: ps_official_build + value: ${{ parameters.OfficialBuild }} + jobs: + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'linux-x64' + JobName: 'linux_x64' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'linux-x64' + JobName: 'linux_x64_minSize' + BuildConfiguration: 'minSize' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'linux-arm' + JobName: 'linux_arm' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'linux-arm64' + JobName: 'linux_arm64' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'fxdependent-linux-x64' + JobName: 'linux_fxd_x64_mariner' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'fxdependent-linux-arm64' + JobName: 'linux_fxd_arm64_mariner' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'fxdependent-noopt-linux-musl-x64' + JobName: 'linux_fxd_x64_alpine' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'fxdependent' + JobName: 'linux_fxd' + + - template: /.pipelines/templates/linux.yml@self + parameters: + Runtime: 'linux-musl-x64' + JobName: 'linux_x64_alpine' + +- stage: windows + displayName: windows - build and sign + dependsOn: ['prep'] + condition: and(succeeded(),eq('${{ parameters.RUN_WINDOWS }}','true')) + variables: + - name: ps_official_build + value: ${{ parameters.OfficialBuild }} + jobs: + - template: /.pipelines/templates/windows-hosted-build.yml@self + parameters: + Architecture: x64 + BuildConfiguration: release + JobName: build_windows_x64_release + - template: /.pipelines/templates/windows-hosted-build.yml@self + parameters: + Architecture: x64 + BuildConfiguration: minSize + JobName: build_windows_x64_minSize_release + - template: /.pipelines/templates/windows-hosted-build.yml@self + parameters: + Architecture: x86 + JobName: build_windows_x86_release + - template: /.pipelines/templates/windows-hosted-build.yml@self + parameters: + Architecture: arm64 + JobName: build_windows_arm64_release + - template: /.pipelines/templates/windows-hosted-build.yml@self + parameters: + Architecture: fxdependent + JobName: build_windows_fxdependent_release + - template: /.pipelines/templates/windows-hosted-build.yml@self + parameters: + Architecture: fxdependentWinDesktop + JobName: build_windows_fxdependentWinDesktop_release + +- stage: test_and_release_artifacts + displayName: Test and Release Artifacts + dependsOn: ['prep'] + condition: and(succeeded(),eq('${{ parameters.RUN_TEST_AND_RELEASE }}','true')) + jobs: + - template: /.pipelines/templates/testartifacts.yml@self + + - job: release_json + displayName: Create and Upload release.json + pool: + type: windows + variables: + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json + - name: ob_sdl_credscan_suppressionsFile + value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json + steps: + - checkout: self + clean: true + - template: /.pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + - template: /.pipelines/templates/rebuild-branch-check.yml@self + - powershell: | + $metadata = Get-Content '$(Build.SourcesDirectory)/PowerShell/tools/metadata.json' -Raw | ConvertFrom-Json + + # Use the rebuild branch check from the template + $isRebuildBranch = '$(RebuildBranchCheck.IsRebuildBranch)' -eq 'true' + + # Don't mark as LTS release for rebuild branches + $LTS = $metadata.LTSRelease.Package -and -not $isRebuildBranch + + if ($isRebuildBranch) { + Write-Verbose -Message "Rebuild branch detected, not marking as LTS release" -Verbose + } + + @{ ReleaseVersion = "$(Version)"; LTSRelease = $LTS } | ConvertTo-Json | Out-File "$(Build.StagingDirectory)\release.json" + Get-Content "$(Build.StagingDirectory)\release.json" + + if (-not (Test-Path "$(ob_outputDirectory)\metadata")) { + New-Item -ItemType Directory -Path "$(ob_outputDirectory)\metadata" + } + + Copy-Item -Path "$(Build.StagingDirectory)\release.json" -Destination "$(ob_outputDirectory)\metadata" -Force + displayName: Create and upload release.json file to build artifact + retryCountOnTaskFailure: 2 + - template: /.pipelines/templates/step/finalize.yml@self diff --git a/.pipelines/templates/stages/PowerShell-Packages-Stages.yml b/.pipelines/templates/stages/PowerShell-Packages-Stages.yml new file mode 100644 index 00000000000..ff40941e31b --- /dev/null +++ b/.pipelines/templates/stages/PowerShell-Packages-Stages.yml @@ -0,0 +1,186 @@ +parameters: + - name: OfficialBuild + type: boolean + +stages: +- stage: prep + displayName: 'Prep BuildInfo+Az' + jobs: + - template: /.pipelines/templates/checkAzureContainer.yml@self + +- stage: mac_package + displayName: 'macOS Pkg+Sign' + dependsOn: [] + jobs: + - template: /.pipelines/templates/mac-package-build.yml@self + parameters: + buildArchitecture: x64 + + - template: /.pipelines/templates/mac-package-build.yml@self + parameters: + buildArchitecture: arm64 + +- stage: windows_package_build + displayName: 'Win Pkg (unsigned)' + dependsOn: [] + jobs: + - template: /.pipelines/templates/packaging/windows/package.yml@self + parameters: + runtime: x64 + + - template: /.pipelines/templates/packaging/windows/package.yml@self + parameters: + runtime: arm64 + + - template: /.pipelines/templates/packaging/windows/package.yml@self + parameters: + runtime: x86 + + - template: /.pipelines/templates/packaging/windows/package.yml@self + parameters: + runtime: fxdependent + + - template: /.pipelines/templates/packaging/windows/package.yml@self + parameters: + runtime: fxdependentWinDesktop + + - template: /.pipelines/templates/packaging/windows/package.yml@self + parameters: + runtime: minsize + +- stage: windows_package_sign + displayName: 'Win Pkg Sign' + dependsOn: [windows_package_build] + jobs: + - template: /.pipelines/templates/packaging/windows/sign.yml@self + parameters: + runtime: x64 + + - template: /.pipelines/templates/packaging/windows/sign.yml@self + parameters: + runtime: arm64 + + - template: /.pipelines/templates/packaging/windows/sign.yml@self + parameters: + runtime: x86 + + - template: /.pipelines/templates/packaging/windows/sign.yml@self + parameters: + runtime: fxdependent + + - template: /.pipelines/templates/packaging/windows/sign.yml@self + parameters: + runtime: fxdependentWinDesktop + + - template: /.pipelines/templates/packaging/windows/sign.yml@self + parameters: + runtime: minsize + +- stage: linux_package + displayName: 'Linux Pkg+Sign' + dependsOn: [] + jobs: + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_x64' + signedDrop: 'drop_linux_sign_linux_x64' + packageType: deb + jobName: deb + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_fxd_x64_mariner' + signedDrop: 'drop_linux_sign_linux_fxd_x64_mariner' + packageType: rpm-fxdependent #mariner-x64 + jobName: mariner_x64 + signingProfile: 'CP-459159-pgpdetached' + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_fxd_arm64_mariner' + signedDrop: 'drop_linux_sign_linux_fxd_arm64_mariner' + packageType: rpm-fxdependent-arm64 #mariner-arm64 + jobName: mariner_arm64 + signingProfile: 'CP-459159-pgpdetached' + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_x64' + signedDrop: 'drop_linux_sign_linux_x64' + packageType: rpm + jobName: rpm + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_arm' + signedDrop: 'drop_linux_sign_linux_arm' + packageType: tar-arm + jobName: tar_arm + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_arm64' + signedDrop: 'drop_linux_sign_linux_arm64' + packageType: tar-arm64 + jobName: tar_arm64 + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_x64_alpine' + signedDrop: 'drop_linux_sign_linux_x64_alpine' + packageType: tar-alpine + jobName: tar_alpine + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_fxd' + signedDrop: 'drop_linux_sign_linux_fxd' + packageType: fxdependent + jobName: fxdependent + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_x64' + signedDrop: 'drop_linux_sign_linux_x64' + packageType: tar + jobName: tar + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_fxd_x64_alpine' + signedDrop: 'drop_linux_sign_linux_fxd_x64_alpine' + packageType: tar-alpine-fxdependent + jobName: tar_alpine_fxd + + - template: /.pipelines/templates/linux-package-build.yml@self + parameters: + unsignedDrop: 'drop_linux_build_linux_x64_minSize' + signedDrop: 'drop_linux_sign_linux_x64_minSize' + packageType: min-size + jobName: minSize + +- stage: nupkg + displayName: 'NuGet Pkg+Sign' + dependsOn: [] + jobs: + - template: /.pipelines/templates/nupkg.yml@self + +- stage: msixbundle + displayName: 'MSIX Bundle+Sign' + dependsOn: [windows_package_build] # Only depends on unsigned packages + jobs: + - template: /.pipelines/templates/package-create-msix.yml@self + parameters: + OfficialBuild: ${{ parameters.OfficialBuild }} + +- stage: upload + displayName: 'Upload' + dependsOn: [prep, mac_package, windows_package_sign, linux_package, nupkg, msixbundle] # prep needed for BuildInfo JSON + jobs: + - template: /.pipelines/templates/uploadToAzure.yml@self + +- stage: validatePackages + displayName: 'Validate Packages' + dependsOn: [upload] + jobs: + - template: /.pipelines/templates/release-validate-packagenames.yml@self diff --git a/.pipelines/templates/stages/PowerShell-Release-Stages.yml b/.pipelines/templates/stages/PowerShell-Release-Stages.yml new file mode 100644 index 00000000000..52ce428a663 --- /dev/null +++ b/.pipelines/templates/stages/PowerShell-Release-Stages.yml @@ -0,0 +1,323 @@ +parameters: + - name: releaseEnvironment + type: string + - name: SkipPublish + type: boolean + - name: SkipPSInfraInstallers + type: boolean + - name: skipMSIXPublish + type: boolean + +stages: +- stage: setReleaseTagAndChangelog + displayName: 'Set Release Tag and Upload Changelog' + jobs: + - template: /.pipelines/templates/release-SetTagAndChangelog.yml@self + +- stage: validateSdk + displayName: 'Validate SDK' + dependsOn: [] + jobs: + - template: /.pipelines/templates/release-validate-sdk.yml@self + parameters: + jobName: "windowsSDK" + displayName: "Windows SDK Validation" + imageName: PSMMS2019-Secure + poolName: $(windowsPool) + + - template: /.pipelines/templates/release-validate-sdk.yml@self + parameters: + jobName: "MacOSSDK" + displayName: "MacOS SDK Validation" + imageName: macOS-latest + poolName: Azure Pipelines + + - template: /.pipelines/templates/release-validate-sdk.yml@self + parameters: + jobName: "LinuxSDK" + displayName: "Linux SDK Validation" + imageName: PSMMSUbuntu22.04-Secure + poolName: $(ubuntuPool) + +- stage: gbltool + displayName: 'Validate Global tools' + dependsOn: [] + jobs: + - template: /.pipelines/templates/release-validate-globaltools.yml@self + parameters: + jobName: "WindowsGlobalTools" + displayName: "Windows Global Tools Validation" + jobtype: windows + + - template: /.pipelines/templates/release-validate-globaltools.yml@self + parameters: + jobName: "LinuxGlobalTools" + displayName: "Linux Global Tools Validation" + jobtype: linux + globalToolExeName: 'pwsh' + globalToolPackageName: 'PowerShell.Linux.x64' + +- stage: fxdpackages + displayName: 'Validate FXD Packages' + dependsOn: [] + jobs: + - template: /.pipelines/templates/release-validate-fxdpackages.yml@self + parameters: + jobName: 'winfxd' + displayName: 'Validate Win Fxd Packages' + jobtype: 'windows' + artifactName: 'drop_windows_package_package_win_fxdependent' + packageNamePattern: '**/*win-fxdependent.zip' + + - template: /.pipelines/templates/release-validate-fxdpackages.yml@self + parameters: + jobName: 'winfxdDesktop' + displayName: 'Validate WinDesktop Fxd Packages' + jobtype: 'windows' + artifactName: 'drop_windows_package_package_win_fxdependentWinDesktop' + packageNamePattern: '**/*win-fxdependentwinDesktop.zip' + + - template: /.pipelines/templates/release-validate-fxdpackages.yml@self + parameters: + jobName: 'linuxfxd' + displayName: 'Validate Linux Fxd Packages' + jobtype: 'linux' + artifactName: 'drop_linux_package_fxdependent' + packageNamePattern: '**/*linux-x64-fxdependent.tar.gz' + + - template: /.pipelines/templates/release-validate-fxdpackages.yml@self + parameters: + jobName: 'linuxArm64fxd' + displayName: 'Validate Linux ARM64 Fxd Packages' + jobtype: 'linux' + artifactName: 'drop_linux_package_fxdependent' + # this is really an architecture independent package + packageNamePattern: '**/*linux-x64-fxdependent.tar.gz' + arm64: 'yes' + enableCredScan: false + +- stage: ManualValidation + dependsOn: [] + displayName: Manual Validation + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Validate Windows Packages + jobName: ValidateWinPkg + instructions: | + Validate zip package on windows + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Validate OSX Packages + jobName: ValidateOsxPkg + instructions: | + Validate tar.gz package on osx-arm64 + +- stage: ReleaseAutomation + dependsOn: [] + displayName: 'Release Automation' + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Start Release Automation + jobName: StartRA + instructions: | + Kick off Release automation build at: https://dev.azure.com/powershell-rel/Release-Automation/_build?definitionId=10&_a=summary + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Triage results + jobName: TriageRA + dependsOnJob: StartRA + instructions: | + Triage ReleaseAutomation results + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Signoff Tests + dependsOnJob: TriageRA + jobName: SignoffTests + instructions: | + Signoff ReleaseAutomation results + +- stage: UpdateChangeLog + displayName: Update the changelog + dependsOn: + - ManualValidation + - ReleaseAutomation + - fxdpackages + - gbltool + - validateSdk + + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Make sure the changelog is updated + jobName: MergeChangeLog + instructions: | + Update and merge the changelog for the release. + This step is required for creating GitHub draft release. + +- stage: PublishGitHubReleaseAndNuget + displayName: Publish GitHub and Nuget Release + dependsOn: + - setReleaseTagAndChangelog + - UpdateChangeLog + variables: + ob_release_environment: ${{ parameters.releaseEnvironment }} + jobs: + - template: /.pipelines/templates/release-githubNuget.yml@self + parameters: + skipPublish: ${{ parameters.SkipPublish }} + +- stage: PushGitTagAndMakeDraftPublic + displayName: Push Git Tag and Make Draft Public + dependsOn: PublishGitHubReleaseAndNuget + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Push Git Tag + jobName: PushGitTag + instructions: | + Push the git tag to upstream + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Make Draft Public + dependsOnJob: PushGitTag + jobName: DraftPublic + instructions: | + Make the GitHub Release Draft Public + +- stage: BlobPublic + displayName: Make Blob Public + dependsOn: + - UpdateChangeLog + - PushGitTagAndMakeDraftPublic + jobs: + - template: /.pipelines/templates/release-MakeBlobPublic.yml@self + parameters: + SkipPSInfraInstallers: ${{ parameters.SkipPSInfraInstallers }} + +- stage: PublishPMC + displayName: Publish PMC + dependsOn: PushGitTagAndMakeDraftPublic + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Publish to PMC + jobName: ReleaseToPMC + instructions: | + Run PowerShell-Release-Official-Azure.yml pipeline to publish to PMC + +- stage: UpdateDotnetDocker + dependsOn: PushGitTagAndMakeDraftPublic + displayName: Update DotNet SDK Docker images + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Update .NET SDK docker images + jobName: DotnetDocker + instructions: | + Create PR for updating dotnet-docker images to use latest PowerShell version. + 1. Fork and clone https://github.com/dotnet/dotnet-docker.git + 2. git checkout upstream/nightly -b updatePS + 3. dotnet run --project .\eng\update-dependencies\ specific --product-version powershell= --compute-shas + 4. create PR targeting nightly branch + +- stage: UpdateWinGet + dependsOn: PushGitTagAndMakeDraftPublic + displayName: Add manifest entry to winget + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Add manifest entry to winget + jobName: UpdateWinGet + instructions: | + This is typically done by the community 1-2 days after the release. + +- stage: PublishMsix + dependsOn: + - setReleaseTagAndChangelog + - PushGitTagAndMakeDraftPublic + displayName: Publish MSIX to store + variables: + ob_release_environment: ${{ parameters.releaseEnvironment }} + jobs: + - template: /.pipelines/templates/release-MSIX-Publish.yml@self + parameters: + skipMSIXPublish: ${{ parameters.skipMSIXPublish }} + +- stage: PublishVPack + dependsOn: PushGitTagAndMakeDraftPublic + displayName: Release vPack + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Start 2 vPack Release pipelines + jobName: PublishVPack + instructions: | + 1. Kick off PowerShell-vPack-Official pipeline + 2. Kick off PowerShell-MSIXBundle-VPack pipeline + +# Need to verify if the Az PS / CLI team still uses this. Skipping for this release. +# - stage: ReleaseDeps +# dependsOn: GitHubTasks +# displayName: Update pwsh.deps.json links +# jobs: +# - template: templates/release-UpdateDepsJson.yml + +- stage: UploadBuildInfoJson + dependsOn: PushGitTagAndMakeDraftPublic + displayName: Upload BuildInfo.json + jobs: + - template: /.pipelines/templates/release-upload-buildinfo.yml@self + +- stage: ReleaseSymbols + dependsOn: PushGitTagAndMakeDraftPublic + displayName: Release Symbols + jobs: + - template: /.pipelines/templates/release-symbols.yml@self + +- stage: ChangesToMaster + displayName: Ensure changes are in GH master + dependsOn: + - PublishPMC + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Make sure changes are in master + jobName: MergeToMaster + instructions: | + Make sure that changes README.md and metadata.json are merged into master on GitHub. + +- stage: ReleaseToMU + displayName: Release to MU + dependsOn: PushGitTagAndMakeDraftPublic # This only needs the blob to be available + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Release to MU + instructions: | + Notify the PM team to start the process of releasing to MU. + +- stage: ReleaseClose + displayName: Finish Release + dependsOn: + - ReleaseToMU + - ReleaseSymbols + jobs: + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Retain Build + jobName: RetainBuild + instructions: | + Retain the build + + - template: /.pipelines/templates/approvalJob.yml@self + parameters: + displayName: Delete release branch + jobName: DeleteBranch + instructions: | + Delete release branch diff --git a/.pipelines/templates/stages/PowerShell-vPack-Stages.yml b/.pipelines/templates/stages/PowerShell-vPack-Stages.yml new file mode 100644 index 00000000000..f0d49e8b489 --- /dev/null +++ b/.pipelines/templates/stages/PowerShell-vPack-Stages.yml @@ -0,0 +1,236 @@ +parameters: + - name: createVPack + type: boolean + - name: vPackName + type: string + +stages: +- stage: BuildStage + jobs: + - job: BuildJob + pool: + type: windows + + strategy: + matrix: + x86: + architecture: x86 + + x64: + architecture: x64 + + arm64: + architecture: arm64 + + variables: + ArtifactPlatform: 'windows' + ob_artifactBaseName: drop_build_$(architecture) + ob_outputDirectory: '$(BUILD.SOURCESDIRECTORY)\out' + ob_createvpack_enabled: ${{ parameters.createVPack }} + ob_createvpack_owneralias: tplunk + ob_createvpack_versionAs: parts + ob_createvpack_propsFile: true + ob_createvpack_verbose: true + ob_createvpack_packagename: '${{ parameters.vPackName }}.$(architecture)' + ob_createvpack_description: PowerShell $(architecture) $(version) + # I think the variables reload after we transition back to the host so this works. 🤷‍♂️ + ob_createvpack_majorVer: $(pwshMajorVersion) + ob_createvpack_minorVer: $(pwshMinorVersion) + ob_createvpack_patchVer: $(pwshPatchVersion) + ${{ if ne(variables['pwshPrereleaseVersion'], '') }}: + ob_createvpack_prereleaseVer: $(pwshPrereleaseVersion) + ${{ else }}: + ob_createvpack_prereleaseVer: $(Build.SourceVersion) + + steps: + - checkout: self + displayName: Checkout source code - during restore + clean: true + path: s + env: + ob_restore_phase: true + + - template: .pipelines/templates/SetVersionVariables.yml@self + parameters: + ReleaseTagVar: $(ReleaseTagVar) + CreateJson: yes + + - pwsh: | + $version = '$(Version)' + Write-Verbose -Verbose "Version: $version" + if(!$version) { + throw "Version is not set." + } + + $mainVersionParts = $version -split '-' + + Write-Verbose -Verbose "mainVersionParts: $($mainVersionParts[0]) ; $($mainVersionParts[1])" + $versionParts = $mainVersionParts[0] -split '[.]'; + $major = $versionParts[0] + $minor = $versionParts[1] + $patch = $versionParts[2] + + $previewPart = $mainVersionParts[1] + Write-Verbose -Verbose "previewPart: $previewPart" + + Write-Host "major: $major; minor: $minor; patch: $patch;" + + $vstsCommandString = "vso[task.setvariable variable=pwshMajorVersion]$major" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + + $vstsCommandString = "vso[task.setvariable variable=pwshMinorVersion]$minor" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + + $vstsCommandString = "vso[task.setvariable variable=pwshPatchVersion]$patch" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + if($previewPart) { + $vstsCommandString = "vso[task.setvariable variable=pwshPrereleaseVersion]$previewPart" + } else { + Write-Verbose -Verbose "No prerelease part found in version string." + } + displayName: Set ob_createvpack_*Ver + env: + ob_restore_phase: true + + # Validate pwsh*Version variables + - pwsh: | + $variables = @("pwshMajorVersion", "pwshMinorVersion", "pwshPatchVersion") + foreach ($var in $variables) { + if (-not (get-item "Env:\$var" -ErrorAction SilentlyContinue).value) { + throw "Required variable '`$env:$var' is not set." + } + } + displayName: Validate pwsh*Version variables + env: + ob_restore_phase: true + + - pwsh: | + if($env:RELEASETAGVAR -match '-') { + throw "Don't release a preview build without coordinating with Windows Engineering Build Tools Team" + } + displayName: Stop any preview release + env: + ob_restore_phase: true + + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk' + inputs: + packageType: sdk + version: 3.1.x + installationPath: $(Agent.ToolsDirectory)/dotnet + + ### BUILD ### + + - template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self + parameters: + repoRoot: $(repoRoot) + + - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. + env: + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + inputs: + Enabled: true + AnalyzeInPipeline: false # Do not upload results + Language: csharp + + - task: UseDotNet@2 + displayName: 'Install .NET based on global.json' + inputs: + useGlobalJson: true + workingDirectory: $(repoRoot) + env: + ob_restore_phase: true + + - pwsh: | + # Need to set PowerShellRoot variable for obp-file-signing template + $vstsCommandString = "vso[task.setvariable variable=PowerShellRoot]$(repoRoot)" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + + $Architecture = '$(Architecture)' + $runtime = switch ($Architecture) + { + "x64" { "win7-x64" } + "x86" { "win7-x86" } + "arm64" { "win-arm64" } + } + + $params = @{} + if ($env:BuildConfiguration -eq 'minSize') { + $params['ForMinimalSize'] = $true + } + + $vstsCommandString = "vso[task.setvariable variable=Runtime]$runtime" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + + Write-Verbose -Message "Building PowerShell with Runtime: $runtime for '$env:BuildConfiguration' configuration" + Import-Module -Name $(repoRoot)/build.psm1 -Force + $buildWithSymbolsPath = New-Item -ItemType Directory -Path "$(Pipeline.Workspace)/Symbols_$Architecture" -Force + + Start-PSBootstrap -Scenario Package + $null = New-Item -ItemType Directory -Path $buildWithSymbolsPath -Force -Verbose + + $ReleaseTagParam = @{} + + if ($env:RELEASETAGVAR) { + $ReleaseTagParam['ReleaseTag'] = $env:RELEASETAGVAR + } + + Start-PSBuild -Runtime $runtime -Configuration Release -Output $buildWithSymbolsPath -Clean -PSModuleRestore @params @ReleaseTagParam + + $refFolderPath = Join-Path $buildWithSymbolsPath 'ref' + Write-Verbose -Verbose "refFolderPath: $refFolderPath" + $outputPath = Join-Path '$(ob_outputDirectory)' 'psoptions' + $null = New-Item -ItemType Directory -Path $outputPath -Force + $psOptPath = "$outputPath/psoptions.json" + Save-PSOptions -PSOptionsPath $psOptPath + + Write-Verbose -Verbose "Completed building PowerShell for '$env:BuildConfiguration' configuration" + displayName: Build Windows Universal - $(Architecture) -$(BuildConfiguration) Symbols folder + env: + __DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY) + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + + - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. + env: + ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(repoRoot)\src' + ob_restore_phase: true + + - template: /.pipelines/templates/obp-file-signing.yml@self + parameters: + binPath: '$(Pipeline.Workspace)/Symbols_$(Architecture)' + SigningProfile: $(windows_build_tools_cert_id) + OfficialBuild: false + vPackScenario: true + + ### END OF BUILD ### + + - pwsh: | + Get-ChildItem env:/ob_createvpack_*Ver + Get-ChildItem -Path "$(Pipeline.Workspace)\Symbols_$(Architecture)\*" -Recurse + Get-Content "$(Pipeline.Workspace)\PowerShell\preview.json" -ErrorAction SilentlyContinue | Write-Host + displayName: Debug Output Directory and Version + condition: succeededOrFailed() + + - pwsh: | + Get-ChildItem -Path env: | Out-String -width 9999 -Stream | write-Verbose -Verbose + displayName: Capture Environment + condition: succeededOrFailed() + + - pwsh: | + $vpackFiles = Get-ChildItem -Path "$(Pipeline.Workspace)\Symbols_$(Architecture)\*" -Recurse + if($vpackFiles.Count -eq 0) { + throw "No files found in $(Pipeline.Workspace)\Symbols_$(Architecture)" + } + $vpackFiles + displayName: Debug Output Directory and Version + condition: succeededOrFailed() diff --git a/.pipelines/templates/variables/PowerShell-Coordinated_Packages-Variables.yml b/.pipelines/templates/variables/PowerShell-Coordinated_Packages-Variables.yml new file mode 100644 index 00000000000..de3ac0ba1b6 --- /dev/null +++ b/.pipelines/templates/variables/PowerShell-Coordinated_Packages-Variables.yml @@ -0,0 +1,67 @@ +parameters: + - name: InternalSDKBlobURL + type: string + default: ' ' + - name: ReleaseTagVar + type: string + default: 'fromBranch' + - name: SKIP_SIGNING + type: string + default: 'NO' + - name: ENABLE_MSBUILD_BINLOGS + type: boolean + default: false + - name: FORCE_CODEQL + type: boolean + default: false + +variables: + - name: PS_RELEASE_BUILD + value: 1 + - name: DOTNET_CLI_TELEMETRY_OPTOUT + value: 1 + - name: POWERSHELL_TELEMETRY_OPTOUT + value: 1 + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - name: branchCounterKey + value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])] + - name: branchCounter + value: $[counter(variables['branchCounterKey'], 1)] + - name: BUILDSECMON_OPT_IN + value: true + - name: __DOTNET_RUNTIME_FEED + value: ${{ parameters.InternalSDKBlobURL }} + - name: LinuxContainerImage + value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 + - name: WindowsContainerImage + value: onebranch.azurecr.io/windows/ltsc2019/vse2022:latest + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] + - name: ReleaseTagVar + value: ${{ parameters.ReleaseTagVar }} + - name: SKIP_SIGNING + value: ${{ parameters.SKIP_SIGNING }} + - group: mscodehub-feed-read-general + - group: mscodehub-feed-read-akv + - name: ENABLE_MSBUILD_BINLOGS + value: ${{ parameters.ENABLE_MSBUILD_BINLOGS }} + - ${{ if eq(parameters['FORCE_CODEQL'],'true') }}: + # Cadence is hours before CodeQL will allow a re-upload of the database + - name: CodeQL.Cadence + value: 1 + - name: CODEQL_ENABLED + ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), eq(parameters['FORCE_CODEQL'],'true')) }}: + value: true + ${{ else }}: + value: false + # Fix for BinSkim ICU package error in Linux containers + - name: DOTNET_SYSTEM_GLOBALIZATION_INVARIANT + value: true + # Disable BinSkim at job level to override NonOfficial template defaults + - name: ob_sdl_binskim_enabled + value: false diff --git a/.pipelines/templates/variables/PowerShell-Packages-Variables.yml b/.pipelines/templates/variables/PowerShell-Packages-Variables.yml new file mode 100644 index 00000000000..7d1818909b5 --- /dev/null +++ b/.pipelines/templates/variables/PowerShell-Packages-Variables.yml @@ -0,0 +1,50 @@ +parameters: + - name: debug + type: boolean + default: false + - name: ForceAzureBlobDelete + type: string + default: 'false' + - name: ReleaseTagVar + type: string + default: 'fromBranch' + - name: disableNetworkIsolation + type: boolean + default: false + +variables: + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] # needed for onebranch.pipeline.version task + - name: system.debug + value: ${{ parameters.debug }} + - name: ENABLE_PRS_DELAYSIGN + value: 1 + - name: ROOT + value: $(Build.SourcesDirectory) + - name: ForceAzureBlobDelete + value: ${{ parameters.ForceAzureBlobDelete }} + - name: NUGET_XMLDOC_MODE + value: none + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - name: ReleaseTagVar + value: ${{ parameters.ReleaseTagVar }} + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: WindowsContainerImage + value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' # Docker image which is used to build the project + - name: LinuxContainerImage + value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 + - group: mscodehub-feed-read-general + - group: mscodehub-feed-read-akv + - name: branchCounterKey + value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])] + - name: branchCounter + value: $[counter(variables['branchCounterKey'], 1)] + - group: MSIXSigningProfile + - name: disableNetworkIsolation + value: ${{ parameters.disableNetworkIsolation }} diff --git a/.pipelines/templates/variables/PowerShell-Release-Azure-Variables.yml b/.pipelines/templates/variables/PowerShell-Release-Azure-Variables.yml new file mode 100644 index 00000000000..3b47e5eff2b --- /dev/null +++ b/.pipelines/templates/variables/PowerShell-Release-Azure-Variables.yml @@ -0,0 +1,35 @@ +parameters: + - name: debug + type: boolean + default: false + +variables: + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] + - name: system.debug + value: ${{ parameters.debug }} + - name: ENABLE_PRS_DELAYSIGN + value: 1 + - name: ROOT + value: $(Build.SourcesDirectory) + - name: REPOROOT + value: $(Build.SourcesDirectory) + - name: OUTPUTROOT + value: $(REPOROOT)\out + - name: NUGET_XMLDOC_MODE + value: none + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: ob_sdl_tsa_configFile + value: $(Build.SourcesDirectory)\.config\tsaoptions.json + - name: WindowsContainerImage + value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + - name: LinuxContainerImage + value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 + - group: PoolNames diff --git a/.pipelines/templates/variables/PowerShell-Release-Variables.yml b/.pipelines/templates/variables/PowerShell-Release-Variables.yml new file mode 100644 index 00000000000..930c559eafe --- /dev/null +++ b/.pipelines/templates/variables/PowerShell-Release-Variables.yml @@ -0,0 +1,41 @@ +parameters: + - name: debug + type: boolean + default: false + - name: ReleaseTagVar + type: string + default: 'fromBranch' + +variables: + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] + - name: system.debug + value: ${{ parameters.debug }} + - name: ENABLE_PRS_DELAYSIGN + value: 1 + - name: ROOT + value: $(Build.SourcesDirectory) + - name: REPOROOT + value: $(Build.SourcesDirectory) + - name: OUTPUTROOT + value: $(REPOROOT)\out + - name: NUGET_XMLDOC_MODE + value: none + - name: nugetMultiFeedWarnLevel + value: none + - name: NugetSecurityAnalysisWarningLevel + value: none + - name: skipNugetSecurityAnalysis + value: true + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' + - name: WindowsContainerImage + value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + - name: LinuxContainerImage + value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 + - name: ReleaseTagVar + value: ${{ parameters.ReleaseTagVar }} + - group: PoolNames + # Fix for BinSkim ICU package error in Linux containers + - name: DOTNET_SYSTEM_GLOBALIZATION_INVARIANT + value: true diff --git a/.pipelines/templates/variables/PowerShell-vPack-Variables.yml b/.pipelines/templates/variables/PowerShell-vPack-Variables.yml new file mode 100644 index 00000000000..276911a35b3 --- /dev/null +++ b/.pipelines/templates/variables/PowerShell-vPack-Variables.yml @@ -0,0 +1,39 @@ +parameters: + - name: debug + type: boolean + default: false + - name: ReleaseTagVar + type: string + default: 'fromBranch' + - name: netiso + type: string + default: 'R1' + +variables: + - name: CDP_DEFINITION_BUILD_COUNT + value: $[counter('', 0)] + - name: system.debug + value: ${{ parameters.debug }} + - name: BuildSolution + value: $(Build.SourcesDirectory)\dirs.proj + - name: BuildConfiguration + value: Release + - name: WindowsContainerImage + value: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' + - name: Codeql.Enabled + value: false # pipeline is not building artifacts; it repackages existing artifacts into a vpack + - name: DOTNET_CLI_TELEMETRY_OPTOUT + value: 1 + - name: POWERSHELL_TELEMETRY_OPTOUT + value: 1 + - name: nugetMultiFeedWarnLevel + value: none + - name: ReleaseTagVar + value: ${{ parameters.ReleaseTagVar }} + - group: Azure Blob variable group + - group: certificate_logical_to_actual # used within signing task + - group: DotNetPrivateBuildAccess + - name: netiso + value: ${{ parameters.netiso }} +# We shouldn't be using PATs anymore +# - group: mscodehub-feed-read-general diff --git a/.pipelines/templates/variable/release-shared.yml b/.pipelines/templates/variables/release-shared.yml similarity index 100% rename from .pipelines/templates/variable/release-shared.yml rename to .pipelines/templates/variables/release-shared.yml