From 571858bc92fedce883a451ddb286bd685052832a Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 11 Sep 2024 14:44:21 -0700 Subject: [PATCH 01/10] Enable and Disable CodeQL in the right places during the official build --- .pipelines/templates/linux.yml | 10 ++++++---- .pipelines/templates/windows-hosted-build.yml | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.pipelines/templates/linux.yml b/.pipelines/templates/linux.yml index 2e6f9c3e5e3..08b63b17da1 100644 --- a/.pipelines/templates/linux.yml +++ b/.pipelines/templates/linux.yml @@ -33,9 +33,9 @@ jobs: value: ${{ parameters.Runtime }} - name: ob_sdl_sbom_packageName value: 'Microsoft.Powershell.Linux.${{ parameters.Runtime }}' - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}: - - name: ob_sdl_codeql_compiled_enabled - value: true + # We add this manually, so we need it disabled the OneBranch auto-injected one. + - name: ob_sdl_codeql_compiled_enabled + value: false steps: - checkout: self @@ -59,7 +59,9 @@ jobs: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. inputs: Enabled: true - AnalyzeInPipeline: true + # AnalyzeInPipeline: false = upload results + # AnalyzeInPipeline: true = do not upload results + AnalyzeInPipeline: false Language: csharp - pwsh: | diff --git a/.pipelines/templates/windows-hosted-build.yml b/.pipelines/templates/windows-hosted-build.yml index 735ed7cc48f..2787cf85ea2 100644 --- a/.pipelines/templates/windows-hosted-build.yml +++ b/.pipelines/templates/windows-hosted-build.yml @@ -34,9 +34,9 @@ jobs: value: ${{ parameters.BuildConfiguration }} - name: ob_sdl_sbom_packageName value: 'Microsoft.Powershell.Windows.${{ parameters.Architecture }}' - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}: - - name: ob_sdl_codeql_compiled_enabled - value: true + # We add this manually, so we need it disabled the OneBranch auto-injected one. + - name: ob_sdl_codeql_compiled_enabled + value: false steps: - checkout: self @@ -60,7 +60,9 @@ jobs: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. inputs: Enabled: true - AnalyzeInPipeline: true + # AnalyzeInPipeline: false = upload results + # AnalyzeInPipeline: true = do not upload results + AnalyzeInPipeline: false Language: csharp - pwsh: | From 1395da5e2f0b2329b8efe93d60450afbd694ce4f Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 11 Sep 2024 15:18:16 -0700 Subject: [PATCH 02/10] fix codeql condition --- .pipelines/templates/linux.yml | 4 ++-- .pipelines/templates/windows-hosted-build.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pipelines/templates/linux.yml b/.pipelines/templates/linux.yml index 08b63b17da1..38ec9b6a165 100644 --- a/.pipelines/templates/linux.yml +++ b/.pipelines/templates/linux.yml @@ -54,7 +54,7 @@ jobs: repoRoot: $(PowerShellRoot) - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. - condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') + condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/master'),startsWith(variables['Build.SourceBranch'], 'refs/heads/release/',startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/')) env: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. inputs: @@ -112,7 +112,7 @@ jobs: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. - condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') + condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/master'),startsWith(variables['Build.SourceBranch'], 'refs/heads/release/',startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/')) env: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. diff --git a/.pipelines/templates/windows-hosted-build.yml b/.pipelines/templates/windows-hosted-build.yml index 2787cf85ea2..42ee10e2b8e 100644 --- a/.pipelines/templates/windows-hosted-build.yml +++ b/.pipelines/templates/windows-hosted-build.yml @@ -55,7 +55,7 @@ jobs: repoRoot: $(PowerShellRoot) - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. - condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') + condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/master'),startsWith(variables['Build.SourceBranch'], 'refs/heads/release/',startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/')) env: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. inputs: @@ -186,7 +186,7 @@ jobs: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. - condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') + condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/master'),startsWith(variables['Build.SourceBranch'], 'refs/heads/release/',startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/')) env: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. From fdb3b9fa9b45f591900aa1fbd692569fb141476b Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 11 Sep 2024 15:21:59 -0700 Subject: [PATCH 03/10] fix syntax error --- .pipelines/templates/linux.yml | 12 ++++++++++-- .pipelines/templates/windows-hosted-build.yml | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.pipelines/templates/linux.yml b/.pipelines/templates/linux.yml index 38ec9b6a165..d8261207290 100644 --- a/.pipelines/templates/linux.yml +++ b/.pipelines/templates/linux.yml @@ -54,7 +54,11 @@ jobs: repoRoot: $(PowerShellRoot) - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. - condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/master'),startsWith(variables['Build.SourceBranch'], 'refs/heads/release/',startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/')) + condition: or( + eq(variables['Build.SourceBranch'], 'refs/heads/master'), + startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), + startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/') + ) env: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. inputs: @@ -112,7 +116,11 @@ jobs: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. - condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/master'),startsWith(variables['Build.SourceBranch'], 'refs/heads/release/',startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/')) + condition: or( + eq(variables['Build.SourceBranch'], 'refs/heads/master'), + startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), + startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/') + ) env: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. diff --git a/.pipelines/templates/windows-hosted-build.yml b/.pipelines/templates/windows-hosted-build.yml index 42ee10e2b8e..26321e48fcd 100644 --- a/.pipelines/templates/windows-hosted-build.yml +++ b/.pipelines/templates/windows-hosted-build.yml @@ -55,7 +55,11 @@ jobs: repoRoot: $(PowerShellRoot) - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. - condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/master'),startsWith(variables['Build.SourceBranch'], 'refs/heads/release/',startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/')) + condition: or( + eq(variables['Build.SourceBranch'], 'refs/heads/master'), + startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), + startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/') + ) env: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. inputs: @@ -186,7 +190,11 @@ jobs: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. - condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/master'),startsWith(variables['Build.SourceBranch'], 'refs/heads/release/',startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/')) + condition: or( + eq(variables['Build.SourceBranch'], 'refs/heads/master'), + startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), + startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/') + ) env: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. From 2233e1958de962f89214817c0a06ca8bd7b76c4f Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 11 Sep 2024 16:21:36 -0700 Subject: [PATCH 04/10] Enable codeQL issue filing --- .config/tsaoptions.json | 1 + .../PowerShell-Coordinated_Packages-Official.yml | 15 ++++++++++----- .pipelines/templates/linux.yml | 12 ++---------- .pipelines/templates/windows-hosted-build.yml | 12 ++---------- 4 files changed, 15 insertions(+), 25 deletions(-) diff --git a/.config/tsaoptions.json b/.config/tsaoptions.json index 7552bd7226c..786ef4331a2 100644 --- a/.config/tsaoptions.json +++ b/.config/tsaoptions.json @@ -1,4 +1,5 @@ { + "codebaseName": "TFSMSAzure_PowerShell", "instanceUrl": "https://msazure.visualstudio.com", "projectName": "One", "areaPath": "One\\MGMT\\Compute\\Powershell\\Powershell\\PowerShell Core\\pwsh", diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index eddc5283df0..c53b05bc0e7 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -74,6 +74,12 @@ variables: - group: mscodehub-feed-read-akv - name: ENABLE_MSBUILD_BINLOGS value: ${{ parameters.ENABLE_MSBUILD_BINLOGS }} + - name: CODEQL_ENABLED + ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/')) }}: + value: true + ${{ else }}: + value: false + extends: template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates @@ -91,11 +97,10 @@ extends: enabled: false sbom: enabled: true - compiled: - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}: - enabled: true - ${{ else }}: - enabled: false + codeql: + compiled: + enabled: $(CODEQL_ENABLED) + tsaEnabled: true # This enables TSA bug filing only for CodeQL 3000 credscan: enabled: true scanFolder: $(Build.SourcesDirectory) diff --git a/.pipelines/templates/linux.yml b/.pipelines/templates/linux.yml index d8261207290..7daa73f3a30 100644 --- a/.pipelines/templates/linux.yml +++ b/.pipelines/templates/linux.yml @@ -54,11 +54,7 @@ jobs: repoRoot: $(PowerShellRoot) - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. - condition: or( - eq(variables['Build.SourceBranch'], 'refs/heads/master'), - startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), - startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/') - ) + condition: eq(variables['CODEQL_ENABLED'], 'true') env: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. inputs: @@ -116,11 +112,7 @@ jobs: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. - condition: or( - eq(variables['Build.SourceBranch'], 'refs/heads/master'), - startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), - startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/') - ) + condition: eq(variables['CODEQL_ENABLED'], 'true') env: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. diff --git a/.pipelines/templates/windows-hosted-build.yml b/.pipelines/templates/windows-hosted-build.yml index 26321e48fcd..d8d5811df66 100644 --- a/.pipelines/templates/windows-hosted-build.yml +++ b/.pipelines/templates/windows-hosted-build.yml @@ -55,11 +55,7 @@ jobs: repoRoot: $(PowerShellRoot) - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. - condition: or( - eq(variables['Build.SourceBranch'], 'refs/heads/master'), - startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), - startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/') - ) + condition: eq(variables['CODEQL_ENABLED'], 'true') env: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. inputs: @@ -190,11 +186,7 @@ jobs: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. - condition: or( - eq(variables['Build.SourceBranch'], 'refs/heads/master'), - startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), - startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/') - ) + condition: eq(variables['CODEQL_ENABLED'], 'true') env: ob_restore_phase: true # Set ob_restore_phase to run this step before '🔒 Setup Signing' step. From 701ef9db3715f7371cbae220c8cad5091fec8f38 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 11 Sep 2024 16:58:00 -0700 Subject: [PATCH 05/10] use parameter instead of branch to force codeql --- .../PowerShell-Coordinated_Packages-Official.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index c53b05bc0e7..9a2412dd625 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -26,6 +26,10 @@ parameters: displayName: Enable MSBuild Binary Logs type: boolean default: false + - name: FORCE_CODEQL + displayName: Enable CodeQL and set cadence to 1 hour + type: boolean + default: false resources: repositories: @@ -74,8 +78,12 @@ variables: - group: mscodehub-feed-read-akv - name: ENABLE_MSBUILD_BINLOGS value: ${{ parameters.ENABLE_MSBUILD_BINLOGS }} + - ${{ if eq(parameters['FORCE_CODEQL'],'true') }}: + - name: CodeQL.Cadence + value: 1 + # TODO, this really should only be master - name: CODEQL_ENABLED - ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/rebuild/')) }}: + ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), eq(parameters['FORCE_CODEQL'],'true')) }}: value: true ${{ else }}: value: false From 0b116b3d4a1df7338a477086ba29bdc7dff5ba7e Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 11 Sep 2024 17:00:30 -0700 Subject: [PATCH 06/10] try to name the build better --- .pipelines/PowerShell-Coordinated_Packages-Official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index 9a2412dd625..1677374440a 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -1,4 +1,4 @@ -name: UnifiedPackageBuild-$(Build.BuildId) +name: UnifiedPackageBuild-$(BUILD.SOURCEBRANCHNAME)-$(Build.BuildId) trigger: none parameters: From 7e8fac4424c4315e27498c99c3ed8f440344ca00 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 12 Sep 2024 10:05:28 -0700 Subject: [PATCH 07/10] Update .pipelines/PowerShell-Coordinated_Packages-Official.yml --- .pipelines/PowerShell-Coordinated_Packages-Official.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index 1677374440a..695037ff803 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -79,6 +79,7 @@ variables: - name: ENABLE_MSBUILD_BINLOGS value: ${{ parameters.ENABLE_MSBUILD_BINLOGS }} - ${{ if eq(parameters['FORCE_CODEQL'],'true') }}: + # Cadence is hours before CodeQL will allow a re-upload of the database - name: CodeQL.Cadence value: 1 # TODO, this really should only be master From 47c87962177dc6547c80ee65b82ba61bf38c2ea2 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 12 Sep 2024 10:07:46 -0700 Subject: [PATCH 08/10] add debugging to display name of debugging parameters --- .../PowerShell-Coordinated_Packages-Official.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index 695037ff803..90b43f65c3d 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -11,23 +11,23 @@ parameters: type: string default: 'fromBranch' - name: SKIP_SIGNING - displayName: Skip Signing + displayName: Debugging - Skip Signing type: string default: 'NO' - name: RUN_TEST_AND_RELEASE - displayName: Run Test and Release Artifacts Stage + displayName: Debugging - Run Test and Release Artifacts Stage type: boolean default: true - name: RUN_WINDOWS - displayName: Enable Windows Stage + displayName: Debugging - Enable Windows Stage type: boolean default: true - name: ENABLE_MSBUILD_BINLOGS - displayName: Enable MSBuild Binary Logs + displayName: Debugging - Enable MSBuild Binary Logs type: boolean default: false - name: FORCE_CODEQL - displayName: Enable CodeQL and set cadence to 1 hour + displayName: Debugging - Enable CodeQL and set cadence to 1 hour type: boolean default: false From da584c002a16c67c88a42913fd4ec159d46dbd7d Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 12 Sep 2024 10:08:58 -0700 Subject: [PATCH 09/10] fix spacing --- .pipelines/PowerShell-Coordinated_Packages-Official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index 90b43f65c3d..58380f70ba6 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -79,7 +79,7 @@ variables: - name: ENABLE_MSBUILD_BINLOGS value: ${{ parameters.ENABLE_MSBUILD_BINLOGS }} - ${{ if eq(parameters['FORCE_CODEQL'],'true') }}: - # Cadence is hours before CodeQL will allow a re-upload of the database + # Cadence is hours before CodeQL will allow a re-upload of the database - name: CodeQL.Cadence value: 1 # TODO, this really should only be master From 43fee4c913f7446375d0d473adb75e3ecfb4481a Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 12 Sep 2024 10:10:43 -0700 Subject: [PATCH 10/10] Update .pipelines/PowerShell-Coordinated_Packages-Official.yml --- .pipelines/PowerShell-Coordinated_Packages-Official.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/PowerShell-Coordinated_Packages-Official.yml b/.pipelines/PowerShell-Coordinated_Packages-Official.yml index 58380f70ba6..a050300b1f5 100644 --- a/.pipelines/PowerShell-Coordinated_Packages-Official.yml +++ b/.pipelines/PowerShell-Coordinated_Packages-Official.yml @@ -82,7 +82,6 @@ variables: # Cadence is hours before CodeQL will allow a re-upload of the database - name: CodeQL.Cadence value: 1 - # TODO, this really should only be master - name: CODEQL_ENABLED ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), eq(parameters['FORCE_CODEQL'],'true')) }}: value: true