diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index be7de6d8fc5..224c73eb60f 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -24,14 +24,14 @@ pr: include: - '*' exclude: - - test/common/markdown/* - - tools/releaseBuild/* - - tools/releaseBuild/azureDevOps/templates/* - - .vsts-ci/misc-analysis.yml - - .github/ISSUE_TEMPLATE/* - .dependabot/config.yml + - .github/ISSUE_TEMPLATE/* + - .vsts-ci/misc-analysis.yml - .vsts-ci/windows.yml - .vsts-ci/windows/* + - test/common/markdown/* + - tools/releaseBuild/* + - tools/releaseBuild/azureDevOps/templates/* variables: DOTNET_CLI_TELEMETRY_OPTOUT: 1 diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml index 5f487275bf2..6b050122ff8 100644 --- a/.vsts-ci/mac.yml +++ b/.vsts-ci/mac.yml @@ -25,14 +25,15 @@ pr: include: - '*' exclude: - - test/common/markdown/* - - .vsts-ci/misc-analysis.yml - - .github/ISSUE_TEMPLATE/* - .dependabot/config.yml - - tools/releaseBuild/* - - tools/releaseBuild/azureDevOps/templates/* + - .github/ISSUE_TEMPLATE/* + - .vsts-ci/misc-analysis.yml - /.vsts-ci/windows.yml - /.vsts-ci/windows/* + - test/common/markdown/* + - tools/packaging/* + - tools/releaseBuild/* + - tools/releaseBuild/azureDevOps/templates/* variables: DOTNET_CLI_TELEMETRY_OPTOUT: 1 diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index bdaa015832e..ac6d350afaa 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -24,12 +24,13 @@ pr: include: - '*' exclude: - - .vsts-ci/misc-analysis.yml - - .github/ISSUE_TEMPLATE/* - .dependabot/config.yml + - .github/ISSUE_TEMPLATE/* + - .vsts-ci/misc-analysis.yml + - test/common/markdown/* + - tools/packaging/* - tools/releaseBuild/* - tools/releaseBuild/azureDevOps/templates/* - - test/common/markdown/* variables: GIT_CONFIG_PARAMETERS: "'core.autocrlf=false'" diff --git a/tools/ci.psm1 b/tools/ci.psm1 index 41e972b9bcf..95bacf612c4 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -502,7 +502,11 @@ function Invoke-CIFinish # the packaging tests find the MSI package using env:PSMsiX64Path $env:PSMsiX64Path = $artifacts | Where-Object { $_.EndsWith(".msi")} - $env:PSExePath = $artifacts | Where-Object { $_.EndsWith(".exe") } + $architechture = $Runtime.Split('-')[1] + $exePath = New-ExePackage -ProductVersion ($preReleaseVersion -replace '^v') -ProductTargetArchitecture $architechture -MsiLocationPath $env:PSMsiX64Path + Write-Verbose "exe Path: $exePath" -Verbose + $artifacts.Add($exePath) + $env:PSExePath = $exePath $env:PSMsiChannel = $Channel $env:PSMsiRuntime = $Runtime diff --git a/tools/packaging/packaging.psd1 b/tools/packaging/packaging.psd1 index 2a1df399f13..00f4bc90d95 100644 --- a/tools/packaging/packaging.psd1 +++ b/tools/packaging/packaging.psd1 @@ -1,12 +1,26 @@ @{ -GUID="41857994-4283-4757-a932-0b0edb104913" -Author="PowerShell" -CompanyName="Microsoft Corporation" -Copyright="Copyright (c) Microsoft Corporation." -ModuleVersion="1.0.0" -PowerShellVersion="5.0" -CmdletsToExport=@() -FunctionsToExport=@('Start-PSPackage','New-PSSignedBuildZip', 'New-PSBuildZip', 'New-MSIPatch', 'Expand-PSSignedBuild', 'Publish-NugetToMyGet', 'New-DotnetSdkContainerFxdPackage', 'New-GlobalToolNupkg', 'New-ILNugetPackage', 'Update-PSSignedBuildFolder') -RootModule="packaging.psm1" -RequiredModules = @("build") + GUID = "41857994-4283-4757-a932-0b0edb104913" + Author = "PowerShell" + CompanyName = "Microsoft Corporation" + Copyright = "Copyright (c) Microsoft Corporation." + ModuleVersion = "1.0.0" + PowerShellVersion = "5.0" + CmdletsToExport = @() + FunctionsToExport = @( + 'Expand-ExePackageEngine' + 'Expand-PSSignedBuild' + 'Compress-ExePackageEngine' + 'New-DotnetSdkContainerFxdPackage' + 'New-ExePackage' + 'New-GlobalToolNupkg' + 'New-ILNugetPackage' + 'New-MSIPatch' + 'New-PSBuildZip' + 'New-PSSignedBuildZip' + 'Publish-NugetToMyGet' + 'Start-PSPackage' + 'Update-PSSignedBuildFolder' + ) + RootModule = "packaging.psm1" + RequiredModules = @("build") } diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index e9ede940673..130f7fb5a0f 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -2832,14 +2832,16 @@ function Get-WixPath $wixPyroExePath = Join-Path $wixToolsetBinPath "pyro.exe" $wixCandleExePath = Join-Path $wixToolsetBinPath "Candle.exe" $wixLightExePath = Join-Path $wixToolsetBinPath "Light.exe" + $wixInsigniaExePath = Join-Path $wixToolsetBinPath "Insignia.exe" return [PSCustomObject] @{ - WixHeatExePath = $wixHeatExePath - WixMeltExePath = $wixMeltExePath - WixTorchExePath = $wixTorchExePath - WixPyroExePath = $wixPyroExePath - WixCandleExePath = $wixCandleExePath - WixLightExePath = $wixLightExePath + WixHeatExePath = $wixHeatExePath + WixMeltExePath = $wixMeltExePath + WixTorchExePath = $wixTorchExePath + WixPyroExePath = $wixPyroExePath + WixCandleExePath = $wixCandleExePath + WixLightExePath = $wixLightExePath + WixInsigniaExePath = $wixInsigniaExePath } } @@ -3046,8 +3048,15 @@ function New-MSIPackage $wixPaths = Get-WixPath - $ProductSemanticVersion = Get-PackageSemanticVersion -Version $ProductVersion - $ProductVersion = Get-PackageVersionAsMajorMinorBuildRevision -Version $ProductVersion + $windowsNames = Get-WindowsNames -ProductName $ProductName -ProductNameSuffix $ProductNameSuffix -ProductVersion $ProductVersion + $productSemanticVersionWithName = $windowsNames.ProductSemanticVersionWithName + $ProductSemanticVersion = $windowsNames.ProductSemanticVersion + $packageName = $windowsNames.PackageName + $ProductVersion = $windowsNames.ProductVersion + Write-Verbose "Create MSI for Product $productSemanticVersionWithName" -Verbose + Write-Verbose "ProductSemanticVersion = $productSemanticVersion" -Verbose + Write-Verbose "packageName = $packageName" -Verbose + Write-Verbose "ProductVersion = $ProductVersion" -Verbose $simpleProductVersion = [string]([Version]$ProductVersion).Major $isPreview = Test-IsPreview -Version $ProductSemanticVersion @@ -3068,10 +3077,7 @@ function New-MSIPackage Write-Verbose "Place dependencies such as icons to $assetsInSourcePath" Copy-Item "$AssetsPath\*.ico" $assetsInSourcePath -Force - $productVersionWithName = $ProductName + '_' + $ProductVersion - $productSemanticVersionWithName = $ProductName + '-' + $ProductSemanticVersion - Write-Verbose "Create MSI for Product $productSemanticVersionWithName" $fileArchitecture = 'amd64' $ProductProgFilesDir = "ProgramFiles64Folder" @@ -3086,11 +3092,6 @@ function New-MSIPackage # cleanup any garbage on the system Remove-Item -ErrorAction SilentlyContinue $wixFragmentPath -Force - $packageName = $productSemanticVersionWithName - if ($ProductNameSuffix) { - $packageName += "-$ProductNameSuffix" - } - $msiLocationPath = Join-Path $CurrentLocation "$packageName.msi" $msiPdbLocationPath = Join-Path $CurrentLocation "$packageName.wixpdb" @@ -3150,6 +3151,85 @@ function New-MSIPackage $errorMessage = "Failed to create $msiLocationPath" throw $errorMessage } +} + +function Get-WindowsNames { + param( + # Name of the Product + [ValidateNotNullOrEmpty()] + [string] $ProductName = 'PowerShell', + + # Suffix of the Name + [string] $ProductNameSuffix, + + # Version of the Product + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string] $ProductVersion + ) + + Write-Verbose -Message "Getting Windows Names for ProductName: $ProductName; ProductNameSuffix: $ProductNameSuffix; ProductVersion: $ProductVersion" -Verbose + + $ProductSemanticVersion = Get-PackageSemanticVersion -Version $ProductVersion + $ProductVersion = Get-PackageVersionAsMajorMinorBuildRevision -Version $ProductVersion + + $productVersionWithName = $ProductName + '_' + $ProductVersion + $productSemanticVersionWithName = $ProductName + '-' + $ProductSemanticVersion + + $packageName = $productSemanticVersionWithName + if ($ProductNameSuffix) { + $packageName += "-$ProductNameSuffix" + } + + return [PSCustomObject]@{ + PackageName = $packageName + ProductVersionWithName = $productVersionWithName + ProductSemanticVersion = $ProductSemanticVersion + ProductSemanticVersionWithName = $productSemanticVersionWithName + ProductVersion = $ProductVersion + } +} + +function New-ExePackage { + param( + # Name of the Product + [ValidateNotNullOrEmpty()] + [string] $ProductName = 'PowerShell', + + # Version of the Product + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + + [string] $ProductVersion, + + # File describing the MSI Package creation semantics + [ValidateNotNullOrEmpty()] + [ValidateScript({Test-Path $_})] + [string] $BundleWxsPath = "$RepoRoot\assets\wix\bundle.wxs", + + # Architecture to use when creating the MSI + [Parameter(Mandatory = $true)] + [ValidateSet("x86", "x64")] + [ValidateNotNullOrEmpty()] + [string] $ProductTargetArchitecture, + + # Location of the signed MSI + [Parameter(Mandatory = $true)] + [string] + $MsiLocationPath, + + [string] $CurrentLocation = (Get-Location) + ) + + $productNameSuffix = "win-$ProductTargetArchitecture" + + $windowsNames = Get-WindowsNames -ProductName $ProductName -ProductNameSuffix $productNameSuffix -ProductVersion $ProductVersion + $productSemanticVersionWithName = $windowsNames.ProductSemanticVersionWithName + $packageName = $windowsNames.PackageName + $isPreview = Test-IsPreview -Version $windowsNames.ProductSemanticVersion + + Write-Verbose "Create EXE for Product $productSemanticVersionWithName" -verbose + Write-Verbose "packageName = $packageName" -Verbose $exeLocationPath = Join-Path $CurrentLocation "$packageName.exe" $exePdbLocationPath = Join-Path $CurrentLocation "$packageName.exe.wixpdb" @@ -3157,20 +3237,72 @@ function New-MSIPackage Start-MsiBuild -WxsFile $BundleWxsPath -ProductTargetArchitecture $ProductTargetArchitecture -Argument @{ IsPreview = $isPreview - TargetPath = $msiLocationPath + TargetPath = $MsiLocationPath WindowsVersion = $windowsVersion } -MsiLocationPath $exeLocationPath -MsiPdbLocationPath $exePdbLocationPath - if (Test-Path $exeLocationPath) - { - Write-Verbose "You can find the MSI @ $exeLocationPath" -Verbose - $exeLocationPath - } - else - { - $errorMessage = "Failed to create $exeLocationPath" - throw $errorMessage - } + return $exeLocationPath +} + +<# +Allows you to extract the engine of exe package, mainly for signing +Any existing signature will be removed. + #> +function Expand-ExePackageEngine { + param( + # Location of the unsigned EXE + [Parameter(Mandatory = $true)] + [string] + $ExePath, + + # Location to put the expanded engine. + [Parameter(Mandatory = $true)] + [string] + $EnginePath + ) + + <# + 2. detach the engine from TestInstaller.exe: + insignia -ib TestInstaller.exe -o engine.exe + #> + + $wixPaths = Get-WixPath + + $resolvedExePath = (Resolve-Path -Path $ExePath).ProviderPath + $resolvedEnginePath = [System.IO.Path]::GetFullPath($EnginePath) + + Start-NativeExecution -VerboseOutputOnError { & $wixPaths.wixInsigniaExePath -ib $resolvedExePath -o $resolvedEnginePath} +} + +<# +Allows you to replace the engine (installer) in the exe package. +Used to replace the engine with a signed version +#> +function Compress-ExePackageEngine { + param( + # Location of the unsigned EXE + [Parameter(Mandatory = $true)] + [string] + $ExePath, + + # Location of the signed engine + [Parameter(Mandatory = $true)] + [string] + $EnginePath + ) + + + <# + 4. re-attach the signed engine.exe to the bundle: + insignia -ab engine.exe TestInstaller.exe -o TestInstaller.exe + #> + + $wixPaths = Get-WixPath + + $resolvedEnginePath = (Resolve-Path -Path $EnginePath).ProviderPath + $resolvedExePath = (Resolve-Path -Path $ExePath).ProviderPath + + Start-NativeExecution -VerboseOutputOnError { & $wixPaths.wixInsigniaExePath -ab $resolvedEnginePath $resolvedExePath -o $resolvedExePath} } function New-MsiArgsArray { diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 5954b080bf7..3760d9ff698 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -161,6 +161,7 @@ jobs: **\*.rpm useMinimatch: true shouldSign: $(SHOULD_SIGN) + displayName: Sign RPM # requires windows - task: AzureFileCopy@4 diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index e0a11d36a06..7c1ed7e1431 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -78,6 +78,7 @@ jobs: **\*.zip useMinimatch: true shouldSign: $(SHOULD_SIGN) + displayName: Sign macOS Binaries - pwsh: | $destination = "$(System.ArtifactsDirectory)\azureMacOs" diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index 529e6e48356..a10a908afcb 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -64,6 +64,7 @@ jobs: **\*.zip useMinimatch: true shouldSign: $(SHOULD_SIGN) + displayName: Sign pkg - template: upload-final-results.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index e606981f282..e7e8b84a4b7 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -144,6 +144,7 @@ jobs: **\*.nupkg useMinimatch: true shouldSign: $(SHOULD_SIGN) + displayName: Sign NuPkg - pwsh: | if (-not (Test-Path '$(System.ArtifactsDirectory)\signed\')) { $null = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)\signed\' } diff --git a/tools/releaseBuild/azureDevOps/templates/upload.yml b/tools/releaseBuild/azureDevOps/templates/upload.yml index b1d75ad7f99..3f121e2f51f 100644 --- a/tools/releaseBuild/azureDevOps/templates/upload.yml +++ b/tools/releaseBuild/azureDevOps/templates/upload.yml @@ -6,23 +6,6 @@ parameters: pdb: no steps: -- template: upload-final-results.yml - parameters: - artifactPath: $(Build.StagingDirectory)\signedPackages - artifactFilter: PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}.msi - condition: and(succeeded(), eq('${{ parameters.msi }}', 'yes')) - -- task: AzureFileCopy@4 - displayName: 'upload signed msi to Azure - ${{ parameters.architecture }}' - inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-${{ parameters.version }}-win-${{ parameters.architecture }}.msi' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - resourceGroup: '$(StorageResourceGroup)' - condition: and(succeeded(), eq('${{ parameters.msi }}', 'yes')) - - template: upload-final-results.yml parameters: artifactPath: $(System.ArtifactsDirectory)\signed diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index e1b5e378def..df9c7c8c091 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -46,11 +46,11 @@ jobs: signOutputPath: $(Build.StagingDirectory)\signedPackages certificateId: "CP-230012" pattern: | - **\*.msi **\*.msix **\*.exe useMinimatch: true shouldSign: $(SHOULD_SIGN) + displayName: Sign exe and msix - powershell: | new-item -itemtype Directory -path '$(Build.StagingDirectory)\signedPackages' diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index 15168d0aca6..18247fb982d 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -155,6 +155,7 @@ jobs: **\*.exe useMinimatch: true shouldSign: $(SHOULD_SIGN) + displayName: Sign our binaries - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force @@ -198,10 +199,11 @@ jobs: **\*.dll useMinimatch: true shouldSign: $(SHOULD_SIGN) + displayName: Sign ThirdParty binaries - powershell: | Get-ChildItem '$(System.ArtifactsDirectory)\thirdPartySigned\*' - displayName: Captrue ThirdParty Signed files + displayName: Capture ThirdParty Signed files condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - powershell: | @@ -261,7 +263,82 @@ jobs: Write-Host "Uploading $packagePath" Write-Host "##vso[artifact.upload containerfolder=signed;artifactname=signed]$packagePath" } - displayName: Upload packages + displayName: Upload unsigned packages + + - ${{ if and(ne(variables['BuildConfiguration'],'minSize'), in(variables['Architecture'], 'x64', 'x86')) }}: + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\pkgSigned + signOutputPath: $(Build.StagingDirectory)\signedPackages + certificateId: "CP-230012" + pattern: | + **\*.msi + useMinimatch: true + shouldSign: $(SHOULD_SIGN) + displayName: Sign MSI + + - pwsh: | + Get-ChildItem '$(System.ArtifactsDirectory)\signedPackages' | ForEach-Object { + $packagePath = $_.FullName + Write-Host "Uploading $packagePath" + Write-Host "##vso[artifact.upload containerfolder=finalResults;artifactname=finalResults]$packagePath" + } + displayName: Upload signed MSI to finalResults + + - task: AzureFileCopy@4 + displayName: 'upload signed msi to Azure - ${{ parameters.architecture }}' + inputs: + SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(version)-win-${{ parameters.architecture }}.msi' + azureSubscription: '$(AzureFileCopySubscription)' + Destination: AzureBlob + storage: '$(StorageAccount)' + ContainerName: '$(AzureVersion)' + resourceGroup: '$(StorageResourceGroup)' + + - pwsh: | + cd $(PowerShellRoot) + Import-Module $(PowerShellRoot)/build.psm1 -Force + Import-Module $(PowerShellRoot)/tools/packaging -Force + + $msiPath = '$(Build.StagingDirectory)\signedPackages\PowerShell-$(version)-win-${{ parameters.architecture }}.msi' + + New-ExePackage -ProductVersion '$(version)' -MsiLocationPath $msiPath -ProductTargetArchitecture ${{ parameters.architecture }} + $exePath = Get-ChildItem '.\PowerShell-*.exe' | Select-Object -First 1 -ExpandProperty fullname + $enginePath = Join-Path -Path '$(System.ArtifactsDirectory)\unsignedEngine' -ChildPath engine.exe + # Expand Burn Engine so we can sign it. + Expand-ExePackageEngine -ExePath $exePath -EnginePath $enginePath + displayName: Create exe wrapper + + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\unsignedEngine + signOutputPath: $(System.ArtifactsDirectory)\signedEngine + certificateId: "CP-230012" + pattern: | + **\*.exe + useMinimatch: true + shouldSign: $(SHOULD_SIGN) + displayName: Sign Burn Engine + + - pwsh: | + cd '$(PowerShellRoot)' + Import-Module '$(PowerShellRoot)/build.psm1' -Force + Import-Module '$(PowerShellRoot)/tools/packaging' -Force + + $exePath = Get-ChildItem '.\PowerShell-*.exe' | Select-Object -First 1 -ExpandProperty fullname + $enginePath = Join-Path -Path '$(System.ArtifactsDirectory)\signedEngine' -ChildPath engine.exe + $enginePath | Get-AuthenticodeSignature | out-string | Write-Verbose -verbose + Compress-ExePackageEngine -ExePath $exePath -EnginePath $enginePath + displayName: Re-attach the signed Burn engine in exe wrapper + + - pwsh: | + cd '$(PowerShellRoot)' + Get-ChildItem '.\PowerShell-*.exe' | ForEach-Object { + $packagePath = $_.FullName + Write-Host "Uploading $packagePath" + Write-Host "##vso[artifact.upload containerfolder=signed;artifactname=signed]$packagePath" + } + displayName: Upload unsigned exe - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' diff --git a/tools/releaseBuild/setReleaseTag.ps1 b/tools/releaseBuild/setReleaseTag.ps1 index 1659dbc74cb..df913471590 100644 --- a/tools/releaseBuild/setReleaseTag.ps1 +++ b/tools/releaseBuild/setReleaseTag.ps1 @@ -70,7 +70,7 @@ if($ReleaseTag -eq 'fromBranch' -or !$ReleaseTag) { $msixType = 'release' Write-Verbose "release branch:" -Verbose - $releaseTag = $Branch -replace $releaseBranchRegex + $releaseTag = $Branch -replace '^.*((release|rebuild)/)' $vstsCommandString = "vso[task.setvariable variable=$Variable]$releaseTag" Write-Verbose -Message "setting $Variable to $releaseTag" -Verbose Write-Host -Object "##$vstsCommandString"