diff --git a/build.psm1 b/build.psm1 index db3bdb5fae7..2b3b414c1ef 100644 --- a/build.psm1 +++ b/build.psm1 @@ -33,14 +33,21 @@ function Sync-PSTags $AddRemoteIfMissing ) - $PowerShellRemoteUrl = "https://github.com/PowerShell/PowerShell.git" + $powerShellRemoteUrls = @( + 'https://github.com/PowerShell/PowerShell' + 'git@github.com:PowerShell/PowerShell' + ) + $defaultRemoteUrl = "$($powerShellRemoteUrls[0]).git" + $upstreamRemoteDefaultName = 'upstream' $remotes = Start-NativeExecution {git --git-dir="$PSScriptRoot/.git" remote} $upstreamRemote = $null foreach($remote in $remotes) { $url = Start-NativeExecution {git --git-dir="$PSScriptRoot/.git" remote get-url $remote} - if($url -eq $PowerShellRemoteUrl) + if ($url.EndsWith('.git')) { $url = $url.Substring(0, $url.Length - 4) } + + if($url -in $powerShellRemoteUrls) { $upstreamRemote = $remote break @@ -49,12 +56,12 @@ function Sync-PSTags if(!$upstreamRemote -and $AddRemoteIfMissing.IsPresent -and $remotes -notcontains $upstreamRemoteDefaultName) { - $null = Start-NativeExecution {git --git-dir="$PSScriptRoot/.git" remote add $upstreamRemoteDefaultName $PowerShellRemoteUrl} + $null = Start-NativeExecution {git --git-dir="$PSScriptRoot/.git" remote add $upstreamRemoteDefaultName $defaultRemoteUrl} $upstreamRemote = $upstreamRemoteDefaultName } elseif(!$upstreamRemote) { - Write-Error "Please add a remote to PowerShell\PowerShell. Example: git remote add $upstreamRemoteDefaultName $PowerShellRemoteUrl" -ErrorAction Stop + Write-Error "Please add a remote to PowerShell\PowerShell. Example: git remote add $upstreamRemoteDefaultName $defaultRemoteUrl" -ErrorAction Stop } $null = Start-NativeExecution {git --git-dir="$PSScriptRoot/.git" fetch --tags --quiet $upstreamRemote} @@ -135,6 +142,10 @@ function Get-EnvironmentInformation $environment += @{'UsingHomebrew' = [bool](Get-Command brew -ErrorAction ignore)} $environment += @{'UsingMacports' = [bool](Get-Command port -ErrorAction ignore)} + $environment += @{ + 'OSArchitecture' = if ((uname -v) -match 'ARM64') { 'arm64' } else { 'x64' } + } + if (-not($environment.UsingHomebrew -or $environment.UsingMacports)) { throw "Neither Homebrew nor MacPorts is installed on this system, visit https://brew.sh/ or https://www.macports.org/ to continue" } diff --git a/tools/ci.psm1 b/tools/ci.psm1 index afdf5b81bab..31c9a7bffe3 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -16,8 +16,8 @@ if(Test-Path $dotNetPath) } # import build into the global scope so it can be used by packaging -Import-Module (Join-Path $repoRoot 'build.psm1') -Scope Global -Import-Module (Join-Path $repoRoot 'tools\packaging') -Scope Global +Import-Module (Join-Path $repoRoot 'build.psm1') -Verbose -Scope Global +Import-Module (Join-Path $repoRoot 'tools\packaging') -Verbose -Scope Global # import the windows specific functcion only in Windows PowerShell or on Windows if($PSVersionTable.PSEdition -eq 'Desktop' -or $IsWindows) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 861dfc95c24..6b9c91da5d6 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -37,6 +37,10 @@ function Start-PSPackage { [ValidateScript({$Environment.IsWindows})] [string] $WindowsRuntime, + [ValidateSet('osx-x64', 'osx-arm64')] + [ValidateScript({$Environment.IsMacOS})] + [string] $MacOSRuntime, + [Switch] $Force, [Switch] $SkipReleaseChecks, @@ -71,12 +75,18 @@ function Start-PSPackage { # Runtime and Configuration settings required by the package ($Runtime, $Configuration) = if ($WindowsRuntime) { $WindowsRuntime, "Release" + } elseif ($MacOSRuntime) { + $MacOSRuntime, "Release" } elseif ($Type -eq "tar-alpine") { New-PSOptions -Configuration "Release" -Runtime "alpine-x64" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } } elseif ($Type -eq "tar-arm") { New-PSOptions -Configuration "Release" -Runtime "Linux-ARM" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } } elseif ($Type -eq "tar-arm64") { - New-PSOptions -Configuration "Release" -Runtime "Linux-ARM64" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } + if ($IsMacOS) { + New-PSOptions -Configuration "Release" -Runtime "osx-arm64" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } + } else { + New-PSOptions -Configuration "Release" -Runtime "Linux-ARM64" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } + } } else { New-PSOptions -Configuration "Release" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } } @@ -97,6 +107,8 @@ function Start-PSPackage { } elseif ($Type -eq 'fxdependent-win-desktop') { $NameSuffix = "win-fxdependentWinDesktop" Write-Log "Packaging : '$Type'; Packaging Configuration: '$Configuration'" + } elseif ($MacOSRuntime) { + $NameSuffix = $MacOSRuntime } else { Write-Log "Packaging RID: '$Runtime'; Packaging Configuration: '$Configuration'" } @@ -417,6 +429,10 @@ function Start-PSPackage { Force = $Force } + if ($MacOSRuntime) { + $Arguments['Architecture'] = $MacOSRuntime.Split('-')[1] + } + if ($PSCmdlet.ShouldProcess("Create tar.gz Package")) { New-TarballPackage @Arguments } diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index f309431b977..ca4f098d208 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -38,18 +38,36 @@ stages: dependsOn: ['prep'] jobs: - template: templates/mac.yml + parameters: + buildArchitecture: x64 + + - template: templates/mac.yml + parameters: + buildArchitecture: arm64 + + - template: templates/mac-file-signing.yml + parameters: + buildArchitecture: x64 - template: templates/mac-file-signing.yml parameters: - parentJob: build_macOS + buildArchitecture: arm64 - template: templates/mac-package-build.yml parameters: - parentJob: MacFileSigningJob + buildArchitecture: x64 + + - template: templates/mac-package-build.yml + parameters: + buildArchitecture: arm64 + + - template: templates/mac-package-signing.yml + parameters: + buildArchitecture: x64 - template: templates/mac-package-signing.yml parameters: - parentJob: package_macOS + buildArchitecture: arm64 - stage: linux dependsOn: ['prep'] diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index 34bee8c365b..bc111a46b07 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -1,10 +1,10 @@ parameters: - parentJob: '' + buildArchitecture: 'x64' jobs: - - job: MacFileSigningJob - displayName: macOS File signing - dependsOn: ${{ parameters.parentJob }} + - job: MacFileSigningJob_${{ parameters.buildArchitecture }} + displayName: macOS File signing ${{ parameters.buildArchitecture }} + dependsOn: build_macOS_${{ parameters.buildArchitecture }} condition: succeeded() pool: name: Package ES Standard Build @@ -41,7 +41,7 @@ jobs: continueOnError: true - pwsh: | - $zipPath = Get-Item '$(System.ArtifactsDirectory)\Symbols\macosBinResults\*symbol*.zip' + $zipPath = Get-Item '$(System.ArtifactsDirectory)\Symbols\macosBinResults\*symbol*${{ parameters.buildArchitecture }}*.zip' Write-Verbose -Verbose "Zip Path: $zipPath" $expandedFolder = $zipPath.BaseName @@ -63,7 +63,7 @@ jobs: - pwsh: | $null = new-item -type directory -path "$(Build.StagingDirectory)\macos" - $zipFile = "$(Build.StagingDirectory)\macos\powershell-files-$(Version)-osx-x64.zip" + $zipFile = "$(Build.StagingDirectory)\macos\powershell-files-$(Version)-osx-${{ parameters.buildArchitecture }}.zip" Get-ChildItem "$(System.ArtifactsDirectory)\$(SymbolsFolder)" -Recurse -Include pwsh, *.dylib | Compress-Archive -Destination $zipFile Write-Host $zipFile @@ -81,7 +81,7 @@ jobs: displayName: Sign macOS Binaries - pwsh: | - $destination = "$(System.ArtifactsDirectory)\azureMacOs" + $destination = "$(System.ArtifactsDirectory)\azureMacOs_${{ parameters.buildArchitecture }}" New-Item -Path $destination -Type Directory $zipPath = Get-ChildItem "$(Build.StagingDirectory)\signedMacOSPackages\powershell-*.zip" -Recurse | select-object -expandproperty fullname foreach ($z in $zipPath) { Expand-Archive -Path $z -DestinationPath $destination } @@ -90,15 +90,15 @@ jobs: - template: upload-final-results.yml parameters: - artifactPath: $(System.ArtifactsDirectory)\azureMacOs + artifactPath: $(System.ArtifactsDirectory)\azureMacOs_${{ parameters.buildArchitecture }} artifactFilter: "*" - artifactName: signedMacOsBins + artifactName: signedMacOsBins_${{ parameters.buildArchitecture }} condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - ${{ if eq(variables['SHOULD_SIGN'], 'true') }}: - template: EsrpScan.yml@ComplianceRepo parameters: - scanPath: $(System.ArtifactsDirectory)\azureMacOs + scanPath: $(System.ArtifactsDirectory)\azureMacOs_${{ parameters.buildArchitecture }} pattern: | **\* diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml index 342f8d3af8f..1bd853dec53 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -1,11 +1,11 @@ parameters: parentJob: '' - jobName: 'package_macOS' + buildArchitecture: x64 jobs: -- job: ${{ parameters.jobName }} - displayName: Package macOS - dependsOn: ${{ parameters.parentJob }} +- job: package_macOS_${{ parameters.buildArchitecture }} + displayName: Package macOS ${{ parameters.buildArchitecture }} + dependsOn: MacFileSigningJob_${{ parameters.buildArchitecture }} condition: succeeded() pool: Hosted Mac Internal variables: @@ -40,13 +40,13 @@ jobs: displayName: Download macosBinResults inputs: artifactName: 'macosBinResults' - itemPattern: '**/*.zip' + itemPattern: '**/*${{ parameters.buildArchitecture }}.zip' downloadPath: '$(System.ArtifactsDirectory)/Symbols' - task: DownloadBuildArtifacts@0 displayName: Download signedMacOsBins inputs: - artifactName: 'signedMacOsBins' + artifactName: 'signedMacOsBins_${{ parameters.buildArchitecture }}' itemPattern: '**/*' downloadPath: '$(System.ArtifactsDirectory)/macOsBins' condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) @@ -58,7 +58,7 @@ jobs: continueOnError: true - pwsh: | - $zipPath = Get-Item '$(System.ArtifactsDirectory)\Symbols\macosBinResults\*symbol*.zip' + $zipPath = Get-Item '$(System.ArtifactsDirectory)\Symbols\macosBinResults\*symbol*${{ parameters.buildArchitecture }}.zip' Write-Verbose -Verbose "Zip Path: $zipPath" $expandedFolder = $zipPath.BaseName @@ -71,7 +71,7 @@ jobs: - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force - $signedFilesPath = '$(System.ArtifactsDirectory)/macOsBins/signedMacOsBins/' + $signedFilesPath = '$(System.ArtifactsDirectory)/macOsBins/signedMacOsBins_${{ parameters.buildArchitecture }}/' $BuildPath = '$(System.ArtifactsDirectory)\$(SymbolsFolder)' Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath @@ -102,7 +102,7 @@ jobs: displayName: 'Bootstrap VM' - pwsh: | - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" + $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" -Runtime 'osx-${{ parameters.buildArchitecture }}' displayName: 'Package' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index a10a908afcb..c8a7ab7e2bc 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -1,10 +1,10 @@ parameters: - parentJob: '' + buildArchitecture: x64 jobs: -- job: MacPackageSigningJob - displayName: macOS Package signing - dependsOn: ${{ parameters.parentJob }} +- job: MacPackageSigningJob_${{ parameters.buildArchitecture }} + displayName: macOS Package signing ${{ parameters.buildArchitecture }} + dependsOn: package_macOS_${{ parameters.buildArchitecture }} condition: succeeded() pool: name: Package ES Standard Build @@ -41,15 +41,15 @@ jobs: - pwsh: | $null = new-item -type directory -path "$(Build.StagingDirectory)\macos" - $zipFile = "$(Build.StagingDirectory)\macos\powershell-$(Version)-osx-x64.zip" - Compress-Archive -Path "$(System.ArtifactsDirectory)\macosPkgResults\powershell-$(Version)-osx-x64.pkg" -Destination $zipFile + $zipFile = "$(Build.StagingDirectory)\macos\powershell-$(Version)-osx-${{ parameters.buildArchitecture }}.zip" + Compress-Archive -Path "$(System.ArtifactsDirectory)\macosPkgResults\powershell-$(Version)-osx-${{ parameters.buildArchitecture }}.pkg" -Destination $zipFile Write-Host $zipFile - $ltsPkgPath = "$(System.ArtifactsDirectory)\macosPkgResults\powershell-lts-$(Version)-osx-x64.pkg" + $ltsPkgPath = "$(System.ArtifactsDirectory)\macosPkgResults\powershell-lts-$(Version)-osx-${{ parameters.buildArchitecture }}.pkg" if(Test-Path $ltsPkgPath) { - $ltsZipFile = "$(Build.StagingDirectory)\macos\powershell-lts-$(Version)-osx-x64.zip" + $ltsZipFile = "$(Build.StagingDirectory)\macos\powershell-lts-$(Version)-osx-${{ parameters.buildArchitecture }}.zip" Compress-Archive -Path $ltsPkgPath -Destination $ltsZipFile Write-Host $ltsZipFile } @@ -69,7 +69,7 @@ jobs: - template: upload-final-results.yml parameters: artifactPath: $(System.ArtifactsDirectory)\macosPkgResults - artifactFilter: "*.tar.gz" + artifactFilter: "*${{ parameters.buildArchitecture }}.tar.gz" - pwsh: | $destination = "$(System.ArtifactsDirectory)\azureMacOs" diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index 7cbb7fa1ff5..54f4354ed1d 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -1,9 +1,9 @@ parameters: - jobName: 'build_macOS' + buildArchitecture: 'x64' jobs: -- job: ${{ parameters.jobName }} - displayName: Build macOS +- job: build_macOS_${{ parameters.buildArchitecture }} + displayName: Build macOS ${{ parameters.buildArchitecture }} condition: succeeded() pool: Hosted Mac Internal variables: @@ -45,7 +45,7 @@ jobs: - pwsh: | $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)' - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults + $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults -Runtime 'osx-${{ parameters.buildArchitecture }}' $env:AzDevOpsFeedPAT2 = $null displayName: 'Build' diff --git a/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 b/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 index 75bd3a1ed8c..3b243dc71b4 100644 --- a/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 +++ b/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 @@ -42,15 +42,23 @@ param ( [ValidatePattern("-signed.zip$")] [string]$BuildZip, - [string]$ArtifactName = 'result' + [Parameter(Mandatory, ParameterSetName = 'packageSigned')] + [Parameter(Mandatory, ParameterSetName = 'IncludeSymbols')] + [Parameter(Mandatory, ParameterSetName = 'Build')] + [ValidateSet('osx-x64', 'osx-arm64')] + [string]$Runtime, + + [string]$ArtifactName = 'result', + + [switch]$SkipReleaseChecks ) $repoRoot = $location -if ($Build.IsPresent -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { - $releaseTagParam = @{ } +if ($Build -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { + $releaseTagParam = @{} if ($ReleaseTag) { - $releaseTagParam = @{ 'ReleaseTag' = $ReleaseTag } + $releaseTagParam['ReleaseTag'] = $ReleaseTag #Remove the initial 'v' from the ReleaseTag $version = $ReleaseTag -replace '^v' @@ -65,13 +73,14 @@ if ($Build.IsPresent -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { Push-Location try { + $pspackageParams = @{ SkipReleaseChecks = $SkipReleaseChecks; MacOSRuntime = $Runtime } Write-Verbose -Message "Init..." -Verbose Set-Location $repoRoot Import-Module "$repoRoot/build.psm1" Import-Module "$repoRoot/tools/packaging" Sync-PSTags -AddRemoteIfMissing - if ($BootStrap.IsPresent) { + if ($BootStrap) { Start-PSBootstrap -Package } @@ -81,38 +90,38 @@ try { Remove-Item -Path $BuildZip - Start-PSPackage @releaseTagParam + Start-PSPackage @pspackageParams @releaseTagParam switch ($ExtraPackage) { - "tar" { Start-PSPackage -Type tar @releaseTagParam } + "tar" { Start-PSPackage -Type tar @pspackageParams @releaseTagParam } } if ($LTS) { - Start-PSPackage @releaseTagParam -LTS + Start-PSPackage @pspackageParams @releaseTagParam -LTS switch ($ExtraPackage) { - "tar" { Start-PSPackage -Type tar @releaseTagParam -LTS } + "tar" { Start-PSPackage -Type tar @pspackageParams @releaseTagParam -LTS } } } } - if ($Build.IsPresent) { - if ($Symbols.IsPresent) { - Start-PSBuild -Configuration 'Release' -Crossgen -NoPSModuleRestore @releaseTagParam - $pspackageParams = @{} + if ($Build) { + $runCrossgen = $Runtime -eq 'osx-x64' + if ($Symbols) { + Start-PSBuild -Clean -Configuration 'Release' -Crossgen:$runCrossgen -NoPSModuleRestore @releaseTagParam -Runtime $Runtime $pspackageParams['Type']='zip' $pspackageParams['IncludeSymbols']=$Symbols.IsPresent Write-Verbose "Starting powershell packaging(zip)..." -Verbose Start-PSPackage @pspackageParams @releaseTagParam } else { - Start-PSBuild -Configuration 'Release' -Crossgen -PSModuleRestore @releaseTagParam - Start-PSPackage @releaseTagParam + Start-PSBuild -Configuration 'Release' -Crossgen:$runCrossgen -PSModuleRestore @releaseTagParam -Runtime $Runtime + Start-PSPackage @pspackageParams @releaseTagParam switch ($ExtraPackage) { - "tar" { Start-PSPackage -Type tar @releaseTagParam } + "tar" { Start-PSPackage -Type tar @pspackageParams @releaseTagParam } } if ($LTS) { Start-PSPackage @releaseTagParam -LTS switch ($ExtraPackage) { - "tar" { Start-PSPackage -Type tar @releaseTagParam -LTS } + "tar" { Start-PSPackage -Type tar @pspackageParams @releaseTagParam -LTS } } } } @@ -121,7 +130,7 @@ try { Pop-Location } -if ($Build.IsPresent -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { +if ($Build -or $PSCmdlet.ParameterSetName -eq 'packageSigned') { $macPackages = Get-ChildItem "$repoRoot/powershell*" -Include *.pkg, *.tar.gz, *.zip foreach ($macPackage in $macPackages) { $filePath = $macPackage.FullName