diff --git a/.spelling b/.spelling index f9406e0c2f0..cc0d9a735b3 100644 --- a/.spelling +++ b/.spelling @@ -757,6 +757,7 @@ rkitover robo210 ronn rpalo +RPMs runspace runspaceinit runspaces diff --git a/CHANGELOG/7.1.md b/CHANGELOG/7.1.md index a51cfcb7a41..6f92dd03b54 100644 --- a/CHANGELOG/7.1.md +++ b/CHANGELOG/7.1.md @@ -1,5 +1,37 @@ # 7.1 Changelog +## [7.1.1] - 2021-01-14 + +### General Cmdlet Updates and Fixes + +- Avoid an exception if file system does not support reparse points (#13634) (Thanks @iSazonov!) +- Make AppLocker Enforce mode take precedence over UMCI Audit mode (#14353) + +### Code Cleanup + +- Fix syntax error in Windows packaging script (#14377) + +### Build and Packaging Improvements + +
+ + + +
+ +[7.1.1]: https://github.com/PowerShell/PowerShell/compare/v7.1.0...v7.1.1 + ## [7.1.0] - 2020-11-11 ### Engine Updates and Fixes diff --git a/assets/files.wxs b/assets/files.wxs index 418a32db376..4aa9ab2dfd3 100644 --- a/assets/files.wxs +++ b/assets/files.wxs @@ -3057,8 +3057,8 @@ - - + + @@ -4045,7 +4045,7 @@ - + diff --git a/build.psm1 b/build.psm1 index 5e079e4fa2d..73319669cbf 100644 --- a/build.psm1 +++ b/build.psm1 @@ -337,7 +337,8 @@ function Start-PSBuild { try { # Excluded sqlite3 folder is due to this Roslyn issue: https://github.com/dotnet/roslyn/issues/23060 # Excluded src/Modules/nuget.config as this is required for release build. - git clean -fdX --exclude .vs/PowerShell/v16/Server/sqlite3 --exclude src/Modules/nuget.config + # Excluded nuget.config as this is required for release build. + git clean -fdX --exclude .vs/PowerShell/v16/Server/sqlite3 --exclude src/Modules/nuget.config --exclude nuget.config } finally { Pop-Location } @@ -628,7 +629,9 @@ function Restore-PSPackage [switch] $Force, - [switch] $InteractiveAuth + [switch] $InteractiveAuth, + + [switch] $PSModule ) if (-not $ProjectDirs) @@ -652,7 +655,10 @@ function Restore-PSPackage 'Microsoft.NET.Sdk' } - if ($Options.Runtime -notlike 'fxdependent*') { + if ($PSModule.IsPresent) { + $RestoreArguments = @("--verbosity") + } + elseif ($Options.Runtime -notlike 'fxdependent*') { $RestoreArguments = @("--runtime", $Options.Runtime, "/property:SDKToUse=$sdkToUse", "--verbosity") } else { $RestoreArguments = @("/property:SDKToUse=$sdkToUse", "--verbosity") @@ -2505,7 +2511,7 @@ function Copy-PSGalleryModules Find-DotNet - Restore-PSPackage -ProjectDirs (Split-Path $CsProjPath) -Force:$Force.IsPresent + Restore-PSPackage -ProjectDirs (Split-Path $CsProjPath) -Force:$Force.IsPresent -PSModule $cache = dotnet nuget locals global-packages -l if ($cache -match "global-packages: (.*)") { @@ -3287,8 +3293,10 @@ function New-NugetConfigFile - + + + <[FEEDNAME]> diff --git a/global.json b/global.json index ee8f712f95d..10c378d31f5 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "5.0.100" + "version": "5.0.102" } } diff --git a/nuget.config b/nuget.config index a1713070c69..9b513db4b6b 100644 --- a/nuget.config +++ b/nuget.config @@ -5,8 +5,8 @@ - - - + + + diff --git a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj index f89ec614d82..502555d3716 100644 --- a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj +++ b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj @@ -16,7 +16,7 @@ - + @@ -30,6 +30,7 @@ + diff --git a/src/Modules/PSGalleryModules.csproj b/src/Modules/PSGalleryModules.csproj index 3f7b71a07a3..63807dc1e50 100644 --- a/src/Modules/PSGalleryModules.csproj +++ b/src/Modules/PSGalleryModules.csproj @@ -1,6 +1,14 @@ - + + PowerShell + Microsoft Corporation + (c) Microsoft Corporation. + + net5.0 + + true + diff --git a/src/Modules/nuget.config b/src/Modules/nuget.config new file mode 100644 index 00000000000..f5a7f806a36 --- /dev/null +++ b/src/Modules/nuget.config @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/src/System.Management.Automation/System.Management.Automation.csproj b/src/System.Management.Automation/System.Management.Automation.csproj index e8d5ff5911d..5a3065af061 100644 --- a/src/System.Management.Automation/System.Management.Automation.csproj +++ b/src/System.Management.Automation/System.Management.Automation.csproj @@ -23,7 +23,7 @@ - + diff --git a/src/System.Management.Automation/namespaces/FileSystemProvider.cs b/src/System.Management.Automation/namespaces/FileSystemProvider.cs index be4b95dc935..75921d838df 100644 --- a/src/System.Management.Automation/namespaces/FileSystemProvider.cs +++ b/src/System.Management.Automation/namespaces/FileSystemProvider.cs @@ -7840,8 +7840,6 @@ public static class InternalSymbolicLinkLinkCodeMethods // data is 16KB, plus there's a header. private const int MAX_REPARSE_SIZE = (16 * 1024) + REPARSE_GUID_DATA_BUFFER_HEADER_SIZE; - private const int ERROR_NOT_A_REPARSE_POINT = 4390; - private const int FSCTL_GET_REPARSE_POINT = 0x000900A8; private const int FSCTL_SET_REPARSE_POINT = 0x000900A4; @@ -8156,16 +8154,20 @@ private static string WinInternalGetLinkType(string filePath) // Get Buffer size IntPtr dangerousHandle = handle.DangerousGetHandle(); - bool result = DeviceIoControl(dangerousHandle, FSCTL_GET_REPARSE_POINT, - IntPtr.Zero, 0, outBuffer, outBufferSize, out bytesReturned, IntPtr.Zero); + bool result = DeviceIoControl( + dangerousHandle, + FSCTL_GET_REPARSE_POINT, + InBuffer: IntPtr.Zero, + nInBufferSize: 0, + outBuffer, + outBufferSize, + out bytesReturned, + lpOverlapped: IntPtr.Zero); if (!result) { - int lastError = Marshal.GetLastWin32Error(); - if (lastError == ERROR_NOT_A_REPARSE_POINT) - linkType = null; - else - throw new Win32Exception(lastError); + // It's not a reparse point or the file system doesn't support reparse points. + return IsHardLink(ref dangerousHandle) ? "HardLink" : null; } REPARSE_DATA_BUFFER_SYMBOLICLINK reparseDataBuffer = Marshal.PtrToStructure(outBuffer); @@ -8185,7 +8187,7 @@ private static string WinInternalGetLinkType(string filePath) break; default: - linkType = IsHardLink(ref dangerousHandle) ? "HardLink" : null; + linkType = null; break; } @@ -8402,16 +8404,20 @@ private static string WinInternalGetTarget(SafeFileHandle handle) // According to MSDN guidance DangerousAddRef() and DangerousRelease() have been used. handle.DangerousAddRef(ref success); - bool result = DeviceIoControl(handle.DangerousGetHandle(), FSCTL_GET_REPARSE_POINT, - IntPtr.Zero, 0, outBuffer, outBufferSize, out bytesReturned, IntPtr.Zero); + bool result = DeviceIoControl( + handle.DangerousGetHandle(), + FSCTL_GET_REPARSE_POINT, + InBuffer: IntPtr.Zero, + nInBufferSize: 0, + outBuffer, + outBufferSize, + out bytesReturned, + lpOverlapped: IntPtr.Zero); if (!result) { - int lastError = Marshal.GetLastWin32Error(); - if (lastError == ERROR_NOT_A_REPARSE_POINT) - return null; - - throw new Win32Exception(lastError); + // It's not a reparse point or the file system doesn't support reparse points. + return null; } string targetDir = null; diff --git a/src/System.Management.Automation/security/wldpNativeMethods.cs b/src/System.Management.Automation/security/wldpNativeMethods.cs index e9665f4dfa8..55cf43be260 100644 --- a/src/System.Management.Automation/security/wldpNativeMethods.cs +++ b/src/System.Management.Automation/security/wldpNativeMethods.cs @@ -79,28 +79,31 @@ public static SystemEnforcementMode GetSystemLockdownPolicy() /// An EnforcementMode that describes policy. public static SystemEnforcementMode GetLockdownPolicy(string path, SafeHandle handle) { - // Check the WLDP API - SystemEnforcementMode lockdownPolicy = GetWldpPolicy(path, handle); - if (lockdownPolicy == SystemEnforcementMode.Enforce) + // Check the WLDP File policy via API + var wldpFilePolicy = GetWldpPolicy(path, handle); + if (wldpFilePolicy == SystemEnforcementMode.Enforce) { - return lockdownPolicy; + return wldpFilePolicy; + } + + // Check the AppLocker File policy via API + // This needs to be checked before WLDP audit policy + // So, that we don't end up in Audit mode, + // when we should be enforce mode. + var appLockerFilePolicy = GetAppLockerPolicy(path, handle); + if (appLockerFilePolicy == SystemEnforcementMode.Enforce) + { + return appLockerFilePolicy; } // At this point, LockdownPolicy = Audit or Allowed. // If there was a WLDP policy, but WLDP didn't block it, // then it was explicitly allowed. Therefore, return the result for the file. SystemEnforcementMode systemWldpPolicy = s_cachedWldpSystemPolicy.GetValueOrDefault(SystemEnforcementMode.None); - if ((systemWldpPolicy == SystemEnforcementMode.Enforce) || - (systemWldpPolicy == SystemEnforcementMode.Audit)) - { - return lockdownPolicy; - } - - // Check the AppLocker API - lockdownPolicy = GetAppLockerPolicy(path, handle); - if (lockdownPolicy == SystemEnforcementMode.Enforce) + if ((systemWldpPolicy == SystemEnforcementMode.Audit) || + (systemWldpPolicy == SystemEnforcementMode.Enforce)) { - return lockdownPolicy; + return wldpFilePolicy; } // If there was a system-wide AppLocker policy, but AppLocker didn't block it, @@ -108,7 +111,7 @@ public static SystemEnforcementMode GetLockdownPolicy(string path, SafeHandle ha if (s_cachedSaferSystemPolicy.GetValueOrDefault(SaferPolicy.Allowed) == SaferPolicy.Disallowed) { - return lockdownPolicy; + return appLockerFilePolicy; } // If it's not set to 'Enforce' by the platform, allow debug overrides diff --git a/test/hosting/NuGet.Config b/test/hosting/NuGet.Config index fcdcf931709..765346e5343 100644 --- a/test/hosting/NuGet.Config +++ b/test/hosting/NuGet.Config @@ -2,9 +2,6 @@ - - - diff --git a/test/powershell/Modules/PSDesiredStateConfiguration/PSDesiredStateConfiguration.Tests.ps1 b/test/powershell/Modules/PSDesiredStateConfiguration/PSDesiredStateConfiguration.Tests.ps1 index c2fc8cf1e55..30a92d6a8ef 100644 --- a/test/powershell/Modules/PSDesiredStateConfiguration/PSDesiredStateConfiguration.Tests.ps1 +++ b/test/powershell/Modules/PSDesiredStateConfiguration/PSDesiredStateConfiguration.Tests.ps1 @@ -129,6 +129,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "https://github.com/PowerShell/PSDesiredStateConfiguration/issues/26" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + $resource = Get-DscResource -Name $name $resource | Should -Not -BeNullOrEmpty $resource.Name | Should -Be $Name @@ -148,6 +152,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "https://github.com/PowerShell/PSDesiredStateConfiguration/issues/26" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($PendingBecause) { Set-ItResult -Pending -Because $PendingBecause } @@ -225,6 +233,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Will only find script from PSDesiredStateConfiguration without modulename" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } @@ -254,6 +266,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "https://github.com/PowerShell/PSDesiredStateConfiguration/issues/12 and https://github.com/PowerShell/PowerShellGet/pull/529" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($PendingBecause) { Set-ItResult -Pending -Because $PendingBecause } @@ -315,6 +331,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($PendingBecause) { Set-ItResult -Pending -Because $PendingBecause } @@ -340,6 +360,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if ($PendingBecause) { Set-ItResult -Pending -Because $PendingBecause } @@ -402,6 +426,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + if (!$IsLinux) { $result = Invoke-DscResource -Name PSModule -ModuleName $psGetModuleSpecification -Method set -Property @{ Name = 'PsDscResources' @@ -427,6 +455,10 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } + # using create scriptBlock because $using: doesn't work with existing Invoke-DscResource # Verified in Windows PowerShell on 20190814 $result = Invoke-DscResource -Name Script -ModuleName PSDscResources -Method Set -Property @{TestScript = { Write-Output 'test'; return $false }; GetScript = { return @{ } }; SetScript = [scriptblock]::Create("`$global:DSCMachineStatus = $value;return") } @@ -438,6 +470,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $result = Invoke-DscResource -Name Script -ModuleName PSDscResources -Method Test -Property @{TestScript = { Write-Output 'test'; return $false }; GetScript = { return @{ } }; SetScript = { return } } $result | Should -Not -BeNullOrEmpty @@ -448,6 +483,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $result = Invoke-DscResource -Name Script -ModuleName PSDscResources -Method Test -Property @{TestScript = { Write-Verbose 'test'; return $true }; GetScript = { return @{ } }; SetScript = { return } } $result | Should -BeTrue -Because "Test method return true" @@ -457,6 +495,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $module = Get-Module PsDscResources -ListAvailable $moduleSpecification = @{ModuleName = $module.Name; ModuleVersion = $module.Version.ToString() } @@ -520,6 +561,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } { Invoke-DscResource -Name santoheusnaasonteuhsantoheu -Method Test -Property @{TestScript = { Write-Host 'test'; return $true }; GetScript = { return @{ } }; SetScript = { return } } -ErrorAction Stop @@ -531,6 +575,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($IsLinux) { Set-ItResult -Pending -Because "https://github.com/PowerShell/PSDesiredStateConfiguration/issues/12 and https://github.com/PowerShell/PowerShellGet/pull/529" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $result = Invoke-DscResource -Name PSModule -ModuleName $psGetModuleSpecification -Method Get -Property @{ Name = 'PsDscResources' } $result | Should -Not -BeNullOrEmpty @@ -577,6 +624,9 @@ Describe "Test PSDesiredStateConfiguration" -tags CI { if ($MissingLibmi) { Set-ItResult -Pending -Because "Libmi not available for this platform" } + if ($IsMacOS) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } $testString = '890574209347509120348' $result = Invoke-DscResource -Name XmlFileContentResource -ModuleName XmlContentDsc -Property @{Path = $resolvedXmlPath; XPath = '/configuration/appSetting/Test1'; Ensure = 'Present'; Attributes = @{ TestValue2 = $testString; Name = $testString } } -Method Set diff --git a/test/powershell/Modules/PSDesiredStateConfiguration/configuration.Tests.ps1 b/test/powershell/Modules/PSDesiredStateConfiguration/configuration.Tests.ps1 index 7b621578e4d..953b28acec7 100644 --- a/test/powershell/Modules/PSDesiredStateConfiguration/configuration.Tests.ps1 +++ b/test/powershell/Modules/PSDesiredStateConfiguration/configuration.Tests.ps1 @@ -15,6 +15,9 @@ Describe "DSC MOF Compilation" -tags "CI" { if ($IsLinux) { Set-ItResult -Pending -Because "https://github.com/PowerShell/PowerShellGet/pull/529" } + if ($IsMacOs) { + Set-ItResult -Pending -Because "macOS is incompatible with libmi" + } Write-Verbose "DSC_HOME: ${env:DSC_HOME}" -Verbose [Scriptblock]::Create(@" diff --git a/test/powershell/engine/Remoting/PSSession.Tests.ps1 b/test/powershell/engine/Remoting/PSSession.Tests.ps1 index e985ff6ab36..3b8259f3707 100644 --- a/test/powershell/engine/Remoting/PSSession.Tests.ps1 +++ b/test/powershell/engine/Remoting/PSSession.Tests.ps1 @@ -5,6 +5,11 @@ # PSSession tests for non-Windows platforms # +function GetRandomString() +{ + return [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetRandomFileName()) +} + Describe "New-PSSessionOption parameters for non-Windows platforms" -Tag "CI" { BeforeAll { @@ -36,12 +41,13 @@ Describe "SkipCACheck and SkipCNCheck PSSession options are required for New-PSS BeforeAll { $originalDefaultParameterValues = $PSDefaultParameterValues.Clone() - if ($IsWindows) { + # Skip this test for macOS because the latest OS release is incompatible with our shipped libmi for WinRM/OMI. + if ($IsWindows -or $IsMacOS) { $PSDefaultParameterValues['it:skip'] = $true } else { $userName = "User_$(Get-Random -Maximum 99999)" - $userPassword = "Password_$(Get-Random -Maximum 99999)" + $userPassword = GetRandomString $cred = [pscredential]::new($userName, (ConvertTo-SecureString -String $userPassword -AsPlainText -Force)) $soSkipCA = New-PSSessionOption -SkipCACheck $soSkipCN = New-PSSessionOption -SkipCNCheck @@ -54,7 +60,7 @@ Describe "SkipCACheck and SkipCNCheck PSSession options are required for New-PSS $testCases = @( @{ - Name = 'Verifies expected error when session options is missing' + Name = 'Verifies expected error when session option is missing' ScriptBlock = { New-PSSession -cn localhost -Credential $cred -Authentication Basic -UseSSL } ExpectedErrorCode = 825 }, diff --git a/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 b/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 index e48eaecb526..27d2fceb454 100644 --- a/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 +++ b/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 @@ -3,19 +3,25 @@ Import-Module HelpersCommon +function GetRandomString() +{ + return [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetRandomFileName()) +} + Describe "New-PSSession basic test" -Tag @("CI") { It "New-PSSession should not crash powershell" { $platformInfo = Get-PlatformInfo if ( ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') + ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($IsMacOS) ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, and CentOS 8" + Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" return } - { New-PSSession -ComputerName nonexistcomputer -Authentication Basic } | + { New-PSSession -ComputerName (GetRandomString) -Authentication Basic } | Should -Throw -ErrorId "InvalidOperation,Microsoft.PowerShell.Commands.NewPSSessionCommand" } } @@ -26,13 +32,14 @@ Describe "Basic Auth over HTTP not allowed on Unix" -Tag @("CI") { if ( ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') + ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($IsMacOS) ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, and CentOS 8" + Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" return } - $password = ConvertTo-SecureString -String "password" -AsPlainText -Force + $password = ConvertTo-SecureString -String (GetRandomString) -AsPlainText -Force $credential = [PSCredential]::new('username', $password) $err = ({New-PSSession -ComputerName 'localhost' -Credential $credential -Authentication Basic} | Should -Throw -PassThru -ErrorId 'System.Management.Automation.Remoting.PSRemotingDataStructureException,Microsoft.PowerShell.Commands.NewPSSessionCommand') @@ -42,18 +49,20 @@ Describe "Basic Auth over HTTP not allowed on Unix" -Tag @("CI") { $err.Exception.ErrorCode | Should -Be 801 } + # Skip this test for macOS because the latest OS release is incompatible with our shipped libmi for WinRM/OMI. It "New-PSSession should NOT throw a ConnectFailed exception when specifying Basic Auth over HTTPS on Unix" -Skip:($IsWindows) { $platformInfo = Get-PlatformInfo if ( ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') + ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($IsMacOS) ) { - Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, and CentOS 8" + Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" return } - $password = ConvertTo-SecureString -String "password" -AsPlainText -Force + $password = ConvertTo-SecureString -String (GetRandomString) -AsPlainText -Force $credential = [PSCredential]::new('username', $password) # use a Uri that specifies HTTPS to test Basic Auth logic. diff --git a/test/tools/Modules/nuget.config b/test/tools/Modules/nuget.config new file mode 100644 index 00000000000..b0fc73009da --- /dev/null +++ b/test/tools/Modules/nuget.config @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml index fab28643168..221a385b8b8 100644 --- a/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml +++ b/tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml @@ -18,7 +18,7 @@ steps: - pwsh: | Import-Module -Force "$(Build.SourcesDirectory)/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1" - SyncGalleryToAzArtifacts -AzDevOpsFeedUserName $(AzDevOpsFeedUserName) -AzDevOpsPAT $(AzDevOpsFeedPAT) -Destination $(Build.ArtifactStagingDirectory) + SyncGalleryToAzArtifacts -AzDevOpsFeedUserName $(AzDevOpsFeedUserName) -AzDevOpsPAT $(AzDevOpsFeedPAT2) -Destination $(Build.ArtifactStagingDirectory) displayName: Download packages from PSGallery that need to be updated condition: succeededOrFailed() diff --git a/tools/releaseBuild/azureDevOps/WindowsBuild.yml b/tools/releaseBuild/azureDevOps/WindowsBuild.yml deleted file mode 100644 index 535e44dc8a7..00000000000 --- a/tools/releaseBuild/azureDevOps/WindowsBuild.yml +++ /dev/null @@ -1,464 +0,0 @@ -jobs: - -- job: BuildJob - displayName: Build - condition: succeeded() - pool: - name: Package ES CodeHub Lab E - strategy: - matrix: - Build (x64,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x64 - Build (x86,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x86 - Build (arm,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: arm - Build (arm64,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: arm64 - Build (fxdependent,release): - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: fxdependent - - steps: - - - checkout: self - clean: true - persistCredentials: true - - - task: PkgESSetupBuild@10 - displayName: 'Initialize build' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - useDfs: false - productName: PowerShellCore - branchVersion: true - disableWorkspace: true - disableBuildTools: true - disableNugetPack: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag $(ReleaseTag) - displayName: 'Set ReleaseTag Variable' - - - powershell: | - Write-Verbose -Verbose "$(Architecture)" - - if ('$(Architecture)' -eq 'fxdependent' -and '$(ReleaseTag)' -match '6.0.*') - { - $vstsCommandString = "vso[task.setvariable variable=SkipFxDependent]true" - } - else - { - $vstsCommandString = "vso[task.setvariable variable=SkipFxDependent]false" - } - - Write-Verbose -Message "$vstsCommandString " -Verbose - Write-Host -Object "##$vstsCommandString" - displayName: 'Skip FxDependent for PS v6.0.*' - - - powershell: | - Import-Module $(Build.SourcesDirectory)/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination $(Build.SourcesDirectory)/src/Modules - - if(-not (Test-Path "$(Build.SourcesDirectory)/src/Modules/nuget.config")) - { - throw "nuget.config is not created" - } - displayName: 'Add nuget.config for AzDevOps feed for PSGallery modules ' - condition: ne(Variables['SkipFxDependent'], 'true') - - - powershell: | - $version = $env:ReleaseTag.Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Set Version Varibale' - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - docker container prune --force - docker container ls --all --format '{{ json .ID }}' | ConvertFrom-Json | ForEach-Object {docker container rm --force --volumes $_} - displayName: 'remove all containers [Port to PSRelease]' - continueOnError: true - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - docker image ls --format '{{ json .}}'|ConvertFrom-Json| ForEach-Object { - if($_.tag -eq '') - { - $formatString = 'yyyy-MM-dd HH:mm:ss zz00' - $createdAtString = $_.CreatedAt.substring(0,$_.CreatedAt.Length -4) - $createdAt = [DateTime]::ParseExact($createdAtString, $formatString,[System.Globalization.CultureInfo]::InvariantCulture) - if($createdAt -lt (Get-Date).adddays(-1)) - { - docker image rm $_.ID - } - } - } - exit 0 - displayName: 'remove old images [Port to PSRelease]' - continueOnError: true - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - Write-verbose "--docker info---" -verbose - docker info - Write-verbose "--docker image ls---" -verbose - docker image ls - Write-verbose "--docker container ls --all---" -verbose - docker container ls --all - Write-verbose "--git branch ---" -verbose - git branch - exit 0 - displayName: 'Get Environment' - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTag) -Name win-$(Architecture)-symbols - displayName: 'Build Windows Universal - $(Architecture) Symbols zip' - continueOnError: true - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - if ("$env:Architecture" -eq 'fxdependent') - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 -SkipPwshExe - } - else - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 - } - displayName: 'Update Signing Xml' - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - $vstsCommandString = "vso[task.setvariable variable=Symbols]${env:Symbols_$(Architecture)}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Get Symbols path [Update build.json]' - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(Architecture)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(Build.SourcesDirectory)\tools\releaseBuild\signing.xml' - inPathRoot: '$(Symbols)' - outPathRoot: '$(Symbols)\signed' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - New-Item -ItemType Directory -Path $(Symbols)\signed -Force - displayName: 'Create empty signed folder' - condition: and(succeeded(), ne(variables['Build.Reason'], 'Manual'), ne(variables['SkipFxDependent'], 'true')) - - - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTag) -Name win-$(Architecture)-package -BuildPath $(Symbols) -SignedFilesPath $(Symbols)\signed - displayName: 'Build Windows Universal - $(Architecture) Package' - continueOnError: true - condition: and(succeeded(), ne(variables['SkipFxDependent'], 'true')) - -- job: ComponentRegistrationJob - displayName: Component Registration - dependsOn: BuildJob - condition: succeeded() - pool: - name: Package ES CodeHub Lab E - strategy: - matrix: - release-anycpu: - BuildConfiguration: release - BuildPlatform: any cpu - - steps: - - - powershell: | - ./tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag $(ReleaseTag) - displayName: 'Set ReleaseTag Variable' - - - powershell: | - $version = $env:ReleaseTag.Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Set Version Variable' - - - powershell: | - docker container prune --force - docker container ls --all --format '{{ json .ID }}' | ConvertFrom-Json | ForEach-Object {docker container rm --force --volumes $_} - displayName: 'Remove all containers' - continueOnError: true - - - powershell: | - docker image ls --format '{{ json .}}'|ConvertFrom-Json| ForEach-Object { - if($_.tag -eq '') - { - $formatString = 'yyyy-MM-dd HH:mm:ss zz00' - $createdAtString = $_.CreatedAt.substring(0,$_.CreatedAt.Length -4) - $createdAt = [DateTime]::ParseExact($createdAtString, $formatString,[System.Globalization.CultureInfo]::InvariantCulture) - if($createdAt -lt (Get-Date).adddays(-1)) - { - docker image rm $_.ID - } - } - } - exit 0 - displayName: 'Remove old images' - continueOnError: true - - - powershell: | - Write-verbose "--docker info---" -verbose - docker info - Write-verbose "--docker image ls---" -verbose - docker image ls - Write-verbose "--docker container ls --all---" -verbose - docker container ls --all - Write-verbose "--git branch ---" -verbose - git branch - exit 0 - displayName: 'Get Environment' - - - powershell: | - ./tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTag) -Name win-x64-component-registration - displayName: 'Build Windows Universal - Component Registration' - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(componentregistration)' - snapshotForceEnabled: true - -- job: PackageSigningJob - displayName: Package signing - dependsOn: BuildJob - condition: succeeded() - pool: - name: Package ES CodeHub Lab E - strategy: - matrix: - release-anycpu: - BuildConfiguration: release - BuildPlatform: any cpu - - steps: - - - powershell: | - tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag $(ReleaseTag) - displayName: 'Set ReleaseTag Variable' - continueOnError: true - - - powershell: | - $version = $env:ReleaseTag.Substring(1) - $vstsCommandString = "vso[task.setvariable variable=Version]$version" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - $azureVersion = $env:ReleaseTag.ToLowerInvariant() -replace '\.', '-' - $vstsCommandString = "vso[task.setvariable variable=AzureVersion]$azureVersion" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - - displayName: 'Set Version Variable' - continueOnError: true - - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifacts' - inputs: - downloadType: specific - continueOnError: true - - - powershell: | - dir "$(System.ArtifactsDirectory)\*" -Recurse - displayName: 'dir artifacts directory' - continueOnError: true - - - powershell: | - Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-x86.zip" -Destination "$(Build.StagingDirectory)\symbols\x86" - displayName: 'Expand symbols zip - x86' - continueOnError: true - - - powershell: | - Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-x64.zip" -Destination "$(Build.StagingDirectory)\symbols\x64" - displayName: 'Expand symbols zip - x64' - continueOnError: true - - - powershell: | - Expand-Archive -Path "$(System.ArtifactsDirectory)\results\PowerShell-$(Version)-symbols-win-fxdependent.zip" -Destination "$(Build.StagingDirectory)\symbols\fxdependent" - displayName: 'Expand symbols zip - fxdependent' - continueOnError: true - - - powershell: | - tools/releaseBuild/generatePackgeSigning.ps1 -AuthenticodeFiles "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.msi","$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.msi" -path "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Generate Package Signing Xml' - - - powershell: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'print signing xml' - continueOnError: true - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x64.msi" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[Create script] upload signed msi - x64' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - task: AzureFileCopy@4 - displayName: 'upload signed msi to Azure - x64' - inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x64.msi' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x86.msi" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed msi - x86' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed msi to Azure - x86' - inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\PowerShell-$(Version)-win-x86.msi' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[Create script] upload signed zip - x64' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - x64' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - x86' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - x86' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - arm' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - arm' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - arm64' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - arm64' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - - - powershell: | - $packagePath = "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-fxdependent.zip" - if(Test-Path -Path $packagePath) - { - Write-Host "##vso[artifact.upload containerfolder=signedResults;artifactname=signedResults]$packagePath" - } - displayName: '[create script] upload signed zip - fxdependent' - continueOnError: true - - - task: AzureFileCopy@4 - displayName: 'upload signed zip to Azure - fxdependent' - inputs: - SourcePath: '$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-fxdependent.zip' - azureSubscription: '$(AzureFileCopySubscription)' - Destination: AzureBlob - storage: '$(StorageAccount)' - ContainerName: '$(AzureVersion)' - continueOnError: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index 2f38adc5367..5e87ff58ec7 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -10,14 +10,19 @@ pr: - master - release* -variables: - DOTNET_CLI_TELEMETRY_OPTOUT: 1 - POWERSHELL_TELEMETRY_OPTOUT: 1 - -# Set AzDevOps Agent to clean the machine after the end of the build resources: -- repo: self - clean: true + repositories: + - repository: ComplianceRepo + type: github + endpoint: ComplianceGHRepo + name: PowerShell/compliance + ref: master + +variables: + - name: DOTNET_CLI_TELEMETRY_OPTOUT + value: 1 + - name: POWERSHELL_TELEMETRY_OPTOUT + value: 1 stages: - stage: prep @@ -151,6 +156,8 @@ stages: pool: vmImage: 'windows-latest' steps: + - checkout: self + clean: true - template: templates/SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml b/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml index ec688f8a91c..0b8e5f42417 100644 --- a/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml +++ b/tools/releaseBuild/azureDevOps/templates/SetVersionVariables.yml @@ -4,9 +4,29 @@ parameters: CreateJson: 'no' steps: +- powershell: | + $path = "./build.psm1" + + if(Test-Path -Path $path) + { + $vstsCommandString = "vso[task.setvariable variable=repoRoot]." + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + } + else{ + $path = "./PowerShell/build.psm1" + if(Test-Path -Path $path) + { + $vstsCommandString = "vso[task.setvariable variable=repoRoot]./PowerShell" + Write-Host ("sending " + $vstsCommandString) + Write-Host "##$vstsCommandString" + } + } + displayName: 'Set repo Root' + - powershell: | $createJson = ("${{ parameters.ReleaseTagVarName }}" -ne "no") - $releaseTag = tools/releaseBuild/setReleaseTag.ps1 -ReleaseTag ${{ parameters.ReleaseTagVar }} -Variable "${{ parameters.ReleaseTagVarName }}" -CreateJson:$createJson + $releaseTag = & "$env:REPOROOT/tools/releaseBuild/setReleaseTag.ps1" -ReleaseTag ${{ parameters.ReleaseTagVar }} -Variable "${{ parameters.ReleaseTagVarName }}" -CreateJson:$createJson $version = $releaseTag.Substring(1) $vstsCommandString = "vso[task.setvariable variable=Version]$version" Write-Host ("sending " + $vstsCommandString) diff --git a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml index c320727cc13..65a89a8a79e 100644 --- a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml +++ b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml @@ -6,6 +6,8 @@ jobs: pool: vmImage: windows-latest steps: + - checkout: self + clean: true - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml new file mode 100644 index 00000000000..0a52f887671 --- /dev/null +++ b/tools/releaseBuild/azureDevOps/templates/cloneToOfficialPath.yml @@ -0,0 +1,16 @@ +steps: + - powershell: | + $dirSeparatorChar = [system.io.path]::DirectorySeparatorChar + $nativePath = "${dirSeparatorChar}PowerShell" + Write-Host "##vso[task.setvariable variable=PowerShellRoot]$nativePath" + + if ((Test-Path "$nativePath")) { + Remove-Item -Path "$nativePath" -Force -Recurse -Verbose -ErrorAction ignore + } + else { + Write-Verbose -Verbose -Message "No cleanup required." + } + + git clone --quiet $env:REPOROOT $nativePath + displayName: Clone PowerShell Repo to /PowerShell + errorActionPreference: silentlycontinue diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index f1ede279141..9157501ea3d 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -15,6 +15,9 @@ jobs: timeoutInMinutes: 180 steps: + - checkout: self + clean: true + - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml index ae697e70e33..5fdb9f7d977 100644 --- a/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml +++ b/tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml @@ -1,11 +1,25 @@ +parameters: + - name: "repoRoot" + default: $(REPOROOT) + steps: -- powershell: | - Import-Module $(Build.SourcesDirectory)/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination $(Build.SourcesDirectory)/src/Modules +- pwsh: | + Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force + New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination '${{ parameters.repoRoot }}/src/Modules' - if(-not (Test-Path "$(Build.SourcesDirectory)/src/Modules/nuget.config")) + if(-not (Test-Path "${{ parameters.repoRoot }}/src/Modules/nuget.config")) { throw "nuget.config is not created" } displayName: 'Add nuget.config for Azure DevOps feed for PSGallery modules' condition: and(succeededOrFailed(), ne(variables['AzDevOpsFeed'], '')) +- pwsh: | + Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force + New-NugetConfigFile -NugetFeedUrl $(AzDevOpsPackageFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination '${{ parameters.repoRoot }}' + + if(-not (Test-Path "${{ parameters.repoRoot }}/nuget.config")) + { + throw "nuget.config is not created" + } + displayName: 'Add nuget.config for Azure DevOps feed for packages' + condition: and(succeededOrFailed(), ne(variables['AzDevOpsPackageFeed'], '')) diff --git a/tools/releaseBuild/azureDevOps/templates/json.yml b/tools/releaseBuild/azureDevOps/templates/json.yml index 1a0bfba950d..b9949200549 100644 --- a/tools/releaseBuild/azureDevOps/templates/json.yml +++ b/tools/releaseBuild/azureDevOps/templates/json.yml @@ -17,6 +17,9 @@ jobs: # inputs: # # displayName: '' + - checkout: self + clean: true + - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index c959a72cd3b..0fac7ae4f24 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -7,12 +7,22 @@ jobs: - job: build_${{ parameters.buildName }} displayName: Build ${{ parameters.buildName }} condition: succeeded() - pool: Hosted Ubuntu 1604 + pool: + vmImage: ubuntu-16.04 dependsOn: ${{ parameters.parentJob }} variables: - build: ${{ parameters.buildName }} - runCodesignValidationInjection: false + - name: runCodesignValidationInjection + value: false + - name: build + value: ${{ parameters.buildName }} + - group: ESRP + steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true - template: SetVersionVariables.yml parameters: @@ -32,16 +42,19 @@ jobs: displayName: 'Skip Alpine or fxdependent for PS v6.0.*' - template: insert-nuget-config-azfeed.yml - + parameters: + repoRoot: $(REPOROOT) - powershell: | - import-module ./build.psm1 + import-module "$env:REPOROOT/build.psm1" Sync-PSTags -AddRemoteIfMissing displayName: SyncTags condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name '$(build)' + $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)' + & "$env:REPOROOT/tools/releaseBuild/vstsbuild.ps1" -ReleaseTag $(ReleaseTagVar) -Name '$(build)' + $env:AzDevOpsFeedPAT2 = $null displayName: 'Build and package' condition: and(succeeded(), ne(variables['SkipBuild'], 'true')) @@ -50,10 +63,19 @@ jobs: displayName: ${{ parameters.uploadDisplayName }} ${{ parameters.buildName }} dependsOn: build_${{ parameters.buildName }} condition: succeeded() - pool: Package ES CodeHub Lab E + pool: + vmImage: windows-latest variables: - buildName: ${{ parameters.buildName }} + - name: buildName + value: ${{ parameters.buildName }} + - group: ESRP + steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true - template: shouldSign.yml @@ -84,32 +106,26 @@ jobs: downloadPath: '$(System.ArtifactsDirectory)\rpm' condition: and(eq(variables['buildName'], 'RPM'),succeeded()) - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run Defender Scan' - - - powershell: | - $authenticodefiles = @() - Get-ChildItem -Path '$(System.ArtifactsDirectory)\rpm\*.rpm' -recurse | ForEach-Object { $authenticodefiles += $_.FullName} - tools/releaseBuild/generatePackgeSigning.ps1 -LinuxFiles $authenticodeFiles -path "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Generate RPM Signing Xml' - condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - - - powershell: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Capture RPM signing xml' - condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - - - task: PkgESCodeSign@10 - displayName: 'CodeSign RPM $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - + - template: EsrpScan.yml@ComplianceRepo + parameters: + scanPath: $(System.ArtifactsDirectory) + pattern: | + **\*.rpm + **\*.deb + **\*.tar.gz + + - ${{ if eq(variables['buildName'], 'RPM') }}: + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\rpm + signOutputPath: $(Build.StagingDirectory)\signedPackages + certificateId: "CP-450779-Pgp" + pattern: | + **\*.rpm + useMinimatch: true + shouldSign: $(SHOULD_SIGN) + + # requires windows - task: AzureFileCopy@4 displayName: 'Upload to Azure - DEB and tar.gz' inputs: @@ -123,6 +139,7 @@ jobs: parameters: artifactPath: $(System.ArtifactsDirectory)\finished\release + # requires windows - task: AzureFileCopy@4 displayName: 'Upload to Azure - RPM - Unsigned' inputs: @@ -133,10 +150,11 @@ jobs: ContainerName: '$(AzureVersion)' condition: and(and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) + # requires windows - task: AzureFileCopy@4 displayName: 'Upload to Azure - RPM - Signed' inputs: - SourcePath: '$(Build.StagingDirectory)\signedPackages\*' + SourcePath: '$(Build.StagingDirectory)\signedPackages\release\*' azureSubscription: '$(AzureFileCopySubscription)' Destination: AzureBlob storage: '$(StorageAccount)' @@ -150,7 +168,7 @@ jobs: - template: upload-final-results.yml parameters: - artifactPath: '$(Build.StagingDirectory)\signedPackages' + artifactPath: '$(Build.StagingDirectory)\signedPackages\release' condition: and(and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')),eq(variables['buildName'], 'RPM')) - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index 620247b10f0..89f0c245879 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -9,10 +9,14 @@ jobs: pool: name: Package ES CodeHub Lab E variables: - BuildConfiguration: release - BuildPlatform: any cpu + - group: ESRP steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true - template: shouldSign.yml @@ -61,27 +65,15 @@ jobs: Write-Host $zipFile displayName: 'Compress macOS binary files' - - pwsh: | - $pkgFiles = "$(Build.StagingDirectory)\macos\powershell-files-$(Version)-osx-x64.zip", "$(Build.StagingDirectory)\macos\powershell-lts-$(Version)-osx-x64.zip" - tools/releaseBuild/generatePackgeSigning.ps1 -MacDeveloperFiles $pkgFiles -path "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Generate macOS binary Signing Xml' - - - pwsh: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Capture macOS signing xml' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedMacOSPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(Build.StagingDirectory)\macos + signOutputPath: $(Build.StagingDirectory)\signedMacOSPackages + certificateId: "CP-401337-Apple" + pattern: | + **\*.zip + useMinimatch: true + shouldSign: $(SHOULD_SIGN) - pwsh: | $destination = "$(System.ArtifactsDirectory)\azureMacOs" @@ -98,8 +90,12 @@ jobs: artifactName: signedMacOsBins condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run Defender Scan' + - ${{ if eq(variables['SHOULD_SIGN'], 'true') }}: + - template: EsrpScan.yml@ComplianceRepo + parameters: + scanPath: $(System.ArtifactsDirectory)\azureMacOs + pattern: | + **\* - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml index 57208f26ce9..0c819d0519f 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -13,6 +13,9 @@ jobs: HOMEBREW_NO_ANALYTICS: 1 runCodesignValidationInjection: false steps: + - checkout: self + clean: true + - pwsh: | # create folder sudo mkdir /PowerShell @@ -21,11 +24,6 @@ jobs: sudo chown $env:USER /PowerShell displayName: 'Create /PowerShell' - - pwsh: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" - git clone $env:BUILD_REPOSITORY_LOCALPATH /PowerShell - displayName: Clone PowerShell Repo to /PowerShell - - template: shouldSign.yml #- task: @ @@ -36,6 +34,7 @@ jobs: parameters: ReleaseTagVar: $(ReleaseTagVar) + - template: cloneToOfficialPath.yml - task: DownloadBuildArtifacts@0 displayName: Download macosBinResults diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index ef3fdc94e05..c405230e0b8 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -9,10 +9,14 @@ jobs: pool: name: Package ES CodeHub Lab E variables: - BuildConfiguration: release - BuildPlatform: any cpu + - group: ESRP steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true - template: shouldSign.yml @@ -48,27 +52,15 @@ jobs: } displayName: 'Compress macOS Package' - - pwsh: | - $pkgFiles = "$(Build.StagingDirectory)\macos\powershell-$(Version)-osx-x64.zip", "$(Build.StagingDirectory)\macos\powershell-lts-$(Version)-osx-x64.zip" - tools/releaseBuild/generatePackgeSigning.ps1 -MacDeveloperFiles $pkgFiles -path "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Generate macOS Package Signing Xml' - - - pwsh: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Capture macOS signing xml' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedMacOSPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(Build.StagingDirectory)\macos + signOutputPath: $(Build.StagingDirectory)\signedMacOSPackages + certificateId: "CP-401337-Apple" + pattern: | + **\*.zip + useMinimatch: true + shouldSign: $(SHOULD_SIGN) - template: upload-final-results.yml parameters: @@ -118,9 +110,6 @@ jobs: ContainerName: '$(AzureVersion)' condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run Defender Scan' - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index 4526c8d1267..6690f678888 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -15,6 +15,8 @@ jobs: # inputs: # # displayName: '' + - checkout: self + clean: true - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) @@ -27,29 +29,20 @@ jobs: sudo chown $env:USER /PowerShell displayName: 'Create /PowerShell' - - pwsh: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" - git clone $env:BUILD_REPOSITORY_LOCALPATH /PowerShell - displayName: Clone PowerShell Repo to /PowerShell + - template: cloneToOfficialPath.yml - pwsh: | tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap displayName: 'Bootstrap VM' - - pwsh: | - Import-Module $(Build.SourcesDirectory)/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination "$(PowerShellRoot)/src/Modules" - - if(-not (Test-Path "$(PowerShellRoot)/src/Modules/nuget.config")) - { - throw "nuget.config is not created" - } - displayName: 'Add nuget.config for AzDevOps feed for PSGallery modules ' + - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml + parameters: + repoRoot: $(PowerShellRoot) - pwsh: | - $env:AZDEVOPSFEEDPAT = '$(AzDevOpsFeedPAT)' + $env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)' $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults - $env:AZDEVOPSFEEDPAT = $null + $env:AzDevOpsFeedPAT2 = $null displayName: 'Build' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index 77b07cfd2de..b78b294b5b9 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -12,17 +12,35 @@ jobs: timeoutInMinutes: 90 variables: - runCodesignValidationInjection: false - GenAPIToolPath: '$(System.ArtifactsDirectory)/GenAPI' - PackagePath: '$(System.ArtifactsDirectory)/UnifiedPackagePath' - winFxdPath: '$(System.ArtifactsDirectory)/winFxd' - winFxdWinDesktopPath: '$(System.ArtifactsDirectory)/winFxdWinDesktop' - linuxFxdPath: '$(System.ArtifactsDirectory)/linuxFxd' + - name: runCodesignValidationInjection + value: false + - name: build + value: ${{ parameters.buildName }} + - group: ESRP + - name: GenAPIToolPath + value: '$(System.ArtifactsDirectory)/GenAPI' + - name: PackagePath + value: '$(System.ArtifactsDirectory)/UnifiedPackagePath' + - name: winFxdPath + value: '$(System.ArtifactsDirectory)/winFxd' + - name: winFxdWinDesktopPath + value: '$(System.ArtifactsDirectory)/winFxdWinDesktop' + - name: linuxFxdPath + value: '$(System.ArtifactsDirectory)/linuxFxd' steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true + + - template: SetVersionVariables.yml + parameters: + ReleaseTagVar: $(ReleaseTagVar) - powershell: | - $content = Get-Content "$(Build.SourcesDirectory)/global.json" -Raw | ConvertFrom-Json + $content = Get-Content "$env:REPOROOT/global.json" -Raw | ConvertFrom-Json $vstsCommandString = "vso[task.setvariable variable=SDKVersion]$($content.sdk.version)" Write-Host "sending " + $vstsCommandString Write-Host "##$vstsCommandString" @@ -35,7 +53,7 @@ jobs: condition: ne(variables['SDKVersion'], '5.0.100') - pwsh: | - Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force + Import-Module $env:REPOROOT\build.psm1 -Force Install-Dotnet -Version '5.0.100-rtm.20526.5' displayName: Install-DotNet condition: eq(variables['SDKVersion'], '5.0.100') @@ -95,8 +113,8 @@ jobs: displayName: 'Install NuGet.exe' - powershell: | - Import-Module $env:BUILD_SOURCESDIRECTORY\build.psm1 - Import-Module $env:BUILD_SOURCESDIRECTORY\tools\packaging + Import-Module $env:REPOROOT\build.psm1 + Import-Module $env:REPOROOT\tools\packaging Find-Dotnet New-ILNugetPackage -PackagePath "$(PackagePath)" -PackageVersion "$(Version)" -WinFxdBinPath '$(winFxdPath)' -LinuxFxdBinPath '$(linuxFxdPath)' -GenAPIToolPath "$(GenAPIToolPath)" displayName: 'Create Nuget Package Folders' @@ -108,8 +126,8 @@ jobs: displayName: Capture fxd folders - powershell: | - Import-Module $env:BUILD_SOURCESDIRECTORY\build.psm1 - Import-Module $env:BUILD_SOURCESDIRECTORY\tools\packaging + Import-Module $env:REPOROOT\build.psm1 + Import-Module $env:REPOROOT\tools\packaging Find-Dotnet # Create unified package first New-GlobalToolNupkg -UnifiedPackage -LinuxBinPath "$(linuxFxdPath)" -WindowsBinPath "$(winFxdPath)" -WindowsDesktopBinPath "$(winFxdWinDesktopPath)" -PackageVersion "$(Version)" -DestinationPath "$(PackagePath)\globaltool" @@ -122,28 +140,15 @@ jobs: Get-ChildItem "$(PackagePath)" -Recurse displayName: Capture generated packages - - powershell: | - $packages = Get-ChildItem "$(PackagePath)\*.nupkg", "$(PackagePath)\globaltool\*.nupkg" | Select-Object -ExpandProperty FullName - - if($packages.Count -lt 1) - { - throw "No packages created" - } - - $(Build.SourcesDirectory)\tools\releaseBuild\generatePackgeSigning.ps1 -Path $(PackagePath)\NugetSigning.xml -NuPkgFiles $packages - displayName: Create signing file - - - task: PkgESCodeSign@10 - displayName: 'CodeSign Nuget Packages' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(PackagePath)\NugetSigning.xml' - inPathRoot: '$(PackagePath)' - outPathRoot: '$(System.ArtifactsDirectory)\signed' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(PackagePath) + signOutputPath: $(System.ArtifactsDirectory)\signed + certificateId: "CP-401405" + pattern: | + **\*.nupkg + useMinimatch: true + shouldSign: $(SHOULD_SIGN) - pwsh: | if (-not (Test-Path '$(System.ArtifactsDirectory)\signed\')) { $null = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)\signed\' } @@ -153,7 +158,7 @@ jobs: condition: eq(variables['SHOULD_SIGN'], 'false') - powershell: | - Import-Module $(Build.SourcesDirectory)\build.psm1 -Force + Import-Module "${env:REPOROOT}\build.psm1" -Force Get-ChildItem -Recurse "$(System.ArtifactsDirectory)\signed\*.nupkg" -Verbose | ForEach-Object { Start-NativeExecution -sb { nuget.exe verify -All $_.FullName } } displayName: Verify all packages are signed condition: eq(variables['SHOULD_SIGN'], 'true') diff --git a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml index b1465105af9..053354d34c0 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-GlobalToolTest.yml @@ -12,6 +12,9 @@ jobs: pool: vmImage: ${{ parameters.imageName }} steps: + - checkout: self + clean: true + - task: DownloadPipelineArtifact@2 inputs: source: specific diff --git a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml index 08a450c9967..9ca03317a97 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml @@ -9,6 +9,9 @@ jobs: pool: vmImage: ${{ parameters.imageName }} steps: + - checkout: self + clean: true + - task: DownloadPipelineArtifact@2 inputs: source: specific @@ -33,13 +36,6 @@ jobs: path: '$(Pipeline.Workspace)/releasePipeline/metadata' - pwsh: | - $dotnetMetadataPath = "$(Build.SourcesDirectory)/DotnetRuntimeMetadata.json" - $dotnetMetadataJson = Get-Content $dotnetMetadataPath -Raw | ConvertFrom-Json - - # Channel is like: $Channel = "5.0.1xx-preview2" - $Channel = $dotnetMetadataJson.sdk.channel - - $sdkVersion = (Get-Content "$(Build.SourcesDirectory)/global.json" -Raw | ConvertFrom-Json).sdk.version Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force Find-Dotnet @@ -51,9 +47,12 @@ jobs: Write-Verbose -Message "Register new package source 'dotnet5'" -verbose } - ## Install latest version from the channel + ## Remove old .NET SDKs if any exists + if (Test-Path -Type Container $Home/.dotnet) + { + Remove-Item $Home/.dotnet -Recurse -Force + } - #Install-Dotnet -Channel "$Channel" -Version $sdkVersion Start-PSBootstrap Write-Verbose -Message "Installing .NET SDK completed." -Verbose diff --git a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml index 84f2d0f7772..4b469e281db 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-UpdateDepsJson.yml @@ -6,6 +6,9 @@ jobs: variables: - group: 'Azure Blob variable group' steps: + - checkout: self + clean: true + - task: DownloadPipelineArtifact@2 inputs: source: specific diff --git a/tools/releaseBuild/azureDevOps/templates/testartifacts.yml b/tools/releaseBuild/azureDevOps/templates/testartifacts.yml index dd51a95f57c..8967062f96d 100644 --- a/tools/releaseBuild/azureDevOps/templates/testartifacts.yml +++ b/tools/releaseBuild/azureDevOps/templates/testartifacts.yml @@ -6,6 +6,9 @@ jobs: condition: succeeded() pool: 'Hosted Ubuntu 1604' steps: + - checkout: self + clean: true + - pwsh: | Import-Module ./build.psm1 diff --git a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml index 4a333dfdadf..ce40d54feda 100644 --- a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml +++ b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml @@ -7,6 +7,8 @@ jobs: condition: succeeded() pool: Package ES CodeHub Lab E steps: + - checkout: self + clean: true - template: ./SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/windows-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-build.yml deleted file mode 100644 index cf263f43a7d..00000000000 --- a/tools/releaseBuild/azureDevOps/templates/windows-build.yml +++ /dev/null @@ -1,126 +0,0 @@ -parameters: - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x64 - -jobs: -- job: build_windows_${{ parameters.Architecture }} - displayName: Build Windows - ${{ parameters.Architecture }} - condition: succeeded() - pool: - name: Package ES CodeHub Lab E - variables: - BuildConfiguration: ${{ parameters.BuildConfiguration }} - BuildPlatform: ${{ parameters.BuildPlatform }} - Architecture: ${{ parameters.Architecture }} - - steps: - - - checkout: self - clean: true - persistCredentials: true - - - template: shouldSign.yml - - template: SetVersionVariables.yml - parameters: - ReleaseTagVar: $(ReleaseTagVar) - - - task: PkgESSetupBuild@10 - displayName: 'Initialize build' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - useDfs: false - productName: PowerShellCore - branchVersion: true - disableWorkspace: true - disableBuildTools: true - disableNugetPack: true - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - template: insert-nuget-config-azfeed.yml - - - powershell: | - docker container prune --force - docker container ls --all --format '{{ json .ID }}' | ConvertFrom-Json | ForEach-Object {docker container rm --force --volumes $_} - displayName: 'Remove all containers [Port to PSRelease]' - # Cleanup is not critical it passes every time it runs - continueOnError: true - - - powershell: | - docker image ls --format '{{ json .}}'|ConvertFrom-Json| ForEach-Object { - if($_.tag -eq '') - { - $formatString = 'yyyy-MM-dd HH:mm:ss zz00' - $createdAtString = $_.CreatedAt.substring(0,$_.CreatedAt.Length -4) - $createdAt = [DateTime]::ParseExact($createdAtString, $formatString,[System.Globalization.CultureInfo]::InvariantCulture) - if($createdAt -lt (Get-Date).adddays(-1)) - { - docker image rm $_.ID - } - } - } - exit 0 - displayName: 'Remove old images [Port to PSRelease]' - # Cleanup is not critical it passes every time it runs - continueOnError: true - - - powershell: | - Write-verbose "--docker info---" -verbose - docker info - Write-verbose "--docker image ls---" -verbose - docker image ls - Write-verbose "--docker container ls --all---" -verbose - docker container ls --all - exit 0 - displayName: 'Capture docker info' - # Diagnostics is not critical it passes every time it runs - continueOnError: true - - - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name win-$(Architecture)-symbols - displayName: 'Build Windows Universal - $(Architecture) Symbols zip' - - - powershell: | - if ("$env:Architecture" -like 'fxdependent*') - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 -SkipPwshExe - } - else - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 - } - displayName: 'Update Signing Xml' - - - powershell: | - $vstsCommandString = "vso[task.setvariable variable=Symbols]${env:Symbols_$(Architecture)}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Get Symbols path [Update build.json]' - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(Architecture)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(Build.SourcesDirectory)\tools\releaseBuild\signing.xml' - inPathRoot: '$(Symbols)' - outPathRoot: '$(Symbols)\signed' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - - powershell: | - New-Item -ItemType Directory -Path $(Symbols)\signed -Force - displayName: 'Create empty signed folder' - condition: and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')) - - - powershell: | - tools/releaseBuild/vstsbuild.ps1 -ReleaseTag $(ReleaseTagVar) -Name win-$(Architecture)-package -BuildPath $(Symbols) -SignedFilesPath $(Symbols)\signed - displayName: 'Build Windows Universal - $(Architecture) Package' - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(Build.SourcesDirectory)' - snapshotForceEnabled: true diff --git a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml index 6ae54703d87..d819aac1b79 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml @@ -10,6 +10,8 @@ jobs: name: Package ES CodeHub Lab E steps: + - checkout: self + clean: true - template: SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml index 39fbfe7a745..cc05417b17b 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml @@ -1,8 +1,13 @@ parameters: - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x64 - parentJob: '' + - name: BuildConfiguration + default: release + - name: BuildPlatform + default: any cpu + - name: Architecture + default: x64 + - name: parentJob + default: '' + jobs: - job: build_windows_${{ parameters.Architecture }} displayName: Build Windows - ${{ parameters.Architecture }} @@ -21,19 +26,16 @@ jobs: - checkout: self clean: true - persistCredentials: true - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) - - template: insert-nuget-config-azfeed.yml + - template: cloneToOfficialPath.yml - - powershell: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" - $null = New-Item -ItemType Directory -Path /Powershell -Force - git clone $env:BUILD_REPOSITORY_LOCALPATH /PowerShell - displayName: Clone PowerShell Repo to /PowerShell + - template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml + parameters: + repoRoot: $(PowerShellRoot) - powershell: | diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index 473762bda10..070a0840991 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -8,12 +8,18 @@ jobs: ${{ parameters.parentJobs }} condition: succeeded() pool: - name: Package ES CodeHub Lab E + vmImage: windows-latest variables: - BuildConfiguration: release - BuildPlatform: any cpu + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: ESRP steps: + - checkout: self + clean: true + + - checkout: ComplianceRepo + clean: true - template: shouldSign.yml - template: SetVersionVariables.yml @@ -34,36 +40,16 @@ jobs: # Diagnostics is not critical it passes every time it runs continueOnError: true - - powershell: | - $authenticodefiles = @( - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.msi" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.msi" - ) - - $msixFiles = @( - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x86.msix" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-x64.msix" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm32.msix" - "$(System.ArtifactsDirectory)\signed\PowerShell-$(Version)-win-arm64.msix" - ) - - tools/releaseBuild/generatePackgeSigning.ps1 -AuthenticodeFiles $authenticodeFiles -path "$(System.ArtifactsDirectory)\package.xml" -MsixCertType $env:MSIX_TYPE -MsixFiles $msixFiles - displayName: 'Generate Package Signing Xml' - - - powershell: | - Get-Content "$(System.ArtifactsDirectory)\package.xml" - displayName: 'Capture signing xml' - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(System.ArtifactsDirectory)\package.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\package.xml' - outPathRoot: '$(Build.StagingDirectory)\signedPackages' - binVersion: $(SigingVersion) - binVersionOverride: $(SigningVersionOverride) - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\signed + signOutputPath: $(Build.StagingDirectory)\signedPackages + certificateId: "CP-230012" + pattern: | + **\*.msi + **\*.msix + useMinimatch: true + shouldSign: $(SHOULD_SIGN) - powershell: | new-item -itemtype Directory -path '$(Build.StagingDirectory)\signedPackages' @@ -108,8 +94,13 @@ jobs: msi: no msix: no - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run Defender Scan' + - template: EsrpScan.yml@ComplianceRepo + parameters: + scanPath: $(Build.StagingDirectory) + pattern: | + **\*.msix + **\*.msi + **\*.zip - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index fcdc8b01c01..4987514291f 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -1,8 +1,12 @@ parameters: - BuildConfiguration: release - BuildPlatform: any cpu - Architecture: x64 - parentJob: '' + - name: BuildConfiguration + default: release + - name: BuildPlatform + default: any cpu + - name: Architecture + default: x64 + - name: parentJob + default: '' jobs: - job: sign_windows_${{ parameters.Architecture }} @@ -10,37 +14,31 @@ jobs: condition: succeeded() dependsOn: ${{ parameters.parentJob }} pool: - name: Package ES CodeHub Lab E + vmImage: windows-latest variables: - BuildConfiguration: ${{ parameters.BuildConfiguration }} - BuildPlatform: ${{ parameters.BuildPlatform }} - Architecture: ${{ parameters.Architecture }} - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + - name: BuildConfiguration + value: ${{ parameters.BuildConfiguration }} + - name: BuildPlatform + value: ${{ parameters.BuildPlatform }} + - name: Architecture + value: ${{ parameters.Architecture }} + - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE + value: 1 + - group: ESRP steps: - checkout: self clean: true - persistCredentials: true + + - checkout: ComplianceRepo + clean: true - template: shouldSign.yml - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) - - task: PkgESSetupBuild@10 - displayName: 'Initialize build' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - useDfs: false - productName: PowerShellCore - branchVersion: true - disableWorkspace: true - disableBuildTools: true - disableNugetPack: true - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - powershell: | $pkgFilter = if ( '$(Architecture)' -eq 'arm' ) { "arm32" @@ -60,22 +58,9 @@ jobs: itemPattern: '**/*$(PkgFilter).zip' downloadPath: '$(System.ArtifactsDirectory)\Symbols' - - powershell: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" + - template: cloneToOfficialPath.yml - if ((Test-Path "\PowerShell")) { - Remove-Item -Path "\PowerShell" -Force -Recurse -Verbose - } - else { - Write-Verbose -Verbose -Message "No cleanup required." - } - - git clone --quiet $env:BUILD_REPOSITORY_LOCALPATH '\PowerShell' - - displayName: Clone PowerShell Repo to /PowerShell - errorActionPreference: silentlycontinue - - - powershell: | + - pwsh: | # cleanup previous install if((Test-Path "${env:ProgramFiles(x86)}\WiX Toolset xcopy")) { Remove-Item "${env:ProgramFiles(x86)}\WiX Toolset xcopy" -Recurse -Force @@ -83,7 +68,7 @@ jobs: $toolsDir = New-Item -ItemType Directory -Path '$(Build.ArtifactStagingDirectory)\tools' $wixUri = 'https://github.com/wixtoolset/wix3/releases/download/wix311rtm/wix311-binaries.zip' - Invoke-RestMethod -Uri $wixUri -OutFile '$(Build.ArtifactStagingDirectory)\tools\wix.zip' + Invoke-RestMethod -Uri $wixUri -OutFile '$(Build.ArtifactStagingDirectory)\tools\wix.zip' -MaximumRetryCount 5 -RetryIntervalSec 10 Import-Module '$(PowerShellRoot)/tools/releaseBuild/Images/microsoft_powershell_windowsservercore/wix.psm1' Install-WixZip -zipPath '$(Build.ArtifactStagingDirectory)\tools\wix.zip' @@ -105,33 +90,75 @@ jobs: Expand-Archive -Path $zipPath -Destination "$(System.ArtifactsDirectory)\$expandedFolder" -Force displayName: Expand symbols zip - - powershell: | - if ("$env:Architecture" -like 'fxdependent*') - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 -SkipPwshExe - } - else - { - $(Build.SourcesDirectory)\tools\releaseBuild\updateSigning.ps1 - } - displayName: 'Update Signing Xml' - - - task: PkgESCodeSign@10 - displayName: 'CodeSign $(Architecture)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(Build.SourcesDirectory)\tools\releaseBuild\signing.xml' - inPathRoot: '$(System.ArtifactsDirectory)\$(SymbolsFolder)' - outPathRoot: '$(System.ArtifactsDirectory)\signed' - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - pwsh: | - New-Item -ItemType Directory -Path $(System.ArtifactsDirectory)\signed -Force - displayName: 'Create empty signed folder' - condition: and(succeeded(), ne(variables['SHOULD_SIGN'], 'true')) + $fullSymbolsFolder = "$(System.ArtifactsDirectory)\$($env:SYMBOLSFOLDER)" + + $filesToSignDirectory = "$(System.ArtifactsDirectory)\toBeSigned" + $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force + + $signedFilesDirectory = "$(System.ArtifactsDirectory)\signed" + $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force + + $itemsToCopyWithRecurse = @( + "$($fullSymbolsFolder)\*.ps1" + "$($fullSymbolsFolder)\Microsoft.PowerShell*.dll" + ) + + $itemsToCopy = @{ + "$($fullSymbolsFolder)\*.ps1" = "" + "$($fullSymbolsFolder)\Microsoft.Management.Infrastructure.CimCmdlets.dll" = "" + "$($fullSymbolsFolder)\Microsoft.WSMan.*.dll" = "" + "$($fullSymbolsFolder)\Modules\CimCmdlets\CimCmdlets.psd1" = "Modules\CimCmdlets" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Diagnostics.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Event.format.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\GetEvent.types.ps1xml" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Diagnostics\Microsoft.PowerShell.Diagnostics.psd1" = "Modules\Microsoft.PowerShell.Diagnostics" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Host\Microsoft.PowerShell.Host.psd1" = "Modules\Microsoft.PowerShell.Host" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Management\Microsoft.PowerShell.Management.psd1" = "Modules\Microsoft.PowerShell.Management" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Security\Microsoft.PowerShell.Security.psd1" = "Modules\Microsoft.PowerShell.Security" + "$($fullSymbolsFolder)\Modules\Microsoft.PowerShell.Utility\Microsoft.PowerShell.Utility.psd1" = "Modules\Microsoft.PowerShell.Utility" + "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\Microsoft.WSMan.Management.psd1" = "Modules\Microsoft.WSMan.Management" + "$($fullSymbolsFolder)\Modules\Microsoft.WSMan.Management\WSMan.format.ps1xml" = "Modules\Microsoft.WSMan.Management" + "$($fullSymbolsFolder)\Modules\PSDiagnostics\PSDiagnostics.ps?1" = "Modules\PSDiagnostics" + "$($fullSymbolsFolder)\pwsh.dll" = "" + "$($fullSymbolsFolder)\System.Management.Automation.dll" = "" + } - - powershell: | + $itemsToExclude = @( + ) + + if ("$env:Architecture" -notlike 'fxdependent*') + { + $itemsToCopy += @{"$($fullSymbolsFolder)\pwsh.exe" = ""} + } + + Write-Verbose -verbose "recusively copying $($itemsToCopyWithRecurse | out-string) to $filesToSignDirectory" + Copy-Item -Path $itemsToCopyWithRecurse -Destination $filesToSignDirectory -Recurse -verbose -exclude $itemsToExclude + + foreach($pattern in $itemsToCopy.Keys) { + $destinationFolder = Join-Path $filesToSignDirectory -ChildPath $itemsToCopy.$pattern + $null = New-Item -ItemType Directory -Path $destinationFolder -Force + Write-Verbose -verbose "copying $pattern to $destinationFolder" + Copy-Item -Path $pattern -Destination $destinationFolder -Recurse -verbose + } + displayName: 'Prepare files to be signed' + + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\toBeSigned + signOutputPath: $(System.ArtifactsDirectory)\signed + certificateId: "CP-230012" + pattern: | + **\*.dll + **\*.psd1 + **\*.psm1 + **\*.ps1xml + **\*.ps1 + **\*.exe + useMinimatch: true + shouldSign: $(SHOULD_SIGN) + + - pwsh: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force $signedFilesPath = '$(System.ArtifactsDirectory)\signed\' @@ -141,29 +168,41 @@ jobs: $dlls = Get-ChildItem $BuildPath\*.dll -Recurse $signatures = $dlls | Get-AuthenticodeSignature $missingSignatures = $signatures | Where-Object { $_.status -eq 'notsigned'}| select-object -ExpandProperty Path - tools/releaseBuild/generatePackgeSigning.ps1 -ThirdPartyFiles $missingSignatures -path "$(System.ArtifactsDirectory)\thirtdparty.xml" - displayName: Create ThirdParty Signing Xml - condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - - task: PkgESCodeSign@10 - displayName: 'CodeSign ThirdParty $(Architecture)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(System.ArtifactsDirectory)\thirtdparty.xml' - inPathRoot: '$(System.ArtifactsDirectory)\$(SymbolsFolder)' - outPathRoot: '$(System.ArtifactsDirectory)\signedThirdParty' + Write-Verbose -verbose "to be signed:`r`n $($missingSignatures | Out-String)" + + $filesToSignDirectory = "$(System.ArtifactsDirectory)\thirdPartyToBeSigned" + $null = New-Item -ItemType Directory -Path $filesToSignDirectory -Force + + $signedFilesDirectory = "$(System.ArtifactsDirectory)\thirdPartySigned" + $null = New-Item -ItemType Directory -Path $signedFilesDirectory -Force + + $missingSignatures | ForEach-Object { + Copy-Item -Path $_ -Destination $filesToSignDirectory + } + + displayName: Create ThirdParty Signing Folder condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) + - template: EsrpSign.yml@ComplianceRepo + parameters: + buildOutputPath: $(System.ArtifactsDirectory)\thirdPartyToBeSigned + signOutputPath: $(System.ArtifactsDirectory)\thirdPartySigned + certificateId: "CP-231522" + pattern: | + **\*.dll + useMinimatch: true + shouldSign: $(SHOULD_SIGN) + - powershell: | - Get-ChildItem '$(System.ArtifactsDirectory)\signedThirdParty\*' + Get-ChildItem '$(System.ArtifactsDirectory)\thirdPartySigned\*' displayName: Captrue ThirdParty Signed files condition: and(succeeded(), eq(variables['SHOULD_SIGN'], 'true')) - powershell: | Import-Module $(PowerShellRoot)/build.psm1 -Force Import-Module $(PowerShellRoot)/tools/packaging -Force - $signedFilesPath = '$(System.ArtifactsDirectory)\signedThirdParty\' + $signedFilesPath = '$(System.ArtifactsDirectory)\thirdPartySigned' $BuildPath = '$(System.ArtifactsDirectory)\$(SymbolsFolder)' Update-PSSignedBuildFolder -BuildPath $BuildPath -SignedFilesPath $SignedFilesPath diff --git a/tools/releaseBuild/azureDevOps/vpackRelease.yml b/tools/releaseBuild/azureDevOps/vpackRelease.yml index 48b9e833844..70091eac8b5 100644 --- a/tools/releaseBuild/azureDevOps/vpackRelease.yml +++ b/tools/releaseBuild/azureDevOps/vpackRelease.yml @@ -17,11 +17,6 @@ variables: value: 1 - group: Azure Blob variable group -# Set AzDevOps Agent to clean the machine after the end of the build -resources: -- repo: self - clean: true - jobs: - job: rename displayName: Name the build @@ -29,6 +24,8 @@ jobs: pool: vmImage: 'windows-latest' steps: + - checkout: self + clean: true - template: ./templates/SetVersionVariables.yml parameters: diff --git a/tools/releaseBuild/macOS/vsts.yml b/tools/releaseBuild/macOS/vsts.yml deleted file mode 100644 index b00fc9ae0b0..00000000000 --- a/tools/releaseBuild/macOS/vsts.yml +++ /dev/null @@ -1,47 +0,0 @@ -resources: -- repo: self - clean: true -queue: - name: Hosted Mac Internal -name: macOS_$(Build.SourceBranchName)_$(Build.BuildId) -steps: -#- task: @ -# inputs: -# -# displayName: '' -- task: ShellScript@2 - inputs: - scriptPath: 'tools/releaseBuild/setReleaseTag.sh' - args: '-ReleaseTag $(ReleaseTagVar) -Variable "ReleaseTagVar"' - displayName: 'Calculate Release Tag' -- task: ShellScript@2 - inputs: - scriptPath: 'tools/installpsh-osx.sh' - displayName: 'Install pwsh' -- task: ShellScript@2 - inputs: - scriptPath: 'tools/releaseBuild/macOS/createPowerShell.sh' - displayName: 'Create /PowerShell' -- powershell: | - Write-Host "##vso[task.setvariable variable=PowerShellRoot]/PowerShell" - git clone $env:BUILD_REPOSITORY_LOCALPATH /PowerShell - displayName: Clone PowerShell Repo to /PowerShell -- task: ShellScript@2 - inputs: - scriptPath: 'tools/releaseBuild/macOS/PowerShellPackageVsts.sh' - args: '-location $(PowerShellRoot) -BootStrap' - displayName: 'Bootstrap VM' -- powershell: | - Import-Module $(Build.SourcesDirectory)/build.psm1 -Force - New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT) -FeedName AzDevOpsFeed -Destination "$(PowerShellRoot)/src/Modules" - - if(-not (Test-Path "$(PowerShellRoot)/src/Modules/nuget.config")) - { - throw "nuget.config is not created" - } - displayName: 'Add nuget.config for AzDevOps feed for PSGallery modules ' -- powershell: | - $env:AZDEVOPSFEEDPAT = '$(AzDevOpsFeedPAT)' - $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -ExtraPackage "tar" -location $(PowerShellRoot) -Build - $env:AZDEVOPSFEEDPAT = $null - displayName: 'Build and Package' diff --git a/tools/releaseBuild/setReleaseTag.ps1 b/tools/releaseBuild/setReleaseTag.ps1 index 1927eb7ae60..121425d963b 100644 --- a/tools/releaseBuild/setReleaseTag.ps1 +++ b/tools/releaseBuild/setReleaseTag.ps1 @@ -79,7 +79,7 @@ if($ReleaseTag -eq 'fromBranch' -or !$ReleaseTag) New-BuildInfoJson -ReleaseTag $releaseTag } } - elseif($branchOnly -eq 'master' -or $branchOnly -like '*dailytest*') + elseif(($branchOnly -eq 'master' -and $env:BUILD_REASON -ne 'Manual') -or $branchOnly -like '*dailytest*') { $isDaily = $true Write-Verbose "daily build" -Verbose