From c25d0e1a21f9b491cf3d850c14d86fc720928e22 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 12 May 2021 10:50:42 +0200 Subject: [PATCH 1/6] add support for comparing .NET Core 2.1 vs .NET 5 vs .NET 6 --- test/perf/benchmarks/powershell-perf.csproj | 29 +++++++++++++++------ test/perf/perf.psm1 | 9 +++++-- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/test/perf/benchmarks/powershell-perf.csproj b/test/perf/benchmarks/powershell-perf.csproj index 860948202a6..a3d91ac10f7 100644 --- a/test/perf/benchmarks/powershell-perf.csproj +++ b/test/perf/benchmarks/powershell-perf.csproj @@ -12,16 +12,22 @@ $(NoWarn);CS8002 true - - $(PERF_TARGET_VERSION) - AnyCPU portable true + + + + netcoreapp2.1;net5.0;net6.0 + + + $(PERF_TARGET_VERSION) + 7.1.3 + 6.2.7 @@ -33,7 +39,7 @@ - + @@ -41,6 +47,13 @@ + + + + + + + diff --git a/test/perf/perf.psm1 b/test/perf/perf.psm1 index 6bea86ff899..1c208baa7d5 100644 --- a/test/perf/perf.psm1 +++ b/test/perf/perf.psm1 @@ -36,6 +36,9 @@ function Start-Benchmarking [ValidateSet('flat', 'tree')] [string] $List, + [ValidateSet('netcoreapp2.1', 'net5.0', 'net6.0')] + [string] $TargetFramework = 'net6.0', + [string[]] $Filter = '*', [string] $Artifacts, [switch] $KeepFiles @@ -61,8 +64,10 @@ function Start-Benchmarking $savedOFS = $OFS; $OFS = $null if ($TargetPSVersion) { - Write-Log -message "Run benchmarks targeting the 'Microsoft.PowerShell.SDK' version $TargetPSVersion..." + Write-Log -message "Run benchmarks targeting $TargetFramework and the 'Microsoft.PowerShell.SDK' version $TargetPSVersion..." $env:PERF_TARGET_VERSION = $TargetPSVersion + } elseif ($TargetFramework -ne "net6.0") { + Write-Log -message "Run benchmarks targeting $TargetFramework and the corresponding 'Microsoft.PowerShell.SDK' version..." } else { Write-Log -message "Run benchmarks targeting the current PowerShell code base..." } @@ -71,7 +76,7 @@ function Start-Benchmarking if ($List) { $runArgs += '--list', $List } if ($KeepFiles) { $runArgs += "--keepFiles" } - dotnet run -c release --filter $Filter --artifacts $Artifacts --envVars POWERSHELL_TELEMETRY_OPTOUT:1 $runArgs + dotnet run -c release -f $TargetFramework --filter $Filter --artifacts $Artifacts --envVars POWERSHELL_TELEMETRY_OPTOUT:1 $runArgs if (Test-Path $Artifacts) { Write-Log -message "`nBenchmark artifacts can be found at $Artifacts" From fa53c79b50b1921f0f572a0ec99ee2d10d5786c9 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 13 May 2021 09:41:56 +0200 Subject: [PATCH 2/6] fix the possibility to benchmark specific SDK version --- test/perf/benchmarks/powershell-perf.csproj | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/perf/benchmarks/powershell-perf.csproj b/test/perf/benchmarks/powershell-perf.csproj index a3d91ac10f7..92873ccd33d 100644 --- a/test/perf/benchmarks/powershell-perf.csproj +++ b/test/perf/benchmarks/powershell-perf.csproj @@ -16,16 +16,19 @@ portable true - - - netcoreapp2.1;net5.0;net6.0 - $(PERF_TARGET_VERSION) + + + + netcoreapp2.1;net5.0;net6.0 + 7.1.3 6.2.7 From ec2a65f2089370927c3c06af76034f2e74ceca12 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 13 May 2021 09:43:45 +0200 Subject: [PATCH 3/6] fix the comment --- test/perf/benchmarks/powershell-perf.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/perf/benchmarks/powershell-perf.csproj b/test/perf/benchmarks/powershell-perf.csproj index 92873ccd33d..0fbaeee661d 100644 --- a/test/perf/benchmarks/powershell-perf.csproj +++ b/test/perf/benchmarks/powershell-perf.csproj @@ -24,8 +24,8 @@ $(PERF_TARGET_VERSION) + But only when PERF_TARGET_VERSION was not specified. When it is specified, a specific version is being benchmarked + and this version can be benchmarked only against single runtime version (because Microsoft.PowerShell.SDK targets a single TFM). --> netcoreapp2.1;net5.0;net6.0 From 69189851527db7e271ba1250b04b6ad549ed3d83 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 13 May 2021 10:37:40 +0200 Subject: [PATCH 4/6] always use `Release` configuration name, as `release` != `Release` and it triggers full build of all dependencies in the benchmarks project --- test/perf/benchmarks/README.md | 4 ++-- test/perf/perf.psm1 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/perf/benchmarks/README.md b/test/perf/benchmarks/README.md index 0da4361630f..f8c513a2d9a 100644 --- a/test/perf/benchmarks/README.md +++ b/test/perf/benchmarks/README.md @@ -24,12 +24,12 @@ This folder contains micro benchmarks that test the performance of PowerShell En You can run the benchmarks directly using `dotnet run` in this directory: 1. To run the benchmarks in Interactive Mode, where you will be asked which benchmark(s) to run: ``` - dotnet run -c release + dotnet run -c Release ``` 2. To list all available benchmarks ([read more](https://github.com/dotnet/performance/blob/main/docs/benchmarkdotnet.md#Listing-the-Benchmarks)): ``` - dotnet run -c release --list [flat/tree] + dotnet run -c Release --list [flat/tree] ``` 3. To filter the benchmarks using a glob pattern applied to `namespace.typeName.methodName` ([read more](https://github.com/dotnet/performance/blob/main/docs/benchmarkdotnet.md#Filtering-the-Benchmarks)]): diff --git a/test/perf/perf.psm1 b/test/perf/perf.psm1 index 1c208baa7d5..093383952bd 100644 --- a/test/perf/perf.psm1 +++ b/test/perf/perf.psm1 @@ -76,7 +76,7 @@ function Start-Benchmarking if ($List) { $runArgs += '--list', $List } if ($KeepFiles) { $runArgs += "--keepFiles" } - dotnet run -c release -f $TargetFramework --filter $Filter --artifacts $Artifacts --envVars POWERSHELL_TELEMETRY_OPTOUT:1 $runArgs + dotnet run -c Release -f $TargetFramework --filter $Filter --artifacts $Artifacts --envVars POWERSHELL_TELEMETRY_OPTOUT:1 $runArgs if (Test-Path $Artifacts) { Write-Log -message "`nBenchmark artifacts can be found at $Artifacts" From 85c4558df76c5b5e98d63b21f4b8c707b8d8f32d Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 13 May 2021 10:45:29 +0200 Subject: [PATCH 5/6] revert redundant whitespace change --- test/perf/benchmarks/powershell-perf.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/perf/benchmarks/powershell-perf.csproj b/test/perf/benchmarks/powershell-perf.csproj index 0fbaeee661d..bdb454af6ef 100644 --- a/test/perf/benchmarks/powershell-perf.csproj +++ b/test/perf/benchmarks/powershell-perf.csproj @@ -42,7 +42,7 @@ - + From a691c058def6dba7d92c2119b167a1647a4e8824 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Thu, 13 May 2021 15:25:53 -0700 Subject: [PATCH 6/6] Use netcoreapp3.1 and update script --- test/perf/benchmarks/powershell-perf.csproj | 23 +++--- test/perf/perf.psm1 | 85 +++++++++++++++++---- 2 files changed, 84 insertions(+), 24 deletions(-) diff --git a/test/perf/benchmarks/powershell-perf.csproj b/test/perf/benchmarks/powershell-perf.csproj index bdb454af6ef..78608c2e96f 100644 --- a/test/perf/benchmarks/powershell-perf.csproj +++ b/test/perf/benchmarks/powershell-perf.csproj @@ -17,20 +17,20 @@ true + Do not try passing in the value using '/property:' at command line, because + 1. 'dotnet run' doesn't respect '/property:' arguments; + 2. BenchmarkDotnet generates temporary project files that reference to this .csproj file, + and '/property:' arguments won't be forwarded when building those temp projects. --> $(PERF_TARGET_VERSION) + But we do it only when 'PERF_TARGET_VERSION' was not specified. When it is specified, a specific version is being benchmarked + and this version can be benchmarked only against single runtime version (because 'Microsoft.PowerShell.SDK' targets a single TFM). --> - netcoreapp2.1;net5.0;net6.0 + netcoreapp3.1;net5.0;net6.0 7.1.3 - 6.2.7 + 7.0.6 @@ -49,14 +49,17 @@ + - - + + diff --git a/test/perf/perf.psm1 b/test/perf/perf.psm1 index 093383952bd..a623a2290dd 100644 --- a/test/perf/perf.psm1 +++ b/test/perf/perf.psm1 @@ -14,9 +14,15 @@ function Start-Benchmarking The version of 'Microsoft.PowerShell.SDK' package that we want the benchmark to target. The supported versions are 7.0.x and above, including preview versions. + .PARAMETER TargetFramework + The target framework to run benchmarks against. + .PARAMETER List List the available benchmarks, in either 'flat' or 'tree' views. + .PARAMETER Runtime + Run benchmarks against multiple .NET runtimes. + .PARAMETER Filter One or more wildcard patterns to filter the benchmarks to be executed or to be listed. @@ -26,18 +32,25 @@ function Start-Benchmarking .PARAMETER KeepFiles Indicates to keep all temporary files produced for running benchmarks. #> - [CmdletBinding()] + [CmdletBinding(DefaultParameterSetName = 'Default')] param( + [Parameter(ParameterSetName = 'Default')] [ValidatePattern( '^7\.(0|1|2)\.\d+(-preview\.\d{1,2})?$', ErrorMessage = 'The package version is invalid or not supported')] [string] $TargetPSVersion, + [Parameter(ParameterSetName = 'Default')] + [ValidateSet('netcoreapp3.1', 'net5.0', 'net6.0')] + [string] $TargetFramework = 'net6.0', + + [Parameter(ParameterSetName = 'Default')] [ValidateSet('flat', 'tree')] [string] $List, - [ValidateSet('netcoreapp2.1', 'net5.0', 'net6.0')] - [string] $TargetFramework = 'net6.0', + [Parameter(Mandatory, ParameterSetName = 'Runtimes')] + [ValidateSet('netcoreapp3.1', 'net5.0', 'net6.0')] + [string[]] $Runtime, [string[]] $Filter = '*', [string] $Artifacts, @@ -56,6 +69,34 @@ function Start-Benchmarking if (Test-Path -Path $Artifacts) { Remove-Item -Path $Artifacts -Recurse -Force -ErrorAction Stop } + + if ($TargetPSVersion) { + ## Validate the specified 'TargetPSVersion' and 'TargetFramework' are compatible. + $TargetFramework -match '\d.\d' > $null + $targetDotNetVersion = $Matches[0] + + $minimalDotNetVersion = switch -Wildcard ($TargetPSVersion) { + '7.0.*' { '3.1' } + '7.1.*' { '5.0' } + '7.2.*' { '6.0' } + } + + if ($targetDotNetVersion -lt $minimalDotNetVersion) { + $dotnetVer = $minimalDotNetVersion -eq '3.1' ? 'netcoreapp3.1' : "net$minimalDotNetVersion" + throw "The '$TargetPSVersion' version of 'Microsoft.PowerShell.SDK' requires '$dotnetVer' as the minimal target framework." + } + } + + if ($Runtime) { + ## Remove duplicate values. + $hash = [ordered]@{} + foreach ($item in $Runtime) { + if (-not $hash.Contains($item)) { + $hash.Add($item, $null) + } + } + $Runtime = $hash.Keys + } } End { @@ -63,20 +104,36 @@ function Start-Benchmarking Push-Location -Path "$PSScriptRoot/benchmarks" $savedOFS = $OFS; $OFS = $null - if ($TargetPSVersion) { - Write-Log -message "Run benchmarks targeting $TargetFramework and the 'Microsoft.PowerShell.SDK' version $TargetPSVersion..." - $env:PERF_TARGET_VERSION = $TargetPSVersion - } elseif ($TargetFramework -ne "net6.0") { - Write-Log -message "Run benchmarks targeting $TargetFramework and the corresponding 'Microsoft.PowerShell.SDK' version..." - } else { - Write-Log -message "Run benchmarks targeting the current PowerShell code base..." - } + ## Aggregate BDN arguments. + $runArgs = @('--filter') + foreach ($entry in $Filter) { $runArgs += $entry } + $runArgs += '--artifacts', $Artifacts + $runArgs += '--envVars', 'POWERSHELL_TELEMETRY_OPTOUT:1' - $runArgs = @() if ($List) { $runArgs += '--list', $List } if ($KeepFiles) { $runArgs += "--keepFiles" } - dotnet run -c Release -f $TargetFramework --filter $Filter --artifacts $Artifacts --envVars POWERSHELL_TELEMETRY_OPTOUT:1 $runArgs + if ($PSCmdlet.ParameterSetName -eq 'Default') { + if ($TargetPSVersion) { + Write-Log -message "Run benchmarks targeting '$TargetFramework' and the 'Microsoft.PowerShell.SDK' version '$TargetPSVersion' ..." + $env:PERF_TARGET_VERSION = $TargetPSVersion + } + elseif ($TargetFramework -eq 'net6.0') { + Write-Log -message "Run benchmarks targeting the current PowerShell code base ..." + } + else { + Write-Log -message "Run benchmarks targeting '$TargetFramework' and the corresponding latest version of 'Microsoft.PowerShell.SDK' ..." + } + + ## Use 'Release' instead of 'release' (note the capital case) because BDN uses 'Release' when building the auto-generated + ## project, and MSBuild somehow recognizes 'release' and 'Release' as two different configurations and thus will rebuild + ## all dependencies unnecessarily. + dotnet run -c Release -f $TargetFramework $runArgs + } + else { + Write-Log -message "Run benchmarks targeting multiple .NET runtimes: $Runtime ..." + dotnet run -c Release -f net6.0 --runtimes $Runtime $runArgs + } if (Test-Path $Artifacts) { Write-Log -message "`nBenchmark artifacts can be found at $Artifacts" @@ -147,7 +204,7 @@ function Compare-BenchmarkResult if ($Noise) { $runArgs += "--noise $Noise" } if ($Top -gt 0) { $runArgs += "--top $Top" } - dotnet run -c release --base $BaseResultPath --diff $DiffResultPath --threshold $Threshold "$runArgs" + dotnet run -c Release --base $BaseResultPath --diff $DiffResultPath --threshold $Threshold "$runArgs" } finally { $OFS = $savedOFS