PowerShell · daxian-dbw · May 14, 2021 · May 12, 2021 · May 13, 2021 · May 13, 2021
@@ -24,12 +24,12 @@ This folder contains micro benchmarks that test the performance of PowerShell En
 You can run the benchmarks directly using `dotnet run` in this directory:
 1. To run the benchmarks in Interactive Mode, where you will be asked which benchmark(s) to run:
    ```
-   dotnet run -c release
+   dotnet run -c Release
    ```
 
 2. To list all available benchmarks ([read more](https://github.com/dotnet/performance/blob/main/docs/benchmarkdotnet.md#Listing-the-Benchmarks)):
    ```
-   dotnet run -c release --list [flat/tree]
+   dotnet run -c Release --list [flat/tree]
    ```
 
 3. To filter the benchmarks using a glob pattern applied to `namespace.typeName.methodName` ([read more](https://github.com/dotnet/performance/blob/main/docs/benchmarkdotnet.md#Filtering-the-Benchmarks)]):

@@ -12,16 +12,25 @@
     <NoWarn>$(NoWarn);CS8002</NoWarn>
     <SuppressNETCoreSdkPreviewMessage>true</SuppressNETCoreSdkPreviewMessage>
 
-    <!-- To run benchmarks targeting a specific package version, set the version to 'PERF_TARGET_VERSION' as an environment variable.
-         Do not try passing in the value using '/property:' at command line, because
-           1. 'dotnet run' doesn't respect '/property:' arguments;
-           2. BenchmarkDotnet generates temporary project files that reference to this .csproj file,
-              and '/property:' arguments won't be forwarded when building those temp projects. -->
-    <PerfTargetVersion>$(PERF_TARGET_VERSION)</PerfTargetVersion>
-
     <PlatformTarget>AnyCPU</PlatformTarget>
     <DebugType>portable</DebugType>
     <DebugSymbols>true</DebugSymbols>
+
+    <!-- To run benchmarks targeting a specific package version, set the version to 'PERF_TARGET_VERSION' as an environment variable.
+      Do not try passing in the value using '/property:' at command line, because
+        1. 'dotnet run' doesn't respect '/property:' arguments;
+        2. BenchmarkDotnet generates temporary project files that reference to this .csproj file,
+           and '/property:' arguments won't be forwarded when building those temp projects. -->
+    <PerfTargetVersion>$(PERF_TARGET_VERSION)</PerfTargetVersion>
+
+    <!-- Test.Common.props sets TargetFramework to net6.0, we need to clear this value to be able to target multiple TFMs.
+      But we do it only when 'PERF_TARGET_VERSION' was not specified. When it is specified, a specific version is being benchmarked
+      and this version can be benchmarked only against single runtime version (because 'Microsoft.PowerShell.SDK' targets a single TFM). -->
+    <TargetFramework Condition="'$(PerfTargetVersion)' == ''"></TargetFramework>
+    <TargetFrameworks Condition="'$(TargetFramework)' == ''">netcoreapp3.1;net5.0;net6.0</TargetFrameworks>
+
+    <PerfTargetVersion Condition="'$(PerfTargetVersion)' == '' AND '$(TargetFramework)' == 'net5.0'">7.1.3</PerfTargetVersion>
+    <PerfTargetVersion Condition="'$(PerfTargetVersion)' == '' AND '$(TargetFramework)' == 'netcoreapp3.1'">7.0.6</PerfTargetVersion>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -40,7 +49,17 @@
   </ItemGroup>
 
   <ItemGroup Condition="'$(PerfTargetVersion)' != ''">
+    <!-- We have to specify all package references explicitly due to a .NET SDK bug: https://github.com/dotnet/sdk/issues/17013
+      If the project references project A which references project B and we want to reference their packages for different TFMs,
+      we have to explicitly reference package A and B. -->
     <PackageReference Include="Microsoft.PowerShell.SDK" Version="$(PerfTargetVersion)" />
+    <PackageReference Include="Microsoft.PowerShell.Commands.Diagnostics" Version="$(PerfTargetVersion)" />
+    <PackageReference Include="Microsoft.PowerShell.Commands.Management" Version="$(PerfTargetVersion)" />
+    <PackageReference Include="Microsoft.PowerShell.Commands.Utility" Version="$(PerfTargetVersion)" />
+    <PackageReference Include="Microsoft.PowerShell.ConsoleHost" Version="$(PerfTargetVersion)" />
+    <PackageReference Include="Microsoft.PowerShell.Security" Version="$(PerfTargetVersion)" />
+    <PackageReference Include="Microsoft.PowerShell.CoreCLR.Eventing" Version="$(PerfTargetVersion)" />
+    <PackageReference Include="System.Management.Automation" Version="$(PerfTargetVersion)" />
   </ItemGroup>
 
 </Project>
@@ -14,9 +14,15 @@ function Start-Benchmarking
     The version of 'Microsoft.PowerShell.SDK' package that we want the benchmark to target.
     The supported versions are 7.0.x and above, including preview versions.
 
+    .PARAMETER TargetFramework
+    The target framework to run benchmarks against.
+
     .PARAMETER List
     List the available benchmarks, in either 'flat' or 'tree' views.
 
+    .PARAMETER Runtime
+    Run benchmarks against multiple .NET runtimes.
+
     .PARAMETER Filter
     One or more wildcard patterns to filter the benchmarks to be executed or to be listed.
 
@@ -26,16 +32,26 @@ function Start-Benchmarking
     .PARAMETER KeepFiles
     Indicates to keep all temporary files produced for running benchmarks.
     #>
-    [CmdletBinding()]
+    [CmdletBinding(DefaultParameterSetName = 'Default')]
     param(
+        [Parameter(ParameterSetName = 'Default')]
         [ValidatePattern(
             '^7\.(0|1|2)\.\d+(-preview\.\d{1,2})?$',
             ErrorMessage = 'The package version is invalid or not supported')]
         [string] $TargetPSVersion,
 
+        [Parameter(ParameterSetName = 'Default')]
+        [ValidateSet('netcoreapp3.1', 'net5.0', 'net6.0')]
+        [string] $TargetFramework = 'net6.0',
+
+        [Parameter(ParameterSetName = 'Default')]
         [ValidateSet('flat', 'tree')]
         [string] $List,
 
+        [Parameter(Mandatory, ParameterSetName = 'Runtimes')]
+        [ValidateSet('netcoreapp3.1', 'net5.0', 'net6.0')]
+        [string[]] $Runtime,
+
         [string[]] $Filter = '*',
         [string] $Artifacts,
         [switch] $KeepFiles
@@ -53,25 +69,71 @@ function Start-Benchmarking
         if (Test-Path -Path $Artifacts) {
             Remove-Item -Path $Artifacts -Recurse -Force -ErrorAction Stop
         }
+
+        if ($TargetPSVersion) {
+            ## Validate the specified 'TargetPSVersion' and 'TargetFramework' are compatible.
+            $TargetFramework -match '\d.\d' > $null
+            $targetDotNetVersion = $Matches[0]
+
+            $minimalDotNetVersion = switch -Wildcard ($TargetPSVersion) {
+                '7.0.*' { '3.1' }
+                '7.1.*' { '5.0' }
+                '7.2.*' { '6.0' }
+            }
+
+            if ($targetDotNetVersion -lt $minimalDotNetVersion) {
+                $dotnetVer = $minimalDotNetVersion -eq '3.1' ? 'netcoreapp3.1' : "net$minimalDotNetVersion"
+                throw "The '$TargetPSVersion' version of 'Microsoft.PowerShell.SDK' requires '$dotnetVer' as the minimal target framework."
+            }
+        }
+
+        if ($Runtime) {
+            ## Remove duplicate values.
+            $hash = [ordered]@{}
+            foreach ($item in $Runtime) {
+                if (-not $hash.Contains($item)) {
+                    $hash.Add($item, $null)
+                }
+            }
+            $Runtime = $hash.Keys
+        }
     }
 
     End {
         try {
             Push-Location -Path "$PSScriptRoot/benchmarks"
             $savedOFS = $OFS; $OFS = $null
 
-            if ($TargetPSVersion) {
-                Write-Log -message "Run benchmarks targeting the 'Microsoft.PowerShell.SDK' version $TargetPSVersion..."
-                $env:PERF_TARGET_VERSION = $TargetPSVersion
-            } else {
-                Write-Log -message "Run benchmarks targeting the current PowerShell code base..."
-            }
+            ## Aggregate BDN arguments.
+            $runArgs = @('--filter')
+            foreach ($entry in $Filter) { $runArgs += $entry }
+            $runArgs += '--artifacts', $Artifacts
+            $runArgs += '--envVars', 'POWERSHELL_TELEMETRY_OPTOUT:1'
 
-            $runArgs = @()
             if ($List) { $runArgs += '--list', $List }
             if ($KeepFiles) { $runArgs += "--keepFiles" }
 
-            dotnet run -c release --filter $Filter --artifacts $Artifacts --envVars POWERSHELL_TELEMETRY_OPTOUT:1 $runArgs
+            if ($PSCmdlet.ParameterSetName -eq 'Default') {
+                if ($TargetPSVersion) {
+                    Write-Log -message "Run benchmarks targeting '$TargetFramework' and the 'Microsoft.PowerShell.SDK' version '$TargetPSVersion' ..."
+                    $env:PERF_TARGET_VERSION = $TargetPSVersion
+                }
+                elseif ($TargetFramework -eq 'net6.0') {
+                    Write-Log -message "Run benchmarks targeting the current PowerShell code base ..."
+                }
+                else {
+                    Write-Log -message "Run benchmarks targeting '$TargetFramework' and the corresponding latest version of 'Microsoft.PowerShell.SDK' ..."
+                }
+
+                ## Use 'Release' instead of 'release' (note the capital case) because BDN uses 'Release' when building the auto-generated
+                ## project, and MSBuild somehow recognizes 'release' and 'Release' as two different configurations and thus will rebuild
+                ## all dependencies unnecessarily.
+                dotnet run -c Release -f $TargetFramework $runArgs
+            }
+            else {
+                Write-Log -message "Run benchmarks targeting multiple .NET runtimes: $Runtime ..."
+                dotnet run -c Release -f net6.0 --runtimes $Runtime $runArgs
+            }
 
             if (Test-Path $Artifacts) {
                 Write-Log -message "`nBenchmark artifacts can be found at $Artifacts"
@@ -142,7 +204,7 @@ function Compare-BenchmarkResult
         if ($Noise) { $runArgs += "--noise $Noise" }
         if ($Top -gt 0) { $runArgs += "--top $Top" }
 
-        dotnet run -c release --base $BaseResultPath --diff $DiffResultPath --threshold $Threshold "$runArgs"
+        dotnet run -c Release --base $BaseResultPath --diff $DiffResultPath --threshold $Threshold "$runArgs"
     }
     finally {
         $OFS = $savedOFS