Add the performance benchmark project for PowerShell performance testing #15242
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1309,3 +1309,6 @@ codesign | |
| release-BuildJson | ||
| yml | ||
| centos-7 | ||
| - test/perf/benchmarks/README.md | ||
| benchmarked | ||
| BenchmarkDotNet | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT License. | ||
|
|
||
| namespace MicroBenchmarks | ||
| { | ||
| public static class Categories | ||
| { | ||
| /// <summary> | ||
| /// Benchmarks belonging to this category are executed for CI jobs. | ||
| /// </summary> | ||
| public const string Components = "Components"; | ||
|
|
||
| /// <summary> | ||
| /// Benchmarks belonging to this category are executed for CI jobs. | ||
| /// </summary> | ||
| public const string Engine = "Engine"; | ||
|
|
||
| /// <summary> | ||
| /// Benchmarks belonging to this category are targeting internal APIs. | ||
| /// </summary> | ||
| public const string Internal = "Internal"; | ||
|
|
||
| /// <summary> | ||
| /// Benchmarks belonging to this category are targeting public APIs. | ||
| /// </summary> | ||
| public const string Public = "Public"; | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System.Management.Automation.Language; | ||
| using BenchmarkDotNet.Attributes; | ||
| using MicroBenchmarks; | ||
|
|
||
| namespace Engine | ||
| { | ||
| [BenchmarkCategory(Categories.Engine, Categories.Public)] | ||
| public class Parsing | ||
| { | ||
| [Benchmark] | ||
| public Ast UsingStatement() | ||
| { | ||
| const string Script = @" | ||
| using module moduleA | ||
| using Assembly assemblyA | ||
| using namespace System.IO"; | ||
| return Parser.ParseInput(Script, out _, out _); | ||
| } | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System.Collections.Generic; | ||
| using System.Collections.ObjectModel; | ||
| using System.Management.Automation; | ||
| using System.Management.Automation.Runspaces; | ||
| using System.Runtime.InteropServices; | ||
| using BenchmarkDotNet.Attributes; | ||
| using MicroBenchmarks; | ||
|
|
||
| namespace Engine | ||
| { | ||
| [BenchmarkCategory(Categories.Engine, Categories.Public)] | ||
| public class Scripting | ||
| { | ||
| private Runspace runspace; | ||
| private ScriptBlock scriptBlock; | ||
|
|
||
| private void SetupRunspace() | ||
| { | ||
| // Unless you want to run commands from any built-in modules, using 'CreateDefault2' is enough. | ||
| runspace = RunspaceFactory.CreateRunspace(InitialSessionState.CreateDefault2()); | ||
|
daxian-dbw marked this conversation as resolved.
|
||
| runspace.Open(); | ||
| Runspace.DefaultRunspace = runspace; | ||
| } | ||
|
|
||
| #region Invoke-Method | ||
|
|
||
| [ParamsSource(nameof(ValuesForScript))] | ||
| public string InvokeMethodScript { get; set; } | ||
|
|
||
| public IEnumerable<string> ValuesForScript() | ||
| { | ||
| yield return @"'String'.GetType()"; | ||
| yield return @"[System.IO.Path]::HasExtension('')"; | ||
|
|
||
| // Test on COM method invocation. | ||
| if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) | ||
| { | ||
| yield return @"$sh=New-Object -ComObject Shell.Application; $sh.Namespace('c:\')"; | ||
| yield return @"$fs=New-Object -ComObject scripting.filesystemobject; $fs.Drives"; | ||
| } | ||
| } | ||
|
|
||
| [GlobalSetup(Target = nameof(InvokeMethod))] | ||
| public void GlobalSetup() | ||
| { | ||
| SetupRunspace(); | ||
|
Member
There was a problem hiding this comment. Should we make sure that the PowerShell / .NET telemetry is off.
Member
Author
There was a problem hiding this comment. Good question. Maybe we should disable it and I will do that from the command line level. |
||
| scriptBlock = ScriptBlock.Create(InvokeMethodScript); | ||
|
|
||
| // Run it once to get the C# code jitted and the script compiled. | ||
| // The first call to this takes relatively too long, which makes the BDN's heuristic incorrectly | ||
| // believe that there is no need to run many ops in each interation. However, the subsequent runs | ||
| // of this method is much faster than the first run, and this causes 'MinIterationTime' warnings | ||
| // to our benchmarks and make the benchmark results not reliable. | ||
| // Calling this method once in 'GlobalSetup' is a workaround. | ||
| // See https://github.com/dotnet/BenchmarkDotNet/issues/837#issuecomment-828600157 | ||
| scriptBlock.Invoke(); | ||
|
daxian-dbw marked this conversation as resolved.
|
||
| } | ||
|
|
||
| [Benchmark] | ||
| public Collection<PSObject> InvokeMethod() | ||
| { | ||
| return scriptBlock.Invoke(); | ||
| } | ||
|
|
||
| #endregion | ||
|
|
||
| [GlobalCleanup] | ||
| public void GlobalCleanup() | ||
| { | ||
| runspace.Dispose(); | ||
| Runspace.DefaultRunspace = null; | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Collections.Immutable; | ||
| using System.IO; | ||
| using BenchmarkDotNet.Running; | ||
| using BenchmarkDotNet.Extensions; | ||
|
|
||
| namespace MicroBenchmarks | ||
| { | ||
| public class Program | ||
| { | ||
| public static int Main(string[] args) | ||
| { | ||
| var argsList = new List<string>(args); | ||
| int? partitionCount; | ||
| int? partitionIndex; | ||
| List<string> exclusionFilterValue; | ||
| List<string> categoryExclusionFilterValue; | ||
| bool getDiffableDisasm; | ||
|
|
||
| // Parse and remove any additional parameters that we need that aren't part of BDN (BenchmarkDotnet) | ||
| try | ||
| { | ||
| CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-count", out partitionCount); | ||
| CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out partitionIndex); | ||
| CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out exclusionFilterValue); | ||
| CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--category-exclusion-filter", out categoryExclusionFilterValue); | ||
| CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--disasm-diff", out getDiffableDisasm); | ||
|
|
||
| CommandLineOptions.ValidatePartitionParameters(partitionCount, partitionIndex); | ||
|
Contributor
There was a problem hiding this comment. to be honest with you I don't think that you are going to need any of these custom arguments in the near future:
Member
Author
There was a problem hiding this comment. That's good to know. As for the
where is the reflection hack?
Member
Author
There was a problem hiding this comment. The The
Contributor
There was a problem hiding this comment. |
||
| } | ||
| catch (ArgumentException e) | ||
| { | ||
| Console.WriteLine("ArgumentException: {0}", e.Message); | ||
| return 1; | ||
| } | ||
|
|
||
| return BenchmarkSwitcher | ||
| .FromAssembly(typeof(Program).Assembly) | ||
| .Run( | ||
| argsList.ToArray(), | ||
| RecommendedConfig.Create( | ||
| artifactsPath: new DirectoryInfo(Path.Combine(Path.GetDirectoryName(typeof(Program).Assembly.Location), "BenchmarkDotNet.Artifacts")), | ||
| mandatoryCategories: ImmutableHashSet.Create(Categories.Components, Categories.Engine), | ||
| partitionCount: partitionCount, | ||
| partitionIndex: partitionIndex, | ||
| exclusionFilterValue: exclusionFilterValue, | ||
| categoryExclusionFilterValue: categoryExclusionFilterValue, | ||
| getDiffableDisasm: getDiffableDisasm)) | ||
| .ToExitCode(); | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| ## Micro Benchmarks | ||
|
|
||
| This folder contains micro benchmarks that test the performance of PowerShell Engine. | ||
|
|
||
| ### Requirement | ||
|
|
||
| 1. A good suite of benchmarks | ||
| Something that measures only the thing that we are interested in and _produces accurate, stable and repeatable results_. | ||
| 2. A set of machine with the same configurations. | ||
| 3. Automation for regression detection. | ||
|
|
||
| ### Design Decision | ||
|
|
||
| 1. This project is internal visible to `System.Management.Automation`. | ||
| We want to be able to target some internal APIs to get measurements on specific scoped scenarios, | ||
| such as measuring the time to compile AST to a delegate by the compiler. | ||
| 2. This project makes `ProjectReference` to other PowerShell assemblies. | ||
| This makes it easy to run benchmarks with the changes made in the codebase. | ||
| To run benchmarks with a specific version of PowerShell, | ||
| just replace the `ProjectReference` with a `PackageReference` to the `Microsoft.PowerShell.SDK` NuGet package of the corresponding version. | ||
|
|
||
| ### Quick Start | ||
|
|
||
| You can run the benchmarks directly using `dotnet run` in this directory: | ||
| 1. To run the benchmarks in Interactive Mode, where you will be asked which benchmark(s) to run: | ||
| ``` | ||
| dotnet run -c release | ||
| ``` | ||
|
|
||
| 2. To list all available benchmarks ([read more](https://github.com/dotnet/performance/blob/main/docs/benchmarkdotnet.md#Listing-the-Benchmarks)): | ||
| ``` | ||
| dotnet run -c release --list [flat/tree] | ||
| ``` | ||
|
|
||
| 3. To filter the benchmarks using a glob pattern applied to `namespace.typeName.methodName` ([read more](https://github.com/dotnet/performance/blob/main/docs/benchmarkdotnet.md#Filtering-the-Benchmarks)]): | ||
| ``` | ||
| dotnet run -c Release -f net6.0 --filter *script* --list flat | ||
| ``` | ||
|
|
||
| 4. To profile the benchmarked code and produce an ETW Trace file ([read more](https://github.com/dotnet/performance/blob/main/docs/benchmarkdotnet.md#Profiling)) | ||
| ``` | ||
| dotnet run -c Release -f net6.0 --filter *script* --profiler ETW | ||
| ``` | ||
|
|
||
| You can also use the function `Start-Benchmarking` from the module [`perf.psm1`](../perf.psm1) to run the benchmarks: | ||
| ```powershell | ||
| Start-Benchmarking [[-TargetPSVersion] <string>] [[-List] <string>] [[-Filter] <string[]>] [[-Artifacts] <string>] [-KeepFiles] [<CommonParameters>] | ||
| ``` | ||
| Run `Get-Help Start-Benchmarking -Full` to see the description of each parameter. | ||
|
|
||
| ### Regression Detection | ||
|
daxian-dbw marked this conversation as resolved.
|
||
|
|
||
| We use the tool [`ResultsComparer`](../dotnet-tools/ResultsComparer) to compare the provided benchmark results. | ||
| See the [README.md](../dotnet-tools/ResultsComparer/README.md) for `ResultsComparer` for more details. | ||
|
|
||
| The module `perf.psm1` also provides `Compare-BenchmarkResult` that wraps `ResultsComparer`. | ||
| Here is an example of using it: | ||
|
|
||
| ``` | ||
| ## Run benchmarks targeting the current code base | ||
| PS:1> Start-Benchmarking -Filter *script* -Artifacts C:\arena\tmp\BenchmarkDotNet.Artifacts\current\ | ||
|
|
||
| ## Run benchmarks targeting the 7.1.3 version of PS package | ||
| PS:2> Start-Benchmarking -Filter *script* -Artifacts C:\arena\tmp\BenchmarkDotNet.Artifacts\7.1.3 -TargetPSVersion 7.1.3 | ||
|
|
||
| ## Compare the results using 5% threshold | ||
| PS:3> Compare-BenchmarkResult -BaseResultPath C:\arena\tmp\BenchmarkDotNet.Artifacts\7.1.3\ -DiffResultPath C:\arena\tmp\BenchmarkDotNet.Artifacts\current\ -Threshold 1% | ||
| summary: | ||
| better: 4, geomean: 1.057 | ||
| total diff: 4 | ||
|
|
||
| No Slower results for the provided threshold = 1% and noise filter = 0.3ns. | ||
|
|
||
| | Faster | base/diff | Base Median (ns) | Diff Median (ns) | Modality| | ||
| | -------------------------------------------------------------------------------- | ---------:| ----------------:| ----------------:| --------:| | ||
| | Engine.Scripting.InvokeMethod(Script: "$fs=New-Object -ComObject scripting.files | 1.07 | 50635.77 | 47116.42 | | | ||
| | Engine.Scripting.InvokeMethod(Script: "$sh=New-Object -ComObject Shell.Applicati | 1.07 | 1063085.23 | 991602.08 | | | ||
| | Engine.Scripting.InvokeMethod(Script: "'String'.GetType()") | 1.06 | 1329.93 | 1252.51 | | | ||
| | Engine.Scripting.InvokeMethod(Script: "[System.IO.Path]::HasExtension('')") | 1.02 | 1322.04 | 1297.72 | | | ||
|
|
||
| No file given | ||
| ``` | ||
|
|
||
| ## References | ||
|
|
||
| - [Getting started with BenchmarkDotNet](https://benchmarkdotnet.org/articles/guides/getting-started.html) | ||
| - [Micro-benchmark Design Guidelines](https://github.com/dotnet/performance/blob/main/docs/microbenchmark-design-guidelines.md) | ||
| - [Adam SITNIK: Powerful benchmarking in .NET](https://www.youtube.com/watch?v=pdcrSG4tOLI&t=351s) | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.