From 40928f7a97ab244a102381a0ac2206dc0c8a44c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Aug 2021 09:24:35 -0700 Subject: [PATCH 001/645] Bump `Microsoft.CodeAnalysis.NetAnalyzers` to newer version (#15962) --- Analyzers.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Analyzers.props b/Analyzers.props index c8d452755b8..b6270442e99 100644 --- a/Analyzers.props +++ b/Analyzers.props @@ -1,7 +1,7 @@ - + From e70ad6ce251d2e1d4ef79e31756cededde7ccc91 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Fri, 20 Aug 2021 11:23:30 -0700 Subject: [PATCH 002/645] Publish global tool package for stable releases (#15961) --- .../azureDevOps/templates/release-ReleaseToNuGet.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/release-ReleaseToNuGet.yml b/tools/releaseBuild/azureDevOps/templates/release-ReleaseToNuGet.yml index e134e4ccc7b..117cf8f91df 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-ReleaseToNuGet.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-ReleaseToNuGet.yml @@ -29,8 +29,11 @@ steps: $releaseVersion = Get-Content "$ENV:PIPELINE_WORKSPACE/releasePipeline/metadata/release.json" | ConvertFrom-Json | Select-Object -ExpandProperty 'ReleaseVersion' $globalToolPath = "$ENV:PIPELINE_WORKSPACE/releasePipeline/finalResults/PowerShell.$releaseVersion.nupkg" - ### -WhatIf to make sure we do not release global tool. Remove -WhatIf when the PowerShell name reservation is done. - Copy-Item $globalToolPath -Destination "$(Pipeline.Workspace)/release" -WhatIf + + if ($releaseVersion -notlike '*-*') { + # Copy the global tool package for stable releases + Copy-Item $globalToolPath -Destination "$(Pipeline.Workspace)/release" + } Get-ChildItem "$(Pipeline.Workspace)/release" -recurse displayName: Download and capture nupkgs From c39d672560948a06a1391e37bb460d6b223eb749 Mon Sep 17 00:00:00 2001 From: "James Truher [MSFT]" Date: Sun, 22 Aug 2021 10:49:07 -0700 Subject: [PATCH 003/645] Remove 4 assertions which cause debug build test runs to fail (#15963) --- .../host/msh/PendingProgress.cs | 12 ------------ .../help/UpdatableHelpModuleInfo.cs | 1 - .../help/UpdatableHelpUri.cs | 1 - 3 files changed, 14 deletions(-) diff --git a/src/Microsoft.PowerShell.ConsoleHost/host/msh/PendingProgress.cs b/src/Microsoft.PowerShell.ConsoleHost/host/msh/PendingProgress.cs index 6bb85c64729..fad667dba2d 100644 --- a/src/Microsoft.PowerShell.ConsoleHost/host/msh/PendingProgress.cs +++ b/src/Microsoft.PowerShell.ConsoleHost/host/msh/PendingProgress.cs @@ -851,18 +851,6 @@ internal override } // If we get all the way to here, then we've compressed all the nodes and we still don't fit. - -#if DEBUG || ASSERTIONS_TRACE - - Dbg.Assert( - nodesCompressed == CountNodes(), - "We should have compressed every node in the tree."); - Dbg.Assert( - AllNodesHaveGivenStyle(_topLevelNodes, newStyle), - "We should have compressed every node in the tree."); - -#endif - return false; } diff --git a/src/System.Management.Automation/help/UpdatableHelpModuleInfo.cs b/src/System.Management.Automation/help/UpdatableHelpModuleInfo.cs index b46717920f4..ccf0b95f0f6 100644 --- a/src/System.Management.Automation/help/UpdatableHelpModuleInfo.cs +++ b/src/System.Management.Automation/help/UpdatableHelpModuleInfo.cs @@ -28,7 +28,6 @@ internal class UpdatableHelpModuleInfo internal UpdatableHelpModuleInfo(string name, Guid guid, string path, string uri) { Debug.Assert(!string.IsNullOrEmpty(name)); - Debug.Assert(guid != Guid.Empty); Debug.Assert(!string.IsNullOrEmpty(path)); Debug.Assert(!string.IsNullOrEmpty(uri)); diff --git a/src/System.Management.Automation/help/UpdatableHelpUri.cs b/src/System.Management.Automation/help/UpdatableHelpUri.cs index 8b854e82a5d..82bbb4016fb 100644 --- a/src/System.Management.Automation/help/UpdatableHelpUri.cs +++ b/src/System.Management.Automation/help/UpdatableHelpUri.cs @@ -21,7 +21,6 @@ internal class UpdatableHelpUri internal UpdatableHelpUri(string moduleName, Guid moduleGuid, CultureInfo culture, string resolvedUri) { Debug.Assert(!string.IsNullOrEmpty(moduleName)); - Debug.Assert(moduleGuid != Guid.Empty); Debug.Assert(!string.IsNullOrEmpty(resolvedUri)); ModuleName = moduleName; From b28318857336c0981ea59d779717834cb0ff072a Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Mon, 23 Aug 2021 15:08:41 -0700 Subject: [PATCH 004/645] Update `readme` and `metadata.json` for 7.2.0-preview.9 release (#15957) --- README.md | 32 ++++++++++++++++---------------- tools/metadata.json | 4 ++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d30953d7d5b..7f83f92f6bc 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,8 @@ You can also download the PowerShell binary archives for Windows, macOS and Linu | Platform | Downloads (stable) | Downloads (preview) | How to Install | | ---------------| --------------------------------------------------- | ------------------------------------------------| -----------------------------------------------| | Windows | [32-bit][rl-winx86-zip]/[64-bit][rl-winx64-zip] | [32-bit][pv-winx86-zip]/[64-bit][pv-winx64-zip] | [Instructions][in-windows-zip] | -| macOS | [64-bit][rl-macos-tar] | [64-bit][pv-macos-tar] | [Instructions][in-tar-macos] | -| macOS | | [64-bit][pv-macos-tar-arm64] | [Instructions][in-tar-macos] | +| macOS (x64) | [64-bit][rl-macos-tar] | [64-bit][pv-macos-tar] | [Instructions][in-tar-macos] | +| macOS (arm64) | | [64-bit][pv-macos-tar-arm64] | [Instructions][in-tar-macos] | | Linux | [64-bit][rl-linux-tar] | [64-bit][pv-linux-tar] | [Instructions][in-tar-linux] | | Windows (Arm) | [64-bit][rl-winarm64] (preview) | [64-bit][pv-winarm64] | [Instructions][in-arm] | | Raspbian (Arm) | [32-bit][rl-arm32]/[64-bit][rl-arm64] | [32-bit][pv-arm32]/[64-bit][pv-arm64] | [Instructions][in-raspbian] | @@ -94,20 +94,20 @@ You can also download the PowerShell binary archives for Windows, macOS and Linu [rl-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.1.4/powershell-7.1.4-linux-arm64.tar.gz [rl-snap]: https://snapcraft.io/powershell -[pv-windows-64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/PowerShell-7.2.0-preview.8-win-x64.msi -[pv-windows-86]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/PowerShell-7.2.0-preview.8-win-x86.msi -[pv-deb]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/powershell-preview_7.2.0-preview.8-1.deb_amd64.deb -[pv-rpm]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/powershell-preview-7.2.0_preview.8-1.rh.x86_64.rpm -[pv-macos]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/powershell-7.2.0-preview.8-osx-x64.pkg -[pv-macos-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/powershell-7.2.0-preview.8-osx-arm64.pkg -[pv-winarm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/PowerShell-7.2.0-preview.8-win-arm64.zip -[pv-winx86-zip]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/PowerShell-7.2.0-preview.8-win-x86.zip -[pv-winx64-zip]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/PowerShell-7.2.0-preview.8-win-x64.zip -[pv-macos-tar]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/powershell-7.2.0-preview.8-osx-x64.tar.gz -[pv-macos-tar-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/powershell-7.2.0-preview.8-osx-arm64.tar.gz -[pv-linux-tar]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/powershell-7.2.0-preview.8-linux-x64.tar.gz -[pv-arm32]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/powershell-7.2.0-preview.8-linux-arm32.tar.gz -[pv-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.8/powershell-7.2.0-preview.8-linux-arm64.tar.gz +[pv-windows-64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/PowerShell-7.2.0-preview.9-win-x64.msi +[pv-windows-86]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/PowerShell-7.2.0-preview.9-win-x86.msi +[pv-deb]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-preview_7.2.0-preview.9-1.deb_amd64.deb +[pv-rpm]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-preview-7.2.0_preview.9-1.rh.x86_64.rpm +[pv-macos]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-osx-x64.pkg +[pv-macos-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-osx-arm64.pkg +[pv-winarm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/PowerShell-7.2.0-preview.9-win-arm64.zip +[pv-winx86-zip]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/PowerShell-7.2.0-preview.9-win-x86.zip +[pv-winx64-zip]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/PowerShell-7.2.0-preview.9-win-x64.zip +[pv-macos-tar]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-osx-x64.tar.gz +[pv-macos-tar-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-osx-arm64.tar.gz +[pv-linux-tar]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-linux-x64.tar.gz +[pv-arm32]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-linux-arm32.tar.gz +[pv-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-linux-arm64.tar.gz [pv-snap]: https://snapcraft.io/powershell-preview [in-windows]: https://docs.microsoft.com/powershell/scripting/install/installing-powershell-core-on-windows diff --git a/tools/metadata.json b/tools/metadata.json index 0a2ce2ebffd..88db98aa613 100644 --- a/tools/metadata.json +++ b/tools/metadata.json @@ -1,9 +1,9 @@ { "StableReleaseTag": "v7.1.4", - "PreviewReleaseTag": "v7.2.0-preview.8", + "PreviewReleaseTag": "v7.2.0-preview.9", "ServicingReleaseTag": "v7.0.7", "ReleaseTag": "v7.1.4", "LTSReleaseTag" : ["v7.0.7"], - "NextReleaseTag": "v7.2.0-preview.9", + "NextReleaseTag": "v7.2.0-preview.10", "LTSRelease": false } From ab5e1779a3b31cd652a367fa55554747f4261c93 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Mon, 23 Aug 2021 17:20:29 -0700 Subject: [PATCH 005/645] Revert "Use `GetValueOrDefault()` for nullable `PSLanguageMode` (#13849)" (#15980) --- src/System.Management.Automation/engine/ExternalScriptInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/System.Management.Automation/engine/ExternalScriptInfo.cs b/src/System.Management.Automation/engine/ExternalScriptInfo.cs index 9bf3318a2c8..bbd49c76d36 100644 --- a/src/System.Management.Automation/engine/ExternalScriptInfo.cs +++ b/src/System.Management.Automation/engine/ExternalScriptInfo.cs @@ -242,7 +242,7 @@ private static ScriptBlock ParseScriptContents(Parser parser, string fileName, s // If we are in ConstrainedLanguage mode but the defining language mode is FullLanguage, then we need // to parse the script contents in FullLanguage mode context. Otherwise we will get bogus parsing errors // such as "Configuration keyword not allowed". - if (definingLanguageMode.GetValueOrDefault() == PSLanguageMode.FullLanguage) + if (definingLanguageMode.HasValue && (definingLanguageMode == PSLanguageMode.FullLanguage)) { var context = LocalPipeline.GetExecutionContextFromTLS(); if ((context != null) && (context.LanguageMode == PSLanguageMode.ConstrainedLanguage)) From 07e466871a8f6e50e23ca82c46d8c9e67196fd99 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Aug 2021 08:09:55 -0700 Subject: [PATCH 006/645] Bump `Microsoft.CodeAnalysis.NetAnalyzers` (#15985) --- Analyzers.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Analyzers.props b/Analyzers.props index b6270442e99..d85c827bf21 100644 --- a/Analyzers.props +++ b/Analyzers.props @@ -1,7 +1,7 @@ - + From bd84f4265bfb9e3e80d4af6a5730db19d7ae5846 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Tue, 24 Aug 2021 09:55:42 -0700 Subject: [PATCH 007/645] Update metadata to start using .NET 6 RC1 builds (#15981) --- DotnetRuntimeMetadata.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DotnetRuntimeMetadata.json b/DotnetRuntimeMetadata.json index e7a597bae5a..18778f58555 100644 --- a/DotnetRuntimeMetadata.json +++ b/DotnetRuntimeMetadata.json @@ -1,11 +1,11 @@ { "sdk": { - "channel": "6.0.1xx-preview6", + "channel": "6.0.1xx-rc1", "quality": "signed", "qualityFallback": "daily", - "packageVersionPattern": "6.0.0-preview.6", + "packageVersionPattern": "6.0.0-rc.1", "sdkImageVersion": "6.0.100", - "nextChannel": "6.0.1xx-preview7" + "nextChannel": "6.0.1xx-rc1" }, "internalfeed" : { "url": null From 56d22bc3865fca445145fd685d6d6f6d63194701 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Tue, 24 Aug 2021 10:20:49 -0700 Subject: [PATCH 008/645] Enable sending Teams notification when workflow fails (#15982) --- .github/workflows/daily.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 1535fced837..6e3a38ac1b1 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -43,6 +43,11 @@ jobs: Write-Verbose "CREATE_PR=true" -Verbose "CREATE_PR=true" | Out-File $env:GITHUB_ENV -Append } + - name: Microsoft Teams Notifier + uses: skitionek/notify-microsoft-teams@master + if: failure() + with: + webhook_url: ${{ secrets.PS_BUILD_TEAMS_CHANNEL }} - name: Create Pull Request uses: peter-evans/create-pull-request@v2 id: cpr From a4d14576b3a0c9a1898daedbdf29fbf5436a63e3 Mon Sep 17 00:00:00 2001 From: Robert Holt Date: Tue, 24 Aug 2021 15:27:22 -0700 Subject: [PATCH 009/645] Make the native command error handling optionally honor `ErrorActionPreference` (#15897) --- .../ExperimentalFeature.cs | 4 + .../engine/InitialSessionState.cs | 309 ++++++++++-------- .../engine/MshCommandRuntime.cs | 18 +- .../engine/NativeCommandProcessor.cs | 146 ++++++++- .../engine/SpecialVariables.cs | 48 +-- .../resources/CommandBaseStrings.resx | 9 + .../resources/RunspaceInit.resx | 3 + .../NativeCommandErrorHandling.Tests.ps1 | 178 ++++++++++ 8 files changed, 534 insertions(+), 181 deletions(-) create mode 100644 test/powershell/engine/Basic/NativeCommandErrorHandling.Tests.ps1 diff --git a/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs b/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs index 9268003ee87..dd0f7209635 100644 --- a/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs +++ b/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs @@ -22,6 +22,7 @@ public class ExperimentalFeature internal const string EngineSource = "PSEngine"; internal const string PSNativeCommandArgumentPassingFeatureName = "PSNativeCommandArgumentPassing"; + internal const string PSNativeCommandErrorActionPreferenceFeatureName = "PSNativeCommandErrorActionPreference"; #endregion @@ -122,6 +123,9 @@ static ExperimentalFeature() new ExperimentalFeature( name: "PSAnsiRenderingFileInfo", description: "Enable coloring for FileInfo objects"), + new ExperimentalFeature( + name: PSNativeCommandErrorActionPreferenceFeatureName, + description: "Native commands with non-zero exit codes issue errors according to $ErrorActionPreference when $PSNativeCommandUseErrorActionPreference is $true"), }; EngineExperimentalFeatures = new ReadOnlyCollection(engineFeatures); diff --git a/src/System.Management.Automation/engine/InitialSessionState.cs b/src/System.Management.Automation/engine/InitialSessionState.cs index 99aa639d09c..07c0d250004 100644 --- a/src/System.Management.Automation/engine/InitialSessionState.cs +++ b/src/System.Management.Automation/engine/InitialSessionState.cs @@ -4471,154 +4471,173 @@ .ForwardHelpCategory Cmdlet internal const ActionPreference DefaultInformationPreference = ActionPreference.SilentlyContinue; internal const ErrorView DefaultErrorView = ErrorView.ConciseView; + internal const bool DefaultPSNativeCommandUseErrorActionPreference = false; internal const bool DefaultWhatIfPreference = false; internal const ConfirmImpact DefaultConfirmPreference = ConfirmImpact.High; - internal static readonly SessionStateVariableEntry[] BuiltInVariables = new SessionStateVariableEntry[] - { - // Engine variables that should be precreated before running profile - // Bug fix for Win7:2202228 Engine halts if initial command fulls up variable table - // Anytime a new variable that the engine depends on to run is added, this table - // must be updated... - new SessionStateVariableEntry(SpecialVariables.LastToken, null, string.Empty), - new SessionStateVariableEntry(SpecialVariables.FirstToken, null, string.Empty), - new SessionStateVariableEntry(SpecialVariables.StackTrace, null, string.Empty), - - // Variable which controls the output rendering - new SessionStateVariableEntry( - SpecialVariables.PSStyle, - PSStyle.Instance, - RunspaceInit.PSStyleDescription, - ScopedItemOptions.None), - - // Variable which controls the encoding for piping data to a NativeCommand - new SessionStateVariableEntry( - SpecialVariables.OutputEncoding, - Utils.utf8NoBom, - RunspaceInit.OutputEncodingDescription, - ScopedItemOptions.None, - new ArgumentTypeConverterAttribute(typeof(System.Text.Encoding))), - - // Preferences - // - // NTRAID#Windows Out Of Band Releases-931461-2006/03/13 - // ArgumentTypeConverterAttribute is applied to these variables, - // but this only reaches the global variable. If these are - // redefined in script scope etc, the type conversion - // is not applicable. - // - // Variables typed to ActionPreference - new SessionStateVariableEntry( - SpecialVariables.ConfirmPreference, - DefaultConfirmPreference, - RunspaceInit.ConfirmPreferenceDescription, - ScopedItemOptions.None, - new ArgumentTypeConverterAttribute(typeof(ConfirmImpact))), - new SessionStateVariableEntry( - SpecialVariables.DebugPreference, - DefaultDebugPreference, - RunspaceInit.DebugPreferenceDescription, - ScopedItemOptions.None, - new ArgumentTypeConverterAttribute(typeof(ActionPreference))), - new SessionStateVariableEntry( - SpecialVariables.ErrorActionPreference, - DefaultErrorActionPreference, - RunspaceInit.ErrorActionPreferenceDescription, - ScopedItemOptions.None, - new ArgumentTypeConverterAttribute(typeof(ActionPreference))), - new SessionStateVariableEntry( - SpecialVariables.ProgressPreference, - DefaultProgressPreference, - RunspaceInit.ProgressPreferenceDescription, - ScopedItemOptions.None, - new ArgumentTypeConverterAttribute(typeof(ActionPreference))), - new SessionStateVariableEntry( - SpecialVariables.VerbosePreference, - DefaultVerbosePreference, - RunspaceInit.VerbosePreferenceDescription, - ScopedItemOptions.None, - new ArgumentTypeConverterAttribute(typeof(ActionPreference))), - new SessionStateVariableEntry( - SpecialVariables.WarningPreference, - DefaultWarningPreference, - RunspaceInit.WarningPreferenceDescription, - ScopedItemOptions.None, - new ArgumentTypeConverterAttribute(typeof(ActionPreference))), - new SessionStateVariableEntry( - SpecialVariables.InformationPreference, - DefaultInformationPreference, - RunspaceInit.InformationPreferenceDescription, - ScopedItemOptions.None, - new ArgumentTypeConverterAttribute(typeof(ActionPreference))), - new SessionStateVariableEntry( - SpecialVariables.ErrorView, - DefaultErrorView, - RunspaceInit.ErrorViewDescription, - ScopedItemOptions.None, - new ArgumentTypeConverterAttribute(typeof(ErrorView))), - new SessionStateVariableEntry( - SpecialVariables.NestedPromptLevel, - 0, - RunspaceInit.NestedPromptLevelDescription), - new SessionStateVariableEntry( - SpecialVariables.WhatIfPreference, - DefaultWhatIfPreference, - RunspaceInit.WhatIfPreferenceDescription), - new SessionStateVariableEntry( - FormatEnumerationLimit, - DefaultFormatEnumerationLimit, - RunspaceInit.FormatEnumerationLimitDescription), - - // variable for PSEmailServer - new SessionStateVariableEntry( - SpecialVariables.PSEmailServer, - string.Empty, - RunspaceInit.PSEmailServerDescription), - - // Start: Variables which control remoting behavior - new SessionStateVariableEntry( - Microsoft.PowerShell.Commands.PSRemotingBaseCmdlet.DEFAULT_SESSION_OPTION, - new System.Management.Automation.Remoting.PSSessionOption(), - RemotingErrorIdStrings.PSDefaultSessionOptionDescription, - ScopedItemOptions.None), - new SessionStateVariableEntry( - SpecialVariables.PSSessionConfigurationName, - "http://schemas.microsoft.com/powershell/Microsoft.PowerShell", - RemotingErrorIdStrings.PSSessionConfigurationName, - ScopedItemOptions.None), - new SessionStateVariableEntry( - SpecialVariables.PSSessionApplicationName, - "wsman", - RemotingErrorIdStrings.PSSessionAppName, - ScopedItemOptions.None), - // End: Variables which control remoting behavior - - #region Platform - new SessionStateVariableEntry( - SpecialVariables.IsLinux, - Platform.IsLinux, - string.Empty, - ScopedItemOptions.ReadOnly | ScopedItemOptions.AllScope), - - new SessionStateVariableEntry( - SpecialVariables.IsMacOS, - Platform.IsMacOS, - string.Empty, - ScopedItemOptions.ReadOnly | ScopedItemOptions.AllScope), - - new SessionStateVariableEntry( - SpecialVariables.IsWindows, - Platform.IsWindows, - string.Empty, - ScopedItemOptions.ReadOnly | ScopedItemOptions.AllScope), - - new SessionStateVariableEntry( - SpecialVariables.IsCoreCLR, - Platform.IsCoreCLR, - string.Empty, - ScopedItemOptions.ReadOnly | ScopedItemOptions.AllScope), - #endregion - }; + static InitialSessionState() + { + var builtinVariables = new List() + { + // Engine variables that should be precreated before running profile + // Bug fix for Win7:2202228 Engine halts if initial command fulls up variable table + // Anytime a new variable that the engine depends on to run is added, this table + // must be updated... + new SessionStateVariableEntry(SpecialVariables.LastToken, null, string.Empty), + new SessionStateVariableEntry(SpecialVariables.FirstToken, null, string.Empty), + new SessionStateVariableEntry(SpecialVariables.StackTrace, null, string.Empty), + + // Variable which controls the output rendering + new SessionStateVariableEntry( + SpecialVariables.PSStyle, + PSStyle.Instance, + RunspaceInit.PSStyleDescription, + ScopedItemOptions.None), + + // Variable which controls the encoding for piping data to a NativeCommand + new SessionStateVariableEntry( + SpecialVariables.OutputEncoding, + Utils.utf8NoBom, + RunspaceInit.OutputEncodingDescription, + ScopedItemOptions.None, + new ArgumentTypeConverterAttribute(typeof(System.Text.Encoding))), + + // Preferences + // + // NTRAID#Windows Out Of Band Releases-931461-2006/03/13 + // ArgumentTypeConverterAttribute is applied to these variables, + // but this only reaches the global variable. If these are + // redefined in script scope etc, the type conversion + // is not applicable. + // + // Variables typed to ActionPreference + new SessionStateVariableEntry( + SpecialVariables.ConfirmPreference, + DefaultConfirmPreference, + RunspaceInit.ConfirmPreferenceDescription, + ScopedItemOptions.None, + new ArgumentTypeConverterAttribute(typeof(ConfirmImpact))), + new SessionStateVariableEntry( + SpecialVariables.DebugPreference, + DefaultDebugPreference, + RunspaceInit.DebugPreferenceDescription, + ScopedItemOptions.None, + new ArgumentTypeConverterAttribute(typeof(ActionPreference))), + new SessionStateVariableEntry( + SpecialVariables.ErrorActionPreference, + DefaultErrorActionPreference, + RunspaceInit.ErrorActionPreferenceDescription, + ScopedItemOptions.None, + new ArgumentTypeConverterAttribute(typeof(ActionPreference))), + new SessionStateVariableEntry( + SpecialVariables.ProgressPreference, + DefaultProgressPreference, + RunspaceInit.ProgressPreferenceDescription, + ScopedItemOptions.None, + new ArgumentTypeConverterAttribute(typeof(ActionPreference))), + new SessionStateVariableEntry( + SpecialVariables.VerbosePreference, + DefaultVerbosePreference, + RunspaceInit.VerbosePreferenceDescription, + ScopedItemOptions.None, + new ArgumentTypeConverterAttribute(typeof(ActionPreference))), + new SessionStateVariableEntry( + SpecialVariables.WarningPreference, + DefaultWarningPreference, + RunspaceInit.WarningPreferenceDescription, + ScopedItemOptions.None, + new ArgumentTypeConverterAttribute(typeof(ActionPreference))), + new SessionStateVariableEntry( + SpecialVariables.InformationPreference, + DefaultInformationPreference, + RunspaceInit.InformationPreferenceDescription, + ScopedItemOptions.None, + new ArgumentTypeConverterAttribute(typeof(ActionPreference))), + new SessionStateVariableEntry( + SpecialVariables.ErrorView, + DefaultErrorView, + RunspaceInit.ErrorViewDescription, + ScopedItemOptions.None, + new ArgumentTypeConverterAttribute(typeof(ErrorView))), + new SessionStateVariableEntry( + SpecialVariables.NestedPromptLevel, + 0, + RunspaceInit.NestedPromptLevelDescription), + new SessionStateVariableEntry( + SpecialVariables.WhatIfPreference, + DefaultWhatIfPreference, + RunspaceInit.WhatIfPreferenceDescription), + new SessionStateVariableEntry( + FormatEnumerationLimit, + DefaultFormatEnumerationLimit, + RunspaceInit.FormatEnumerationLimitDescription), + + // variable for PSEmailServer + new SessionStateVariableEntry( + SpecialVariables.PSEmailServer, + string.Empty, + RunspaceInit.PSEmailServerDescription), + + // Start: Variables which control remoting behavior + new SessionStateVariableEntry( + Microsoft.PowerShell.Commands.PSRemotingBaseCmdlet.DEFAULT_SESSION_OPTION, + new System.Management.Automation.Remoting.PSSessionOption(), + RemotingErrorIdStrings.PSDefaultSessionOptionDescription, + ScopedItemOptions.None), + new SessionStateVariableEntry( + SpecialVariables.PSSessionConfigurationName, + "http://schemas.microsoft.com/powershell/Microsoft.PowerShell", + RemotingErrorIdStrings.PSSessionConfigurationName, + ScopedItemOptions.None), + new SessionStateVariableEntry( + SpecialVariables.PSSessionApplicationName, + "wsman", + RemotingErrorIdStrings.PSSessionAppName, + ScopedItemOptions.None), + // End: Variables which control remoting behavior + + #region Platform + new SessionStateVariableEntry( + SpecialVariables.IsLinux, + Platform.IsLinux, + string.Empty, + ScopedItemOptions.ReadOnly | ScopedItemOptions.AllScope), + + new SessionStateVariableEntry( + SpecialVariables.IsMacOS, + Platform.IsMacOS, + string.Empty, + ScopedItemOptions.ReadOnly | ScopedItemOptions.AllScope), + + new SessionStateVariableEntry( + SpecialVariables.IsWindows, + Platform.IsWindows, + string.Empty, + ScopedItemOptions.ReadOnly | ScopedItemOptions.AllScope), + + new SessionStateVariableEntry( + SpecialVariables.IsCoreCLR, + Platform.IsCoreCLR, + string.Empty, + ScopedItemOptions.ReadOnly | ScopedItemOptions.AllScope), + #endregion + }; + + if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandErrorActionPreferenceFeatureName)) + { + builtinVariables.Add( + new SessionStateVariableEntry( + SpecialVariables.PSNativeCommandUseErrorActionPreference, + DefaultPSNativeCommandUseErrorActionPreference, + RunspaceInit.PSNativeCommandUseErrorActionPreferenceDescription, + ScopedItemOptions.None, + new ArgumentTypeConverterAttribute(typeof(bool)))); + } + + BuiltInVariables = builtinVariables.ToArray(); + } + + internal static readonly SessionStateVariableEntry[] BuiltInVariables; /// /// Returns a new array of alias entries everytime it's called. This diff --git a/src/System.Management.Automation/engine/MshCommandRuntime.cs b/src/System.Management.Automation/engine/MshCommandRuntime.cs index 3a440478d1a..a801278aa04 100644 --- a/src/System.Management.Automation/engine/MshCommandRuntime.cs +++ b/src/System.Management.Automation/engine/MshCommandRuntime.cs @@ -2804,8 +2804,16 @@ private void DoWriteError(object obj) _WriteErrorSkipAllowCheck(errorRecord, preference); } - // NOTICE-2004/06/08-JonN 959638 - // Use this variant to skip the ThrowIfWriteNotPermitted check + /// + /// Write an error, skipping the ThrowIfWriteNotPermitted check. + /// + /// The error record to write. + /// The configured error action preference. + /// + /// True when this method is called to write from a native command's stderr stream. + /// When errors are written through a native stderr stream, they do not interact with the error preference system, + /// but must still present as errors in PowerShell. + /// /// /// The pipeline has already been terminated, or was terminated /// during the execution of this method. @@ -2819,7 +2827,7 @@ private void DoWriteError(object obj) /// but the command failure will ultimately be /// , /// - internal void _WriteErrorSkipAllowCheck(ErrorRecord errorRecord, ActionPreference? actionPreference = null, bool isNativeError = false) + internal void _WriteErrorSkipAllowCheck(ErrorRecord errorRecord, ActionPreference? actionPreference = null, bool isFromNativeStdError = false) { ThrowIfStopping(); @@ -2839,7 +2847,7 @@ internal void _WriteErrorSkipAllowCheck(ErrorRecord errorRecord, ActionPreferenc this.PipelineProcessor.LogExecutionError(_thisCommand.MyInvocation, errorRecord); } - if (!isNativeError) + if (!isFromNativeStdError) { this.PipelineProcessor.ExecutionFailed = true; @@ -2905,7 +2913,7 @@ internal void _WriteErrorSkipAllowCheck(ErrorRecord errorRecord, ActionPreferenc // when tracing), so don't add the member again. // We don't add a note property on messages that comes from stderr stream. - if (!isNativeError) + if (!isFromNativeStdError) { errorWrap.WriteStream = WriteStreamType.Error; } diff --git a/src/System.Management.Automation/engine/NativeCommandProcessor.cs b/src/System.Management.Automation/engine/NativeCommandProcessor.cs index 3407532acdb..18d5b16c82f 100644 --- a/src/System.Management.Automation/engine/NativeCommandProcessor.cs +++ b/src/System.Management.Automation/engine/NativeCommandProcessor.cs @@ -3,21 +3,22 @@ #pragma warning disable 1634, 1691 +using System.Collections; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.ComponentModel; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; using System.IO; -using System.ComponentModel; +using System.Management.Automation.Internal; +using System.Management.Automation.Runspaces; +using System.Runtime.InteropServices; +using System.Runtime.Serialization; using System.Text; -using System.Collections; using System.Threading; -using System.Management.Automation.Internal; using System.Xml; -using System.Runtime.InteropServices; using Dbg = System.Management.Automation.Diagnostics; -using System.Runtime.Serialization; -using System.Globalization; -using System.Diagnostics.CodeAnalysis; -using System.Collections.Concurrent; -using System.Collections.Generic; namespace System.Management.Automation { @@ -130,6 +131,100 @@ internal ProcessOutputObject(object data, MinishellStream stream) } } + #nullable enable + /// + /// This exception is used by the NativeCommandProcessor to indicate an error + /// when a native command retuns a non-zero exit code. + /// + [Serializable] + public sealed class NativeCommandExitException : RuntimeException + { + // NOTE: + // When implementing the native error action preference integration, + // reusing ApplicationFailedException was rejected. + // Instead of reusing a type already used in another scenario + // it was decided instead to use a fresh type to avoid conflating the two scenarios: + // * ApplicationFailedException: PowerShell was not able to complete execution of the application. + // * NativeCommandExitException: the application completed execution but returned a non-zero exit code. + + #region Constructors + + /// + /// Initializes a new instance of the class with information on the native + /// command, a specified error message and a specified error ID. + /// + /// The full path of the native command. + /// The exit code returned by the native command. + /// The process ID of the process before it ended. + /// The error message. + /// The PowerShell runtime error ID. + internal NativeCommandExitException(string path, int exitCode, int processId, string message, string errorId) + : base(message) + { + SetErrorId(errorId); + SetErrorCategory(ErrorCategory.NotSpecified); + Path = path; + ExitCode = exitCode; + ProcessId = processId; + } + + /// + /// Initializes a new instance of the class with serialized data. + /// + /// + /// + private NativeCommandExitException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + if (info is null) + { + throw new PSArgumentNullException(nameof(info)); + } + + Path = info.GetString(nameof(Path)); + ExitCode = info.GetInt32(nameof(ExitCode)); + ProcessId = info.GetInt32(nameof(ProcessId)); + } + + #endregion Constructors + + /// + /// Serializes the exception data. + /// + /// Serialization information. + /// Streaming context. + public override void GetObjectData(SerializationInfo info, StreamingContext context) + { + if (info is null) + { + throw new PSArgumentNullException(nameof(info)); + } + + base.GetObjectData(info, context); + + info.AddValue(nameof(Path), Path); + info.AddValue(nameof(ExitCode), ExitCode); + info.AddValue(nameof(ProcessId), ProcessId); + } + + /// + /// Gets the path of the native command. + /// + public string? Path { get; } + + /// + /// Gets the exit code returned by the native command. + /// + public int ExitCode { get; } + + /// + /// Gets the native command's process ID. + /// + public int ProcessId { get; } + + } + #nullable restore + /// /// Provides way to create and execute native commands. /// @@ -762,8 +857,35 @@ internal override void Complete() } this.Command.Context.SetVariable(SpecialVariables.LastExitCodeVarPath, _nativeProcess.ExitCode); - if (_nativeProcess.ExitCode != 0) - this.commandRuntime.PipelineProcessor.ExecutionFailed = true; + if (_nativeProcess.ExitCode == 0) + { + return; + } + + this.commandRuntime.PipelineProcessor.ExecutionFailed = true; + + if (!ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandErrorActionPreferenceFeatureName) + || !(bool)Command.Context.GetVariableValue(SpecialVariables.PSNativeCommandUseErrorActionPreferenceVarPath, defaultValue: false)) + { + return; + } + + const string errorId = nameof(CommandBaseStrings.ProgramExitedWithNonZeroCode); + + string errorMsg = StringUtil.Format( + CommandBaseStrings.ProgramExitedWithNonZeroCode, + NativeCommandName, + _nativeProcess.ExitCode); + + var exception = new NativeCommandExitException( + Path, + _nativeProcess.ExitCode, + _nativeProcess.Id, + errorMsg, + errorId); + + var errorRecord = new ErrorRecord(exception, errorId, ErrorCategory.NotSpecified, targetObject: NativeCommandName); + this.commandRuntime._WriteErrorSkipAllowCheck(errorRecord); } } catch (Win32Exception e) @@ -1087,7 +1209,7 @@ private void ProcessOutputRecord(ProcessOutputObject outputValue) ErrorRecord record = outputValue.Data as ErrorRecord; Dbg.Assert(record != null, "ProcessReader should ensure that data is ErrorRecord"); record.SetInvocationInfo(this.Command.MyInvocation); - this.commandRuntime._WriteErrorSkipAllowCheck(record, isNativeError: true); + this.commandRuntime._WriteErrorSkipAllowCheck(record, isFromNativeStdError: true); } else if (outputValue.Stream == MinishellStream.Output) { diff --git a/src/System.Management.Automation/engine/SpecialVariables.cs b/src/System.Management.Automation/engine/SpecialVariables.cs index 0f95349664f..0137ace9d66 100644 --- a/src/System.Management.Automation/engine/SpecialVariables.cs +++ b/src/System.Management.Automation/engine/SpecialVariables.cs @@ -255,6 +255,11 @@ internal static class SpecialVariables internal static readonly VariablePath InformationPreferenceVarPath = new VariablePath(InformationPreference); + internal const string PSNativeCommandUseErrorActionPreference = nameof(PSNativeCommandUseErrorActionPreference); + + internal static readonly VariablePath PSNativeCommandUseErrorActionPreferenceVarPath = + new(PSNativeCommandUseErrorActionPreference); + #endregion Preference Variables // Native command argument passing style @@ -321,25 +326,30 @@ internal static class SpecialVariables /* PSCommandPath */ typeof(string), }; - internal static readonly string[] PreferenceVariables = { - SpecialVariables.DebugPreference, - SpecialVariables.VerbosePreference, - SpecialVariables.ErrorActionPreference, - SpecialVariables.WhatIfPreference, - SpecialVariables.WarningPreference, - SpecialVariables.InformationPreference, - SpecialVariables.ConfirmPreference, - }; - - internal static readonly Type[] PreferenceVariableTypes = { - /* DebugPreference */ typeof(ActionPreference), - /* VerbosePreference */ typeof(ActionPreference), - /* ErrorPreference */ typeof(ActionPreference), - /* WhatIfPreference */ typeof(SwitchParameter), - /* WarningPreference */ typeof(ActionPreference), - /* InformationPreference */ typeof(ActionPreference), - /* ConfirmPreference */ typeof(ConfirmImpact), - }; + // This array and the one below it exist to optimize the way common parameters work in advanced functions. + // Common parameters work by setting preference variables in the scope of the function and restoring the old value afterward. + // Variables that don't correspond to common cmdlet parameters don't need to be added here. + internal static readonly string[] PreferenceVariables = + { + SpecialVariables.DebugPreference, + SpecialVariables.VerbosePreference, + SpecialVariables.ErrorActionPreference, + SpecialVariables.WhatIfPreference, + SpecialVariables.WarningPreference, + SpecialVariables.InformationPreference, + SpecialVariables.ConfirmPreference, + }; + + internal static readonly Type[] PreferenceVariableTypes = + { + /* DebugPreference */ typeof(ActionPreference), + /* VerbosePreference */ typeof(ActionPreference), + /* ErrorPreference */ typeof(ActionPreference), + /* WhatIfPreference */ typeof(SwitchParameter), + /* WarningPreference */ typeof(ActionPreference), + /* InformationPreference */ typeof(ActionPreference), + /* ConfirmPreference */ typeof(ConfirmImpact), + }; // The following variables are created in every session w/ AllScope. We avoid creating local slots when we // see an assignment to any of these variables so that they get handled properly (either throwing an exception diff --git a/src/System.Management.Automation/resources/CommandBaseStrings.resx b/src/System.Management.Automation/resources/CommandBaseStrings.resx index 656dc226f22..dc2dd4bbbf5 100644 --- a/src/System.Management.Automation/resources/CommandBaseStrings.resx +++ b/src/System.Management.Automation/resources/CommandBaseStrings.resx @@ -156,6 +156,15 @@ Pause the current pipeline and return to the command prompt. Type "{0}" to resume the pipeline. + + + Program "{0}" ended with non-zero exit code: {1}. + Performing the operation "{0}" on target "{1}". diff --git a/src/System.Management.Automation/resources/RunspaceInit.resx b/src/System.Management.Automation/resources/RunspaceInit.resx index a2497961901..f12ba3f71c5 100644 --- a/src/System.Management.Automation/resources/RunspaceInit.resx +++ b/src/System.Management.Automation/resources/RunspaceInit.resx @@ -186,6 +186,9 @@ Dictates what type of prompt should be displayed for the current nesting level + + If true, $ErrorActionPreference applies to native executables, so that non-zero exit codes will generate cmdlet-style errors governed by error action settings + If true, WhatIf is considered to be enabled for all commands. diff --git a/test/powershell/engine/Basic/NativeCommandErrorHandling.Tests.ps1 b/test/powershell/engine/Basic/NativeCommandErrorHandling.Tests.ps1 new file mode 100644 index 00000000000..83bf3d66c37 --- /dev/null +++ b/test/powershell/engine/Basic/NativeCommandErrorHandling.Tests.ps1 @@ -0,0 +1,178 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Functional tests to verify that native executables throw errors (non-terminating and terminating) appropriately +# when $PSNativeCommandUseErrorActionPreference is $true + +Describe 'Native command error handling tests' -Tags 'CI' { + BeforeAll { + $originalDefaultParameterValues = $PSDefaultParameterValues.Clone() + if (-not [ExperimentalFeature]::IsEnabled('PSNativeCommandErrorActionPreference')) + { + $PSDefaultParameterValues['It:Skip'] = $true + return + } + + $exeName = $IsWindows ? 'testexe.exe' : 'testexe' + + $errorActionPrefTestCases = @( + @{ ErrorActionPref = 'Stop' } + @{ ErrorActionPref = 'Continue' } + @{ ErrorActionPref = 'SilentlyContinue' } + @{ ErrorActionPref = 'Ignore' } + ) + } + + AfterAll { + $global:PSDefaultParameterValues = $originalDefaultParameterValues + } + + BeforeEach { + $Error.Clear() + } + + Context 'PSNativeCommandUseErrorActionPreference is $true' { + BeforeEach { + $PSNativeCommandUseErrorActionPreference = $true + } + + It 'Non-zero exit code throws teminating error for $ErrorActionPreference = ''Stop''' { + $ErrorActionPreference = 'Stop' + + { testexe -returncode 1 } | Should -Throw -ErrorId 'ProgramExitedWithNonZeroCode' + + $error.Count | Should -Be 1 + $error[0].FullyQualifiedErrorId | Should -BeExactly 'ProgramExitedWithNonZeroCode' + $error[0].TargetObject | Should -BeExactly $exeName + } + + It 'Non-zero exit code outputs a non-teminating error for $ErrorActionPreference = ''Continue''' { + $ErrorActionPreference = 'Continue' + + $stderr = testexe -returncode 1 2>&1 + + $error[0].FullyQualifiedErrorId | Should -BeExactly 'ProgramExitedWithNonZeroCode' + $error[0].TargetObject | Should -BeExactly $exeName + $stderr[1].Exception.Message | Should -BeExactly "Program `"$exeName`" ended with non-zero exit code: 1." + } + + It 'Non-zero exit code generates a non-teminating error for $ErrorActionPreference = ''SilentlyContinue''' { + $ErrorActionPreference = 'SilentlyContinue' + + testexe -returncode 1 > $null + + $error.Count | Should -Be 1 + $error[0].FullyQualifiedErrorId | Should -BeExactly 'ProgramExitedWithNonZeroCode' + $error[0].TargetObject | Should -BeExactly $exeName + } + + It 'Non-zero exit code does not generates an error record for $ErrorActionPreference = ''Ignore''' { + $ErrorActionPreference = 'Ignore' + + testexe -returncode 1 > $null + + $LASTEXITCODE | Should -Be 1 + $error.Count | Should -Be 0 + } + + It 'Zero exit code generates no error for $ErrorActionPreference = ''''' -TestCases $errorActionPrefTestCases { + param($ErrorActionPref) + + $ErrorActionPreference = $ErrorActionPref + + $output = testexe -returncode 0 + + $output | Should -BeExactly '0' + $LASTEXITCODE | Should -Be 0 + $Error.Count | Should -Be 0 + } + + It 'Works as expected with a try/catch block when $ErrorActionPreference = ''''' -TestCase $errorActionPrefTestCases { + param($ErrorActionPref) + + $ErrorActionPreference = $ErrorActionPref + + $threw = $false + $continued = $false + $hitFinally = $false + try + { + testexe -returncode 17 2>&1 > $null + $continued = $true + } + catch + { + $threw = $true + $exception = $_.Exception + } + finally + { + $hitFinally = $true + } + + $hitFinally | Should -BeTrue + $continued | Should -Be ($ErrorActionPreference -ne 'Stop') + $threw | Should -Be ($ErrorActionPreference -eq 'Stop') + + if ($threw) + { + $exception.Path | Should -BeExactly (Get-Command -Name testexe -CommandType Application).Path + $exception.ExitCode | Should -Be $LASTEXITCODE + $exception.ProcessId | Should -BeGreaterThan 0 + } + } + + It 'Works with trap when $ErrorActionPreference = ''''' -TestCases $errorActionPrefTestCases { + param($ErrorActionPref) + + $ErrorActionPreference = $ErrorActionPref + + trap + { + $hitTrap = $true + $exception = $_ + continue + } + + $hitTrap = $false + + # Expect this to be trapped + testexe -returncode 17 2>&1 > $null + + if ($ErrorActionPreference -eq 'Stop') + { + $hitTrap | Should -BeTrue + $exception.ExitCode | Should -Be $LASTEXITCODE + $exception.Path | Should -BeExactly (Get-Command -Name testexe -CommandType Application).Path + $exception.ProcessId | Should -BeGreaterThan 0 + } + else + { + $hitTrap | Should -BeFalse + $exception | Should -BeNullOrEmpty + } + } + } + + Context 'PSNativeCommandUseErrorActionPreference is $false' { + BeforeEach { + $PSNativeCommandUseErrorActionPreference = $false + } + + It 'Non-zero exit code generates no error for $ErrorActionPreference = ''''' -TestCases $errorActionPrefTestCases { + param($ErrorActionPref) + + $ErrorActionPreference = $ErrorActionPref + + if ($ErrorActionPref -eq 'Stop') { + { testexe -returncode 1 } | Should -Not -Throw + } + else { + testexe -returncode 1 > $null + } + + $LASTEXITCODE | Should -Be 1 + $Error.Count | Should -Be 0 + } + } +} From 321b726306fcee2e79c955fdf00e06df10a7d220 Mon Sep 17 00:00:00 2001 From: Daryl Graves <46203429+DarylGraves@users.noreply.github.com> Date: Wed, 25 Aug 2021 17:45:00 +0100 Subject: [PATCH 010/645] Format-Wide: Fix NullReferenceException (#15990) --- .../FormatAndOutput/common/TableWriter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/System.Management.Automation/FormatAndOutput/common/TableWriter.cs b/src/System.Management.Automation/FormatAndOutput/common/TableWriter.cs index 74d5f543e07..fb5ef0bf930 100644 --- a/src/System.Management.Automation/FormatAndOutput/common/TableWriter.cs +++ b/src/System.Management.Automation/FormatAndOutput/common/TableWriter.cs @@ -458,7 +458,7 @@ private string GenerateRow(string[] values, ReadOnlySpan alignment, Display } sb.Append(GenerateRowField(values[k], _si.columnInfo[k].width, alignment[k], dc, addPadding)); - if (values[k].Contains(ESC)) + if (values[k] is not null && values[k].Contains(ESC)) { // Reset the console output if the content of this column contains ESC sb.Append(PSStyle.Instance.Reset); From b09f6973fdc6ea062366dd718cb2978516cc4573 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 25 Aug 2021 14:19:55 -0700 Subject: [PATCH 011/645] Find packages separately for each source in `UpdateDotnetRuntime.ps1` script (#15998) --- tools/UpdateDotnetRuntime.ps1 | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/UpdateDotnetRuntime.ps1 b/tools/UpdateDotnetRuntime.ps1 index 29aefe3ee71..f02f35da866 100644 --- a/tools/UpdateDotnetRuntime.ps1 +++ b/tools/UpdateDotnetRuntime.ps1 @@ -107,12 +107,24 @@ function Update-PackageVersion { } $packages.GetEnumerator() | ForEach-Object { - $pkgs = Find-Package -Name $_.Key -AllVersions -AllowPrereleaseVersions -Source $source + $pkgName = $_.Key + + $pkgs = [System.Collections.Generic.Dictionary[string, Microsoft.PackageManagement.Packaging.SoftwareIdentity]]::new() + + # We have to find packages for all sources separately as Find-Package does not return all packages when both sources are provided at the same time. + # Since there will be a lot of duplicates we add the package to a dictionary so we only get a unique set by version. + $source | ForEach-Object { + Find-Package -Name $pkgName -AllVersions -AllowPrereleaseVersions -Source $_ | ForEach-Object { + if (-not $pkgs.ContainsKey($_.Version)) { + $pkgs.Add($_.Version, $_) + } + } + } foreach ($v in $_.Value) { $version = $v.Version - foreach ($p in $pkgs) { + foreach ($p in $pkgs.Values) { # some packages are directly updated on nuget.org so need to check that too. if ($p.Version -like "$versionPattern*" -or $p.Source -eq 'nuget.org') { if ([System.Management.Automation.SemanticVersion] ($version) -lt [System.Management.Automation.SemanticVersion] ($p.Version)) { From 15836f289cab3f2a2f8c6afb0bd8fe2cc3adba1b Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 25 Aug 2021 15:59:58 -0700 Subject: [PATCH 012/645] Ignore error from `Find-Package` (#15999) --- tools/UpdateDotnetRuntime.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/UpdateDotnetRuntime.ps1 b/tools/UpdateDotnetRuntime.ps1 index f02f35da866..cbaa69bfd02 100644 --- a/tools/UpdateDotnetRuntime.ps1 +++ b/tools/UpdateDotnetRuntime.ps1 @@ -114,7 +114,7 @@ function Update-PackageVersion { # We have to find packages for all sources separately as Find-Package does not return all packages when both sources are provided at the same time. # Since there will be a lot of duplicates we add the package to a dictionary so we only get a unique set by version. $source | ForEach-Object { - Find-Package -Name $pkgName -AllVersions -AllowPrereleaseVersions -Source $_ | ForEach-Object { + Find-Package -Name $pkgName -AllVersions -AllowPrereleaseVersions -Source $_ -ErrorAction SilentlyContinue | ForEach-Object { if (-not $pkgs.ContainsKey($_.Version)) { $pkgs.Add($_.Version, $_) } From 403767d7f7b910a6f315505287fe9a72c3960c52 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Wed, 25 Aug 2021 17:05:54 -0700 Subject: [PATCH 013/645] Fix a casting error when using `$PSNativeCommandUsesErrorActionPreference` (#15993) --- .../engine/ExecutionContext.cs | 62 ++----------------- .../engine/InitialSessionState.cs | 53 +++++----------- .../engine/MshCommandRuntime.cs | 3 +- .../engine/NativeCommandParameterBinder.cs | 6 +- .../engine/NativeCommandProcessor.cs | 2 +- .../engine/SpecialVariables.cs | 6 +- .../NativeCommandErrorHandling.Tests.ps1 | 26 ++++++++ 7 files changed, 55 insertions(+), 103 deletions(-) diff --git a/src/System.Management.Automation/engine/ExecutionContext.cs b/src/System.Management.Automation/engine/ExecutionContext.cs index cfcc417de4a..8ffd25949c5 100644 --- a/src/System.Management.Automation/engine/ExecutionContext.cs +++ b/src/System.Management.Automation/engine/ExecutionContext.cs @@ -525,9 +525,7 @@ internal object GetVariableValue(VariablePath path) /// internal object GetVariableValue(VariablePath path, object defaultValue) { - CmdletProviderContext context; - SessionStateScope scope; - return EngineSessionState.GetVariableValue(path, out context, out scope) ?? defaultValue; + return EngineSessionState.GetVariableValue(path, out _, out _) ?? defaultValue; } /// @@ -612,19 +610,15 @@ private void CheckActionPreference(VariablePath preferenceVariablePath, ActionPr /// internal bool GetBooleanPreference(VariablePath preferenceVariablePath, bool defaultPref, out bool defaultUsed) { - CmdletProviderContext context = null; - SessionStateScope scope = null; - object val = EngineSessionState.GetVariableValue(preferenceVariablePath, out context, out scope); - if (val == null) + object val = EngineSessionState.GetVariableValue(preferenceVariablePath, out _, out _); + if (val is null) { defaultUsed = true; return defaultPref; } - bool converted = defaultPref; - defaultUsed = !LanguagePrimitives.TryConvertTo - (val, out converted); - return (defaultUsed) ? defaultPref : converted; + defaultUsed = !LanguagePrimitives.TryConvertTo(val, out bool converted); + return defaultUsed ? defaultPref : converted; } #endregion GetSetVariable methods @@ -1568,23 +1562,6 @@ internal ExecutionContext(AutomationEngine engine, PSHost hostInterface, Initial private void InitializeCommon(AutomationEngine engine, PSHost hostInterface) { Engine = engine; -#if !CORECLR// System.AppDomain is not in CoreCLR - // Set the assembly resolve handler if it isn't already set... - if (!_assemblyEventHandlerSet) - { - // we only want to set the event handler once for the entire app domain... - lock (lockObject) - { - // Need to check again inside the lock due to possibility of a race condition... - if (!_assemblyEventHandlerSet) - { - AppDomain currentAppDomain = AppDomain.CurrentDomain; - currentAppDomain.AssemblyResolve += new ResolveEventHandler(PowerShellAssemblyResolveHandler); - _assemblyEventHandlerSet = true; - } - } - } -#endif Events = new PSLocalEventManager(this); transactionManager = new PSTransactionManager(); _debugger = new ScriptDebugger(this); @@ -1609,35 +1586,6 @@ private void InitializeCommon(AutomationEngine engine, PSHost hostInterface) } private static readonly object lockObject = new object(); - -#if !CORECLR // System.AppDomain is not in CoreCLR - private static bool _assemblyEventHandlerSet = false; - - /// - /// AssemblyResolve event handler that will look in the assembly cache to see - /// if the named assembly has been loaded. This is necessary so that assemblies loaded - /// with LoadFrom, which are in a different loaded context than Load, can still be used to - /// resolve types. - /// - /// The event sender. - /// The event args. - /// The resolve assembly or null if not found. - private static Assembly PowerShellAssemblyResolveHandler(object sender, ResolveEventArgs args) - { - ExecutionContext ecFromTLS = Runspaces.LocalPipeline.GetExecutionContextFromTLS(); - if (ecFromTLS != null) - { - if (ecFromTLS.AssemblyCache != null) - { - Assembly assembly; - ecFromTLS.AssemblyCache.TryGetValue(args.Name, out assembly); - return assembly; - } - } - - return null; - } -#endif } /// diff --git a/src/System.Management.Automation/engine/InitialSessionState.cs b/src/System.Management.Automation/engine/InitialSessionState.cs index 07c0d250004..815f7ea5628 100644 --- a/src/System.Management.Automation/engine/InitialSessionState.cs +++ b/src/System.Management.Automation/engine/InitialSessionState.cs @@ -1308,37 +1308,6 @@ private static void MakeDisallowedEntriesPrivate(InitialSessionStateEntryColl } } - #region VariableHelper - /// - /// A helper for adding variables to session state. - /// Experimental features can be handled here. - /// - /// The variables to add to session state. - private void AddVariables(IEnumerable variables) - { - Variables.Add(variables); - - // If the PSNativeCommandArgumentPassing feature is enabled, create the variable which controls the behavior - // Since the BuiltInVariables list is static, and this should be done dynamically - // we need to do this here. Also, since the defaults are different based on platform we need a - // bit more logic. - if (ExperimentalFeature.IsEnabled("PSNativeCommandArgumentPassing")) - { - NativeArgumentPassingStyle style = NativeArgumentPassingStyle.Standard; - if (Platform.IsWindows) { - style = NativeArgumentPassingStyle.Windows; - } - Variables.Add( - new SessionStateVariableEntry( - SpecialVariables.NativeArgumentPassing, - style, - RunspaceInit.NativeCommandArgumentPassingDescription, - ScopedItemOptions.None, - new ArgumentTypeConverterAttribute(typeof(NativeArgumentPassingStyle)))); - } - } - #endregion - /// /// Creates an initial session state from a PSSC configuration file. /// @@ -1444,7 +1413,7 @@ private static InitialSessionState CreateRestrictedForRemoteServer() } // Add built-in variables. - iss.AddVariables(BuiltInVariables); + iss.Variables.Add(BuiltInVariables); // wrap some commands in a proxy function to restrict their parameters foreach (KeyValuePair proxyFunction in CommandMetadata.GetRestrictedCommands(SessionCapabilities.RemoteServer)) @@ -1531,7 +1500,7 @@ public static InitialSessionState CreateDefault() InitialSessionState ss = new InitialSessionState(); - ss.AddVariables(BuiltInVariables); + ss.Variables.Add(BuiltInVariables); ss.Commands.Add(new SessionStateApplicationEntry("*")); ss.Commands.Add(new SessionStateScriptEntry("*")); ss.Commands.Add(BuiltInFunctions); @@ -1598,7 +1567,7 @@ public static InitialSessionState CreateDefault2() { InitialSessionState ss = new InitialSessionState(); - ss.AddVariables(BuiltInVariables); + ss.Variables.Add(BuiltInVariables); ss.Commands.Add(new SessionStateApplicationEntry("*")); ss.Commands.Add(new SessionStateScriptEntry("*")); ss.Commands.Add(BuiltInFunctions); @@ -1639,7 +1608,7 @@ public InitialSessionState Clone() { InitialSessionState ss = new InitialSessionState(); - ss.AddVariables(this.Variables.Clone()); + ss.Variables.Add(this.Variables.Clone()); ss.EnvironmentVariables.Add(this.EnvironmentVariables.Clone()); ss.Commands.Add(this.Commands.Clone()); ss.Assemblies.Add(this.Assemblies.Clone()); @@ -4471,7 +4440,6 @@ .ForwardHelpCategory Cmdlet internal const ActionPreference DefaultInformationPreference = ActionPreference.SilentlyContinue; internal const ErrorView DefaultErrorView = ErrorView.ConciseView; - internal const bool DefaultPSNativeCommandUseErrorActionPreference = false; internal const bool DefaultWhatIfPreference = false; internal const ConfirmImpact DefaultConfirmPreference = ConfirmImpact.High; @@ -4628,12 +4596,23 @@ static InitialSessionState() builtinVariables.Add( new SessionStateVariableEntry( SpecialVariables.PSNativeCommandUseErrorActionPreference, - DefaultPSNativeCommandUseErrorActionPreference, + value: false, RunspaceInit.PSNativeCommandUseErrorActionPreferenceDescription, ScopedItemOptions.None, new ArgumentTypeConverterAttribute(typeof(bool)))); } + if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandArgumentPassingFeatureName)) + { + builtinVariables.Add( + new SessionStateVariableEntry( + SpecialVariables.NativeArgumentPassing, + Platform.IsWindows ? NativeArgumentPassingStyle.Windows : NativeArgumentPassingStyle.Standard, + RunspaceInit.NativeCommandArgumentPassingDescription, + ScopedItemOptions.None, + new ArgumentTypeConverterAttribute(typeof(NativeArgumentPassingStyle)))); + } + BuiltInVariables = builtinVariables.ToArray(); } diff --git a/src/System.Management.Automation/engine/MshCommandRuntime.cs b/src/System.Management.Automation/engine/MshCommandRuntime.cs index a801278aa04..f76723d64e1 100644 --- a/src/System.Management.Automation/engine/MshCommandRuntime.cs +++ b/src/System.Management.Automation/engine/MshCommandRuntime.cs @@ -3250,8 +3250,7 @@ internal SwitchParameter WhatIf { if (!IsWhatIfFlagSet && !_isWhatIfPreferenceCached) { - bool defaultUsed = false; - _whatIfFlag = Context.GetBooleanPreference(SpecialVariables.WhatIfPreferenceVarPath, _whatIfFlag, out defaultUsed); + _whatIfFlag = Context.GetBooleanPreference(SpecialVariables.WhatIfPreferenceVarPath, _whatIfFlag, out _); _isWhatIfPreferenceCached = true; } diff --git a/src/System.Management.Automation/engine/NativeCommandParameterBinder.cs b/src/System.Management.Automation/engine/NativeCommandParameterBinder.cs index 511449bc7e5..59715a02125 100644 --- a/src/System.Management.Automation/engine/NativeCommandParameterBinder.cs +++ b/src/System.Management.Automation/engine/NativeCommandParameterBinder.cs @@ -20,8 +20,6 @@ namespace System.Management.Automation /// internal class NativeCommandParameterBinder : ParameterBinderBase { - private readonly VariablePath s_nativeArgumentPassingVarPath = new VariablePath(SpecialVariables.NativeArgumentPassing); - #region ctor /// @@ -193,13 +191,13 @@ internal NativeArgumentPassingStyle ArgumentPassingStyle { get { - if (ExperimentalFeature.IsEnabled("PSNativeCommandArgumentPassing")) + if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandArgumentPassingFeatureName)) { try { // This will default to the new behavior if it is set to anything other than Legacy var preference = LanguagePrimitives.ConvertTo( - Context.GetVariableValue(s_nativeArgumentPassingVarPath, NativeArgumentPassingStyle.Standard)); + Context.GetVariableValue(SpecialVariables.NativeArgumentPassingVarPath, NativeArgumentPassingStyle.Standard)); return preference; } catch diff --git a/src/System.Management.Automation/engine/NativeCommandProcessor.cs b/src/System.Management.Automation/engine/NativeCommandProcessor.cs index 18d5b16c82f..7df3dd82546 100644 --- a/src/System.Management.Automation/engine/NativeCommandProcessor.cs +++ b/src/System.Management.Automation/engine/NativeCommandProcessor.cs @@ -865,7 +865,7 @@ internal override void Complete() this.commandRuntime.PipelineProcessor.ExecutionFailed = true; if (!ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandErrorActionPreferenceFeatureName) - || !(bool)Command.Context.GetVariableValue(SpecialVariables.PSNativeCommandUseErrorActionPreferenceVarPath, defaultValue: false)) + || !Command.Context.GetBooleanPreference(SpecialVariables.PSNativeCommandUseErrorActionPreferenceVarPath, defaultPref: false, out _)) { return; } diff --git a/src/System.Management.Automation/engine/SpecialVariables.cs b/src/System.Management.Automation/engine/SpecialVariables.cs index 0137ace9d66..d093cdbbb50 100644 --- a/src/System.Management.Automation/engine/SpecialVariables.cs +++ b/src/System.Management.Automation/engine/SpecialVariables.cs @@ -204,6 +204,7 @@ internal static class SpecialVariables internal static readonly VariablePath PSModuleAutoLoadingPreferenceVarPath = new VariablePath("global:" + PSModuleAutoLoading); #region Platform Variables + internal const string IsLinux = "IsLinux"; internal static readonly VariablePath IsLinuxPath = new VariablePath("IsLinux"); @@ -221,6 +222,7 @@ internal static class SpecialVariables internal static readonly VariablePath IsCoreCLRPath = new VariablePath("IsCoreCLR"); #endregion + #region Preference Variables internal const string DebugPreference = "DebugPreference"; @@ -255,13 +257,13 @@ internal static class SpecialVariables internal static readonly VariablePath InformationPreferenceVarPath = new VariablePath(InformationPreference); + #endregion Preference Variables + internal const string PSNativeCommandUseErrorActionPreference = nameof(PSNativeCommandUseErrorActionPreference); internal static readonly VariablePath PSNativeCommandUseErrorActionPreferenceVarPath = new(PSNativeCommandUseErrorActionPreference); - #endregion Preference Variables - // Native command argument passing style internal const string NativeArgumentPassing = "PSNativeCommandArgumentPassing"; diff --git a/test/powershell/engine/Basic/NativeCommandErrorHandling.Tests.ps1 b/test/powershell/engine/Basic/NativeCommandErrorHandling.Tests.ps1 index 83bf3d66c37..b8d6b20e126 100644 --- a/test/powershell/engine/Basic/NativeCommandErrorHandling.Tests.ps1 +++ b/test/powershell/engine/Basic/NativeCommandErrorHandling.Tests.ps1 @@ -56,6 +56,19 @@ Describe 'Native command error handling tests' -Tags 'CI' { $stderr[1].Exception.Message | Should -BeExactly "Program `"$exeName`" ended with non-zero exit code: 1." } + It "Non-boolean value should not cause type casting error when the native command exited with non-zero code" { + $ErrorActionPreference = 'Continue' + + $PSNativeCommandUseErrorActionPreference = 'Yeah' + $PSNativeCommandUseErrorActionPreference | Should -BeExactly 'Yeah' + + $stderr = testexe -returncode 1 2>&1 + + $error[0].FullyQualifiedErrorId | Should -BeExactly 'ProgramExitedWithNonZeroCode' + $error[0].TargetObject | Should -BeExactly $exeName + $stderr[1].Exception.Message | Should -BeExactly "Program `"$exeName`" ended with non-zero exit code: 1." + } + It 'Non-zero exit code generates a non-teminating error for $ErrorActionPreference = ''SilentlyContinue''' { $ErrorActionPreference = 'SilentlyContinue' @@ -174,5 +187,18 @@ Describe 'Native command error handling tests' -Tags 'CI' { $LASTEXITCODE | Should -Be 1 $Error.Count | Should -Be 0 } + + It "Non-boolean value should not cause type casting error when the native command exited with non-zero code" { + $ErrorActionPreference = 'Continue' + + $PSNativeCommandUseErrorActionPreference = 0 + $PSNativeCommandUseErrorActionPreference | Should -Be 0 + $PSNativeCommandUseErrorActionPreference | Should -BeOfType 'System.Int32' + + testexe -returncode 1 > $null + + $LASTEXITCODE | Should -Be 1 + $Error.Count | Should -Be 0 + } } } From d33b223674996e0159fd77416fd59706fba73856 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Aug 2021 13:07:43 -0700 Subject: [PATCH 014/645] Update .NET SDK version from `6.0.100-preview.6.21355.2` to `6.0.100-rc.1.21426.1` (#15648) --- assets/wix/files.wxs | 14 +++++++------- global.json | 2 +- ...crosoft.PowerShell.Commands.Management.csproj | 2 +- .../Microsoft.PowerShell.Commands.Utility.csproj | 6 +++--- .../Microsoft.PowerShell.CoreCLR.Eventing.csproj | 2 +- .../Microsoft.PowerShell.SDK.csproj | 8 ++++---- .../Microsoft.WSMan.Management.csproj | 2 +- .../System.Management.Automation.csproj | 16 ++++++++-------- .../engine/Api/TypeInference.Tests.ps1 | 2 +- test/tools/TestService/TestService.csproj | 2 +- test/tools/WebListener/WebListener.csproj | 4 ++-- 11 files changed, 30 insertions(+), 30 deletions(-) diff --git a/assets/wix/files.wxs b/assets/wix/files.wxs index f205226a49d..84539cac520 100644 --- a/assets/wix/files.wxs +++ b/assets/wix/files.wxs @@ -1608,9 +1608,6 @@ - - - @@ -3054,8 +3051,11 @@ - - + + + + + @@ -4027,7 +4027,6 @@ - @@ -4038,10 +4037,11 @@ - + + diff --git a/global.json b/global.json index 26b49e6e30a..21a9b9bc28f 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "6.0.100-preview.6.21355.2" + "version": "6.0.100-rc.1.21426.1" } } diff --git a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj index 1fb088cfdd7..e8c7e78918e 100644 --- a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj +++ b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj @@ -47,7 +47,7 @@ - + diff --git a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj index be3bf2892e9..19645f35ae0 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj +++ b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj @@ -31,9 +31,9 @@ - - - + + + diff --git a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj index fc257238436..a309820ffab 100644 --- a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj +++ b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj index 0907eaa3736..5875f9904dd 100644 --- a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj +++ b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj @@ -18,9 +18,9 @@ - - - + + + @@ -30,7 +30,7 @@ - + diff --git a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj index d32fb47a456..fbd381b9d6f 100644 --- a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj +++ b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/System.Management.Automation/System.Management.Automation.csproj b/src/System.Management.Automation/System.Management.Automation.csproj index 37d3686dbaa..4b6917b168e 100644 --- a/src/System.Management.Automation/System.Management.Automation.csproj +++ b/src/System.Management.Automation/System.Management.Automation.csproj @@ -16,16 +16,16 @@ - - - + + + - - + + - - - + + + diff --git a/test/powershell/engine/Api/TypeInference.Tests.ps1 b/test/powershell/engine/Api/TypeInference.Tests.ps1 index 9ce41f54e29..1d65047c34c 100644 --- a/test/powershell/engine/Api/TypeInference.Tests.ps1 +++ b/test/powershell/engine/Api/TypeInference.Tests.ps1 @@ -504,7 +504,7 @@ Describe "Type inference Tests" -tags "CI" { It "Infers typeof Select-Object when Parameter is ExcludeProperty" { $res = [AstTypeInference]::InferTypeOf( { [io.fileinfo]::new("file") | Select-Object -ExcludeProperty *Time*, E* }.Ast) $res.Count | Should -Be 1 - $res[0].Name | Should -BeExactly "System.Management.Automation.PSObject#Attributes:BaseName:Directory:DirectoryName:FullName:IsReadOnly:Length:LengthString:LinkType:Mode:ModeWithoutHardLink:Name:NameString:Target:VersionInfo" + $res[0].Name | Should -BeExactly "System.Management.Automation.PSObject#Attributes:BaseName:Directory:DirectoryName:FullName:IsReadOnly:Length:LengthString:LinkTarget:LinkType:Mode:ModeWithoutHardLink:Name:NameString:Target:VersionInfo" $names = $res[0].Members.Name $names -contains "BaseName" | Should -BeTrue $names -contains "Name" | Should -BeTrue diff --git a/test/tools/TestService/TestService.csproj b/test/tools/TestService/TestService.csproj index da9976c3a42..a3ce2faa839 100644 --- a/test/tools/TestService/TestService.csproj +++ b/test/tools/TestService/TestService.csproj @@ -13,7 +13,7 @@ - + diff --git a/test/tools/WebListener/WebListener.csproj b/test/tools/WebListener/WebListener.csproj index 4217232a67d..6939ea252a3 100644 --- a/test/tools/WebListener/WebListener.csproj +++ b/test/tools/WebListener/WebListener.csproj @@ -7,8 +7,8 @@ - - + + From 02b532fb1da782b3594404c779e9226145fee897 Mon Sep 17 00:00:00 2001 From: Steven Liekens Date: Fri, 27 Aug 2021 00:11:15 +0200 Subject: [PATCH 015/645] Fix link header parsing to handle unquoted `rel` types (#15973) --- .../utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 85860aa19e6..d0abc8f3cd7 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -1830,7 +1830,7 @@ internal void ParseLinkHeader(HttpResponseMessage response, System.Uri requestUr // we only support the URL in angle brackets and `rel`, other attributes are ignored // user can still parse it themselves via the Headers property - const string pattern = "<(?.*?)>;\\s*rel=(\"?)(?.*?)\\1[^\\w -.]?"; + const string pattern = "<(?.*?)>;\\s*rel=(?\")?(?(?(quoted).*?|[^,;]*))(?(quoted)\")"; IEnumerable links; if (response.Headers.TryGetValues("Link", out links)) { From fde119e28b7bc52aa9b7186e9369a8d88736c94e Mon Sep 17 00:00:00 2001 From: "Joel Sallow (/u/ta11ow)" <32407840+vexx32@users.noreply.github.com> Date: Fri, 27 Aug 2021 00:41:01 -0400 Subject: [PATCH 016/645] Use `CurrentCulture` when handling conversions to `DateTime` in `Add-History` (#16005) --- .../engine/hostifaces/History.cs | 5 +- .../History.Tests.ps1 | 52 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/System.Management.Automation/engine/hostifaces/History.cs b/src/System.Management.Automation/engine/hostifaces/History.cs index e1100b97487..dc74a46213f 100644 --- a/src/System.Management.Automation/engine/hostifaces/History.cs +++ b/src/System.Management.Automation/engine/hostifaces/History.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; +using System.Globalization; using System.IO; using System.Management.Automation; using System.Management.Automation.Host; @@ -1449,14 +1450,14 @@ void ProcessRecord() // Read StartExecutionTime property object temp = GetPropertyValue(mshObject, "StartExecutionTime"); - if (temp == null || !LanguagePrimitives.TryConvertTo(temp, out DateTime startExecutionTime)) + if (temp == null || !LanguagePrimitives.TryConvertTo(temp, CultureInfo.CurrentCulture, out DateTime startExecutionTime)) { break; } // Read EndExecutionTime property temp = GetPropertyValue(mshObject, "EndExecutionTime"); - if (temp == null || !LanguagePrimitives.TryConvertTo(temp, out DateTime endExecutionTime)) + if (temp == null || !LanguagePrimitives.TryConvertTo(temp, CultureInfo.CurrentCulture, out DateTime endExecutionTime)) { break; } diff --git a/test/powershell/Modules/Microsoft.PowerShell.Core/History.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Core/History.Tests.ps1 index 6841778af19..dc1b8b81b4d 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Core/History.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Core/History.Tests.ps1 @@ -49,6 +49,58 @@ Describe "History cmdlet test cases" -Tags "CI" { } } + Context 'Conversions and Culture tests' { + + BeforeAll { + $cultureTestCases = @( + @{ + Culture = 'en-us' + StartTime = '08/18/2021 16:43:50' + EndTime = '08/18/2021 16:44:50' + } + @{ + Culture = 'en-au' + StartTime = '18/08/2021 16:43:50' + EndTime = '18/08/2021 16:44:50' + } + ) + + $oldCulture = [cultureinfo]::CurrentCulture + } + + AfterEach { + [cultureinfo]::CurrentCulture = $oldCulture + } + + It "respects current culture settings when handling datetime conversions" -TestCases $cultureTestCases { + param($Culture, $StartTime, $EndTime) + + [cultureinfo]::CurrentCulture = [cultureinfo]::GetCultureInfo($Culture) + + $history = [PSCustomObject] @{ + CommandLine = "test-command" + ExecutionStatus = [Management.Automation.Runspaces.PipelineState]::Completed + StartExecutionTime = $StartTime + EndExecutionTime = $EndTime + } + + { $history | Add-History -ErrorAction Stop } | Should -Not -Throw -Because 'the datetime should be converted according to the current culture' + } + + It "throws an error when asked to convert a date format that doesn't match the current culture" { + [cultureinfo]::CurrentCulture = [cultureinfo]::GetCultureInfo('en-au') + $history = [PSCustomObject] @{ + CommandLine = "test-command" + ExecutionStatus = [Management.Automation.Runspaces.PipelineState]::Completed + StartExecutionTime = '08/18/2021 16:43:50' + EndExecutionTime = '08/18/2021 16:44:50' + } + + $errorMessage = 'Cannot add history because the input object has a format that is not valid.' + { $history | Add-History -ErrorAction Stop } | Should -Throw -ExpectedMessage $errorMessage + } + } + It "Tests Invoke-History on a cmdlet that generates output on all streams" { $streamSpammer = ' function StreamSpammer From 102ccffd1479d8ab6a47755fcc7b33b8e28d936b Mon Sep 17 00:00:00 2001 From: Ilya Date: Fri, 27 Aug 2021 09:53:27 +0500 Subject: [PATCH 017/645] Enable two previously disabled `Get-Process` tests (#15845) --- .../Microsoft.PowerShell.Management/Get-Process.Tests.ps1 | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Process.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Process.Tests.ps1 index 2169c23b705..345b2affa2e 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Process.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Process.Tests.ps1 @@ -70,8 +70,7 @@ Describe "Get-Process" -Tags "CI" { { (Get-Process -Id $idleProcessPid).Name } | Should -Not -Throw } - It "Test for process property = Name" -Pending { - # Bug in .Net 5.0 Preview4. See https://github.com/PowerShell/PowerShell/pull/12894 + It "Test for process property = Name" { (Get-Process -Id $PID).Name | Should -BeExactly "pwsh" } @@ -129,8 +128,7 @@ Describe "Get-Process Formatting" -Tags "Feature" { } Describe "Process Parent property" -Tags "CI" { - It "Has Parent process property" -Pending { - # Bug in .Net 5.0 Preview4. See https://github.com/PowerShell/PowerShell/pull/12894 + It "Has Parent process property" { $powershellexe = (Get-Process -Id $PID).mainmodule.filename & $powershellexe -noprofile -command '(Get-Process -Id $PID).Parent' | Should -Not -BeNullOrEmpty } From 008f4b057fdc1482d3200de0a61d23570b4d30e4 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 27 Aug 2021 14:05:39 -0700 Subject: [PATCH 018/645] Use Alpine 3.12 for building PowerShell for alpine (#16008) --- .../releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile b/tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile index c0541b7bbd8..fb1070fcd52 100644 --- a/tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile +++ b/tools/releaseBuild/Images/microsoft_powershell_alpine3/Dockerfile @@ -1,6 +1,6 @@ # Docker image file that describes an Centos7 image with PowerShell installed from Microsoft YUM Repo -FROM mcr.microsoft.com/powershell:alpine-3.8 +FROM mcr.microsoft.com/powershell:alpine-3.12 LABEL maintainer="PowerShell Team " # Install dependencies and clean up From e3d8353e3cb815568ab896751821263735aa2964 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Sat, 28 Aug 2021 12:51:37 -0700 Subject: [PATCH 019/645] Fix the mac build by updating the pool image name (#16010) --- tools/releaseBuild/azureDevOps/templates/mac-package-build.yml | 3 ++- tools/releaseBuild/azureDevOps/templates/mac.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml index 1bd853dec53..34437b295a1 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml @@ -7,7 +7,8 @@ jobs: displayName: Package macOS ${{ parameters.buildArchitecture }} dependsOn: MacFileSigningJob_${{ parameters.buildArchitecture }} condition: succeeded() - pool: Hosted Mac Internal + pool: + vmImage: internal-macos-10.14 variables: # Turn off Homebrew analytics - name: HOMEBREW_NO_ANALYTICS diff --git a/tools/releaseBuild/azureDevOps/templates/mac.yml b/tools/releaseBuild/azureDevOps/templates/mac.yml index 54f4354ed1d..6bf7814d93b 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac.yml @@ -5,7 +5,8 @@ jobs: - job: build_macOS_${{ parameters.buildArchitecture }} displayName: Build macOS ${{ parameters.buildArchitecture }} condition: succeeded() - pool: Hosted Mac Internal + pool: + vmImage: internal-macos-10.14 variables: # Turn off Homebrew analytics - name: HOMEBREW_NO_ANALYTICS From fa50e4dcaee6a4e23d1bd54d8ee3451fd7f72353 Mon Sep 17 00:00:00 2001 From: xtqqczze <45661989+xtqqczze@users.noreply.github.com> Date: Mon, 30 Aug 2021 16:51:07 +0100 Subject: [PATCH 020/645] Use `bool?.GetValueOrDefault()` in `FormatWideCommand` (#15988) --- .../format-wide/Format-Wide.cs | 49 ++++++------------- 1 file changed, 14 insertions(+), 35 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-wide/Format-Wide.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-wide/Format-Wide.cs index 8d80462c12f..b3e593160b5 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-wide/Format-Wide.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-wide/Format-Wide.cs @@ -47,17 +47,8 @@ public object Property [Parameter] public SwitchParameter AutoSize { - get - { - if (_autosize.HasValue) - return _autosize.Value; - return false; - } - - set - { - _autosize = value; - } + get => _autosize.GetValueOrDefault(); + set => _autosize = value; } private bool? _autosize = null; @@ -70,17 +61,8 @@ public SwitchParameter AutoSize [ValidateRangeAttribute(1, int.MaxValue)] public int Column { - get - { - if (_column.HasValue) - return _column.Value; - return -1; - } - - set - { - _column = value; - } + get => _column.GetValueOrDefault(-1); + set => _column = value; } private int? _column = null; @@ -110,22 +92,19 @@ internal override FormattingCommandLineParameters GetCommandLineParameters() } // we cannot specify -column and -autosize, they are mutually exclusive - if (_autosize.HasValue && _column.HasValue) + if (AutoSize && _column.HasValue) { - if (_autosize.Value) - { - // the user specified -autosize:true AND a column number - string msg = StringUtil.Format(FormatAndOut_format_xxx.CannotSpecifyAutosizeAndColumnsError); + // the user specified -autosize:true AND a column number + string msg = StringUtil.Format(FormatAndOut_format_xxx.CannotSpecifyAutosizeAndColumnsError); - ErrorRecord errorRecord = new( - new InvalidDataException(), - "FormatCannotSpecifyAutosizeAndColumns", - ErrorCategory.InvalidArgument, - null); + ErrorRecord errorRecord = new( + new InvalidDataException(), + "FormatCannotSpecifyAutosizeAndColumns", + ErrorCategory.InvalidArgument, + null); - errorRecord.ErrorDetails = new ErrorDetails(msg); - this.ThrowTerminatingError(errorRecord); - } + errorRecord.ErrorDetails = new ErrorDetails(msg); + this.ThrowTerminatingError(errorRecord); } parameters.groupByParameter = this.ProcessGroupByParameter(); From 3bb6b2736a845f0c644787ece9251f91de3d9619 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Mon, 30 Aug 2021 13:56:52 -0700 Subject: [PATCH 021/645] Update Ubuntu images to use Ubuntu 20.04 (#15906) --- .vsts-ci/linux-daily.yml | 6 ++--- .vsts-ci/linux.yml | 16 ++++++------ .vsts-ci/linux/templates/packaging.yml | 2 +- .vsts-ci/misc-analysis.yml | 2 +- .vsts-ci/templates/nix-test.yml | 12 +++++++++ build.psm1 | 25 ++++++++++++++++--- .../Parser/ParameterBinding.Tests.ps1 | 8 +++++- .../Language/Scripting/ScriptHelp.Tests.ps1 | 14 +++++++++-- .../Test-Connection.Tests.ps1 | 3 ++- .../Add-Type.Tests.ps1 | 8 +++++- .../Set-PSBreakpoint.Tests.ps1 | 9 ++++++- .../engine/Basic/Assembly.LoadFrom.Tests.ps1 | 9 ++++++- .../Basic/Assembly.LoadNative.Tests.ps1 | 8 +++++- .../engine/Remoting/PSSession.Tests.ps1 | 3 ++- .../Remoting/RemoteSession.Basic.Tests.ps1 | 3 +++ .../Modules/WebListener/WebListener.psm1 | 2 +- tools/packaging/packaging.psm1 | 10 ++++---- 17 files changed, 109 insertions(+), 31 deletions(-) diff --git a/.vsts-ci/linux-daily.yml b/.vsts-ci/linux-daily.yml index 6ab1832dfd9..e6a5375b0c9 100644 --- a/.vsts-ci/linux-daily.yml +++ b/.vsts-ci/linux-daily.yml @@ -47,7 +47,7 @@ stages: jobs: - template: templates/ci-build.yml parameters: - pool: ubuntu-16.04 + pool: ubuntu-20.04 jobName: linux_build displayName: linux Build @@ -56,7 +56,7 @@ stages: jobs: - job: linux_test pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-20.04 displayName: Linux Test steps: @@ -149,7 +149,7 @@ stages: - job: CodeCovTestPackage displayName: CodeCoverage and Test Packages pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-20.04 steps: - pwsh: | Import-Module .\tools\ci.psm1 diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index bed0c4fd296..0cc59284859 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -52,7 +52,7 @@ stages: jobs: - template: templates/ci-build.yml parameters: - pool: ubuntu-16.04 + pool: ubuntu-20.04 jobName: linux_build displayName: linux Build @@ -62,34 +62,34 @@ stages: - template: templates/nix-test.yml parameters: name: Linux - pool: ubuntu-16.04 + pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: CI - template: templates/nix-test.yml parameters: name: Linux - pool: ubuntu-16.04 + pool: ubuntu-20.04 purpose: ElevatedPesterTests tagSet: CI - template: templates/nix-test.yml parameters: name: Linux - pool: ubuntu-16.04 + pool: ubuntu-20.04 purpose: UnelevatedPesterTests tagSet: Others - template: templates/nix-test.yml parameters: name: Linux - pool: ubuntu-16.04 + pool: ubuntu-20.04 purpose: ElevatedPesterTests tagSet: Others - template: templates/verify-xunit.yml parameters: - pool: ubuntu-16.04 + pool: ubuntu-20.04 - stage: PackageLinux displayName: Package Linux @@ -97,7 +97,7 @@ stages: jobs: - template: linux/templates/packaging.yml parameters: - pool: ubuntu-16.04 + pool: ubuntu-20.04 - stage: CodeCovTestPackage displayName: CodeCoverage and Test Packages @@ -106,7 +106,7 @@ stages: - job: CodeCovTestPackage displayName: CodeCoverage and Test Packages pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-20.04 steps: - pwsh: | Import-Module .\tools\ci.psm1 diff --git a/.vsts-ci/linux/templates/packaging.yml b/.vsts-ci/linux/templates/packaging.yml index 530232cbf67..74a234aff78 100644 --- a/.vsts-ci/linux/templates/packaging.yml +++ b/.vsts-ci/linux/templates/packaging.yml @@ -1,5 +1,5 @@ parameters: - pool: 'ubuntu-16.04' + pool: 'ubuntu-20.04' parentJobs: [] name: 'Linux' diff --git a/.vsts-ci/misc-analysis.yml b/.vsts-ci/misc-analysis.yml index d760a6e4931..5de5e28265a 100644 --- a/.vsts-ci/misc-analysis.yml +++ b/.vsts-ci/misc-analysis.yml @@ -51,7 +51,7 @@ jobs: displayName: Markdown and Common Tests pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-20.04 variables: - name: repoPath diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 6a1b6e6f9de..89c1a39dd44 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -63,6 +63,18 @@ jobs: continueOnError: true - pwsh: | + Import-Module .\build.psm1 -Force + $environment = Get-EnvironmentInformation + $isUbuntu20 = $environment.IsUbuntu -and $environment.IsUbuntu20 + if ($isUbuntu20) { + $env:LC_ALL='en_US.UTF-8' + $env:LANG='en_US.UTF-8' + sudo locale-gen $LANG + sudo update-locale + } + + locale + Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' $options = (Get-PSOptions) diff --git a/build.psm1 b/build.psm1 index f9887250490..c87e1147571 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1223,7 +1223,13 @@ function Start-PSPester { if ($Unelevate) { - $outputBufferFilePath = [System.IO.Path]::GetTempFileName() + if ($environment.IsWindows) { + $outputBufferFilePath = [System.IO.Path]::GetTempFileName() + } + else { + # Azure DevOps agents do not have Temp folder setup on Ubuntu 20.04, hence using HOME directory + $outputBufferFilePath = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) + } } $command += "Invoke-Pester " @@ -1305,7 +1311,14 @@ function Start-PSPester { $PSFlags = @("-noprofile") if (-not [string]::IsNullOrEmpty($ExperimentalFeatureName)) { - $configFile = [System.IO.Path]::GetTempFileName() + + if ($environment.IsWindows) { + $configFile = [System.IO.Path]::GetTempFileName() + } + else { + $configFile = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) + } + $configFile = [System.IO.Path]::ChangeExtension($configFile, ".json") ## Create the config.json file to enable the given experimental feature. @@ -1384,7 +1397,13 @@ function Start-PSPester { { if ($PassThru.IsPresent) { - $passThruFile = [System.IO.Path]::GetTempFileName() + if ($environment.IsWindows) { + $passThruFile = [System.IO.Path]::GetTempFileName() + } + else { + $passThruFile = Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName()) + } + try { $command += "| Export-Clixml -Path '$passThruFile' -Force" diff --git a/test/powershell/Language/Parser/ParameterBinding.Tests.ps1 b/test/powershell/Language/Parser/ParameterBinding.Tests.ps1 index 7fd26912ead..493ea9083a0 100644 --- a/test/powershell/Language/Parser/ParameterBinding.Tests.ps1 +++ b/test/powershell/Language/Parser/ParameterBinding.Tests.ps1 @@ -168,7 +168,13 @@ Describe "Custom type conversion in parameter binding" -Tags 'Feature' { } } '@ - $asmFile = [System.IO.Path]::GetTempFileName() + ".dll" + if ($IsWindows) { + $asmFile = [System.IO.Path]::GetTempFileName() + ".dll" + } + else { + $asmFile = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName() + ".dll")) + } + Add-Type -TypeDefinition $code -OutputAssembly $asmFile ## Helper function to execute script diff --git a/test/powershell/Language/Scripting/ScriptHelp.Tests.ps1 b/test/powershell/Language/Scripting/ScriptHelp.Tests.ps1 index 8824d962090..7553fc96207 100644 --- a/test/powershell/Language/Scripting/ScriptHelp.Tests.ps1 +++ b/test/powershell/Language/Scripting/ScriptHelp.Tests.ps1 @@ -180,7 +180,12 @@ Describe 'get-help HelpFunc1' -Tags "Feature" { Describe 'get-help file' -Tags "CI" { BeforeAll { try { - $tmpfile = [IO.Path]::ChangeExtension([IO.Path]::GetTempFileName(), "ps1") + if ($IsWindows) { + $tmpfile = [IO.Path]::ChangeExtension([IO.Path]::GetTempFileName(), "ps1") + } + else { + $tmpfile = Join-Path $env:HOME $([IO.Path]::ChangeExtension([IO.Path]::GetRandomFileName(), "ps1")) + } } catch { return } @@ -233,7 +238,12 @@ Describe 'get-help file' -Tags "CI" { Describe 'get-help other tests' -Tags "CI" { BeforeAll { try { - $tempFile = [IO.Path]::ChangeExtension([IO.Path]::GetTempFileName(), "ps1") + if ($IsWindows) { + $tempFile = [IO.Path]::ChangeExtension([IO.Path]::GetTempFileName(), "ps1") + } + else { + $tempFile = Join-Path $env:HOME $([IO.Path]::ChangeExtension([IO.Path]::GetRandomFileName(), "ps1")) + } } catch { return } diff --git a/test/powershell/Modules/Microsoft.PowerShell.Management/Test-Connection.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Management/Test-Connection.Tests.ps1 index fc5cf7a69d4..c41b60c3ecb 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Management/Test-Connection.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Management/Test-Connection.Tests.ps1 @@ -96,7 +96,8 @@ Describe "Test-Connection" -tags "CI" { { Test-Connection "fakeHost" -Count 1 -ErrorAction Stop } | Should -Throw -ErrorId "TestConnectionException,Microsoft.PowerShell.Commands.TestConnectionCommand" # Error code = 11001 - Host not found. - if ((Get-PlatformInfo).Platform -match "raspbian") { + $platform = Get-PlatformInfo + if ($platform.Platform -match "raspbian" -or ( $platform.Platform -match 'ubuntu' -and $platform.Version -eq '20.04')) { $code = 11 } elseif (!$IsWindows) { $code = -131073 diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Add-Type.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Add-Type.Tests.ps1 index b818dc64012..72d42af1d12 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Add-Type.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Add-Type.Tests.ps1 @@ -155,7 +155,13 @@ public class SMAAttributeTest$guid : PSCmdlet ## The assembly files cannot be removed once they are loaded, unless the current PowerShell session exits. ## If we use $TestDrive here, then Pester will try to remove them afterward and result in errors. - $TempPath = [System.IO.Path]::GetTempFileName() + if ($IsWindows) { + $TempPath = [System.IO.Path]::GetTempFileName() + } + else { + $TempPath = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) + } + if (Test-Path $TempPath) { Remove-Item -Path $TempPath -Force -Recurse } New-Item -Path $TempPath -ItemType Directory -Force > $null diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Set-PSBreakpoint.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Set-PSBreakpoint.Tests.ps1 index 21c84fe4770..a5b809d8f15 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Set-PSBreakpoint.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Set-PSBreakpoint.Tests.ps1 @@ -154,7 +154,14 @@ set-psbreakpoint -command foo } It "Fail to set psbreakpoints when script is a file of wrong type" { - $tempFile = [System.IO.Path]::GetTempFileName() + + if ($IsWindows) { + $tempFile = [System.IO.Path]::GetTempFileName() + } + else { + $tempFile = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) + } + $ErrorActionPreference = "Stop" { Set-PSBreakpoint -Script $tempFile -Line 1 diff --git a/test/powershell/engine/Basic/Assembly.LoadFrom.Tests.ps1 b/test/powershell/engine/Basic/Assembly.LoadFrom.Tests.ps1 index 67ee0677c51..383f20bfb10 100644 --- a/test/powershell/engine/Basic/Assembly.LoadFrom.Tests.ps1 +++ b/test/powershell/engine/Basic/Assembly.LoadFrom.Tests.ps1 @@ -35,7 +35,14 @@ Describe "Assembly.LoadFrom Validation Test" -Tags "CI" { ## The assembly files cannot be removed once they are loaded, unless the current PowerShell session exits. ## If we use $TestDrive here, then Pester will try to remove them afterward and result in errors. - $TempPath = [System.IO.Path]::GetTempFileName() + + if ($IsWindows) { + $TempPath = [System.IO.Path]::GetTempFileName() + } + else { + $TempPath = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) + } + if (Test-Path $TempPath) { Remove-Item -Path $TempPath -Force -Recurse } New-Item -Path $TempPath -ItemType Directory -Force > $null diff --git a/test/powershell/engine/Basic/Assembly.LoadNative.Tests.ps1 b/test/powershell/engine/Basic/Assembly.LoadNative.Tests.ps1 index bb383573672..9196ccc0e13 100644 --- a/test/powershell/engine/Basic/Assembly.LoadNative.Tests.ps1 +++ b/test/powershell/engine/Basic/Assembly.LoadNative.Tests.ps1 @@ -6,7 +6,13 @@ Describe "Can load a native assembly" -Tags "CI" { BeforeAll { ## The assembly files cannot be removed once they are loaded, unless the current PowerShell session exits. ## If we use $TestDrive here, then Pester will try to remove them afterward and result in errors. - $TempPath = [System.IO.Path]::GetTempFileName() + if ($IsWindows) { + $TempPath = [System.IO.Path]::GetTempFileName() + } + else { + $TempPath = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) + } + if (Test-Path $TempPath) { Remove-Item -Path $TempPath -Force -Recurse } New-Item -Path $TempPath -ItemType Directory -Force > $null diff --git a/test/powershell/engine/Remoting/PSSession.Tests.ps1 b/test/powershell/engine/Remoting/PSSession.Tests.ps1 index 3b8259f3707..25c7ab9c582 100644 --- a/test/powershell/engine/Remoting/PSSession.Tests.ps1 +++ b/test/powershell/engine/Remoting/PSSession.Tests.ps1 @@ -83,7 +83,8 @@ Describe "SkipCACheck and SkipCNCheck PSSession options are required for New-PSS if ( ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID - ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') + ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') ) { Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, and CentOS 8" return diff --git a/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 b/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 index a8054b47182..d56ed1999ac 100644 --- a/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 +++ b/test/powershell/engine/Remoting/RemoteSession.Basic.Tests.ps1 @@ -15,6 +15,7 @@ Describe "New-PSSession basic test" -Tag @("CI") { ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or ($IsMacOS) ) { Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" @@ -33,6 +34,7 @@ Describe "Basic Auth over HTTP not allowed on Unix" -Tag @("CI") { ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or ($IsMacOS) ) { Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" @@ -56,6 +58,7 @@ Describe "Basic Auth over HTTP not allowed on Unix" -Tag @("CI") { ($platformInfo.Platform -match "alpine|raspbian") -or ($platformInfo.Platform -eq "debian" -and ($platformInfo.Version -eq '10' -or $platformInfo.Version -eq '')) -or # debian 11 has empty Version ID ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or + ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or ($IsMacOS) ) { Set-ItResult -Skipped -Because "MI library not available for Alpine, Raspberry Pi, Debian 10 and 11, CentOS 8, and not compatible with macOS" diff --git a/test/tools/Modules/WebListener/WebListener.psm1 b/test/tools/Modules/WebListener/WebListener.psm1 index 6436da97d1c..655b4cdd638 100644 --- a/test/tools/Modules/WebListener/WebListener.psm1 +++ b/test/tools/Modules/WebListener/WebListener.psm1 @@ -102,7 +102,7 @@ function Start-WebListener [int]$HttpPort = 8083, [ValidateRange(1,65535)] - [int]$HttpsPort = 8084, + [int]$HttpsPort = 9084, [ValidateRange(1,65535)] [int]$Tls11Port = 8085, diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 9b1602a095e..000d48067e1 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -1351,21 +1351,21 @@ function New-AfterScripts Write-Verbose -Message "AfterScript Distribution: $Distribution" -Verbose if ($Environment.IsRedHatFamily) { - $AfterInstallScript = [io.path]::GetTempFileName() - $AfterRemoveScript = [io.path]::GetTempFileName() + $AfterInstallScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) + $AfterRemoveScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) $packagingStrings.RedHatAfterInstallScript -f "$Link", $Destination | Out-File -FilePath $AfterInstallScript -Encoding ascii $packagingStrings.RedHatAfterRemoveScript -f "$Link", $Destination | Out-File -FilePath $AfterRemoveScript -Encoding ascii } elseif ($Environment.IsDebianFamily -or $Environment.IsSUSEFamily) { - $AfterInstallScript = [io.path]::GetTempFileName() - $AfterRemoveScript = [io.path]::GetTempFileName() + $AfterInstallScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) + $AfterRemoveScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) $packagingStrings.UbuntuAfterInstallScript -f "$Link", $Destination | Out-File -FilePath $AfterInstallScript -Encoding ascii $packagingStrings.UbuntuAfterRemoveScript -f "$Link", $Destination | Out-File -FilePath $AfterRemoveScript -Encoding ascii } elseif ($Environment.IsMacOS) { # NOTE: The macos pkgutil doesn't support uninstall actions so we did not implement it. # Handling uninstall can be done in Homebrew so we'll take advantage of that in the brew formula. - $AfterInstallScript = [io.path]::GetTempFileName() + $AfterInstallScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) $packagingStrings.MacOSAfterInstallScript -f "$Link" | Out-File -FilePath $AfterInstallScript -Encoding ascii } From 6de53f89cf68d957f3c235d847264308ea31ff5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Aug 2021 12:44:55 +0500 Subject: [PATCH 022/645] Bump Microsoft.CodeAnalysis.NetAnalyzers (#16021) Bumps [Microsoft.CodeAnalysis.NetAnalyzers](https://github.com/dotnet/roslyn-analyzers) from 6.0.0-rc2.21423.3 to 6.0.0-rc2.21430.2. - [Release notes](https://github.com/dotnet/roslyn-analyzers/releases) - [Changelog](https://github.com/dotnet/roslyn-analyzers/blob/main/PostReleaseActivities.md) - [Commits](https://github.com/dotnet/roslyn-analyzers/commits) --- updated-dependencies: - dependency-name: Microsoft.CodeAnalysis.NetAnalyzers dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Analyzers.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Analyzers.props b/Analyzers.props index d85c827bf21..0dc31a83b3e 100644 --- a/Analyzers.props +++ b/Analyzers.props @@ -1,7 +1,7 @@ - + From 15f2730c4c99bdcb79caddb73c539e65d2400cf7 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Tue, 31 Aug 2021 15:16:10 -0700 Subject: [PATCH 023/645] Move from `PkgES` hosted agents to 1ES hosted agents (#16023) --- tools/releaseBuild/azureDevOps/releaseBuild.yml | 4 +++- .../azureDevOps/templates/checkAzureContainer.yml | 4 +++- .../releaseBuild/azureDevOps/templates/compliance.yml | 10 +++++++++- tools/releaseBuild/azureDevOps/templates/json.yml | 4 +++- tools/releaseBuild/azureDevOps/templates/linux.yml | 8 ++++++-- .../azureDevOps/templates/mac-file-signing.yml | 4 +++- .../azureDevOps/templates/mac-package-signing.yml | 4 +++- tools/releaseBuild/azureDevOps/templates/nuget.yml | 5 ++++- .../templates/windows-component-governance.yml | 4 +++- .../azureDevOps/templates/windows-hosted-build.yml | 4 +++- .../azureDevOps/templates/windows-package-signing.yml | 4 +++- .../azureDevOps/templates/windows-packaging.yml | 4 +++- 12 files changed, 46 insertions(+), 13 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index ca4f098d208..bd45d7ffb8a 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -189,7 +189,9 @@ stages: - job: release_json displayName: Create and Upload release.json pool: - vmImage: 'windows-latest' + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 steps: - checkout: self clean: true diff --git a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml index 9000d42bd61..8019a5de3b9 100644 --- a/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml +++ b/tools/releaseBuild/azureDevOps/templates/checkAzureContainer.yml @@ -8,7 +8,9 @@ jobs: - group: Azure Blob variable group displayName: Delete blob is exists pool: - vmImage: windows-latest + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 steps: - checkout: self clean: true diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index 497dfc8613b..9e6c7ad1c61 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -8,11 +8,17 @@ jobs: value : false - name: NugetSecurityAnalysisWarningLevel value: none + + # Defines the variables APIScanClient, APIScanTenant and APIScanSecret + - group: PS-PS-APIScan + displayName: Compliance dependsOn: ${{ parameters.parentJobs }} pool: - name: Package ES Standard Build + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 # APIScan can take a long time timeoutInMinutes: 180 @@ -115,6 +121,8 @@ jobs: softwareVersionNum: '$(ReleaseTagVar)' isLargeApp: false preserveTempFiles: true + env: + AzureServicesAuthConnectionString: RunAs=App;AppId=$(APIScanClient);TenantId=$(APIScanTenant);AppKey=$(APIScanSecret) continueOnError: true - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 diff --git a/tools/releaseBuild/azureDevOps/templates/json.yml b/tools/releaseBuild/azureDevOps/templates/json.yml index 0dca065e25d..570a658756a 100644 --- a/tools/releaseBuild/azureDevOps/templates/json.yml +++ b/tools/releaseBuild/azureDevOps/templates/json.yml @@ -13,7 +13,9 @@ jobs: ${{ parameters.parentJobs }} condition: succeeded() pool: - vmImage: windows-latest + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 steps: #- task: @ diff --git a/tools/releaseBuild/azureDevOps/templates/linux.yml b/tools/releaseBuild/azureDevOps/templates/linux.yml index 952b12183ce..bf7662a9711 100644 --- a/tools/releaseBuild/azureDevOps/templates/linux.yml +++ b/tools/releaseBuild/azureDevOps/templates/linux.yml @@ -8,7 +8,9 @@ jobs: displayName: Build ${{ parameters.buildName }} condition: succeeded() pool: - vmImage: ubuntu-16.04 + name: PowerShell1ES + demands: + - ImageOverride -equals MMSUbuntu20.04 dependsOn: ${{ parameters.parentJob }} variables: - name: runCodesignValidationInjection @@ -66,7 +68,9 @@ jobs: dependsOn: build_${{ parameters.buildName }} condition: succeeded() pool: - vmImage: windows-latest + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 variables: - name: buildName value: ${{ parameters.buildName }} diff --git a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml index bc111a46b07..1f43d39ab61 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-file-signing.yml @@ -7,7 +7,9 @@ jobs: dependsOn: build_macOS_${{ parameters.buildArchitecture }} condition: succeeded() pool: - name: Package ES Standard Build + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 variables: - group: ESRP - name: runCodesignValidationInjection diff --git a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml index c8a7ab7e2bc..cba0ed6f9c3 100644 --- a/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/mac-package-signing.yml @@ -7,7 +7,9 @@ jobs: dependsOn: package_macOS_${{ parameters.buildArchitecture }} condition: succeeded() pool: - name: Package ES Standard Build + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 variables: - group: ESRP - name: runCodesignValidationInjection diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index c6f21ec81f0..cd3972d5fab 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -7,7 +7,10 @@ jobs: ${{ parameters.parentJobs }} displayName: Build NuGet packages condition: succeeded() - pool: Package ES Standard Build + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 timeoutInMinutes: 90 diff --git a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml index 572be144c51..b21ab208f1b 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-component-governance.yml @@ -10,7 +10,9 @@ jobs: condition: succeeded() pool: - name: Package ES Standard Build + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 steps: - checkout: self diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml index 2c5f4cc138d..2820654d167 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml @@ -14,7 +14,9 @@ jobs: condition: succeeded() dependsOn: ${{ parameters.parentJob }} pool: - vmImage: windows-latest + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 variables: - name: runCodesignValidationInjection value: false diff --git a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml index e06ea954c5d..dd547da6a60 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-package-signing.yml @@ -8,7 +8,9 @@ jobs: ${{ parameters.parentJobs }} condition: succeeded() pool: - vmImage: windows-latest + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 variables: - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE value: 1 diff --git a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml index 0aa0f235165..5ce2c90a8e3 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-packaging.yml @@ -14,7 +14,9 @@ jobs: condition: succeeded() dependsOn: ${{ parameters.parentJob }} pool: - vmImage: windows-latest + name: PowerShell1ES + demands: + - ImageOverride -equals MMS2019 variables: - name: BuildConfiguration value: ${{ parameters.BuildConfiguration }} From 946341b2ebe6a61f081f4c9143668dc7be1f9119 Mon Sep 17 00:00:00 2001 From: Paul Higinbotham Date: Wed, 1 Sep 2021 09:31:19 -0700 Subject: [PATCH 024/645] Remove duplicate remote server mediator code (#16027) --- .../host/msh/ConsoleHost.cs | 29 ++-- .../server/OutOfProcServerMediator.cs | 142 +++++------------- 2 files changed, 52 insertions(+), 119 deletions(-) diff --git a/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs b/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs index 06d61548048..b0ae4735026 100644 --- a/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs +++ b/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs @@ -193,7 +193,20 @@ internal static int Start(string bannerText, string helpText) { ApplicationInsightsTelemetry.SendPSCoreStartupTelemetry("ServerMode"); ProfileOptimization.StartProfile("StartupProfileData-ServerMode"); - System.Management.Automation.Remoting.Server.OutOfProcessMediator.Run(s_cpp.InitialCommand, s_cpp.WorkingDirectory); + System.Management.Automation.Remoting.Server.StdIOProcessMediator.Run( + initialCommand: s_cpp.InitialCommand, + workingDirectory: s_cpp.WorkingDirectory, + configurationName: null); + exitCode = 0; + } + else if (s_cpp.SSHServerMode) + { + ApplicationInsightsTelemetry.SendPSCoreStartupTelemetry("SSHServer"); + ProfileOptimization.StartProfile("StartupProfileData-SSHServerMode"); + System.Management.Automation.Remoting.Server.StdIOProcessMediator.Run( + initialCommand: s_cpp.InitialCommand, + workingDirectory: null, + configurationName: null); exitCode = 0; } else if (s_cpp.NamedPipeServerMode) @@ -201,22 +214,16 @@ internal static int Start(string bannerText, string helpText) ApplicationInsightsTelemetry.SendPSCoreStartupTelemetry("NamedPipe"); ProfileOptimization.StartProfile("StartupProfileData-NamedPipeServerMode"); System.Management.Automation.Remoting.RemoteSessionNamedPipeServer.RunServerMode( - s_cpp.ConfigurationName); - exitCode = 0; - } - else if (s_cpp.SSHServerMode) - { - ApplicationInsightsTelemetry.SendPSCoreStartupTelemetry("SSHServer"); - ProfileOptimization.StartProfile("StartupProfileData-SSHServerMode"); - System.Management.Automation.Remoting.Server.SSHProcessMediator.Run(s_cpp.InitialCommand); + configurationName: s_cpp.ConfigurationName); exitCode = 0; } else if (s_cpp.SocketServerMode) { ApplicationInsightsTelemetry.SendPSCoreStartupTelemetry("SocketServerMode"); ProfileOptimization.StartProfile("StartupProfileData-SocketServerMode"); - System.Management.Automation.Remoting.Server.HyperVSocketMediator.Run(s_cpp.InitialCommand, - s_cpp.ConfigurationName); + System.Management.Automation.Remoting.Server.HyperVSocketMediator.Run( + initialCommand: s_cpp.InitialCommand, + configurationName: s_cpp.ConfigurationName); exitCode = 0; } else diff --git a/src/System.Management.Automation/engine/remoting/server/OutOfProcServerMediator.cs b/src/System.Management.Automation/engine/remoting/server/OutOfProcServerMediator.cs index 06c992b842b..fe17a31f0cb 100644 --- a/src/System.Management.Automation/engine/remoting/server/OutOfProcServerMediator.cs +++ b/src/System.Management.Automation/engine/remoting/server/OutOfProcServerMediator.cs @@ -8,7 +8,6 @@ #if !UNIX using System.Security.Principal; #endif -using Microsoft.Win32.SafeHandles; using Dbg = System.Management.Automation.Diagnostics; @@ -73,14 +72,6 @@ protected void ProcessingThreadStart(object state) { try { -#if !CORECLR - // CurrentUICulture is not available in Thread Class in CSS - // WinBlue: 621775. Thread culture is not properly set - // for local background jobs causing experience differences - // between local console and local background jobs. - Thread.CurrentThread.CurrentUICulture = Microsoft.PowerShell.NativeCultureResolver.UICulture; - Thread.CurrentThread.CurrentCulture = Microsoft.PowerShell.NativeCultureResolver.Culture; -#endif string data = state as string; OutOfProcessUtils.ProcessData(data, callbacks); } @@ -307,7 +298,10 @@ protected void OnCloseAckPacketReceived(Guid psGuid) #region Methods - protected OutOfProcessServerSessionTransportManager CreateSessionTransportManager(string configurationName, PSRemotingCryptoHelperServer cryptoHelper, string workingDirectory) + protected OutOfProcessServerSessionTransportManager CreateSessionTransportManager( + string configurationName, + PSRemotingCryptoHelperServer cryptoHelper, + string workingDirectory) { PSSenderInfo senderInfo; #if !UNIX @@ -335,7 +329,11 @@ protected OutOfProcessServerSessionTransportManager CreateSessionTransportManage return tm; } - protected void Start(string initialCommand, PSRemotingCryptoHelperServer cryptoHelper, string workingDirectory = null, string configurationName = null) + protected void Start( + string initialCommand, + PSRemotingCryptoHelperServer cryptoHelper, + string workingDirectory, + string configurationName) { _initialCommand = initialCommand; @@ -417,35 +415,13 @@ protected void Start(string initialCommand, PSRemotingCryptoHelperServer cryptoH } #endregion - - #region Static Methods - - internal static void AppDomainUnhandledException(object sender, UnhandledExceptionEventArgs args) - { - // args can never be null. - Exception exception = (Exception)args.ExceptionObject; - // log the exception to crimson event logs - PSEtwLog.LogOperationalError(PSEventId.AppDomainUnhandledException, - PSOpcode.Close, PSTask.None, - PSKeyword.UseAlwaysOperational, - exception.GetType().ToString(), exception.Message, - exception.StackTrace); - - PSEtwLog.LogAnalyticError(PSEventId.AppDomainUnhandledException_Analytic, - PSOpcode.Close, PSTask.None, - PSKeyword.ManagedPlugin | PSKeyword.UseAlwaysAnalytic, - exception.GetType().ToString(), exception.Message, - exception.StackTrace); - } - - #endregion } - internal sealed class OutOfProcessMediator : OutOfProcessMediatorBase + internal sealed class StdIOProcessMediator : OutOfProcessMediatorBase { #region Private Data - private static OutOfProcessMediator s_singletonInstance; + private static StdIOProcessMediator s_singletonInstance; #endregion @@ -453,10 +429,10 @@ internal sealed class OutOfProcessMediator : OutOfProcessMediatorBase /// /// The mediator will take actions from the StdIn stream and responds to them. - /// It will replace StdIn,StdOut and StdErr stream with TextWriter.Null's. This is + /// It will replace StdIn,StdOut and StdErr stream with TextWriter.Null. This is /// to make sure these streams are totally used by our Mediator. /// - private OutOfProcessMediator() : base(true) + private StdIOProcessMediator() : base(true) { // Create input stream reader from Console standard input stream. // We don't use the provided Console.In TextReader because it can have @@ -465,9 +441,6 @@ private OutOfProcessMediator() : base(true) // stream encoding. This way the stream encoding is determined by the // stream BOM as needed. originalStdIn = new StreamReader(Console.OpenStandardInput(), true); - - // replacing StdIn with Null so that no other app messes with the - // original stream. Console.SetIn(TextReader.Null); // replacing StdOut with Null so that no other app messes with the @@ -490,61 +463,11 @@ private OutOfProcessMediator() : base(true) /// /// Specifies the initialization script. /// Specifies the initial working directory. The working directory is set before the initial command. - internal static void Run(string initialCommand, string workingDirectory) - { - lock (SyncObject) - { - if (s_singletonInstance != null) - { - Dbg.Assert(false, "Run should not be called multiple times"); - return; - } - - s_singletonInstance = new OutOfProcessMediator(); - } - -#if !CORECLR // AppDomain is not available in CoreCLR - // Setup unhandled exception to log events - AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomainUnhandledException); -#endif - s_singletonInstance.Start(initialCommand, new PSRemotingCryptoHelperServer(), workingDirectory); - } - - #endregion - } - - internal sealed class SSHProcessMediator : OutOfProcessMediatorBase - { - #region Private Data - - private static SSHProcessMediator s_singletonInstance; - - #endregion - - #region Constructors - - private SSHProcessMediator() : base(true) - { - originalStdIn = new StreamReader(Console.OpenStandardInput(), true); - originalStdOut = new OutOfProcessTextWriter( - new StreamWriter(Console.OpenStandardOutput())); - originalStdErr = new OutOfProcessTextWriter( - new StreamWriter(Console.OpenStandardError())); - - // Disable console from writing to the PSRP streams. - Console.SetIn(TextReader.Null); - Console.SetOut(TextWriter.Null); - Console.SetError(TextWriter.Null); - } - - #endregion - - #region Static Methods - - /// - /// - /// - internal static void Run(string initialCommand) + /// Specifies an optional configuration name that configures the endpoint session. + internal static void Run( + string initialCommand, + string workingDirectory, + string configurationName) { lock (SyncObject) { @@ -554,10 +477,14 @@ internal static void Run(string initialCommand) return; } - s_singletonInstance = new SSHProcessMediator(); + s_singletonInstance = new StdIOProcessMediator(); } - s_singletonInstance.Start(initialCommand, new PSRemotingCryptoHelperServer()); + s_singletonInstance.Start( + initialCommand: initialCommand, + cryptoHelper: new PSRemotingCryptoHelperServer(), + workingDirectory: workingDirectory, + configurationName: configurationName); } #endregion @@ -626,11 +553,11 @@ internal static void Run( s_singletonInstance = new NamedPipeProcessMediator(namedPipeServer); } -#if !CORECLR - // AppDomain is not available in CoreCLR - AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomainUnhandledException); -#endif - s_singletonInstance.Start(initialCommand, new PSRemotingCryptoHelperServer(), namedPipeServer.ConfigurationName); + s_singletonInstance.Start( + initialCommand: initialCommand, + cryptoHelper: new PSRemotingCryptoHelperServer(), + workingDirectory: null, + configurationName: namedPipeServer.ConfigurationName); } #endregion @@ -716,12 +643,11 @@ internal static void Run( s_instance = new HyperVSocketMediator(); } -#if !CORECLR - // AppDomain is not available in CoreCLR - AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomainUnhandledException); -#endif - - s_instance.Start(initialCommand, new PSRemotingCryptoHelperServer(), configurationName); + s_instance.Start( + initialCommand: initialCommand, + cryptoHelper: new PSRemotingCryptoHelperServer(), + workingDirectory: null, + configurationName: configurationName); } #endregion From 7fc60d1d5fa2f59c58ba261c7f220ccd3d6fcf9b Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 1 Sep 2021 10:35:27 -0700 Subject: [PATCH 025/645] Bump .NET to `6.0.100-rc.1.21430.44` (#16028) --- DotnetRuntimeMetadata.json | 2 +- assets/wix/files.wxs | 6 +++--- global.json | 2 +- nuget.config | 1 + ...crosoft.PowerShell.Commands.Management.csproj | 2 +- .../Microsoft.PowerShell.Commands.Utility.csproj | 4 ++-- .../Microsoft.PowerShell.CoreCLR.Eventing.csproj | 2 +- .../Microsoft.PowerShell.SDK.csproj | 8 ++++---- .../Microsoft.WSMan.Management.csproj | 2 +- .../System.Management.Automation.csproj | 16 ++++++++-------- test/tools/TestService/TestService.csproj | 2 +- test/tools/WebListener/WebListener.csproj | 4 ++-- tools/UpdateDotnetRuntime.ps1 | 1 + 13 files changed, 27 insertions(+), 25 deletions(-) diff --git a/DotnetRuntimeMetadata.json b/DotnetRuntimeMetadata.json index 18778f58555..ed45c403569 100644 --- a/DotnetRuntimeMetadata.json +++ b/DotnetRuntimeMetadata.json @@ -8,6 +8,6 @@ "nextChannel": "6.0.1xx-rc1" }, "internalfeed" : { - "url": null + "url": "https://pkgs.dev.azure.com/dnceng/public/_packaging/6.0.100-rc.1.21430.44-shipping/nuget/v2" } } diff --git a/assets/wix/files.wxs b/assets/wix/files.wxs index 84539cac520..e8fdfdefcb4 100644 --- a/assets/wix/files.wxs +++ b/assets/wix/files.wxs @@ -3054,8 +3054,8 @@ - - + + @@ -4041,7 +4041,7 @@ - + diff --git a/global.json b/global.json index 21a9b9bc28f..35448cc72b6 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "6.0.100-rc.1.21426.1" + "version": "6.0.100-rc.1.21430.44" } } diff --git a/nuget.config b/nuget.config index a8ed27bf99e..0e9671d1a70 100644 --- a/nuget.config +++ b/nuget.config @@ -4,6 +4,7 @@ + diff --git a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj index e8c7e78918e..67394beef83 100644 --- a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj +++ b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj @@ -47,7 +47,7 @@ - + diff --git a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj index 19645f35ae0..74921e9ea62 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj +++ b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj @@ -32,8 +32,8 @@ - - + + diff --git a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj index a309820ffab..1d5a865f535 100644 --- a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj +++ b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj index 5875f9904dd..e03dbe4701a 100644 --- a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj +++ b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj @@ -18,9 +18,9 @@ - - - + + + @@ -30,7 +30,7 @@ - + diff --git a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj index fbd381b9d6f..c3fcff08ffc 100644 --- a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj +++ b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/System.Management.Automation/System.Management.Automation.csproj b/src/System.Management.Automation/System.Management.Automation.csproj index 4b6917b168e..0f167c7ef35 100644 --- a/src/System.Management.Automation/System.Management.Automation.csproj +++ b/src/System.Management.Automation/System.Management.Automation.csproj @@ -16,16 +16,16 @@ - - - + + + - - + + - - - + + + diff --git a/test/tools/TestService/TestService.csproj b/test/tools/TestService/TestService.csproj index a3ce2faa839..a44ac7810c6 100644 --- a/test/tools/TestService/TestService.csproj +++ b/test/tools/TestService/TestService.csproj @@ -13,7 +13,7 @@ - + diff --git a/test/tools/WebListener/WebListener.csproj b/test/tools/WebListener/WebListener.csproj index 6939ea252a3..4a0c0c38cba 100644 --- a/test/tools/WebListener/WebListener.csproj +++ b/test/tools/WebListener/WebListener.csproj @@ -7,8 +7,8 @@ - - + + diff --git a/tools/UpdateDotnetRuntime.ps1 b/tools/UpdateDotnetRuntime.ps1 index cbaa69bfd02..15c482ba630 100644 --- a/tools/UpdateDotnetRuntime.ps1 +++ b/tools/UpdateDotnetRuntime.ps1 @@ -181,6 +181,7 @@ function Get-DotnetUpdate { NewVersion = $SDKVersionOverride Message = $null FeedUrl = $feedUrl + Quality = $quality } } From 6cde546203bd2ce8d94ca70b0660e43b03b44899 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 1 Sep 2021 10:36:58 -0700 Subject: [PATCH 026/645] Add `.stylecop` to `filetypexml` and format it (#16025) --- CHANGELOG/6.0.md | 2 +- CHANGELOG/6.1.md | 2 +- CHANGELOG/6.2.md | 2 +- CHANGELOG/7.0.md | 2 +- .../CommandCompletion/CompletionCompleters.cs | 2 +- .../releaseBuild/azureDevOps/releaseBuild.yml | 8 + .../azureDevOps/templates/compliance.yml | 5 +- tools/terms/FileTypeSet.xml | 379 ------------------ tools/terms/TermsExclusion.xml | 11 + 9 files changed, 26 insertions(+), 387 deletions(-) delete mode 100644 tools/terms/FileTypeSet.xml create mode 100644 tools/terms/TermsExclusion.xml diff --git a/CHANGELOG/6.0.md b/CHANGELOG/6.0.md index 2c4da7b7098..948197f0619 100644 --- a/CHANGELOG/6.0.md +++ b/CHANGELOG/6.0.md @@ -104,7 +104,7 @@ work is required for Microsoft to continue to sign and release packages from the project as official Microsoft packages. - Remove `PerformWSManPluginReportCompletion`, which was not used, from `pwrshplugin.dll` (#5498) (Thanks @bergmeister!) -- Remove exclusion for hang and add context exception for remaining instances (#5595) +- Remove exclusion for unresponsive condition and add context exception for remaining instances (#5595) - Replace `strlen` with `strnlen` in native code (#5510) ## [6.0.0-rc] - 2017-11-16 diff --git a/CHANGELOG/6.1.md b/CHANGELOG/6.1.md index f8e12f47001..59cf2842d78 100644 --- a/CHANGELOG/6.1.md +++ b/CHANGELOG/6.1.md @@ -428,7 +428,7 @@ - Fix crash when terminal is reset (#6777) - Fix a module-loading regression that caused an infinite loop (#6843) - Further improve `PSMethod` to `Delegate` conversion (#6851) -- Blacklist `System.Windows.Forms` from loading to prevent a crash (#6822) +- Block list `System.Windows.Forms` from loading to prevent a crash (#6822) - Fix `Format-Table` where rows were being trimmed unnecessarily if there's only one row of headers (#6772) - Fix `SetDate` function in `libpsl-native` to avoid corrupting memory during `P/Invoke` (#6881) - Fix tab completions for hash table (#6839) (Thanks @iSazonov!) diff --git a/CHANGELOG/6.2.md b/CHANGELOG/6.2.md index 06ad8f41482..bf54f978eba 100644 --- a/CHANGELOG/6.2.md +++ b/CHANGELOG/6.2.md @@ -844,7 +844,7 @@ ### Documentation and Help Content -- Replace ambiguous `hang` term (#7902, #7931) (Thanks @iSazonov!) +- Replace ambiguous term (#7902, #7931) (Thanks @iSazonov!) - Updating incorrect example of `PowerShell.Create()` (#7926) (Thanks @1RedOne!) - Update `governance.md` (#7927) (Thanks @tommymaynard!) - Add `cURL` to the Bash users list in `README.md` (#7948) (Thanks @vmsilvamolina!) diff --git a/CHANGELOG/7.0.md b/CHANGELOG/7.0.md index fe0eca4c5a7..9b7c0a9f45b 100644 --- a/CHANGELOG/7.0.md +++ b/CHANGELOG/7.0.md @@ -327,7 +327,7 @@ Move to .NET Core 3.1.202 SDK and update packages. - Skip null data in output data received handler to fix a `NullReferenceException` (#11448) (Thanks @iSazonov!) - Add `ssh` parameter sets for the parameter `-JobName` in `Invoke-Command` (#11444) - Adding `PowerShell Editor Services` and `PSScriptAnalyzer` to tracked modules (#11514) -- Fix key exchange hang with `SecureString` for the `OutOfProc` transports (#11380, #11406) +- Fix condition when key exchange stops responding with `SecureString` for the `OutOfProc` transports (#11380, #11406) - Add setting to disable the implicit `WinPS` module loading (#11332) ### General Cmdlet Updates and Fixes diff --git a/src/System.Management.Automation/engine/CommandCompletion/CompletionCompleters.cs b/src/System.Management.Automation/engine/CommandCompletion/CompletionCompleters.cs index b525098ac6c..d8360e714d8 100644 --- a/src/System.Management.Automation/engine/CommandCompletion/CompletionCompleters.cs +++ b/src/System.Management.Automation/engine/CommandCompletion/CompletionCompleters.cs @@ -7652,7 +7652,7 @@ public PropertyNameCompleter() /// /// Initializes a new instance of the class. /// - /// The name of the property of the input object for witch to complete with property names. + /// The name of the property of the input object for which to complete with property names. public PropertyNameCompleter(string parameterNameOfInput) { _parameterNameOfInput = parameterNameOfInput; diff --git a/tools/releaseBuild/azureDevOps/releaseBuild.yml b/tools/releaseBuild/azureDevOps/releaseBuild.yml index bd45d7ffb8a..4ffa04fe3f2 100644 --- a/tools/releaseBuild/azureDevOps/releaseBuild.yml +++ b/tools/releaseBuild/azureDevOps/releaseBuild.yml @@ -10,6 +10,11 @@ pr: - master - release* +parameters: + - name: ForceAzureBlobDelete + displayName: Delete Azure Blob + default: false + resources: repositories: - repository: ComplianceRepo @@ -29,6 +34,9 @@ variables: value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])] - name: branchCounter value: $[counter(variables['branchCounterKey'], 1)] + - name: ForceAzureBlobDelete + value: ${{ parameters.ForceAzureBlobDelete }} + stages: - stage: prep jobs: diff --git a/tools/releaseBuild/azureDevOps/templates/compliance.yml b/tools/releaseBuild/azureDevOps/templates/compliance.yml index 9e6c7ad1c61..280ae3551f6 100644 --- a/tools/releaseBuild/azureDevOps/templates/compliance.yml +++ b/tools/releaseBuild/azureDevOps/templates/compliance.yml @@ -85,12 +85,11 @@ jobs: inputs: targetType: F optionsFC: 0 - optionsXS: 0 + optionsXS: 1 optionsPE: '1|2|3|4' optionsHMENABLE: 0 optionsRulesDBPath: '$(Build.SourcesDirectory)\tools\terms\PowerShell-Terms-Rules.mdb' - optionsFTPATH: '$(Build.SourcesDirectory)\tools\terms\FileTypeSet.xml' - toolVersion: 5.8.2.1 + optionsUEPath: $(Build.SourcesDirectory)\tools\terms\TermsExclusion.xml continueOnError: true # add RoslynAnalyzers diff --git a/tools/terms/FileTypeSet.xml b/tools/terms/FileTypeSet.xml deleted file mode 100644 index 2b0821d6777..00000000000 --- a/tools/terms/FileTypeSet.xml +++ /dev/null @@ -1,379 +0,0 @@ - - - - -Pure Text Files - -.txt -.des -.pwd -.asm -.cmd -.ini -.poc -.pwt -.hpj -.sql -.inf -.log -.def -.url -.bat -.aspx -.idl -.sys -.resources -.strings -.md -.yml -.yaml -.spelling -.gitignore -.gitattributes -.gitmodules -.csv -.tsv - - - -CodeFiles - -.frm -.inc -.cpp -.cls -.c -.hpp -.vbs -.java -.cs -.cxx -.h -.jav -.bas -.hxx -.js -.pl -.rc -.vb -.json -.resjson -.fs -.fsi -.fsx -.m -.psm1 -.config -.ps1 -.psd1 -.cmake -.sh -.cshtml -.plist -.mof -.mc - - - -XML Files - -.xml -.hxa -.hxk -.hxl -.xsl -.hxc -.hxt -.hxm -.resx -.hxe -.hxf -.hxv -.acctb -.accfl -.xaml -.ttml -.ddue -.sln -.props -.ps1xml -.csproj -.xsd -.svg -.clixml -.nuspec -.cdxml -.manifest - - - -Microsoft Word Documents - -.doc -.dot -.wiz - - - -Microsoft Access Database Compatible - -.mdb -.mda -.mde -.mpd -.mdt - - - -Microsoft PowerPoint Presentation - -.ppt -.pot -.pps - - - -Microsoft Publisher Files - -.pub - - - -Microsoft Excel Workbooks - -.xls -.xlt - - - -Pure Binary Files - -.com -.bin -.tlb -.drv -.fon -.blg -.gif -.png -.icns -.ico -.bmp -.pfx - - - -Localization resource databases - -.edb -.lcl -.xlf -.xliff - - - -Microsoft Project Files - -.mpp -.mpt - - - -Microsoft Visio Files - -.vsd -.vdx -.vss -.vst - - - -Windows Installer databases - -.msi -.msm - - - -Zip Files - -.zip -.accdt -.axtr - - - -Cabinet / MS Compression Files - -.cab - - - -Table driven IME lexicons - -.mb - - - -IME ( IMD ) Files - -.imd - - - -TrueType Font Files - -.ttf - - - -Microsoft Outlook Mail Files - -.msg -.oft - - - -HTML Help 2.0 Files / InfoTech5.x Storage System Files - -.its -.hxh -.hxr -.hxw -.hxi -.hxs -.hxq - - - -Adobe Acrobat PDF Files - -.pdf - - - -HTML Files / Web Page - -.htm -.dtd -.hhk -.htw -.asp -.htc -.htx -.html -.hhc -.css -.stm - - - -Rich Text Files - -.rtf - - - -Windows 3.x Write Files - -.wri - - - -MHTML Files - -.eml -.nws -.mht - - - -Word 2007 Files - -.docx -.docm -.dotx -.dotm - - - -Excel 2007 Files - -.xlsx -.xlsm -.xltx -.xltm -.xlsb -.xlam - - - -Power Point 2007 Files - -.pptx -.pptm -.potx -.potm -.ppsx -.ppsm -.ppam - - - -Access 2007 Files - -.accdb -.accde -.accdr - - - -Win32/64-based executable (image) Files - -.exe -.dll -.ocx -.scr -.acm -.rll -.cpl -.mui -.ax -.ime - - - -HTML Help 1.0 Files - -.chm - - - -LocStudio lsg - -.lsg - - - -Microsoft Office OneNote Files - -.one -.onepkg - - - -Custom Parsers - - - - -Visio 2011 Files - -.vstx -.vsdx -.vssx - - - - - diff --git a/tools/terms/TermsExclusion.xml b/tools/terms/TermsExclusion.xml new file mode 100644 index 00000000000..e314fc1b7a2 --- /dev/null +++ b/tools/terms/TermsExclusion.xml @@ -0,0 +1,11 @@ + + + + .GIT + + + + + + + From 3f571d8a8f39a21e2eecb8737a490b8c1d161c69 Mon Sep 17 00:00:00 2001 From: Ikko Ashimine Date: Thu, 2 Sep 2021 21:42:06 +0900 Subject: [PATCH 027/645] Fix typo in build.psm1 (#16038) targetting -> targeting --- build.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index c87e1147571..7441f48eb25 100644 --- a/build.psm1 +++ b/build.psm1 @@ -865,7 +865,7 @@ function New-PSOptions { } } - # We plan to release packages targetting win7-x64 and win7-x86 RIDs, + # We plan to release packages targeting win7-x64 and win7-x86 RIDs, # which supports all supported windows platforms. # So we, will change the RID to win7- $Runtime = $RID -replace "win\d+", "win7" From d3d88976190ee39fdf0b5040388fd8d13e4cb140 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Sep 2021 09:59:58 -0700 Subject: [PATCH 028/645] Bump Microsoft.CodeAnalysis.CSharp from 4.0.0-3.final to 4.0.0-4.21430.4 (#16036) --- .../Microsoft.PowerShell.Commands.Utility.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj index 74921e9ea62..1aff829f1d4 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj +++ b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj @@ -31,7 +31,7 @@ - + From 14ced821c94547ae1d5c66840c4bdd560224b04e Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Fri, 3 Sep 2021 15:18:50 -0700 Subject: [PATCH 029/645] Fix the GitHub Action for updating .NET daily builds (#16042) --- .github/workflows/daily.yml | 5 +++-- tools/UpdateDotnetRuntime.ps1 | 27 ++++++++++++++++++++------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 6e3a38ac1b1..a1c883c98f0 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -34,7 +34,7 @@ jobs: Write-Verbose "OLD_VERSION=$currentVersion" -Verbose "OLD_VERSION=$currentVersion" | Out-File $env:GITHUB_ENV -Append - ./tools/UpdateDotnetRuntime.ps1 -UpdateMSIPackaging + ./tools/UpdateDotnetRuntime.ps1 -UpdateMSIPackaging -UseInternalFeed $newVersion = (Get-Content .\global.json | ConvertFrom-Json).sdk.version Write-Verbose "NEW_VERSION=$newVersion" -Verbose "NEW_VERSION=$newVersion" | Out-File $env:GITHUB_ENV -Append @@ -48,8 +48,9 @@ jobs: if: failure() with: webhook_url: ${{ secrets.PS_BUILD_TEAMS_CHANNEL }} + overwrite: "{title: `Failure in updating .NET build. Look at ${workflow_link}`}" - name: Create Pull Request - uses: peter-evans/create-pull-request@v2 + uses: peter-evans/create-pull-request@v3 id: cpr if: env.CREATE_PR == 'true' with: diff --git a/tools/UpdateDotnetRuntime.ps1 b/tools/UpdateDotnetRuntime.ps1 index 15c482ba630..5126d109999 100644 --- a/tools/UpdateDotnetRuntime.ps1 +++ b/tools/UpdateDotnetRuntime.ps1 @@ -187,17 +187,23 @@ function Get-DotnetUpdate { try { - $latestSDKVersionString = Invoke-RestMethod -Uri "http://aka.ms/dotnet/$channel/$quality/sdk-productVersion.txt" -ErrorAction Stop | ForEach-Object { $_.Trim() } - $selectedQuality = $quality + try { + $latestSDKVersionString = Invoke-RestMethod -Uri "http://aka.ms/dotnet/$channel/$quality/sdk-productVersion.txt" -ErrorAction Stop | ForEach-Object { $_.Trim() } + $selectedQuality = $quality + } catch { + if ($_.exception.Response.StatusCode -eq 'NotFound') { + Write-Verbose "Build not found for Channel: $Channel and Quality: $Quality" -Verbose + } else { + throw $_ + } + } - if (-not $latestSDKVersionString.StartsWith($sdkImageVersion)) - { + if (-not $latestSDKVersionString -or -not $latestSDKVersionString.StartsWith($sdkImageVersion)) { # we did not get a version number so fall back to daily $latestSDKVersionString = Invoke-RestMethod -Uri "http://aka.ms/dotnet/$channel/$qualityFallback/sdk-productVersion.txt" -ErrorAction Stop | ForEach-Object { $_.Trim() } $selectedQuality = $qualityFallback - if (-not $latestSDKVersionString.StartsWith($sdkImageVersion)) - { + if (-not $latestSDKVersionString.StartsWith($sdkImageVersion)) { throw "No build found!" } } @@ -272,8 +278,15 @@ if ($dotnetUpdate.ShouldUpdate) { if ($feedname -eq 'dotnet-internal') { # This NuGet feed is for internal to Microsoft use only. $dotnetInternalFeed = $dotnetMetadataJson.internalfeed.url - $updatedNugetFile = $nugetFileContent -replace "", " `r`n " + + $updatedNugetFile = if ($nugetFileContent.Contains('dotnet-internal')) { + $nugetFileContent -replace ".`r`n " + } else { + $nugetFileContent -replace "", " `r`n " + } + $updatedNugetFile | Out-File "$PSScriptRoot/../nuget.config" -Force + Register-PackageSource -Name 'dotnet-internal' -Location $dotnetInternalFeed -ProviderName NuGet Write-Verbose -Message "Register new package source 'dotnet-internal'" -verbose } From 66b5d5017059bb1ae49b72b6e1dcb2411ee0f670 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Fri, 3 Sep 2021 15:36:31 -0700 Subject: [PATCH 030/645] Set $? correctly for command expression with redirections (#16046) --- .../engine/parser/Compiler.cs | 20 ++++- .../Language/Scripting/DollarHook.Tests.ps1 | 80 +++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/Compiler.cs b/src/System.Management.Automation/engine/parser/Compiler.cs index 14057f05f68..bcddbee0b24 100644 --- a/src/System.Management.Automation/engine/parser/Compiler.cs +++ b/src/System.Management.Automation/engine/parser/Compiler.cs @@ -3279,9 +3279,9 @@ private bool ShouldSetExecutionStatusToSuccess(StatementAst statementAst) /// True is the compiler should add the success setting, false otherwise. private bool ShouldSetExecutionStatusToSuccess(PipelineAst pipelineAst) { - ExpressionAst expressionAst = pipelineAst.GetPureExpression(); + ExpressionAst expressionAst = GetSingleExpressionFromPipeline(pipelineAst); - // If the pipeline is not a simple expression, it will set $? + // If the pipeline is not a single expression, it will set $? if (expressionAst == null) { return false; @@ -3291,6 +3291,22 @@ private bool ShouldSetExecutionStatusToSuccess(PipelineAst pipelineAst) return ShouldSetExecutionStatusToSuccess(expressionAst); } + /// + /// If the pipeline contains a single expression, the expression is returned, otherwise null is returned. + /// This method is different from in that it allows the single + /// expression to have redirections. + /// + private static ExpressionAst GetSingleExpressionFromPipeline(PipelineAst pipelineAst) + { + var pipelineElements = pipelineAst.PipelineElements; + if (pipelineElements.Count == 1 && pipelineElements[0] is CommandExpressionAst expr) + { + return expr.Expression; + } + + return null; + } + /// /// Determines whether an assignment statement must have an explicit setting /// for $? = $true after it by the compiler. diff --git a/test/powershell/Language/Scripting/DollarHook.Tests.ps1 b/test/powershell/Language/Scripting/DollarHook.Tests.ps1 index 86fb70306d2..198b93a1de4 100644 --- a/test/powershell/Language/Scripting/DollarHook.Tests.ps1 +++ b/test/powershell/Language/Scripting/DollarHook.Tests.ps1 @@ -85,4 +85,84 @@ Describe 'Tests for setting $? for execution success' -Tag 'CI' { $script:hookValues | Should -Be $HookResults $output | Should -Be $PipelineResults } + + It 'Sets $? correctly for single expression with redirection ''''' -TestCases @( + @{ Expression = 'Write-Error "Bad"; Hook $?; "b" > $null; Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; "b" >> $null; Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; "b" > TESTDRIVE:\out.txt; Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; "b" >> TESTDRIVE:\out.txt; Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; "b" 2> $null; Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; "b" 2>> $null; Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; "b" 2> TESTDRIVE:\out.txt; Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; "b" 2>> TESTDRIVE:\out.txt; Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; "b" 2>&1; Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; "b" 2>&1; Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; "b" 2>&1 > $null; Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; "b" 2>&1 > TESTDRIVE:\out.txt; Hook $?'; HookResults = $($false, $true); } + + @{ Expression = 'Write-Error "Bad"; Hook $?; ("b" > $null); Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; ("b" >> $null); Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; ("b" > TESTDRIVE:\out.txt); Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; ("b" >> TESTDRIVE:\out.txt); Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; ("b" 2> $null); Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; ("b" 2>> $null); Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; ("b" 2> TESTDRIVE:\out.txt); Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; ("b" 2>> TESTDRIVE:\out.txt); Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; ("b" 2>&1); Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; ("b" 2>&1 > $null); Hook $?'; HookResults = $($false, $true); } + @{ Expression = 'Write-Error "Bad"; Hook $?; ("b" 2>&1 > TESTDRIVE:\out.txt); Hook $?'; HookResults = $($false, $true); } + ) { + param([string]$Expression, [object[]]$HookResults) + + Invoke-Expression $Expression 2>&1 >$null + + $script:hookValues | Should -Be $HookResults + } + + Context 'Validate $? with potential terminating error' { + + ## Script execution directly in Pester tests will be enclosed in try/catch by the Pester, + ## and therefore, general exceptions thrown from an expression like "1/0" will be turned + ## into a terminating exception, which will stop the execution of remaining scripts. + ## + ## For those test cases, we have to use a PowerShell instance, so as to keep the default + ## error handling behavior for the general exceptions. + + BeforeAll { + $pwsh = [powershell]::Create() + + function Invoke([string] $script) + { + $pwsh.Commands.Clear() + $pwsh.Streams.ClearStreams() + $pwsh.AddScript($script).Invoke() + } + + $root = Join-Path ([System.IO.Path]::GetTempPath()) ([guid]::NewGuid().ToString()) + $null = Invoke "New-PSDrive -Name TESTDRIVE -PSProvider FileSystem -Root $root" + } + + Afterall { + $null = Invoke "Remove-PSDrive -Name TESTDRIVE -PSProvider FileSystem -Force" + $pwsh.Dispose() + } + + It 'Sets $? correctly for single expression with redirection ''''' -TestCases @( + @{ Expression = '1/0 > $null; $?'; Result = $false; } + @{ Expression = '1/0 >> $null; $?'; Result = $false; } + @{ Expression = '1/0 > TESTDRIVE:\out.txt; $?'; Result = $false; } + @{ Expression = '1/0 >> TESTDRIVE:\out.txt; $?'; Result = $false; } + @{ Expression = '1/0 2>&1; $?'; Result = $false; } + @{ Expression = '1/0 2>&1 > $null; $?'; Result = $false; } + @{ Expression = '1/0 2>&1 > TESTDRIVE:\out.txt; $?'; Result = $false; } + + @{ Expression = '"b" > NonExistDrive:\nowhere.txt; $?'; Result = $false; } + @{ Expression = '"b" >> NonExistDrive:\nowhere.txt; $?'; Result = $false; } + @{ Expression = '"b" 2>&1 > NonExistDrive:\nowhere.txt; $?'; Result = $false; } + ) { + param([string]$Expression, $Result) + + Invoke $Expression | Should -Be $Result + } + } } From 17986d88c10eb6271b1aad2ca955889e45e2b285 Mon Sep 17 00:00:00 2001 From: Robert Holt Date: Fri, 3 Sep 2021 16:05:30 -0700 Subject: [PATCH 031/645] Improve `CommandInvocationIntrinsics` API documentation and style (#14369) --- .../engine/MshCmdlet.cs | 65 ++++++++++++------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/src/System.Management.Automation/engine/MshCmdlet.cs b/src/System.Management.Automation/engine/MshCmdlet.cs index 2c899531866..c11e674660c 100644 --- a/src/System.Management.Automation/engine/MshCmdlet.cs +++ b/src/System.Management.Automation/engine/MshCmdlet.cs @@ -674,40 +674,46 @@ internal IEnumerable GetCommands(string name, CommandTypes commandT } /// - /// Executes a piece of text as a script synchronously. + /// Executes a piece of text as a script synchronously in the caller's session state. + /// The given text will be executed in a child scope rather than dot-sourced. /// /// The script text to evaluate. - /// A collection of MshCobjects generated by the script. + /// A collection of MshCobjects generated by the script. Never null, but may be empty. /// Thrown if there was a parsing error in the script. /// Represents a script-level exception. /// public Collection InvokeScript(string script) { - return InvokeScript(script, true, PipelineResultTypes.None, null); + return InvokeScript(script, useNewScope: true, PipelineResultTypes.None, input: null); } /// - /// Executes a piece of text as a script synchronously. + /// Executes a piece of text as a script synchronously in the caller's session state. + /// The given text will be executed in a child scope rather than dot-sourced. /// /// The script text to evaluate. - /// The arguments to the script. - /// A collection of MshCobjects generated by the script. + /// The arguments to the script, available as $args. + /// A collection of MshCobjects generated by the script. Never null, but may be empty. /// Thrown if there was a parsing error in the script. /// Represents a script-level exception. /// public Collection InvokeScript(string script, params object[] args) { - return InvokeScript(script, true, PipelineResultTypes.None, null, args); + return InvokeScript(script, useNewScope: true, PipelineResultTypes.None, input: null, args); } /// + /// Executes a given scriptblock synchonously in the given session state. + /// The scriptblock will be executed in the calling scope (dot-sourced) rather than in a new child scope. /// - /// - /// - /// - /// + /// The session state in which to execute the scriptblock. + /// The scriptblock to execute. + /// The arguments to the scriptblock, available as $args. + /// A collection of the PSObjects emitted by the executing scriptblock. Never null, but may be empty. public Collection InvokeScript( - SessionState sessionState, ScriptBlock scriptBlock, params object[] args) + SessionState sessionState, + ScriptBlock scriptBlock, + params object[] args) { if (scriptBlock == null) { @@ -739,13 +745,18 @@ public Collection InvokeScript( /// /// Invoke a scriptblock in the current runspace, controlling if it gets a new scope. /// - /// If true, a new scope will be created. + /// If true, executes the scriptblock in a new child scope, otherwise the scriptblock is dot-sourced into the calling scope. /// The scriptblock to execute. /// Optionall input to the command. /// Arguments to pass to the scriptblock. - /// The result of the evaluation. + /// + /// A collection of the PSObjects generated by executing the script. Never null, but may be empty. + /// public Collection InvokeScript( - bool useLocalScope, ScriptBlock scriptBlock, IList input, params object[] args) + bool useLocalScope, + ScriptBlock scriptBlock, + IList input, + params object[] args) { if (scriptBlock == null) { @@ -771,21 +782,25 @@ public Collection InvokeScript( /// /// The script to evaluate. /// If true, evaluate the script in its own scope. - /// If false, the script will be evaluated in the current scope i.e. it will be "dotted" + /// If false, the script will be evaluated in the current scope i.e. it will be dot-sourced. /// If set to Output, all output will be streamed /// to the output pipe of the calling cmdlet. If set to None, the result will be returned /// to the caller as a collection of PSObjects. No other flags are supported at this time and /// will result in an exception if used. /// The list of objects to use as input to the script. - /// The array of arguments to the command. - /// A collection of MshCobjects generated by the script. This will be - /// empty if output was redirected. + /// The array of arguments to the command, available as $args. + /// A collection of PSObjects generated by the script. This will be + /// empty if output was redirected. Never null. /// Thrown if there was a parsing error in the script. /// Represents a script-level exception. /// Thrown if any redirect other than output is attempted. /// - public Collection InvokeScript(string script, bool useNewScope, - PipelineResultTypes writeToPipeline, IList input, params object[] args) + public Collection InvokeScript( + string script, + bool useNewScope, + PipelineResultTypes writeToPipeline, + IList input, + params object[] args) { if (script == null) throw new ArgumentNullException(nameof(script)); @@ -796,8 +811,12 @@ public Collection InvokeScript(string script, bool useNewScope, return InvokeScript(sb, useNewScope, writeToPipeline, input, args); } - private Collection InvokeScript(ScriptBlock sb, bool useNewScope, - PipelineResultTypes writeToPipeline, IList input, params object[] args) + private Collection InvokeScript( + ScriptBlock sb, + bool useNewScope, + PipelineResultTypes writeToPipeline, + IList input, + params object[] args) { if (_cmdlet != null) _cmdlet.ThrowIfStopping(); From 1e823e6677abb2c1aa4a795d24f1bbe67b51c2e9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 Sep 2021 14:11:13 -0700 Subject: [PATCH 032/645] Update .NET SDK version from `6.0.100-rc.1.21430.44` to `6.0.100-rc.1.21455.2` (#16041) --- assets/wix/files.wxs | 6 +++--- global.json | 2 +- nuget.config | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/assets/wix/files.wxs b/assets/wix/files.wxs index e8fdfdefcb4..8e821f1c6fc 100644 --- a/assets/wix/files.wxs +++ b/assets/wix/files.wxs @@ -3054,8 +3054,8 @@ - - + + @@ -4041,7 +4041,7 @@ - + diff --git a/global.json b/global.json index 35448cc72b6..0c2a82ceaf6 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "6.0.100-rc.1.21430.44" + "version": "6.0.100-rc.1.21455.2" } } diff --git a/nuget.config b/nuget.config index 0e9671d1a70..ba38f1bad11 100644 --- a/nuget.config +++ b/nuget.config @@ -10,3 +10,4 @@ + From 5872e47254d921353596d755e92566354ab70f45 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Tue, 7 Sep 2021 23:20:30 -0700 Subject: [PATCH 033/645] Fix regression in `Move-Item` to only fallback to CopyAndDelete in specific cases (#16029) If Move fails, check if known case before attempting CopyAndDelete: - if an item is attempted to be renamed across filesystem mount boundaries. - if the source and destination do not have the same root path. --- .../namespaces/FileSystemProvider.cs | 11 ++++++++++- .../FileSystem.Tests.ps1 | 16 ++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/System.Management.Automation/namespaces/FileSystemProvider.cs b/src/System.Management.Automation/namespaces/FileSystemProvider.cs index e95669c2961..b7923ab7352 100644 --- a/src/System.Management.Automation/namespaces/FileSystemProvider.cs +++ b/src/System.Management.Automation/namespaces/FileSystemProvider.cs @@ -6121,7 +6121,16 @@ private void MoveDirectoryInfoUnchecked(DirectoryInfo directory, string destinat directory.MoveTo(destinationPath); } - catch (IOException) +#if UNIX + // This is the errno returned by the rename() syscall + // when an item is attempted to be renamed across filesystem mount boundaries. + // 0x80131620 is returned if the source and destination do not have the same root path + catch (IOException e) when (e.HResult == 18 || e.HResult == -2146232800) +#else + // 0x80070005 ACCESS_DENIED is returned when trying to move files across volumes like DFS + // 0x80131620 is returned if the source and destination do not have the same root path + catch (IOException e) when (e.HResult == -2147024891 || e.HResult == -2146232800) +#endif { // Rather than try to ascertain whether we can rename a directory ahead of time, // it's both faster and more correct to try to rename it and fall back to copy/deleting it diff --git a/test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1 index 70a18ff604f..e5d150013cd 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1 @@ -170,11 +170,23 @@ Describe "Basic FileSystem Provider Tests" -Tags "CI" { } It "Verify Move-Item will not move to an existing file" { - { Move-Item -Path $testDir -Destination $testFile -ErrorAction Stop } | Should -Throw -ErrorId 'DirectoryExist,Microsoft.PowerShell.Commands.MoveItemCommand' - $error[0].Exception | Should -BeOfType System.IO.IOException + if ($IsWindows) { + $expectedError = 'MoveDirectoryItemIOError,Microsoft.PowerShell.Commands.MoveItemCommand' + } + else { + $expectedError = 'DirectoryExist,Microsoft.PowerShell.Commands.MoveItemCommand' + } + + $e = { Move-Item -Path $testDir -Destination $testFile -ErrorAction Stop } | Should -Throw -ErrorId $expectedError -PassThru + $e.Exception | Should -BeOfType System.IO.IOException $testDir | Should -Exist } + It 'Verify Move-Item fails for non-existing destination path' { + $e = { Move-Item -Path $testDir -Destination TestDrive:/0/2/0 -ErrorAction Stop } | Should -Throw -ErrorId 'MoveDirectoryItemIOError,Microsoft.PowerShell.Commands.MoveItemCommand' -PassThru + $e.Exception | Should -BeOfType System.IO.IOException + } + It "Verify Move-Item throws correct error for non-existent source" { { Move-Item -Path /does/not/exist -Destination $testFile -ErrorAction Stop } | Should -Throw -ErrorId 'PathNotFound,Microsoft.PowerShell.Commands.MoveItemCommand' } From 4c5f987e6c1f850429cf7d808a249a84ffbf88c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Sep 2021 14:19:07 -0700 Subject: [PATCH 034/645] Bump `Microsoft.CodeAnalysis.NetAnalyzers` (#16045) --- Analyzers.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Analyzers.props b/Analyzers.props index 0dc31a83b3e..ab1655e1527 100644 --- a/Analyzers.props +++ b/Analyzers.props @@ -1,7 +1,7 @@ - + From 12dbdd9f05ff32e4d42e923e13a21894ce3e1c73 Mon Sep 17 00:00:00 2001 From: Robert Holt Date: Wed, 8 Sep 2021 14:30:44 -0700 Subject: [PATCH 035/645] Ensure locale is set correctly on Ubuntu 20.04 in CI (#16067) --- .vsts-ci/linux-daily.yml | 6 ++++++ .vsts-ci/templates/nix-test.yml | 11 +---------- build.psm1 | 20 ++++++++++++++++++++ 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/.vsts-ci/linux-daily.yml b/.vsts-ci/linux-daily.yml index e6a5375b0c9..14cdcc1372c 100644 --- a/.vsts-ci/linux-daily.yml +++ b/.vsts-ci/linux-daily.yml @@ -122,6 +122,9 @@ stages: condition: succeededOrFailed() - pwsh: | + Import-Module .\build.psm1 + Set-CorrectLocale + Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' Invoke-CITest -Purpose UnelevatedPesterTests -TagSet Others @@ -129,6 +132,9 @@ stages: condition: succeededOrFailed() - pwsh: | + Import-Module .\build.psm1 + Set-CorrectLocale + Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' Invoke-CITest -Purpose ElevatedPesterTests -TagSet Others diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index 89c1a39dd44..d9ed3e35712 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -64,16 +64,7 @@ jobs: - pwsh: | Import-Module .\build.psm1 -Force - $environment = Get-EnvironmentInformation - $isUbuntu20 = $environment.IsUbuntu -and $environment.IsUbuntu20 - if ($isUbuntu20) { - $env:LC_ALL='en_US.UTF-8' - $env:LANG='en_US.UTF-8' - sudo locale-gen $LANG - sudo update-locale - } - - locale + Set-CorrectLocale Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' diff --git a/build.psm1 b/build.psm1 index 7441f48eb25..bbb98110d2a 100644 --- a/build.psm1 +++ b/build.psm1 @@ -3386,3 +3386,23 @@ function New-NugetConfigFile Set-Content -Path (Join-Path $Destination 'nuget.config') -Value $content -Force } + +function Set-CorrectLocale +{ + if (-not $IsLinux) + { + return + } + + $environment = Get-EnvironmentInformation + if ($environment.IsUbuntu -and $environment.IsUbuntu20) + { + $env:LC_ALL = 'en_US.UTF-8' + $env:LANG = 'en_US.UTF-8' + sudo locale-gen $env:LANG + sudo update-locale + } + + # Output the locale to log it + locale +} From 0d7ba2fe5a11d642b7f8b32e25a2539642fafe86 Mon Sep 17 00:00:00 2001 From: Paul Higinbotham Date: Thu, 9 Sep 2021 10:36:05 -0700 Subject: [PATCH 036/645] Fix `ConvertTo-SecureString` with key regression due to .NET breaking change (#16068) --- .../security/SecureStringHelper.cs | 151 ++++++++---------- .../SecureString.Tests.ps1 | 15 +- 2 files changed, 79 insertions(+), 87 deletions(-) diff --git a/src/System.Management.Automation/security/SecureStringHelper.cs b/src/System.Management.Automation/security/SecureStringHelper.cs index 6d38859162f..ddb94e52ee8 100644 --- a/src/System.Management.Automation/security/SecureStringHelper.cs +++ b/src/System.Management.Automation/security/SecureStringHelper.cs @@ -217,8 +217,6 @@ internal static SecureString Unprotect(string input) /// A string (see summary). internal static EncryptionResult Encrypt(SecureString input, SecureString key) { - EncryptionResult output = null; - // // get clear text key from the SecureString key // @@ -227,14 +225,14 @@ internal static EncryptionResult Encrypt(SecureString input, SecureString key) // // encrypt the data // - output = Encrypt(input, keyBlob); - - // - // clear the clear text key - // - Array.Clear(keyBlob, 0, keyBlob.Length); - - return output; + try + { + return Encrypt(input, keyBlob); + } + finally + { + Array.Clear(keyBlob); + } } /// @@ -254,48 +252,46 @@ internal static EncryptionResult Encrypt(SecureString input, byte[] key, byte[] Utils.CheckSecureStringArg(input, "input"); Utils.CheckKeyArg(key, "key"); - byte[] encryptedData = null; - MemoryStream ms = null; - ICryptoTransform encryptor = null; - CryptoStream cs = null; - // // prepare the crypto stuff. Initialization Vector is // randomized by default. // - Aes aes = Aes.Create(); - if (iv == null) - iv = aes.IV; - - encryptor = aes.CreateEncryptor(key, iv); - ms = new MemoryStream(); - - using (cs = new CryptoStream(ms, encryptor, CryptoStreamMode.Write)) + using (Aes aes = Aes.Create()) { + if (iv is null) + { + iv = aes.IV; + } + // // get clear text data from the input SecureString // byte[] data = GetData(input); + try + { + using (ICryptoTransform encryptor = aes.CreateEncryptor(key, iv)) + using (var sourceStream = new MemoryStream(data)) + using (var encryptedStream = new MemoryStream()) + { + // + // encrypt it + // + using (var cryptoStream = new CryptoStream(encryptedStream, encryptor, CryptoStreamMode.Write)) + { + sourceStream.CopyTo(cryptoStream); + } - // - // encrypt it - // - cs.Write(data, 0, data.Length); - cs.FlushFinalBlock(); - - // - // clear the clear text data array - // - Array.Clear(data, 0, data.Length); - - // - // convert the encrypted blob to a string - // - encryptedData = ms.ToArray(); - - EncryptionResult output = new EncryptionResult(ByteArrayToString(encryptedData), Convert.ToBase64String(iv)); - - return output; + // + // return encrypted data + // + byte[] encryptedData = encryptedStream.ToArray(); + return new EncryptionResult(ByteArrayToString(encryptedData), Convert.ToBase64String(iv)); + } + } + finally + { + Array.Clear(data, 0, data.Length); + } } } @@ -311,8 +307,6 @@ internal static EncryptionResult Encrypt(SecureString input, byte[] key, byte[] /// SecureString . internal static SecureString Decrypt(string input, SecureString key, byte[] IV) { - SecureString output = null; - // // get clear text key from the SecureString key // @@ -321,14 +315,14 @@ internal static SecureString Decrypt(string input, SecureString key, byte[] IV) // // decrypt the data // - output = Decrypt(input, keyBlob, IV); - - // - // clear the clear text key - // - Array.Clear(keyBlob, 0, keyBlob.Length); - - return output; + try + { + return Decrypt(input, keyBlob, IV); + } + finally + { + Array.Clear(keyBlob); + } } /// @@ -346,44 +340,33 @@ internal static SecureString Decrypt(string input, byte[] key, byte[] IV) Utils.CheckArgForNullOrEmpty(input, "input"); Utils.CheckKeyArg(key, "key"); - byte[] decryptedData = null; - byte[] encryptedData = null; - SecureString s = null; - // // prepare the crypto stuff // - Aes aes = Aes.Create(); - encryptedData = ByteArrayFromString(input); - - var decryptor = aes.CreateDecryptor(key, IV ?? aes.IV); - - MemoryStream ms = new MemoryStream(encryptedData); - - using (CryptoStream cs = new CryptoStream(ms, decryptor, CryptoStreamMode.Read)) + using (var aes = Aes.Create()) { - byte[] tempDecryptedData = new byte[encryptedData.Length]; - - int numBytesRead = 0; - - // - // decrypt the data - // - numBytesRead = cs.Read(tempDecryptedData, 0, - tempDecryptedData.Length); - - decryptedData = new byte[numBytesRead]; - - for (int i = 0; i < numBytesRead; i++) + using (ICryptoTransform decryptor = aes.CreateDecryptor(key, IV ?? aes.IV)) + using (var encryptedStream = new MemoryStream(ByteArrayFromString(input))) + using (var targetStream = new MemoryStream()) { - decryptedData[i] = tempDecryptedData[i]; - } - - s = New(decryptedData); - Array.Clear(decryptedData, 0, decryptedData.Length); - Array.Clear(tempDecryptedData, 0, tempDecryptedData.Length); + // + // decrypt the data and return as SecureString + // + using (var sourceStream = new CryptoStream(encryptedStream, decryptor, CryptoStreamMode.Read)) + { + sourceStream.CopyTo(targetStream); + } - return s; + byte[] decryptedData = targetStream.ToArray(); + try + { + return New(decryptedData); + } + finally + { + Array.Clear(decryptedData); + } + } } } diff --git a/test/powershell/Modules/Microsoft.PowerShell.Security/SecureString.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Security/SecureString.Tests.ps1 index 6b8f45d509f..df1fa9294e1 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Security/SecureString.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Security/SecureString.Tests.ps1 @@ -11,12 +11,12 @@ Describe "SecureString conversion tests" -Tags "CI" { $string.ToCharArray() | ForEach-Object { $securestring.AppendChar($_) } } - It "using null arguments to ConvertFrom-SecureString produces an exception" { + It "Using null arguments to ConvertFrom-SecureString produces an exception" { { ConvertFrom-SecureString -SecureString $null -Key $null } | Should -Throw -ErrorId "ParameterArgumentValidationErrorNullNotAllowed,Microsoft.PowerShell.Commands.ConvertFromSecureStringCommand" } - It "using a bad key produces an exception" { + It "Using a bad key produces an exception" { $badkey = [byte[]]@(1,2) { ConvertFrom-SecureString -SecureString $secureString -Key $badkey } | Should -Throw -ErrorId "Argument,Microsoft.PowerShell.Commands.ConvertFromSecureStringCommand" @@ -27,9 +27,18 @@ Describe "SecureString conversion tests" -Tags "CI" { $ss | Should -BeOfType SecureString } - It "can convert back from a secure string" { + It "Can convert back from a secure string" { $ss1 = ConvertTo-SecureString -AsPlainText -Force $string $ss2 = ConvertFrom-SecureString $ss1 | ConvertTo-SecureString $ss2 | ConvertFrom-SecureString -AsPlainText | Should -Be $string } + + It "Can encode secure string with key" { + $testString = '[8Chars][8Chars][Not8]' + $key = [System.Text.Encoding]::UTF8.GetBytes("1234"*8) + $ss1 = $testString | ConvertTo-SecureString -AsPlainText -Force + $encodedStr = $ss1 | ConvertFrom-SecureString -Key $key + $ss2 = $encodedStr | ConvertTo-SecureString -Key $key + $ss2 | ConvertFrom-SecureString -AsPlainText | Should -BeExactly $testString + } } From 024f4097630e028440b35e08933b0fefc31da399 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Thu, 9 Sep 2021 11:50:36 -0700 Subject: [PATCH 037/645] Add `isOutputRedirected` parameter to `GetFormatStyleString()` method (#14397) --- .../host/msh/ConsoleHostUserInterface.cs | 6 +++--- src/System.Management.Automation/engine/Utils.cs | 5 ++--- .../engine/Formatting/OutputRendering.Tests.ps1 | 10 ---------- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterface.cs b/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterface.cs index 756e294b226..87a65dfdcbd 100644 --- a/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterface.cs +++ b/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterface.cs @@ -1215,7 +1215,7 @@ public override void WriteDebugLine(string message) { if (SupportsVirtualTerminal) { - WriteLine(Utils.GetFormatStyleString(Utils.FormatStyle.Debug) + StringUtil.Format(ConsoleHostUserInterfaceStrings.DebugFormatString, message) + PSStyle.Instance.Reset); + WriteLine(Utils.GetFormatStyleString(Utils.FormatStyle.Debug, Console.IsOutputRedirected) + StringUtil.Format(ConsoleHostUserInterfaceStrings.DebugFormatString, message) + PSStyle.Instance.Reset); } else { @@ -1276,7 +1276,7 @@ public override void WriteVerboseLine(string message) { if (SupportsVirtualTerminal) { - WriteLine(Utils.GetFormatStyleString(Utils.FormatStyle.Verbose) + StringUtil.Format(ConsoleHostUserInterfaceStrings.VerboseFormatString, message) + PSStyle.Instance.Reset); + WriteLine(Utils.GetFormatStyleString(Utils.FormatStyle.Verbose, Console.IsOutputRedirected) + StringUtil.Format(ConsoleHostUserInterfaceStrings.VerboseFormatString, message) + PSStyle.Instance.Reset); } else { @@ -1320,7 +1320,7 @@ public override void WriteWarningLine(string message) { if (SupportsVirtualTerminal) { - WriteLine(Utils.GetFormatStyleString(Utils.FormatStyle.Warning) + StringUtil.Format(ConsoleHostUserInterfaceStrings.WarningFormatString, message) + PSStyle.Instance.Reset); + WriteLine(Utils.GetFormatStyleString(Utils.FormatStyle.Warning, Console.IsOutputRedirected) + StringUtil.Format(ConsoleHostUserInterfaceStrings.WarningFormatString, message) + PSStyle.Instance.Reset); } else { diff --git a/src/System.Management.Automation/engine/Utils.cs b/src/System.Management.Automation/engine/Utils.cs index eb2470e6d17..91f6083596d 100644 --- a/src/System.Management.Automation/engine/Utils.cs +++ b/src/System.Management.Automation/engine/Utils.cs @@ -1617,13 +1617,12 @@ internal enum FormatStyle Debug, } - internal static string GetFormatStyleString(FormatStyle formatStyle) + internal static string GetFormatStyleString(FormatStyle formatStyle, bool isOutputRedirected) { // redirected console gets plaintext output to preserve existing behavior if (!InternalTestHooks.BypassOutputRedirectionCheck && ((PSStyle.Instance.OutputRendering == OutputRendering.PlainText) || - (formatStyle == FormatStyle.Error && Console.IsErrorRedirected) || - (formatStyle != FormatStyle.Error && Console.IsOutputRedirected))) + isOutputRedirected)) { return string.Empty; } diff --git a/test/powershell/engine/Formatting/OutputRendering.Tests.ps1 b/test/powershell/engine/Formatting/OutputRendering.Tests.ps1 index 106197b10f5..14085cc23ba 100644 --- a/test/powershell/engine/Formatting/OutputRendering.Tests.ps1 +++ b/test/powershell/engine/Formatting/OutputRendering.Tests.ps1 @@ -2,14 +2,6 @@ # Licensed under the MIT License. Describe 'OutputRendering tests' { - BeforeAll { - $th = New-TestHost - $rs = [runspacefactory]::Createrunspace($th) - $rs.open() - $ps = [powershell]::Create() - $ps.Runspace = $rs - } - BeforeEach { if ($null -ne $PSStyle) { $oldOutputRendering = $PSStyle.OutputRendering @@ -20,8 +12,6 @@ Describe 'OutputRendering tests' { if ($null -ne $PSStyle) { $PSStyle.OutputRendering = $oldOutputRendering } - - $ps.Commands.Clear() } It 'OutputRendering works for "" to the host' -TestCases @( From 74e283c4f9777f73e45e08f377b0b28415f65535 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Thu, 9 Sep 2021 13:59:31 -0700 Subject: [PATCH 038/645] Set locale correctly on Linux CI (#16073) --- .vsts-ci/linux-daily.yml | 6 ------ .vsts-ci/templates/nix-test.yml | 3 --- tools/ci.psm1 | 3 +++ 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.vsts-ci/linux-daily.yml b/.vsts-ci/linux-daily.yml index 14cdcc1372c..e6a5375b0c9 100644 --- a/.vsts-ci/linux-daily.yml +++ b/.vsts-ci/linux-daily.yml @@ -122,9 +122,6 @@ stages: condition: succeededOrFailed() - pwsh: | - Import-Module .\build.psm1 - Set-CorrectLocale - Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' Invoke-CITest -Purpose UnelevatedPesterTests -TagSet Others @@ -132,9 +129,6 @@ stages: condition: succeededOrFailed() - pwsh: | - Import-Module .\build.psm1 - Set-CorrectLocale - Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' Invoke-CITest -Purpose ElevatedPesterTests -TagSet Others diff --git a/.vsts-ci/templates/nix-test.yml b/.vsts-ci/templates/nix-test.yml index d9ed3e35712..6a1b6e6f9de 100644 --- a/.vsts-ci/templates/nix-test.yml +++ b/.vsts-ci/templates/nix-test.yml @@ -63,9 +63,6 @@ jobs: continueOnError: true - pwsh: | - Import-Module .\build.psm1 -Force - Set-CorrectLocale - Import-Module .\tools\ci.psm1 Restore-PSOptions -PSOptionsPath '$(System.ArtifactsDirectory)\build\psoptions.json' $options = (Get-PSOptions) diff --git a/tools/ci.psm1 b/tools/ci.psm1 index c7b2735abb0..f53c9271c01 100644 --- a/tools/ci.psm1 +++ b/tools/ci.psm1 @@ -215,6 +215,9 @@ function Invoke-CITest [string] $TagSet ) + # Set locale correctly for Linux CIs + Set-CorrectLocale + # Pester doesn't allow Invoke-Pester -TagAll@('CI', 'RequireAdminOnWindows') currently # https://github.com/pester/Pester/issues/608 # To work-around it, we exlude all categories, but 'CI' from the list From dad7a34ee743c81dd96e742ed5516ac782388972 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Fri, 10 Sep 2021 09:05:07 -0700 Subject: [PATCH 039/645] Move `GetOuputString()` and `GetFormatStyleString()` to `PSHostUserInterface` as public API (#16075) --- .../host/msh/ConsoleHostUserInterface.cs | 8 +- .../FormatAndOutput/common/ILineOutput.cs | 3 +- .../engine/Utils.cs | 93 ----------- .../engine/hostifaces/MshHostUserInterface.cs | 155 ++++++++++++++++++ 4 files changed, 161 insertions(+), 98 deletions(-) diff --git a/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterface.cs b/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterface.cs index 87a65dfdcbd..ad20eadf3d8 100644 --- a/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterface.cs +++ b/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHostUserInterface.cs @@ -729,7 +729,7 @@ private void WriteImpl(string value, bool newLine) } TextWriter writer = Console.IsOutputRedirected ? Console.Out : _parent.ConsoleTextWriter; - value = Utils.GetOutputString(value, isHost: true, SupportsVirtualTerminal, Console.IsOutputRedirected); + value = GetOutputString(value, SupportsVirtualTerminal, Console.IsOutputRedirected); if (_parent.IsRunningAsync) { @@ -1215,7 +1215,7 @@ public override void WriteDebugLine(string message) { if (SupportsVirtualTerminal) { - WriteLine(Utils.GetFormatStyleString(Utils.FormatStyle.Debug, Console.IsOutputRedirected) + StringUtil.Format(ConsoleHostUserInterfaceStrings.DebugFormatString, message) + PSStyle.Instance.Reset); + WriteLine(GetFormatStyleString(FormatStyle.Debug, Console.IsOutputRedirected) + StringUtil.Format(ConsoleHostUserInterfaceStrings.DebugFormatString, message) + PSStyle.Instance.Reset); } else { @@ -1276,7 +1276,7 @@ public override void WriteVerboseLine(string message) { if (SupportsVirtualTerminal) { - WriteLine(Utils.GetFormatStyleString(Utils.FormatStyle.Verbose, Console.IsOutputRedirected) + StringUtil.Format(ConsoleHostUserInterfaceStrings.VerboseFormatString, message) + PSStyle.Instance.Reset); + WriteLine(GetFormatStyleString(FormatStyle.Verbose, Console.IsOutputRedirected) + StringUtil.Format(ConsoleHostUserInterfaceStrings.VerboseFormatString, message) + PSStyle.Instance.Reset); } else { @@ -1320,7 +1320,7 @@ public override void WriteWarningLine(string message) { if (SupportsVirtualTerminal) { - WriteLine(Utils.GetFormatStyleString(Utils.FormatStyle.Warning, Console.IsOutputRedirected) + StringUtil.Format(ConsoleHostUserInterfaceStrings.WarningFormatString, message) + PSStyle.Instance.Reset); + WriteLine(GetFormatStyleString(FormatStyle.Warning, Console.IsOutputRedirected) + StringUtil.Format(ConsoleHostUserInterfaceStrings.WarningFormatString, message) + PSStyle.Instance.Reset); } else { diff --git a/src/System.Management.Automation/FormatAndOutput/common/ILineOutput.cs b/src/System.Management.Automation/FormatAndOutput/common/ILineOutput.cs index 32d4565da5d..8c672b62063 100644 --- a/src/System.Management.Automation/FormatAndOutput/common/ILineOutput.cs +++ b/src/System.Management.Automation/FormatAndOutput/common/ILineOutput.cs @@ -4,6 +4,7 @@ using System.Globalization; using System.IO; using System.Management.Automation; +using System.Management.Automation.Host; using System.Management.Automation.Internal; using System.Text; @@ -413,7 +414,7 @@ internal override void WriteLine(string s) { CheckStopProcessing(); - s = Utils.GetOutputString(s, isHost: false); + s = PSHostUserInterface.GetOutputString(s, isHost: false); if (_suppressNewline) { diff --git a/src/System.Management.Automation/engine/Utils.cs b/src/System.Management.Automation/engine/Utils.cs index 91f6083596d..f277dcb1e49 100644 --- a/src/System.Management.Automation/engine/Utils.cs +++ b/src/System.Management.Automation/engine/Utils.cs @@ -1559,99 +1559,6 @@ internal static bool IsComObject(object obj) return oldMode; } - - #region PSAnsiRendering - - internal static bool ShouldOutputPlainText(bool isHost, bool? supportsVirtualTerminal) - { - var outputRendering = OutputRendering.Ansi; - - if (supportsVirtualTerminal != false) - { - switch (PSStyle.Instance.OutputRendering) - { - case OutputRendering.Host: - outputRendering = isHost ? OutputRendering.Ansi : OutputRendering.PlainText; - break; - default: - outputRendering = PSStyle.Instance.OutputRendering; - break; - } - } - - return outputRendering == OutputRendering.PlainText; - } - - internal static string GetOutputString(string s, bool isHost, bool? supportsVirtualTerminal = null, bool isOutputRedirected = false) - { - var sd = new ValueStringDecorated(s); - - if (sd.IsDecorated) - { - var outputRendering = OutputRendering.Ansi; - if (InternalTestHooks.BypassOutputRedirectionCheck) - { - isOutputRedirected = false; - } - - if (isOutputRedirected || ShouldOutputPlainText(isHost, supportsVirtualTerminal)) - { - outputRendering = OutputRendering.PlainText; - } - - s = sd.ToString(outputRendering); - } - - return s; - } - - internal enum FormatStyle - { - Reset, - FormatAccent, - TableHeader, - ErrorAccent, - Error, - Warning, - Verbose, - Debug, - } - - internal static string GetFormatStyleString(FormatStyle formatStyle, bool isOutputRedirected) - { - // redirected console gets plaintext output to preserve existing behavior - if (!InternalTestHooks.BypassOutputRedirectionCheck && - ((PSStyle.Instance.OutputRendering == OutputRendering.PlainText) || - isOutputRedirected)) - { - return string.Empty; - } - - PSStyle psstyle = PSStyle.Instance; - switch (formatStyle) - { - case FormatStyle.Reset: - return psstyle.Reset; - case FormatStyle.FormatAccent: - return psstyle.Formatting.FormatAccent; - case FormatStyle.TableHeader: - return psstyle.Formatting.TableHeader; - case FormatStyle.ErrorAccent: - return psstyle.Formatting.ErrorAccent; - case FormatStyle.Error: - return psstyle.Formatting.Error; - case FormatStyle.Warning: - return psstyle.Formatting.Warning; - case FormatStyle.Verbose: - return psstyle.Formatting.Verbose; - case FormatStyle.Debug: - return psstyle.Formatting.Debug; - default: - return string.Empty; - } - } - - #endregion } } diff --git a/src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs b/src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs index 2d8c16f2b7b..cf2dea14a5f 100644 --- a/src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs +++ b/src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs @@ -232,6 +232,161 @@ public virtual void WriteLine(ConsoleColor foregroundColor, ConsoleColor backgro /// public virtual void WriteInformation(InformationRecord record) { } + private static bool ShouldOutputPlainText(bool isHost, bool? supportsVirtualTerminal) + { + var outputRendering = OutputRendering.Ansi; + + if (supportsVirtualTerminal != false) + { + switch (PSStyle.Instance.OutputRendering) + { + case OutputRendering.Host: + outputRendering = isHost ? OutputRendering.Ansi : OutputRendering.PlainText; + break; + default: + outputRendering = PSStyle.Instance.OutputRendering; + break; + } + } + + return outputRendering == OutputRendering.PlainText; + } + + /// + /// The format styles that are supported by the host. + /// + public enum FormatStyle + { + /// + /// Reset the formatting to the default. + /// + Reset, + + /// + /// Highlight text used in output formatting. + /// + FormatAccent, + + /// + /// Highlight for table headers. + /// + TableHeader, + + /// + /// Highlight for detailed error view. + /// + ErrorAccent, + + /// + /// Style for error messages. + /// + Error, + + /// + /// Style for warning messages. + /// + Warning, + + /// + /// Style for verbose messages. + /// + Verbose, + + /// + /// Style for debug messages. + /// + Debug, + } + + /// + /// Get the ANSI escape sequence for the given format style. + /// + /// + /// The format style to get the escape sequence for. + /// + /// + /// True if the output is redirected. + /// + /// + /// The ANSI escape sequence for the given format style. + /// + public static string GetFormatStyleString(FormatStyle formatStyle, bool isOutputRedirected) + { + // redirected console gets plaintext output to preserve existing behavior + if (!InternalTestHooks.BypassOutputRedirectionCheck && + (PSStyle.Instance.OutputRendering == OutputRendering.PlainText || + isOutputRedirected)) + { + return string.Empty; + } + + PSStyle psstyle = PSStyle.Instance; + switch (formatStyle) + { + case FormatStyle.Reset: + return psstyle.Reset; + case FormatStyle.FormatAccent: + return psstyle.Formatting.FormatAccent; + case FormatStyle.TableHeader: + return psstyle.Formatting.TableHeader; + case FormatStyle.ErrorAccent: + return psstyle.Formatting.ErrorAccent; + case FormatStyle.Error: + return psstyle.Formatting.Error; + case FormatStyle.Warning: + return psstyle.Formatting.Warning; + case FormatStyle.Verbose: + return psstyle.Formatting.Verbose; + case FormatStyle.Debug: + return psstyle.Formatting.Debug; + default: + return string.Empty; + } + } + + /// + /// Get the appropriate output string based on different criteria. + /// + /// + /// The text to format. + /// + /// + /// True if the host supports virtual terminal. + /// + /// + /// True if the output is redirected. + /// + /// + /// The formatted text. + /// + public static string GetOutputString(string text, bool supportsVirtualTerminal, bool isOutputRedirected) + { + return GetOutputString(text, isHost: true, supportsVirtualTerminal: supportsVirtualTerminal, isOutputRedirected: isOutputRedirected); + } + + internal static string GetOutputString(string text, bool isHost, bool? supportsVirtualTerminal = null, bool isOutputRedirected = false) + { + var sd = new ValueStringDecorated(text); + + if (sd.IsDecorated) + { + var outputRendering = OutputRendering.Ansi; + if (InternalTestHooks.BypassOutputRedirectionCheck) + { + isOutputRedirected = false; + } + + if (isOutputRedirected || ShouldOutputPlainText(isHost, supportsVirtualTerminal)) + { + outputRendering = OutputRendering.PlainText; + } + + text = sd.ToString(outputRendering); + } + + return text; + } + // Gets the state associated with PowerShell transcription. // // Ideally, this would be associated with the host instance, but remoting recycles host instances From 8b962ca1dbae2cdb49586d3c8dc5f51820d6724c Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Mon, 13 Sep 2021 15:56:12 -0700 Subject: [PATCH 040/645] Update minimum required OS version for macOS (#16088) --- tools/packaging/packaging.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 000d48067e1..7b652f9b3bf 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -1115,7 +1115,7 @@ function New-MacOsDistributionPackage # 2 - package path # 3 - minimum os version # 4 - Package Identifier - $PackagingStrings.OsxDistributionTemplate -f "PowerShell - $packageVersion", $packageVersion, $packageName, '10.13', $packageId | Out-File -Encoding ascii -FilePath $distributionXmlPath -Force + $PackagingStrings.OsxDistributionTemplate -f "PowerShell - $packageVersion", $packageVersion, $packageName, '10.14', $packageId | Out-File -Encoding ascii -FilePath $distributionXmlPath -Force Write-Log "Applying distribution.xml to package..." Push-Location $tempDir From 55ccbb6803162965eead3a08aace37f11ea7630b Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Mon, 13 Sep 2021 16:45:10 -0700 Subject: [PATCH 041/645] Update .NET to `6.0.100-rc.1.21458.32` (#16066) --- DotnetRuntimeMetadata.json | 2 +- global.json | 2 +- ...crosoft.PowerShell.Commands.Management.csproj | 2 +- .../Microsoft.PowerShell.Commands.Utility.csproj | 4 ++-- .../Microsoft.PowerShell.CoreCLR.Eventing.csproj | 2 +- .../Microsoft.PowerShell.SDK.csproj | 8 ++++---- .../Microsoft.WSMan.Management.csproj | 2 +- .../System.Management.Automation.csproj | 16 ++++++++-------- test/tools/TestService/TestService.csproj | 2 +- test/tools/WebListener/WebListener.csproj | 4 ++-- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/DotnetRuntimeMetadata.json b/DotnetRuntimeMetadata.json index ed45c403569..7a24b57b192 100644 --- a/DotnetRuntimeMetadata.json +++ b/DotnetRuntimeMetadata.json @@ -8,6 +8,6 @@ "nextChannel": "6.0.1xx-rc1" }, "internalfeed" : { - "url": "https://pkgs.dev.azure.com/dnceng/public/_packaging/6.0.100-rc.1.21430.44-shipping/nuget/v2" + "url": "https://pkgs.dev.azure.com/dnceng/public/_packaging/6.0.100-rc.1.21458.32-shipping/nuget/v2" } } diff --git a/global.json b/global.json index 0c2a82ceaf6..5f03d7d7c48 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "6.0.100-rc.1.21455.2" + "version": "6.0.100-rc.1.21458.32" } } diff --git a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj index 67394beef83..05b7b4e7472 100644 --- a/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj +++ b/src/Microsoft.PowerShell.Commands.Management/Microsoft.PowerShell.Commands.Management.csproj @@ -47,7 +47,7 @@ - + diff --git a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj index 1aff829f1d4..df92bc5a746 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj +++ b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj @@ -32,8 +32,8 @@ - - + + diff --git a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj index 1d5a865f535..e30019fac81 100644 --- a/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj +++ b/src/Microsoft.PowerShell.CoreCLR.Eventing/Microsoft.PowerShell.CoreCLR.Eventing.csproj @@ -8,7 +8,7 @@ - + diff --git a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj index e03dbe4701a..4b8485ac58b 100644 --- a/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj +++ b/src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj @@ -18,9 +18,9 @@ - - - + + + @@ -30,7 +30,7 @@ - + diff --git a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj index c3fcff08ffc..56c0310e2a0 100644 --- a/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj +++ b/src/Microsoft.WSMan.Management/Microsoft.WSMan.Management.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/System.Management.Automation/System.Management.Automation.csproj b/src/System.Management.Automation/System.Management.Automation.csproj index 0f167c7ef35..2ebdabb9593 100644 --- a/src/System.Management.Automation/System.Management.Automation.csproj +++ b/src/System.Management.Automation/System.Management.Automation.csproj @@ -16,16 +16,16 @@ - - - + + + - - + + - - - + + + diff --git a/test/tools/TestService/TestService.csproj b/test/tools/TestService/TestService.csproj index a44ac7810c6..d0d4649878f 100644 --- a/test/tools/TestService/TestService.csproj +++ b/test/tools/TestService/TestService.csproj @@ -13,7 +13,7 @@ - + diff --git a/test/tools/WebListener/WebListener.csproj b/test/tools/WebListener/WebListener.csproj index 4a0c0c38cba..f59d5bc8d12 100644 --- a/test/tools/WebListener/WebListener.csproj +++ b/test/tools/WebListener/WebListener.csproj @@ -7,8 +7,8 @@ - - + + From 1704ca68857688ced6d7fe2a3b2e8a14ec184d7d Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Tue, 14 Sep 2021 13:51:18 -0700 Subject: [PATCH 042/645] Add benchmark to test compiler performance (#16083) --- test/perf/benchmarks/Engine.Compiler.cs | 78 + test/perf/benchmarks/Program.cs | 2 +- test/perf/benchmarks/assets/compiler.test.ps1 | 2688 +++++++++++++++++ test/perf/benchmarks/powershell-perf.csproj | 3 + .../BenchmarkDotNet.Extensions.csproj | 4 +- .../dotnet-tools/ResultsComparer/Program.cs | 2 +- tools/UpdateDotnetRuntime.ps1 | 1 + 7 files changed, 2774 insertions(+), 4 deletions(-) create mode 100644 test/perf/benchmarks/Engine.Compiler.cs create mode 100644 test/perf/benchmarks/assets/compiler.test.ps1 diff --git a/test/perf/benchmarks/Engine.Compiler.cs b/test/perf/benchmarks/Engine.Compiler.cs new file mode 100644 index 00000000000..10e0054a01c --- /dev/null +++ b/test/perf/benchmarks/Engine.Compiler.cs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#if NET6_0 + +using System; +using System.Collections.Generic; +using System.IO; +using System.Management.Automation; +using System.Management.Automation.Language; + +using BenchmarkDotNet.Attributes; +using MicroBenchmarks; + +namespace Engine +{ + [BenchmarkCategory(Categories.Engine, Categories.Internal)] + public class Compiler + { + private static readonly Dictionary s_scriptBlocksDict; + private static readonly List s_functionNames; + private ScriptBlockAst _currentAst; + + static Compiler() + { + string pattern = string.Format("{0}test{0}perf{0}benchmarks", Path.DirectorySeparatorChar); + string location = typeof(Compiler).Assembly.Location; + string testFilePath = null; + + int start = location.IndexOf(pattern, StringComparison.Ordinal); + if (start > 0) + { + testFilePath = Path.Join(location.AsSpan(0, start + pattern.Length), "assets", "compiler.test.ps1"); + } + + var topScriptBlockAst = Parser.ParseFile(testFilePath, tokens: out _, errors: out _); + var allFunctions = topScriptBlockAst.FindAll(ast => ast is FunctionDefinitionAst, searchNestedScriptBlocks: false); + + s_scriptBlocksDict = new Dictionary(capacity: 16); + s_functionNames = new List(capacity: 16); + + foreach (FunctionDefinitionAst function in allFunctions) + { + s_functionNames.Add(function.Name); + s_scriptBlocksDict.Add(function.Name, function.Body); + } + } + + [ParamsSource(nameof(FunctionName))] + public string FunctionsToCompile { get; set; } + + public IEnumerable FunctionName() => s_functionNames; + + [GlobalSetup(Target = nameof(CompileFunction))] + public void GlobalSetup() + { + _currentAst = s_scriptBlocksDict[FunctionsToCompile]; + + // Run it once to get the C# code jitted. + // The first call to this takes relatively too long, which makes the BDN's heuristic incorrectly + // believe that there is no need to run many ops in each interation. However, the subsequent runs + // of this method is much faster than the first run, and this causes 'MinIterationTime' warnings + // to our benchmarks and make the benchmark results not reliable. + // Calling this method once in 'GlobalSetup' is a workaround. + // See https://github.com/dotnet/BenchmarkDotNet/issues/837#issuecomment-828600157 + CompileFunction(); + } + + [Benchmark] + public bool CompileFunction() + { + var compiledData = new CompiledScriptBlockData(_currentAst, isFilter: false); + return compiledData.Compile(true); + } + } +} + +#endif diff --git a/test/perf/benchmarks/Program.cs b/test/perf/benchmarks/Program.cs index 2b3aafdb127..53f9a3ce95b 100644 --- a/test/perf/benchmarks/Program.cs +++ b/test/perf/benchmarks/Program.cs @@ -10,7 +10,7 @@ namespace MicroBenchmarks { - public class Program + public sealed class Program { public static int Main(string[] args) { diff --git a/test/perf/benchmarks/assets/compiler.test.ps1 b/test/perf/benchmarks/assets/compiler.test.ps1 new file mode 100644 index 00000000000..c7ca697f2cc --- /dev/null +++ b/test/perf/benchmarks/assets/compiler.test.ps1 @@ -0,0 +1,2688 @@ +## Copyright (c) Microsoft Corporation. +## Licensed under the MIT License. + +function Get-EnvInformation +{ + $environment = @{'IsWindows' = [System.Environment]::OSVersion.Platform -eq [System.PlatformID]::Win32NT} + # PowerShell will likely not be built on pre-1709 nanoserver + if ('System.Management.Automation.Platform' -as [type]) { + $environment += @{'IsCoreCLR' = [System.Management.Automation.Platform]::IsCoreCLR} + $environment += @{'IsLinux' = [System.Management.Automation.Platform]::IsLinux} + $environment += @{'IsMacOS' = [System.Management.Automation.Platform]::IsMacOS} + } else { + $environment += @{'IsCoreCLR' = $false} + $environment += @{'IsLinux' = $false} + $environment += @{'IsMacOS' = $false} + } + + if ($environment.IsWindows) + { + $environment += @{'IsAdmin' = (New-Object Security.Principal.WindowsPrincipal ([Security.Principal.WindowsIdentity]::GetCurrent())).IsInRole([Security.Principal.WindowsBuiltinRole]::Administrator)} + $environment += @{'nugetPackagesRoot' = "${env:USERPROFILE}\.nuget\packages", "${env:NUGET_PACKAGES}"} + } + else + { + $environment += @{'nugetPackagesRoot' = "${env:HOME}/.nuget/packages"} + } + + if ($environment.IsMacOS) { + $environment += @{'UsingHomebrew' = [bool](Get-Command brew -ErrorAction ignore)} + $environment += @{'UsingMacports' = [bool](Get-Command port -ErrorAction ignore)} + + $environment += @{ + 'OSArchitecture' = if ((uname -v) -match 'ARM64') { 'arm64' } else { 'x64' } + } + + if (-not($environment.UsingHomebrew -or $environment.UsingMacports)) { + throw "Neither Homebrew nor MacPorts is installed on this system, visit https://brew.sh/ or https://www.macports.org/ to continue" + } + } + + if ($environment.IsLinux) { + $LinuxInfo = Get-Content /etc/os-release -Raw | ConvertFrom-StringData + $lsb_release = Get-Command lsb_release -Type Application -ErrorAction Ignore | Select-Object -First 1 + if ($lsb_release) { + $LinuxID = & $lsb_release -is + } + else { + $LinuxID = "" + } + + $environment += @{'LinuxInfo' = $LinuxInfo} + $environment += @{'IsDebian' = $LinuxInfo.ID -match 'debian' -or $LinuxInfo.ID -match 'kali'} + $environment += @{'IsDebian9' = $environment.IsDebian -and $LinuxInfo.VERSION_ID -match '9'} + $environment += @{'IsDebian10' = $environment.IsDebian -and $LinuxInfo.VERSION_ID -match '10'} + $environment += @{'IsDebian11' = $environment.IsDebian -and $LinuxInfo.PRETTY_NAME -match 'bullseye'} + $environment += @{'IsUbuntu' = $LinuxInfo.ID -match 'ubuntu' -or $LinuxID -match 'Ubuntu'} + $environment += @{'IsUbuntu16' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '16.04'} + $environment += @{'IsUbuntu18' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '18.04'} + $environment += @{'IsUbuntu20' = $environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '20.04'} + $environment += @{'IsCentOS' = $LinuxInfo.ID -match 'centos' -and $LinuxInfo.VERSION_ID -match '7'} + $environment += @{'IsFedora' = $LinuxInfo.ID -match 'fedora' -and $LinuxInfo.VERSION_ID -ge 24} + $environment += @{'IsOpenSUSE' = $LinuxInfo.ID -match 'opensuse'} + $environment += @{'IsSLES' = $LinuxInfo.ID -match 'sles'} + $environment += @{'IsRedHat' = $LinuxInfo.ID -match 'rhel'} + $environment += @{'IsRedHat7' = $environment.IsRedHat -and $LinuxInfo.VERSION_ID -match '7' } + $environment += @{'IsOpenSUSE13' = $environment.IsOpenSUSE -and $LinuxInfo.VERSION_ID -match '13'} + $environment += @{'IsOpenSUSE42.1' = $environment.IsOpenSUSE -and $LinuxInfo.VERSION_ID -match '42.1'} + $environment += @{'IsDebianFamily' = $environment.IsDebian -or $environment.IsUbuntu} + $environment += @{'IsRedHatFamily' = $environment.IsCentOS -or $environment.IsFedora -or $environment.IsRedHat} + $environment += @{'IsSUSEFamily' = $environment.IsSLES -or $environment.IsOpenSUSE} + $environment += @{'IsAlpine' = $LinuxInfo.ID -match 'alpine'} + + # Workaround for temporary LD_LIBRARY_PATH hack for Fedora 24 + # https://github.com/PowerShell/PowerShell/issues/2511 + if ($environment.IsFedora -and (Test-Path ENV:\LD_LIBRARY_PATH)) { + Remove-Item -Force ENV:\LD_LIBRARY_PATH + Get-ChildItem ENV: + } + + if( -not( + $environment.IsDebian -or + $environment.IsUbuntu -or + $environment.IsRedHatFamily -or + $environment.IsSUSEFamily -or + $environment.IsAlpine) + ) { + if ($SkipLinuxDistroCheck) { + Write-Warning "The current OS : $($LinuxInfo.ID) is not supported for building PowerShell." + } else { + throw "The current OS : $($LinuxInfo.ID) is not supported for building PowerShell. Import this module with '-ArgumentList `$true' to bypass this check." + } + } + } + + return [PSCustomObject] $environment +} + +function Start-PSBuild { + [CmdletBinding(DefaultParameterSetName="Default")] + param( + # When specified this switch will stops running dev powershell + # to help avoid compilation error, because file are in use. + [switch]$StopDevPowerShell, + + [switch]$Restore, + # Accept a path to the output directory + # When specified, --output will be passed to dotnet + [string]$Output, + [switch]$ResGen, + [switch]$TypeGen, + [switch]$Clean, + [Parameter(ParameterSetName="Legacy")] + [switch]$PSModuleRestore, + [Parameter(ParameterSetName="Default")] + [switch]$NoPSModuleRestore, + [switch]$CI, + [switch]$ForMinimalSize, + + # Skips the step where the pwsh that's been built is used to create a configuration + # Useful when changing parsing/compilation, since bugs there can mean we can't get past this step + [switch]$SkipExperimentalFeatureGeneration, + + # this switch will re-build only System.Management.Automation.dll + # it's useful for development, to do a quick changes in the engine + [switch]$SMAOnly, + + # These runtimes must match those in project.json + # We do not use ValidateScript since we want tab completion + # If this parameter is not provided it will get determined automatically. + [ValidateSet("alpine-x64", + "fxdependent", + "fxdependent-win-desktop", + "linux-arm", + "linux-arm64", + "linux-x64", + "osx-arm64", + "osx-x64", + "win-arm", + "win-arm64", + "win7-x64", + "win7-x86")] + [string]$Runtime, + + [ValidateSet('Debug', 'Release', 'CodeCoverage', '')] # We might need "Checked" as well + [string]$Configuration, + + [switch]$CrossGen, + + [ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d{1,2})?)?$")] + [ValidateNotNullOrEmpty()] + [string]$ReleaseTag, + [switch]$Detailed, + [switch]$InteractiveAuth, + [switch]$SkipRoslynAnalyzers + ) + + if ($ReleaseTag -and $ReleaseTag -notmatch "^v\d+\.\d+\.\d+(-(preview|rc)(\.\d{1,2})?)?$") { + Write-Warning "Only preview or rc are supported for releasing pre-release version of PowerShell" + } + + if ($PSCmdlet.ParameterSetName -eq "Default" -and !$NoPSModuleRestore) + { + $PSModuleRestore = $true + } + + if ($Runtime -eq "linux-arm" -and $environment.IsLinux -and -not $environment.IsUbuntu) { + throw "Cross compiling for linux-arm is only supported on Ubuntu environment" + } + + if ("win-arm","win-arm64" -contains $Runtime -and -not $environment.IsWindows) { + throw "Cross compiling for win-arm or win-arm64 is only supported on Windows environment" + } + + if ($ForMinimalSize) { + if ($CrossGen) { + throw "Build for the minimal size requires the minimal disk footprint, so `CrossGen` is not allowed" + } + + if ($Runtime -and "linux-x64", "win7-x64", "osx-x64" -notcontains $Runtime) { + throw "Build for the minimal size is enabled only for following runtimes: 'linux-x64', 'win7-x64', 'osx-x64'" + } + } + + function Stop-DevPowerShell { + Get-Process pwsh* | + Where-Object { + $_.Modules | + Where-Object { + $_.FileName -eq (Resolve-Path $script:Options.Output).Path + } + } | + Stop-Process -Verbose + } + + if ($Clean) { + Write-Log -message "Cleaning your working directory. You can also do it with 'git clean -fdX --exclude .vs/PowerShell/v16/Server/sqlite3'" + Push-Location $PSScriptRoot + try { + # Excluded sqlite3 folder is due to this Roslyn issue: https://github.com/dotnet/roslyn/issues/23060 + # Excluded src/Modules/nuget.config as this is required for release build. + # Excluded nuget.config as this is required for release build. + git clean -fdX --exclude .vs/PowerShell/v16/Server/sqlite3 --exclude src/Modules/nuget.config --exclude nuget.config + } finally { + Pop-Location + } + } + + # Add .NET CLI tools to PATH + Find-Dotnet + + # Verify we have git in place to do the build, and abort if the precheck failed + $precheck = precheck 'git' "Build dependency 'git' not found in PATH. See " + if (-not $precheck) { + return + } + + # Verify we have .NET SDK in place to do the build, and abort if the precheck failed + $precheck = precheck 'dotnet' "Build dependency 'dotnet' not found in PATH. Run Start-PSBootstrap. Also see " + if (-not $precheck) { + return + } + + # Verify if the dotnet in-use is the required version + $dotnetCLIInstalledVersion = Start-NativeExecution -sb { dotnet --version } -IgnoreExitcode + If ($dotnetCLIInstalledVersion -ne $dotnetCLIRequiredVersion) { + Write-Warning @" +The currently installed .NET Command Line Tools is not the required version. + +Installed version: $dotnetCLIInstalledVersion +Required version: $dotnetCLIRequiredVersion + +Fix steps: + +1. Remove the installed version from: + - on windows '`$env:LOCALAPPDATA\Microsoft\dotnet' + - on macOS and linux '`$env:HOME/.dotnet' +2. Run Start-PSBootstrap or Install-Dotnet +3. Start-PSBuild -Clean +`n +"@ + return + } + + # set output options + $OptionsArguments = @{ + CrossGen=$CrossGen + Output=$Output + Runtime=$Runtime + Configuration=$Configuration + Verbose=$true + SMAOnly=[bool]$SMAOnly + PSModuleRestore=$PSModuleRestore + ForMinimalSize=$ForMinimalSize + } + $script:Options = New-PSOptions @OptionsArguments + + if ($StopDevPowerShell) { + Stop-DevPowerShell + } + + # setup arguments + # adding ErrorOnDuplicatePublishOutputFiles=false due to .NET SDk issue: https://github.com/dotnet/sdk/issues/15748 + # removing --no-restore due to .NET SDK issue: https://github.com/dotnet/sdk/issues/18999 + # $Arguments = @("publish","--no-restore","/property:GenerateFullPaths=true", "/property:ErrorOnDuplicatePublishOutputFiles=false") + $Arguments = @("publish","/property:GenerateFullPaths=true", "/property:ErrorOnDuplicatePublishOutputFiles=false") + if ($Output -or $SMAOnly) { + $Arguments += "--output", (Split-Path $Options.Output) + } + + # Add --self-contained due to "warning NETSDK1179: One of '--self-contained' or '--no-self-contained' options are required when '--runtime' is used." + if ($Options.Runtime -like 'fxdependent*') { + $Arguments += "--no-self-contained" + } + else { + $Arguments += "--self-contained" + } + + if ($Options.Runtime -like 'win*' -or ($Options.Runtime -like 'fxdependent*' -and $environment.IsWindows)) { + $Arguments += "/property:IsWindows=true" + } + else { + $Arguments += "/property:IsWindows=false" + } + + # Framework Dependent builds do not support ReadyToRun as it needs a specific runtime to optimize for. + # The property is set in Powershell.Common.props file. + # We override the property through the build command line. + if($Options.Runtime -like 'fxdependent*' -or $ForMinimalSize) { + $Arguments += "/property:PublishReadyToRun=false" + } + + $Arguments += "--configuration", $Options.Configuration + $Arguments += "--framework", $Options.Framework + + if ($Detailed.IsPresent) + { + $Arguments += '--verbosity', 'd' + } + + if (-not $SMAOnly -and $Options.Runtime -notlike 'fxdependent*') { + # libraries should not have runtime + $Arguments += "--runtime", $Options.Runtime + } + + if ($ReleaseTag) { + $ReleaseTagToUse = $ReleaseTag -Replace '^v' + $Arguments += "/property:ReleaseTag=$ReleaseTagToUse" + } + + if ($SkipRoslynAnalyzers) { + $Arguments += "/property:RunAnalyzersDuringBuild=false" + } + + # handle Restore + Restore-PSPackage -Options $Options -Force:$Restore -InteractiveAuth:$InteractiveAuth + + # handle ResGen + # Heuristic to run ResGen on the fresh machine + if ($ResGen -or -not (Test-Path "$PSScriptRoot/src/Microsoft.PowerShell.ConsoleHost/gen")) { + Write-Log -message "Run ResGen (generating C# bindings for resx files)" + Start-ResGen + } + + # Handle TypeGen + # .inc file name must be different for Windows and Linux to allow build on Windows and WSL. + $incFileName = "powershell_$($Options.Runtime).inc" + if ($TypeGen -or -not (Test-Path "$PSScriptRoot/src/TypeCatalogGen/$incFileName")) { + Write-Log -message "Run TypeGen (generating CorePsTypeCatalog.cs)" + Start-TypeGen -IncFileName $incFileName + } + + # Get the folder path where pwsh.exe is located. + if ((Split-Path $Options.Output -Leaf) -like "pwsh*") { + $publishPath = Split-Path $Options.Output -Parent + } + else { + $publishPath = $Options.Output + } + + try { + # Relative paths do not work well if cwd is not changed to project + Push-Location $Options.Top + + if ($Options.Runtime -notlike 'fxdependent*') { + $sdkToUse = 'Microsoft.NET.Sdk' + if ($Options.Runtime -like 'win7-*' -and !$ForMinimalSize) { + ## WPF/WinForm and the PowerShell GraphicalHost assemblies are included + ## when 'Microsoft.NET.Sdk.WindowsDesktop' is used. + $sdkToUse = 'Microsoft.NET.Sdk.WindowsDesktop' + } + + $Arguments += "/property:SDKToUse=$sdkToUse" + + Write-Log -message "Run dotnet $Arguments from $PWD" + Start-NativeExecution { dotnet $Arguments } + Write-Log -message "PowerShell output: $($Options.Output)" + + if ($CrossGen) { + # fxdependent package cannot be CrossGen'ed + Start-CrossGen -PublishPath $publishPath -Runtime $script:Options.Runtime + Write-Log -message "pwsh.exe with ngen binaries is available at: $($Options.Output)" + } + } else { + $globalToolSrcFolder = Resolve-Path (Join-Path $Options.Top "../Microsoft.PowerShell.GlobalTool.Shim") | Select-Object -ExpandProperty Path + + if ($Options.Runtime -eq 'fxdependent') { + $Arguments += "/property:SDKToUse=Microsoft.NET.Sdk" + } elseif ($Options.Runtime -eq 'fxdependent-win-desktop') { + $Arguments += "/property:SDKToUse=Microsoft.NET.Sdk.WindowsDesktop" + } + + Write-Log -message "Run dotnet $Arguments from $PWD" + Start-NativeExecution { dotnet $Arguments } + Write-Log -message "PowerShell output: $($Options.Output)" + + try { + Push-Location $globalToolSrcFolder + $Arguments += "--output", $publishPath + Write-Log -message "Run dotnet $Arguments from $PWD to build global tool entry point" + Start-NativeExecution { dotnet $Arguments } + } + finally { + Pop-Location + } + } + } finally { + Pop-Location + } + + # No extra post-building task will run if '-SMAOnly' is specified, because its purpose is for a quick update of S.M.A.dll after full build. + if ($SMAOnly) { + return + } + + # publish reference assemblies + try { + Push-Location "$PSScriptRoot/src/TypeCatalogGen" + $refAssemblies = Get-Content -Path $incFileName | Where-Object { $_ -like "*microsoft.netcore.app*" } | ForEach-Object { $_.TrimEnd(';') } + $refDestFolder = Join-Path -Path $publishPath -ChildPath "ref" + + if (Test-Path $refDestFolder -PathType Container) { + Remove-Item $refDestFolder -Force -Recurse -ErrorAction Stop + } + New-Item -Path $refDestFolder -ItemType Directory -Force -ErrorAction Stop > $null + Copy-Item -Path $refAssemblies -Destination $refDestFolder -Force -ErrorAction Stop + } finally { + Pop-Location + } + + if ($ReleaseTag) { + $psVersion = $ReleaseTag + } + else { + $psVersion = git --git-dir="$PSScriptRoot/.git" describe + } + + if ($environment.IsLinux) { + if ($environment.IsRedHatFamily -or $environment.IsDebian) { + # Symbolic links added here do NOT affect packaging as we do not build on Debian. + # add two symbolic links to system shared libraries that libmi.so is dependent on to handle + # platform specific changes. This is the only set of platforms needed for this currently + # as Ubuntu has these specific library files in the platform and macOS builds for itself + # against the correct versions. + + if ($environment.IsDebian10 -or $environment.IsDebian11){ + $sslTarget = "/usr/lib/x86_64-linux-gnu/libssl.so.1.1" + $cryptoTarget = "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.1" + } + elseif ($environment.IsDebian9){ + # NOTE: Debian 8 doesn't need these symlinks + $sslTarget = "/usr/lib/x86_64-linux-gnu/libssl.so.1.0.2" + $cryptoTarget = "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.2" + } + else { #IsRedHatFamily + $sslTarget = "/lib64/libssl.so.10" + $cryptoTarget = "/lib64/libcrypto.so.10" + } + + if ( ! (Test-Path "$publishPath/libssl.so.1.0.0")) { + $null = New-Item -Force -ItemType SymbolicLink -Target $sslTarget -Path "$publishPath/libssl.so.1.0.0" -ErrorAction Stop + } + if ( ! (Test-Path "$publishPath/libcrypto.so.1.0.0")) { + $null = New-Item -Force -ItemType SymbolicLink -Target $cryptoTarget -Path "$publishPath/libcrypto.so.1.0.0" -ErrorAction Stop + } + } + } + + # download modules from powershell gallery. + # - PowerShellGet, PackageManagement, Microsoft.PowerShell.Archive + if ($PSModuleRestore) { + Restore-PSModuleToBuild -PublishPath $publishPath + } + + # publish powershell.config.json + $config = @{} + if ($environment.IsWindows) { + $config = @{ "Microsoft.PowerShell:ExecutionPolicy" = "RemoteSigned"; + "WindowsPowerShellCompatibilityModuleDenyList" = @("PSScheduledJob","BestPractices","UpdateServices") } + } + + # When building preview, we want the configuration to enable all experiemental features by default + # ARM is cross compiled, so we can't run pwsh to enumerate Experimental Features + if (-not $SkipExperimentalFeatureGeneration -and + (Test-IsPreview $psVersion) -and + -not (Test-IsReleaseCandidate $psVersion) -and + -not $Runtime.Contains("arm") -and + -not ($Runtime -like 'fxdependent*')) { + + $json = & $publishPath\pwsh -noprofile -command { + # Special case for DSC code in PS; + # this experimental feature requires new DSC module that is not inbox, + # so we don't want default DSC use case be broken + [System.Collections.ArrayList] $expFeatures = Get-ExperimentalFeature | Where-Object Name -NE PS7DscSupport | ForEach-Object -MemberName Name + + $expFeatures | Out-String | Write-Verbose -Verbose + + # Make sure ExperimentalFeatures from modules in PSHome are added + # https://github.com/PowerShell/PowerShell/issues/10550 + $ExperimentalFeaturesFromGalleryModulesInPSHome = @() + $ExperimentalFeaturesFromGalleryModulesInPSHome | ForEach-Object { + if (!$expFeatures.Contains($_)) { + $null = $expFeatures.Add($_) + } + } + + ConvertTo-Json $expFeatures + } + + $config += @{ ExperimentalFeatures = ([string[]] ($json | ConvertFrom-Json)) } + } + + if ($config.Count -gt 0) { + $configPublishPath = Join-Path -Path $publishPath -ChildPath "powershell.config.json" + Set-Content -Path $configPublishPath -Value ($config | ConvertTo-Json) -Force -ErrorAction Stop + } + + # Restore the Pester module + if ($CI) { + Restore-PSPester -Destination (Join-Path $publishPath "Modules") + } +} + +function New-PSOptions { + [CmdletBinding()] + param( + [ValidateSet("Debug", "Release", "CodeCoverage", '')] + [string]$Configuration, + + [ValidateSet("net6.0")] + [string]$Framework = "net6.0", + + # These are duplicated from Start-PSBuild + # We do not use ValidateScript since we want tab completion + [ValidateSet("", + "alpine-x64", + "fxdependent", + "fxdependent-win-desktop", + "linux-arm", + "linux-arm64", + "linux-x64", + "osx-arm64", + "osx-x64", + "win-arm", + "win-arm64", + "win7-x64", + "win7-x86")] + [string]$Runtime, + + [switch]$CrossGen, + + # Accept a path to the output directory + # If not null or empty, name of the executable will be appended to + # this path, otherwise, to the default path, and then the full path + # of the output executable will be assigned to the Output property + [string]$Output, + + [switch]$SMAOnly, + + [switch]$PSModuleRestore, + + [switch]$ForMinimalSize + ) + + # Add .NET CLI tools to PATH + Find-Dotnet + + if (-not $Configuration) { + $Configuration = 'Debug' + } + + Write-Verbose "Using configuration '$Configuration'" + Write-Verbose "Using framework '$Framework'" + + if (-not $Runtime) { + if ($environment.IsLinux) { + $Runtime = "linux-x64" + } elseif ($environment.IsMacOS) { + if ($PSVersionTable.OS.Contains('ARM64')) { + $Runtime = "osx-arm64" + } + else { + $Runtime = "osx-x64" + } + } else { + $RID = dotnet --info | ForEach-Object { + if ($_ -match "RID") { + $_ -split "\s+" | Select-Object -Last 1 + } + } + + # We plan to release packages targeting win7-x64 and win7-x86 RIDs, + # which supports all supported windows platforms. + # So we, will change the RID to win7- + $Runtime = $RID -replace "win\d+", "win7" + } + + if (-not $Runtime) { + Throw "Could not determine Runtime Identifier, please update dotnet" + } else { + Write-Verbose "Using runtime '$Runtime'" + } + } + + $PowerShellDir = if ($Runtime -like 'win*' -or ($Runtime -like 'fxdependent*' -and $environment.IsWindows)) { + "powershell-win-core" + } else { + "powershell-unix" + } + + $Top = [IO.Path]::Combine($PSScriptRoot, "src", $PowerShellDir) + Write-Verbose "Top project directory is $Top" + + $Executable = if ($Runtime -like 'fxdependent*') { + "pwsh.dll" + } elseif ($environment.IsLinux -or $environment.IsMacOS) { + "pwsh" + } elseif ($environment.IsWindows) { + "pwsh.exe" + } + + # Build the Output path + if (!$Output) { + if ($Runtime -like 'fxdependent*') { + $Output = [IO.Path]::Combine($Top, "bin", $Configuration, $Framework, "publish", $Executable) + } else { + $Output = [IO.Path]::Combine($Top, "bin", $Configuration, $Framework, $Runtime, "publish", $Executable) + } + } else { + $Output = [IO.Path]::Combine($Output, $Executable) + } + + if ($SMAOnly) + { + $Top = [IO.Path]::Combine($PSScriptRoot, "src", "System.Management.Automation") + } + + $RootInfo = @{RepoPath = $PSScriptRoot} + + # the valid root is the root of the filesystem and the folder PowerShell + $RootInfo['ValidPath'] = Join-Path -Path ([system.io.path]::GetPathRoot($RootInfo.RepoPath)) -ChildPath 'PowerShell' + + if($RootInfo.RepoPath -ne $RootInfo.ValidPath) + { + $RootInfo['Warning'] = "Please ensure your repo is at the root of the file system and named 'PowerShell' (example: '$($RootInfo.ValidPath)'), when building and packaging for release!" + $RootInfo['IsValid'] = $false + } + else + { + $RootInfo['IsValid'] = $true + } + + return New-PSOptionsObject ` + -RootInfo ([PSCustomObject]$RootInfo) ` + -Top $Top ` + -Runtime $Runtime ` + -Crossgen $Crossgen.IsPresent ` + -Configuration $Configuration ` + -PSModuleRestore $PSModuleRestore.IsPresent ` + -Framework $Framework ` + -Output $Output ` + -ForMinimalSize $ForMinimalSize +} + +function Start-PSPester { + [CmdletBinding(DefaultParameterSetName='default')] + param( + [Parameter(Position=0)] + [string[]]$Path = @("$PSScriptRoot/test/powershell"), + [string]$OutputFormat = "NUnitXml", + [string]$OutputFile = "pester-tests.xml", + [string[]]$ExcludeTag = 'Slow', + [string[]]$Tag = @("CI","Feature"), + [switch]$ThrowOnFailure, + [string]$BinDir = (Split-Path (Get-PSOptions -DefaultToNew).Output), + [string]$powershell = (Join-Path $BinDir 'pwsh'), + [string]$Pester = ([IO.Path]::Combine($BinDir, "Modules", "Pester")), + [Parameter(ParameterSetName='Unelevate',Mandatory=$true)] + [switch]$Unelevate, + [switch]$Quiet, + [switch]$Terse, + [Parameter(ParameterSetName='PassThru',Mandatory=$true)] + [switch]$PassThru, + [Parameter(ParameterSetName='PassThru',HelpMessage='Run commands on Linux with sudo.')] + [switch]$Sudo, + [switch]$IncludeFailingTest, + [switch]$IncludeCommonTests, + [string]$ExperimentalFeatureName, + [Parameter(HelpMessage='Title to publish the results as.')] + [string]$Title = 'PowerShell 7 Tests', + [Parameter(ParameterSetName='Wait', Mandatory=$true, + HelpMessage='Wait for the debugger to attach to PowerShell before Pester starts. Debug builds only!')] + [switch]$Wait, + [switch]$SkipTestToolBuild + ) + + if (-not (Get-Module -ListAvailable -Name $Pester -ErrorAction SilentlyContinue | Where-Object { $_.Version -ge "4.2" } )) + { + Restore-PSPester + } + + if ($IncludeFailingTest.IsPresent) + { + $Path += "$PSScriptRoot/tools/failingTests" + } + + if($IncludeCommonTests.IsPresent) + { + $path = += "$PSScriptRoot/test/common" + } + + # we need to do few checks and if user didn't provide $ExcludeTag explicitly, we should alternate the default + if ($Unelevate) + { + if (-not $environment.IsWindows) + { + throw '-Unelevate is currently not supported on non-Windows platforms' + } + + if (-not $environment.IsAdmin) + { + throw '-Unelevate cannot be applied because the current user is not Administrator' + } + + if (-not $PSBoundParameters.ContainsKey('ExcludeTag')) + { + $ExcludeTag += 'RequireAdminOnWindows' + } + } + elseif ($environment.IsWindows -and (-not $environment.IsAdmin)) + { + if (-not $PSBoundParameters.ContainsKey('ExcludeTag')) + { + $ExcludeTag += 'RequireAdminOnWindows' + } + } + elseif (-not $environment.IsWindows -and (-not $Sudo.IsPresent)) + { + if (-not $PSBoundParameters.ContainsKey('ExcludeTag')) + { + $ExcludeTag += 'RequireSudoOnUnix' + } + } + elseif (-not $environment.IsWindows -and $Sudo.IsPresent) + { + if (-not $PSBoundParameters.ContainsKey('Tag')) + { + $Tag = 'RequireSudoOnUnix' + } + } + + Write-Verbose "Running pester tests at '$path' with tag '$($Tag -join ''', ''')' and ExcludeTag '$($ExcludeTag -join ''', ''')'" -Verbose + if(!$SkipTestToolBuild.IsPresent) + { + $publishArgs = @{ } + # if we are building for Alpine, we must include the runtime as linux-x64 + # will not build runnable test tools + if ( $environment.IsLinux -and $environment.IsAlpine ) { + $publishArgs['runtime'] = 'alpine-x64' + } + Publish-PSTestTools @publishArgs | ForEach-Object {Write-Host $_} + } + + # All concatenated commands/arguments are suffixed with the delimiter (space) + + # Disable telemetry for all startups of pwsh in tests + $command = "`$env:POWERSHELL_TELEMETRY_OPTOUT = 'yes';" + if ($Terse) + { + $command += "`$ProgressPreference = 'silentlyContinue'; " + } + + # Autoload (in subprocess) temporary modules used in our tests + $newPathFragment = $TestModulePath + $TestModulePathSeparator + $command += '$env:PSModulePath = '+"'$newPathFragment'" + '+$env:PSModulePath;' + + # Windows needs the execution policy adjusted + if ($environment.IsWindows) { + $command += "Set-ExecutionPolicy -Scope Process Unrestricted; " + } + + $command += "Import-Module '$Pester'; " + + if ($Unelevate) + { + if ($environment.IsWindows) { + $outputBufferFilePath = [System.IO.Path]::GetTempFileName() + } + else { + # Azure DevOps agents do not have Temp folder setup on Ubuntu 20.04, hence using HOME directory + $outputBufferFilePath = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) + } + } + + $command += "Invoke-Pester " + + $command += "-OutputFormat ${OutputFormat} -OutputFile ${OutputFile} " + if ($ExcludeTag -and ($ExcludeTag -ne "")) { + $command += "-ExcludeTag @('" + (${ExcludeTag} -join "','") + "') " + } + if ($Tag) { + $command += "-Tag @('" + (${Tag} -join "','") + "') " + } + # sometimes we need to eliminate Pester output, especially when we're + # doing a daily build as the log file is too large + if ( $Quiet ) { + $command += "-Quiet " + } + if ( $PassThru ) { + $command += "-PassThru " + } + + $command += "'" + ($Path -join "','") + "'" + if ($Unelevate) + { + $command += " *> $outputBufferFilePath; '__UNELEVATED_TESTS_THE_END__' >> $outputBufferFilePath" + } + + Write-Verbose $command + + $script:nonewline = $true + $script:inerror = $false + function Write-Terse([string] $line) + { + $trimmedline = $line.Trim() + if ($trimmedline.StartsWith("[+]")) { + Write-Host "+" -NoNewline -ForegroundColor Green + $script:nonewline = $true + $script:inerror = $false + } + elseif ($trimmedline.StartsWith("[?]")) { + Write-Host "?" -NoNewline -ForegroundColor Cyan + $script:nonewline = $true + $script:inerror = $false + } + elseif ($trimmedline.StartsWith("[!]")) { + Write-Host "!" -NoNewline -ForegroundColor Gray + $script:nonewline = $true + $script:inerror = $false + } + elseif ($trimmedline.StartsWith("Executing script ")) { + # Skip lines where Pester reports that is executing a test script + return + } + elseif ($trimmedline -match "^\d+(\.\d+)?m?s$") { + # Skip the time elapse like '12ms', '1ms', '1.2s' and '12.53s' + return + } + else { + if ($script:nonewline) { + Write-Host "`n" -NoNewline + } + if ($trimmedline.StartsWith("[-]") -or $script:inerror) { + Write-Host $line -ForegroundColor Red + $script:inerror = $true + } + elseif ($trimmedline.StartsWith("VERBOSE:")) { + Write-Host $line -ForegroundColor Yellow + $script:inerror = $false + } + elseif ($trimmedline.StartsWith("Describing") -or $trimmedline.StartsWith("Context")) { + Write-Host $line -ForegroundColor Magenta + $script:inerror = $false + } + else { + Write-Host $line -ForegroundColor Gray + } + $script:nonewline = $false + } + } + + $PSFlags = @("-noprofile") + if (-not [string]::IsNullOrEmpty($ExperimentalFeatureName)) { + + if ($environment.IsWindows) { + $configFile = [System.IO.Path]::GetTempFileName() + } + else { + $configFile = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName())) + } + + $configFile = [System.IO.Path]::ChangeExtension($configFile, ".json") + + ## Create the config.json file to enable the given experimental feature. + ## On Windows, we need to have 'RemoteSigned' declared for ExecutionPolicy because the ExecutionPolicy is 'Restricted' by default. + ## On Unix, ExecutionPolicy is not supported, so we don't need to declare it. + if ($environment.IsWindows) { + $content = @" +{ + "Microsoft.PowerShell:ExecutionPolicy":"RemoteSigned", + "ExperimentalFeatures": [ + "$ExperimentalFeatureName" + ] +} +"@ + } else { + $content = @" +{ + "ExperimentalFeatures": [ + "$ExperimentalFeatureName" + ] +} +"@ + } + + Set-Content -Path $configFile -Value $content -Encoding Ascii -Force + $PSFlags = @("-settings", $configFile, "-noprofile") + } + + # -Wait is only available on Debug builds + # It is used to allow the debugger to attach before PowerShell + # runs pester in this case + if($Wait.IsPresent){ + $PSFlags += '-wait' + } + + # To ensure proper testing, the module path must not be inherited by the spawned process + try { + $originalModulePath = $env:PSModulePath + $originalTelemetry = $env:POWERSHELL_TELEMETRY_OPTOUT + $env:POWERSHELL_TELEMETRY_OPTOUT = 'yes' + if ($Unelevate) + { + Start-UnelevatedProcess -process $powershell -arguments ($PSFlags + "-c $Command") + $currentLines = 0 + while ($true) + { + $lines = Get-Content $outputBufferFilePath | Select-Object -Skip $currentLines + if ($Terse) + { + foreach ($line in $lines) + { + Write-Terse -line $line + } + } + else + { + $lines | Write-Host + } + if ($lines | Where-Object { $_ -eq '__UNELEVATED_TESTS_THE_END__'}) + { + break + } + + $count = ($lines | Measure-Object).Count + if ($count -eq 0) + { + Start-Sleep -Seconds 1 + } + else + { + $currentLines += $count + } + } + } + else + { + if ($PassThru.IsPresent) + { + if ($environment.IsWindows) { + $passThruFile = [System.IO.Path]::GetTempFileName() + } + else { + $passThruFile = Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName()) + } + + try + { + $command += "| Export-Clixml -Path '$passThruFile' -Force" + + $passThruCommand = { & $powershell $PSFlags -c $command } + if ($Sudo.IsPresent) { + # -E says to preserve the environment + $passThruCommand = { & sudo -E $powershell $PSFlags -c $command } + } + + $writeCommand = { Write-Host $_ } + if ($Terse) + { + $writeCommand = { Write-Terse $_ } + } + + Start-NativeExecution -sb $passThruCommand | ForEach-Object $writeCommand + Import-Clixml -Path $passThruFile | Where-Object {$_.TotalCount -is [Int32]} + } + finally + { + Remove-Item $passThruFile -ErrorAction SilentlyContinue -Force + } + } + else + { + if ($Terse) + { + Start-NativeExecution -sb {& $powershell $PSFlags -c $command} | ForEach-Object { Write-Terse -line $_ } + } + else + { + Start-NativeExecution -sb {& $powershell $PSFlags -c $command} + } + } + } + } finally { + $env:PSModulePath = $originalModulePath + $env:POWERSHELL_TELEMETRY_OPTOUT = $originalTelemetry + if ($Unelevate) + { + Remove-Item $outputBufferFilePath + } + } + + Publish-TestResults -Path $OutputFile -Title $Title + + if($ThrowOnFailure) + { + Test-PSPesterResults -TestResultsFile $OutputFile + } +} + +function Install-Dotnet { + [CmdletBinding()] + param( + [string]$Channel = $dotnetCLIChannel, + [string]$Version = $dotnetCLIRequiredVersion, + [string]$Quality = $dotnetCLIQuality, + [switch]$NoSudo, + [string]$InstallDir, + [string]$AzureFeed, + [string]$FeedCredential + ) + + # This allows sudo install to be optional; needed when running in containers / as root + # Note that when it is null, Invoke-Expression (but not &) must be used to interpolate properly + $sudo = if (!$NoSudo) { "sudo" } + + $installObtainUrl = "https://dotnet.microsoft.com/download/dotnet-core/scripts/v1" + $uninstallObtainUrl = "https://raw.githubusercontent.com/dotnet/cli/master/scripts/obtain" + + # Install for Linux and OS X + if ($environment.IsLinux -or $environment.IsMacOS) { + $wget = Get-Command -Name wget -CommandType Application -TotalCount 1 -ErrorAction Stop + + # Uninstall all previous dotnet packages + $uninstallScript = if ($environment.IsLinux -and $environment.IsUbuntu) { + "dotnet-uninstall-debian-packages.sh" + } elseif ($environment.IsMacOS) { + "dotnet-uninstall-pkgs.sh" + } + + if ($uninstallScript) { + Start-NativeExecution { + & $wget $uninstallObtainUrl/uninstall/$uninstallScript + Invoke-Expression "$sudo bash ./$uninstallScript" + } + } else { + Write-Warning "This script only removes prior versions of dotnet for Ubuntu and OS X" + } + + # Install new dotnet 1.1.0 preview packages + $installScript = "dotnet-install.sh" + Start-NativeExecution { + Write-Verbose -Message "downloading install script from $installObtainUrl/$installScript ..." -Verbose + & $wget $installObtainUrl/$installScript + + if ((Get-ChildItem "./$installScript").Length -eq 0) { + throw "./$installScript was 0 length" + } + + if ($Version) { + $bashArgs = @("./$installScript", '-v', $Version, '-q', $Quality) + } + elseif ($Channel) { + $bashArgs = @("./$installScript", '-c', $Channel, '-q', $Quality) + } + + if ($InstallDir) { + $bashArgs += @('-i', $InstallDir) + } + + if ($AzureFeed) { + $bashArgs += @('-AzureFeed', $AzureFeed, '-FeedCredential', $FeedCredential) + } + + bash @bashArgs + } + } elseif ($environment.IsWindows) { + Remove-Item -ErrorAction SilentlyContinue -Recurse -Force ~\AppData\Local\Microsoft\dotnet + $installScript = "dotnet-install.ps1" + Invoke-WebRequest -Uri $installObtainUrl/$installScript -OutFile $installScript + if (-not $environment.IsCoreCLR) { + $installArgs = @{ + Quality = $Quality + } + + if ($Version) { + $installArgs += @{ Version = $Version } + } elseif ($Channel) { + $installArgs += @{ Channel = $Channel } + } + + if ($InstallDir) { + $installArgs += @{ InstallDir = $InstallDir } + } + + if ($AzureFeed) { + $installArgs += @{ + AzureFeed = $AzureFeed + $FeedCredential = $FeedCredential + } + } + + & ./$installScript @installArgs + } + else { + # dotnet-install.ps1 uses APIs that are not supported in .NET Core, so we run it with Windows PowerShell + $fullPSPath = Join-Path -Path $env:windir -ChildPath "System32\WindowsPowerShell\v1.0\powershell.exe" + $fullDotnetInstallPath = Join-Path -Path $PWD.Path -ChildPath $installScript + Start-NativeExecution { + + if ($Version) { + $psArgs = @('-NoLogo', '-NoProfile', '-File', $fullDotnetInstallPath, '-Version', $Version, '-Quality', $Quality) + } + elseif ($Channel) { + $psArgs = @('-NoLogo', '-NoProfile', '-File', $fullDotnetInstallPath, '-Channel', $Channel, '-Quality', $Quality) + } + + if ($InstallDir) { + $psArgs += @('-InstallDir', $InstallDir) + } + + if ($AzureFeed) { + $psArgs += @('-AzureFeed', $AzureFeed, '-FeedCredential', $FeedCredential) + } + + & $fullPSPath @psArgs + } + } + } +} + +function Start-PSBootstrap { + [CmdletBinding()] + param( + [string]$Channel = $dotnetCLIChannel, + # we currently pin dotnet-cli version, and will + # update it when more stable version comes out. + [string]$Version = $dotnetCLIRequiredVersion, + [switch]$Package, + [switch]$NoSudo, + [switch]$BuildLinuxArm, + [switch]$Force + ) + + Write-Log -message "Installing PowerShell build dependencies" + + Push-Location $PSScriptRoot/tools + + try { + if ($environment.IsLinux -or $environment.IsMacOS) { + # This allows sudo install to be optional; needed when running in containers / as root + # Note that when it is null, Invoke-Expression (but not &) must be used to interpolate properly + $sudo = if (!$NoSudo) { "sudo" } + + if ($BuildLinuxArm -and $environment.IsLinux -and -not $environment.IsUbuntu) { + Write-Error "Cross compiling for linux-arm is only supported on Ubuntu environment" + return + } + + # Install ours and .NET's dependencies + $Deps = @() + if ($environment.IsLinux -and $environment.IsUbuntu) { + # Build tools + $Deps += "curl", "g++", "cmake", "make" + + if ($BuildLinuxArm) { + $Deps += "gcc-arm-linux-gnueabihf", "g++-arm-linux-gnueabihf" + } + + # .NET Core required runtime libraries + $Deps += "libunwind8" + if ($environment.IsUbuntu16) { $Deps += "libicu55" } + elseif ($environment.IsUbuntu18) { $Deps += "libicu60"} + + # Packaging tools + if ($Package) { $Deps += "ruby-dev", "groff", "libffi-dev" } + + # Install dependencies + # change the fontend from apt-get to noninteractive + $originalDebianFrontEnd=$env:DEBIAN_FRONTEND + $env:DEBIAN_FRONTEND='noninteractive' + try { + Start-NativeExecution { + Invoke-Expression "$sudo apt-get update -qq" + Invoke-Expression "$sudo apt-get install -y -qq $Deps" + } + } + finally { + # change the apt frontend back to the original + $env:DEBIAN_FRONTEND=$originalDebianFrontEnd + } + } elseif ($environment.IsLinux -and $environment.IsRedHatFamily) { + # Build tools + $Deps += "which", "curl", "gcc-c++", "cmake", "make" + + # .NET Core required runtime libraries + $Deps += "libicu", "libunwind" + + # Packaging tools + if ($Package) { $Deps += "ruby-devel", "rpm-build", "groff", 'libffi-devel' } + + $PackageManager = Get-RedHatPackageManager + + $baseCommand = "$sudo $PackageManager" + + # On OpenSUSE 13.2 container, sudo does not exist, so don't use it if not needed + if($NoSudo) + { + $baseCommand = $PackageManager + } + + # Install dependencies + Start-NativeExecution { + Invoke-Expression "$baseCommand $Deps" + } + } elseif ($environment.IsLinux -and $environment.IsSUSEFamily) { + # Build tools + $Deps += "gcc", "cmake", "make" + + # Packaging tools + if ($Package) { $Deps += "ruby-devel", "rpmbuild", "groff", 'libffi-devel' } + + $PackageManager = "zypper --non-interactive install" + $baseCommand = "$sudo $PackageManager" + + # On OpenSUSE 13.2 container, sudo does not exist, so don't use it if not needed + if($NoSudo) + { + $baseCommand = $PackageManager + } + + # Install dependencies + Start-NativeExecution { + Invoke-Expression "$baseCommand $Deps" + } + } elseif ($environment.IsMacOS) { + if ($environment.UsingHomebrew) { + $PackageManager = "brew" + } elseif ($environment.UsingMacports) { + $PackageManager = "$sudo port" + } + + # Build tools + $Deps += "cmake" + + # .NET Core required runtime libraries + $Deps += "openssl" + + # Install dependencies + # ignore exitcode, because they may be already installed + Start-NativeExecution ([ScriptBlock]::Create("$PackageManager install $Deps")) -IgnoreExitcode + } elseif ($environment.IsLinux -and $environment.IsAlpine) { + $Deps += 'libunwind', 'libcurl', 'bash', 'cmake', 'clang', 'build-base', 'git', 'curl' + + Start-NativeExecution { + Invoke-Expression "apk add $Deps" + } + } + + # Install [fpm](https://github.com/jordansissel/fpm) and [ronn](https://github.com/rtomayko/ronn) + if ($Package) { + try { + # We cannot guess if the user wants to run gem install as root on linux and windows, + # but macOs usually requires sudo + $gemsudo = '' + if($environment.IsMacOS -or $env:TF_BUILD) { + $gemsudo = $sudo + } + Start-NativeExecution ([ScriptBlock]::Create("$gemsudo gem install ffi -v 1.12.0 --no-document")) + Start-NativeExecution ([ScriptBlock]::Create("$gemsudo gem install fpm -v 1.11.0 --no-document")) + Start-NativeExecution ([ScriptBlock]::Create("$gemsudo gem install ronn -v 0.7.3 --no-document")) + } catch { + Write-Warning "Installation of fpm and ronn gems failed! Must resolve manually." + } + } + } + + # Try to locate dotnet-SDK before installing it + Find-Dotnet + + # Install dotnet-SDK + $dotNetExists = precheck 'dotnet' $null + $dotNetVersion = [string]::Empty + if($dotNetExists) { + $dotNetVersion = Start-NativeExecution -sb { dotnet --version } -IgnoreExitcode + } + + if(!$dotNetExists -or $dotNetVersion -ne $dotnetCLIRequiredVersion -or $Force.IsPresent) { + if($Force.IsPresent) { + Write-Log -message "Installing dotnet due to -Force." + } + elseif(!$dotNetExists) { + Write-Log -message "dotnet not present. Installing dotnet." + } + else { + Write-Log -message "dotnet out of date ($dotNetVersion). Updating dotnet." + } + + $DotnetArguments = @{ Channel=$Channel; Version=$Version; NoSudo=$NoSudo } + Install-Dotnet @DotnetArguments + } + else { + Write-Log -message "dotnet is already installed. Skipping installation." + } + + # Install Windows dependencies if `-Package` or `-BuildWindowsNative` is specified + if ($environment.IsWindows) { + ## The VSCode build task requires 'pwsh.exe' to be found in Path + if (-not (Get-Command -Name pwsh.exe -CommandType Application -ErrorAction Ignore)) + { + Write-Log -message "pwsh.exe not found. Install latest PowerShell release and add it to Path" + $psInstallFile = [System.IO.Path]::Combine($PSScriptRoot, "tools", "install-powershell.ps1") + & $psInstallFile -AddToPath + } + } + } finally { + Pop-Location + } +} + +function Start-CrossGen { + [CmdletBinding()] + param( + [Parameter(Mandatory= $true)] + [ValidateNotNullOrEmpty()] + [String] + $PublishPath, + + [Parameter(Mandatory=$true)] + [ValidateSet("alpine-x64", + "linux-arm", + "linux-arm64", + "linux-x64", + "osx-arm64", + "osx-x64", + "win-arm", + "win-arm64", + "win7-x64", + "win7-x86")] + [string] + $Runtime + ) + + function New-CrossGenAssembly { + param ( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [String[]] + $AssemblyPath, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [String] + $CrossgenPath, + + [Parameter(Mandatory = $true)] + [ValidateSet("alpine-x64", + "linux-arm", + "linux-arm64", + "linux-x64", + "osx-arm64", + "osx-x64", + "win-arm", + "win-arm64", + "win7-x64", + "win7-x86")] + [string] + $Runtime + ) + + $platformAssembliesPath = Split-Path $AssemblyPath[0] -Parent + + $targetOS, $targetArch = $Runtime -split '-' + + # Special cases where OS / Arch does not conform with runtime names + switch ($Runtime) { + 'alpine-x64' { + $targetOS = 'linux' + $targetArch = 'x64' + } + 'win-arm' { + $targetOS = 'windows' + $targetArch = 'arm' + } + 'win-arm64' { + $targetOS = 'windows' + $targetArch = 'arm64' + } + 'win7-x64' { + $targetOS = 'windows' + $targetArch = 'x64' + } + 'win7-x86' { + $targetOS = 'windows' + $targetArch = 'x86' + } + } + + $generatePdb = $targetos -eq 'windows' + + # The path to folder must end with directory separator + $dirSep = [System.IO.Path]::DirectorySeparatorChar + $platformAssembliesPath = if (-not $platformAssembliesPath.EndsWith($dirSep)) { $platformAssembliesPath + $dirSep } + + Start-NativeExecution { + $crossgen2Params = @( + "-r" + $platformAssembliesPath + "--out-near-input" + "--single-file-compilation" + "-O" + "--targetos" + $targetOS + "--targetarch" + $targetArch + ) + + if ($generatePdb) { + $crossgen2Params += "--pdb" + } + + $crossgen2Params += $AssemblyPath + + & $CrossgenPath $crossgen2Params + } + } + + if (-not (Test-Path $PublishPath)) { + throw "Path '$PublishPath' does not exist." + } + + # Get the path to crossgen + $crossGenExe = if ($environment.IsWindows) { "crossgen2.exe" } else { "crossgen2" } + + # The crossgen tool is only published for these particular runtimes + $crossGenRuntime = if ($environment.IsWindows) { + # for windows the tool architecture is the host machine architecture, so it is always x64. + # we can cross compile for x86, arm and arm64 + "win-x64" + } else { + $Runtime + } + + if (-not $crossGenRuntime) { + throw "crossgen is not available for this platform" + } + + $dotnetRuntimeVersion = $script:Options.Framework -replace 'net' + + # Get the CrossGen.exe for the correct runtime with the latest version + $crossGenPath = Get-ChildItem $script:Environment.nugetPackagesRoot $crossGenExe -Recurse | ` + Where-Object { $_.FullName -match $crossGenRuntime } | ` + Where-Object { $_.FullName -match $dotnetRuntimeVersion } | ` + Where-Object { (Split-Path $_.FullName -Parent).EndsWith('tools') } | ` + Sort-Object -Property FullName -Descending | ` + Select-Object -First 1 | ` + ForEach-Object { $_.FullName } + if (-not $crossGenPath) { + throw "Unable to find latest version of crossgen2.exe. 'Please run Start-PSBuild -Clean' first, and then try again." + } + Write-Verbose "Matched CrossGen2.exe: $crossGenPath" -Verbose + + # Common assemblies used by Add-Type or assemblies with high JIT and no pdbs to crossgen + $commonAssembliesForAddType = @( + "Microsoft.CodeAnalysis.CSharp.dll" + "Microsoft.CodeAnalysis.dll" + "System.Linq.Expressions.dll" + "Microsoft.CSharp.dll" + "System.Runtime.Extensions.dll" + "System.Linq.dll" + "System.Collections.Concurrent.dll" + "System.Collections.dll" + "Newtonsoft.Json.dll" + "System.IO.FileSystem.dll" + "System.Diagnostics.Process.dll" + "System.Threading.Tasks.Parallel.dll" + "System.Security.AccessControl.dll" + "System.Text.Encoding.CodePages.dll" + "System.Private.Uri.dll" + "System.Threading.dll" + "System.Security.Principal.Windows.dll" + "System.Console.dll" + "Microsoft.Win32.Registry.dll" + "System.IO.Pipes.dll" + "System.Diagnostics.FileVersionInfo.dll" + "System.Collections.Specialized.dll" + "Microsoft.ApplicationInsights.dll" + ) + + $fullAssemblyList = $commonAssembliesForAddType + + $assemblyFullPaths = @() + $assemblyFullPaths += foreach ($assemblyName in $fullAssemblyList) { + Join-Path $PublishPath $assemblyName + } + + New-CrossGenAssembly -CrossgenPath $crossGenPath -AssemblyPath $assemblyFullPaths -Runtime $Runtime + + # + # With the latest dotnet.exe, the default load context is only able to load TPAs, and TPA + # only contains IL assembly names. In order to make the default load context able to load + # the NI PS assemblies, we need to replace the IL PS assemblies with the corresponding NI + # PS assemblies, but with the same IL assembly names. + # + Write-Verbose "PowerShell Ngen assemblies have been generated. Deploying ..." -Verbose + foreach ($assemblyName in $fullAssemblyList) { + + # Remove the IL assembly and its symbols. + $assemblyPath = Join-Path $PublishPath $assemblyName + $symbolsPath = [System.IO.Path]::ChangeExtension($assemblyPath, ".pdb") + + Remove-Item $assemblyPath -Force -ErrorAction Stop + + # Rename the corresponding ni.dll assembly to be the same as the IL assembly + $niAssemblyPath = [System.IO.Path]::ChangeExtension($assemblyPath, "ni.dll") + Rename-Item $niAssemblyPath $assemblyPath -Force -ErrorAction Stop + + # No symbols are available for Microsoft.CodeAnalysis.CSharp.dll, Microsoft.CodeAnalysis.dll, + # Microsoft.CodeAnalysis.VisualBasic.dll, and Microsoft.CSharp.dll. + if ($commonAssembliesForAddType -notcontains $assemblyName) { + Remove-Item $symbolsPath -Force -ErrorAction Stop + } + } +} + +function Use-PSClass { + [CmdletBinding()] + param ( + [Parameter(ValueFromPipeline = $true, Mandatory = $true, Position = 0)] + [string[]]$Logfile, + [Parameter()][switch]$IncludeEmpty, + [Parameter()][switch]$MultipleLog + ) + <# +Convert our test logs to +xunit schema - top level assemblies +Pester conversion +foreach $r in "test-results"."test-suite".results."test-suite" +assembly + name = $r.Description + config-file = log file (this is the only way we can determine between admin/nonadmin log) + test-framework = Pester + environment = top-level "test-results.environment.platform + run-date = date (doesn't exist in pester except for beginning) + run-time = time + time = +#> + + BEGIN { + # CLASSES + class assemblies { + # attributes + [datetime]$timestamp + # child elements + [System.Collections.Generic.List[testAssembly]]$assembly + assemblies() { + $this.timestamp = [datetime]::now + $this.assembly = [System.Collections.Generic.List[testAssembly]]::new() + } + static [assemblies] op_Addition([assemblies]$ls, [assemblies]$rs) { + $newAssembly = [assemblies]::new() + $newAssembly.assembly.AddRange($ls.assembly) + $newAssembly.assembly.AddRange($rs.assembly) + return $newAssembly + } + [string]ToString() { + $sb = [text.stringbuilder]::new() + $sb.AppendLine('' -f $this.timestamp) + foreach ( $a in $this.assembly ) { + $sb.Append("$a") + } + $sb.AppendLine(""); + return $sb.ToString() + } + # use Write-Output to emit these into the pipeline + [array]GetTests() { + return $this.Assembly.collection.test + } + } + + class testAssembly { + # attributes + [string]$name # path to pester file + [string]${config-file} + [string]${test-framework} # Pester + [string]$environment + [string]${run-date} + [string]${run-time} + [decimal]$time + [int]$total + [int]$passed + [int]$failed + [int]$skipped + [int]$errors + testAssembly ( ) { + $this."config-file" = "no config" + $this."test-framework" = "Pester" + $this.environment = $script:environment + $this."run-date" = $script:rundate + $this."run-time" = $script:runtime + $this.collection = [System.Collections.Generic.List[collection]]::new() + } + # child elements + [error[]]$error + [System.Collections.Generic.List[collection]]$collection + [string]ToString() { + $sb = [System.Text.StringBuilder]::new() + $sb.AppendFormat(' ") + if ( $this.error ) { + $sb.AppendLine(" ") + foreach ( $e in $this.error ) { + $sb.AppendLine($e.ToString()) + } + $sb.AppendLine(" ") + } else { + $sb.AppendLine(" ") + } + foreach ( $col in $this.collection ) { + $sb.AppendLine($col.ToString()) + } + $sb.AppendLine(" ") + return $sb.ToString() + } + } + + class collection { + # attributes + [string]$name + [decimal]$time + [int]$total + [int]$passed + [int]$failed + [int]$skipped + # child element + [System.Collections.Generic.List[test]]$test + # constructor + collection () { + $this.test = [System.Collections.Generic.List[test]]::new() + } + [string]ToString() { + $sb = [Text.StringBuilder]::new() + if ( $this.test.count -eq 0 ) { + $sb.AppendLine(" ") + } else { + $sb.AppendFormat(' ' + "`n", + $this.total, $this.passed, $this.failed, $this.skipped, [security.securityelement]::escape($this.name), $this.time) + foreach ( $t in $this.test ) { + $sb.AppendLine(" " + $t.ToString()); + } + $sb.Append(" ") + } + return $sb.ToString() + } + } + + class errors { + [error[]]$error + } + class error { + # attributes + [string]$type + [string]$name + # child elements + [failure]$failure + [string]ToString() { + $sb = [system.text.stringbuilder]::new() + $sb.AppendLine('' -f $this.type, [security.securityelement]::escape($this.Name)) + $sb.AppendLine($this.failure -as [string]) + $sb.AppendLine("") + return $sb.ToString() + } + } + + class cdata { + [string]$text + cdata ( [string]$s ) { $this.text = $s } + [string]ToString() { + return '' + } + } + + class failure { + [string]${exception-type} + [cdata]$message + [cdata]${stack-trace} + failure ( [string]$message, [string]$stack ) { + $this."exception-type" = "Pester" + $this.Message = [cdata]::new($message) + $this."stack-trace" = [cdata]::new($stack) + } + [string]ToString() { + $sb = [text.stringbuilder]::new() + $sb.AppendLine(" ") + $sb.AppendLine(" " + ($this.message -as [string]) + "") + $sb.AppendLine(" " + ($this."stack-trace" -as [string]) + "") + $sb.Append(" ") + return $sb.ToString() + } + } + + enum resultenum { + Pass + Fail + Skip + } + + class trait { + # attributes + [string]$name + [string]$value + } + class traits { + [trait[]]$trait + } + class test { + # attributes + [string]$name + [string]$type + [string]$method + [decimal]$time + [resultenum]$result + # child elements + [trait[]]$traits + [failure]$failure + [cdata]$reason # skip reason + [string]ToString() { + $sb = [text.stringbuilder]::new() + $sb.appendformat(' ") + $sb.AppendLine($this.failure -as [string]) + $sb.append(' ') + } else { + $sb.Append("/>") + } + return $sb.ToString() + } + } + + function convert-pesterlog ( [xml]$x, $logpath, [switch]$includeEmpty ) { + <#$resultMap = @{ + Success = "Pass" + Ignored = "Skip" + Failure = "Fail" + }#> + + $resultMap = @{ + Success = "Pass" + Ignored = "Skip" + Failure = "Fail" + Inconclusive = "Skip" + } + + $configfile = $logpath + $runtime = $x."test-results".time + $environment = $x."test-results".environment.platform + "-" + $x."test-results".environment."os-version" + $rundate = $x."test-results".date + $suites = $x."test-results"."test-suite".results."test-suite" + $assemblies = [assemblies]::new() + foreach ( $suite in $suites ) { + $tCases = $suite.SelectNodes(".//test-case") + # only create an assembly group if we have tests + if ( $tCases.count -eq 0 -and ! $includeEmpty ) { continue } + $tGroup = $tCases | Group-Object result + $total = $tCases.Count + $asm = [testassembly]::new() + $asm.environment = $environment + $asm."run-date" = $rundate + $asm."run-time" = $runtime + $asm.Name = $suite.name + $asm."config-file" = $configfile + $asm.time = $suite.time + $asm.total = $suite.SelectNodes(".//test-case").Count + $asm.Passed = $tGroup| Where-Object -FilterScript {$_.Name -eq "Success"} | ForEach-Object -Process {$_.Count} + $asm.Failed = $tGroup| Where-Object -FilterScript {$_.Name -eq "Failure"} | ForEach-Object -Process {$_.Count} + $asm.Skipped = $tGroup| Where-Object -FilterScript { $_.Name -eq "Ignored" } | ForEach-Object -Process {$_.Count} + $asm.Skipped += $tGroup| Where-Object -FilterScript { $_.Name -eq "Inconclusive" } | ForEach-Object -Process {$_.Count} + $c = [collection]::new() + $c.passed = $asm.Passed + $c.failed = $asm.failed + $c.skipped = $asm.skipped + $c.total = $asm.total + $c.time = $asm.time + $c.name = $asm.name + foreach ( $tc in $suite.SelectNodes(".//test-case")) { + if ( $tc.result -match "Success|Ignored|Failure" ) { + $t = [test]::new() + $t.name = $tc.Name + $t.time = $tc.time + $t.method = $tc.description # the pester actually puts the name of the "it" as description + $t.type = $suite.results."test-suite".description | Select-Object -First 1 + $t.result = $resultMap[$tc.result] + if ( $tc.failure ) { + $t.failure = [failure]::new($tc.failure.message, $tc.failure."stack-trace") + } + $null = $c.test.Add($t) + } + } + $null = $asm.collection.add($c) + $assemblies.assembly.Add($asm) + } + $assemblies + } + + # convert it to our object model + # a simple conversion + function convert-xunitlog { + param ( $x, $logpath ) + $asms = [assemblies]::new() + $asms.timestamp = $x.assemblies.timestamp + foreach ( $assembly in $x.assemblies.assembly ) { + $asm = [testAssembly]::new() + $asm.environment = $assembly.environment + $asm."test-framework" = $assembly."test-framework" + $asm."run-date" = $assembly."run-date" + $asm."run-time" = $assembly."run-time" + $asm.total = $assembly.total + $asm.passed = $assembly.passed + $asm.failed = $assembly.failed + $asm.skipped = $assembly.skipped + $asm.time = $assembly.time + $asm.name = $assembly.name + foreach ( $coll in $assembly.collection ) { + $c = [collection]::new() + $c.name = $coll.name + $c.total = $coll.total + $c.passed = $coll.passed + $c.failed = $coll.failed + $c.skipped = $coll.skipped + $c.time = $coll.time + foreach ( $t in $coll.test ) { + $test = [test]::new() + $test.name = $t.name + $test.type = $t.type + $test.method = $t.method + $test.time = $t.time + $test.result = $t.result + $c.test.Add($test) + } + $null = $asm.collection.add($c) + } + $null = $asms.assembly.add($asm) + } + $asms + } + $Logs = @() + } + + PROCESS { + #### MAIN #### + foreach ( $log in $Logfile ) { + foreach ( $logpath in (Resolve-Path $log).path ) { + Write-Progress "converting file $logpath" + if ( ! $logpath) { throw "Cannot resolve $Logfile" } + $x = [xml](Get-Content -Raw -ReadCount 0 $logpath) + + if ( $x.psobject.properties['test-results'] ) { + $Logs += convert-pesterlog $x $logpath -includeempty:$includeempty + } elseif ( $x.psobject.properties['assemblies'] ) { + $Logs += convert-xunitlog $x $logpath -includeEmpty:$includeEmpty + } else { + Write-Error "Cannot determine log type" + } + } + } + } + + END { + if ( $MultipleLog ) { + $Logs + } else { + $combinedLog = $Logs[0] + for ( $i = 1; $i -lt $logs.count; $i++ ) { + $combinedLog += $Logs[$i] + } + $combinedLog + } + } +} + +function Start-PSPackage { + [CmdletBinding(DefaultParameterSetName='Version',SupportsShouldProcess=$true)] + param( + # PowerShell packages use Semantic Versioning https://semver.org/ + [Parameter(ParameterSetName = "Version")] + [string]$Version, + + [Parameter(ParameterSetName = "ReleaseTag")] + [ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d{1,2})?)?$")] + [ValidateNotNullOrEmpty()] + [string]$ReleaseTag, + + # Package name + [ValidatePattern("^powershell")] + [string]$Name = "powershell", + + # Ubuntu, CentOS, Fedora, macOS, and Windows packages are supported + [ValidateSet("msix", "deb", "osxpkg", "rpm", "msi", "zip", "zip-pdb", "nupkg", "tar", "tar-arm", "tar-arm64", "tar-alpine", "fxdependent", "fxdependent-win-desktop", "min-size")] + [string[]]$Type, + + # Generate windows downlevel package + [ValidateSet("win7-x86", "win7-x64", "win-arm", "win-arm64")] + [ValidateScript({$Environment.IsWindows})] + [string] $WindowsRuntime, + + [ValidateSet('osx-x64', 'osx-arm64')] + [ValidateScript({$Environment.IsMacOS})] + [string] $MacOSRuntime, + + [Switch] $Force, + + [Switch] $SkipReleaseChecks, + + [switch] $NoSudo, + + [switch] $LTS + ) + + DynamicParam { + if ($Type -in ('zip', 'min-size') -or $Type -like 'fxdependent*') { + # Add a dynamic parameter '-IncludeSymbols' when the specified package type is 'zip' only. + # The '-IncludeSymbols' parameter can be used to indicate that the package should only contain powershell binaries and symbols. + $ParameterAttr = New-Object "System.Management.Automation.ParameterAttribute" + $Attributes = New-Object "System.Collections.ObjectModel.Collection``1[System.Attribute]" + $Attributes.Add($ParameterAttr) > $null + + $Parameter = New-Object "System.Management.Automation.RuntimeDefinedParameter" -ArgumentList ("IncludeSymbols", [switch], $Attributes) + $Dict = New-Object "System.Management.Automation.RuntimeDefinedParameterDictionary" + $Dict.Add("IncludeSymbols", $Parameter) > $null + return $Dict + } + } + + End { + $IncludeSymbols = $null + if ($PSBoundParameters.ContainsKey('IncludeSymbols')) { + Write-Log 'setting IncludeSymbols' + $IncludeSymbols = $PSBoundParameters['IncludeSymbols'] + } + + # Runtime and Configuration settings required by the package + ($Runtime, $Configuration) = if ($WindowsRuntime) { + $WindowsRuntime, "Release" + } elseif ($MacOSRuntime) { + $MacOSRuntime, "Release" + } elseif ($Type -eq "tar-alpine") { + New-PSOptions -Configuration "Release" -Runtime "alpine-x64" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } + } elseif ($Type -eq "tar-arm") { + New-PSOptions -Configuration "Release" -Runtime "Linux-ARM" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } + } elseif ($Type -eq "tar-arm64") { + if ($IsMacOS) { + New-PSOptions -Configuration "Release" -Runtime "osx-arm64" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } + } else { + New-PSOptions -Configuration "Release" -Runtime "Linux-ARM64" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } + } + } else { + New-PSOptions -Configuration "Release" -WarningAction SilentlyContinue | ForEach-Object { $_.Runtime, $_.Configuration } + } + + if ($Environment.IsWindows) { + # Runtime will be one of win7-x64, win7-x86, "win-arm" and "win-arm64" on Windows. + # Build the name suffix for universal win-plat packages. + switch ($Runtime) { + "win-arm" { $NameSuffix = "win-arm32" } + "win-arm64" { $NameSuffix = "win-arm64" } + default { $NameSuffix = $_ -replace 'win\d+', 'win' } + } + } + + if ($Type -eq 'fxdependent') { + $NameSuffix = "win-fxdependent" + Write-Log "Packaging : '$Type'; Packaging Configuration: '$Configuration'" + } elseif ($Type -eq 'fxdependent-win-desktop') { + $NameSuffix = "win-fxdependentWinDesktop" + Write-Log "Packaging : '$Type'; Packaging Configuration: '$Configuration'" + } elseif ($MacOSRuntime) { + $NameSuffix = $MacOSRuntime + } else { + Write-Log "Packaging RID: '$Runtime'; Packaging Configuration: '$Configuration'" + } + + $Script:Options = Get-PSOptions + $actualParams = @() + + $crossGenCorrect = $false + if ($Runtime -match "arm" -or $Type -eq 'min-size') { + ## crossgen doesn't support arm32/64; + ## For the min-size package, we intentionally avoid crossgen. + $crossGenCorrect = $true + } + elseif ($Script:Options.CrossGen) { + $actualParams += '-CrossGen' + $crossGenCorrect = $true + } + + $PSModuleRestoreCorrect = $false + + # Require PSModuleRestore for packaging without symbols + # But Disallow it when packaging with symbols + if (!$IncludeSymbols.IsPresent -and $Script:Options.PSModuleRestore) { + $actualParams += '-PSModuleRestore' + $PSModuleRestoreCorrect = $true + } + elseif ($IncludeSymbols.IsPresent -and !$Script:Options.PSModuleRestore) { + $PSModuleRestoreCorrect = $true + } + else { + $actualParams += '-PSModuleRestore' + } + + $precheckFailed = if ($Type -like 'fxdependent*' -or $Type -eq 'tar-alpine') { + ## We do not check for runtime and crossgen for framework dependent package. + -not $Script:Options -or ## Start-PSBuild hasn't been executed yet + -not $PSModuleRestoreCorrect -or ## Last build didn't specify '-PSModuleRestore' correctly + $Script:Options.Configuration -ne $Configuration -or ## Last build was with configuration other than 'Release' + $Script:Options.Framework -ne $script:netCoreRuntime ## Last build wasn't for CoreCLR + } else { + -not $Script:Options -or ## Start-PSBuild hasn't been executed yet + -not $crossGenCorrect -or ## Last build didn't specify '-CrossGen' correctly + -not $PSModuleRestoreCorrect -or ## Last build didn't specify '-PSModuleRestore' correctly + $Script:Options.Runtime -ne $Runtime -or ## Last build wasn't for the required RID + $Script:Options.Configuration -ne $Configuration -or ## Last build was with configuration other than 'Release' + $Script:Options.Framework -ne $script:netCoreRuntime ## Last build wasn't for CoreCLR + } + + # Make sure the most recent build satisfies the package requirement + if ($precheckFailed) { + # It's possible that the most recent build doesn't satisfy the package requirement but + # an earlier build does. + # It's also possible that the last build actually satisfies the package requirement but + # then `Start-PSPackage` runs from a new PS session or `build.psm1` was reloaded. + # + # In these cases, the user will be asked to build again even though it's technically not + # necessary. However, we want it that way -- being very explict when generating packages. + # This check serves as a simple gate to ensure that the user knows what he is doing, and + # also ensure `Start-PSPackage` does what the user asks/expects, because once packages + # are generated, it'll be hard to verify if they were built from the correct content. + + + $params = @('-Clean') + + # CrossGen cannot be done for framework dependent package as it is runtime agnostic. + if ($Type -notlike 'fxdependent*') { + $params += '-CrossGen' + } + + if (!$IncludeSymbols.IsPresent) { + $params += '-PSModuleRestore' + } + + $actualParams += '-Runtime ' + $Script:Options.Runtime + + if ($Type -eq 'fxdependent') { + $params += '-Runtime', 'fxdependent' + } elseif ($Type -eq 'fxdependent-win-desktop') { + $params += '-Runtime', 'fxdependent-win-desktop' + } else { + $params += '-Runtime', $Runtime + } + + $params += '-Configuration', $Configuration + $actualParams += '-Configuration ' + $Script:Options.Configuration + + Write-Warning "Build started with unexpected parameters 'Start-PSBuild $actualParams" + throw "Please ensure you have run 'Start-PSBuild $params'!" + } + + if ($SkipReleaseChecks.IsPresent) { + Write-Warning "Skipping release checks." + } + elseif (!$Script:Options.RootInfo.IsValid){ + throw $Script:Options.RootInfo.Warning + } + + # If ReleaseTag is specified, use the given tag to calculate Version + if ($PSCmdlet.ParameterSetName -eq "ReleaseTag") { + $Version = $ReleaseTag -Replace '^v' + } + + # Use Git tag if not given a version + if (-not $Version) { + $Version = (git --git-dir="$RepoRoot/.git" describe) -Replace '^v' + } + + $Source = Split-Path -Path $Script:Options.Output -Parent + + # Copy the ThirdPartyNotices.txt so it's part of the package + Copy-Item "$RepoRoot/ThirdPartyNotices.txt" -Destination $Source -Force + + # Copy the default.help.txt so it's part of the package + Copy-Item "$RepoRoot/assets/default.help.txt" -Destination "$Source/en-US" -Force + + # If building a symbols package, we add a zip of the parent to publish + if ($IncludeSymbols.IsPresent) + { + $publishSource = $Source + $buildSource = Split-Path -Path $Source -Parent + $Source = New-TempFolder + $symbolsSource = New-TempFolder + + try + { + # Copy files which go into the root package + Get-ChildItem -Path $publishSource | Copy-Item -Destination $Source -Recurse + + $signingXml = [xml] (Get-Content (Join-Path $PSScriptRoot "..\releaseBuild\signing.xml" -Resolve)) + # Only include the files we sign for compliance scanning, those are the files we build. + $filesToInclude = $signingXml.SignConfigXML.job.file.src | Where-Object { -not $_.endswith('pwsh.exe') -and ($_.endswith(".dll") -or $_.endswith(".exe")) } | ForEach-Object { ($_ -split '\\')[-1] } + $filesToInclude += $filesToInclude | ForEach-Object { $_ -replace '.dll', '.pdb' } + Get-ChildItem -Path $buildSource | Where-Object { $_.Name -in $filesToInclude } | Copy-Item -Destination $symbolsSource -Recurse + + # Zip symbols.zip to the root package + $zipSource = Join-Path $symbolsSource -ChildPath '*' + $zipPath = Join-Path -Path $Source -ChildPath 'symbols.zip' + Save-PSOptions -PSOptionsPath (Join-Path -Path $source -ChildPath 'psoptions.json') -Options $Script:Options + Compress-Archive -Path $zipSource -DestinationPath $zipPath + } + finally + { + Remove-Item -Path $symbolsSource -Recurse -Force -ErrorAction SilentlyContinue + } + } + + Write-Log "Packaging Source: '$Source'" + + # Decide package output type + if (-not $Type) { + $Type = if ($Environment.IsLinux) { + if ($Environment.LinuxInfo.ID -match "ubuntu") { + "deb", "nupkg", "tar" + } elseif ($Environment.IsRedHatFamily) { + "rpm", "nupkg" + } elseif ($Environment.IsSUSEFamily) { + "rpm", "nupkg" + } else { + throw "Building packages for $($Environment.LinuxInfo.PRETTY_NAME) is unsupported!" + } + } elseif ($Environment.IsMacOS) { + "osxpkg", "nupkg", "tar" + } elseif ($Environment.IsWindows) { + "msi", "nupkg", "msix" + } + Write-Warning "-Type was not specified, continuing with $Type!" + } + Write-Log "Packaging Type: $Type" + + # Add the symbols to the suffix + # if symbols are specified to be included + if ($IncludeSymbols.IsPresent -and $NameSuffix) { + $NameSuffix = "symbols-$NameSuffix" + } + elseif ($IncludeSymbols.IsPresent) { + $NameSuffix = "symbols" + } + + switch ($Type) { + "zip" { + $Arguments = @{ + PackageNameSuffix = $NameSuffix + PackageSourcePath = $Source + PackageVersion = $Version + Force = $Force + } + + if ($PSCmdlet.ShouldProcess("Create Zip Package")) { + New-ZipPackage @Arguments + } + } + "zip-pdb" { + $Arguments = @{ + PackageNameSuffix = $NameSuffix + PackageSourcePath = $Source + PackageVersion = $Version + Force = $Force + } + + if ($PSCmdlet.ShouldProcess("Create Symbols Zip Package")) { + New-PdbZipPackage @Arguments + } + } + "min-size" { + # Remove symbol files, xml document files. + Remove-Item "$Source\*.pdb", "$Source\*.xml" -Force + + # Add suffix '-gc' because this package is for the Guest Config team. + if ($Environment.IsWindows) { + $Arguments = @{ + PackageNameSuffix = "$NameSuffix-gc" + PackageSourcePath = $Source + PackageVersion = $Version + Force = $Force + } + + if ($PSCmdlet.ShouldProcess("Create Zip Package")) { + New-ZipPackage @Arguments + } + } + elseif ($Environment.IsLinux) { + $Arguments = @{ + PackageSourcePath = $Source + Name = $Name + PackageNameSuffix = 'gc' + Version = $Version + Force = $Force + } + + if ($PSCmdlet.ShouldProcess("Create tar.gz Package")) { + New-TarballPackage @Arguments + } + } + } + { $_ -like "fxdependent*" } { + ## Remove PDBs from package to reduce size. + if(-not $IncludeSymbols.IsPresent) { + Get-ChildItem $Source -Filter *.pdb | Remove-Item -Force + } + + if ($Environment.IsWindows) { + $Arguments = @{ + PackageNameSuffix = $NameSuffix + PackageSourcePath = $Source + PackageVersion = $Version + Force = $Force + } + + if ($PSCmdlet.ShouldProcess("Create Zip Package")) { + New-ZipPackage @Arguments + } + } elseif ($Environment.IsLinux) { + $Arguments = @{ + PackageSourcePath = $Source + Name = $Name + PackageNameSuffix = 'fxdependent' + Version = $Version + Force = $Force + } + + if ($PSCmdlet.ShouldProcess("Create tar.gz Package")) { + New-TarballPackage @Arguments + } + } + } + "msi" { + $TargetArchitecture = "x64" + if ($Runtime -match "-x86") { + $TargetArchitecture = "x86" + } + Write-Verbose "TargetArchitecture = $TargetArchitecture" -Verbose + + $Arguments = @{ + ProductNameSuffix = $NameSuffix + ProductSourcePath = $Source + ProductVersion = $Version + AssetsPath = "$RepoRoot\assets" + ProductTargetArchitecture = $TargetArchitecture + Force = $Force + } + + if ($PSCmdlet.ShouldProcess("Create MSI Package")) { + New-MSIPackage @Arguments + } + } + "msix" { + $Arguments = @{ + ProductNameSuffix = $NameSuffix + ProductSourcePath = $Source + ProductVersion = $Version + Architecture = $WindowsRuntime.Split('-')[1] + Force = $Force + } + + if ($PSCmdlet.ShouldProcess("Create MSIX Package")) { + New-MSIXPackage @Arguments + } + } + 'nupkg' { + $Arguments = @{ + PackageNameSuffix = $NameSuffix + PackageSourcePath = $Source + PackageVersion = $Version + PackageRuntime = $Runtime + PackageConfiguration = $Configuration + Force = $Force + } + + if ($PSCmdlet.ShouldProcess("Create NuPkg Package")) { + New-NugetContentPackage @Arguments + } + } + "tar" { + $Arguments = @{ + PackageSourcePath = $Source + Name = $Name + Version = $Version + Force = $Force + } + + if ($MacOSRuntime) { + $Arguments['Architecture'] = $MacOSRuntime.Split('-')[1] + } + + if ($PSCmdlet.ShouldProcess("Create tar.gz Package")) { + New-TarballPackage @Arguments + } + } + "tar-arm" { + $Arguments = @{ + PackageSourcePath = $Source + Name = $Name + Version = $Version + Force = $Force + Architecture = "arm32" + ExcludeSymbolicLinks = $true + } + + if ($PSCmdlet.ShouldProcess("Create tar.gz Package")) { + New-TarballPackage @Arguments + } + } + "tar-arm64" { + $Arguments = @{ + PackageSourcePath = $Source + Name = $Name + Version = $Version + Force = $Force + Architecture = "arm64" + ExcludeSymbolicLinks = $true + } + + if ($PSCmdlet.ShouldProcess("Create tar.gz Package")) { + New-TarballPackage @Arguments + } + } + "tar-alpine" { + $Arguments = @{ + PackageSourcePath = $Source + Name = $Name + Version = $Version + Force = $Force + Architecture = "alpine-x64" + ExcludeSymbolicLinks = $true + } + + if ($PSCmdlet.ShouldProcess("Create tar.gz Package")) { + New-TarballPackage @Arguments + } + } + 'deb' { + $Arguments = @{ + Type = 'deb' + PackageSourcePath = $Source + Name = $Name + Version = $Version + Force = $Force + NoSudo = $NoSudo + LTS = $LTS + } + foreach ($Distro in $Script:DebianDistributions) { + $Arguments["Distribution"] = $Distro + if ($PSCmdlet.ShouldProcess("Create DEB Package for $Distro")) { + New-UnixPackage @Arguments + } + } + } + 'rpm' { + $Arguments = @{ + Type = 'rpm' + PackageSourcePath = $Source + Name = $Name + Version = $Version + Force = $Force + NoSudo = $NoSudo + LTS = $LTS + } + foreach ($Distro in $Script:RedhatDistributions) { + $Arguments["Distribution"] = $Distro + if ($PSCmdlet.ShouldProcess("Create RPM Package for $Distro")) { + New-UnixPackage @Arguments + } + } + } + default { + $Arguments = @{ + Type = $_ + PackageSourcePath = $Source + Name = $Name + Version = $Version + Force = $Force + NoSudo = $NoSudo + LTS = $LTS + } + + if ($PSCmdlet.ShouldProcess("Create $_ Package")) { + New-UnixPackage @Arguments + } + } + } + + if ($IncludeSymbols.IsPresent) + { + # Source is a temporary folder when -IncludeSymbols is present. So, we should remove it. + Remove-Item -Path $Source -Recurse -Force -ErrorAction SilentlyContinue + } + } +} + +function New-UnixPackage { + [CmdletBinding(SupportsShouldProcess=$true)] + param( + [Parameter(Mandatory)] + [ValidateSet("deb", "osxpkg", "rpm")] + [string]$Type, + + [Parameter(Mandatory)] + [string]$PackageSourcePath, + + # Must start with 'powershell' but may have any suffix + [Parameter(Mandatory)] + [ValidatePattern("^powershell")] + [string]$Name, + + [Parameter(Mandatory)] + [string]$Version, + + # Package iteration version (rarely changed) + # This is a string because strings are appended to it + [string]$Iteration = "1", + + [Switch] + $Force, + + [switch] + $NoSudo, + + [switch] + $LTS, + + [string] + $CurrentLocation = (Get-Location) + ) + + DynamicParam { + if ($Type -eq "deb" -or $Type -eq 'rpm') { + # Add a dynamic parameter '-Distribution' when the specified package type is 'deb'. + # The '-Distribution' parameter can be used to indicate which Debian distro this pacakge is targeting. + $ParameterAttr = New-Object "System.Management.Automation.ParameterAttribute" + if($type -eq 'deb') + { + $ValidateSetAttr = New-Object "System.Management.Automation.ValidateSetAttribute" -ArgumentList $Script:DebianDistributions + } + else + { + $ValidateSetAttr = New-Object "System.Management.Automation.ValidateSetAttribute" -ArgumentList $Script:RedHatDistributions + } + $Attributes = New-Object "System.Collections.ObjectModel.Collection``1[System.Attribute]" + $Attributes.Add($ParameterAttr) > $null + $Attributes.Add($ValidateSetAttr) > $null + + $Parameter = New-Object "System.Management.Automation.RuntimeDefinedParameter" -ArgumentList ("Distribution", [string], $Attributes) + $Dict = New-Object "System.Management.Automation.RuntimeDefinedParameterDictionary" + $Dict.Add("Distribution", $Parameter) > $null + return $Dict + } + } + + End { + # This allows sudo install to be optional; needed when running in containers / as root + # Note that when it is null, Invoke-Expression (but not &) must be used to interpolate properly + $sudo = if (!$NoSudo) { "sudo" } + + # Validate platform + $ErrorMessage = "Must be on {0} to build '$Type' packages!" + switch ($Type) { + "deb" { + $packageVersion = Get-LinuxPackageSemanticVersion -Version $Version + if (!$Environment.IsUbuntu -and !$Environment.IsDebian) { + throw ($ErrorMessage -f "Ubuntu or Debian") + } + + if ($PSBoundParameters.ContainsKey('Distribution')) { + $DebDistro = $PSBoundParameters['Distribution'] + } elseif ($Environment.IsUbuntu16) { + $DebDistro = "ubuntu.16.04" + } elseif ($Environment.IsUbuntu18) { + $DebDistro = "ubuntu.18.04" + } elseif ($Environment.IsUbuntu20) { + $DebDistro = "ubuntu.20.04" + } elseif ($Environment.IsDebian9) { + $DebDistro = "debian.9" + } else { + throw "The current Debian distribution is not supported." + } + + # iteration is "debian_revision" + # usage of this to differentiate distributions is allowed by non-standard + $Iteration += ".$DebDistro" + } + "rpm" { + if ($PSBoundParameters.ContainsKey('Distribution')) { + $DebDistro = $PSBoundParameters['Distribution'] + + } elseif ($Environment.IsRedHatFamily) { + $DebDistro = "rhel.7" + } else { + throw "The current distribution is not supported." + } + + $packageVersion = Get-LinuxPackageSemanticVersion -Version $Version + } + "osxpkg" { + $packageVersion = $Version + if (!$Environment.IsMacOS) { + throw ($ErrorMessage -f "macOS") + } + + $DebDistro = 'macOS' + } + } + + # Determine if the version is a preview version + $IsPreview = Test-IsPreview -Version $Version -IsLTS:$LTS + + # Preview versions have preview in the name + $Name = if($LTS) { + "powershell-lts" + } + elseif ($IsPreview) { + "powershell-preview" + } + else { + "powershell" + } + + # Verify dependencies are installed and in the path + Test-Dependencies + + $Description = $packagingStrings.Description + + # Break the version down into its components, we are interested in the major version + $VersionMatch = [regex]::Match($Version, '(\d+)(?:.(\d+)(?:.(\d+)(?:-preview(?:.(\d+))?)?)?)?') + $MajorVersion = $VersionMatch.Groups[1].Value + + # Suffix is used for side-by-side preview/release package installation + $Suffix = if ($IsPreview) { $MajorVersion + "-preview" } elseif ($LTS) { $MajorVersion + "-lts" } else { $MajorVersion } + + # Setup staging directory so we don't change the original source directory + $Staging = "$PSScriptRoot/staging" + if ($PSCmdlet.ShouldProcess("Create staging folder")) { + New-StagingFolder -StagingPath $Staging -PackageSourcePath $PackageSourcePath + } + + # Follow the Filesystem Hierarchy Standard for Linux and macOS + $Destination = if ($Environment.IsLinux) { + "/opt/microsoft/powershell/$Suffix" + } elseif ($Environment.IsMacOS) { + "/usr/local/microsoft/powershell/$Suffix" + } + + # Destination for symlink to powershell executable + $Link = Get-PwshExecutablePath -IsPreview:$IsPreview + $links = @(New-LinkInfo -LinkDestination $Link -LinkTarget "$Destination/pwsh") + + if($LTS) { + $links += New-LinkInfo -LinkDestination (Get-PwshExecutablePath -IsLTS:$LTS) -LinkTarget "$Destination/pwsh" + } + + if ($PSCmdlet.ShouldProcess("Create package file system")) + { + # Generate After Install and After Remove scripts + $AfterScriptInfo = New-AfterScripts -Link $Link -Distribution $DebDistro -Destination $Destination + + # there is a weird bug in fpm + # if the target of the powershell symlink exists, `fpm` aborts + # with a `utime` error on macOS. + # so we move it to make symlink broken + # refers to executable, does not vary by channel + $symlink_dest = "$Destination/pwsh" + $hack_dest = "./_fpm_symlink_hack_powershell" + if ($Environment.IsMacOS) { + if (Test-Path $symlink_dest) { + Write-Warning "Move $symlink_dest to $hack_dest (fpm utime bug)" + Start-NativeExecution ([ScriptBlock]::Create("$sudo mv $symlink_dest $hack_dest")) + } + } + + # Generate gzip of man file + $ManGzipInfo = New-ManGzip -IsPreview:$IsPreview -IsLTS:$LTS + + # Change permissions for packaging + Write-Log "Setting permissions..." + Start-NativeExecution { + find $Staging -type d | xargs chmod 755 + find $Staging -type f | xargs chmod 644 + chmod 644 $ManGzipInfo.GzipFile + # refers to executable, does not vary by channel + chmod 755 "$Staging/pwsh" #only the executable file should be granted the execution permission + } + } + + # Add macOS powershell launcher + if ($Type -eq "osxpkg") + { + Write-Log "Adding macOS launch application..." + if ($PSCmdlet.ShouldProcess("Add macOS launch application")) + { + # Generate launcher app folder + $AppsFolder = New-MacOSLauncher -Version $Version + } + } + + $packageDependenciesParams = @{} + if ($DebDistro) + { + $packageDependenciesParams['Distribution']=$DebDistro + } + + # Setup package dependencies + $Dependencies = @(Get-PackageDependencies @packageDependenciesParams) + + $Arguments = Get-FpmArguments ` + -Name $Name ` + -Version $packageVersion ` + -Iteration $Iteration ` + -Description $Description ` + -Type $Type ` + -Dependencies $Dependencies ` + -AfterInstallScript $AfterScriptInfo.AfterInstallScript ` + -AfterRemoveScript $AfterScriptInfo.AfterRemoveScript ` + -Staging $Staging ` + -Destination $Destination ` + -ManGzipFile $ManGzipInfo.GzipFile ` + -ManDestination $ManGzipInfo.ManFile ` + -LinkInfo $Links ` + -AppsFolder $AppsFolder ` + -Distribution $DebDistro ` + -ErrorAction Stop + + # Build package + try { + if ($PSCmdlet.ShouldProcess("Create $type package")) { + Write-Log "Creating package with fpm..." + $Output = Start-NativeExecution { fpm $Arguments } + } + } finally { + if ($Environment.IsMacOS) { + Write-Log "Starting Cleanup for mac packaging..." + if ($PSCmdlet.ShouldProcess("Cleanup macOS launcher")) + { + Clear-MacOSLauncher + } + + # this is continuation of a fpm hack for a weird bug + if (Test-Path $hack_dest) { + Write-Warning "Move $hack_dest to $symlink_dest (fpm utime bug)" + Start-NativeExecution -sb ([ScriptBlock]::Create("$sudo mv $hack_dest $symlink_dest")) -VerboseOutputOnError + } + } + if ($AfterScriptInfo.AfterInstallScript) { + Remove-Item -ErrorAction 'silentlycontinue' $AfterScriptInfo.AfterInstallScript -Force + } + if ($AfterScriptInfo.AfterRemoveScript) { + Remove-Item -ErrorAction 'silentlycontinue' $AfterScriptInfo.AfterRemoveScript -Force + } + Remove-Item -Path $ManGzipInfo.GzipFile -Force -ErrorAction SilentlyContinue + } + + # Magic to get path output + $createdPackage = Get-Item (Join-Path $CurrentLocation (($Output[-1] -split ":path=>")[-1] -replace '["{}]')) + + if ($Environment.IsMacOS) { + if ($PSCmdlet.ShouldProcess("Add distribution information and Fix PackageName")) + { + $createdPackage = New-MacOsDistributionPackage -FpmPackage $createdPackage -IsPreview:$IsPreview + } + } + + if (Test-Path $createdPackage) + { + Write-Verbose "Created package: $createdPackage" -Verbose + return $createdPackage + } + else + { + throw "Failed to create $createdPackage" + } + } +} diff --git a/test/perf/benchmarks/powershell-perf.csproj b/test/perf/benchmarks/powershell-perf.csproj index 78608c2e96f..93c164b98b8 100644 --- a/test/perf/benchmarks/powershell-perf.csproj +++ b/test/perf/benchmarks/powershell-perf.csproj @@ -46,6 +46,7 @@ + @@ -59,6 +60,8 @@ + + diff --git a/test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj b/test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj index 1383cfc1d62..f0c3a35115e 100644 --- a/test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj +++ b/test/perf/dotnet-tools/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj @@ -6,8 +6,8 @@ - - + + diff --git a/test/perf/dotnet-tools/ResultsComparer/Program.cs b/test/perf/dotnet-tools/ResultsComparer/Program.cs index 655c9f3458e..68615f61c78 100644 --- a/test/perf/dotnet-tools/ResultsComparer/Program.cs +++ b/test/perf/dotnet-tools/ResultsComparer/Program.cs @@ -20,7 +20,7 @@ namespace ResultsComparer { - public class Program + public sealed class Program { private const string FullBdnJsonFileExtension = "full.json"; diff --git a/tools/UpdateDotnetRuntime.ps1 b/tools/UpdateDotnetRuntime.ps1 index 5126d109999..37e2d43dec8 100644 --- a/tools/UpdateDotnetRuntime.ps1 +++ b/tools/UpdateDotnetRuntime.ps1 @@ -79,6 +79,7 @@ function Update-PackageVersion { "$PSScriptRoot/packaging/projects/reference/Microsoft.PowerShell.ConsoleHost/Microsoft.PowerShell.ConsoleHost.csproj" "$PSScriptRoot/../src/" "$PSScriptRoot/../test/tools/" + "$PSScriptRoot/../test/perf/dotnet-tools/" ) Get-ChildItem -Path $paths -Recurse -Filter "*.csproj" -Exclude 'PSGalleryModules.csproj', 'PSGalleryTestModules.csproj' | ForEach-Object { From ed249d0fe616f1da40b1d2f30804415c8f24efb6 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Tue, 14 Sep 2021 14:08:00 -0700 Subject: [PATCH 043/645] Remove support for AppExeCLinks to retrieve target (#16044) --- .../namespaces/FileSystemProvider.cs | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/src/System.Management.Automation/namespaces/FileSystemProvider.cs b/src/System.Management.Automation/namespaces/FileSystemProvider.cs index b7923ab7352..34c399f7bed 100644 --- a/src/System.Management.Automation/namespaces/FileSystemProvider.cs +++ b/src/System.Management.Automation/namespaces/FileSystemProvider.cs @@ -7993,18 +7993,6 @@ private struct REPARSE_DATA_BUFFER_MOUNTPOINT public byte[] PathBuffer; } - [StructLayout(LayoutKind.Sequential)] - private struct REPARSE_DATA_BUFFER_APPEXECLINK - { - public uint ReparseTag; - public ushort ReparseDataLength; - public ushort Reserved; - public uint StringCount; - - [MarshalAs(UnmanagedType.ByValArray, SizeConst = 0x3FF0)] - public byte[] StringList; - } - [StructLayout(LayoutKind.Sequential)] private struct BY_HANDLE_FILE_INFORMATION { @@ -8233,10 +8221,6 @@ private static string WinInternalGetLinkType(string filePath) linkType = "Junction"; break; - case IO_REPARSE_TAG_APPEXECLINK: - linkType = "AppExeCLink"; - break; - default: linkType = null; break; @@ -8515,16 +8499,6 @@ private static string WinInternalGetTarget(SafeFileHandle handle) targetDir = Encoding.Unicode.GetString(reparseMountPointDataBuffer.PathBuffer, reparseMountPointDataBuffer.SubstituteNameOffset, reparseMountPointDataBuffer.SubstituteNameLength); break; - case IO_REPARSE_TAG_APPEXECLINK: - REPARSE_DATA_BUFFER_APPEXECLINK reparseAppExeDataBuffer = Marshal.PtrToStructure(outBuffer); - // The target file is at index 2 - if (reparseAppExeDataBuffer.StringCount >= 3) - { - string temp = Encoding.Unicode.GetString(reparseAppExeDataBuffer.StringList); - targetDir = temp.Split('\0')[2]; - } - break; - default: return null; } From f1b122588b885eb89865728dca2f783e0c87a7fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Sep 2021 08:58:57 +0500 Subject: [PATCH 044/645] Bump Microsoft.CodeAnalysis.NetAnalyzers (#16070) Bumps [Microsoft.CodeAnalysis.NetAnalyzers](https://github.com/dotnet/roslyn-analyzers) from 6.0.0-rc2.21452.5 to 6.0.0-rc2.21458.5. - [Release notes](https://github.com/dotnet/roslyn-analyzers/releases) - [Changelog](https://github.com/dotnet/roslyn-analyzers/blob/main/PostReleaseActivities.md) - [Commits](https://github.com/dotnet/roslyn-analyzers/commits) --- updated-dependencies: - dependency-name: Microsoft.CodeAnalysis.NetAnalyzers dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Analyzers.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Analyzers.props b/Analyzers.props index ab1655e1527..b67dba7625b 100644 --- a/Analyzers.props +++ b/Analyzers.props @@ -1,7 +1,7 @@ - + From 4c958d8631433e96bfbfcc543b27ab3336e759fe Mon Sep 17 00:00:00 2001 From: Kellen Stuart <37430166+Kellen-Stuart@users.noreply.github.com> Date: Wed, 15 Sep 2021 09:32:42 -0600 Subject: [PATCH 045/645] Update build documentation to reflect .NET 6 (#15751) --- docs/building/windows-core.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/building/windows-core.md b/docs/building/windows-core.md index ec273d00391..65d76a37adc 100644 --- a/docs/building/windows-core.md +++ b/docs/building/windows-core.md @@ -59,11 +59,11 @@ Import-Module ./build.psm1 Start-PSBuild ``` -Congratulations! If everything went right, PowerShell is now built and executable as `./src/powershell-win-core/bin/Debug/net5.0/win7-x64/publish/pwsh.exe`. +Congratulations! If everything went right, PowerShell is now built and executable as `./src/powershell-win-core/bin/Debug/net6.0/win7-x64/publish/pwsh.exe`. This location is of the form `./[project]/bin/[configuration]/[framework]/[rid]/publish/[binary name]`, and our project is `powershell`, configuration is `Debug` by default, -framework is `net5.0`, runtime identifier is `win7-x64` by default, +framework is `net6.0`, runtime identifier is `win7-x64` by default, and binary name is `pwsh`. The function `Get-PSOutput` will return the path to the executable; thus you can execute the development copy via `& (Get-PSOutput)`. From fd3579badbe577f574d497b6571dff2d9f74a230 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Wed, 15 Sep 2021 10:38:26 -0700 Subject: [PATCH 046/645] Use `PlainText` when writing to a host that doesn't support VT (#16092) --- .../engine/hostifaces/MshHostUserInterface.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs b/src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs index cf2dea14a5f..ea101ade659 100644 --- a/src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs +++ b/src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs @@ -234,7 +234,7 @@ public virtual void WriteInformation(InformationRecord record) { } private static bool ShouldOutputPlainText(bool isHost, bool? supportsVirtualTerminal) { - var outputRendering = OutputRendering.Ansi; + var outputRendering = OutputRendering.PlainText; if (supportsVirtualTerminal != false) { From 6d2feeee388ff388da760de8d5f9c06739cb30a7 Mon Sep 17 00:00:00 2001 From: Sean Wheeler Date: Wed, 15 Sep 2021 17:42:13 -0500 Subject: [PATCH 047/645] Change 'snippet' tag to 'code' tag in XML comments (#16106) --- .../engine/MshCommandRuntime.cs | 24 ++++++++-------- .../engine/cmdlet.cs | 28 +++++++++---------- .../hostifaces/MshHostRawUserInterface.cs | 4 +-- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/System.Management.Automation/engine/MshCommandRuntime.cs b/src/System.Management.Automation/engine/MshCommandRuntime.cs index f76723d64e1..7b693ce0ed4 100644 --- a/src/System.Management.Automation/engine/MshCommandRuntime.cs +++ b/src/System.Management.Automation/engine/MshCommandRuntime.cs @@ -1063,7 +1063,7 @@ internal int OutBuffer /// , /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype1")] @@ -1086,7 +1086,7 @@ internal int OutBuffer /// } /// } /// } - /// + /// /// /// /// @@ -1157,7 +1157,7 @@ public bool ShouldProcess(string target) /// , /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype2")] @@ -1180,7 +1180,7 @@ public bool ShouldProcess(string target) /// } /// } /// } - /// + /// /// /// /// @@ -1260,7 +1260,7 @@ public bool ShouldProcess(string target, string action) /// , /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype3")] @@ -1286,7 +1286,7 @@ public bool ShouldProcess(string target, string action) /// } /// } /// } - /// + /// /// /// /// @@ -1375,7 +1375,7 @@ public bool ShouldProcess( /// , /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype3")] @@ -1403,7 +1403,7 @@ public bool ShouldProcess( /// } /// } /// } - /// + /// /// /// /// @@ -1681,7 +1681,7 @@ internal ShouldProcessPossibleOptimization CalculatePossibleShouldProcessOptimiz /// to ShouldProcess for the Cmdlet instance. /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype4")] @@ -1725,7 +1725,7 @@ internal ShouldProcessPossibleOptimization CalculatePossibleShouldProcessOptimiz /// } /// } /// } - /// + /// /// /// /// @@ -1863,7 +1863,7 @@ public bool ShouldContinue( /// to ShouldProcess for the Cmdlet instance. /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype4")] @@ -1912,7 +1912,7 @@ public bool ShouldContinue( /// } /// } /// } - /// + /// /// /// /// diff --git a/src/System.Management.Automation/engine/cmdlet.cs b/src/System.Management.Automation/engine/cmdlet.cs index ba7548f982d..953156873fa 100644 --- a/src/System.Management.Automation/engine/cmdlet.cs +++ b/src/System.Management.Automation/engine/cmdlet.cs @@ -801,7 +801,7 @@ public void WriteInformation(InformationRecord informationRecord) /// , /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype1")] @@ -824,7 +824,7 @@ public void WriteInformation(InformationRecord informationRecord) /// } /// } /// } - /// + /// /// /// /// @@ -897,7 +897,7 @@ public bool ShouldProcess(string target) /// , /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype2")] @@ -920,7 +920,7 @@ public bool ShouldProcess(string target) /// } /// } /// } - /// + /// /// /// /// @@ -1001,7 +1001,7 @@ public bool ShouldProcess(string target, string action) /// , /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype3")] @@ -1027,7 +1027,7 @@ public bool ShouldProcess(string target, string action) /// } /// } /// } - /// + /// /// /// /// @@ -1117,7 +1117,7 @@ public bool ShouldProcess( /// , /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype3")] @@ -1145,7 +1145,7 @@ public bool ShouldProcess( /// } /// } /// } - /// + /// /// /// /// @@ -1233,7 +1233,7 @@ public bool ShouldProcess( /// to ShouldProcess for the Cmdlet instance. /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype4")] @@ -1277,7 +1277,7 @@ public bool ShouldProcess( /// } /// } /// } - /// + /// /// /// /// @@ -1362,7 +1362,7 @@ public bool ShouldContinue(string query, string caption) /// to ShouldProcess for the Cmdlet instance. /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype4")] @@ -1411,7 +1411,7 @@ public bool ShouldContinue(string query, string caption) /// } /// } /// } - /// + /// /// /// /// @@ -1502,7 +1502,7 @@ public bool ShouldContinue( /// to ShouldProcess for the Cmdlet instance. /// /// - /// + /// /// namespace Microsoft.Samples.MSH.Cmdlet /// { /// [Cmdlet(VerbsCommon.Remove,"myobjecttype4")] @@ -1551,7 +1551,7 @@ public bool ShouldContinue( /// } /// } /// } - /// + /// /// /// /// diff --git a/src/System.Management.Automation/engine/hostifaces/MshHostRawUserInterface.cs b/src/System.Management.Automation/engine/hostifaces/MshHostRawUserInterface.cs index f4c3aedfb69..09d58ea7357 100644 --- a/src/System.Management.Automation/engine/hostifaces/MshHostRawUserInterface.cs +++ b/src/System.Management.Automation/engine/hostifaces/MshHostRawUserInterface.cs @@ -1458,7 +1458,7 @@ public abstract /// Provided for clearing regions -- less chatty than passing an array of cells. /// /// - /// + /// /// using System; /// using System.Management.Automation; /// using System.Management.Automation.Host; @@ -1474,7 +1474,7 @@ public abstract /// } /// } /// } - /// + /// /// /// /// From 49e21be29dbea81998484b2a6d7f6ff2cd6fef99 Mon Sep 17 00:00:00 2001 From: xtqqczze <45661989+xtqqczze@users.noreply.github.com> Date: Thu, 16 Sep 2021 04:51:48 +0100 Subject: [PATCH 048/645] Update more docs for `net6.0` TFM (#16102) --- docs/building/linux.md | 2 +- docs/building/macos.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/building/linux.md b/docs/building/linux.md index 7b42b04d342..fdf355fcde1 100644 --- a/docs/building/linux.md +++ b/docs/building/linux.md @@ -71,7 +71,7 @@ Start-PSBuild Congratulations! If everything went right, PowerShell is now built. The `Start-PSBuild` script will output the location of the executable: -`./src/powershell-unix/bin/Debug/net5.0/linux-x64/publish/pwsh`. +`./src/powershell-unix/bin/Debug/net6.0/linux-x64/publish/pwsh`. You should now be running the PowerShell Core that you just built, if you run the above executable. You can run our cross-platform Pester tests with `Start-PSPester`, and our xUnit tests with `Start-PSxUnit`. diff --git a/docs/building/macos.md b/docs/building/macos.md index e398451e8da..8898a917877 100644 --- a/docs/building/macos.md +++ b/docs/building/macos.md @@ -36,4 +36,4 @@ We cannot do this for you in the build module due to #[847][]. Start a PowerShell session by running `pwsh`, and then use `Start-PSBuild` from the module. -After building, PowerShell will be at `./src/powershell-unix/bin/Debug/net5.0/osx-x64/publish/pwsh`. +After building, PowerShell will be at `./src/powershell-unix/bin/Debug/net6.0/osx-x64/publish/pwsh`. From e81048b6ab9a9577b044c664d612ed1032c46297 Mon Sep 17 00:00:00 2001 From: Steve Lee Date: Fri, 17 Sep 2021 10:37:18 -0700 Subject: [PATCH 049/645] Remove Joey from Committee and WG membership (#16119) --- docs/community/governance.md | 1 - docs/community/working-group-definitions.md | 1 - 2 files changed, 2 deletions(-) diff --git a/docs/community/governance.md b/docs/community/governance.md index e97985a586b..dcad933fb36 100644 --- a/docs/community/governance.md +++ b/docs/community/governance.md @@ -27,7 +27,6 @@ The PowerShell Committee and its members (aka Committee Members) are the primary * Bruce Payette ([BrucePay](https://github.com/BrucePay)) * Jim Truher ([JamesWTruher](https://github.com/JamesWTruher)) -* Joey Aiello ([joeyaiello](https://github.com/joeyaiello)) * Paul Higinbotham ([paulhigin](https://github.com/paulhigin)) * Rob Holt ([rjmholt](https://github.com/rjmholt)) * Steve Lee ([SteveL-MSFT](https://github.com/SteveL-MSFT)) diff --git a/docs/community/working-group-definitions.md b/docs/community/working-group-definitions.md index b7e9cf323a7..f3efc89df3c 100644 --- a/docs/community/working-group-definitions.md +++ b/docs/community/working-group-definitions.md @@ -19,7 +19,6 @@ Today, DSC is integrated into the PowerShell language, and we need to manage it * @TravisEz13 * @theJasonHelmick -* @joeyaiello * @anmenaga ## Developer Experience From c4c312cc3198081d3452ed0cd0ed524e1f364feb Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Thu, 23 Sep 2021 07:04:14 -0700 Subject: [PATCH 050/645] Update `Microsoft.CodeAnalysis.CSharp` version (#16138) --- .../Microsoft.PowerShell.Commands.Utility.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj index df92bc5a746..ca77b060e53 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj +++ b/src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj @@ -31,7 +31,7 @@ - + From 6589eeb610b5d7b0d3d1de20d67cae32db10a014 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 23 Sep 2021 07:06:06 -0700 Subject: [PATCH 051/645] Change path for Component Governance for build to the path we actually use to build (#16137) --- .../releaseBuild/azureDevOps/templates/windows-hosted-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml index 2820654d167..8cc5831b8e5 100644 --- a/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml +++ b/tools/releaseBuild/azureDevOps/templates/windows-hosted-build.yml @@ -76,5 +76,5 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Detection' inputs: - sourceScanPath: '$(Build.SourcesDirectory)' + sourceScanPath: '$(PowerShellRoot)' snapshotForceEnabled: true From 9eb63debcd0d5f7f0cfe83b3d554bdda8fb05e61 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 23 Sep 2021 17:31:57 -0700 Subject: [PATCH 052/645] Add sha256 hashes to release (#16147) * Add sha256 hashes to release * Update tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml --- .../templates/release-CreateGitHubDraft.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml b/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml index 3b316e84cbf..4e3e8e998a4 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml @@ -16,6 +16,17 @@ steps: git clone https://$(AzureDevOpsPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools '$(Pipeline.Workspace)/tools' displayName: Clone Internal-Tools repository +- pwsh: | + $Path = "$(System.ArtifactsDirectory)" + $OutputPath = Join-Path $Path ‘hashes.sha256’ + $null = New-Item $OutputPath -ItemType File -Force + foreach ($file in Get-ChildItem -Path $Path -File ){ + Get-FileHash -Algorithm SHA256 -Path $file | + ForEach-Object { "$($_.Hash) *$($file.Name)" } | + Out-File -Path $OutputPath -Append + } + displayName: Add sha256 hashes + - pwsh: | Import-module '$(Pipeline.Workspace)/tools/Scripts/GitHubRelease.psm1' Publish-ReleaseDraft -Tag '$(ReleaseTag)' -Name '$(ReleaseTag) Release of PowerShell' -Description '<-- Update Me -->' -User PowerShell -Repository PowerShell -PackageFolder $(System.ArtifactsDirectory) -Token $(GitHubReleasePat) From 3ed7571f94705a81b861efca7854dbcd9c2225bd Mon Sep 17 00:00:00 2001 From: Andy Schwartzmeyer Date: Thu, 23 Sep 2021 22:28:57 -0700 Subject: [PATCH 053/645] Update Windows PowerShell issues link (#16105) --- .github/ISSUE_TEMPLATE/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 4e050986fa5..8471badfb0e 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ blank_issues_enabled: false contact_links: - name: Windows PowerShell - url: https://windowsserver.uservoice.com/forums/301869-powershell + url: https://support.microsoft.com/windows/send-feedback-to-microsoft-with-the-feedback-hub-app-f59187f8-8739-22d6-ba93-f66612949332 about: Windows PowerShell issues or suggestions. - name: Support url: https://github.com/PowerShell/PowerShell/blob/master/.github/SUPPORT.md From 7f36609cf462139481ff06c55d9dd35804de2d12 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 28 Sep 2021 16:58:26 -0700 Subject: [PATCH 054/645] fix issue with hash file getting created before we have finished get-childitem (#16170) --- .../templates/release-CreateGitHubDraft.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml b/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml index 4e3e8e998a4..6e968cca572 100644 --- a/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml +++ b/tools/releaseBuild/azureDevOps/templates/release-CreateGitHubDraft.yml @@ -19,12 +19,19 @@ steps: - pwsh: | $Path = "$(System.ArtifactsDirectory)" $OutputPath = Join-Path $Path ‘hashes.sha256’ - $null = New-Item $OutputPath -ItemType File -Force - foreach ($file in Get-ChildItem -Path $Path -File ){ - Get-FileHash -Algorithm SHA256 -Path $file | - ForEach-Object { "$($_.Hash) *$($file.Name)" } | - Out-File -Path $OutputPath -Append - } + $srcPaths = @($Path) + $packages = Get-ChildItem -Path $srcPaths -Include * -Recurse + $checksums = $packages | + ForEach-Object { + Write-Verbose -Verbose "Generating checksum file for $($_.FullName)" + $packageName = $_.Name + $hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash.ToLower() + # the '*' before the packagename signifies it is a binary + "$hash *$packageName" + } + $checksums | Out-File -FilePath $OutputPath -Force + $fileContent = Get-Content -Path $OutputPath -Raw | Out-String + Write-Verbose -Verbose -Message $fileContent displayName: Add sha256 hashes - pwsh: | From f4ee1d7f12a2516273dd7d7e9c7b1654ab0fec39 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Tue, 28 Sep 2021 17:13:11 -0700 Subject: [PATCH 055/645] Update `README.md` and `metadata.json` for next preview release (#16107) --- README.md | 28 ++++++++++++++-------------- tools/metadata.json | 4 ++-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 7f83f92f6bc..d0e918001a8 100644 --- a/README.md +++ b/README.md @@ -94,20 +94,20 @@ You can also download the PowerShell binary archives for Windows, macOS and Linu [rl-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.1.4/powershell-7.1.4-linux-arm64.tar.gz [rl-snap]: https://snapcraft.io/powershell -[pv-windows-64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/PowerShell-7.2.0-preview.9-win-x64.msi -[pv-windows-86]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/PowerShell-7.2.0-preview.9-win-x86.msi -[pv-deb]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-preview_7.2.0-preview.9-1.deb_amd64.deb -[pv-rpm]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-preview-7.2.0_preview.9-1.rh.x86_64.rpm -[pv-macos]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-osx-x64.pkg -[pv-macos-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-osx-arm64.pkg -[pv-winarm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/PowerShell-7.2.0-preview.9-win-arm64.zip -[pv-winx86-zip]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/PowerShell-7.2.0-preview.9-win-x86.zip -[pv-winx64-zip]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/PowerShell-7.2.0-preview.9-win-x64.zip -[pv-macos-tar]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-osx-x64.tar.gz -[pv-macos-tar-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-osx-arm64.tar.gz -[pv-linux-tar]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-linux-x64.tar.gz -[pv-arm32]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-linux-arm32.tar.gz -[pv-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.9/powershell-7.2.0-preview.9-linux-arm64.tar.gz +[pv-windows-64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/PowerShell-7.2.0-preview.10-win-x64.msi +[pv-windows-86]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/PowerShell-7.2.0-preview.10-win-x86.msi +[pv-deb]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/powershell-preview_7.2.0-preview.10-1.deb_amd64.deb +[pv-rpm]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/powershell-preview-7.2.0_preview.10-1.rh.x86_64.rpm +[pv-macos]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/powershell-7.2.0-preview.10-osx-x64.pkg +[pv-macos-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/powershell-7.2.0-preview.10-osx-arm64.pkg +[pv-winarm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/PowerShell-7.2.0-preview.10-win-arm64.zip +[pv-winx86-zip]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/PowerShell-7.2.0-preview.10-win-x86.zip +[pv-winx64-zip]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/PowerShell-7.2.0-preview.10-win-x64.zip +[pv-macos-tar]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/powershell-7.2.0-preview.10-osx-x64.tar.gz +[pv-macos-tar-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/powershell-7.2.0-preview.10-osx-arm64.tar.gz +[pv-linux-tar]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/powershell-7.2.0-preview.10-linux-x64.tar.gz +[pv-arm32]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/powershell-7.2.0-preview.10-linux-arm32.tar.gz +[pv-arm64]: https://github.com/PowerShell/PowerShell/releases/download/v7.2.0-preview.10/powershell-7.2.0-preview.10-linux-arm64.tar.gz [pv-snap]: https://snapcraft.io/powershell-preview [in-windows]: https://docs.microsoft.com/powershell/scripting/install/installing-powershell-core-on-windows diff --git a/tools/metadata.json b/tools/metadata.json index 88db98aa613..52b88af8a37 100644 --- a/tools/metadata.json +++ b/tools/metadata.json @@ -1,9 +1,9 @@ { "StableReleaseTag": "v7.1.4", - "PreviewReleaseTag": "v7.2.0-preview.9", + "PreviewReleaseTag": "v7.2.0-preview.10", "ServicingReleaseTag": "v7.0.7", "ReleaseTag": "v7.1.4", "LTSReleaseTag" : ["v7.0.7"], - "NextReleaseTag": "v7.2.0-preview.10", + "NextReleaseTag": "v7.2.0-preview.11", "LTSRelease": false } From 4336ebaf29bc85e0ac96aee0dfd2dd10c2a248f9 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sun, 3 Oct 2021 13:35:19 -0700 Subject: [PATCH 056/645] Fix Microsoft update spelling issue. (#16178) --- assets/wix/Product.wxs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/wix/Product.wxs b/assets/wix/Product.wxs index da25dee7b04..3691f5268ca 100644 --- a/assets/wix/Product.wxs +++ b/assets/wix/Product.wxs @@ -350,7 +350,7 @@ - + From 6b42f445eded6daefc9fa3d6826b29022e94038d Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Sun, 3 Oct 2021 13:36:55 -0700 Subject: [PATCH 057/645] Move vPack build to 1ES Pool (#16169) --- tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml index c7ef645a878..8cc31ab8df9 100644 --- a/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml +++ b/tools/releaseBuild/azureDevOps/templates/vpackReleaseJob.yml @@ -5,7 +5,7 @@ jobs: - job: vpack_${{ parameters.architecture }} displayName: Build and Publish VPack - ${{ parameters.architecture }} condition: succeeded() - pool: Package ES Standard Build + pool: PowerShell1ES steps: - checkout: self clean: true From cfe45defc1ed87a7a7b916f154e56124d81a320b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 3 Oct 2021 13:48:25 -0700 Subject: [PATCH 058/645] Bump `Microsoft.CodeAnalysis.NetAnalyzers` from `6.0.0-rc2.21458.5` to `6.0.0-rtm.21480.8` (#16183) Bumps [Microsoft.CodeAnalysis.NetAnalyzers](https://github.com/dotnet/roslyn-analyzers) from 6.0.0-rc2.21458.5 to 6.0.0-rtm.21480.8. - [Release notes](https://github.com/dotnet/roslyn-analyzers/releases) - [Changelog](https://github.com/dotnet/roslyn-analyzers/blob/main/PostReleaseActivities.md) - [Commits](https://github.com/dotnet/roslyn-analyzers/commits) --- updated-dependencies: - dependency-name: Microsoft.CodeAnalysis.NetAnalyzers dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Analyzers.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Analyzers.props b/Analyzers.props index b67dba7625b..770043a87e2 100644 --- a/Analyzers.props +++ b/Analyzers.props @@ -1,7 +1,7 @@ - + From 4c84665bc7287be8b0d04652f44481e614a0b061 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Oct 2021 10:28:10 -0700 Subject: [PATCH 059/645] Bump Microsoft.CodeAnalysis.NetAnalyzers (#16194) Bumps [Microsoft.CodeAnalysis.NetAnalyzers](https://github.com/dotnet/roslyn-analyzers) from 6.0.0-rtm.21480.8 to 6.0.0-rtm.21504.2. - [Release notes](https://github.com/dotnet/roslyn-analyzers/releases) - [Changelog](https://github.com/dotnet/roslyn-analyzers/blob/main/PostReleaseActivities.md) - [Commits](https://github.com/dotnet/roslyn-analyzers/commits) --- updated-dependencies: - dependency-name: Microsoft.CodeAnalysis.NetAnalyzers dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Analyzers.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Analyzers.props b/Analyzers.props index 770043a87e2..4ea5f3e0618 100644 --- a/Analyzers.props +++ b/Analyzers.props @@ -1,7 +1,7 @@ - + From 2f57bf848b03828ee6c343b55f7ce80df2e5a23e Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Tue, 5 Oct 2021 10:29:16 -0700 Subject: [PATCH 060/645] Change `Target` from a `CodeProperty` to be an `AliasProperty` that points to `FileSystemInfo.LinkTarget` (#16165) --- .../host/msh/ConsoleHost.cs | 2 +- .../CoreCLR/CorePsPlatform.cs | 5 - .../engine/NativeCommandProcessor.cs | 20 +- .../engine/TypeTable_Types_Ps1Xml.cs | 10 +- .../namespaces/FileSystemProvider.cs | 227 +++++------------- .../FileSystem.Tests.ps1 | 36 +++ 6 files changed, 108 insertions(+), 192 deletions(-) diff --git a/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs b/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs index b0ae4735026..c2206ac7de8 100644 --- a/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs +++ b/src/Microsoft.PowerShell.ConsoleHost/host/msh/ConsoleHost.cs @@ -432,7 +432,7 @@ private static void SpinUpBreakHandlerThread(bool shouldEndSession) host.ShouldEndSession = shouldEndSession; } - // Creation of the tread and starting it should be an atomic operation. + // Creation of the thread and starting it should be an atomic operation. // otherwise the code in Run method can get instance of the breakhandlerThread // after it is created and before started and call join on it. This will result // in ThreadStateException. diff --git a/src/System.Management.Automation/CoreCLR/CorePsPlatform.cs b/src/System.Management.Automation/CoreCLR/CorePsPlatform.cs index 884e91e3356..ce37ae12d17 100644 --- a/src/System.Management.Automation/CoreCLR/CorePsPlatform.cs +++ b/src/System.Management.Automation/CoreCLR/CorePsPlatform.cs @@ -516,11 +516,6 @@ internal static bool NonWindowsIsHardLink(FileSystemInfo fileInfo) return Unix.IsHardLink(fileInfo); } - internal static string NonWindowsInternalGetTarget(string path) - { - return Unix.NativeMethods.FollowSymLink(path); - } - internal static string NonWindowsGetUserFromPid(int path) { return Unix.NativeMethods.GetUserFromPid(path); diff --git a/src/System.Management.Automation/engine/NativeCommandProcessor.cs b/src/System.Management.Automation/engine/NativeCommandProcessor.cs index 7df3dd82546..2c5efb35668 100644 --- a/src/System.Management.Automation/engine/NativeCommandProcessor.cs +++ b/src/System.Management.Automation/engine/NativeCommandProcessor.cs @@ -1101,15 +1101,21 @@ private static bool IsWindowsApplication(string fileName) #if UNIX return false; #else - if (!Platform.IsWindowsDesktop) { return false; } + if (!Platform.IsWindowsDesktop) + { + return false; + } - // SHGetFileInfo() does not understand reparse points and returns 0 ("non exe or error") - // so we are trying to get a real path before. - // It is a workaround for Microsoft Store applications. - string realPath = Microsoft.PowerShell.Commands.InternalSymbolicLinkLinkCodeMethods.WinInternalGetTarget(fileName); - if (realPath is not null) + // The function 'SHGetFileInfo()' does not understand reparse points and returns 0 ("non exe or error") + // for a symbolic link file, so we try to get the immediate link target in that case. + // Why not get the final target (use 'returnFinalTarget: true')? Because: + // 1. When starting a process on Windows, if the 'FileName' is a symbolic link, the immediate link target will automatically be used, + // but the OS does not do recursive resolution when the immediate link target is also a symbolic link. + // 2. Keep the same behavior as before adopting the 'LinkTarget' and 'ResolveLinkTarget' APIs in .NET 6. + string linkTarget = File.ResolveLinkTarget(fileName, returnFinalTarget: false)?.FullName; + if (linkTarget is not null) { - fileName = realPath; + fileName = linkTarget; } SHFILEINFO shinfo = new SHFILEINFO(); diff --git a/src/System.Management.Automation/engine/TypeTable_Types_Ps1Xml.cs b/src/System.Management.Automation/engine/TypeTable_Types_Ps1Xml.cs index 7113c0953cb..648bafe152c 100644 --- a/src/System.Management.Automation/engine/TypeTable_Types_Ps1Xml.cs +++ b/src/System.Management.Automation/engine/TypeTable_Types_Ps1Xml.cs @@ -680,10 +680,7 @@ private void Process_Types_Ps1Xml(string filePath, ConcurrentBag errors) AddMember( errors, typeName, - new PSCodeProperty( - @"Target", - GetMethodInfo(typeof(Microsoft.PowerShell.Commands.InternalSymbolicLinkLinkCodeMethods), @"GetTarget"), - setterCodeReference: null), + new PSAliasProperty(@"Target", @"LinkTarget", conversionType: null), typeMembers, isOverride: false); @@ -808,10 +805,7 @@ private void Process_Types_Ps1Xml(string filePath, ConcurrentBag errors) AddMember( errors, typeName, - new PSCodeProperty( - @"Target", - GetMethodInfo(typeof(Microsoft.PowerShell.Commands.InternalSymbolicLinkLinkCodeMethods), @"GetTarget"), - setterCodeReference: null), + new PSAliasProperty(@"Target", @"LinkTarget", conversionType: null), typeMembers, isOverride: false); diff --git a/src/System.Management.Automation/namespaces/FileSystemProvider.cs b/src/System.Management.Automation/namespaces/FileSystemProvider.cs index 34c399f7bed..7e6d302b22a 100644 --- a/src/System.Management.Automation/namespaces/FileSystemProvider.cs +++ b/src/System.Management.Automation/namespaces/FileSystemProvider.cs @@ -2069,7 +2069,7 @@ public static string NameString(PSObject instance) { if (InternalSymbolicLinkLinkCodeMethods.IsReparsePointLikeSymlink(fileInfo)) { - return $"{PSStyle.Instance.FileInfo.SymbolicLink}{fileInfo.Name}{PSStyle.Instance.Reset} -> {InternalSymbolicLinkLinkCodeMethods.GetTarget(instance)}"; + return $"{PSStyle.Instance.FileInfo.SymbolicLink}{fileInfo.Name}{PSStyle.Instance.Reset} -> {fileInfo.LinkTarget}"; } else if (fileInfo.Attributes.HasFlag(FileAttributes.Directory)) { @@ -2096,7 +2096,7 @@ public static string NameString(PSObject instance) { return instance?.BaseObject is FileSystemInfo fileInfo ? InternalSymbolicLinkLinkCodeMethods.IsReparsePointLikeSymlink(fileInfo) - ? $"{fileInfo.Name} -> {InternalSymbolicLinkLinkCodeMethods.GetTarget(instance)}" + ? $"{fileInfo.Name} -> {fileInfo.LinkTarget}" : fileInfo.Name : string.Empty; } @@ -8105,15 +8105,18 @@ internal unsafe struct WIN32_FIND_DATA /// /// The object of FileInfo or DirectoryInfo type. /// The target of the reparse point. + [Obsolete("This method is now obsolete. Please use the .NET API 'FileSystemInfo.LinkTarget'", error: true)] public static string GetTarget(PSObject instance) { if (instance.BaseObject is FileSystemInfo fileSysInfo) { -#if !UNIX - return WinInternalGetTarget(fileSysInfo.FullName); -#else - return UnixInternalGetTarget(fileSysInfo.FullName); -#endif + if (!fileSysInfo.Exists) + { + throw new ArgumentException( + StringUtil.Format(SessionStateStrings.PathNotFound, fileSysInfo.FullName)); + } + + return fileSysInfo.LinkTarget; } return null; @@ -8136,20 +8139,6 @@ public static string GetLinkType(PSObject instance) return null; } -#if UNIX - private static string UnixInternalGetTarget(string filePath) - { - string link = Platform.NonWindowsInternalGetTarget(filePath); - - if (string.IsNullOrEmpty(link)) - { - throw new Win32Exception(Marshal.GetLastWin32Error()); - } - - return link; - } -#endif - private static string InternalGetLinkType(FileSystemInfo fileInfo) { if (Platform.IsWindows) @@ -8165,16 +8154,11 @@ private static string InternalGetLinkType(FileSystemInfo fileInfo) [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2001:AvoidCallingProblematicMethods")] private static string WinInternalGetLinkType(string filePath) { - if (!Platform.IsWindows) - { - throw new PlatformNotSupportedException(); - } - // We set accessMode parameter to zero because documentation says: // If this parameter is zero, the application can query certain metadata // such as file, directory, or device attributes without accessing // that file or device, even if GENERIC_READ access would have been denied. - using (SafeFileHandle handle = OpenReparsePoint(filePath, FileDesiredAccess.GenericZero)) + using (SafeFileHandle handle = WinOpenReparsePoint(filePath, FileDesiredAccess.GenericZero)) { int outBufferSize = Marshal.SizeOf(); @@ -8439,176 +8423,77 @@ internal static bool WinIsHardLink(ref IntPtr handle) return succeeded && (handleInfo.NumberOfLinks > 1); } -#if !UNIX - internal static string WinInternalGetTarget(string path) - { - // We set accessMode parameter to zero because documentation says: - // If this parameter is zero, the application can query certain metadata - // such as file, directory, or device attributes without accessing - // that file or device, even if GENERIC_READ access would have been denied. - using (SafeFileHandle handle = OpenReparsePoint(path, FileDesiredAccess.GenericZero)) - { - return WinInternalGetTarget(handle); - } - } - - [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2001:AvoidCallingProblematicMethods")] - private static string WinInternalGetTarget(SafeFileHandle handle) - { - int outBufferSize = Marshal.SizeOf(); - - IntPtr outBuffer = Marshal.AllocHGlobal(outBufferSize); - bool success = false; - - try - { - int bytesReturned; - - // OACR warning 62001 about using DeviceIOControl has been disabled. - // According to MSDN guidance DangerousAddRef() and DangerousRelease() have been used. - handle.DangerousAddRef(ref success); - - bool result = DeviceIoControl( - handle.DangerousGetHandle(), - FSCTL_GET_REPARSE_POINT, - InBuffer: IntPtr.Zero, - nInBufferSize: 0, - outBuffer, - outBufferSize, - out bytesReturned, - lpOverlapped: IntPtr.Zero); - - if (!result) - { - // It's not a reparse point or the file system doesn't support reparse points. - return null; - } - - string targetDir = null; - - REPARSE_DATA_BUFFER_SYMBOLICLINK reparseDataBuffer = Marshal.PtrToStructure(outBuffer); - - switch (reparseDataBuffer.ReparseTag) - { - case IO_REPARSE_TAG_SYMLINK: - targetDir = Encoding.Unicode.GetString(reparseDataBuffer.PathBuffer, reparseDataBuffer.SubstituteNameOffset, reparseDataBuffer.SubstituteNameLength); - break; - - case IO_REPARSE_TAG_MOUNT_POINT: - REPARSE_DATA_BUFFER_MOUNTPOINT reparseMountPointDataBuffer = Marshal.PtrToStructure(outBuffer); - targetDir = Encoding.Unicode.GetString(reparseMountPointDataBuffer.PathBuffer, reparseMountPointDataBuffer.SubstituteNameOffset, reparseMountPointDataBuffer.SubstituteNameLength); - break; - - default: - return null; - } - - if (targetDir != null && targetDir.StartsWith(NonInterpretedPathPrefix, StringComparison.OrdinalIgnoreCase)) - { - targetDir = targetDir.Substring(NonInterpretedPathPrefix.Length); - } - - return targetDir; - } - finally - { - if (success) - { - handle.DangerousRelease(); - } - - Marshal.FreeHGlobal(outBuffer); - } - } -#endif - internal static bool CreateJunction(string path, string target) { - // this is a purely Windows specific feature, no feature flag - // used for that reason + // this is a purely Windows specific feature, no feature flag used for that reason. if (Platform.IsWindows) { return WinCreateJunction(path, target); } - else - { - return false; - } + + return false; } [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2001:AvoidCallingProblematicMethods")] private static bool WinCreateJunction(string path, string target) { - if (!string.IsNullOrEmpty(path)) + if (string.IsNullOrEmpty(path)) { - if (!string.IsNullOrEmpty(target)) - { - using (SafeHandle handle = OpenReparsePoint(path, FileDesiredAccess.GenericWrite)) - { - byte[] mountPointBytes = Encoding.Unicode.GetBytes(NonInterpretedPathPrefix + Path.GetFullPath(target)); + throw new ArgumentNullException(nameof(path)); + } - REPARSE_DATA_BUFFER_MOUNTPOINT mountPoint = new REPARSE_DATA_BUFFER_MOUNTPOINT(); - mountPoint.ReparseTag = IO_REPARSE_TAG_MOUNT_POINT; - mountPoint.ReparseDataLength = (ushort)(mountPointBytes.Length + 12); // Added space for the header and null endo - mountPoint.SubstituteNameOffset = 0; - mountPoint.SubstituteNameLength = (ushort)mountPointBytes.Length; - mountPoint.PrintNameOffset = (ushort)(mountPointBytes.Length + 2); // 2 as unicode null take 2 bytes. - mountPoint.PrintNameLength = 0; - mountPoint.PathBuffer = new byte[0x3FF0]; // Buffer for max size. - Array.Copy(mountPointBytes, mountPoint.PathBuffer, mountPointBytes.Length); + if (string.IsNullOrEmpty(target)) + { + throw new ArgumentNullException(nameof(target)); + } - int nativeBufferSize = Marshal.SizeOf(mountPoint); - IntPtr nativeBuffer = Marshal.AllocHGlobal(nativeBufferSize); - bool success = false; + using (SafeHandle handle = WinOpenReparsePoint(path, FileDesiredAccess.GenericWrite)) + { + byte[] mountPointBytes = Encoding.Unicode.GetBytes(NonInterpretedPathPrefix + Path.GetFullPath(target)); - try - { - Marshal.StructureToPtr(mountPoint, nativeBuffer, false); + var mountPoint = new REPARSE_DATA_BUFFER_MOUNTPOINT(); + mountPoint.ReparseTag = IO_REPARSE_TAG_MOUNT_POINT; + mountPoint.ReparseDataLength = (ushort)(mountPointBytes.Length + 12); // Added space for the header and null endo + mountPoint.SubstituteNameOffset = 0; + mountPoint.SubstituteNameLength = (ushort)mountPointBytes.Length; + mountPoint.PrintNameOffset = (ushort)(mountPointBytes.Length + 2); // 2 as unicode null take 2 bytes. + mountPoint.PrintNameLength = 0; + mountPoint.PathBuffer = new byte[0x3FF0]; // Buffer for max size. + Array.Copy(mountPointBytes, mountPoint.PathBuffer, mountPointBytes.Length); - int bytesReturned = 0; + int nativeBufferSize = Marshal.SizeOf(mountPoint); + IntPtr nativeBuffer = Marshal.AllocHGlobal(nativeBufferSize); + bool success = false; - // OACR warning 62001 about using DeviceIOControl has been disabled. - // According to MSDN guidance DangerousAddRef() and DangerousRelease() have been used. - handle.DangerousAddRef(ref success); + try + { + Marshal.StructureToPtr(mountPoint, nativeBuffer, false); - bool result = DeviceIoControl(handle.DangerousGetHandle(), FSCTL_SET_REPARSE_POINT, nativeBuffer, mountPointBytes.Length + 20, IntPtr.Zero, 0, out bytesReturned, IntPtr.Zero); + int bytesReturned = 0; - if (!result) - { - throw new Win32Exception(Marshal.GetLastWin32Error()); - } + // OACR warning 62001 about using DeviceIOControl has been disabled. + // According to MSDN guidance DangerousAddRef() and DangerousRelease() have been used. + handle.DangerousAddRef(ref success); - return result; - } - finally - { - Marshal.FreeHGlobal(nativeBuffer); + bool result = DeviceIoControl(handle.DangerousGetHandle(), FSCTL_SET_REPARSE_POINT, nativeBuffer, mountPointBytes.Length + 20, IntPtr.Zero, 0, out bytesReturned, IntPtr.Zero); - if (success) - { - handle.DangerousRelease(); - } - } + if (!result) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); } + + return result; } - else + finally { - throw new ArgumentNullException(nameof(target)); + Marshal.FreeHGlobal(nativeBuffer); + + if (success) + { + handle.DangerousRelease(); + } } } - else - { - throw new ArgumentNullException(nameof(path)); - } - } - - private static SafeFileHandle OpenReparsePoint(string reparsePoint, FileDesiredAccess accessMode) - { -#if UNIX - throw new PlatformNotSupportedException(); -#else - return WinOpenReparsePoint(reparsePoint, accessMode); -#endif } private static SafeFileHandle WinOpenReparsePoint(string reparsePoint, FileDesiredAccess accessMode) diff --git a/test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1 index e5d150013cd..10ec92b6be6 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Management/FileSystem.Tests.ps1 @@ -773,6 +773,42 @@ Describe "Hard link and symbolic link tests" -Tags "CI", "RequireAdminOnWindows" $childB.Name | Should -BeExactly $childA.Name } } + + Context "Show immediate target" { + BeforeAll { + $testDir = Join-Path $TestDrive "immediate-target" + New-Item -ItemType Directory $testDir > $null + + $testFile = Join-Path $testDir "target" + Set-Content -Path $testFile -Value "Hello world" + + Push-Location $testDir + New-Item -ItemType SymbolicLink -Path 'firstLink' -Value 'target' > $null + New-Item -ItemType SymbolicLink -Path 'secondLink' -Value 'firstLink' > $null + Pop-Location + } + + AfterAll { + Remove-Item $testDir -Recurse -Force + } + + It "Property 'Target' should show the immediate target" { + $firstLink = Get-Item (Join-Path $testDir 'firstLink') + $firstLink.Target | Should -BeExactly 'target' + $str = [Microsoft.PowerShell.Commands.FileSystemProvider]::NameString($firstLink) + [System.Management.Automation.Internal.StringDecorated]::new($str).ToString([System.Management.Automation.OutputRendering]::PlainText) | Should -BeExactly 'firstLink -> target' + + $secondLink = Get-Item (Join-Path $testDir 'secondLink') + $secondLink.Target | Should -BeExactly 'firstLink' + $str = [Microsoft.PowerShell.Commands.FileSystemProvider]::NameString($secondLink) + [System.Management.Automation.Internal.StringDecorated]::new($str).ToString([System.Management.Automation.OutputRendering]::PlainText) | Should -BeExactly 'secondLink -> firstLink' + } + + It "Get-Content should be able to resolve the final target" { + Get-Content (Join-Path $testDir 'firstLink') | Should -BeExactly "Hello world" + Get-Content (Join-Path $testDir 'secondLink') | Should -BeExactly "Hello world" + } + } } Describe "Copy-Item can avoid copying an item onto itself" -Tags "CI", "RequireAdminOnWindows" { From 53ac646cc0f23e2e150f23cabf98ab4c85b5b0db Mon Sep 17 00:00:00 2001 From: dwtaber <38249738+dwtaber@users.noreply.github.com> Date: Wed, 6 Oct 2021 13:43:56 -0400 Subject: [PATCH 061/645] Invoke-Command: improve handling of variables with $using: expression (#16113) --- src/System.Management.Automation/engine/parser/ast.cs | 6 ++++-- .../engine/remoting/commands/PSRemotingCmdlet.cs | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/ast.cs b/src/System.Management.Automation/engine/parser/ast.cs index 3e1f9ce60a5..1f03eb68df5 100644 --- a/src/System.Management.Automation/engine/parser/ast.cs +++ b/src/System.Management.Automation/engine/parser/ast.cs @@ -1261,7 +1261,8 @@ internal string ToStringForSerialization(Tuple, stri var varAst = ast as VariableExpressionAst; if (varAst != null) { - string varName = varAst.VariablePath.UserPath; + VariablePath varPath = varAst.VariablePath; + string varName = varPath.IsDriveQualified ? $"{varPath.DriveName}_{varPath.UnqualifiedPath}" : $"{varPath.UnqualifiedPath}"; string varSign = varAst.Splatted ? "@" : "$"; string newVarName = varSign + UsingExpressionAst.UsingPrefix + varName; @@ -2329,7 +2330,8 @@ internal string GetParamTextWithDollarUsingHandling(IEnumerator= endOffset) { break; } - string varName = varAst.VariablePath.UserPath; + VariablePath varPath = varAst.VariablePath; + string varName = varPath.IsDriveQualified ? $"{varPath.DriveName}_{varPath.UnqualifiedPath}" : $"{varPath.UnqualifiedPath}"; string varSign = varAst.Splatted ? "@" : "$"; string newVarName = varSign + UsingExpressionAst.UsingPrefix + varName; diff --git a/src/System.Management.Automation/engine/remoting/commands/PSRemotingCmdlet.cs b/src/System.Management.Automation/engine/remoting/commands/PSRemotingCmdlet.cs index d4f5ca427ab..67d0ced9f15 100644 --- a/src/System.Management.Automation/engine/remoting/commands/PSRemotingCmdlet.cs +++ b/src/System.Management.Automation/engine/remoting/commands/PSRemotingCmdlet.cs @@ -2374,7 +2374,8 @@ private string GetConvertedScript(out List newParameterNames, out List Date: Thu, 7 Oct 2021 10:26:21 -0700 Subject: [PATCH 062/645] Fix '-PipelineVariable' to set variable in the right scope (#16199) --- .../engine/MshCommandRuntime.cs | 52 +++++++++++-------- .../engine/pipeline.cs | 16 +++++- .../ParameterBinding.Tests.ps1 | 34 ++++++++++++ 3 files changed, 78 insertions(+), 24 deletions(-) diff --git a/src/System.Management.Automation/engine/MshCommandRuntime.cs b/src/System.Management.Automation/engine/MshCommandRuntime.cs index 7b693ce0ed4..20965258462 100644 --- a/src/System.Management.Automation/engine/MshCommandRuntime.cs +++ b/src/System.Management.Automation/engine/MshCommandRuntime.cs @@ -928,7 +928,8 @@ private bool InitShouldLogPipelineExecutionDetail() /// internal string PipelineVariable { get; set; } - private PSVariable _pipelineVarReference = null; + private PSVariable _pipelineVarReference; + private bool _shouldRemovePipelineVariable; internal void SetupOutVariable() { @@ -972,7 +973,7 @@ internal void SetupPipelineVariable() // This can't use the common SetupVariable implementation, as this needs to persist for an entire // pipeline. - if (string.IsNullOrEmpty(this.PipelineVariable)) + if (string.IsNullOrEmpty(PipelineVariable)) { return; } @@ -983,12 +984,24 @@ internal void SetupPipelineVariable() _state = new SessionState(Context.EngineSessionState); // Create the pipeline variable - _pipelineVarReference = new PSVariable(this.PipelineVariable); - _state.PSVariable.Set(_pipelineVarReference); + _pipelineVarReference = new PSVariable(PipelineVariable); + object varToUse = _state.Internal.SetVariable( + _pipelineVarReference, + force: false, + CommandOrigin.Internal); - // Get the reference again in case we re-used one from the - // same scope. - _pipelineVarReference = _state.PSVariable.Get(this.PipelineVariable); + if (ReferenceEquals(_pipelineVarReference, varToUse)) + { + // The returned variable is the exact same instance, which means we set a new variable. + // In this case, we will try removing the pipeline variable in the end. + _shouldRemovePipelineVariable = true; + } + else + { + // A variable with the same name already exists in the same scope and it was returned. + // In this case, we update the reference and don't remove the variable in the end. + _pipelineVarReference = (PSVariable)varToUse; + } if (_thisCommand is not PSScriptCmdlet) { @@ -996,6 +1009,15 @@ internal void SetupPipelineVariable() } } + internal void RemovePipelineVariable() + { + if (_shouldRemovePipelineVariable) + { + // Remove pipeline variable when a pipeline is being torn down. + _state.PSVariable.Remove(PipelineVariable); + } + } + /// /// Configures the number of objects to buffer before calling the downstream Cmdlet. /// @@ -3765,25 +3787,12 @@ internal void SetVariableListsInPipe() if (this.PipelineVariable != null) { - // _state can be null if the current script block is dynamicparam, etc. - if (_state != null) - { - // Create the pipeline variable - _state.PSVariable.Set(_pipelineVarReference); - - // Get the reference again in case we re-used one from the - // same scope. - _pipelineVarReference = _state.PSVariable.Get(this.PipelineVariable); - } - this.OutputPipe.SetPipelineVariable(_pipelineVarReference); } } internal void RemoveVariableListsInPipe() { - // Diagnostics.Assert(thisCommand is PSScriptCmdlet, "this is only done for script cmdlets"); - if (_outVarList != null) { this.OutputPipe.RemoveVariableList(VariableStreamKind.Output, _outVarList); @@ -3807,9 +3816,6 @@ internal void RemoveVariableListsInPipe() if (this.PipelineVariable != null) { this.OutputPipe.RemovePipelineVariable(); - // '_state' could be null when a 'DynamicParam' block runs because the 'DynamicParam' block runs in 'DoPrepare', - // before 'PipelineProcessor.SetupParameterVariables' is called, where '_state' is initialized. - _state?.PSVariable.Remove(this.PipelineVariable); } } } diff --git a/src/System.Management.Automation/engine/pipeline.cs b/src/System.Management.Automation/engine/pipeline.cs index 58ea07de6ae..78fb19dbc65 100644 --- a/src/System.Management.Automation/engine/pipeline.cs +++ b/src/System.Management.Automation/engine/pipeline.cs @@ -1298,7 +1298,21 @@ private void DisposeCommands() // pipeline failure and continue disposing cmdlets. try { - commandProcessor.CommandRuntime.RemoveVariableListsInPipe(); + // Only cmdlets can have variables defined via the common parameters. + // We handle the cleanup of those variables only if we need to. + if (commandProcessor is CommandProcessor) + { + if (commandProcessor.Command is not PSScriptCmdlet) + { + // For script cmdlets, the variable lists were already removed when exiting a scope. + // So we only need to take care of binary cmdlets here. + commandProcessor.CommandRuntime.RemoveVariableListsInPipe(); + } + + // Remove the pipeline variable if we need to. + commandProcessor.CommandRuntime.RemovePipelineVariable(); + } + commandProcessor.Dispose(); } // 2005/04/13-JonN: The only vaguely plausible reason diff --git a/test/powershell/engine/ParameterBinding/ParameterBinding.Tests.ps1 b/test/powershell/engine/ParameterBinding/ParameterBinding.Tests.ps1 index 5bacbae7b2d..bf48dd34640 100644 --- a/test/powershell/engine/ParameterBinding/ParameterBinding.Tests.ps1 +++ b/test/powershell/engine/ParameterBinding/ParameterBinding.Tests.ps1 @@ -264,6 +264,40 @@ Describe "Parameter Binding Tests" -Tags "CI" { { Copy-Item "~\$guid*" -Destination ~ -ToSession $null } | Should -Throw -ErrorId 'ParameterArgumentValidationError' } + It 'PipelineVariable should not cause variable-removal exception (issue #16155)' { + function Invoke-AddOne { + param ( + [Parameter(ValueFromPipeline)] + [int]$Number + ) + + Begin { + $testValue = 'prefix-' + } + + Process { + $testValue + $Number + } + } + + 1,2,3 | Invoke-AddOne -PipelineVariable testValue | ForEach-Object { $testValue } | Should -Be @('prefix-1', 'prefix-2', 'prefix-3') + { Get-Variable -Name testValue -ErrorAction Stop } | Should -Throw -ErrorId 'VariableNotFound,Microsoft.PowerShell.Commands.GetVariableCommand' + + $results = & { $test = 'str'; 1,2,3 | Invoke-AddOne -PipelineVariable test | ForEach-Object { $test }; Get-Variable test } + $results | Should -HaveCount 4 + $results[0] | Should -BeExactly 'prefix-1' + $results[1] | Should -BeExactly 'prefix-2' + $results[2] | Should -BeExactly 'prefix-3' + $results[3] -is [psvariable] | Should -BeTrue + + $results = & { Set-Variable -Name test -Value 'str'; 1,2,3 | Invoke-AddOne -PipelineVariable test | ForEach-Object { $test }; Get-Variable test } + $results | Should -HaveCount 4 + $results[0] | Should -BeExactly 'prefix-1' + $results[1] | Should -BeExactly 'prefix-2' + $results[2] | Should -BeExactly 'prefix-3' + $results[3] -is [psvariable] | Should -BeTrue + } + Context "PipelineVariable Behaviour" { BeforeAll { From 41093f6d7a6c0b2a2f6f5e896e26dce1e3943ddf Mon Sep 17 00:00:00 2001 From: Sean Wheeler Date: Fri, 8 Oct 2021 11:01:28 -0500 Subject: [PATCH 063/645] Fixes #16176 - replace snippet tag with code tag in comments (#16177) From ce951602a4edf4b8d6c813e2f3bef0a280cabfb8 Mon Sep 17 00:00:00 2001 From: Ikko Ashimine Date: Tue, 12 Oct 2021 02:15:16 +0900 Subject: [PATCH 064/645] Fix typo in `TypeTable.cs` (#16220) occured -> occurred --- src/System.Management.Automation/engine/TypeTable.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/System.Management.Automation/engine/TypeTable.cs b/src/System.Management.Automation/engine/TypeTable.cs index 0bcd276a45b..edc870f8933 100644 --- a/src/System.Management.Automation/engine/TypeTable.cs +++ b/src/System.Management.Automation/engine/TypeTable.cs @@ -1747,7 +1747,7 @@ internal void AddError(string typeName, int errorLineNumber, string resourceStri /// /// This exception is used by TypeTable constructor to indicate errors - /// occured during construction time. + /// occurred during construction time. /// [Serializable] public class TypeTableLoadException : RuntimeException @@ -1796,7 +1796,7 @@ public TypeTableLoadException(string message, Exception innerException) /// time. /// /// - /// The errors that occured + /// The errors that occurred /// internal TypeTableLoadException(ConcurrentBag loadErrors) : base(TypesXmlStrings.TypeTableLoadErrors) From fa4bfb447ee4c7840276ca14aaf82357fe79d648 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Mon, 11 Oct 2021 13:43:42 -0700 Subject: [PATCH 065/645] Use Ubuntu 20.04 for SSH remoting test (#16225) --- .vsts-ci/sshremoting-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.vsts-ci/sshremoting-tests.yml b/.vsts-ci/sshremoting-tests.yml index 016f3bfddca..46af221f8eb 100644 --- a/.vsts-ci/sshremoting-tests.yml +++ b/.vsts-ci/sshremoting-tests.yml @@ -34,6 +34,8 @@ resources: clean: true jobs: - job: SSHRemotingTests + pool: + vmImage: ubuntu-20.04 container: mcr.microsoft.com/powershell/test-deps:ubuntu-18.04 displayName: SSH Remoting Tests From a32700a1c15a227bde54a0b80fa83cbe47bd2f27 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Mon, 11 Oct 2021 14:49:09 -0700 Subject: [PATCH 066/645] Add `clean` block to script block as a peer to `begin`, `process`, and `end` to allow easy resource cleanup (#15177) --- .../utility/ImplicitRemotingCommands.cs | 15 +- .../engine/CommandBase.cs | 7 + .../engine/CommandInfo.cs | 2 +- .../engine/CommandMetadata.cs | 21 +- .../engine/CommandProcessor.cs | 17 +- .../engine/CommandProcessorBase.cs | 227 ++- .../engine/ExecutionContext.cs | 11 +- .../ExperimentalFeature.cs | 4 + .../engine/MshCommandRuntime.cs | 13 +- .../engine/Pipe.cs | 7 + .../engine/ProxyCommand.cs | 24 + .../engine/ScriptCommandProcessor.cs | 48 +- .../engine/debugger/Breakpoint.cs | 11 +- .../engine/hostifaces/Pipeline.cs | 2 +- .../engine/lang/scriptblock.cs | 49 +- .../engine/parser/Compiler.cs | 10 + .../engine/parser/Parser.cs | 33 +- .../engine/parser/SemanticChecks.cs | 9 +- .../engine/parser/VariableAnalysis.cs | 24 +- .../engine/parser/ast.cs | 173 ++- .../engine/parser/token.cs | 17 +- .../engine/parser/tokenizer.cs | 36 +- .../engine/pipeline.cs | 801 +++++----- .../engine/runtime/CompiledScriptBlock.cs | 84 +- .../engine/runtime/Operations/MiscOps.cs | 31 +- .../resources/ParserStrings.resx | 8 +- .../Language/Parser/Parser.Tests.ps1 | 2 + .../Language/Parser/Parsing.Tests.ps1 | 6 +- .../CleanBlockErrorHandling.Tests.ps1 | 1374 +++++++++++++++++ .../Scripting/PipelineBehaviour.Tests.ps1 | 591 +++++++ 30 files changed, 3034 insertions(+), 623 deletions(-) create mode 100644 test/powershell/Language/Scripting/CleanBlockErrorHandling.Tests.ps1 create mode 100644 test/powershell/Language/Scripting/PipelineBehaviour.Tests.ps1 diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs index 325632c44ce..19bc5d44aec 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs @@ -2823,13 +2823,14 @@ private void GenerateHelperFunctions(TextWriter writer) $clientSideParameters = Get-PSImplicitRemotingClientSideParameters $PSBoundParameters ${8} - $scriptCmd = {{ & $script:InvokeCommand ` - @clientSideParameters ` - -HideComputerName ` - -Session (Get-PSImplicitRemotingSession -CommandName '{0}') ` - -Arg ('{0}', $PSBoundParameters, $positionalArguments) ` - -Script {{ param($name, $boundParams, $unboundParams) & $name @boundParams @unboundParams }} ` - }} + $scriptCmd = {{ + & $script:InvokeCommand ` + @clientSideParameters ` + -HideComputerName ` + -Session (Get-PSImplicitRemotingSession -CommandName '{0}') ` + -Arg ('{0}', $PSBoundParameters, $positionalArguments) ` + -Script {{ param($name, $boundParams, $unboundParams) & $name @boundParams @unboundParams }} ` + }} $steppablePipeline = $scriptCmd.GetSteppablePipeline($myInvocation.CommandOrigin) $steppablePipeline.Begin($myInvocation.ExpectingInput, $ExecutionContext) diff --git a/src/System.Management.Automation/engine/CommandBase.cs b/src/System.Management.Automation/engine/CommandBase.cs index 048eafd56c8..bb7aec959f6 100644 --- a/src/System.Management.Automation/engine/CommandBase.cs +++ b/src/System.Management.Automation/engine/CommandBase.cs @@ -233,6 +233,13 @@ internal virtual void DoStopProcessing() { } + /// + /// When overridden in the derived class, performs clean-up after the command execution. + /// + internal virtual void DoCleanResource() + { + } + #endregion Override /// diff --git a/src/System.Management.Automation/engine/CommandInfo.cs b/src/System.Management.Automation/engine/CommandInfo.cs index 7e28fece99d..1acca145ff9 100644 --- a/src/System.Management.Automation/engine/CommandInfo.cs +++ b/src/System.Management.Automation/engine/CommandInfo.cs @@ -529,7 +529,7 @@ private void GetMergedCommandParameterMetadata(out MergedCommandParameterMetadat processor = scriptCommand != null ? new CommandProcessor(scriptCommand, _context, useLocalScope: true, fromScriptFile: false, sessionState: scriptCommand.ScriptBlock.SessionStateInternal ?? Context.EngineSessionState) - : new CommandProcessor((CmdletInfo)this, _context) { UseLocalScope = true }; + : new CommandProcessor((CmdletInfo)this, _context); ParameterBinderController.AddArgumentsToCommandProcessor(processor, Arguments); CommandProcessorBase oldCurrentCommandProcessor = Context.CurrentCommandProcessor; diff --git a/src/System.Management.Automation/engine/CommandMetadata.cs b/src/System.Management.Automation/engine/CommandMetadata.cs index eb59f9140a2..bbe2b6666b2 100644 --- a/src/System.Management.Automation/engine/CommandMetadata.cs +++ b/src/System.Management.Automation/engine/CommandMetadata.cs @@ -875,8 +875,11 @@ internal string GetProxyCommand(string helpComment, bool generateDynamicParamete end {{{5}}} + +clean +{{{6}}} <# -{6} +{7} #> ", GetDecl(), @@ -885,6 +888,7 @@ internal string GetProxyCommand(string helpComment, bool generateDynamicParamete GetBeginBlock(), GetProcessBlock(), GetEndBlock(), + GetCleanBlock(), CodeGeneration.EscapeBlockCommentContent(helpComment)); return result; @@ -1063,6 +1067,11 @@ internal string GetBeginBlock() internal string GetProcessBlock() { + // The reason we wrap scripts in 'try { } catch { throw }' (here and elsewhere) is to turn + // an exception that could be thrown from .NET method invocation into a terminating error + // that can be propagated up. + // By default, an exception thrown from .NET method is not terminating, but when enclosed + // in try/catch, it will be turned into a terminating error. return @" try { $steppablePipeline.Process($_) @@ -1113,6 +1122,16 @@ internal string GetEndBlock() "; } + internal string GetCleanBlock() + { + // Here we don't need to enclose the script in a 'try/catch' like elsewhere, because + // 1. the 'Clean' block doesn't propagate up any exception (terminating error); + // 2. only one expression in the script, so nothing else needs to be stopped when invoking the method fails. + return @" + $steppablePipeline.Clean() +"; + } + #endregion #region Helper methods for restricting commands needed by implicit and interactive remoting diff --git a/src/System.Management.Automation/engine/CommandProcessor.cs b/src/System.Management.Automation/engine/CommandProcessor.cs index fb1fe81fc54..62eebe5d937 100644 --- a/src/System.Management.Automation/engine/CommandProcessor.cs +++ b/src/System.Management.Automation/engine/CommandProcessor.cs @@ -309,13 +309,11 @@ internal override void DoBegin() internal override void ProcessRecord() { // Invoke the Command method with the request object - if (!this.RanBeginAlready) { RanBeginAlready = true; try { - // NOTICE-2004/06/08-JonN 959638 using (commandRuntime.AllowThisCommandToWrite(true)) { if (Context._debuggingMode > 0 && Command is not PSScriptCmdlet) @@ -326,12 +324,9 @@ internal override void ProcessRecord() Command.DoBeginProcessing(); } } - // 2004/03/18-JonN This is understood to be - // an FXCOP violation, cleared by KCwalina. - catch (Exception e) // Catch-all OK, 3rd party callout. + catch (Exception e) { - // This cmdlet threw an exception, so - // wrap it and bubble it up. + // This cmdlet threw an exception, so wrap it and bubble it up. throw ManageInvocationException(e); } } @@ -366,6 +361,7 @@ internal override void ProcessRecord() // NOTICE-2004/06/08-JonN 959638 using (commandRuntime.AllowThisCommandToWrite(true)) + using (ParameterBinderBase.bindingTracer.TraceScope("CALLING ProcessRecord")) { if (CmdletParameterBinderController.ObsoleteParameterWarningList != null && CmdletParameterBinderController.ObsoleteParameterWarningList.Count > 0) @@ -400,14 +396,13 @@ internal override void ProcessRecord() } catch (LoopFlowException) { - // Win8:84066 - Don't wrap LoopFlowException, we incorrectly raise a PipelineStoppedException + // Don't wrap LoopFlowException, we incorrectly raise a PipelineStoppedException // which gets caught by a script try/catch if we wrap here. throw; } - // 2004/03/18-JonN This is understood to be - // an FXCOP violation, cleared by KCwalina. - catch (Exception e) // Catch-all OK, 3rd party callout. + catch (Exception e) { + // Catch-all OK, 3rd party callout. exceptionToThrow = e; } finally diff --git a/src/System.Management.Automation/engine/CommandProcessorBase.cs b/src/System.Management.Automation/engine/CommandProcessorBase.cs index c6825cc6925..8033e2b3677 100644 --- a/src/System.Management.Automation/engine/CommandProcessorBase.cs +++ b/src/System.Management.Automation/engine/CommandProcessorBase.cs @@ -5,8 +5,7 @@ using System.Collections.ObjectModel; using System.Management.Automation.Internal; using System.Management.Automation.Language; - -using Dbg = System.Management.Automation.Diagnostics; +using System.Runtime.InteropServices; namespace System.Management.Automation { @@ -46,6 +45,7 @@ internal CommandProcessorBase(CommandInfo commandInfo) string errorTemplate = expAttribute.ExperimentAction == ExperimentAction.Hide ? DiscoveryExceptions.ScriptDisabledWhenFeatureOn : DiscoveryExceptions.ScriptDisabledWhenFeatureOff; + string errorMsg = StringUtil.Format(errorTemplate, expAttribute.ExperimentName); ErrorRecord errorRecord = new ErrorRecord( new InvalidOperationException(errorMsg), @@ -54,6 +54,8 @@ internal CommandProcessorBase(CommandInfo commandInfo) commandInfo); throw new CmdletInvocationException(errorRecord); } + + HasCleanBlock = scriptCommand.ScriptBlock.HasCleanBlock; } CommandInfo = commandInfo; @@ -87,6 +89,11 @@ internal bool AddedToPipelineAlready /// internal CommandInfo CommandInfo { get; set; } + /// + /// Gets whether the command has a 'Clean' block defined. + /// + internal bool HasCleanBlock { get; } + /// /// This indicates whether this command processor is created from /// a script file. @@ -371,13 +378,10 @@ internal void RestorePreviousScope() Context.EngineSessionState = _previousCommandSessionState; - if (_previousScope != null) - { - // Restore the scope but use the same session state instance we - // got it from because the command may have changed the execution context - // session state... - CommandSessionState.CurrentScope = _previousScope; - } + // Restore the scope but use the same session state instance we + // got it from because the command may have changed the execution context + // session state... + CommandSessionState.CurrentScope = _previousScope; } private SessionStateScope _previousScope; @@ -452,16 +456,14 @@ internal void DoPrepare(IDictionary psDefaultParameterValues) HandleObsoleteCommand(ObsoleteAttribute); } } - catch (Exception) + catch (InvalidComObjectException e) { - if (_useLocalScope) - { - // If we had an exception during Prepare, we're done trying to execute the command - // so the scope we created needs to release any resources it hold.s - CommandSessionState.RemoveScope(CommandScope); - } + // This type of exception could be thrown from parameter binding. + string msg = StringUtil.Format(ParserStrings.InvalidComObjectException, e.Message); + var newEx = new RuntimeException(msg, e); - throw; + newEx.SetErrorId("InvalidComObjectException"); + throw newEx; } finally { @@ -508,26 +510,23 @@ internal virtual void DoBegin() // The RedirectShellErrorOutputPipe flag is used by the V2 hosting API to force the // redirection. // - if (this.RedirectShellErrorOutputPipe || _context.ShellFunctionErrorOutputPipe != null) + if (RedirectShellErrorOutputPipe || _context.ShellFunctionErrorOutputPipe is not null) { - _context.ShellFunctionErrorOutputPipe = this.commandRuntime.ErrorOutputPipe; + _context.ShellFunctionErrorOutputPipe = commandRuntime.ErrorOutputPipe; } _context.CurrentCommandProcessor = this; + SetCurrentScopeToExecutionScope(); + using (commandRuntime.AllowThisCommandToWrite(true)) + using (ParameterBinderBase.bindingTracer.TraceScope("CALLING BeginProcessing")) { - using (ParameterBinderBase.bindingTracer.TraceScope( - "CALLING BeginProcessing")) + if (Context._debuggingMode > 0 && Command is not PSScriptCmdlet) { - SetCurrentScopeToExecutionScope(); - - if (Context._debuggingMode > 0 && Command is not PSScriptCmdlet) - { - Context.Debugger.CheckCommand(this.Command.MyInvocation); - } - - Command.DoBeginProcessing(); + Context.Debugger.CheckCommand(Command.MyInvocation); } + + Command.DoBeginProcessing(); } } catch (Exception e) @@ -589,20 +588,14 @@ internal virtual void Complete() try { using (commandRuntime.AllowThisCommandToWrite(true)) + using (ParameterBinderBase.bindingTracer.TraceScope("CALLING EndProcessing")) { - using (ParameterBinderBase.bindingTracer.TraceScope( - "CALLING EndProcessing")) - { - this.Command.DoEndProcessing(); - } + this.Command.DoEndProcessing(); } } - // 2004/03/18-JonN This is understood to be - // an FXCOP violation, cleared by KCwalina. catch (Exception e) { - // This cmdlet threw an exception, so - // wrap it and bubble it up. + // This cmdlet threw an exception, wrap it as needed and bubble it up. throw ManageInvocationException(e); } } @@ -631,46 +624,121 @@ internal void DoComplete() // The RedirectShellErrorOutputPipe flag is used by the V2 hosting API to force the // redirection. // - if (this.RedirectShellErrorOutputPipe || _context.ShellFunctionErrorOutputPipe != null) + if (RedirectShellErrorOutputPipe || _context.ShellFunctionErrorOutputPipe is not null) { - _context.ShellFunctionErrorOutputPipe = this.commandRuntime.ErrorOutputPipe; + _context.ShellFunctionErrorOutputPipe = commandRuntime.ErrorOutputPipe; } _context.CurrentCommandProcessor = this; - SetCurrentScopeToExecutionScope(); Complete(); } finally { - OnRestorePreviousScope(); - _context.ShellFunctionErrorOutputPipe = oldErrorOutputPipe; _context.CurrentCommandProcessor = oldCurrentCommandProcessor; - // Destroy the local scope at this point if there is one... - if (_useLocalScope && CommandScope != null) - { - CommandSessionState.RemoveScope(CommandScope); - } + RestorePreviousScope(); + } + } - // and the previous scope... - if (_previousScope != null) + protected virtual void CleanResource() + { + try + { + using (commandRuntime.AllowThisCommandToWrite(permittedToWriteToPipeline: true)) + using (ParameterBinderBase.bindingTracer.TraceScope("CALLING CleanResource")) { - // Restore the scope but use the same session state instance we - // got it from because the command may have changed the execution context - // session state... - CommandSessionState.CurrentScope = _previousScope; + Command.DoCleanResource(); } + } + catch (HaltCommandException) + { + throw; + } + catch (FlowControlException) + { + throw; + } + catch (Exception e) + { + // This cmdlet threw an exception, so wrap it and bubble it up. + throw ManageInvocationException(e); + } + } + + internal void DoCleanup() + { + // The property 'PropagateExceptionsToEnclosingStatementBlock' controls whether a general exception + // (an exception thrown from a .NET method invocation, or an expression like '1/0') will be turned + // into a terminating error, which will be propagated up and thus stop the rest of the running script. + // It is usually used by TryStatement and TrapStatement, which makes the general exception catch-able. + // + // For the 'Clean' block, we don't want to bubble up the general exception when the command is enclosed + // in a TryStatement or has TrapStatement accompanying, because no exception can escape from 'Clean' and + // thus it's pointless to bubble up the general exception in this case. + // + // Therefore we set this property to 'false' here to mask off the previous setting that could be from a + // TryStatement or TrapStatement. Example: + // PS:1> function b { end {} clean { 1/0; Write-Host 'clean' } } + // PS:2> b + // RuntimeException: Attempted to divide by zero. + // clean + // ## Note that, outer 'try/trap' doesn't affect the general exception happens in 'Clean' block. + // ## so its behavior is consistent regardless of whether the command is enclosed by 'try/catch' or not. + // PS:3> try { b } catch { 'outer catch' } + // RuntimeException: Attempted to divide by zero. + // clean + // + // Be noted that, this doesn't affect the TryStatement/TrapStatement within the 'Clean' block. Example: + // ## 'try/trap' within 'Clean' block makes the general exception catch-able. + // PS:3> function a { end {} clean { try { 1/0; Write-Host 'clean' } catch { Write-Host "caught: $_" } } } + // PS:4> a + // caught: Attempted to divide by zero. + bool oldExceptionPropagationState = _context.PropagateExceptionsToEnclosingStatementBlock; + _context.PropagateExceptionsToEnclosingStatementBlock = false; - // Restore the previous session state - if (_previousCommandSessionState != null) + Pipe oldErrorOutputPipe = _context.ShellFunctionErrorOutputPipe; + CommandProcessorBase oldCurrentCommandProcessor = _context.CurrentCommandProcessor; + + try + { + if (RedirectShellErrorOutputPipe || _context.ShellFunctionErrorOutputPipe is not null) { - Context.EngineSessionState = _previousCommandSessionState; + _context.ShellFunctionErrorOutputPipe = commandRuntime.ErrorOutputPipe; } + + _context.CurrentCommandProcessor = this; + SetCurrentScopeToExecutionScope(); + CleanResource(); + } + finally + { + _context.PropagateExceptionsToEnclosingStatementBlock = oldExceptionPropagationState; + _context.ShellFunctionErrorOutputPipe = oldErrorOutputPipe; + _context.CurrentCommandProcessor = oldCurrentCommandProcessor; + + RestorePreviousScope(); } } + internal void ReportCleanupError(Exception exception) + { + var error = exception is IContainsErrorRecord icer + ? icer.ErrorRecord + : new ErrorRecord(exception, "Clean.ReportException", ErrorCategory.NotSpecified, targetObject: null); + + PSObject errorWrap = PSObject.AsPSObject(error); + errorWrap.WriteStream = WriteStreamType.Error; + + var errorPipe = commandRuntime.ErrorMergeTo == MshCommandRuntime.MergeDataStream.Output + ? commandRuntime.OutputPipe + : commandRuntime.ErrorOutputPipe; + + errorPipe.Add(errorWrap); + _context.QuestionMarkVariableValue = false; + } + /// /// For diagnostic purposes. /// @@ -777,23 +845,16 @@ internal PipelineStoppedException ManageInvocationException(Exception e) { do // false loop { - ProviderInvocationException pie = e as ProviderInvocationException; - if (pie != null) + if (e is ProviderInvocationException pie) { - // If a ProviderInvocationException occurred, - // discard the ProviderInvocationException and - // re-wrap in CmdletProviderInvocationException - e = new CmdletProviderInvocationException( - pie, - Command.MyInvocation); + // If a ProviderInvocationException occurred, discard the ProviderInvocationException + // and re-wrap it in CmdletProviderInvocationException. + e = new CmdletProviderInvocationException(pie, Command.MyInvocation); break; } - // 1021203-2005/05/09-JonN - // HaltCommandException will cause the command - // to stop, but not be reported as an error. - // 906445-2005/05/16-JonN - // FlowControlException should not be wrapped + // HaltCommandException will cause the command to stop, but not be reported as an error. + // FlowControlException should not be wrapped. if (e is PipelineStoppedException || e is CmdletInvocationException || e is ActionPreferenceStopException @@ -813,9 +874,7 @@ internal PipelineStoppedException ManageInvocationException(Exception e) } // wrap all other exceptions - e = new CmdletInvocationException( - e, - Command.MyInvocation); + e = new CmdletInvocationException(e, Command.MyInvocation); } while (false); // commandRuntime.ManageException will always throw PipelineStoppedException @@ -943,15 +1002,27 @@ public void Dispose() private void Dispose(bool disposing) { if (_disposed) + { return; + } if (disposing) { - // 2004/03/05-JonN Look into using metadata to check - // whether IDisposable is implemented, in order to avoid - // this expensive reflection cast. - IDisposable id = Command as IDisposable; - if (id != null) + if (UseLocalScope) + { + // Clean up the PS drives that are associated with this local scope. + // This operation may be needed at multiple stages depending on whether the 'clean' block is declared: + // 1. when there is a 'clean' block, it needs to be done only after 'clean' block runs, because the scope + // needs to be preserved until the 'clean' block finish execution. + // 2. when there is no 'clean' block, it needs to be done when + // (1) there is any exception thrown from 'DoPrepare()', 'DoBegin()', 'DoExecute()', or 'DoComplete'; + // (2) OR, the command runs to the end successfully; + // Doing this cleanup at those multiple stages is cumbersome. Since we will always dispose the command in + // the end, doing this cleanup here will cover all the above cases. + CommandSessionState.RemoveScope(CommandScope); + } + + if (Command is IDisposable id) { id.Dispose(); } diff --git a/src/System.Management.Automation/engine/ExecutionContext.cs b/src/System.Management.Automation/engine/ExecutionContext.cs index 8ffd25949c5..18ba461a68e 100644 --- a/src/System.Management.Automation/engine/ExecutionContext.cs +++ b/src/System.Management.Automation/engine/ExecutionContext.cs @@ -783,11 +783,6 @@ internal Pipe RedirectErrorPipe(Pipe newPipe) return oldPipe; } - internal void RestoreErrorPipe(Pipe pipe) - { - ShellFunctionErrorOutputPipe = pipe; - } - /// /// Reset all of the redirection book keeping variables. This routine should be called when starting to /// execute a script. @@ -840,15 +835,13 @@ internal void ResetRedirection() internal void AppendDollarError(object obj) { ErrorRecord objAsErrorRecord = obj as ErrorRecord; - if (objAsErrorRecord == null && obj is not Exception) + if (objAsErrorRecord is null && obj is not Exception) { Diagnostics.Assert(false, "Object to append was neither an ErrorRecord nor an Exception in ExecutionContext.AppendDollarError"); return; } - object old = this.DollarErrorVariable; - ArrayList arraylist = old as ArrayList; - if (arraylist == null) + if (DollarErrorVariable is not ArrayList arraylist) { Diagnostics.Assert(false, "$error should be a global constant ArrayList"); return; diff --git a/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs b/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs index dd0f7209635..cb0be3193a7 100644 --- a/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs +++ b/src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs @@ -23,6 +23,7 @@ public class ExperimentalFeature internal const string EngineSource = "PSEngine"; internal const string PSNativeCommandArgumentPassingFeatureName = "PSNativeCommandArgumentPassing"; internal const string PSNativeCommandErrorActionPreferenceFeatureName = "PSNativeCommandErrorActionPreference"; + internal const string PSCleanBlockFeatureName = "PSCleanBlock"; #endregion @@ -126,6 +127,9 @@ static ExperimentalFeature() new ExperimentalFeature( name: PSNativeCommandErrorActionPreferenceFeatureName, description: "Native commands with non-zero exit codes issue errors according to $ErrorActionPreference when $PSNativeCommandUseErrorActionPreference is $true"), + new ExperimentalFeature( + name: PSCleanBlockFeatureName, + description: "Add support of a 'Clean' block to functions and script cmdlets for easy resource cleanup"), }; EngineExperimentalFeatures = new ReadOnlyCollection(engineFeatures); diff --git a/src/System.Management.Automation/engine/MshCommandRuntime.cs b/src/System.Management.Automation/engine/MshCommandRuntime.cs index 20965258462..16fa8652bcc 100644 --- a/src/System.Management.Automation/engine/MshCommandRuntime.cs +++ b/src/System.Management.Automation/engine/MshCommandRuntime.cs @@ -942,10 +942,8 @@ internal void SetupOutVariable() // Handle the creation of OutVariable in the case of Out-Default specially, // as it needs to handle much of its OutVariable support itself. - if ( - (!string.IsNullOrEmpty(this.OutVariable)) && - (!(this.OutVariable.StartsWith('+'))) && - string.Equals("Out-Default", _thisCommand.CommandInfo.Name, StringComparison.OrdinalIgnoreCase)) + if (!OutVariable.StartsWith('+') && + string.Equals("Out-Default", _commandInfo.Name, StringComparison.OrdinalIgnoreCase)) { if (_state == null) _state = new SessionState(Context.EngineSessionState); @@ -2426,7 +2424,6 @@ public Exception ManageException(Exception e) } // Log a command health event - MshLog.LogCommandHealthEvent( Context, e, @@ -3765,8 +3762,10 @@ internal void SetVariableListsInPipe() { Diagnostics.Assert(_thisCommand is PSScriptCmdlet, "this is only done for script cmdlets"); - if (_outVarList != null) + if (_outVarList != null && !OutputPipe.IgnoreOutVariableList) { + // A null pipe is used when executing the 'Clean' block of a PSScriptCmdlet. + // In such a case, we don't capture output to the out variable list. this.OutputPipe.AddVariableList(VariableStreamKind.Output, _outVarList); } @@ -3793,7 +3792,7 @@ internal void SetVariableListsInPipe() internal void RemoveVariableListsInPipe() { - if (_outVarList != null) + if (_outVarList != null && !OutputPipe.IgnoreOutVariableList) { this.OutputPipe.RemoveVariableList(VariableStreamKind.Output, _outVarList); } diff --git a/src/System.Management.Automation/engine/Pipe.cs b/src/System.Management.Automation/engine/Pipe.cs index 8ee000a4faf..9e24c6471db 100644 --- a/src/System.Management.Automation/engine/Pipe.cs +++ b/src/System.Management.Automation/engine/Pipe.cs @@ -109,6 +109,13 @@ public override string ToString() /// internal int OutBufferCount { get; set; } = 0; + /// + /// Gets whether the out variable list should be ignored. + /// This is used for scenarios like the `clean` block, where writing to output stream is intentionally + /// disabled and thus out variables should also be ignored. + /// + internal bool IgnoreOutVariableList { get; set; } + /// /// If true, then all input added to this pipe will simply be discarded... /// diff --git a/src/System.Management.Automation/engine/ProxyCommand.cs b/src/System.Management.Automation/engine/ProxyCommand.cs index 5703adec664..6a2e6dce66d 100644 --- a/src/System.Management.Automation/engine/ProxyCommand.cs +++ b/src/System.Management.Automation/engine/ProxyCommand.cs @@ -247,6 +247,30 @@ public static string GetEnd(CommandMetadata commandMetadata) return commandMetadata.GetEndBlock(); } + /// + /// This method constructs a string representing the clean block of the command + /// specified by . The returned string only contains the + /// script, it is not enclosed in "clean { }". + /// + /// + /// An instance of CommandMetadata representing a command. + /// + /// + /// A string representing the end block of the command. + /// + /// + /// If is null. + /// + public static string GetClean(CommandMetadata commandMetadata) + { + if (commandMetadata == null) + { + throw PSTraceSource.NewArgumentNullException(nameof(commandMetadata)); + } + + return commandMetadata.GetCleanBlock(); + } + private static T GetProperty(PSObject obj, string property) where T : class { T result = null; diff --git a/src/System.Management.Automation/engine/ScriptCommandProcessor.cs b/src/System.Management.Automation/engine/ScriptCommandProcessor.cs index 99b685e07d2..54d4f45849b 100644 --- a/src/System.Management.Automation/engine/ScriptCommandProcessor.cs +++ b/src/System.Management.Automation/engine/ScriptCommandProcessor.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.Management.Automation.Internal; using System.Management.Automation.Language; +using System.Management.Automation.Runspaces; using System.Reflection; using Dbg = System.Management.Automation.Diagnostics; @@ -47,7 +48,7 @@ protected ScriptCommandProcessorBase(IScriptCommandInfo commandInfo, ExecutionCo protected bool _dontUseScopeCommandOrigin; /// - /// If true, then an exit exception will be rethrown to instead of caught and processed... + /// If true, then an exit exception will be rethrown instead of caught and processed... /// protected bool _rethrowExitException; @@ -237,6 +238,7 @@ internal sealed class DlrScriptCommandProcessor : ScriptCommandProcessorBase private MutableTuple _localsTuple; private bool _runOptimizedCode; private bool _argsBound; + private bool _anyClauseExecuted; private FunctionContext _functionContext; internal DlrScriptCommandProcessor(ScriptBlock scriptBlock, ExecutionContext context, bool useNewScope, CommandOrigin origin, SessionStateInternal sessionState, object dollarUnderbar) @@ -327,8 +329,7 @@ internal override void DoBegin() ScriptBlock.LogScriptBlockStart(_scriptBlock, Context.CurrentRunspace.InstanceId); - // Even if there is no begin, we need to set up the execution scope for this - // script... + // Even if there is no begin, we need to set up the execution scope for this script... SetCurrentScopeToExecutionScope(); CommandProcessorBase oldCurrentCommandProcessor = Context.CurrentCommandProcessor; try @@ -410,6 +411,7 @@ internal override void Complete() if (_scriptBlock.HasEndBlock) { var endBlock = _runOptimizedCode ? _scriptBlock.EndBlock : _scriptBlock.UnoptimizedEndBlock; + if (this.CommandRuntime.InputPipe.ExternalReader == null) { if (IsPipelineInputExpected()) @@ -433,7 +435,33 @@ internal override void Complete() } finally { - ScriptBlock.LogScriptBlockEnd(_scriptBlock, Context.CurrentRunspace.InstanceId); + if (!_scriptBlock.HasCleanBlock) + { + ScriptBlock.LogScriptBlockEnd(_scriptBlock, Context.CurrentRunspace.InstanceId); + } + } + } + + protected override void CleanResource() + { + if (_scriptBlock.HasCleanBlock && _anyClauseExecuted) + { + // The 'Clean' block doesn't write to pipeline. + Pipe oldOutputPipe = _functionContext._outputPipe; + _functionContext._outputPipe = new Pipe { NullPipe = true }; + + try + { + RunClause( + clause: _runOptimizedCode ? _scriptBlock.CleanBlock : _scriptBlock.UnoptimizedCleanBlock, + dollarUnderbar: AutomationNull.Value, + inputToProcess: AutomationNull.Value); + } + finally + { + _functionContext._outputPipe = oldOutputPipe; + ScriptBlock.LogScriptBlockEnd(_scriptBlock, Context.CurrentRunspace.InstanceId); + } } } @@ -459,6 +487,7 @@ private void RunClause(Action clause, object dollarUnderbar, ob { ExecutionContext.CheckStackDepth(); + _anyClauseExecuted = true; Pipe oldErrorOutputPipe = this.Context.ShellFunctionErrorOutputPipe; // If the script block has a different language mode than the current, @@ -553,7 +582,7 @@ private void RunClause(Action clause, object dollarUnderbar, ob } finally { - this.Context.RestoreErrorPipe(oldErrorOutputPipe); + Context.ShellFunctionErrorOutputPipe = oldErrorOutputPipe; if (oldLanguageMode.HasValue) { @@ -584,15 +613,12 @@ private void RunClause(Action clause, object dollarUnderbar, ob } catch (RuntimeException e) { - ManageScriptException(e); // always throws - // This quiets the compiler which wants to see a return value - // in all codepaths. - throw; + // This method always throws. + ManageScriptException(e); } catch (Exception e) { - // This cmdlet threw an exception, so - // wrap it and bubble it up. + // This cmdlet threw an exception, so wrap it and bubble it up. throw ManageInvocationException(e); } } diff --git a/src/System.Management.Automation/engine/debugger/Breakpoint.cs b/src/System.Management.Automation/engine/debugger/Breakpoint.cs index 5daf3bdc18b..e51d79f4afa 100644 --- a/src/System.Management.Automation/engine/debugger/Breakpoint.cs +++ b/src/System.Management.Automation/engine/debugger/Breakpoint.cs @@ -531,15 +531,16 @@ internal bool TrySetBreakpoint(string scriptFile, FunctionContext functionContex // Not found. First, we check if the line/column is before any real code. If so, we'll // move the breakpoint to the first interesting sequence point (could be a dynamicparam, - // begin, process, or end block.) + // begin, process, end, or clean block.) if (scriptBlock != null) { var ast = scriptBlock.Ast; var bodyAst = ((IParameterMetadataProvider)ast).Body; - if ((bodyAst.DynamicParamBlock == null || bodyAst.DynamicParamBlock.Extent.IsAfter(Line, Column)) && - (bodyAst.BeginBlock == null || bodyAst.BeginBlock.Extent.IsAfter(Line, Column)) && - (bodyAst.ProcessBlock == null || bodyAst.ProcessBlock.Extent.IsAfter(Line, Column)) && - (bodyAst.EndBlock == null || bodyAst.EndBlock.Extent.IsAfter(Line, Column))) + if ((bodyAst.DynamicParamBlock == null || bodyAst.DynamicParamBlock.Extent.IsAfter(Line, Column)) + && (bodyAst.BeginBlock == null || bodyAst.BeginBlock.Extent.IsAfter(Line, Column)) + && (bodyAst.ProcessBlock == null || bodyAst.ProcessBlock.Extent.IsAfter(Line, Column)) + && (bodyAst.EndBlock == null || bodyAst.EndBlock.Extent.IsAfter(Line, Column)) + && (bodyAst.CleanBlock == null || bodyAst.CleanBlock.Extent.IsAfter(Line, Column))) { SetBreakpoint(functionContext, 0); return true; diff --git a/src/System.Management.Automation/engine/hostifaces/Pipeline.cs b/src/System.Management.Automation/engine/hostifaces/Pipeline.cs index 874691852cb..c7b962b7668 100644 --- a/src/System.Management.Automation/engine/hostifaces/Pipeline.cs +++ b/src/System.Management.Automation/engine/hostifaces/Pipeline.cs @@ -445,7 +445,7 @@ internal void SetHadErrors(bool status) /// /// This flag is used to force the redirection. By default it is false to maintain compatibility with /// V1, but the V2 hosting interface (PowerShell class) sets this flag to true to ensure the global - /// error output pipe is always set and $ErrorActionPreference when invoking the Pipeline. + /// error output pipe is always set and $ErrorActionPreference is checked when invoking the Pipeline. /// internal bool RedirectShellErrorOutputPipe { get; set; } = false; diff --git a/src/System.Management.Automation/engine/lang/scriptblock.cs b/src/System.Management.Automation/engine/lang/scriptblock.cs index 0c6c1d5dff1..07f53c132ff 100644 --- a/src/System.Management.Automation/engine/lang/scriptblock.cs +++ b/src/System.Management.Automation/engine/lang/scriptblock.cs @@ -1280,7 +1280,42 @@ public Array End() { // then pop this pipeline and dispose it... _context.PopPipelineProcessor(true); - _pipeline.Dispose(); + Dispose(); + } + } + + /// + /// Clean resources for script commands of this steppable pipeline. + /// + /// + /// The way we handle 'Clean' blocks in a steppable pipeline makes sure that: + /// 1. The 'Clean' blocks get to run if any exception is thrown from 'Begin/Process/End'. + /// 2. The 'Clean' blocks get to run if 'End' finished successfully. + /// However, this is not enough for a steppable pipeline, because the function, where the steppable + /// pipeline gets used, may fail (think about a proxy function). And that may lead to the situation + /// where "no exception was thrown from the steppable pipeline" but "the steppable pipeline didn't + /// run to the end". In that case, 'Clean' won't run unless it's triggered explicitly on the steppable + /// pipeline. This method allows a user to do that from the 'Clean' block of the proxy function. + /// + public void Clean() + { + if (_pipeline.Commands is null) + { + // The pipeline commands have been disposed. In this case, 'Clean' + // should have already been called on the pipeline processor. + return; + } + + try + { + _context.PushPipelineProcessor(_pipeline); + _pipeline.DoCleanup(); + } + finally + { + // then pop this pipeline and dispose it... + _context.PopPipelineProcessor(true); + Dispose(); } } @@ -1293,23 +1328,13 @@ public Array End() /// When this object is disposed, the contained pipeline should also be disposed. /// public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - private void Dispose(bool disposing) { if (_disposed) { return; } - if (disposing) - { - _pipeline.Dispose(); - } - + _pipeline.Dispose(); _disposed = true; } diff --git a/src/System.Management.Automation/engine/parser/Compiler.cs b/src/System.Management.Automation/engine/parser/Compiler.cs index bcddbee0b24..9956fb29e30 100644 --- a/src/System.Management.Automation/engine/parser/Compiler.cs +++ b/src/System.Management.Automation/engine/parser/Compiler.cs @@ -2026,6 +2026,7 @@ internal void Compile(CompiledScriptBlockData scriptBlock, bool optimize) scriptBlock.BeginBlock = CompileTree(_beginBlockLambda, compileInterpretChoice); scriptBlock.ProcessBlock = CompileTree(_processBlockLambda, compileInterpretChoice); scriptBlock.EndBlock = CompileTree(_endBlockLambda, compileInterpretChoice); + scriptBlock.CleanBlock = CompileTree(_cleanBlockLambda, compileInterpretChoice); scriptBlock.LocalsMutableTupleType = LocalVariablesTupleType; scriptBlock.LocalsMutableTupleCreator = MutableTuple.TupleCreator(LocalVariablesTupleType); scriptBlock.NameToIndexMap = nameToIndexMap; @@ -2036,6 +2037,7 @@ internal void Compile(CompiledScriptBlockData scriptBlock, bool optimize) scriptBlock.UnoptimizedBeginBlock = CompileTree(_beginBlockLambda, compileInterpretChoice); scriptBlock.UnoptimizedProcessBlock = CompileTree(_processBlockLambda, compileInterpretChoice); scriptBlock.UnoptimizedEndBlock = CompileTree(_endBlockLambda, compileInterpretChoice); + scriptBlock.UnoptimizedCleanBlock = CompileTree(_cleanBlockLambda, compileInterpretChoice); scriptBlock.UnoptimizedLocalsMutableTupleType = LocalVariablesTupleType; scriptBlock.UnoptimizedLocalsMutableTupleCreator = MutableTuple.TupleCreator(LocalVariablesTupleType); } @@ -2221,6 +2223,7 @@ internal LoopGotoTargets(string label, LabelTarget breakLabel, LabelTarget conti private Expression> _beginBlockLambda; private Expression> _processBlockLambda; private Expression> _endBlockLambda; + private Expression> _cleanBlockLambda; private readonly List _loopTargets = new List(); private bool _generatingWhileOrDoLoop; @@ -2463,6 +2466,13 @@ public object VisitScriptBlock(ScriptBlockAst scriptBlockAst) } _endBlockLambda = CompileNamedBlock(scriptBlockAst.EndBlock, funcName, rootForDefiningTypesAndUsings); + rootForDefiningTypesAndUsings = null; + } + + if (scriptBlockAst.CleanBlock != null) + { + _cleanBlockLambda = CompileNamedBlock(scriptBlockAst.CleanBlock, funcName + "", rootForDefiningTypesAndUsings); + rootForDefiningTypesAndUsings = null; } return null; diff --git a/src/System.Management.Automation/engine/parser/Parser.cs b/src/System.Management.Automation/engine/parser/Parser.cs index 3afc55e9861..c5f9d81f4b8 100644 --- a/src/System.Management.Automation/engine/parser/Parser.cs +++ b/src/System.Management.Automation/engine/parser/Parser.cs @@ -724,12 +724,13 @@ internal static bool TryParseAsConstantHashtable(string input, out Hashtable res ParseError[] parseErrors; var ast = Parser.ParseInput(input, out throwAwayTokens, out parseErrors); - if ((ast == null) || - parseErrors.Length > 0 || - ast.BeginBlock != null || - ast.ProcessBlock != null || - ast.DynamicParamBlock != null || - ast.EndBlock.Traps != null) + if (ast == null + || parseErrors.Length > 0 + || ast.BeginBlock != null + || ast.ProcessBlock != null + || ast.CleanBlock != null + || ast.DynamicParamBlock != null + || ast.EndBlock.Traps != null) { return false; } @@ -1713,9 +1714,9 @@ private ScriptBlockAst NamedBlockListRule(Token lCurly, List NamedBlockAst beginBlock = null; NamedBlockAst processBlock = null; NamedBlockAst endBlock = null; - IScriptExtent startExtent = lCurly != null - ? lCurly.Extent - : paramBlockAst?.Extent; + NamedBlockAst cleanBlock = null; + + IScriptExtent startExtent = lCurly?.Extent ?? paramBlockAst?.Extent; IScriptExtent endExtent = null; IScriptExtent extent = null; IScriptExtent scriptBlockExtent = null; @@ -1757,6 +1758,7 @@ private ScriptBlockAst NamedBlockListRule(Token lCurly, List case TokenKind.Begin: case TokenKind.Process: case TokenKind.End: + case TokenKind.Clean: break; } @@ -1797,6 +1799,10 @@ private ScriptBlockAst NamedBlockListRule(Token lCurly, List { endBlock = new NamedBlockAst(extent, TokenKind.End, statementBlock, false); } + else if (blockNameToken.Kind == TokenKind.Clean && cleanBlock == null) + { + cleanBlock = new NamedBlockAst(extent, TokenKind.Clean, statementBlock, false); + } else if (blockNameToken.Kind == TokenKind.Dynamicparam && dynamicParamBlock == null) { dynamicParamBlock = new NamedBlockAst(extent, TokenKind.Dynamicparam, statementBlock, false); @@ -1818,7 +1824,14 @@ private ScriptBlockAst NamedBlockListRule(Token lCurly, List CompleteScriptBlockBody(lCurly, ref extent, out scriptBlockExtent); return_script_block_ast: - return new ScriptBlockAst(scriptBlockExtent, usingStatements, paramBlockAst, beginBlock, processBlock, endBlock, + return new ScriptBlockAst( + scriptBlockExtent, + usingStatements, + paramBlockAst, + beginBlock, + processBlock, + endBlock, + cleanBlock, dynamicParamBlock); } diff --git a/src/System.Management.Automation/engine/parser/SemanticChecks.cs b/src/System.Management.Automation/engine/parser/SemanticChecks.cs index 927b2c33ae8..b792530193b 100644 --- a/src/System.Management.Automation/engine/parser/SemanticChecks.cs +++ b/src/System.Management.Automation/engine/parser/SemanticChecks.cs @@ -406,10 +406,11 @@ public override AstVisitAction VisitFunctionMember(FunctionMemberAst functionMem ParserStrings.ParamBlockNotAllowedInMethod); } - if (body.BeginBlock != null || - body.ProcessBlock != null || - body.DynamicParamBlock != null || - !body.EndBlock.Unnamed) + if (body.BeginBlock != null + || body.ProcessBlock != null + || body.CleanBlock != null + || body.DynamicParamBlock != null + || !body.EndBlock.Unnamed) { _parser.ReportError(Parser.ExtentFromFirstOf(body.DynamicParamBlock, body.BeginBlock, body.ProcessBlock, body.EndBlock), nameof(ParserStrings.NamedBlockNotAllowedInMethod), diff --git a/src/System.Management.Automation/engine/parser/VariableAnalysis.cs b/src/System.Management.Automation/engine/parser/VariableAnalysis.cs index b38954ebfc6..09b336099cb 100644 --- a/src/System.Management.Automation/engine/parser/VariableAnalysis.cs +++ b/src/System.Management.Automation/engine/parser/VariableAnalysis.cs @@ -945,25 +945,11 @@ public object VisitScriptBlock(ScriptBlockAst scriptBlockAst) { _currentBlock = _entryBlock; - if (scriptBlockAst.DynamicParamBlock != null) - { - scriptBlockAst.DynamicParamBlock.Accept(this); - } - - if (scriptBlockAst.BeginBlock != null) - { - scriptBlockAst.BeginBlock.Accept(this); - } - - if (scriptBlockAst.ProcessBlock != null) - { - scriptBlockAst.ProcessBlock.Accept(this); - } - - if (scriptBlockAst.EndBlock != null) - { - scriptBlockAst.EndBlock.Accept(this); - } + scriptBlockAst.DynamicParamBlock?.Accept(this); + scriptBlockAst.BeginBlock?.Accept(this); + scriptBlockAst.ProcessBlock?.Accept(this); + scriptBlockAst.EndBlock?.Accept(this); + scriptBlockAst.CleanBlock?.Accept(this); _currentBlock.FlowsTo(_exitBlock); diff --git a/src/System.Management.Automation/engine/parser/ast.cs b/src/System.Management.Automation/engine/parser/ast.cs index 1f03eb68df5..b40a8e66a1e 100644 --- a/src/System.Management.Automation/engine/parser/ast.cs +++ b/src/System.Management.Automation/engine/parser/ast.cs @@ -818,6 +818,46 @@ public ScriptBlockAst(IScriptExtent extent, NamedBlockAst processBlock, NamedBlockAst endBlock, NamedBlockAst dynamicParamBlock) + : this( + extent, + usingStatements, + attributes, + paramBlock, + beginBlock, + processBlock, + endBlock, + cleanBlock: null, + dynamicParamBlock) + { + } + + /// + /// Initializes a new instance of the class. + /// This construction uses explicitly named begin/process/end/clean blocks. + /// + /// The extent of the script block. + /// The list of using statments, may be null. + /// The set of attributes for the script block. + /// The ast for the param block, may be null. + /// The ast for the begin block, may be null. + /// The ast for the process block, may be null. + /// The ast for the end block, may be null. + /// The ast for the clean block, may be null. + /// The ast for the dynamicparam block, may be null. + /// + /// If is null. + /// + [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "param")] + public ScriptBlockAst( + IScriptExtent extent, + IEnumerable usingStatements, + IEnumerable attributes, + ParamBlockAst paramBlock, + NamedBlockAst beginBlock, + NamedBlockAst processBlock, + NamedBlockAst endBlock, + NamedBlockAst cleanBlock, + NamedBlockAst dynamicParamBlock) : base(extent) { SetUsingStatements(usingStatements); @@ -856,6 +896,12 @@ public ScriptBlockAst(IScriptExtent extent, SetParent(endBlock); } + if (cleanBlock != null) + { + this.CleanBlock = cleanBlock; + SetParent(cleanBlock); + } + if (dynamicParamBlock != null) { this.DynamicParamBlock = dynamicParamBlock; @@ -888,6 +934,35 @@ public ScriptBlockAst(IScriptExtent extent, { } + /// + /// Initializes a new instance of the class. + /// This construction uses explicitly named begin/process/end/clean blocks. + /// + /// The extent of the script block. + /// The list of using statments, may be null. + /// The ast for the param block, may be null. + /// The ast for the begin block, may be null. + /// The ast for the process block, may be null. + /// The ast for the end block, may be null. + /// The ast for the clean block, may be null. + /// The ast for the dynamicparam block, may be null. + /// + /// If is null. + /// + [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "param")] + public ScriptBlockAst( + IScriptExtent extent, + IEnumerable usingStatements, + ParamBlockAst paramBlock, + NamedBlockAst beginBlock, + NamedBlockAst processBlock, + NamedBlockAst endBlock, + NamedBlockAst cleanBlock, + NamedBlockAst dynamicParamBlock) + : this(extent, usingStatements, null, paramBlock, beginBlock, processBlock, endBlock, cleanBlock, dynamicParamBlock) + { + } + /// /// Construct a ScriptBlockAst that uses explicitly named begin/process/end blocks. /// @@ -911,6 +986,33 @@ public ScriptBlockAst(IScriptExtent extent, { } + /// + /// Initializes a new instance of the class. + /// This construction uses explicitly named begin/process/end/clean blocks. + /// + /// The extent of the script block. + /// The ast for the param block, may be null. + /// The ast for the begin block, may be null. + /// The ast for the process block, may be null. + /// The ast for the end block, may be null. + /// The ast for the clean block, may be null. + /// The ast for the dynamicparam block, may be null. + /// + /// If is null. + /// + [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "param")] + public ScriptBlockAst( + IScriptExtent extent, + ParamBlockAst paramBlock, + NamedBlockAst beginBlock, + NamedBlockAst processBlock, + NamedBlockAst endBlock, + NamedBlockAst cleanBlock, + NamedBlockAst dynamicParamBlock) + : this(extent, null, paramBlock, beginBlock, processBlock, endBlock, cleanBlock, dynamicParamBlock) + { + } + /// /// Construct a ScriptBlockAst that does not use explicitly named blocks. /// @@ -1115,6 +1217,11 @@ private void SetUsingStatements(IEnumerable usingStatements) /// public NamedBlockAst EndBlock { get; } + /// + /// Gets the ast representing the clean block for a script block, or null if no clean block was specified. + /// + public NamedBlockAst CleanBlock { get; } + /// /// The ast representing the dynamicparam block for a script block, or null if no dynamicparam block was specified. /// @@ -1194,17 +1301,25 @@ public override Ast Copy() var newBeginBlock = CopyElement(this.BeginBlock); var newProcessBlock = CopyElement(this.ProcessBlock); var newEndBlock = CopyElement(this.EndBlock); + var newCleanBlock = CopyElement(this.CleanBlock); var newDynamicParamBlock = CopyElement(this.DynamicParamBlock); var newAttributes = CopyElements(this.Attributes); var newUsingStatements = CopyElements(this.UsingStatements); - var scriptBlockAst = new ScriptBlockAst(this.Extent, newUsingStatements, newAttributes, newParamBlock, newBeginBlock, newProcessBlock, - newEndBlock, newDynamicParamBlock) + return new ScriptBlockAst( + this.Extent, + newUsingStatements, + newAttributes, + newParamBlock, + newBeginBlock, + newProcessBlock, + newEndBlock, + newCleanBlock, + newDynamicParamBlock) { IsConfiguration = this.IsConfiguration, ScriptRequirements = this.ScriptRequirements }; - return scriptBlockAst; } internal string ToStringForSerialization() @@ -1367,17 +1482,27 @@ internal override AstVisitAction InternalVisit(AstVisitor visitor) } } - if (action == AstVisitAction.Continue && ParamBlock != null) - action = ParamBlock.InternalVisit(visitor); - if (action == AstVisitAction.Continue && DynamicParamBlock != null) - action = DynamicParamBlock.InternalVisit(visitor); - if (action == AstVisitAction.Continue && BeginBlock != null) - action = BeginBlock.InternalVisit(visitor); - if (action == AstVisitAction.Continue && ProcessBlock != null) - action = ProcessBlock.InternalVisit(visitor); - if (action == AstVisitAction.Continue && EndBlock != null) - action = EndBlock.InternalVisit(visitor); + if (action == AstVisitAction.Continue) + { + _ = VisitAndShallContinue(ParamBlock) && + VisitAndShallContinue(DynamicParamBlock) && + VisitAndShallContinue(BeginBlock) && + VisitAndShallContinue(ProcessBlock) && + VisitAndShallContinue(EndBlock) && + VisitAndShallContinue(CleanBlock); + } + return visitor.CheckForPostAction(this, action); + + bool VisitAndShallContinue(Ast ast) + { + if (ast is not null) + { + action = ast.InternalVisit(visitor); + } + + return action == AstVisitAction.Continue; + } } #endregion Visitors @@ -1581,9 +1706,12 @@ bool IParameterMetadataProvider.UsesCmdletBinding() internal PipelineAst GetSimplePipeline(bool allowMultiplePipelines, out string errorId, out string errorMsg) { - if (BeginBlock != null || ProcessBlock != null || DynamicParamBlock != null) + if (BeginBlock != null + || ProcessBlock != null + || CleanBlock != null + || DynamicParamBlock != null) { - errorId = "CanConvertOneClauseOnly"; + errorId = nameof(AutomationExceptions.CanConvertOneClauseOnly); errorMsg = AutomationExceptions.CanConvertOneClauseOnly; return null; } @@ -1749,7 +1877,7 @@ internal static bool UsesCmdletBinding(IEnumerable parameters) public class NamedBlockAst : Ast { /// - /// Construct the ast for a begin, process, end, or dynamic param block. + /// Construct the ast for a begin, process, end, clean, or dynamic param block. /// /// /// The extent of the block. If is false, the extent includes @@ -1761,6 +1889,7 @@ public class NamedBlockAst : Ast /// /// /// + /// /// /// /// @@ -1779,8 +1908,7 @@ public NamedBlockAst(IScriptExtent extent, TokenKind blockName, StatementBlockAs { // Validate the block name. If the block is unnamed, it must be an End block (for a function) // or Process block (for a filter). - if (!blockName.HasTrait(TokenFlags.ScriptBlockBlockName) - || (unnamed && (blockName == TokenKind.Begin || blockName == TokenKind.Dynamicparam))) + if (HasInvalidBlockName(blockName, unnamed)) { throw PSTraceSource.NewArgumentException(nameof(blockName)); } @@ -1838,6 +1966,7 @@ public NamedBlockAst(IScriptExtent extent, TokenKind blockName, StatementBlockAs /// /// /// + /// /// /// /// @@ -1877,6 +2006,14 @@ public override Ast Copy() return new NamedBlockAst(this.Extent, this.BlockKind, statementBlock, this.Unnamed); } + private static bool HasInvalidBlockName(TokenKind blockName, bool unnamed) + { + return !blockName.HasTrait(TokenFlags.ScriptBlockBlockName) + || (unnamed + && blockName != TokenKind.Process + && blockName != TokenKind.End); + } + // Used by the debugger for command breakpoints internal IScriptExtent OpenCurlyExtent { get; } diff --git a/src/System.Management.Automation/engine/parser/token.cs b/src/System.Management.Automation/engine/parser/token.cs index 893ec9fc8e0..43a3e86ebb7 100644 --- a/src/System.Management.Automation/engine/parser/token.cs +++ b/src/System.Management.Automation/engine/parser/token.cs @@ -588,6 +588,9 @@ public enum TokenKind /// The 'default' keyword Default = 169, + /// The 'clean' keyword. + Clean = 170, + #endregion Keywords } @@ -659,7 +662,7 @@ public enum TokenFlags Keyword = 0x00000010, /// - /// The token one of the keywords that is a part of a script block: 'begin', 'process', 'end', or 'dynamicparam'. + /// The token is one of the keywords that is a part of a script block: 'begin', 'process', 'end', 'clean', or 'dynamicparam'. /// ScriptBlockBlockName = 0x00000020, @@ -948,6 +951,7 @@ public static class TokenTraits /* Hidden */ TokenFlags.Keyword, /* Base */ TokenFlags.Keyword, /* Default */ TokenFlags.Keyword, + /* Clean */ TokenFlags.Keyword | TokenFlags.ScriptBlockBlockName, #endregion Flags for keywords }; @@ -1147,6 +1151,7 @@ public static class TokenTraits /* Hidden */ "hidden", /* Base */ "base", /* Default */ "default", + /* Clean */ "clean", #endregion Text for keywords }; @@ -1154,10 +1159,12 @@ public static class TokenTraits #if DEBUG static TokenTraits() { - Diagnostics.Assert(s_staticTokenFlags.Length == ((int)TokenKind.Default + 1), - "Table size out of sync with enum - _staticTokenFlags"); - Diagnostics.Assert(s_tokenText.Length == ((int)TokenKind.Default + 1), - "Table size out of sync with enum - _tokenText"); + Diagnostics.Assert( + s_staticTokenFlags.Length == ((int)TokenKind.Clean + 1), + "Table size out of sync with enum - _staticTokenFlags"); + Diagnostics.Assert( + s_tokenText.Length == ((int)TokenKind.Clean + 1), + "Table size out of sync with enum - _tokenText"); // Some random assertions to make sure the enum and the traits are in sync Diagnostics.Assert(GetTraits(TokenKind.Begin) == (TokenFlags.Keyword | TokenFlags.ScriptBlockBlockName), "Table out of sync with enum - flags Begin"); diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 161e53cef0c..43046c9d29e 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -635,23 +635,23 @@ private static readonly Dictionary s_operatorTable /*A*/ "configuration", "public", "private", "static", /*A*/ /*B*/ "interface", "enum", "namespace", "module", /*B*/ /*C*/ "type", "assembly", "command", "hidden", /*C*/ - /*D*/ "base", "default", /*D*/ + /*D*/ "base", "default", "clean", /*D*/ }; private static readonly TokenKind[] s_keywordTokenKind = new TokenKind[] { - /*1*/ TokenKind.ElseIf, TokenKind.If, TokenKind.Else, TokenKind.Switch, /*1*/ - /*2*/ TokenKind.Foreach, TokenKind.From, TokenKind.In, TokenKind.For, /*2*/ - /*3*/ TokenKind.While, TokenKind.Until, TokenKind.Do, TokenKind.Try, /*3*/ - /*4*/ TokenKind.Catch, TokenKind.Finally, TokenKind.Trap, TokenKind.Data, /*4*/ - /*5*/ TokenKind.Return, TokenKind.Continue, TokenKind.Break, TokenKind.Exit, /*5*/ - /*6*/ TokenKind.Throw, TokenKind.Begin, TokenKind.Process, TokenKind.End, /*6*/ - /*7*/ TokenKind.Dynamicparam, TokenKind.Function, TokenKind.Filter, TokenKind.Param, /*7*/ - /*8*/ TokenKind.Class, TokenKind.Define, TokenKind.Var, TokenKind.Using, /*8*/ - /*9*/ TokenKind.Workflow, TokenKind.Parallel, TokenKind.Sequence, TokenKind.InlineScript, /*9*/ - /*A*/ TokenKind.Configuration, TokenKind.Public, TokenKind.Private, TokenKind.Static, /*A*/ - /*B*/ TokenKind.Interface, TokenKind.Enum, TokenKind.Namespace, TokenKind.Module, /*B*/ - /*C*/ TokenKind.Type, TokenKind.Assembly, TokenKind.Command, TokenKind.Hidden, /*C*/ - /*D*/ TokenKind.Base, TokenKind.Default, /*D*/ + /*1*/ TokenKind.ElseIf, TokenKind.If, TokenKind.Else, TokenKind.Switch, /*1*/ + /*2*/ TokenKind.Foreach, TokenKind.From, TokenKind.In, TokenKind.For, /*2*/ + /*3*/ TokenKind.While, TokenKind.Until, TokenKind.Do, TokenKind.Try, /*3*/ + /*4*/ TokenKind.Catch, TokenKind.Finally, TokenKind.Trap, TokenKind.Data, /*4*/ + /*5*/ TokenKind.Return, TokenKind.Continue, TokenKind.Break, TokenKind.Exit, /*5*/ + /*6*/ TokenKind.Throw, TokenKind.Begin, TokenKind.Process, TokenKind.End, /*6*/ + /*7*/ TokenKind.Dynamicparam, TokenKind.Function, TokenKind.Filter, TokenKind.Param, /*7*/ + /*8*/ TokenKind.Class, TokenKind.Define, TokenKind.Var, TokenKind.Using, /*8*/ + /*9*/ TokenKind.Workflow, TokenKind.Parallel, TokenKind.Sequence, TokenKind.InlineScript, /*9*/ + /*A*/ TokenKind.Configuration, TokenKind.Public, TokenKind.Private, TokenKind.Static, /*A*/ + /*B*/ TokenKind.Interface, TokenKind.Enum, TokenKind.Namespace, TokenKind.Module, /*B*/ + /*C*/ TokenKind.Type, TokenKind.Assembly, TokenKind.Command, TokenKind.Hidden, /*C*/ + /*D*/ TokenKind.Base, TokenKind.Default, TokenKind.Clean, /*D*/ }; internal static readonly string[] _operatorText = new string[] { @@ -699,8 +699,16 @@ static Tokenizer() Diagnostics.Assert(s_keywordText.Length == s_keywordTokenKind.Length, "Keyword table sizes must match"); Diagnostics.Assert(_operatorText.Length == s_operatorTokenKind.Length, "Operator table sizes must match"); + bool isCleanBlockFeatureEnabled = ExperimentalFeature.IsEnabled(ExperimentalFeature.PSCleanBlockFeatureName); + for (int i = 0; i < s_keywordText.Length; ++i) { + if (!isCleanBlockFeatureEnabled && s_keywordText[i] == "clean") + { + // Skip adding the 'clean' keyword when the feature is disabled. + continue; + } + s_keywordTable.Add(s_keywordText[i], s_keywordTokenKind[i]); } diff --git a/src/System.Management.Automation/engine/pipeline.cs b/src/System.Management.Automation/engine/pipeline.cs index 78fb19dbc65..185e60e072f 100644 --- a/src/System.Management.Automation/engine/pipeline.cs +++ b/src/System.Management.Automation/engine/pipeline.cs @@ -66,7 +66,6 @@ internal class PipelineProcessor : IDisposable public void Dispose() { Dispose(true); - GC.SuppressFinalize(this); } private void Dispose(bool disposing) @@ -267,9 +266,12 @@ internal int Add(CommandProcessorBase commandProcessor) internal void AddRedirectionPipe(PipelineProcessor pipelineProcessor) { - if (pipelineProcessor == null) throw PSTraceSource.NewArgumentNullException(nameof(pipelineProcessor)); - if (_redirectionPipes == null) - _redirectionPipes = new List(); + if (pipelineProcessor is null) + { + throw PSTraceSource.NewArgumentNullException(nameof(pipelineProcessor)); + } + + _redirectionPipes ??= new List(); _redirectionPipes.Add(pipelineProcessor); } @@ -347,18 +349,15 @@ private int AddCommand(CommandProcessorBase commandProcessor, int readFromComman } else { - CommandProcessorBase prevcommandProcessor = _commands[readFromCommand - 1] as CommandProcessorBase; - if (prevcommandProcessor == null || prevcommandProcessor.CommandRuntime == null) - { - // "PipelineProcessor.AddCommand(): previous request object == null" - throw PSTraceSource.NewInvalidOperationException(); - } + var prevcommandProcessor = _commands[readFromCommand - 1] as CommandProcessorBase; + ValidateCommandProcessorNotNull(prevcommandProcessor, errorMessage: null); + + Pipe UpstreamPipe = (readErrorQueue) + ? prevcommandProcessor.CommandRuntime.ErrorOutputPipe + : prevcommandProcessor.CommandRuntime.OutputPipe; - Pipe UpstreamPipe = (readErrorQueue) ? - prevcommandProcessor.CommandRuntime.ErrorOutputPipe : prevcommandProcessor.CommandRuntime.OutputPipe; if (UpstreamPipe == null) { - // "PipelineProcessor.AddCommand(): UpstreamPipe == null" throw PSTraceSource.NewInvalidOperationException(); } @@ -381,11 +380,8 @@ private int AddCommand(CommandProcessorBase commandProcessor, int readFromComman for (int i = 0; i < _commands.Count; i++) { prevcommandProcessor = _commands[i]; - if (prevcommandProcessor == null || prevcommandProcessor.CommandRuntime == null) - { - // "PipelineProcessor.AddCommand(): previous request object == null" - throw PSTraceSource.NewInvalidOperationException(); - } + ValidateCommandProcessorNotNull(prevcommandProcessor, errorMessage: null); + // check whether the error output is already claimed if (prevcommandProcessor.CommandRuntime.ErrorOutputPipe.DownstreamCmdlet != null) continue; @@ -470,194 +466,305 @@ internal Array SynchronousExecuteEnumerate(object input) throw new PipelineStoppedException(); } - ExceptionDispatchInfo toRethrowInfo; + bool pipelineSucceeded = false; + ExceptionDispatchInfo toRethrowInfo = null; + CommandProcessorBase commandRequestingUpstreamCommandsToStop = null; + try { - CommandProcessorBase commandRequestingUpstreamCommandsToStop = null; try { - // If the caller specified an input object array, - // we run assuming there is an incoming "stream" - // of objects. This will prevent the one default call - // to ProcessRecord on the first command. - Start(input != AutomationNull.Value); + try + { + // If the caller specified an input object array, we run assuming there is an incoming "stream" + // of objects. This will prevent the one default call to ProcessRecord on the first command. + Start(incomingStream: input != AutomationNull.Value); - // Start has already validated firstcommandProcessor - CommandProcessorBase firstCommandProcessor = _commands[0]; + // Start has already validated firstcommandProcessor + CommandProcessorBase firstCommandProcessor = _commands[0]; - // Add any input to the first command. - if (ExternalInput != null) + // Add any input to the first command. + if (ExternalInput is not null) + { + firstCommandProcessor.CommandRuntime.InputPipe.ExternalReader = ExternalInput; + } + + Inject(input, enumerate: true); + } + catch (PipelineStoppedException) { - firstCommandProcessor.CommandRuntime.InputPipe.ExternalReader - = ExternalInput; + if (_firstTerminatingError?.SourceException is StopUpstreamCommandsException exception) + { + _firstTerminatingError = null; + commandRequestingUpstreamCommandsToStop = exception.RequestingCommandProcessor; + } + else + { + throw; + } } - Inject(input, enumerate: true); + DoCompleteCore(commandRequestingUpstreamCommandsToStop); + pipelineSucceeded = true; } - catch (PipelineStoppedException) + finally { - StopUpstreamCommandsException stopUpstreamCommandsException = - _firstTerminatingError != null - ? _firstTerminatingError.SourceException as StopUpstreamCommandsException - : null; - if (stopUpstreamCommandsException == null) - { - throw; - } - else - { - _firstTerminatingError = null; - commandRequestingUpstreamCommandsToStop = stopUpstreamCommandsException.RequestingCommandProcessor; - } + // Clean up resources for script commands, no matter the pipeline succeeded or not. + // This method catches and handles all exceptions inside, so it will never throw. + Clean(); } - DoCompleteCore(commandRequestingUpstreamCommandsToStop); - - // By this point, we are sure all commandProcessors hosted by the current pipelineProcess are done execution, - // so if there are any redirection pipelineProcessors associated with any of those commandProcessors, we should - // call DoComplete on them. - if (_redirectionPipes != null) + if (pipelineSucceeded) { - foreach (PipelineProcessor redirectPipelineProcessor in _redirectionPipes) + // Now, we are sure all 'commandProcessors' hosted by the current 'pipelineProcessor' are done execution, + // so if there are any redirection 'pipelineProcessors' associated with any of those 'commandProcessors', + // they must have successfully executed 'StartStepping' and 'Step', and thus we should call 'DoComplete' + // on them for completeness. + if (_redirectionPipes is not null) { - redirectPipelineProcessor.DoCompleteCore(null); + foreach (PipelineProcessor redirectPipelineProcessor in _redirectionPipes) + { + // The 'Clean' block for each 'commandProcessor' might still write to a pipe that is associated + // with the redirection 'pipelineProcessor' (e.g. a redirected error pipe), which would trigger + // the call to 'pipelineProcessor.Step'. + // It's possible (though very unlikely) that the call to 'pipelineProcessor.Step' failed with an + // exception, and in such case, the 'pipelineProcessor' would have been disposed, and therefore + // the call to 'DoComplete' will simply return, because '_commands' was already set to null. + redirectPipelineProcessor.DoCompleteCore(null); + } } - } - return RetrieveResults(); + // The 'Clean' blocks write nothing to the output pipe, so the results won't be affected by them. + return RetrieveResults(); + } } catch (RuntimeException e) { - // The error we want to report is the first terminating error - // which occurred during pipeline execution, regardless - // of whether other errors occurred afterward. - toRethrowInfo = _firstTerminatingError ?? ExceptionDispatchInfo.Capture(e); - this.LogExecutionException(toRethrowInfo.SourceException); - } - // NTRAID#Windows Out Of Band Releases-929020-2006/03/14-JonN - catch (System.Runtime.InteropServices.InvalidComObjectException comException) - { - // The error we want to report is the first terminating error - // which occurred during pipeline execution, regardless - // of whether other errors occurred afterward. - if (_firstTerminatingError != null) - { - toRethrowInfo = _firstTerminatingError; - } - else - { - string message = StringUtil.Format(ParserStrings.InvalidComObjectException, comException.Message); - var rte = new RuntimeException(message, comException); - rte.SetErrorId("InvalidComObjectException"); - toRethrowInfo = ExceptionDispatchInfo.Capture(rte); - } - - this.LogExecutionException(toRethrowInfo.SourceException); + toRethrowInfo = GetFirstError(e); } finally { DisposeCommands(); } - // By rethrowing the exception outside of the handler, - // we allow the CLR on X64/IA64 to free from the stack - // the exception records related to this exception. + // By rethrowing the exception outside of the handler, we allow the CLR on X64/IA64 to free from + // the stack the exception records related to this exception. - // The only reason we should get here is if - // an exception should be rethrown. + // The only reason we should get here is if an exception should be rethrown. Diagnostics.Assert(toRethrowInfo != null, "Alternate protocol path failure"); toRethrowInfo.Throw(); - return null; // UNREACHABLE + + // UNREACHABLE + return null; + } + + private ExceptionDispatchInfo GetFirstError(RuntimeException e) + { + // The error we want to report is the first terminating error which occurred during pipeline execution, + // regardless of whether other errors occurred afterward. + var firstError = _firstTerminatingError ?? ExceptionDispatchInfo.Capture(e); + LogExecutionException(firstError.SourceException); + return firstError; + } + + private void ThrowFirstErrorIfExisting(bool logException) + { + if (_firstTerminatingError != null) + { + if (logException) + { + LogExecutionException(_firstTerminatingError.SourceException); + } + + _firstTerminatingError.Throw(); + } } private void DoCompleteCore(CommandProcessorBase commandRequestingUpstreamCommandsToStop) { - // Call DoComplete() for all the commands. DoComplete() will internally call Complete() + if (_commands is null) + { + // This could happen to a redirection pipeline, either for an expression (e.g. 1 > a.txt) + // or for a command (e.g. command > a.txt). + // An exception may be thrown from the call to 'StartStepping' or 'Step' on the pipeline, + // which causes the pipeline commands to be disposed. + return; + } + + // Call DoComplete() for all the commands, which will internally call Complete() MshCommandRuntime lastCommandRuntime = null; - if (_commands != null) + for (int i = 0; i < _commands.Count; i++) { - for (int i = 0; i < _commands.Count; i++) - { - CommandProcessorBase commandProcessor = _commands[i]; + CommandProcessorBase commandProcessor = _commands[i]; - if (commandProcessor == null) - { - // "null command " + i - throw PSTraceSource.NewInvalidOperationException(); - } + if (commandProcessor is null) + { + // An internal error that should not happen. + throw PSTraceSource.NewInvalidOperationException(); + } - if (object.ReferenceEquals(commandRequestingUpstreamCommandsToStop, commandProcessor)) - { - commandRequestingUpstreamCommandsToStop = null; - continue; // do not call DoComplete/EndProcessing on the command that initiated stopping - } + if (object.ReferenceEquals(commandRequestingUpstreamCommandsToStop, commandProcessor)) + { + // Do not call DoComplete/EndProcessing on the command that initiated stopping. + commandRequestingUpstreamCommandsToStop = null; + continue; + } - if (commandRequestingUpstreamCommandsToStop != null) - { - continue; // do not call DoComplete/EndProcessing on commands that were stopped upstream - } + if (commandRequestingUpstreamCommandsToStop is not null) + { + // Do not call DoComplete/EndProcessing on commands that were stopped upstream. + continue; + } - try + try + { + commandProcessor.DoComplete(); + } + catch (PipelineStoppedException) + { + if (_firstTerminatingError?.SourceException is StopUpstreamCommandsException exception) { - commandProcessor.DoComplete(); + _firstTerminatingError = null; + commandRequestingUpstreamCommandsToStop = exception.RequestingCommandProcessor; } - catch (PipelineStoppedException) + else { - StopUpstreamCommandsException stopUpstreamCommandsException = - _firstTerminatingError != null - ? _firstTerminatingError.SourceException as StopUpstreamCommandsException - : null; - if (stopUpstreamCommandsException == null) - { - throw; - } - else - { - _firstTerminatingError = null; - commandRequestingUpstreamCommandsToStop = stopUpstreamCommandsException.RequestingCommandProcessor; - } + throw; } + } - EtwActivity.SetActivityId(commandProcessor.PipelineActivityId); - - // Log a command stopped event - MshLog.LogCommandLifecycleEvent( - commandProcessor.Command.Context, - CommandState.Stopped, - commandProcessor.Command.MyInvocation); + EtwActivity.SetActivityId(commandProcessor.PipelineActivityId); - // Log the execution of a command (not script chunks, as they - // are not commands in and of themselves) - if (commandProcessor.CommandInfo.CommandType != CommandTypes.Script) - { - commandProcessor.CommandRuntime.PipelineProcessor.LogExecutionComplete( - commandProcessor.Command.MyInvocation, commandProcessor.CommandInfo.Name); - } + // Log a command stopped event + MshLog.LogCommandLifecycleEvent( + commandProcessor.Command.Context, + CommandState.Stopped, + commandProcessor.Command.MyInvocation); - lastCommandRuntime = commandProcessor.CommandRuntime; + // Log the execution of a command (not script chunks, as they are not commands in and of themselves). + if (commandProcessor.CommandInfo.CommandType != CommandTypes.Script) + { + LogExecutionComplete(commandProcessor.Command.MyInvocation, commandProcessor.CommandInfo.Name); } + + lastCommandRuntime = commandProcessor.CommandRuntime; } // Log the pipeline completion. - if (lastCommandRuntime != null) + if (lastCommandRuntime is not null) { // Only log the pipeline completion if this wasn't a nested pipeline, as // pipeline state in transcription is associated with the toplevel pipeline - if ((this.LocalPipeline == null) || (!this.LocalPipeline.IsNested)) + if (LocalPipeline is null || !LocalPipeline.IsNested) { lastCommandRuntime.PipelineProcessor.LogPipelineComplete(); } } // If a terminating error occurred, report it now. - if (_firstTerminatingError != null) + // This pipeline could have been stopped asynchronously, by 'Ctrl+c' manually or + // 'PowerShell.Stop' programatically. We need to check and see if that's the case. + // An example: + // - 'Start-Sleep' is running in this pipeline, and 'pipelineProcessor.Stop' gets + // called on a different thread, which sets a 'PipelineStoppedException' object + // to '_firstTerminatingError' and runs 'StopProcessing' on 'Start-Sleep'. + // - The 'StopProcessing' will cause 'Start-Sleep' to return from 'ProcessRecord' + // call, and thus the pipeline execution will move forward to run 'DoComplete' + // for the 'Start-Sleep' command and thus the code flow will reach here. + // For this given example, we need to check '_firstTerminatingError' and throw out + // the 'PipelineStoppedException' if the pipeline was indeed being stopped. + ThrowFirstErrorIfExisting(logException: true); + } + + /// + /// Clean up resources for script commands in this pipeline processor. + /// + /// + /// Exception from a 'Clean' block is not allowed to propagate up and terminate the pipeline + /// so that other 'Clean' blocks can run without being affected. Therefore, this method will + /// catch and handle all exceptions inside, and it will never throw. + /// + private void Clean() + { + if (!_executionStarted || _commands is null) { - this.LogExecutionException(_firstTerminatingError.SourceException); - _firstTerminatingError.Throw(); + // Simply return if the pipeline execution wasn't even started, or the commands of + // the pipeline have already been disposed. + return; + } + + // So far, if '_firstTerminatingError' is not null, then it must be a terminating error + // thrown from one of 'Begin/Process/End' blocks. There can be terminating error thrown + // from 'Clean' block as well, which needs to be handled in this method. + // In order to capture the subsequent first terminating error thrown from 'Clean', we + // need to forget the previous '_firstTerminatingError' value before calling 'DoClean' + // on each command processor, so we have to save the old value here and restore later. + ExceptionDispatchInfo oldFirstTerminatingError = _firstTerminatingError; + + // Suspend a stopping pipeline by setting 'IsStopping' to false and restore it afterwards. + bool oldIsStopping = ExceptionHandlingOps.SuspendStoppingPipelineImpl(LocalPipeline); + + try + { + foreach (CommandProcessorBase commandProcessor in _commands) + { + if (commandProcessor is null || !commandProcessor.HasCleanBlock) + { + continue; + } + + try + { + // Forget the terminating error we saw before, so a terminating error thrown + // from the subsequent 'Clean' block can be recorded and handled properly. + _firstTerminatingError = null; + commandProcessor.DoCleanup(); + } + catch (RuntimeException e) + { + // Retrieve and report the terminating error that was thrown in the 'Clean' block. + ExceptionDispatchInfo firstError = GetFirstError(e); + commandProcessor.ReportCleanupError(firstError.SourceException); + } + catch (Exception ex) + { + // Theoretically, only 'RuntimeException' could be thrown out, but we catch + // all and log them here just to be safe. + // Skip special flow control exceptions and log others. + if (ex is not FlowControlException && ex is not HaltCommandException) + { + MshLog.LogCommandHealthEvent(commandProcessor.Context, ex, Severity.Warning); + } + } + } + } + finally + { + _firstTerminatingError = oldFirstTerminatingError; + ExceptionHandlingOps.RestoreStoppingPipelineImpl(LocalPipeline, oldIsStopping); } } + /// + /// Clean up resources for the script commands of a steppable pipeline. + /// + /// + /// The way we handle 'Clean' blocks in 'StartStepping', 'Step', and 'DoComplete' makes sure that: + /// 1. The 'Clean' blocks get to run if any exception is thrown from the pipeline execution. + /// 2. The 'Clean' blocks get to run if the pipeline runs to the end successfully. + /// However, this is not enough for a steppable pipeline, because the function, where the steppable + /// pipeline gets used, may fail (think about a proxy function). And that may lead to the situation + /// where "no exception was thrown from the steppable pipeline" but "the steppable pipeline didn't + /// run to the end". In that case, 'Clean' won't run unless it's triggered explicitly on the steppable + /// pipeline. This method is how we will expose this functionality to 'SteppablePipeline'. + /// + internal void DoCleanup() + { + Clean(); + DisposeCommands(); + } + /// /// Implements DoComplete as a stand-alone function for completing /// the execution of a steppable pipeline. @@ -665,98 +772,79 @@ private void DoCompleteCore(CommandProcessorBase commandRequestingUpstreamComman /// The results of the execution. internal Array DoComplete() { - if (Stopping) - { - throw new PipelineStoppedException(); - } - if (!_executionStarted) { throw PSTraceSource.NewInvalidOperationException( PipelineStrings.PipelineNotStarted); } - ExceptionDispatchInfo toRethrowInfo; try { - DoCompleteCore(null); + if (Stopping) + { + throw new PipelineStoppedException(); + } - return RetrieveResults(); - } - catch (RuntimeException e) - { - // The error we want to report is the first terminating error - // which occurred during pipeline execution, regardless - // of whether other errors occurred afterward. - toRethrowInfo = _firstTerminatingError ?? ExceptionDispatchInfo.Capture(e); - this.LogExecutionException(toRethrowInfo.SourceException); - } - // NTRAID#Windows Out Of Band Releases-929020-2006/03/14-JonN - catch (System.Runtime.InteropServices.InvalidComObjectException comException) - { - // The error we want to report is the first terminating error - // which occurred during pipeline execution, regardless - // of whether other errors occurred afterward. - if (_firstTerminatingError != null) + ExceptionDispatchInfo toRethrowInfo; + try { - toRethrowInfo = _firstTerminatingError; + DoCompleteCore(null); + return RetrieveResults(); } - else + catch (RuntimeException e) { - string message = StringUtil.Format(ParserStrings.InvalidComObjectException, comException.Message); - var rte = new RuntimeException(message, comException); - rte.SetErrorId("InvalidComObjectException"); - toRethrowInfo = ExceptionDispatchInfo.Capture(rte); + toRethrowInfo = GetFirstError(e); } - this.LogExecutionException(toRethrowInfo.SourceException); + // By rethrowing the exception outside of the handler, we allow the CLR on X64/IA64 to free from the stack + // the exception records related to this exception. + + // The only reason we should get here is an exception should be rethrown. + Diagnostics.Assert(toRethrowInfo != null, "Alternate protocol path failure"); + toRethrowInfo.Throw(); + + // UNREACHABLE + return null; } finally { + Clean(); DisposeCommands(); } - - // By rethrowing the exception outside of the handler, - // we allow the CLR on X64/IA64 to free from the stack - // the exception records related to this exception. - - // The only reason we should get here is if - // an exception should be rethrown. - Diagnostics.Assert(toRethrowInfo != null, "Alternate protocol path failure"); - toRethrowInfo.Throw(); - return null; // UNREACHABLE } /// - /// This routine starts the stepping process. It is optional to - /// call this but can be useful if you want the begin clauses - /// of the pipeline to be run even when there may not be any input - /// to process as is the case for I/O redirection into a file. We - /// still want the file opened, even if there was nothing to write to it. + /// This routine starts the stepping process. It is optional to call this but can be useful + /// if you want the begin clauses of the pipeline to be run even when there may not be any + /// input to process as is the case for I/O redirection into a file. We still want the file + /// opened, even if there was nothing to write to it. /// /// True if you want to write to this pipeline. internal void StartStepping(bool expectInput) { + bool startSucceeded = false; try { Start(expectInput); + startSucceeded = true; - // If a terminating error occurred, report it now. - if (_firstTerminatingError != null) - { - _firstTerminatingError.Throw(); - } + // Check if this pipeline is being stopped asynchronously. + ThrowFirstErrorIfExisting(logException: false); } - catch (PipelineStoppedException) + catch (Exception e) { + Clean(); DisposeCommands(); - // The error we want to report is the first terminating error - // which occurred during pipeline execution, regardless - // of whether other errors occurred afterward. - if (_firstTerminatingError != null) + if (!startSucceeded && e is PipelineStoppedException) { - _firstTerminatingError.Throw(); + // When a terminating error happens during command execution, PowerShell will first save it + // to '_firstTerminatingError', and then throw a 'PipelineStoppedException' to tear down the + // pipeline. So when the caught exception here is 'PipelineStoppedException', it may not be + // the actual original terminating error. + // In this case, we want to report the first terminating error which occurred during pipeline + // execution, regardless of whether other errors occurred afterward. + ThrowFirstErrorIfExisting(logException: false); } throw; @@ -773,35 +861,35 @@ internal void Stop() // Only call StopProcessing if the pipeline is being stopped // for the first time - if (!RecordFailure(new PipelineStoppedException(), null)) + if (!RecordFailure(new PipelineStoppedException(), command: null)) + { return; + } // Retain copy of _commands in case Dispose() is called List commands = _commands; - if (commands == null) + if (commands is null) + { return; + } // Call StopProcessing() for all the commands. - for (int i = 0; i < commands.Count; i++) + foreach (CommandProcessorBase commandProcessor in commands) { - CommandProcessorBase commandProcessor = commands[i]; - if (commandProcessor == null) { throw PSTraceSource.NewInvalidOperationException(); } -#pragma warning disable 56500 + try { commandProcessor.Command.DoStopProcessing(); } catch (Exception) { - // 2004/04/26-JonN We swallow exceptions - // which occur during StopProcessing. + // We swallow exceptions which occur during StopProcessing. continue; } -#pragma warning restore 56500 } } @@ -844,43 +932,35 @@ internal void Stop() /// internal Array Step(object input) { - if (Stopping) - { - throw new PipelineStoppedException(); - } - + bool injectSucceeded = false; try { Start(true); Inject(input, enumerate: false); + injectSucceeded = true; - // If a terminating error occurred, report it now. - if (_firstTerminatingError != null) - { - _firstTerminatingError.Throw(); - } - + // Check if this pipeline is being stopped asynchronously. + ThrowFirstErrorIfExisting(logException: false); return RetrieveResults(); } - catch (PipelineStoppedException) + catch (Exception e) { + Clean(); DisposeCommands(); - // The error we want to report is the first terminating error - // which occurred during pipeline execution, regardless - // of whether other errors occurred afterward. - if (_firstTerminatingError != null) + if (!injectSucceeded && e is PipelineStoppedException) { - _firstTerminatingError.Throw(); + // When a terminating error happens during command execution, PowerShell will first save it + // to '_firstTerminatingError', and then throw a 'PipelineStoppedException' to tear down the + // pipeline. So when the caught exception here is 'PipelineStoppedException', it may not be + // the actual original terminating error. + // In this case, we want to report the first terminating error which occurred during pipeline + // execution, regardless of whether other errors occurred afterward. + ThrowFirstErrorIfExisting(logException: false); } throw; } - catch (Exception) - { - DisposeCommands(); - throw; - } } /// @@ -927,7 +1007,9 @@ private void Start(bool incomingStream) } if (_executionStarted) + { return; + } if (_commands == null || _commands.Count == 0) { @@ -936,12 +1018,7 @@ private void Start(bool incomingStream) } CommandProcessorBase firstcommandProcessor = _commands[0]; - if (firstcommandProcessor == null - || firstcommandProcessor.CommandRuntime == null) - { - throw PSTraceSource.NewInvalidOperationException( - PipelineStrings.PipelineExecuteRequiresAtLeastOneCommand); - } + ValidateCommandProcessorNotNull(firstcommandProcessor, PipelineStrings.PipelineExecuteRequiresAtLeastOneCommand); // Set the execution scope using the current scope if (_executionScope == null) @@ -951,17 +1028,11 @@ private void Start(bool incomingStream) // add ExternalSuccessOutput to the last command CommandProcessorBase LastCommandProcessor = _commands[_commands.Count - 1]; - if (LastCommandProcessor == null - || LastCommandProcessor.CommandRuntime == null) - { - // "PipelineProcessor.Start(): LastCommandProcessor == null" - throw PSTraceSource.NewInvalidOperationException(); - } + ValidateCommandProcessorNotNull(LastCommandProcessor, errorMessage: null); if (ExternalSuccessOutput != null) { - LastCommandProcessor.CommandRuntime.OutputPipe.ExternalWriter - = ExternalSuccessOutput; + LastCommandProcessor.CommandRuntime.OutputPipe.ExternalWriter = ExternalSuccessOutput; } // add ExternalErrorOutput to all commands whose error @@ -981,14 +1052,11 @@ private void Start(bool incomingStream) _executionStarted = true; - // // Allocate the pipeline iteration array; note that the pipeline position for // each command starts at 1 so we need to allocate _commands.Count + 1 items. - // int[] pipelineIterationInfo = new int[_commands.Count + 1]; - // Prepare all commands from Engine's side, - // and make sure they are all valid + // Prepare all commands from Engine's side, and make sure they are all valid for (int i = 0; i < _commands.Count; i++) { CommandProcessorBase commandProcessor = _commands[i]; @@ -1000,8 +1068,6 @@ private void Start(bool incomingStream) // Generate new Activity Id for the thread Guid pipelineActivityId = EtwActivity.CreateActivityId(); - - // commandProcess.PipelineActivityId = new Activity id EtwActivity.SetActivityId(pipelineActivityId); commandProcessor.PipelineActivityId = pipelineActivityId; @@ -1011,20 +1077,16 @@ private void Start(bool incomingStream) CommandState.Started, commandProcessor.Command.MyInvocation); - // Telemetry here - // the type of command should be sent along - // commandProcessor.CommandInfo.CommandType + // Send telemetry that includes the type of command. ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ApplicationType, commandProcessor.Command.CommandInfo.CommandType.ToString()); #if LEGACYTELEMETRY Microsoft.PowerShell.Telemetry.Internal.TelemetryAPI.TraceExecutedCommand(commandProcessor.Command.CommandInfo, commandProcessor.Command.CommandOrigin); #endif - // Log the execution of a command (not script chunks, as they - // are not commands in and of themselves) + // Log the execution of a command (not script chunks, as they are not commands in and of themselves) if (commandProcessor.CommandInfo.CommandType != CommandTypes.Script) { - commandProcessor.CommandRuntime.PipelineProcessor.LogExecutionInfo( - commandProcessor.Command.MyInvocation, commandProcessor.CommandInfo.Name); + LogExecutionInfo(commandProcessor.Command.MyInvocation, commandProcessor.CommandInfo.Name); } InvocationInfo myInfo = commandProcessor.Command.MyInvocation; @@ -1057,8 +1119,7 @@ private void Start(bool incomingStream) } /// - /// Add ExternalErrorOutput to all commands whose error - /// output is not yet claimed. + /// Add ExternalErrorOutput to all commands whose error output is not yet claimed. /// private void SetExternalErrorOutput() { @@ -1067,14 +1128,12 @@ private void SetExternalErrorOutput() for (int i = 0; i < _commands.Count; i++) { CommandProcessorBase commandProcessor = _commands[i]; - Pipe UpstreamPipe = - commandProcessor.CommandRuntime.ErrorOutputPipe; + Pipe errorPipe = commandProcessor.CommandRuntime.ErrorOutputPipe; // check whether a cmdlet is consuming the error pipe - if (!UpstreamPipe.IsRedirected) + if (!errorPipe.IsRedirected) { - UpstreamPipe.ExternalWriter = - ExternalErrorOutput; + errorPipe.ExternalWriter = ExternalErrorOutput; } } } @@ -1085,14 +1144,9 @@ private void SetExternalErrorOutput() /// private void SetupParameterVariables() { - for (int i = 0; i < _commands.Count; i++) + foreach (CommandProcessorBase commandProcessor in _commands) { - CommandProcessorBase commandProcessor = _commands[i]; - if (commandProcessor == null || commandProcessor.CommandRuntime == null) - { - // "null command " + i - throw PSTraceSource.NewInvalidOperationException(); - } + ValidateCommandProcessorNotNull(commandProcessor, errorMessage: null); commandProcessor.CommandRuntime.SetupOutVariable(); commandProcessor.CommandRuntime.SetupErrorVariable(); @@ -1102,6 +1156,16 @@ private void SetupParameterVariables() } } + private static void ValidateCommandProcessorNotNull(CommandProcessorBase commandProcessor, string errorMessage) + { + if (commandProcessor?.CommandRuntime is null) + { + throw errorMessage is null + ? PSTraceSource.NewInvalidOperationException() + : PSTraceSource.NewInvalidOperationException(errorMessage, Array.Empty()); + } + } + /// /// Partially execute the pipeline. The output remains in /// the pipes. @@ -1131,12 +1195,7 @@ private void Inject(object input, bool enumerate) { // Add any input to the first command. CommandProcessorBase firstcommandProcessor = _commands[0]; - if (firstcommandProcessor == null - || firstcommandProcessor.CommandRuntime == null) - { - throw PSTraceSource.NewInvalidOperationException( - PipelineStrings.PipelineExecuteRequiresAtLeastOneCommand); - } + ValidateCommandProcessorNotNull(firstcommandProcessor, PipelineStrings.PipelineExecuteRequiresAtLeastOneCommand); if (input != AutomationNull.Value) { @@ -1174,27 +1233,26 @@ private void Inject(object input, bool enumerate) /// private Array RetrieveResults() { + if (_commands is null) + { + // This could happen to an expression redirection pipeline (e.g. 1 > a.txt). + // An exception may be thrown from the call to 'StartStepping' or 'Step' on the pipeline, + // which causes the pipeline commands to be disposed. + return MshCommandRuntime.StaticEmptyArray; + } + // If the error queue has been linked, it's up to the link to // deal with the output. Don't do anything here... if (!_linkedErrorOutput) { - // Retrieve any accumulated error objects from each of the pipes - // and add them to the error results hash table. - for (int i = 0; i < _commands.Count; i++) + foreach (CommandProcessorBase commandProcessor in _commands) { - CommandProcessorBase commandProcessor = _commands[i]; - if (commandProcessor == null - || commandProcessor.CommandRuntime == null) - { - // "null command or request or ErrorOutputPipe " + i - throw PSTraceSource.NewInvalidOperationException(); - } + ValidateCommandProcessorNotNull(commandProcessor, errorMessage: null); Pipe ErrorPipe = commandProcessor.CommandRuntime.ErrorOutputPipe; if (ErrorPipe.DownstreamCmdlet == null && !ErrorPipe.Empty) { - // 2003/10/02-JonN - // Do not return the same error results more than once + // Clear the error pipe if it's not empty and will not be consumed. ErrorPipe.Clear(); } } @@ -1203,26 +1261,18 @@ private Array RetrieveResults() // If the success queue has been linked, it's up to the link to // deal with the output. Don't do anything here... if (_linkedSuccessOutput) + { return MshCommandRuntime.StaticEmptyArray; + } CommandProcessorBase LastCommandProcessor = _commands[_commands.Count - 1]; - if (LastCommandProcessor == null - || LastCommandProcessor.CommandRuntime == null) - { - // "PipelineProcessor.RetrieveResults(): LastCommandProcessor == null" - throw PSTraceSource.NewInvalidOperationException(); - } + ValidateCommandProcessorNotNull(LastCommandProcessor, errorMessage: null); - Array results = - LastCommandProcessor.CommandRuntime.GetResultsAsArray(); + Array results = LastCommandProcessor.CommandRuntime.GetResultsAsArray(); - // 2003/10/02-JonN // Do not return the same results more than once LastCommandProcessor.CommandRuntime.OutputPipe.Clear(); - - if (results == null) - return MshCommandRuntime.StaticEmptyArray; - return results; + return results is null ? MshCommandRuntime.StaticEmptyArray : results; } /// @@ -1236,12 +1286,7 @@ internal void LinkPipelineSuccessOutput(Pipe pipeToUse) Dbg.Assert(pipeToUse != null, "Caller should verify pipeToUse != null"); CommandProcessorBase LastCommandProcessor = _commands[_commands.Count - 1]; - if (LastCommandProcessor == null - || LastCommandProcessor.CommandRuntime == null) - { - // "PipelineProcessor.RetrieveResults(): LastCommandProcessor == null" - throw PSTraceSource.NewInvalidOperationException(); - } + ValidateCommandProcessorNotNull(LastCommandProcessor, errorMessage: null); LastCommandProcessor.CommandRuntime.OutputPipe = pipeToUse; _linkedSuccessOutput = true; @@ -1251,15 +1296,9 @@ internal void LinkPipelineErrorOutput(Pipe pipeToUse) { Dbg.Assert(pipeToUse != null, "Caller should verify pipeToUse != null"); - for (int i = 0; i < _commands.Count; i++) + foreach (CommandProcessorBase commandProcessor in _commands) { - CommandProcessorBase commandProcessor = _commands[i]; - if (commandProcessor == null - || commandProcessor.CommandRuntime == null) - { - // "null command or request or ErrorOutputPipe " + i - throw PSTraceSource.NewInvalidOperationException(); - } + ValidateCommandProcessorNotNull(commandProcessor, errorMessage: null); if (commandProcessor.CommandRuntime.ErrorOutputPipe.DownstreamCmdlet == null) { @@ -1280,76 +1319,65 @@ internal void LinkPipelineErrorOutput(Pipe pipeToUse) private void DisposeCommands() { // Note that this is not in a lock. - // We do not make Dispose() wait until StopProcessing() - // has completed. + // We do not make Dispose() wait until StopProcessing() has completed. _stopping = true; + if (_commands is null && _redirectionPipes is null) + { + // Commands were already disposed. + return; + } + LogToEventLog(); - if (_commands != null) + if (_commands is not null) { - for (int i = 0; i < _commands.Count; i++) + foreach (CommandProcessorBase commandProcessor in _commands) { - CommandProcessorBase commandProcessor = _commands[i]; - if (commandProcessor != null) + if (commandProcessor is null) { -#pragma warning disable 56500 - // If Dispose throws an exception, record it as a - // pipeline failure and continue disposing cmdlets. - try + continue; + } + + // If Dispose throws an exception, record it as a pipeline failure and continue disposing cmdlets. + try + { + // Only cmdlets can have variables defined via the common parameters. + // We handle the cleanup of those variables only if we need to. + if (commandProcessor is CommandProcessor) { - // Only cmdlets can have variables defined via the common parameters. - // We handle the cleanup of those variables only if we need to. - if (commandProcessor is CommandProcessor) + if (commandProcessor.Command is not PSScriptCmdlet) { - if (commandProcessor.Command is not PSScriptCmdlet) - { - // For script cmdlets, the variable lists were already removed when exiting a scope. - // So we only need to take care of binary cmdlets here. - commandProcessor.CommandRuntime.RemoveVariableListsInPipe(); - } - - // Remove the pipeline variable if we need to. - commandProcessor.CommandRuntime.RemovePipelineVariable(); + // For script cmdlets, the variable lists were already removed when exiting a scope. + // So we only need to take care of binary cmdlets here. + commandProcessor.CommandRuntime.RemoveVariableListsInPipe(); } - commandProcessor.Dispose(); + // Remove the pipeline variable if we need to. + commandProcessor.CommandRuntime.RemovePipelineVariable(); } - // 2005/04/13-JonN: The only vaguely plausible reason - // for a failure here is an exception in Command.Dispose. - // As such, this should be covered by the overall - // exemption. - catch (Exception e) // Catch-all OK, 3rd party callout. - { - InvocationInfo myInvocation = null; - if (commandProcessor.Command != null) - myInvocation = commandProcessor.Command.MyInvocation; - ProviderInvocationException pie = - e as ProviderInvocationException; - if (pie != null) - { - e = new CmdletProviderInvocationException( - pie, - myInvocation); - } - else - { - e = new CmdletInvocationException( - e, - myInvocation); - - // Log a command health event + commandProcessor.Dispose(); + } + catch (Exception e) + { + // The only vaguely plausible reason for a failure here is an exception in 'Command.Dispose'. + // As such, this should be covered by the overall exemption. + InvocationInfo myInvocation = commandProcessor.Command?.MyInvocation; - MshLog.LogCommandHealthEvent( - commandProcessor.Command.Context, - e, - Severity.Warning); - } + if (e is ProviderInvocationException pie) + { + e = new CmdletProviderInvocationException(pie, myInvocation); + } + else + { + e = new CmdletInvocationException(e, myInvocation); - RecordFailure(e, commandProcessor.Command); + // Log a command health event + MshLog.LogCommandHealthEvent(commandProcessor.Command.Context, e, Severity.Warning); } -#pragma warning restore 56500 + + RecordFailure(e, commandProcessor.Command); } } } @@ -1357,25 +1385,31 @@ private void DisposeCommands() _commands = null; // Now dispose any pipes that were used for redirection... - if (_redirectionPipes != null) + if (_redirectionPipes is not null) { foreach (PipelineProcessor redirPipe in _redirectionPipes) { -#pragma warning disable 56500 + if (redirPipe is null) + { + continue; + } + + // Clean resources for script commands. + // It is possible (though very unlikely) that the call to 'Step' on the redirection pipeline failed. + // In such a case, 'Clean' would have run and the 'pipelineProcessor' would have been disposed. + // Therefore, calling 'Clean' again will simply return, because '_commands' was already set to null. + redirPipe.Clean(); + // The complicated logic of disposing the commands is taken care // of through recursion, this routine should not be getting any // exceptions... try { - if (redirPipe != null) - { - redirPipe.Dispose(); - } + redirPipe.Dispose(); } catch (Exception) { } -#pragma warning restore 56500 } } @@ -1399,11 +1433,9 @@ internal bool RecordFailure(Exception e, InternalCommand command) { _firstTerminatingError = ExceptionDispatchInfo.Capture(e); } - // 905900-2005/05/12 - // Drop5: Error Architecture: Log/trace second and subsequent RecordFailure - // Note that the pipeline could have been stopped asynchronously - // before hitting the error, therefore we check whether - // firstTerminatingError is PipelineStoppedException. + // Error Architecture: Log/trace second and subsequent RecordFailure. + // Note that the pipeline could have been stopped asynchronously before hitting the error, + // therefore we check whether '_firstTerminatingError' is 'PipelineStoppedException'. else if (_firstTerminatingError.SourceException is not PipelineStoppedException && command?.Context != null) { @@ -1422,11 +1454,10 @@ internal bool RecordFailure(Exception e, InternalCommand command) ex.GetType().Name, ex.StackTrace ); - InvalidOperationException ioe - = new InvalidOperationException(message, ex); + MshLog.LogCommandHealthEvent( command.Context, - ioe, + new InvalidOperationException(message, ex), Severity.Warning); } } diff --git a/src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs b/src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs index 2e6db041126..79881796cad 100644 --- a/src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs +++ b/src/System.Management.Automation/engine/runtime/CompiledScriptBlock.cs @@ -12,11 +12,9 @@ using System.Management.Automation.Runspaces; using System.Management.Automation.Tracing; using System.Reflection; -using System.Runtime.CompilerServices; using System.Runtime.Serialization; using System.Security.Cryptography.X509Certificates; using System.Text; -using System.Threading.Tasks; #if LEGACYTELEMETRY using Microsoft.PowerShell.Telemetry.Internal; #endif @@ -35,6 +33,7 @@ internal enum ScriptBlockClauseToInvoke Begin, Process, End, + Clean, ProcessBlockOnly, } @@ -247,6 +246,7 @@ bool IsScriptBlockInFactASafeHashtable() if (scriptBlockAst.BeginBlock != null || scriptBlockAst.ProcessBlock != null + || scriptBlockAst.CleanBlock != null || scriptBlockAst.ParamBlock != null || scriptBlockAst.DynamicParamBlock != null || scriptBlockAst.ScriptRequirements != null @@ -316,6 +316,8 @@ private IParameterMetadataProvider DelayParseScriptText() internal Dictionary NameToIndexMap { get; set; } + #region Named Blocks + internal Action DynamicParamBlock { get; set; } internal Action UnoptimizedDynamicParamBlock { get; set; } @@ -332,6 +334,12 @@ private IParameterMetadataProvider DelayParseScriptText() internal Action UnoptimizedEndBlock { get; set; } + internal Action CleanBlock { get; set; } + + internal Action UnoptimizedCleanBlock { get; set; } + + #endregion Named Blocks + internal IScriptExtent[] SequencePoints { get; set; } private RuntimeDefinedParameterDictionary _runtimeDefinedParameterDictionary; @@ -753,7 +761,7 @@ private PipelineAst GetSimplePipeline(Func errorHandler) { errorHandler ??= (static _ => null); - if (HasBeginBlock || HasProcessBlock) + if (HasBeginBlock || HasProcessBlock || HasCleanBlock) { return errorHandler(AutomationExceptions.CanConvertOneClauseOnly); } @@ -891,7 +899,10 @@ public void CheckRestrictedLanguage( Parser parser = new Parser(); var ast = AstInternal; - if (HasBeginBlock || HasProcessBlock || ast.Body.ParamBlock != null) + if (HasBeginBlock + || HasProcessBlock + || HasCleanBlock + || ast.Body.ParamBlock is not null) { Ast errorAst = ast.Body.BeginBlock ?? (Ast)ast.Body.ProcessBlock ?? ast.Body.ParamBlock; parser.ReportError( @@ -974,6 +985,11 @@ internal void InvokeWithPipeImpl( InvocationInfo invocationInfo, params object[] args) { + if (clauseToInvoke == ScriptBlockClauseToInvoke.Clean) + { + throw new PSNotSupportedException(ParserStrings.InvokingCleanBlockNotSupported); + } + if ((clauseToInvoke == ScriptBlockClauseToInvoke.Begin && !HasBeginBlock) || (clauseToInvoke == ScriptBlockClauseToInvoke.Process && !HasProcessBlock) || (clauseToInvoke == ScriptBlockClauseToInvoke.End && !HasEndBlock)) @@ -991,7 +1007,7 @@ internal void InvokeWithPipeImpl( throw new PipelineStoppedException(); } - // Validate at the arguments are consistent. The only public API that gets you here never sets createLocalScope to false... + // Validate that the arguments are consistent. The only public API that gets you here never sets createLocalScope to false... Diagnostics.Assert( createLocalScope || functionsToDefine == null, "When calling ScriptBlock.InvokeWithContext(), if 'functionsToDefine' != null then 'createLocalScope' must be true"); @@ -1195,7 +1211,7 @@ internal void InvokeWithPipeImpl( _sequencePoints = SequencePoints, }; - ScriptBlock.LogScriptBlockStart(this, context.CurrentRunspace.InstanceId); + LogScriptBlockStart(this, context.CurrentRunspace.InstanceId); try { @@ -1203,7 +1219,7 @@ internal void InvokeWithPipeImpl( } finally { - ScriptBlock.LogScriptBlockEnd(this, context.CurrentRunspace.InstanceId); + LogScriptBlockEnd(this, context.CurrentRunspace.InstanceId); } } catch (TargetInvocationException tie) @@ -1362,7 +1378,7 @@ internal static void SetAutomaticVariable(AutomaticVariable variable, object val private Action GetCodeToInvoke(ref bool optimized, ScriptBlockClauseToInvoke clauseToInvoke) { if (clauseToInvoke == ScriptBlockClauseToInvoke.ProcessBlockOnly - && (HasBeginBlock || (HasEndBlock && HasProcessBlock))) + && (HasBeginBlock || HasCleanBlock || (HasEndBlock && HasProcessBlock))) { throw PSTraceSource.NewInvalidOperationException(AutomationExceptions.ScriptBlockInvokeOnOneClauseOnly); } @@ -1379,6 +1395,8 @@ private Action GetCodeToInvoke(ref bool optimized, ScriptBlockC return _scriptBlockData.ProcessBlock; case ScriptBlockClauseToInvoke.End: return _scriptBlockData.EndBlock; + case ScriptBlockClauseToInvoke.Clean: + return _scriptBlockData.CleanBlock; default: return HasProcessBlock ? _scriptBlockData.ProcessBlock : _scriptBlockData.EndBlock; } @@ -1392,6 +1410,8 @@ private Action GetCodeToInvoke(ref bool optimized, ScriptBlockC return _scriptBlockData.UnoptimizedProcessBlock; case ScriptBlockClauseToInvoke.End: return _scriptBlockData.UnoptimizedEndBlock; + case ScriptBlockClauseToInvoke.Clean: + return _scriptBlockData.UnoptimizedCleanBlock; default: return HasProcessBlock ? _scriptBlockData.UnoptimizedProcessBlock : _scriptBlockData.UnoptimizedEndBlock; } @@ -2147,11 +2167,17 @@ internal static void LogScriptBlockEnd(ScriptBlock scriptBlock, Guid runspaceId) internal Action UnoptimizedEndBlock { get => _scriptBlockData.UnoptimizedEndBlock; } + internal Action CleanBlock { get => _scriptBlockData.CleanBlock; } + + internal Action UnoptimizedCleanBlock { get => _scriptBlockData.UnoptimizedCleanBlock; } + internal bool HasBeginBlock { get => AstInternal.Body.BeginBlock != null; } internal bool HasProcessBlock { get => AstInternal.Body.ProcessBlock != null; } internal bool HasEndBlock { get => AstInternal.Body.EndBlock != null; } + + internal bool HasCleanBlock { get => AstInternal.Body.CleanBlock != null; } } [Serializable] @@ -2197,11 +2223,13 @@ internal sealed class PSScriptCmdlet : PSCmdlet, IDynamicParameters, IDisposable private readonly bool _useLocalScope; private readonly bool _runOptimized; private readonly bool _rethrowExitException; - private MshCommandRuntime _commandRuntime; private readonly MutableTuple _localsTuple; - private bool _exitWasCalled; private readonly FunctionContext _functionContext; + private MshCommandRuntime _commandRuntime; + private bool _exitWasCalled; + private bool _anyClauseExecuted; + public PSScriptCmdlet(ScriptBlock scriptBlock, bool useNewScope, bool fromScriptFile, ExecutionContext context) { _scriptBlock = scriptBlock; @@ -2291,6 +2319,34 @@ internal override void DoEndProcessing() } } + internal override void DoCleanResource() + { + if (_scriptBlock.HasCleanBlock && _anyClauseExecuted) + { + // The 'Clean' block doesn't write any output to pipeline, so we use a 'NullPipe' here and + // disallow the output to be collected by an 'out' variable. However, the error, warning, + // and information records should still be collectable by the corresponding variables. + Pipe oldOutputPipe = _commandRuntime.OutputPipe; + _functionContext._outputPipe = _commandRuntime.OutputPipe = new Pipe + { + NullPipe = true, + IgnoreOutVariableList = true, + }; + + try + { + RunClause( + clause: _runOptimized ? _scriptBlock.CleanBlock : _scriptBlock.UnoptimizedCleanBlock, + dollarUnderbar: AutomationNull.Value, + inputToProcess: AutomationNull.Value); + } + finally + { + _functionContext._outputPipe = _commandRuntime.OutputPipe = oldOutputPipe; + } + } + } + private void EnterScope() { _commandRuntime.SetVariableListsInPipe(); @@ -2303,6 +2359,7 @@ private void ExitScope() private void RunClause(Action clause, object dollarUnderbar, object inputToProcess) { + _anyClauseExecuted = true; Pipe oldErrorOutputPipe = this.Context.ShellFunctionErrorOutputPipe; // If the script block has a different language mode than the current, @@ -2356,9 +2413,9 @@ private void RunClause(Action clause, object dollarUnderbar, ob } finally { - this.Context.RestoreErrorPipe(oldErrorOutputPipe); + Context.ShellFunctionErrorOutputPipe = oldErrorOutputPipe; - // Set the language mode + // Restore the language mode if (oldLanguageMode.HasValue) { Context.LanguageMode = oldLanguageMode.Value; @@ -2518,9 +2575,6 @@ public void Dispose() commandRuntime = null; currentObjectInPipeline = null; _input.Clear(); - // _scriptBlock = null; - // _localsTuple = null; - // _functionContext = null; base.InternalDispose(true); _disposed = true; diff --git a/src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs b/src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs index de9ef697fc6..8d50fcf5f1a 100644 --- a/src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs +++ b/src/System.Management.Automation/engine/runtime/Operations/MiscOps.cs @@ -1578,23 +1578,33 @@ private static int FindMatchingHandlerByType(Type exceptionType, Type[] types) internal static bool SuspendStoppingPipeline(ExecutionContext context) { - LocalPipeline lpl = (LocalPipeline)context.CurrentRunspace.GetCurrentlyRunningPipeline(); - if (lpl != null) + var localPipeline = (LocalPipeline)context.CurrentRunspace.GetCurrentlyRunningPipeline(); + return SuspendStoppingPipelineImpl(localPipeline); + } + + internal static void RestoreStoppingPipeline(ExecutionContext context, bool oldIsStopping) + { + var localPipeline = (LocalPipeline)context.CurrentRunspace.GetCurrentlyRunningPipeline(); + RestoreStoppingPipelineImpl(localPipeline, oldIsStopping); + } + + internal static bool SuspendStoppingPipelineImpl(LocalPipeline localPipeline) + { + if (localPipeline is not null) { - bool oldIsStopping = lpl.Stopper.IsStopping; - lpl.Stopper.IsStopping = false; + bool oldIsStopping = localPipeline.Stopper.IsStopping; + localPipeline.Stopper.IsStopping = false; return oldIsStopping; } return false; } - internal static void RestoreStoppingPipeline(ExecutionContext context, bool oldIsStopping) + internal static void RestoreStoppingPipelineImpl(LocalPipeline localPipeline, bool oldIsStopping) { - LocalPipeline lpl = (LocalPipeline)context.CurrentRunspace.GetCurrentlyRunningPipeline(); - if (lpl != null) + if (localPipeline is not null) { - lpl.Stopper.IsStopping = oldIsStopping; + localPipeline.Stopper.IsStopping = oldIsStopping; } } @@ -2855,6 +2865,11 @@ internal static object ForEach(IEnumerator enumerator, object expression, object ScriptBlock sb = expression as ScriptBlock; if (sb != null) { + if (sb.HasCleanBlock) + { + throw new PSNotSupportedException(ParserStrings.ForEachNotSupportCleanBlock); + } + Pipe outputPipe = new Pipe(result); if (sb.HasBeginBlock) { diff --git a/src/System.Management.Automation/resources/ParserStrings.resx b/src/System.Management.Automation/resources/ParserStrings.resx index d21719157ac..a0569a6f4e4 100644 --- a/src/System.Management.Automation/resources/ParserStrings.resx +++ b/src/System.Management.Automation/resources/ParserStrings.resx @@ -497,7 +497,7 @@ The correct form is: foreach ($a in $b) {...} Script command clause '{0}' has already been defined. - unexpected token '{0}', expected 'begin', 'process', 'end', or 'dynamicparam'. + unexpected token '{0}', expected 'begin', 'process', 'end', 'clean', or 'dynamicparam'. Missing closing '}' in statement block or type definition. @@ -1126,6 +1126,9 @@ ModuleVersion : Version of module to import. If used, ModuleName must represent Unable to convert input to the target type [{0}] passed to the ForEach() operator. Please check the specified type and try running your script again. + + Script block with a 'clean' block is not supported by the 'ForEach' method. + The 'numberToReturn' value provided to the third argument of the Where() operator must be greater than zero. Please correct the argument's value and try running your script again. @@ -1479,4 +1482,7 @@ ModuleVersion : Version of module to import. If used, ModuleName must represent Background operators can only be used at the end of a pipeline chain. + + Directly invoking the 'clean' block of a script block is not supported. + diff --git a/test/powershell/Language/Parser/Parser.Tests.ps1 b/test/powershell/Language/Parser/Parser.Tests.ps1 index 0f8a6c74726..0368dfe9c49 100644 --- a/test/powershell/Language/Parser/Parser.Tests.ps1 +++ b/test/powershell/Language/Parser/Parser.Tests.ps1 @@ -67,6 +67,8 @@ Describe "ParserTests (admin\monad\tests\monad\src\engine\core\ParserTests.cs)" } end {} + + clean {} } '@ $functionDefinition>$functionDefinitionFile diff --git a/test/powershell/Language/Parser/Parsing.Tests.ps1 b/test/powershell/Language/Parser/Parsing.Tests.ps1 index c764830954b..97e46bcbe3c 100644 --- a/test/powershell/Language/Parser/Parsing.Tests.ps1 +++ b/test/powershell/Language/Parser/Parsing.Tests.ps1 @@ -148,17 +148,21 @@ Describe 'named blocks parsing' -Tags "CI" { ShouldBeParseError 'begin' MissingNamedStatementBlock 5 ShouldBeParseError 'process' MissingNamedStatementBlock 7 ShouldBeParseError 'end' MissingNamedStatementBlock 3 + ShouldBeParseError 'clean' MissingNamedStatementBlock 5 ShouldBeParseError 'dynamicparam' MissingNamedStatementBlock 12 ShouldBeParseError 'begin process {}' MissingNamedStatementBlock 6 -CheckColumnNumber ShouldBeParseError 'end process {}' MissingNamedStatementBlock 4 -CheckColumnNumber + ShouldBeParseError 'clean process {}' MissingNamedStatementBlock 6 -CheckColumnNumber ShouldBeParseError 'dynamicparam process {}' MissingNamedStatementBlock 13 -CheckColumnNumber ShouldBeParseError 'process begin {}' MissingNamedStatementBlock 8 -CheckColumnNumber - ShouldBeParseError 'begin process end' MissingNamedStatementBlock,MissingNamedStatementBlock,MissingNamedStatementBlock 6,14,18 -CheckColumnNumber + ShouldBeParseError 'begin process end clean' MissingNamedStatementBlock, MissingNamedStatementBlock, MissingNamedStatementBlock, MissingNamedStatementBlock 6, 14, 18, 24 -CheckColumnNumber Test-Ast 'begin' 'begin' 'begin' Test-Ast 'begin end' 'begin end' 'begin' 'end' Test-Ast 'begin end process' 'begin end process' 'begin' 'end' 'process' Test-Ast 'begin {} end' 'begin {} end' 'begin {}' 'end' + Test-Ast 'begin process end clean' 'begin process end clean' 'begin' 'clean' 'end' 'process' + Test-Ast 'begin {} process end clean {}' 'begin {} process end clean {}' 'begin {}' 'clean {}' 'end' 'process' } # diff --git a/test/powershell/Language/Scripting/CleanBlockErrorHandling.Tests.ps1 b/test/powershell/Language/Scripting/CleanBlockErrorHandling.Tests.ps1 new file mode 100644 index 00000000000..68904d4c79f --- /dev/null +++ b/test/powershell/Language/Scripting/CleanBlockErrorHandling.Tests.ps1 @@ -0,0 +1,1374 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +Describe "Error handling within a single 'Clean' block" -Tag 'CI' { + + BeforeAll { + function ErrorInClean { + [CmdletBinding()] + param( + [switch] $ThrowTerminatingError, + [switch] $ErrorActionStop, + [switch] $ThrowException, + [switch] $WriteErrorAPI, + [switch] $WriteErrorCmdlet, + [switch] $MethodInvocationThrowException, + [switch] $ExpressionThrowException + ) + + End { <# use an empty end block to allow the clean block to actually run #> } + + clean { + if ($ThrowTerminatingError) { + $ex = [System.ArgumentException]::new('terminating-exception') + $er = [System.Management.Automation.ErrorRecord]::new($ex, 'ThrowTerminatingError:error', 'InvalidArgument', $null) + $PSCmdlet.ThrowTerminatingError($er) + Write-Verbose -Verbose "verbose-message" + } + elseif ($ErrorActionStop) { + Get-Command NonExist -ErrorAction Stop + Write-Verbose -Verbose "verbose-message" + } + elseif ($ThrowException) { + throw 'throw-exception' + Write-Verbose -Verbose "verbose-message" + } + elseif ($WriteErrorAPI) { + $ex = [System.ArgumentException]::new('arg-exception') + $er = [System.Management.Automation.ErrorRecord]::new($ex, 'WriteErrorAPI:error', 'InvalidArgument', $null) + $PSCmdlet.WriteError($er) + Write-Verbose -Verbose "verbose-message" + } + elseif ($WriteErrorCmdlet) { + Write-Error 'write-error-cmdlet' + Write-Verbose -Verbose "verbose-message" + } + elseif ($MethodInvocationThrowException) { + ## This method call throws exception. + $iss = [initialsessionstate]::Create() + $iss.ImportPSModule($null) + Write-Verbose -Verbose "verbose-message" + } + elseif ($ExpressionThrowException) { + 1/0 ## throw exception. + Write-Verbose -Verbose "verbose-message" + } + } + } + + function ErrorInEnd { + [CmdletBinding()] + param( + [switch] $ThrowTerminatingError, + [switch] $ErrorActionStop, + [switch] $ThrowException, + [switch] $WriteErrorAPI, + [switch] $WriteErrorCmdlet, + [switch] $MethodInvocationThrowException, + [switch] $ExpressionThrowException + ) + + if ($ThrowTerminatingError) { + $ex = [System.ArgumentException]::new('terminating-exception') + $er = [System.Management.Automation.ErrorRecord]::new($ex, 'ThrowTerminatingError:error', 'InvalidArgument', $null) + $PSCmdlet.ThrowTerminatingError($er) + Write-Verbose -Verbose "verbose-message" + } + elseif ($ErrorActionStop) { + Get-Command NonExist -ErrorAction Stop + Write-Verbose -Verbose "verbose-message" + } + elseif ($ThrowException) { + throw 'throw-exception' + Write-Verbose -Verbose "verbose-message" + } + elseif ($WriteErrorAPI) { + $ex = [System.ArgumentException]::new('arg-exception') + $er = [System.Management.Automation.ErrorRecord]::new($ex, 'WriteErrorAPI:error', 'InvalidArgument', $null) + $PSCmdlet.WriteError($er) + Write-Verbose -Verbose "verbose-message" + } + elseif ($WriteErrorCmdlet) { + Write-Error 'write-error-cmdlet' + Write-Verbose -Verbose "verbose-message" + } + elseif ($MethodInvocationThrowException) { + ## This method call throws exception. + $iss = [initialsessionstate]::Create() + $iss.ImportPSModule($null) + Write-Verbose -Verbose "verbose-message" + } + elseif ($ExpressionThrowException) { + 1/0 ## throw exception. + Write-Verbose -Verbose "verbose-message" + } + } + + function DivideByZeroWrappedInTry { + [CmdletBinding()] + param() + + end {} + clean { + try { + 1/0 + Write-Verbose -Verbose 'clean' + } + catch { Write-Verbose -Verbose $_.Exception.InnerException.GetType().FullName } + } + } + + function ArgumentNullWrappedInTry { + [CmdletBinding()] + param() + + end {} + clean { + try { + $iss = [initialsessionstate]::Create() + $iss.ImportPSModule($null) + Write-Verbose -Verbose 'clean' + } + catch { Write-Verbose -Verbose $_.Exception.InnerException.GetType().FullName } + } + } + + function DivideByZeroWithTrap { + [CmdletBinding()] + param() + + end {} + clean { + trap { + Write-Verbose -Verbose $_.Exception.GetType().FullName + continue + } + + 1/0 + Write-Verbose -Verbose 'clean' + } + } + + function ArgumentNullWithTrap { + [CmdletBinding()] + param() + + end {} + clean { + trap { + Write-Verbose -Verbose $_.Exception.GetType().FullName + continue + } + + $iss = [initialsessionstate]::Create() + $iss.ImportPSModule($null) + Write-Verbose -Verbose 'clean' + } + } + + #region Helper + + $pwsh = [PowerShell]::Create() + $text = (Get-Command ErrorInClean).ScriptBlock.Ast.Extent.Text + $pwsh.AddScript($text).Invoke() + + $pwsh.Commands.Clear() + $text = (Get-Command ErrorInEnd).ScriptBlock.Ast.Extent.Text + $pwsh.AddScript($text).Invoke() + + function RunCommand { + param( + [ValidateSet('ErrorInClean', 'ErrorInEnd')] + [string] $Command, + + [ValidateSet('ThrowTerminatingError', 'ErrorActionStop', 'ThrowException', 'WriteErrorAPI', + 'WriteErrorCmdlet', 'MethodInvocationThrowException', 'ExpressionThrowException')] + [string] $ParamNameToUse, + + [ValidateSet('Continue', 'Ignore', 'SilentlyContinue', 'Stop')] + [string] $ErrorAction + ) + + $pwsh.Commands.Clear() + $pwsh.Streams.ClearStreams() + $pwsh.AddCommand($Command).AddParameter($ParamNameToUse, $true) > $null + if ($ErrorAction) { $pwsh.AddParameter('ErrorAction', $ErrorAction) > $null } + $pwsh.Invoke() + } + + function RunScript { + param([string] $Script) + + $pwsh.Commands.Clear() + $pwsh.Streams.ClearStreams() + $pwsh.AddScript($Script).Invoke() + } + + function GetLastError { + $pwsh.Commands.Clear() + $pwsh.AddCommand('Get-Error').Invoke() + } + + function ClearDollarError { + $pwsh.Commands.Clear() + $pwsh.AddScript('$Error.Clear()').Invoke() + } + + #endregion + } + + AfterAll { + $pwsh.Dispose() + } + + It "Terminating error should stop the 'Clean' block execution but should not be propagated up" { + ## 'ThrowTerminatingException' stops the execution within the 'Clean' block, but the error doesn't get + ## propagated out of the 'Clean' block. Instead, the error is written to the 'ErrorOutput' pipe. + RunCommand -Command 'ErrorInClean' -ParamNameToUse 'ThrowTerminatingError' + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'terminating-exception' + + ## 'throw' statement stops the execution within the 'Clean' block by default, but the error doesn't get + ## propagated out of the 'Clean' block. Instead, the error is written to the 'ErrorOutput' pipe. + RunCommand -Command 'ErrorInClean' -ParamNameToUse 'ThrowException' + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'throw-exception' + + ## '-ErrorAction Stop' stops the execution within the 'Clean' block, but the error doesn't get propagated + ## out of the 'Clean' block. Instead, the error is written to the 'ErrorOutput' pipe. + RunCommand -Command 'ErrorInClean' -ParamNameToUse 'ErrorActionStop' + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].FullyQualifiedErrorId | Should -BeExactly 'CommandNotFoundException,Microsoft.PowerShell.Commands.GetCommandCommand' + + ## Turn non-terminating errors into terminating by '-ErrorAction Stop' explicitly. + ## Execution within the 'Clean' block should be stopped. The resulted terminating error should not get + ## propagated, but instead should be written to 'ErrorOutput' pipe. + RunCommand -Command 'ErrorInClean' -ParamNameToUse 'WriteErrorAPI' -ErrorAction Stop + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'arg-exception' + + RunCommand -Command 'ErrorInClean' -ParamNameToUse 'WriteErrorCmdlet' -ErrorAction Stop + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'write-error-cmdlet' + + RunCommand -Command 'ErrorInClean' -ParamNameToUse 'MethodInvocationThrowException' -ErrorAction Stop + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' + + RunCommand -Command 'ErrorInClean' -ParamNameToUse 'ExpressionThrowException' -ErrorAction Stop + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' + } + + It "Terminating error should set `$? correctly" { + RunScript -Script 'ErrorInClean -ThrowTerminatingError; $?' | Should -BeFalse + RunScript -Script 'ErrorInClean -ThrowException; $?' | Should -BeFalse + RunScript -Script 'ErrorInClean -ErrorActionStop; $?' | Should -BeFalse + RunScript -Script 'ErrorInClean -WriteErrorAPI -ErrorAction Stop; $?' | Should -BeFalse + RunScript -Script 'ErrorInClean -WriteErrorCmdlet -ErrorAction Stop; $?' | Should -BeFalse + RunScript -Script 'ErrorInClean -MethodInvocationThrowException -ErrorAction Stop; $?' | Should -BeFalse + RunScript -Script 'ErrorInClean -ExpressionThrowException -ErrorAction Stop; $?' | Should -BeFalse + } + + It "Track the `$? behavior for non-terminating errors within 'Clean' and 'End' blocks" { + RunScript -Script 'ErrorInClean -WriteErrorAPI; $?' | Should -BeFalse + RunScript -Script 'ErrorInEnd -WriteErrorAPI; $?' | Should -BeFalse + + ## The 'Write-Error' is specially weird, in that when a command emits error because of 'Write-Error' within it, + ## the following '$?' won't reflect '$false', but will be '$true'. + ## Frankly, this is counter-intuitive, but it's the existing behavior. The tests below just keeps track of this + ## behavior. Feel free to change this test if someone is fixing this seemingly wrong behavior. + RunScript -Script 'ErrorInClean -WriteErrorCmdlet; $?' | Should -BeTrue + RunScript -Script 'ErrorInEnd -WriteErrorCmdlet; $?' | Should -BeTrue + + ## Similarly, when a command emits error because of a method invocation within it throws an exception, + ## the following '$?' won't reflect '$false', but will be '$true'. + ## Again, this seems wrong, but it's the existing behavior. The tests below just keeps track of this + ## behavior. Feel free to change this test if someone is fixing this seemingly wrong behavior. + RunScript -Script 'ErrorInClean -MethodInvocationThrowException; $?' | Should -BeTrue + RunScript -Script 'ErrorInEnd -MethodInvocationThrowException; $?' | Should -BeTrue + + ## Again, when a command emits error because of an expression within it throws an exception, + ## the following '$?' won't reflect '$false', but will be '$true'. + ## This seems wrong, but it's the existing behavior. The tests below just keeps track of this + ## behavior. Feel free to change this test if someone is fixing this seemingly wrong behavior. + RunScript -Script 'ErrorInClean -ExpressionThrowException; $?' | Should -BeTrue + RunScript -Script 'ErrorInEnd -ExpressionThrowException; $?' | Should -BeTrue + } + + It "Non-terminating error within 'Clean' block should act based on ErrorActionPreference: - Continue" -TestCases @( + @{ ParamName = 'WriteErrorAPI'; AssertScript = { param($err) $err.Exception.Message | Should -BeExactly 'arg-exception' } } + @{ ParamName = 'WriteErrorCmdlet'; AssertScript = { param($err) $err.Exception.Message | Should -BeExactly 'write-error-cmdlet' } } + @{ ParamName = 'MethodInvocationThrowException'; AssertScript = { param($err) $err.Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' } } + @{ ParamName = 'ExpressionThrowException'; AssertScript = { param($err) $err.Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' } } + ) { + param($ParamName, $AssertScript) + + RunCommand -Command 'ErrorInClean' -ParamNameToUse $ParamName + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'verbose-message' + $pwsh.Streams.Error.Count | Should -Be 1 + & $AssertScript $pwsh.Streams.Error[0] + } + + It "Non-terminating error within 'End' block should act based on ErrorActionPreference: - Continue" -TestCases @( + @{ ParamName = 'WriteErrorAPI'; AssertScript = { param($err) $err.Exception.Message | Should -BeExactly 'arg-exception' } } + @{ ParamName = 'WriteErrorCmdlet'; AssertScript = { param($err) $err.Exception.Message | Should -BeExactly 'write-error-cmdlet' } } + @{ ParamName = 'MethodInvocationThrowException'; AssertScript = { param($err) $err.Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' } } + @{ ParamName = 'ExpressionThrowException'; AssertScript = { param($err) $err.Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' } } + ) { + param($ParamName, $AssertScript) + + RunCommand -Command 'ErrorInEnd' -ParamNameToUse $ParamName + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'verbose-message' + $pwsh.Streams.Error.Count | Should -Be 1 + & $AssertScript $pwsh.Streams.Error[0] + } + + It "Non-terminating error within 'Clean' block should act based on ErrorActionPreference: - " -TestCases @( + ### When error action is 'Ignore', non-terminating errors emitted by 'WriteErrorAPI' and 'WriteErrorCmdlet' are not captured in $Error, + ### but non-terminating errors emitted by method exception or expression exception are captured in $Error. + ### This inconsistency is surprising, but it's the existing behavior -- same in other named blocks. + @{ ParamName = 'WriteErrorAPI'; Action = 'Ignore'; AssertScript = $null } + @{ ParamName = 'WriteErrorCmdlet'; Action = 'Ignore'; AssertScript = $null } + @{ ParamName = 'MethodInvocationThrowException'; Action = 'Ignore'; AssertScript = { param($err) $err.Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' } } + @{ ParamName = 'ExpressionThrowException'; Action = 'Ignore'; AssertScript = { param($err) $err.Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' } } + + @{ ParamName = 'WriteErrorAPI'; Action = 'SilentlyContinue'; AssertScript = { param($err) $err.Exception.Message | Should -BeExactly 'arg-exception' } } + @{ ParamName = 'WriteErrorCmdlet'; Action = 'SilentlyContinue'; AssertScript = { param($err) $err.Exception.Message | Should -BeExactly 'write-error-cmdlet' } } + @{ ParamName = 'MethodInvocationThrowException'; Action = 'SilentlyContinue'; AssertScript = { param($err) $err.Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' } } + @{ ParamName = 'ExpressionThrowException'; Action = 'SilentlyContinue'; AssertScript = { param($err) $err.Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' } } + ) { + param($ParamName, $Action, $AssertScript) + + ClearDollarError + RunCommand -Command 'ErrorInClean' -ParamNameToUse $ParamName -ErrorAction $Action + + $pwsh.Streams.Error.Count | Should -Be 0 + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'verbose-message' + + $lastErr = GetLastError + if ($null -eq $AssertScript) { + $lastErr | Should -BeNullOrEmpty + } else { + $lastErr | Should -Not -BeNullOrEmpty + & $AssertScript $lastErr + } + } + + ### These tests are targeting 'End' block but with the same settings as the ones right above. + ### They are used as a comparison to prove the consistent behavior in 'End' and 'Clean'. + It "Non-terminating error within 'End' block should act based on ErrorActionPreference: - " -TestCases @( + @{ ParamName = 'WriteErrorAPI'; Action = 'Ignore'; AssertScript = $null } + @{ ParamName = 'WriteErrorCmdlet'; Action = 'Ignore'; AssertScript = $null } + @{ ParamName = 'MethodInvocationThrowException'; Action = 'Ignore'; AssertScript = { param($err) $err.Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' } } + @{ ParamName = 'ExpressionThrowException'; Action = 'Ignore'; AssertScript = { param($err) $err.Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' } } + + @{ ParamName = 'WriteErrorAPI'; Action = 'SilentlyContinue'; AssertScript = { param($err) $err.Exception.Message | Should -BeExactly 'arg-exception' } } + @{ ParamName = 'WriteErrorCmdlet'; Action = 'SilentlyContinue'; AssertScript = { param($err) $err.Exception.Message | Should -BeExactly 'write-error-cmdlet' } } + @{ ParamName = 'MethodInvocationThrowException'; Action = 'SilentlyContinue'; AssertScript = { param($err) $err.Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' } } + @{ ParamName = 'ExpressionThrowException'; Action = 'SilentlyContinue'; AssertScript = { param($err) $err.Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' } } + ) { + param($ParamName, $Action, $AssertScript) + + ClearDollarError + RunCommand -Command 'ErrorInEnd' -ParamNameToUse $ParamName -ErrorAction $Action + + $pwsh.Streams.Error.Count | Should -Be 0 + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'verbose-message' + + $lastErr = GetLastError + if ($null -eq $AssertScript) { + $lastErr | Should -BeNullOrEmpty + } else { + $lastErr | Should -Not -BeNullOrEmpty + & $AssertScript $lastErr + } + } + + It "'try/catch' and 'trap' should turn general exception thrown from method/expression into terminating error within 'Clean' block. ErrorAction: " -TestCases @( + @{ ErrorAction = 'Continue' } + @{ ErrorAction = 'Ignore' } + @{ ErrorAction = 'SilentlyContinue' } + ) { + param($ErrorAction) + + $verbose = DivideByZeroWrappedInTry -ErrorAction $ErrorAction 4>&1 + $verbose.Count | Should -Be 1 + $verbose.Message | Should -BeExactly 'System.DivideByZeroException' + + $verbose = ArgumentNullWrappedInTry -ErrorAction $ErrorAction 4>&1 + $verbose.Count | Should -Be 1 + $verbose.Message | Should -BeExactly 'System.ArgumentNullException' + + $verbose = DivideByZeroWithTrap -ErrorAction $ErrorAction 4>&1 + $verbose.Count | Should -Be 2 + $verbose[0].Message | Should -BeExactly 'System.DivideByZeroException' + $verbose[1].Message | Should -BeExactly 'clean' + + $verbose = ArgumentNullWithTrap -ErrorAction $ErrorAction 4>&1 + $verbose.Count | Should -Be 2 + $verbose[0].Message | Should -BeExactly 'System.ArgumentNullException' + $verbose[1].Message | Should -BeExactly 'clean' + } + + It "'try/catch' and 'trap' outside the command should NOT affect general exception thrown from method/expression in the 'Clean' block" { + ## The catch block should not run + RunScript -Script "try { ErrorInClean -MethodInvocationThrowException } catch { Write-Debug -Debug 'caught-something' }" + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'verbose-message' + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' + $pwsh.Streams.Debug.Count | Should -Be 0 + + ## The catch block should not run + RunScript -Script "try { ErrorInClean -ExpressionThrowException } catch { Write-Debug -Debug 'caught-something' }" + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'verbose-message' + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' + $pwsh.Streams.Debug.Count | Should -Be 0 + + ## The trap block should not run + RunScript -Script "trap { Write-Debug -Debug 'caught-something'; continue } ErrorInClean -MethodInvocationThrowException" + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'verbose-message' + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' + $pwsh.Streams.Debug.Count | Should -Be 0 + + ## The trap block should not run + RunScript -Script "trap { Write-Debug -Debug 'caught-something'; continue } ErrorInClean -ExpressionThrowException" + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'verbose-message' + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' + $pwsh.Streams.Debug.Count | Should -Be 0 + } + + It "'try/catch' and 'trap' outside the command should NOT affect 'throw' statement in the 'Clean' block. ErrorAction: " -TestCases @( + @{ ErrorAction = 'Ignore' } + @{ ErrorAction = 'SilentlyContinue' } + ) { + param ($ErrorAction) + + ## 'throw' statement should be suppressed by 'Ignore' or 'SilentlyContinue' within a 'Clean' block, + ## even if the command is wrapped in try/catch. + RunScript -Script "try { ErrorInClean -ThrowException -ErrorAction $ErrorAction } catch { Write-Debug -Debug 'caught-something' }" + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'verbose-message' + + ## Nothing written to error stream + $pwsh.Streams.Error.Count | Should -Be 0 + ## Nothing written to debug stream + $pwsh.Streams.Debug.Count | Should -Be 0 + + ## The suppressed 'throw' exception is kept in '$Error' + $err = GetLastError + $err | Should -Not -BeNullOrEmpty + $err.FullyQualifiedErrorId | Should -BeExactly 'throw-exception' + + + ## 'throw' statement should be suppressed by 'Ignore' or 'SilentlyContinue' within a 'Clean' block, + ## even if the command is accompanied by 'trap'. + RunScript -Script "trap { Write-Debug -Debug 'caught-something'; continue } ErrorInClean -ThrowException -ErrorAction $ErrorAction" + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'verbose-message' + + ## Nothing written to error stream + $pwsh.Streams.Error.Count | Should -Be 0 + ## Nothing written to debug stream + $pwsh.Streams.Debug.Count | Should -Be 0 + + ## The suppressed 'throw' exception is kept in '$Error' + $err = GetLastError + $err | Should -Not -BeNullOrEmpty + $err.FullyQualifiedErrorId | Should -BeExactly 'throw-exception' + } + + It "Error out-variable should work for the 'Clean' block" { + ## Terminating errors thrown from 'Clean' block are captured and written to the error pipe. + ## Here we redirect the error pipe to discard the error stream, so the error doesn't pollute + ## the test output. + ErrorInClean -ThrowTerminatingError -ErrorVariable err 2>&1 > $null + $err.Count | Should -Be 1 + $err[0].Message | Should -BeExactly 'terminating-exception' + + ## $err.Count is 3 in this case. It's the same for other named blocks too. + ## This looks like an existing bug because $err.Count should be 1 since only 1 error happened. + ## Opened issue https://github.com/PowerShell/PowerShell/issues/15739 + ErrorInClean -ErrorActionStop -ErrorVariable err 2>&1 > $null + $err[0] | Should -BeOfType 'System.Management.Automation.ActionPreferenceStopException' + + ## $err.Count is 2 in this case. It's the same for other named blocks too. + ## Similarly, this looks like an existing bug and $err.Count should be 1. + ## This is tracked by the same issue above. + ErrorInClean -ThrowException -ErrorVariable err 2>&1 > $null + $err[0].Exception.Message | Should -BeExactly 'throw-exception' + + ErrorInClean -WriteErrorAPI -ErrorVariable err *>&1 > $null + $err.Count | Should -Be 1 + $err[0].Exception.Message | Should -BeExactly 'arg-exception' + + ErrorInClean -WriteErrorCmdlet -ErrorVariable err *>&1 > $null + $err.Count | Should -Be 1 + $err[0].Exception.Message | Should -BeExactly 'write-error-cmdlet' + + ErrorInClean -MethodInvocationThrowException -ErrorVariable err *>&1 > $null + $err.Count | Should -Be 1 + $err[0].Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' + + ErrorInClean -ExpressionThrowException -ErrorVariable err *>&1 > $null + $err.Count | Should -Be 1 + $err[0].Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' + } +} + +Describe "Multiple errors from 'Clean' and another named block" { + + BeforeAll { + function MultipleErrors { + [CmdletBinding()] + param( + [ValidateSet('ThrowTerminatingError', 'ErrorActionStop', 'ThrowException', 'WriteErrorAPI', + 'WriteErrorCmdlet', 'MethodInvocationThrowException', 'ExpressionThrowException')] + [Parameter(Mandatory)] + [string] $ErrorFromEndBlock, + + [ValidateSet('ThrowTerminatingError', 'ErrorActionStop', 'ThrowException', 'WriteErrorAPI', + 'WriteErrorCmdlet', 'MethodInvocationThrowException', 'ExpressionThrowException')] + [Parameter(Mandatory)] + [string] $ErrorFromCleanBlock + ) + + End { + switch ($ErrorFromEndBlock) { + 'ThrowTerminatingError' { + $ex = [System.ArgumentException]::new('end-terminating-exception') + $er = [System.Management.Automation.ErrorRecord]::new($ex, 'ThrowTerminatingError:end-error', 'InvalidArgument', $null) + $PSCmdlet.ThrowTerminatingError($er) + Write-Verbose -Verbose "end-verbose-message" + } + + 'ErrorActionStop' { + Get-Command NonExistEnd -ErrorAction Stop + Write-Verbose -Verbose "end-verbose-message" + } + + 'ThrowException' { + throw 'end-throw-exception' + Write-Verbose -Verbose "end-verbose-message" + } + + 'WriteErrorAPI' { + $ex = [System.ArgumentException]::new('end-arg-exception') + $er = [System.Management.Automation.ErrorRecord]::new($ex, 'WriteErrorAPI:end-error', 'InvalidArgument', $null) + $PSCmdlet.WriteError($er) + Write-Verbose -Verbose "end-verbose-message" + } + + 'WriteErrorCmdlet' { + Write-Error 'end-write-error-cmdlet' + Write-Verbose -Verbose "end-verbose-message" + } + + 'MethodInvocationThrowException' { + ## This method call throws exception. + $iss = [initialsessionstate]::Create() + $iss.ImportPSModule($null) + Write-Verbose -Verbose "end-verbose-message" + } + + 'ExpressionThrowException' { + 1/0 ## throw exception. + Write-Verbose -Verbose "end-verbose-message" + } + } + } + + clean { + switch ($ErrorFromCleanBlock) { + 'ThrowTerminatingError' { + $ex = [System.ArgumentException]::new('clean-terminating-exception') + $er = [System.Management.Automation.ErrorRecord]::new($ex, 'ThrowTerminatingError:clean-error', 'InvalidArgument', $null) + $PSCmdlet.ThrowTerminatingError($er) + Write-Verbose -Verbose "clean-verbose-message" + } + + 'ErrorActionStop' { + Get-Command NonExistClean -ErrorAction Stop + Write-Verbose -Verbose "clean-verbose-message" + } + + 'ThrowException' { + throw 'clean-throw-exception' + Write-Verbose -Verbose "clean-verbose-message" + } + + 'WriteErrorAPI' { + $ex = [System.ArgumentException]::new('clean-arg-exception') + $er = [System.Management.Automation.ErrorRecord]::new($ex, 'WriteErrorAPI:clean-error', 'InvalidArgument', $null) + $PSCmdlet.WriteError($er) + Write-Verbose -Verbose "clean-verbose-message" + } + + 'WriteErrorCmdlet' { + Write-Error 'clean-write-error-cmdlet' + Write-Verbose -Verbose "clean-verbose-message" + } + + 'MethodInvocationThrowException' { + ## This method call throws exception. + $iss = [initialsessionstate]::Create() + $iss.ImportPSModule($null) + Write-Verbose -Verbose "clean-verbose-message" + } + + 'ExpressionThrowException' { + 1/0 ## throw exception. + Write-Verbose -Verbose "clean-verbose-message" + } + } + } + } + + #region Helper + + $pwsh = [PowerShell]::Create() + $text = (Get-Command MultipleErrors).ScriptBlock.Ast.Extent.Text + $pwsh.AddScript($text).Invoke() + + function RunCommand { + param( + [ValidateSet('MultipleErrors')] + [string] $Command, + + [ValidateSet('ThrowTerminatingError', 'ErrorActionStop', 'ThrowException', 'WriteErrorAPI', + 'WriteErrorCmdlet', 'MethodInvocationThrowException', 'ExpressionThrowException')] + [string] $ErrorFromEndBlock, + + [ValidateSet('ThrowTerminatingError', 'ErrorActionStop', 'ThrowException', 'WriteErrorAPI', + 'WriteErrorCmdlet', 'MethodInvocationThrowException', 'ExpressionThrowException')] + [string] $ErrorFromCleanBlock, + + [ValidateSet('Continue', 'Ignore', 'SilentlyContinue', 'Stop')] + [string] $ErrorAction + ) + + $pwsh.Commands.Clear() + $pwsh.Streams.ClearStreams() + $pwsh.AddCommand($Command) > $null + $pwsh.AddParameter('ErrorFromEndBlock', $ErrorFromEndBlock) > $null + $pwsh.AddParameter('ErrorFromCleanBlock', $ErrorFromCleanBlock) > $null + if ($ErrorAction) { $pwsh.AddParameter('ErrorAction', $ErrorAction) > $null } + $pwsh.Invoke() + } + + function RunScript { + param([string] $Script) + + $pwsh.Commands.Clear() + $pwsh.Streams.ClearStreams() + $pwsh.AddScript($Script).Invoke() + } + + function GetLastError { + $pwsh.Commands.Clear() + $pwsh.AddCommand('Get-Error').Invoke() + } + + function GetAllErrors { + $pwsh.Commands.Clear() + $pwsh.AddScript('$Error').Invoke() + } + + function ClearDollarError { + $pwsh.Commands.Clear() + $pwsh.AddScript('$Error.Clear()').Invoke() + } + + #endregion + } + + AfterAll { + $pwsh.Dispose() + } + + It "Terminating errors from both 'End' (ThrowTerminatingError) and 'Clean' (ThrowTerminatingError) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ThrowTerminatingError' -ErrorFromCleanBlock 'ThrowTerminatingError' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException | Should -BeOfType 'System.Management.Automation.CmdletInvocationException' + $failure.Exception.InnerException.InnerException | Should -BeOfType 'System.ArgumentException' + $failure.Exception.InnerException.InnerException.Message | Should -BeExactly 'end-terminating-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception | Should -BeOfType 'System.ArgumentException' + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'clean-terminating-exception' + } + + It "Terminating errors from both 'End' (ErrorActionStop) and 'Clean' (ThrowTerminatingError) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ErrorActionStop' -ErrorFromCleanBlock 'ThrowTerminatingError' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException | Should -BeOfType 'System.Management.Automation.ActionPreferenceStopException' + $failure.Exception.InnerException.Message | should -BeLike "*'NonExistEnd'*" + + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception | Should -BeOfType 'System.ArgumentException' + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'clean-terminating-exception' + } + + It "Terminating errors from both 'End' (ThrowException) and 'Clean' (ThrowTerminatingError) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ThrowException' -ErrorFromCleanBlock 'ThrowTerminatingError' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException.Message | should -BeExactly 'end-throw-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception | Should -BeOfType 'System.ArgumentException' + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'clean-terminating-exception' + } + + It "Terminating errors from both 'End' (ThrowTerminatingError) and 'Clean' (ErrorActionStop) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ThrowTerminatingError' -ErrorFromCleanBlock 'ErrorActionStop' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException | Should -BeOfType 'System.Management.Automation.CmdletInvocationException' + $failure.Exception.InnerException.InnerException | Should -BeOfType 'System.ArgumentException' + $failure.Exception.InnerException.InnerException.Message | Should -BeExactly 'end-terminating-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].FullyQualifiedErrorId | Should -BeExactly 'CommandNotFoundException,Microsoft.PowerShell.Commands.GetCommandCommand' + $pwsh.Streams.Error[0].Exception.Message | Should -BeLike "*'NonExistClean'*" + } + + It "Terminating errors from both 'End' (ErrorActionStop) and 'Clean' (ErrorActionStop) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ErrorActionStop' -ErrorFromCleanBlock 'ErrorActionStop' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException | Should -BeOfType 'System.Management.Automation.ActionPreferenceStopException' + $failure.Exception.InnerException.Message | should -BeLike "*'NonExistEnd'*" + + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].FullyQualifiedErrorId | Should -BeExactly 'CommandNotFoundException,Microsoft.PowerShell.Commands.GetCommandCommand' + $pwsh.Streams.Error[0].Exception.Message | Should -BeLike "*'NonExistClean'*" + } + + It "Terminating errors from both 'End' (ThrowException) and 'Clean' (ErrorActionStop) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ThrowException' -ErrorFromCleanBlock 'ErrorActionStop' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException.Message | should -BeExactly 'end-throw-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].FullyQualifiedErrorId | Should -BeExactly 'CommandNotFoundException,Microsoft.PowerShell.Commands.GetCommandCommand' + $pwsh.Streams.Error[0].Exception.Message | Should -BeLike "*'NonExistClean'*" + } + + It "Terminating errors from both 'End' (ThrowTerminatingError) and 'Clean' (ThrowException) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ThrowTerminatingError' -ErrorFromCleanBlock 'ThrowException' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException | Should -BeOfType 'System.Management.Automation.CmdletInvocationException' + $failure.Exception.InnerException.InnerException | Should -BeOfType 'System.ArgumentException' + $failure.Exception.InnerException.InnerException.Message | Should -BeExactly 'end-terminating-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'clean-throw-exception' + } + + It "Terminating errors from both 'End' (ErrorActionStop) and 'Clean' (ThrowException) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ErrorActionStop' -ErrorFromCleanBlock 'ThrowException' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException | Should -BeOfType 'System.Management.Automation.ActionPreferenceStopException' + $failure.Exception.InnerException.Message | should -BeLike "*'NonExistEnd'*" + + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'clean-throw-exception' + } + + It "Terminating errors from both 'End' (ThrowException) and 'Clean' (ThrowException) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ThrowException' -ErrorFromCleanBlock 'ThrowException' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException.Message | should -BeExactly 'end-throw-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'clean-throw-exception' + } + + It "Terminating errors from both 'End' (ThrowException) and 'Clean' (ThrowException) with ErrorAction '' should work properly" -TestCases @( + @{ ErrorAction = 'Ignore' } + @{ ErrorAction = 'SilentlyContinue' } + ) { + param($ErrorAction) + + ClearDollarError + + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ThrowException' -ErrorFromCleanBlock 'ThrowException' -ErrorAction $ErrorAction + + $pwsh.Streams.Error.Count | Should -Be 0 + $pwsh.Streams.Verbose.Count | Should -Be 2 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + $pwsh.Streams.Verbose[1] | Should -BeExactly 'clean-verbose-message' + + $ers = GetAllErrors + $ers.Count | Should -Be 2 + $ers[0].Exception.Message | Should -BeExactly 'clean-throw-exception' + $ers[1].Exception.Message | Should -BeExactly 'end-throw-exception' + } + + It "Non-terminating error from 'End' (WriteErrorAPI) and terminating error from 'Clean' (ThrowTerminatingError) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'WriteErrorAPI' -ErrorFromCleanBlock 'ThrowTerminatingError' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'end-arg-exception' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'clean-terminating-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + } + + It "Non-terminating error from 'End' (WriteErrorAPI) and terminating error from 'Clean' (ErrorActionStop) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'WriteErrorAPI' -ErrorFromCleanBlock 'ErrorActionStop' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'end-arg-exception' + $pwsh.Streams.Error[1].Exception.Message | Should -BeLike "*'NonExistClean'*" + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + } + + It "Non-terminating error from 'End' (WriteErrorAPI) and terminating error from 'Clean' (ThrowException) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'WriteErrorAPI' -ErrorFromCleanBlock 'ThrowException' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'end-arg-exception' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'clean-throw-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + } + + It "Non-terminating error from 'End' (WriteErrorCmdlet) and terminating error from 'Clean' (ThrowTerminatingError) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'WriteErrorCmdlet' -ErrorFromCleanBlock 'ThrowTerminatingError' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'end-write-error-cmdlet' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'clean-terminating-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + } + + It "Non-terminating error from 'End' (WriteErrorCmdlet) and terminating error from 'Clean' (ErrorActionStop) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'WriteErrorCmdlet' -ErrorFromCleanBlock 'ErrorActionStop' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'end-write-error-cmdlet' + $pwsh.Streams.Error[1].Exception.Message | Should -BeLike "*'NonExistClean'*" + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + } + + It "Non-terminating error from 'End' (WriteErrorCmdlet) and terminating error from 'Clean' (ThrowException) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'WriteErrorCmdlet' -ErrorFromCleanBlock 'ThrowException' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'end-write-error-cmdlet' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'clean-throw-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + } + + It "Non-terminating error from 'End' (MethodInvocationThrowException) and terminating error from 'Clean' (ThrowTerminatingError) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'MethodInvocationThrowException' -ErrorFromCleanBlock 'ThrowTerminatingError' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'clean-terminating-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + } + + It "Non-terminating error from 'End' (MethodInvocationThrowException) and terminating error from 'Clean' (ErrorActionStop) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'MethodInvocationThrowException' -ErrorFromCleanBlock 'ErrorActionStop' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' + $pwsh.Streams.Error[1].Exception.Message | Should -BeLike "*'NonExistClean'*" + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + } + + It "Non-terminating error from 'End' (MethodInvocationThrowException) and terminating error from 'Clean' (ThrowException) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'MethodInvocationThrowException' -ErrorFromCleanBlock 'ThrowException' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'clean-throw-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + } + + It "Non-terminating error from 'End' (ExpressionThrowException) and terminating error from 'Clean' (ThrowTerminatingError) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ExpressionThrowException' -ErrorFromCleanBlock 'ThrowTerminatingError' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'clean-terminating-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + } + + It "Non-terminating error from 'End' (ExpressionThrowException) and terminating error from 'Clean' (ErrorActionStop) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ExpressionThrowException' -ErrorFromCleanBlock 'ErrorActionStop' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' + $pwsh.Streams.Error[1].Exception.Message | Should -BeLike "*'NonExistClean'*" + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + } + + It "Non-terminating error from 'End' (ExpressionThrowException) and terminating error from 'Clean' (ThrowException) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ExpressionThrowException' -ErrorFromCleanBlock 'ThrowException' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'clean-throw-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'end-verbose-message' + } + + It "Terminating error from 'End' (ThrowException) and non-terminating error from 'Clean' (WriteErrorAPI) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ThrowException' -ErrorFromCleanBlock 'WriteErrorAPI' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException.Message | should -BeExactly 'end-throw-exception' + + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'clean-arg-exception' + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'clean-verbose-message' + } + + It "Terminating error from 'End' (ThrowException) and non-terminating error from 'Clean' (WriteErrorCmdlet) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ThrowException' -ErrorFromCleanBlock 'WriteErrorCmdlet' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException.Message | should -BeExactly 'end-throw-exception' + + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'clean-write-error-cmdlet' + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'clean-verbose-message' + } + + It "Terminating error from 'End' (ThrowException) and non-terminating error from 'Clean' (MethodInvocationThrowException) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ThrowException' -ErrorFromCleanBlock 'MethodInvocationThrowException' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException.Message | should -BeExactly 'end-throw-exception' + + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'clean-verbose-message' + } + + It "Terminating error from 'End' (ThrowException) and non-terminating error from 'Clean' (ExpressionThrowException) should work properly" { + $failure = $null + try { + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'ThrowException' -ErrorFromCleanBlock 'ExpressionThrowException' + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException.Message | should -BeExactly 'end-throw-exception' + + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Verbose[0] | Should -BeExactly 'clean-verbose-message' + } + + It "Non-terminating error from 'End' (WriteErrorAPI) and non-terminating error from 'Clean' (WriteErrorCmdlet) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'WriteErrorAPI' -ErrorFromCleanBlock 'WriteErrorCmdlet' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'end-arg-exception' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'clean-write-error-cmdlet' + + $pwsh.Streams.Verbose.Count | Should -Be 2 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'end-verbose-message' + $pwsh.Streams.Verbose[1].Message | Should -BeExactly 'clean-verbose-message' + } + + It "Non-terminating error from 'End' (WriteErrorAPI) and non-terminating error from 'Clean' (MethodInvocationThrowException) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'WriteErrorAPI' -ErrorFromCleanBlock 'MethodInvocationThrowException' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'end-arg-exception' + $pwsh.Streams.Error[1].Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' + + $pwsh.Streams.Verbose.Count | Should -Be 2 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'end-verbose-message' + $pwsh.Streams.Verbose[1].Message | Should -BeExactly 'clean-verbose-message' + } + + It "Non-terminating error from 'End' (WriteErrorCmdlet) and non-terminating error from 'Clean' (WriteErrorAPI) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'WriteErrorCmdlet' -ErrorFromCleanBlock 'WriteErrorAPI' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'end-write-error-cmdlet' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'clean-arg-exception' + + $pwsh.Streams.Verbose.Count | Should -Be 2 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'end-verbose-message' + $pwsh.Streams.Verbose[1].Message | Should -BeExactly 'clean-verbose-message' + } + + It "Non-terminating error from 'End' (WriteErrorCmdlet) and non-terminating error from 'Clean' (ExpressionThrowException) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'WriteErrorCmdlet' -ErrorFromCleanBlock 'ExpressionThrowException' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'end-write-error-cmdlet' + $pwsh.Streams.Error[1].Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' + + $pwsh.Streams.Verbose.Count | Should -Be 2 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'end-verbose-message' + $pwsh.Streams.Verbose[1].Message | Should -BeExactly 'clean-verbose-message' + } + + It "Non-terminating error from 'End' (MethodInvocationThrowException) and non-terminating error from 'Clean' (WriteErrorCmdlet) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'MethodInvocationThrowException' -ErrorFromCleanBlock 'WriteErrorCmdlet' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'clean-write-error-cmdlet' + + $pwsh.Streams.Verbose.Count | Should -Be 2 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'end-verbose-message' + $pwsh.Streams.Verbose[1].Message | Should -BeExactly 'clean-verbose-message' + } + + It "Non-terminating error from 'End' (MethodInvocationThrowException) and non-terminating error from 'Clean' (ExpressionThrowException) should work properly" { + ## No exception should be thrown + RunCommand -Command 'MultipleErrors' -ErrorFromEndBlock 'MethodInvocationThrowException' -ErrorFromCleanBlock 'ExpressionThrowException' + + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.InnerException | Should -BeOfType 'System.ArgumentNullException' + $pwsh.Streams.Error[1].Exception.InnerException | Should -BeOfType 'System.DivideByZeroException' + + $pwsh.Streams.Verbose.Count | Should -Be 2 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'end-verbose-message' + $pwsh.Streams.Verbose[1].Message | Should -BeExactly 'clean-verbose-message' + } +} + +Describe "Error handling within a pipeline (multiple commands with 'Clean' block)" -Tag 'CI' { + BeforeAll { + function test1 { + param([switch] $EmitErrorInProcess) + process { + if ($EmitErrorInProcess) { + throw 'test1-process' + } + Write-Output 'process-obj' + } + clean { + throw 'test1-clean' + Write-Verbose -Verbose 'test1-clean-verbose' + } + } + + function test2 { + param([switch] $EmitErrorInProcess) + process { + if ($EmitErrorInProcess) { + throw 'test2-process' + } + Write-Verbose -Verbose $_ + } + clean { + throw 'test2-clean' + Write-Verbose -Verbose 'test2-clean-verbose' + } + } + + function test-1 { + param([switch] $EmitException) + process { 'output' } + clean { + if ($EmitException) { + throw 'test-1-clean-exception' + } + Write-Verbose -Verbose 'test-1-clean' + } + } + + function test-2 { + param([switch] $EmitException) + process { $_ } + clean { + if ($EmitException) { + throw 'test-2-clean-exception' + } + Write-Verbose -Verbose 'test-2-clean' + } + } + + function test-3 { + param([switch] $EmitException) + process { Write-Warning $_ } + clean { + if ($EmitException) { + throw 'test-3-clean-exception' + } + Write-Verbose -Verbose 'test-3-clean' + } + } + + #region Helper + + $pwsh = [PowerShell]::Create() + $text = (Get-Command test1, test2, test-1, test-2, test-3).ScriptBlock.Ast.Extent.Text + $pwsh.AddScript($text -join "`n").Invoke() + + function RunScript { + param([string] $Script) + + $pwsh.Commands.Clear() + $pwsh.Streams.ClearStreams() + $pwsh.AddScript($Script).Invoke() + } + + #endregion + } + + It "Errors from multiple 'Clean' blocks should work properly" { + ## No exception should be thrown + RunScript -Script "test1 | test2" + + ## Exceptions thrown from 'throw' statement are not propagated up, but instead written to the error stream. + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'test1-clean' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'test2-clean' + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'process-obj' + } + + It "ErrorAction should be honored by 'Clean' blocks" { + try { + RunScript -Script '$ErrorActionPreference = "SilentlyContinue"' + ## No exception should be thrown. + RunScript -Script "test1 | test2" + + ## The exception from 'throw' statement should be suppressed by 'SilentlyContinue'. + $pwsh.Streams.Error.Count | Should -Be 0 + $pwsh.Streams.Verbose.Count | Should -Be 3 + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'process-obj' + $pwsh.Streams.Verbose[1].Message | Should -BeExactly 'test1-clean-verbose' + $pwsh.Streams.Verbose[2].Message | Should -BeExactly 'test2-clean-verbose' + } + finally { + ## Revert error action back to 'Continue' + RunScript -Script '$ErrorActionPreference = "Continue"' + } + } + + It "Errors from 'Clean' blocks should work properly when another named block emits error" { + $failure = $null + try { + RunScript -Script "test1 | test2 -EmitErrorInProcess" + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException.Message | Should -BeExactly 'test2-process' + + $pwsh.Streams.Verbose.Count | Should -Be 0 + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'test1-clean' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'test2-clean' + } + + It "The 'Clean' block should not run when the none of other named blocks from the same command get to run" { + $failure = $null + try { + RunScript -Script "test1 -EmitErrorInProcess | test2" + } catch { + $failure = $_ + } + + $failure | Should -Not -BeNullOrEmpty + $failure.Exception | Should -BeOfType 'System.Management.Automation.MethodInvocationException' + $failure.Exception.InnerException.Message | Should -BeExactly 'test1-process' + + ## Only the 'Clean' block from 'test1' will run. + ## The 'Clean' block from 'test2' won't run because none of the other blocks from 'test2' gets to run + ## due to the terminating exception thrown from 'test1.Process'. + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'test1-clean' + } + + It "Exception from the 'Clean' block at should not affect other 'Clean' blocks" -TestCases @( + @{ Position = 'beginning-of-pipeline'; Script = 'test-1 -EmitException | test-2 | test-3'; ExceptionMessage = 'test-1-clean-exception'; VerboseMessages = @('test-2-clean', 'test-3-clean') } + @{ Position = 'middle-of-pipeline'; Script = 'test-1 | test-2 -EmitException | test-3'; ExceptionMessage = 'test-2-clean-exception'; VerboseMessages = @('test-1-clean', 'test-3-clean') } + @{ Position = 'end-of-pipeline'; Script = 'test-1 | test-2 | test-3 -EmitException'; ExceptionMessage = 'test-3-clean-exception'; VerboseMessages = @('test-1-clean', 'test-2-clean') } + ) { + param($Script, $ExceptionMessage, $VerboseMessages) + + RunScript -Script $Script + $pwsh.Streams.Error.Count | Should -Be 1 + $pwsh.Streams.Verbose.Count | Should -Be 2 + $pwsh.Streams.Warning.Count | Should -Be 1 + + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly $ExceptionMessage + $pwsh.Streams.Verbose[0].Message | Should -BeExactly $VerboseMessages[0] + $pwsh.Streams.Verbose[1].Message | Should -BeExactly $VerboseMessages[1] + $pwsh.Streams.Warning[0].Message | Should -BeExactly 'output' + } + + It "Multiple exceptions from 'Clean' blocks should not affect other 'Clean' blocks" { + RunScript -Script 'test-1 -EmitException | test-2 | test-3 -EmitException' + $pwsh.Streams.Error.Count | Should -Be 2 + $pwsh.Streams.Verbose.Count | Should -Be 1 + $pwsh.Streams.Warning.Count | Should -Be 1 + + $pwsh.Streams.Error[0].Exception.Message | Should -BeExactly 'test-1-clean-exception' + $pwsh.Streams.Error[1].Exception.Message | Should -BeExactly 'test-3-clean-exception' + $pwsh.Streams.Verbose[0].Message | Should -BeExactly 'test-2-clean' + $pwsh.Streams.Warning[0].Message | Should -BeExactly 'output' + } +} diff --git a/test/powershell/Language/Scripting/PipelineBehaviour.Tests.ps1 b/test/powershell/Language/Scripting/PipelineBehaviour.Tests.ps1 new file mode 100644 index 00000000000..9551343da85 --- /dev/null +++ b/test/powershell/Language/Scripting/PipelineBehaviour.Tests.ps1 @@ -0,0 +1,591 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +Describe 'Function Pipeline Behaviour' -Tag 'CI' { + + BeforeAll { + $filePath = "$TestDrive\output.txt" + if (Test-Path $filePath) { + Remove-Item $filePath -Force + } + } + + Context "'Clean' block runs when any other named blocks run" { + + AfterEach { + if (Test-Path $filePath) { + Remove-Item $filePath -Force + } + } + + It "'Clean' block executes only if at least one of the other named blocks executed" { + ## The 'Clean' block is for cleanup purpose. When none of other named blocks execute, + ## there is no point to execute the 'Clean' block, so it will be skipped in this case. + function test-1 { + clean { 'clean-redirected-output' > $filePath } + } + + function test-2 { + End { 'end' } + clean { 'clean-redirected-output' > $filePath } + } + + ## The 'Clean' block is skipped. + test-1 | Should -BeNullOrEmpty + Test-Path -Path $filePath | Should -BeFalse + + ## The 'Clean' block runs. + test-2 | Should -BeExactly 'end' + Test-Path -Path $filePath | Should -BeTrue + Get-Content $filePath | Should -BeExactly 'clean-redirected-output' + } + + It "'Clean' block is skipped when the command doesn't run due to no input from upstream command" { + function test-1 ([switch] $WriteOutput) { + Process { + if ($WriteOutput) { + Write-Output 'process' + } else { + Write-Verbose -Verbose 'process' + } + } + } + + function test-2 { + Process { Write-Output "test-2: $_" } + clean { Write-Warning 'test-2-clean-warning' } + } + + ## No output from 'test-1.Process', so 'test-2.Process' didn't run, and thus 'test-2.Clean' was skipped. + test-1 | test-2 *>&1 | Should -BeNullOrEmpty + + ## Output from 'test-1.Process' would trigger 'test-2.Process' to run, and thus 'test-2.Clean' would run. + $output = test-1 -WriteOutput | test-2 *>&1 + $output | Should -Be @('test-2: process', 'test-2-clean-warning') + } + + It "'Clean' block is skipped when the command doesn't run due to terminating error from upstream Process block" { + function test-1 ([switch] $ThrowException) { + Process { + if ($ThrowException) { + throw 'process' + } else { + Write-Output 'process' + } + } + } + + function test-2 { + Process { Write-Output "test-2: $_" } + clean { 'clean-redirected-output' > $filePath } + } + + $failure = $null + try { test-1 -ThrowException | test-2 } catch { $failure = $_ } + $failure | Should -Not -BeNullOrEmpty + $failure.Exception.Message | Should -BeExactly 'process' + ## 'test-2' didn't run because 'test-1' throws terminating exception, so 'test-2.Clean' didn't run either. + Test-Path -Path $filePath | Should -BeFalse + + test-1 | test-2 | Should -BeExactly 'test-2: process' + Test-Path -Path $filePath | Should -BeTrue + Get-Content $filePath | Should -BeExactly 'clean-redirected-output' + } + + It "'Clean' block is skipped when the command doesn't run due to terminating error from upstream Begin block" { + function test-1 { + Begin { throw 'begin' } + End { 'end' } + } + + function test-2 { + Begin { 'begin' } + Process { Write-Output "test-2: $_" } + clean { 'clean-redirected-output' > $filePath } + } + + $failure = $null + try { test-1 | test-2 } catch { $failure = $_ } + $failure | Should -Not -BeNullOrEmpty + $failure.Exception.Message | Should -BeExactly 'begin' + ## 'test-2' didn't run because 'test-1' throws terminating exception, so 'test-2.Clean' didn't run either. + Test-Path -Path $filePath | Should -BeFalse + } + + It "'Clean' block runs when '' runs" -TestCases @( + @{ Script = { [CmdletBinding()]param() begin { 'output' } clean { Write-Warning 'clean-warning' } }; BlockName = 'Begin' } + @{ Script = { [CmdletBinding()]param() process { 'output' } clean { Write-Warning 'clean-warning' } }; BlockName = 'Process' } + @{ Script = { [CmdletBinding()]param() end { 'output' } clean { Write-Warning 'clean-warning' } }; BlockName = 'End' } + ) { + param($Script, $BlockName) + + & $Script -WarningVariable wv | Should -BeExactly 'output' + $wv | Should -BeExactly 'clean-warning' + } + + It "'Clean' block runs when '' throws terminating error" -TestCases @( + @{ Script = { [CmdletBinding()]param() begin { throw 'failure' } clean { Write-Warning 'clean-warning' } }; BlockName = 'Begin' } + @{ Script = { [CmdletBinding()]param() process { throw 'failure' } clean { Write-Warning 'clean-warning' } }; BlockName = 'Process' } + @{ Script = { [CmdletBinding()]param() end { throw 'failure' } clean { Write-Warning 'clean-warning' } }; BlockName = 'End' } + ) { + param($Script, $BlockName) + + $failure = $null + try { & $Script -WarningVariable wv } catch { $failure = $_ } + $failure | Should -Not -BeNullOrEmpty + $failure.Exception.Message | Should -BeExactly 'failure' + $wv | Should -BeExactly 'clean-warning' + } + + It "'Clean' block runs in pipeline - simple function" { + function test-1 { + param([switch] $EmitError) + process { + if ($EmitError) { + throw 'test-1-process-error' + } else { + Write-Output 'test-1' + } + } + + clean { 'test-1-clean' >> $filePath } + } + + function test-2 { + begin { Write-Verbose -Verbose 'test-2-begin' } + process { $_ } + clean { 'test-2-clean' >> $filePath } + } + + function test-3 { + end { Write-Verbose -Verbose 'test-3-end' } + clean { 'test-3-clean' >> $filePath } + } + + ## All command will run, so all 'Clean' blocks will run + test-1 | test-2 | test-3 + Test-Path $filePath | Should -BeTrue + $content = Get-Content $filePath + $content | Should -Be @('test-1-clean', 'test-2-clean', 'test-3-clean') + + $failure = $null + Remove-Item $filePath -Force + try { + test-1 -EmitError | test-2 | test-3 + } catch { + $failure = $_ + } + + ## Exception is thrown from 'test-1.Process'. By that time, the 'test-2.Begin' has run, + ## so 'test-2.Clean' will run. However, 'test-3.End' won't run, so 'test-3.Clean' won't run. + $failure | Should -Not -BeNullOrEmpty + $failure.Exception.Message | Should -BeExactly 'test-1-process-error' + Test-Path $filePath | Should -BeTrue + $content = Get-Content $filePath + $content | Should -Be @('test-1-clean', 'test-2-clean') + } + + It "'Clean' block runs in pipeline - advanced function" { + function test-1 { + [CmdletBinding()] + param([switch] $EmitError) + process { + if ($EmitError) { + throw 'test-1-process-error' + } else { + Write-Output 'test-1' + } + } + + clean { 'test-1-clean' >> $filePath } + } + + function test-2 { + [CmdletBinding()] + param( + [Parameter(ValueFromPipeline)] + $pipeInput + ) + + begin { Write-Verbose -Verbose 'test-2-begin' } + process { $pipeInput } + clean { 'test-2-clean' >> $filePath } + } + + function test-3 { + [CmdletBinding()] + param( + [Parameter(ValueFromPipeline)] + $pipeInput + ) + + end { Write-Verbose -Verbose 'test-3-end' } + clean { 'test-3-clean' >> $filePath } + } + + ## All command will run, so all 'Clean' blocks will run + test-1 | test-2 | test-3 + Test-Path $filePath | Should -BeTrue + $content = Get-Content $filePath + $content | Should -Be @('test-1-clean', 'test-2-clean', 'test-3-clean') + + + $failure = $null + Remove-Item $filePath -Force + ## Exception will be thrown from 'test-1.Process'. By that time, the 'test-2.Begin' has run, + ## so 'test-2.Clean' will run. However, 'test-3.End' won't run, so 'test-3.Clean' won't run. + try { + test-1 -EmitError | test-2 | test-3 + } catch { + $failure = $_ + } + $failure | Should -Not -BeNullOrEmpty + $failure.Exception.Message | Should -BeExactly 'test-1-process-error' + Test-Path $filePath | Should -BeTrue + $content = Get-Content $filePath + $content | Should -Be @('test-1-clean', 'test-2-clean') + } + + It 'does not execute End {} if the pipeline is halted during Process {}' { + # We don't need Should -Not -Throw as if this reaches end{} and throws the test will fail anyway. + 1..10 | + & { + begin { "BEGIN" } + process { "PROCESS $_" } + end { "END"; throw "This should not be reached." } + } | + Select-Object -First 3 | + Should -Be @( "BEGIN", "PROCESS 1", "PROCESS 2" ) + } + + It "still executes 'Clean' block if the pipeline is halted" { + 1..10 | + & { + process { $_ } + clean { "Clean block hit" > $filePath } + } | + Select-Object -First 1 | + Should -Be 1 + + Test-Path $filePath | Should -BeTrue + Get-Content $filePath | Should -BeExactly 'Clean block hit' + } + + It "Select-Object in pipeline" { + function bar { + process { 'bar_' + $_ } end { 'bar_end' } clean { 'bar_clean' > $filePath } + } + + function zoo { + process { 'zoo_' + $_ } end { 'zoo_end' } clean { 'zoo_clean' >> $filePath } + } + + 1..10 | bar | Select-Object -First 2 | zoo | Should -Be @('zoo_bar_1', 'zoo_bar_2', 'zoo_end') + Test-Path $filePath | Should -BeTrue + $content = Get-Content $filePath + $content | Should -Be @('bar_clean', 'zoo_clean') + } + } + + Context 'Streams from Named Blocks' { + + It 'Permits output from named block: