From def9f84fcd2a01a68ac9f8a1ef494534034602ac Mon Sep 17 00:00:00 2001 From: Ilya Date: Sun, 17 Nov 2019 22:57:57 +0500 Subject: [PATCH 1/9] Use async streams in Invoke-RestMethod --- .../Common/InvokeRestMethodCommand.Common.cs | 151 +++++++++--------- .../Common/WebRequestPSCmdlet.Common.cs | 2 +- .../InvokeWebRequestCommand.CoreClr.cs | 2 +- .../utility/WebCmdlet/StreamHelper.cs | 65 +++----- 4 files changed, 101 insertions(+), 119 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs index b82d47952da..3bd30903172 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs @@ -85,7 +85,7 @@ public int MaximumFollowRelLink #region Helper Methods - private bool TryProcessFeedStream(BufferingStreamReader responseStream) + private bool TryProcessFeedStream(Stream responseStream) { bool isRssOrFeed = false; @@ -382,95 +382,94 @@ internal override void ProcessResponse(HttpResponseMessage response) { if (response == null) { throw new ArgumentNullException("response"); } - using (BufferingStreamReader responseStream = new BufferingStreamReader(StreamHelper.GetResponseStream(response))) + Stream responseStream = StreamHelper.GetResponseStream(response); + + if (ShouldWriteToPipeline) { - if (ShouldWriteToPipeline) + // First see if it is an RSS / ATOM feed, in which case we can + // stream it - unless the user has overridden it with a return type of "XML" + if (TryProcessFeedStream(responseStream)) + { + // Do nothing, content has been processed. + } + else { - // First see if it is an RSS / ATOM feed, in which case we can - // stream it - unless the user has overridden it with a return type of "XML" - if (TryProcessFeedStream(responseStream)) + // determine the response type + RestReturnType returnType = CheckReturnType(response); + + // Try to get the response encoding from the ContentType header. + Encoding encoding = null; + string charSet = response.Content.Headers.ContentType?.CharSet; + if (!string.IsNullOrEmpty(charSet)) { - // Do nothing, content has been processed. + // NOTE: Don't use ContentHelper.GetEncoding; it returns a + // default which bypasses checking for a meta charset value. + StreamHelper.TryGetEncoding(charSet, out encoding); } - else - { - // determine the response type - RestReturnType returnType = CheckReturnType(response); - - // Try to get the response encoding from the ContentType header. - Encoding encoding = null; - string charSet = response.Content.Headers.ContentType?.CharSet; - if (!string.IsNullOrEmpty(charSet)) - { - // NOTE: Don't use ContentHelper.GetEncoding; it returns a - // default which bypasses checking for a meta charset value. - StreamHelper.TryGetEncoding(charSet, out encoding); - } - - if (string.IsNullOrEmpty(charSet) && returnType == RestReturnType.Json) - { - encoding = Encoding.UTF8; - } - object obj = null; - Exception ex = null; + if (string.IsNullOrEmpty(charSet) && returnType == RestReturnType.Json) + { + encoding = Encoding.UTF8; + } - string str = StreamHelper.DecodeStream(responseStream, ref encoding); + object obj = null; + Exception ex = null; - string encodingVerboseName; - try - { - encodingVerboseName = string.IsNullOrEmpty(encoding.HeaderName) ? encoding.EncodingName : encoding.HeaderName; - } - catch (NotSupportedException) - { - encodingVerboseName = encoding.EncodingName; - } - // NOTE: Tests use this verbose output to verify the encoding. - WriteVerbose(string.Format - ( - System.Globalization.CultureInfo.InvariantCulture, - "Content encoding: {0}", - encodingVerboseName) - ); - bool convertSuccess = false; - - if (returnType == RestReturnType.Json) - { - convertSuccess = TryConvertToJson(str, out obj, ref ex) || TryConvertToXml(str, out obj, ref ex); - } - // default to try xml first since it's more common - else - { - convertSuccess = TryConvertToXml(str, out obj, ref ex) || TryConvertToJson(str, out obj, ref ex); - } + string str = StreamHelper.DecodeStream(responseStream, ref encoding); - if (!convertSuccess) - { - // fallback to string - obj = str; - } + string encodingVerboseName; + try + { + encodingVerboseName = string.IsNullOrEmpty(encoding.HeaderName) ? encoding.EncodingName : encoding.HeaderName; + } + catch (NotSupportedException) + { + encodingVerboseName = encoding.EncodingName; + } + // NOTE: Tests use this verbose output to verify the encoding. + WriteVerbose(string.Format + ( + System.Globalization.CultureInfo.InvariantCulture, + "Content encoding: {0}", + encodingVerboseName) + ); + bool convertSuccess = false; + + if (returnType == RestReturnType.Json) + { + convertSuccess = TryConvertToJson(str, out obj, ref ex) || TryConvertToXml(str, out obj, ref ex); + } + // default to try xml first since it's more common + else + { + convertSuccess = TryConvertToXml(str, out obj, ref ex) || TryConvertToJson(str, out obj, ref ex); + } - WriteObject(obj); + if (!convertSuccess) + { + // fallback to string + obj = str; } - } - if (ShouldSaveToOutFile) - { - StreamHelper.SaveStreamToFile(responseStream, QualifiedOutFile, this); + WriteObject(obj); } + } - if (!string.IsNullOrEmpty(StatusCodeVariable)) - { - PSVariableIntrinsics vi = SessionState.PSVariable; - vi.Set(StatusCodeVariable, (int)response.StatusCode); - } + if (ShouldSaveToOutFile) + { + StreamHelper.SaveStreamToFile(responseStream, QualifiedOutFile, this, _cancelToken); + } - if (!string.IsNullOrEmpty(ResponseHeadersVariable)) - { - PSVariableIntrinsics vi = SessionState.PSVariable; - vi.Set(ResponseHeadersVariable, WebResponseHelper.GetHeadersDictionary(response)); - } + if (!string.IsNullOrEmpty(StatusCodeVariable)) + { + PSVariableIntrinsics vi = SessionState.PSVariable; + vi.Set(StatusCodeVariable, (int)response.StatusCode); + } + + if (!string.IsNullOrEmpty(ResponseHeadersVariable)) + { + PSVariableIntrinsics vi = SessionState.PSVariable; + vi.Set(ResponseHeadersVariable, WebResponseHelper.GetHeadersDictionary(response)); } } diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index ddac58fe57d..1aea7aa82ee 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -911,7 +911,7 @@ public abstract partial class WebRequestPSCmdlet : PSCmdlet /// /// Cancellation token source. /// - private CancellationTokenSource _cancelToken = null; + internal CancellationTokenSource _cancelToken = null; /// /// Parse Rel Links. diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs index 0a5441faef5..2130050192a 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs @@ -51,7 +51,7 @@ internal override void ProcessResponse(HttpResponseMessage response) if (ShouldSaveToOutFile) { - StreamHelper.SaveStreamToFile(responseStream, QualifiedOutFile, this); + StreamHelper.SaveStreamToFile(responseStream, QualifiedOutFile, this, _cancelToken); } } diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index 91a9a0e9975..cf44056ff23 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -9,6 +9,8 @@ using System.Net.Http; using System.Text; using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; namespace Microsoft.PowerShell.Commands { @@ -99,7 +101,7 @@ public override long Length /// /// /// - public override System.Threading.Tasks.Task CopyToAsync(Stream destination, int bufferSize, System.Threading.CancellationToken cancellationToken) + public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) { Initialize(); return base.CopyToAsync(destination, bufferSize, cancellationToken); @@ -124,7 +126,7 @@ public override int Read(byte[] buffer, int offset, int count) /// /// /// - public override System.Threading.Tasks.Task ReadAsync(byte[] buffer, int offset, int count, System.Threading.CancellationToken cancellationToken) + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { Initialize(); return base.ReadAsync(buffer, offset, count, cancellationToken); @@ -175,7 +177,7 @@ public override void Write(byte[] buffer, int offset, int count) /// /// /// - public override System.Threading.Tasks.Task WriteAsync(byte[] buffer, int offset, int count, System.Threading.CancellationToken cancellationToken) + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { Initialize(); return base.WriteAsync(buffer, offset, count, cancellationToken); @@ -273,47 +275,27 @@ internal static class StreamHelper #region Static Methods - internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet) + internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, CancellationTokenSource cts) { - byte[] data = new byte[ChunkSize]; - - int read = 0; - long totalWritten = 0; - do + if (cmdlet == null) { - if (cmdlet != null) - { - ProgressRecord record = new ProgressRecord(ActivityId, - WebCmdletStrings.WriteRequestProgressActivity, - StringUtil.Format(WebCmdletStrings.WriteRequestProgressStatus, totalWritten)); - cmdlet.WriteProgress(record); - } - - read = input.Read(data, 0, ChunkSize); + throw new ArgumentNullException(nameof(cmdlet)); + } - if (0 < read) - { - output.Write(data, 0, read); - totalWritten += read; - } - } while (read != 0); + var copyTask = input.CopyToAsync(output, cts.Token); - if (cmdlet != null) + ProgressRecord record = new ProgressRecord( + ActivityId, + WebCmdletStrings.WriteRequestProgressActivity, + StringUtil.Format(WebCmdletStrings.WriteRequestComplete, 0)); + do { - ProgressRecord record = new ProgressRecord(ActivityId, - WebCmdletStrings.WriteRequestProgressActivity, - StringUtil.Format(WebCmdletStrings.WriteRequestComplete, totalWritten)); - record.RecordType = ProgressRecordType.Completed; + record.StatusDescription = StringUtil.Format(WebCmdletStrings.WriteRequestComplete, output.Position); cmdlet.WriteProgress(record); - } - output.Flush(); - } - - internal static void WriteToStream(byte[] input, Stream output) - { - output.Write(input, 0, input.Length); - output.Flush(); + Task.Delay(1000).Wait(cts.Token); + } + while (!cts.IsCancellationRequested && !copyTask.IsCompleted); } /// @@ -323,21 +305,22 @@ internal static void WriteToStream(byte[] input, Stream output) /// /// /// - internal static void SaveStreamToFile(Stream stream, string filePath, PSCmdlet cmdlet) + /// + internal static void SaveStreamToFile(Stream stream, string filePath, PSCmdlet cmdlet, CancellationTokenSource cts) { // If the web cmdlet should resume, append the file instead of overwriting. if (cmdlet is WebRequestPSCmdlet webCmdlet && webCmdlet.ShouldResume) { using (FileStream output = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.Read)) { - WriteToStream(stream, output, cmdlet); + WriteToStream(stream, output, cmdlet, cts); } } else { - using (FileStream output = File.Create(filePath)) + using (FileStream output = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { - WriteToStream(stream, output, cmdlet); + WriteToStream(stream, output, cmdlet,cts); } } } From c3f016f20675fac073cd91d731b6f1bc3a56d2a0 Mon Sep 17 00:00:00 2001 From: Ilya Date: Sun, 17 Nov 2019 23:05:38 +0500 Subject: [PATCH 2/9] Fix typo. --- .../commands/utility/WebCmdlet/StreamHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index cf44056ff23..c9f453689fe 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -320,7 +320,7 @@ internal static void SaveStreamToFile(Stream stream, string filePath, PSCmdlet c { using (FileStream output = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { - WriteToStream(stream, output, cmdlet,cts); + WriteToStream(stream, output, cmdlet, cts); } } } From 9703c43706fa73e4c0ab7c17be542c9ae30d3363 Mon Sep 17 00:00:00 2001 From: Ilya Date: Mon, 18 Nov 2019 11:02:14 +0500 Subject: [PATCH 3/9] Explicitly split ShouldWriteToPipeline and ShouldSaveToOutFile --- .../WebCmdlet/Common/InvokeRestMethodCommand.Common.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs index 3bd30903172..4169b6122bf 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs @@ -382,10 +382,12 @@ internal override void ProcessResponse(HttpResponseMessage response) { if (response == null) { throw new ArgumentNullException("response"); } - Stream responseStream = StreamHelper.GetResponseStream(response); + var baseResponseStream = StreamHelper.GetResponseStream(response); if (ShouldWriteToPipeline) { + using (BufferingStreamReader responseStream = new BufferingStreamReader(baseResponseStream)) + // First see if it is an RSS / ATOM feed, in which case we can // stream it - unless the user has overridden it with a return type of "XML" if (TryProcessFeedStream(responseStream)) @@ -454,10 +456,9 @@ internal override void ProcessResponse(HttpResponseMessage response) WriteObject(obj); } } - - if (ShouldSaveToOutFile) + else if (ShouldSaveToOutFile) { - StreamHelper.SaveStreamToFile(responseStream, QualifiedOutFile, this, _cancelToken); + StreamHelper.SaveStreamToFile(baseResponseStream, QualifiedOutFile, this, _cancelToken); } if (!string.IsNullOrEmpty(StatusCodeVariable)) From 2abf0b99dc1a5a43fe11a624ddd7a6dfd1da6fa0 Mon Sep 17 00:00:00 2001 From: Ilya Date: Mon, 18 Nov 2019 11:59:42 +0500 Subject: [PATCH 4/9] Simplify SaveStreamToFile method --- .../utility/WebCmdlet/StreamHelper.cs | 27 ++++++------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index c9f453689fe..63bc9796590 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -282,7 +282,7 @@ internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, throw new ArgumentNullException(nameof(cmdlet)); } - var copyTask = input.CopyToAsync(output, cts.Token); + Task copyTask = input.CopyToAsync(output, cts.Token); ProgressRecord record = new ProgressRecord( ActivityId, @@ -302,27 +302,16 @@ internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, /// Saves content from stream into filePath. /// Caller need to ensure position is properly set. /// - /// - /// - /// - /// + /// Input stream. + /// Output file name. + /// Current cmdlet (Invoke-WebRequest or Invoke-RestMethod). + /// CancellationTokenSource to track the cmdlet cancellation. internal static void SaveStreamToFile(Stream stream, string filePath, PSCmdlet cmdlet, CancellationTokenSource cts) { // If the web cmdlet should resume, append the file instead of overwriting. - if (cmdlet is WebRequestPSCmdlet webCmdlet && webCmdlet.ShouldResume) - { - using (FileStream output = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.Read)) - { - WriteToStream(stream, output, cmdlet, cts); - } - } - else - { - using (FileStream output = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read)) - { - WriteToStream(stream, output, cmdlet, cts); - } - } + FileMode fileMode = cmdlet is WebRequestPSCmdlet webCmdlet && webCmdlet.ShouldResume ? FileMode.Append : FileMode.Create; + using FileStream output = new FileStream(filePath, fileMode, FileAccess.Write, FileShare.Read); + WriteToStream(stream, output, cmdlet, cts); } private static string StreamToString(Stream stream, Encoding encoding) From a1e0f2aeb4d07575ec2e82603a18637e64e812c0 Mon Sep 17 00:00:00 2001 From: Ilya Date: Wed, 29 Jan 2020 11:52:12 +0500 Subject: [PATCH 5/9] Address feedback --- .../Common/InvokeRestMethodCommand.Common.cs | 4 +-- .../InvokeWebRequestCommand.CoreClr.cs | 2 +- .../utility/WebCmdlet/StreamHelper.cs | 27 ++++++++++++------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs index 4169b6122bf..ee3db835eec 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs @@ -386,7 +386,7 @@ internal override void ProcessResponse(HttpResponseMessage response) if (ShouldWriteToPipeline) { - using (BufferingStreamReader responseStream = new BufferingStreamReader(baseResponseStream)) + using var responseStream = new BufferingStreamReader(baseResponseStream); // First see if it is an RSS / ATOM feed, in which case we can // stream it - unless the user has overridden it with a return type of "XML" @@ -458,7 +458,7 @@ internal override void ProcessResponse(HttpResponseMessage response) } else if (ShouldSaveToOutFile) { - StreamHelper.SaveStreamToFile(baseResponseStream, QualifiedOutFile, this, _cancelToken); + StreamHelper.SaveStreamToFile(baseResponseStream, QualifiedOutFile, this, _cancelToken.Token); } if (!string.IsNullOrEmpty(StatusCodeVariable)) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs index 2130050192a..63baf2946fc 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs @@ -51,7 +51,7 @@ internal override void ProcessResponse(HttpResponseMessage response) if (ShouldSaveToOutFile) { - StreamHelper.SaveStreamToFile(responseStream, QualifiedOutFile, this, _cancelToken); + StreamHelper.SaveStreamToFile(responseStream, QualifiedOutFile, this, _cancelToken.Token); } } diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index 63bc9796590..d5fad6a4e06 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -275,27 +275,34 @@ internal static class StreamHelper #region Static Methods - internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, CancellationTokenSource cts) + internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, CancellationToken cancellationToken) { if (cmdlet == null) { throw new ArgumentNullException(nameof(cmdlet)); } - Task copyTask = input.CopyToAsync(output, cts.Token); + Task copyTask = input.CopyToAsync(output, cancellationToken); ProgressRecord record = new ProgressRecord( ActivityId, WebCmdletStrings.WriteRequestProgressActivity, StringUtil.Format(WebCmdletStrings.WriteRequestComplete, 0)); - do + try { - record.StatusDescription = StringUtil.Format(WebCmdletStrings.WriteRequestComplete, output.Position); - cmdlet.WriteProgress(record); + do + { + record.StatusDescription = StringUtil.Format(WebCmdletStrings.WriteRequestProgressStatus, output.Position); + cmdlet.WriteProgress(record); - Task.Delay(1000).Wait(cts.Token); + Task.Delay(1000).Wait(cancellationToken); + } + while (!copyTask.IsCompleted); + } + catch + { + // Catch OperationCanceledException from Wait() } - while (!cts.IsCancellationRequested && !copyTask.IsCompleted); } /// @@ -305,13 +312,13 @@ internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, /// Input stream. /// Output file name. /// Current cmdlet (Invoke-WebRequest or Invoke-RestMethod). - /// CancellationTokenSource to track the cmdlet cancellation. - internal static void SaveStreamToFile(Stream stream, string filePath, PSCmdlet cmdlet, CancellationTokenSource cts) + /// CancellationToken to track the cmdlet cancellation. + internal static void SaveStreamToFile(Stream stream, string filePath, PSCmdlet cmdlet, CancellationToken cancellationToken) { // If the web cmdlet should resume, append the file instead of overwriting. FileMode fileMode = cmdlet is WebRequestPSCmdlet webCmdlet && webCmdlet.ShouldResume ? FileMode.Append : FileMode.Create; using FileStream output = new FileStream(filePath, fileMode, FileAccess.Write, FileShare.Read); - WriteToStream(stream, output, cmdlet, cts); + WriteToStream(stream, output, cmdlet, cancellationToken); } private static string StreamToString(Stream stream, Encoding encoding) From 5d4b9777e9b77ccde94def59b6ac0c877e4a1036 Mon Sep 17 00:00:00 2001 From: Ilya Date: Thu, 30 Jan 2020 08:19:29 +0500 Subject: [PATCH 6/9] Address feedback 2 --- .../commands/utility/WebCmdlet/StreamHelper.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index d5fad6a4e06..5bde0836fdc 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -287,7 +287,7 @@ internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, ProgressRecord record = new ProgressRecord( ActivityId, WebCmdletStrings.WriteRequestProgressActivity, - StringUtil.Format(WebCmdletStrings.WriteRequestComplete, 0)); + string.Empty); try { do @@ -299,9 +299,13 @@ internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, } while (!copyTask.IsCompleted); } - catch + catch (OperationCanceledException) { - // Catch OperationCanceledException from Wait() + } + finally + { + record.StatusDescription = StringUtil.Format(WebCmdletStrings.WriteRequestComplete, output.Position); + cmdlet.WriteProgress(record); } } From 70fb55c5b6280ca540994af4240e8c811cc427aa Mon Sep 17 00:00:00 2001 From: Ilya Date: Thu, 30 Jan 2020 09:03:20 +0500 Subject: [PATCH 7/9] Update ProgressRecord --- .../commands/utility/WebCmdlet/StreamHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index 5bde0836fdc..1c1559582b5 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -287,7 +287,7 @@ internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, ProgressRecord record = new ProgressRecord( ActivityId, WebCmdletStrings.WriteRequestProgressActivity, - string.Empty); + WebCmdletStrings.WriteRequestProgressStatus); try { do From 3de608cbd89cbad1f126d7dab5f1af57c3189872 Mon Sep 17 00:00:00 2001 From: Ilya Date: Fri, 31 Jan 2020 08:36:48 +0500 Subject: [PATCH 8/9] Add cancellation check --- .../commands/utility/WebCmdlet/StreamHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index 1c1559582b5..f394715c0dc 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -297,7 +297,7 @@ internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, Task.Delay(1000).Wait(cancellationToken); } - while (!copyTask.IsCompleted); + while (!copyTask.IsCompleted && !cancellationToken.IsCancellationRequested); } catch (OperationCanceledException) { From 8beead84e2b7d3c96b9dffccb61ea9214ecc9ca2 Mon Sep 17 00:00:00 2001 From: Ilya Date: Sat, 1 Feb 2020 00:45:52 +0500 Subject: [PATCH 9/9] Fix last WriteProgress --- .../commands/utility/WebCmdlet/StreamHelper.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index f394715c0dc..91662ed7878 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -298,15 +298,16 @@ internal static void WriteToStream(Stream input, Stream output, PSCmdlet cmdlet, Task.Delay(1000).Wait(cancellationToken); } while (!copyTask.IsCompleted && !cancellationToken.IsCancellationRequested); + + if (copyTask.IsCompleted) + { + record.StatusDescription = StringUtil.Format(WebCmdletStrings.WriteRequestComplete, output.Position); + cmdlet.WriteProgress(record); + } } catch (OperationCanceledException) { } - finally - { - record.StatusDescription = StringUtil.Format(WebCmdletStrings.WriteRequestComplete, output.Position); - cmdlet.WriteProgress(record); - } } ///