From b3dc569582dda98788edd545aaf4b608b249f41f Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Mar 2023 02:15:13 +0100 Subject: [PATCH 1/8] Small cleanup --- .../Common/InvokeRestMethodCommand.Common.cs | 15 +++++---------- .../commands/utility/WebCmdlet/StreamHelper.cs | 3 +-- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs index a27e4e70008..61674d31926 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs @@ -89,22 +89,17 @@ internal override void ProcessResponse(HttpResponseMessage response) } else { - // determine the response type + // Determine the response type RestReturnType returnType = CheckReturnType(response); // Try to get the response encoding from the ContentType header. - Encoding encoding = null; - string charSet = response.Content.Headers.ContentType?.CharSet; - if (!string.IsNullOrEmpty(charSet)) - { - StreamHelper.TryGetEncoding(charSet, out encoding); - } + string charSet = WebResponseHelper.GetCharacterSet(response); + + string str = StreamHelper.DecodeStream(responseStream, charSet, out Encoding encoding); object obj = null; Exception ex = null; - string str = StreamHelper.DecodeStream(responseStream, ref encoding); - string encodingVerboseName; try { @@ -122,7 +117,7 @@ internal override void ProcessResponse(HttpResponseMessage response) if (returnType == RestReturnType.Json) { - convertSuccess = TryConvertToJson(str, out obj, ref ex) || TryConvertToXml(str, out obj, ref ex); + convertSuccess = TryConvertToJson(str, out object obj, ref ex) || TryConvertToXml(str, out obj, ref ex); } // default to try xml first since it's more common else diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index 0e219e591b9..1e478edb930 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -449,10 +449,9 @@ internal static string DecodeStream(Stream stream, ref Encoding encoding) if (match.Success) { - Encoding localEncoding = null; string characterSet = match.Groups["charset"].Value; - if (TryGetEncoding(characterSet, out localEncoding)) + if (TryGetEncoding(characterSet, out Encoding localEncoding)) { stream.Seek(0, SeekOrigin.Begin); content = StreamToString(stream, localEncoding); From 1333c50479046b0f52ddbafd965beedf2ef66fa5 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Mar 2023 02:32:10 +0100 Subject: [PATCH 2/8] replace var --- .../WebCmdlet/Common/InvokeRestMethodCommand.Common.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs index 61674d31926..11fda6fc614 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs @@ -75,11 +75,11 @@ internal override void ProcessResponse(HttpResponseMessage response) { ArgumentNullException.ThrowIfNull(response); - var baseResponseStream = StreamHelper.GetResponseStream(response); + Stream baseResponseStream = StreamHelper.GetResponseStream(response); if (ShouldWriteToPipeline) { - using var responseStream = new BufferingStreamReader(baseResponseStream); + using BufferingStreamReader responseStream = new(baseResponseStream); // First see if it is an RSS / ATOM feed, in which case we can // stream it - unless the user has overridden it with a return type of "XML" @@ -117,7 +117,7 @@ internal override void ProcessResponse(HttpResponseMessage response) if (returnType == RestReturnType.Json) { - convertSuccess = TryConvertToJson(str, out object obj, ref ex) || TryConvertToXml(str, out obj, ref ex); + convertSuccess = TryConvertToJson(str, out obj, ref ex) || TryConvertToXml(str, out obj, ref ex); } // default to try xml first since it's more common else @@ -263,7 +263,7 @@ private static bool TryConvertToXml(string xml, out object doc, ref Exception ex XmlReaderSettings settings = GetSecureXmlReaderSettings(); XmlReader xmlReader = XmlReader.Create(new StringReader(xml), settings); - var xmlDoc = new XmlDocument(); + XmlDocument xmlDoc = new(); xmlDoc.PreserveWhitespace = true; xmlDoc.Load(xmlReader); @@ -310,7 +310,7 @@ private static bool TryConvertToJson(string json, out object obj, ref Exception } catch (JsonException ex) { - var msg = string.Format(System.Globalization.CultureInfo.CurrentCulture, WebCmdletStrings.JsonDeserializationFailed, ex.Message); + string msg = string.Format(System.Globalization.CultureInfo.CurrentCulture, WebCmdletStrings.JsonDeserializationFailed, ex.Message); exRef = new ArgumentException(msg, ex); obj = null; } From cf4b994fcbfd28d924aaac6b9aa87eb26a32f334 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Mar 2023 03:00:35 +0100 Subject: [PATCH 3/8] add ? --- .../utility/WebCmdlet/CoreCLR/WebResponseHelper.CoreClr.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebResponseHelper.CoreClr.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebResponseHelper.CoreClr.cs index aaf921c7f0d..b1449de0692 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebResponseHelper.CoreClr.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebResponseHelper.CoreClr.cs @@ -10,7 +10,7 @@ namespace Microsoft.PowerShell.Commands { internal static class WebResponseHelper { - internal static string GetCharacterSet(HttpResponseMessage response) => response.Content.Headers.ContentType.CharSet; + internal static string GetCharacterSet(HttpResponseMessage response) => response.Content.Headers.ContentType?.CharSet; internal static Dictionary> GetHeadersDictionary(HttpResponseMessage response) { From c820546e2601dac735de8e066ae90f2affe1b29a Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Sat, 11 Mar 2023 00:12:40 +0100 Subject: [PATCH 4/8] changes from #19308 --- .../Common/BasicHtmlWebResponseObject.Common.cs | 13 ++++--------- .../commands/utility/WebCmdlet/StreamHelper.cs | 9 +-------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/BasicHtmlWebResponseObject.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/BasicHtmlWebResponseObject.Common.cs index 86d4c4dfbba..8e8c099c661 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/BasicHtmlWebResponseObject.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/BasicHtmlWebResponseObject.Common.cs @@ -167,11 +167,6 @@ protected void InitializeContent() // Fill the Content buffer string characterSet = WebResponseHelper.GetCharacterSet(BaseResponse); - if (string.IsNullOrEmpty(characterSet) && ContentHelper.IsJson(contentType)) - { - characterSet = Encoding.UTF8.HeaderName; - } - Content = StreamHelper.DecodeStream(RawContentStream, characterSet, out Encoding encoding); Encoding = encoding; } @@ -218,7 +213,7 @@ private void InitializeRawContent(HttpResponseMessage baseResponse) { StringBuilder raw = ContentHelper.GetRawContentHeader(baseResponse); raw.Append(Content); - this.RawContent = raw.ToString(); + RawContent = raw.ToString(); } private static void ParseAttributes(string outerHtml, PSObject elementObject) @@ -228,16 +223,16 @@ private static void ParseAttributes(string outerHtml, PSObject elementObject) { // Extract just the opening tag of the HTML element (omitting the closing tag and any contents, // including contained HTML elements) - var match = s_tagRegex.Match(outerHtml); + Match match = s_tagRegex.Match(outerHtml); // Extract all the attribute specifications within the HTML element opening tag - var attribMatches = s_attribsRegex.Matches(match.Value); + MatchCollection attribMatches = s_attribsRegex.Matches(match.Value); foreach (Match attribMatch in attribMatches) { // Extract the name and value for this attribute (allowing for variations like single/double/no // quotes, and no value at all) - var nvMatches = s_attribNameValueRegex.Match(attribMatch.Value); + Match nvMatches = s_attribNameValueRegex.Match(attribMatch.Value); Debug.Assert(nvMatches.Groups.Count == 5); // Name is always captured by group #1 diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index 1e478edb930..322072dde1a 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -383,14 +383,7 @@ private static string StreamToString(Stream stream, Encoding encoding) internal static string DecodeStream(Stream stream, string characterSet, out Encoding encoding) { - try - { - encoding = Encoding.GetEncoding(characterSet); - } - catch (ArgumentException) - { - encoding = null; - } + TryGetEncoding(characterSet, out encoding); return DecodeStream(stream, ref encoding); } From df7ab30bc75147f54370f0b6ca98d85dbe82ab62 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Sat, 11 Mar 2023 00:18:49 +0100 Subject: [PATCH 5/8] merge DecodeStream --- .../utility/WebCmdlet/StreamHelper.cs | 63 +++++++++---------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index 322072dde1a..711258e2f3f 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -382,42 +382,9 @@ private static string StreamToString(Stream stream, Encoding encoding) } internal static string DecodeStream(Stream stream, string characterSet, out Encoding encoding) - { - TryGetEncoding(characterSet, out encoding); - - return DecodeStream(stream, ref encoding); - } - - internal static bool TryGetEncoding(string characterSet, out Encoding encoding) - { - bool result = false; - try - { - encoding = Encoding.GetEncoding(characterSet); - result = true; - } - catch (ArgumentException) - { - encoding = null; - } - - return result; - } - - private static readonly Regex s_metaRegex = new( - @"<]*charset\s*=\s*[""'\n]?(?[A-Za-z].[^\s""'\n<>]*)[\s""'\n>]", - RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.ExplicitCapture | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | RegexOptions.NonBacktracking - ); - - private static readonly Regex s_xmlRegex = new( - @"<\?xml\s.*[^.><]*encoding\s*=\s*[""'\n]?(?[A-Za-z].[^\s""'\n<>]*)[\s""'\n>]", - RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.ExplicitCapture | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | RegexOptions.NonBacktracking - ); - - internal static string DecodeStream(Stream stream, ref Encoding encoding) { bool isDefaultEncoding = false; - if (encoding is null) + if (!TryGetEncoding(characterSet, out encoding)) { // Use the default encoding if one wasn't provided encoding = ContentHelper.GetDefaultEncoding(); @@ -442,7 +409,7 @@ internal static string DecodeStream(Stream stream, ref Encoding encoding) if (match.Success) { - string characterSet = match.Groups["charset"].Value; + characterSet = match.Groups["charset"].Value; if (TryGetEncoding(characterSet, out Encoding localEncoding)) { @@ -456,6 +423,32 @@ internal static string DecodeStream(Stream stream, ref Encoding encoding) return content; } + internal static bool TryGetEncoding(string characterSet, out Encoding encoding) + { + bool result = false; + try + { + encoding = Encoding.GetEncoding(characterSet); + result = true; + } + catch (ArgumentException) + { + encoding = null; + } + + return result; + } + + private static readonly Regex s_metaRegex = new( + @"<]*charset\s*=\s*[""'\n]?(?[A-Za-z].[^\s""'\n<>]*)[\s""'\n>]", + RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.ExplicitCapture | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | RegexOptions.NonBacktracking + ); + + private static readonly Regex s_xmlRegex = new( + @"<\?xml\s.*[^.><]*encoding\s*=\s*[""'\n]?(?[A-Za-z].[^\s""'\n<>]*)[\s""'\n>]", + RegexOptions.Compiled | RegexOptions.Singleline | RegexOptions.ExplicitCapture | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | RegexOptions.NonBacktracking + ); + internal static byte[] EncodeToBytes(string str, Encoding encoding) { // Just use the default encoding if one wasn't provided From 57cbd6e515909cecbe2355fd508e2497588b897d Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Sat, 11 Mar 2023 00:45:01 +0100 Subject: [PATCH 6/8] remove StreamReader --- .../WebCmdlet/Common/WebRequestPSCmdlet.Common.cs | 11 +++-------- .../commands/utility/WebCmdlet/StreamHelper.cs | 2 ++ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index acf8135f73f..c1df22296e7 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -609,20 +609,15 @@ protected override void ProcessRecord() HttpResponseException httpEx = new(message, response); ErrorRecord er = new(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); string detailMsg = string.Empty; - StreamReader reader = null; try { - reader = new StreamReader(StreamHelper.GetResponseStream(response)); - detailMsg = FormatErrorMessage(reader.ReadToEnd(), contentType); + string error = StreamHelper.GetResponseString(response); + detailMsg = FormatErrorMessage(error, contentType); } catch { // Catch all } - finally - { - reader?.Dispose(); - } if (!string.IsNullOrEmpty(detailMsg)) { @@ -1791,7 +1786,7 @@ private static string FormatErrorMessage(string error, string contentType) if (string.IsNullOrEmpty(formattedError)) { // Remove HTML tags making it easier to read - formattedError = System.Text.RegularExpressions.Regex.Replace(error, "<[^>]*>", string.Empty); + formattedError = Regex.Replace(error, "<[^>]*>", string.Empty); } return formattedError; diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index 711258e2f3f..1ddaf5a5794 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -457,6 +457,8 @@ internal static byte[] EncodeToBytes(string str, Encoding encoding) return encoding.GetBytes(str); } + internal static string GetResponseString(HttpResponseMessage response) => response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + internal static Stream GetResponseStream(HttpResponseMessage response) => response.Content.ReadAsStreamAsync().GetAwaiter().GetResult(); #endregion Static Methods From 3af2a034b65b39e038e7f9dccdae3778e5af710a Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Mon, 13 Mar 2023 20:03:11 +0100 Subject: [PATCH 7/8] revert following suggestions @daxian-dbw --- .../WebCmdlet/Common/InvokeRestMethodCommand.Common.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs index 11fda6fc614..fd55b370b86 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs @@ -79,7 +79,7 @@ internal override void ProcessResponse(HttpResponseMessage response) if (ShouldWriteToPipeline) { - using BufferingStreamReader responseStream = new(baseResponseStream); + using var responseStream = new BufferingStreamReader(baseResponseStream); // First see if it is an RSS / ATOM feed, in which case we can // stream it - unless the user has overridden it with a return type of "XML" @@ -203,7 +203,7 @@ private bool TryProcessFeedStream(Stream responseStream) if (isRssOrFeed) { - XmlDocument workingDocument = new(); + var workingDocument = new XmlDocument(); // Performing a Read() here to avoid rechecking // "rss" or "feed" items From 788f71944e0afa886605c93a6565535cd2c410c5 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Mon, 13 Mar 2023 20:14:07 +0100 Subject: [PATCH 8/8] revert --- .../WebCmdlet/Common/InvokeRestMethodCommand.Common.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs index fd55b370b86..0c97211faea 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs @@ -203,7 +203,7 @@ private bool TryProcessFeedStream(Stream responseStream) if (isRssOrFeed) { - var workingDocument = new XmlDocument(); + XmlDocument workingDocument = new(); // Performing a Read() here to avoid rechecking // "rss" or "feed" items @@ -263,7 +263,7 @@ private static bool TryConvertToXml(string xml, out object doc, ref Exception ex XmlReaderSettings settings = GetSecureXmlReaderSettings(); XmlReader xmlReader = XmlReader.Create(new StringReader(xml), settings); - XmlDocument xmlDoc = new(); + var xmlDoc = new XmlDocument(); xmlDoc.PreserveWhitespace = true; xmlDoc.Load(xmlReader);