From 79f5f49aff5563e93d20d61e63677c6357dd7b64 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Fri, 20 Jan 2023 17:49:51 +0100 Subject: [PATCH 1/7] Add UnixSocket to WebCmdlets --- .../Common/WebRequestPSCmdlet.Common.cs | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 73eb34013a5..904128644ab 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -11,6 +11,7 @@ using System.Net; using System.Net.Http; using System.Net.Http.Headers; +using System.Net.Sockets; using System.Security; using System.Security.Authentication; using System.Security.Cryptography; @@ -286,6 +287,13 @@ public virtual string CustomMethod private string _custommethod; + /// + /// Gets or sets the UnixSocket property. + /// + [Parameter] + [ValidateNotNullOrEmpty] + public virtual UnixDomainSocketEndPoint UnixSocket { get; set; } + #endregion #region NoProxy @@ -913,7 +921,21 @@ public abstract partial class WebRequestPSCmdlet : PSCmdlet // and PreserveAuthorizationOnRedirect is NOT set. internal virtual HttpClient GetHttpClient(bool handleRedirect) { - HttpClientHandler handler = new(); + //HttpClientHandler handler = new(); + SocketsHttpHandler handler = new(); + + if (UnixSocket is not null) + { + handler.ConnectCallback = async (context, token) => + { + Socket socket = new(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + UnixDomainSocketEndPoint endpoint = UnixSocket; + await socket.ConnectAsync(endpoint).ConfigureAwait(false); + + return new NetworkStream(socket, ownsSocket: false); + }; + } + handler.CookieContainer = WebSession.Cookies; handler.AutomaticDecompression = DecompressionMethods.All; @@ -921,7 +943,8 @@ internal virtual HttpClient GetHttpClient(bool handleRedirect) if (WebSession.UseDefaultCredentials) { // the UseDefaultCredentials flag overrides other supplied credentials - handler.UseDefaultCredentials = true; + //handler.UseDefaultCredentials = true; + handler.Credentials = CredentialCache.DefaultCredentials; } else if (WebSession.Credentials is not null) { @@ -939,13 +962,15 @@ internal virtual HttpClient GetHttpClient(bool handleRedirect) if (WebSession.Certificates is not null) { - handler.ClientCertificates.AddRange(WebSession.Certificates); + //handler.ClientCertificates.AddRange(WebSession.Certificates); + handler.SslOptions.ClientCertificates.AddRange(WebSession.Certificates); } if (SkipCertificateCheck) { - handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; - handler.ClientCertificateOptions = ClientCertificateOption.Manual; + //handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; + //handler.ClientCertificateOptions = ClientCertificateOption.Manual; + handler.SslOptions.RemoteCertificateValidationCallback = delegate { return true; }; } // This indicates GetResponse will handle redirects. @@ -958,7 +983,8 @@ internal virtual HttpClient GetHttpClient(bool handleRedirect) handler.MaxAutomaticRedirections = WebSession.MaximumRedirection; } - handler.SslProtocols = (SslProtocols)SslProtocol; + //handler.SslProtocols = (SslProtocols)SslProtocol; + handler.SslOptions.EnabledSslProtocols = (SslProtocols)SslProtocol; HttpClient httpClient = new(handler); @@ -1513,7 +1539,7 @@ protected override void ProcessRecord() // Errors with redirection counts of greater than 0 are handled automatically by .NET, but are // impossible to detect programmatically when we hit this limit. By handling this ourselves // (and still writing out the result), users can debug actual HTTP redirect problems. - if (WebSession.MaximumRedirection == 0 && IsRedirectCode(response.StatusCode)) // Indicate "HttpClientHandler.AllowAutoRedirect is false" + if (WebSession.MaximumRedirection == 0 && IsRedirectCode(response.StatusCode)) { ErrorRecord er = new(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request); er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded); From 5e05397d02f6b239f303624140e76d04d8620531 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Sat, 21 Jan 2023 16:33:32 +0100 Subject: [PATCH 2/7] remove commented stuff --- .../utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 904128644ab..17136e70f2f 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -921,7 +921,6 @@ public abstract partial class WebRequestPSCmdlet : PSCmdlet // and PreserveAuthorizationOnRedirect is NOT set. internal virtual HttpClient GetHttpClient(bool handleRedirect) { - //HttpClientHandler handler = new(); SocketsHttpHandler handler = new(); if (UnixSocket is not null) @@ -943,7 +942,6 @@ internal virtual HttpClient GetHttpClient(bool handleRedirect) if (WebSession.UseDefaultCredentials) { // the UseDefaultCredentials flag overrides other supplied credentials - //handler.UseDefaultCredentials = true; handler.Credentials = CredentialCache.DefaultCredentials; } else if (WebSession.Credentials is not null) @@ -962,7 +960,6 @@ internal virtual HttpClient GetHttpClient(bool handleRedirect) if (WebSession.Certificates is not null) { - //handler.ClientCertificates.AddRange(WebSession.Certificates); handler.SslOptions.ClientCertificates.AddRange(WebSession.Certificates); } @@ -983,7 +980,6 @@ internal virtual HttpClient GetHttpClient(bool handleRedirect) handler.MaxAutomaticRedirections = WebSession.MaximumRedirection; } - //handler.SslProtocols = (SslProtocols)SslProtocol; handler.SslOptions.EnabledSslProtocols = (SslProtocols)SslProtocol; HttpClient httpClient = new(handler); From 890b530164998585632f0378c2031a18f61635d1 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Sat, 21 Jan 2023 17:15:31 +0100 Subject: [PATCH 3/7] more comments stuff --- .../WebCmdlet/Common/WebRequestPSCmdlet.Common.cs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 17136e70f2f..1441626470b 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -938,10 +938,10 @@ internal virtual HttpClient GetHttpClient(bool handleRedirect) handler.CookieContainer = WebSession.Cookies; handler.AutomaticDecompression = DecompressionMethods.All; - // set the credentials used by this request + // Set the credentials used by this request if (WebSession.UseDefaultCredentials) { - // the UseDefaultCredentials flag overrides other supplied credentials + // The UseDefaultCredentials flag overrides other supplied credentials handler.Credentials = CredentialCache.DefaultCredentials; } else if (WebSession.Credentials is not null) @@ -965,8 +965,6 @@ internal virtual HttpClient GetHttpClient(bool handleRedirect) if (SkipCertificateCheck) { - //handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; - //handler.ClientCertificateOptions = ClientCertificateOption.Manual; handler.SslOptions.RemoteCertificateValidationCallback = delegate { return true; }; } @@ -995,7 +993,7 @@ internal virtual HttpRequestMessage GetRequest(Uri uri) Uri requestUri = PrepareUri(uri); HttpMethod httpMethod = string.IsNullOrEmpty(CustomMethod) ? GetHttpMethod(Method) : new HttpMethod(CustomMethod); - // create the base WebRequest object + // Create the base WebRequest object var request = new HttpRequestMessage(httpMethod, requestUri); if (HttpVersion is not null) @@ -1003,7 +1001,7 @@ internal virtual HttpRequestMessage GetRequest(Uri uri) request.Version = HttpVersion; } - // pull in session data + // Pull in session data if (WebSession.Headers.Count > 0) { WebSession.ContentHeaders.Clear(); From 4fc3f15f8b2b5b172e0e5e0438df124ee2252243 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Sat, 4 Feb 2023 01:30:36 +0100 Subject: [PATCH 4/7] fix tests --- .../Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 index 43363edb6eb..0b47f2b462e 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 @@ -1408,7 +1408,6 @@ Describe "Invoke-WebRequest tests" -Tags "Feature", "RequireAdminOnWindows" { ConvertFrom-Json $result.Status | Should -Be 'OK' - $result.Thumbprint | Should -Be $certificate.Thumbprint } } @@ -2892,7 +2891,6 @@ Describe "Invoke-RestMethod tests" -Tags "Feature", "RequireAdminOnWindows" { $result = Invoke-RestMethod -Uri $uri -Certificate $certificate -SkipCertificateCheck $result.Status | Should -Be 'OK' - $result.Thumbprint | Should -Be $certificate.Thumbprint } } From 4707c46bf680d7b75a9616ebebbf3bc1dfc9ca5d Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Sat, 4 Feb 2023 23:43:09 +0100 Subject: [PATCH 5/7] Fix handler ClientCertificates to fix tests; revert changes in tests --- .../utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs | 8 ++++---- .../Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index b1fca4c3618..c12b6c3971f 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -962,7 +962,7 @@ internal virtual HttpClient GetHttpClient(bool handleRedirect) if (WebSession.Certificates is not null) { - handler.SslOptions.ClientCertificates.AddRange(WebSession.Certificates); + handler.SslOptions.ClientCertificates = new X509CertificateCollection(WebSession.Certificates); } if (SkipCertificateCheck) @@ -982,10 +982,10 @@ internal virtual HttpClient GetHttpClient(bool handleRedirect) handler.SslOptions.EnabledSslProtocols = (SslProtocols)SslProtocol; - HttpClient httpClient = new(handler); + // Check timeout setting (in seconds) + handler.ConnectTimeout = TimeoutSec is 0 ? TimeSpan.FromMilliseconds(Timeout.Infinite) : new TimeSpan(0, 0, TimeoutSec); - // Check timeout setting (in seconds instead of milliseconds as in HttpWebRequest) - httpClient.Timeout = TimeoutSec is 0 ? TimeSpan.FromMilliseconds(Timeout.Infinite) : new TimeSpan(0, 0, TimeoutSec); + HttpClient httpClient = new(handler); return httpClient; } diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 index 0b47f2b462e..43363edb6eb 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 @@ -1408,6 +1408,7 @@ Describe "Invoke-WebRequest tests" -Tags "Feature", "RequireAdminOnWindows" { ConvertFrom-Json $result.Status | Should -Be 'OK' + $result.Thumbprint | Should -Be $certificate.Thumbprint } } @@ -2891,6 +2892,7 @@ Describe "Invoke-RestMethod tests" -Tags "Feature", "RequireAdminOnWindows" { $result = Invoke-RestMethod -Uri $uri -Certificate $certificate -SkipCertificateCheck $result.Status | Should -Be 'OK' + $result.Thumbprint | Should -Be $certificate.Thumbprint } } From 22474d4ad353535bd8672095014bc0418a1adc3b Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Sat, 4 Feb 2023 23:59:28 +0100 Subject: [PATCH 6/7] revert --- .../utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index c12b6c3971f..af7faf693d4 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -982,11 +982,11 @@ internal virtual HttpClient GetHttpClient(bool handleRedirect) handler.SslOptions.EnabledSslProtocols = (SslProtocols)SslProtocol; - // Check timeout setting (in seconds) - handler.ConnectTimeout = TimeoutSec is 0 ? TimeSpan.FromMilliseconds(Timeout.Infinite) : new TimeSpan(0, 0, TimeoutSec); - HttpClient httpClient = new(handler); + // Check timeout setting (in seconds) + httpClient.Timeout = TimeoutSec is 0 ? TimeSpan.FromMilliseconds(Timeout.Infinite) : new TimeSpan(0, 0, TimeoutSec); + return httpClient; } From be750033db7916997c6dc0b66defdb0161da3942 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Fri, 10 Feb 2023 11:26:06 +0100 Subject: [PATCH 7/7] merge --- .../Common/WebRequestPSCmdlet.Common.cs | 861 +++++++++--------- 1 file changed, 425 insertions(+), 436 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 3d14ee70f48..58e1696368e 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -89,8 +89,47 @@ public enum WebSslProtocol /// /// Base class for Invoke-RestMethod and Invoke-WebRequest commands. /// - public abstract partial class WebRequestPSCmdlet : PSCmdlet + public abstract class WebRequestPSCmdlet : PSCmdlet { + #region Fields + + /// + /// Cancellation token source. + /// + internal CancellationTokenSource _cancelToken = null; + + /// + /// Automatically follow Rel Links. + /// + internal bool _followRelLink = false; + + /// + /// Maximum number of Rel Links to follow. + /// + internal int _maximumFollowRelLink = int.MaxValue; + + /// + /// Parse Rel Links. + /// + internal bool _parseRelLink = false; + + /// + /// Automatically follow Rel Links. + /// + internal Dictionary _relationLink = null; + + /// + /// The current size of the local file being resumed. + /// + private long _resumeFileSize = 0; + + /// + /// The remote endpoint returned a 206 status code indicating successful resume. + /// + private bool _resumeSuccess = false; + + #endregion Fields + #region Virtual Properties #region URI @@ -433,6 +472,213 @@ public virtual string CustomMethod #endregion Virtual Properties + #region Helper Properties + + internal string QualifiedOutFile => QualifyFilePath(OutFile); + + internal bool ShouldCheckHttpStatus => !SkipHttpErrorCheck; + + /// + /// Determines whether writing to a file should Resume and append rather than overwrite. + /// + internal bool ShouldResume => Resume.IsPresent && _resumeSuccess; + + internal bool ShouldSaveToOutFile => !string.IsNullOrEmpty(OutFile); + + internal bool ShouldWriteToPipeline => !ShouldSaveToOutFile || PassThru; + + #endregion Helper Properties + + #region Abstract Methods + + /// + /// Read the supplied WebResponse object and push the resulting output into the pipeline. + /// + /// Instance of a WebResponse object to be processed. + internal abstract void ProcessResponse(HttpResponseMessage response); + + #endregion Abstract Methods + + #region Overrides + + /// + /// The main execution method for cmdlets derived from WebRequestPSCmdlet. + /// + protected override void ProcessRecord() + { + try + { + // Set cmdlet context for write progress + ValidateParameters(); + PrepareSession(); + + // If the request contains an authorization header and PreserveAuthorizationOnRedirect is not set, + // it needs to be stripped on the first redirect. + bool keepAuthorizationOnRedirect = PreserveAuthorizationOnRedirect.IsPresent + && WebSession.Headers.ContainsKey(HttpKnownHeaderNames.Authorization); + + bool handleRedirect = keepAuthorizationOnRedirect || AllowInsecureRedirect; + + using (HttpClient client = GetHttpClient(handleRedirect)) + { + int followedRelLink = 0; + Uri uri = Uri; + do + { + if (followedRelLink > 0) + { + string linkVerboseMsg = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.FollowingRelLinkVerboseMsg, + uri.AbsoluteUri); + + WriteVerbose(linkVerboseMsg); + } + + using (HttpRequestMessage request = GetRequest(uri)) + { + FillRequestStream(request); + try + { + long requestContentLength = request.Content is null ? 0 : request.Content.Headers.ContentLength.Value; + + string reqVerboseMsg = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.WebMethodInvocationVerboseMsg, + request.Version, + request.Method, + requestContentLength); + + WriteVerbose(reqVerboseMsg); + + using HttpResponseMessage response = GetResponse(client, request, handleRedirect); + + string contentType = ContentHelper.GetContentType(response); + string respVerboseMsg = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.WebResponseVerboseMsg, + response.Content.Headers.ContentLength, + contentType); + + WriteVerbose(respVerboseMsg); + + bool _isSuccess = response.IsSuccessStatusCode; + + // Check if the Resume range was not satisfiable because the file already completed downloading. + // This happens when the local file is the same size as the remote file. + if (Resume.IsPresent + && response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable + && response.Content.Headers.ContentRange.HasLength + && response.Content.Headers.ContentRange.Length == _resumeFileSize) + { + _isSuccess = true; + WriteVerbose(string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.OutFileWritingSkipped, + OutFile)); + + // Disable writing to the OutFile. + OutFile = null; + } + + if (ShouldCheckHttpStatus && !_isSuccess) + { + string message = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.ResponseStatusCodeFailure, + (int)response.StatusCode, + response.ReasonPhrase); + + HttpResponseException httpEx = new(message, response); + ErrorRecord er = new(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); + string detailMsg = string.Empty; + StreamReader reader = null; + try + { + reader = new StreamReader(StreamHelper.GetResponseStream(response)); + detailMsg = FormatErrorMessage(reader.ReadToEnd(), contentType); + } + catch + { + // Catch all + } + finally + { + reader?.Dispose(); + } + + if (!string.IsNullOrEmpty(detailMsg)) + { + er.ErrorDetails = new ErrorDetails(detailMsg); + } + + ThrowTerminatingError(er); + } + + if (_parseRelLink || _followRelLink) + { + ParseLinkHeader(response, uri); + } + + ProcessResponse(response); + UpdateSession(response); + + // If we hit our maximum redirection count, generate an error. + // Errors with redirection counts of greater than 0 are handled automatically by .NET, but are + // impossible to detect programmatically when we hit this limit. By handling this ourselves + // (and still writing out the result), users can debug actual HTTP redirect problems. + if (WebSession.MaximumRedirection == 0 && IsRedirectCode(response.StatusCode)) + { + ErrorRecord er = new(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request); + er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded); + WriteError(er); + } + } + catch (HttpRequestException ex) + { + ErrorRecord er = new(ex, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); + if (ex.InnerException is not null) + { + er.ErrorDetails = new ErrorDetails(ex.InnerException.Message); + } + + ThrowTerminatingError(er); + } + + if (_followRelLink) + { + if (!_relationLink.ContainsKey("next")) + { + return; + } + + uri = new Uri(_relationLink["next"]); + followedRelLink++; + } + } + } + while (_followRelLink && (followedRelLink < _maximumFollowRelLink)); + } + } + catch (CryptographicException ex) + { + ErrorRecord er = new(ex, "WebCmdletCertificateException", ErrorCategory.SecurityError, null); + ThrowTerminatingError(er); + } + catch (NotSupportedException ex) + { + ErrorRecord er = new(ex, "WebCmdletIEDomNotSupportedException", ErrorCategory.NotImplemented, null); + ThrowTerminatingError(er); + } + } + + /// + /// To implement ^C. + /// + protected override void StopProcessing() => _cancelToken?.Cancel(); + + #endregion Overrides + #region Virtual Methods internal virtual void ValidateParameters() @@ -692,247 +938,18 @@ internal virtual void PrepareSession() WebSession.RetryIntervalInSeconds = RetryIntervalSec; } } + + internal virtual HttpClient GetHttpClient(bool handleRedirect) + { + SocketsHttpHandler handler = new(); - #endregion Virtual Methods - - #region Helper Properties - - internal string QualifiedOutFile => QualifyFilePath(OutFile); - - internal bool ShouldSaveToOutFile => !string.IsNullOrEmpty(OutFile); - - internal bool ShouldWriteToPipeline => !ShouldSaveToOutFile || PassThru; - - internal bool ShouldCheckHttpStatus => !SkipHttpErrorCheck; - - /// - /// Determines whether writing to a file should Resume and append rather than overwrite. - /// - internal bool ShouldResume => Resume.IsPresent && _resumeSuccess; - - #endregion Helper Properties - - #region Helper Methods - private Uri PrepareUri(Uri uri) - { - uri = CheckProtocol(uri); - - // Before creating the web request, - // preprocess Body if content is a dictionary and method is GET (set as query) - LanguagePrimitives.TryConvertTo(Body, out IDictionary bodyAsDictionary); - if (bodyAsDictionary is not null && (Method == WebRequestMethod.Default || Method == WebRequestMethod.Get || CustomMethod == "GET")) - { - UriBuilder uriBuilder = new(uri); - if (uriBuilder.Query is not null && uriBuilder.Query.Length > 1) - { - uriBuilder.Query = string.Concat(uriBuilder.Query.AsSpan(1), "&", FormatDictionary(bodyAsDictionary)); - } - else - { - uriBuilder.Query = FormatDictionary(bodyAsDictionary); - } - - uri = uriBuilder.Uri; - - // Set body to null to prevent later FillRequestStream - Body = null; - } - - return uri; - } - - private static Uri CheckProtocol(Uri uri) - { - ArgumentNullException.ThrowIfNull(uri); - - if (!uri.IsAbsoluteUri) - { - uri = new Uri("http://" + uri.OriginalString); - } - - return uri; - } - - private string QualifyFilePath(string path) - { - string resolvedFilePath = PathUtils.ResolveFilePath(filePath: path, command: this, isLiteralPath: true); - return resolvedFilePath; - } - - private static string FormatDictionary(IDictionary content) - { - ArgumentNullException.ThrowIfNull(content); - - StringBuilder bodyBuilder = new(); - foreach (string key in content.Keys) - { - if (bodyBuilder.Length > 0) - { - bodyBuilder.Append('&'); - } - - object value = content[key]; - - // URLEncode the key and value - string encodedKey = WebUtility.UrlEncode(key); - string encodedValue = string.Empty; - if (value is not null) - { - encodedValue = WebUtility.UrlEncode(value.ToString()); - } - - bodyBuilder.Append($"{encodedKey}={encodedValue}"); - } - - return bodyBuilder.ToString(); - } - - private ErrorRecord GetValidationError(string msg, string errorId) - { - var ex = new ValidationMetadataException(msg); - var error = new ErrorRecord(ex, errorId, ErrorCategory.InvalidArgument, this); - return error; - } - - private ErrorRecord GetValidationError(string msg, string errorId, params object[] args) - { - msg = string.Format(CultureInfo.InvariantCulture, msg, args); - var ex = new ValidationMetadataException(msg); - var error = new ErrorRecord(ex, errorId, ErrorCategory.InvalidArgument, this); - return error; - } - - private string GetBasicAuthorizationHeader() - { - var password = new NetworkCredential(null, Credential.Password).Password; - string unencoded = string.Create(CultureInfo.InvariantCulture, $"{Credential.UserName}:{password}"); - byte[] bytes = Encoding.UTF8.GetBytes(unencoded); - return string.Create(CultureInfo.InvariantCulture, $"Basic {Convert.ToBase64String(bytes)}"); - } - - private string GetBearerAuthorizationHeader() - { - return string.Create(CultureInfo.InvariantCulture, $"Bearer {new NetworkCredential(string.Empty, Token).Password}"); - } - - private void ProcessAuthentication() - { - if (Authentication == WebAuthenticationType.Basic) - { - WebSession.Headers["Authorization"] = GetBasicAuthorizationHeader(); - } - else if (Authentication == WebAuthenticationType.Bearer || Authentication == WebAuthenticationType.OAuth) - { - WebSession.Headers["Authorization"] = GetBearerAuthorizationHeader(); - } - else - { - Diagnostics.Assert(false, string.Create(CultureInfo.InvariantCulture, $"Unrecognized Authentication value: {Authentication}")); - } - } - - #endregion Helper Methods - } - - // TODO: Merge Partials - - /// - /// Exception class for webcmdlets to enable returning HTTP error response. - /// - public sealed class HttpResponseException : HttpRequestException - { - /// - /// Initializes a new instance of the class. - /// - /// Message for the exception. - /// Response from the HTTP server. - public HttpResponseException(string message, HttpResponseMessage response) : base(message, inner: null, response.StatusCode) - { - Response = response; - } - - /// - /// HTTP error response. - /// - public HttpResponseMessage Response { get; } - } - - /// - /// Base class for Invoke-RestMethod and Invoke-WebRequest commands. - /// - public abstract partial class WebRequestPSCmdlet : PSCmdlet - { - #region Abstract Methods - - /// - /// Read the supplied WebResponse object and push the resulting output into the pipeline. - /// - /// Instance of a WebResponse object to be processed. - internal abstract void ProcessResponse(HttpResponseMessage response); - - #endregion Abstract Methods - - /// - /// Cancellation token source. - /// - internal CancellationTokenSource _cancelToken = null; - - /// - /// Parse Rel Links. - /// - internal bool _parseRelLink = false; - - /// - /// Automatically follow Rel Links. - /// - internal bool _followRelLink = false; - - /// - /// Automatically follow Rel Links. - /// - internal Dictionary _relationLink = null; - - /// - /// Maximum number of Rel Links to follow. - /// - internal int _maximumFollowRelLink = int.MaxValue; - - /// - /// The remote endpoint returned a 206 status code indicating successful resume. - /// - private bool _resumeSuccess = false; - - /// - /// The current size of the local file being resumed. - /// - private long _resumeFileSize = 0; - - private static HttpMethod GetHttpMethod(WebRequestMethod method) => method switch - { - WebRequestMethod.Default or WebRequestMethod.Get => HttpMethod.Get, - WebRequestMethod.Delete => HttpMethod.Delete, - WebRequestMethod.Head => HttpMethod.Head, - WebRequestMethod.Patch => HttpMethod.Patch, - WebRequestMethod.Post => HttpMethod.Post, - WebRequestMethod.Put => HttpMethod.Put, - WebRequestMethod.Options => HttpMethod.Options, - WebRequestMethod.Trace => HttpMethod.Trace, - _ => new HttpMethod(method.ToString().ToUpperInvariant()) - }; - - #region Virtual Methods - - internal virtual HttpClient GetHttpClient(bool handleRedirect) - { - SocketsHttpHandler handler = new(); - - if (UnixSocket is not null) - { - handler.ConnectCallback = async (context, token) => - { - Socket socket = new(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); - UnixDomainSocketEndPoint endpoint = UnixSocket; - await socket.ConnectAsync(endpoint).ConfigureAwait(false); + if (UnixSocket is not null) + { + handler.ConnectCallback = async (context, token) => + { + Socket socket = new(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + UnixDomainSocketEndPoint endpoint = UnixSocket; + await socket.ConnectAsync(endpoint).ConfigureAwait(false); return new NetworkStream(socket, ownsSocket: false); }; @@ -1207,45 +1224,6 @@ internal virtual void FillRequestStream(HttpRequestMessage request) } } - // Returns true if the status code is one of the supported redirection codes. - private static bool IsRedirectCode(HttpStatusCode code) - { - int intCode = (int)code; - return - ( - (intCode >= 300 && intCode < 304) || - intCode == 307 || - intCode == 308 - ); - } - - // Returns true if the status code is a redirection code and the action requires switching from POST to GET on redirection. - // NOTE: Some of these status codes map to the same underlying value but spelling them out for completeness. - private static bool IsRedirectToGet(HttpStatusCode code) - { - return - ( - code == HttpStatusCode.Found || - code == HttpStatusCode.Moved || - code == HttpStatusCode.Redirect || - code == HttpStatusCode.RedirectMethod || - code == HttpStatusCode.SeeOther || - code == HttpStatusCode.Ambiguous || - code == HttpStatusCode.MultipleChoices - ); - } - - // Returns true if the status code shows a server or client error and MaximumRetryCount > 0 - private bool ShouldRetry(HttpStatusCode code) - { - int intCode = (int)code; - - return - ( - (intCode == 304 || (intCode >= 400 && intCode <= 599)) && WebSession.MaximumRetryCount > 0 - ); - } - internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestMessage request, bool handleRedirect) { ArgumentNullException.ThrowIfNull(client); @@ -1388,188 +1366,125 @@ internal virtual void UpdateSession(HttpResponseMessage response) #endregion Virtual Methods - #region Overrides - - /// - /// The main execution method for cmdlets derived from WebRequestPSCmdlet. - /// - protected override void ProcessRecord() + #region Helper Methods + private Uri PrepareUri(Uri uri) { - try - { - // Set cmdlet context for write progress - ValidateParameters(); - PrepareSession(); - - // If the request contains an authorization header and PreserveAuthorizationOnRedirect is not set, - // it needs to be stripped on the first redirect. - bool keepAuthorizationOnRedirect = PreserveAuthorizationOnRedirect.IsPresent - && WebSession.Headers.ContainsKey(HttpKnownHeaderNames.Authorization); - - bool handleRedirect = keepAuthorizationOnRedirect || AllowInsecureRedirect; + uri = CheckProtocol(uri); - using (HttpClient client = GetHttpClient(handleRedirect)) + // Before creating the web request, + // preprocess Body if content is a dictionary and method is GET (set as query) + LanguagePrimitives.TryConvertTo(Body, out IDictionary bodyAsDictionary); + if (bodyAsDictionary is not null && (Method == WebRequestMethod.Default || Method == WebRequestMethod.Get || CustomMethod == "GET")) + { + UriBuilder uriBuilder = new(uri); + if (uriBuilder.Query is not null && uriBuilder.Query.Length > 1) { - int followedRelLink = 0; - Uri uri = Uri; - do - { - if (followedRelLink > 0) - { - string linkVerboseMsg = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.FollowingRelLinkVerboseMsg, - uri.AbsoluteUri); - - WriteVerbose(linkVerboseMsg); - } - - using (HttpRequestMessage request = GetRequest(uri)) - { - FillRequestStream(request); - try - { - long requestContentLength = request.Content is null ? 0 : request.Content.Headers.ContentLength.Value; + uriBuilder.Query = string.Concat(uriBuilder.Query.AsSpan(1), "&", FormatDictionary(bodyAsDictionary)); + } + else + { + uriBuilder.Query = FormatDictionary(bodyAsDictionary); + } - string reqVerboseMsg = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.WebMethodInvocationVerboseMsg, - request.Version, - request.Method, - requestContentLength); + uri = uriBuilder.Uri; - WriteVerbose(reqVerboseMsg); + // Set body to null to prevent later FillRequestStream + Body = null; + } - using HttpResponseMessage response = GetResponse(client, request, handleRedirect); + return uri; + } - string contentType = ContentHelper.GetContentType(response); - string respVerboseMsg = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.WebResponseVerboseMsg, - response.Content.Headers.ContentLength, - contentType); + private static Uri CheckProtocol(Uri uri) + { + ArgumentNullException.ThrowIfNull(uri); - WriteVerbose(respVerboseMsg); + if (!uri.IsAbsoluteUri) + { + uri = new Uri("http://" + uri.OriginalString); + } - bool _isSuccess = response.IsSuccessStatusCode; + return uri; + } - // Check if the Resume range was not satisfiable because the file already completed downloading. - // This happens when the local file is the same size as the remote file. - if (Resume.IsPresent - && response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable - && response.Content.Headers.ContentRange.HasLength - && response.Content.Headers.ContentRange.Length == _resumeFileSize) - { - _isSuccess = true; - WriteVerbose(string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.OutFileWritingSkipped, - OutFile)); + private string QualifyFilePath(string path) + { + string resolvedFilePath = PathUtils.ResolveFilePath(filePath: path, command: this, isLiteralPath: true); + return resolvedFilePath; + } - // Disable writing to the OutFile. - OutFile = null; - } + private static string FormatDictionary(IDictionary content) + { + ArgumentNullException.ThrowIfNull(content); - if (ShouldCheckHttpStatus && !_isSuccess) - { - string message = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.ResponseStatusCodeFailure, - (int)response.StatusCode, - response.ReasonPhrase); + StringBuilder bodyBuilder = new(); + foreach (string key in content.Keys) + { + if (bodyBuilder.Length > 0) + { + bodyBuilder.Append('&'); + } - HttpResponseException httpEx = new(message, response); - ErrorRecord er = new(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); - string detailMsg = string.Empty; - StreamReader reader = null; - try - { - reader = new StreamReader(StreamHelper.GetResponseStream(response)); - detailMsg = FormatErrorMessage(reader.ReadToEnd(), contentType); - } - catch - { - // Catch all - } - finally - { - reader?.Dispose(); - } + object value = content[key]; - if (!string.IsNullOrEmpty(detailMsg)) - { - er.ErrorDetails = new ErrorDetails(detailMsg); - } + // URLEncode the key and value + string encodedKey = WebUtility.UrlEncode(key); + string encodedValue = string.Empty; + if (value is not null) + { + encodedValue = WebUtility.UrlEncode(value.ToString()); + } - ThrowTerminatingError(er); - } + bodyBuilder.Append($"{encodedKey}={encodedValue}"); + } - if (_parseRelLink || _followRelLink) - { - ParseLinkHeader(response, uri); - } + return bodyBuilder.ToString(); + } - ProcessResponse(response); - UpdateSession(response); + private ErrorRecord GetValidationError(string msg, string errorId) + { + var ex = new ValidationMetadataException(msg); + var error = new ErrorRecord(ex, errorId, ErrorCategory.InvalidArgument, this); + return error; + } - // If we hit our maximum redirection count, generate an error. - // Errors with redirection counts of greater than 0 are handled automatically by .NET, but are - // impossible to detect programmatically when we hit this limit. By handling this ourselves - // (and still writing out the result), users can debug actual HTTP redirect problems. - if (WebSession.MaximumRedirection == 0 && IsRedirectCode(response.StatusCode)) - { - ErrorRecord er = new(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request); - er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded); - WriteError(er); - } - } - catch (HttpRequestException ex) - { - ErrorRecord er = new(ex, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); - if (ex.InnerException is not null) - { - er.ErrorDetails = new ErrorDetails(ex.InnerException.Message); - } + private ErrorRecord GetValidationError(string msg, string errorId, params object[] args) + { + msg = string.Format(CultureInfo.InvariantCulture, msg, args); + var ex = new ValidationMetadataException(msg); + var error = new ErrorRecord(ex, errorId, ErrorCategory.InvalidArgument, this); + return error; + } - ThrowTerminatingError(er); - } + private string GetBasicAuthorizationHeader() + { + var password = new NetworkCredential(null, Credential.Password).Password; + string unencoded = string.Create(CultureInfo.InvariantCulture, $"{Credential.UserName}:{password}"); + byte[] bytes = Encoding.UTF8.GetBytes(unencoded); + return string.Create(CultureInfo.InvariantCulture, $"Basic {Convert.ToBase64String(bytes)}"); + } - if (_followRelLink) - { - if (!_relationLink.ContainsKey("next")) - { - return; - } + private string GetBearerAuthorizationHeader() + { + return string.Create(CultureInfo.InvariantCulture, $"Bearer {new NetworkCredential(string.Empty, Token).Password}"); + } - uri = new Uri(_relationLink["next"]); - followedRelLink++; - } - } - } - while (_followRelLink && (followedRelLink < _maximumFollowRelLink)); - } + private void ProcessAuthentication() + { + if (Authentication == WebAuthenticationType.Basic) + { + WebSession.Headers["Authorization"] = GetBasicAuthorizationHeader(); } - catch (CryptographicException ex) + else if (Authentication == WebAuthenticationType.Bearer || Authentication == WebAuthenticationType.OAuth) { - ErrorRecord er = new(ex, "WebCmdletCertificateException", ErrorCategory.SecurityError, null); - ThrowTerminatingError(er); + WebSession.Headers["Authorization"] = GetBearerAuthorizationHeader(); } - catch (NotSupportedException ex) + else { - ErrorRecord er = new(ex, "WebCmdletIEDomNotSupportedException", ErrorCategory.NotImplemented, null); - ThrowTerminatingError(er); + Diagnostics.Assert(false, string.Create(CultureInfo.InvariantCulture, $"Unrecognized Authentication value: {Authentication}")); } } - - /// - /// To implement ^C. - /// - protected override void StopProcessing() => _cancelToken?.Cancel(); - - #endregion Overrides - - #region Helper Methods - + /// /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. /// @@ -1744,7 +1659,7 @@ internal void ParseLinkHeader(HttpResponseMessage response, System.Uri requestUr /// /// The Field Name to use. /// The Field Value to use. - /// The > to update. + /// The to update. /// If true, collection types in will be enumerated. If false, collections will be treated as single value. private void AddMultipartContent(object fieldName, object fieldValue, MultipartFormDataContent formData, bool enumerate) { @@ -1835,6 +1750,7 @@ private static StreamContent GetMultipartStreamContent(object fieldName, Stream private static StreamContent GetMultipartFileContent(object fieldName, FileInfo file) { var result = GetMultipartStreamContent(fieldName: fieldName, stream: new FileStream(file.FullName, FileMode.Open)); + // .NET does not enclose field names in quotes, however, modern browsers and curl do. result.Headers.ContentDisposition.FileName = "\"" + file.Name + "\""; @@ -1894,6 +1810,79 @@ private static string FormatErrorMessage(string error, string contentType) return formattedError; } + // Returns true if the status code is one of the supported redirection codes. + private static bool IsRedirectCode(HttpStatusCode code) + { + int intCode = (int)code; + return + ( + (intCode >= 300 && intCode < 304) || + intCode == 307 || + intCode == 308 + ); + } + + // Returns true if the status code is a redirection code and the action requires switching from POST to GET on redirection. + // NOTE: Some of these status codes map to the same underlying value but spelling them out for completeness. + private static bool IsRedirectToGet(HttpStatusCode code) + { + return + ( + code == HttpStatusCode.Found || + code == HttpStatusCode.Moved || + code == HttpStatusCode.Redirect || + code == HttpStatusCode.RedirectMethod || + code == HttpStatusCode.SeeOther || + code == HttpStatusCode.Ambiguous || + code == HttpStatusCode.MultipleChoices + ); + } + + // Returns true if the status code shows a server or client error and MaximumRetryCount > 0 + private bool ShouldRetry(HttpStatusCode code) + { + int intCode = (int)code; + + return + ( + (intCode == 304 || (intCode >= 400 && intCode <= 599)) && WebSession.MaximumRetryCount > 0 + ); + } + + private static HttpMethod GetHttpMethod(WebRequestMethod method) => method switch + { + WebRequestMethod.Default or WebRequestMethod.Get => HttpMethod.Get, + WebRequestMethod.Delete => HttpMethod.Delete, + WebRequestMethod.Head => HttpMethod.Head, + WebRequestMethod.Patch => HttpMethod.Patch, + WebRequestMethod.Post => HttpMethod.Post, + WebRequestMethod.Put => HttpMethod.Put, + WebRequestMethod.Options => HttpMethod.Options, + WebRequestMethod.Trace => HttpMethod.Trace, + _ => new HttpMethod(method.ToString().ToUpperInvariant()) + }; + #endregion Helper Methods } + + /// + /// Exception class for webcmdlets to enable returning HTTP error response. + /// + public sealed class HttpResponseException : HttpRequestException + { + /// + /// Initializes a new instance of the class. + /// + /// Message for the exception. + /// Response from the HTTP server. + public HttpResponseException(string message, HttpResponseMessage response) : base(message, inner: null, response.StatusCode) + { + Response = response; + } + + /// + /// HTTP error response. + /// + public HttpResponseMessage Response { get; } + } }