From 1f85d03384ef4586dd31eee25790afe31dc14c01 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 16:50:19 +0100 Subject: [PATCH 01/16] move abstract methods --- .../Common/WebRequestPSCmdlet.Common.cs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index bd4e762f5a9..7a13def90b6 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -823,6 +823,16 @@ private void ProcessAuthentication() } #endregion Helper Methods + + #region Abstract Methods + + /// + /// Read the supplied WebResponse object and push the resulting output into the pipeline. + /// + /// Instance of a WebResponse object to be processed. + internal abstract void ProcessResponse(HttpResponseMessage response); + + #endregion Abstract Methods } // TODO: Merge Partials @@ -853,15 +863,6 @@ public HttpResponseException(string message, HttpResponseMessage response) : bas /// public abstract partial class WebRequestPSCmdlet : PSCmdlet { - #region Abstract Methods - - /// - /// Read the supplied WebResponse object and push the resulting output into the pipeline. - /// - /// Instance of a WebResponse object to be processed. - internal abstract void ProcessResponse(HttpResponseMessage response); - - #endregion Abstract Methods /// /// Cancellation token source. From 17334242d5856545ff0d00e42a0d29fecf5c18a4 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 16:56:42 +0100 Subject: [PATCH 02/16] merge helper methods --- .../Common/WebRequestPSCmdlet.Common.cs | 1723 ++++++++--------- 1 file changed, 860 insertions(+), 863 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 7a13def90b6..2484d782caf 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -821,1063 +821,1060 @@ private void ProcessAuthentication() Diagnostics.Assert(false, string.Create(CultureInfo.InvariantCulture, $"Unrecognized Authentication value: {Authentication}")); } } - - #endregion Helper Methods - - #region Abstract Methods - + /// - /// Read the supplied WebResponse object and push the resulting output into the pipeline. + /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. /// - /// Instance of a WebResponse object to be processed. - internal abstract void ProcessResponse(HttpResponseMessage response); - - #endregion Abstract Methods - } + /// The WebRequest who's content is to be set. + /// A byte array containing the content data. + /// + /// Because this function sets the request's ContentLength property and writes content data into the request's stream, + /// it should be called one time maximum on a given request. + /// + internal void SetRequestContent(HttpRequestMessage request, byte[] content) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(content); - // TODO: Merge Partials + ByteArrayContent byteArrayContent = new(content); + request.Content = byteArrayContent; + } - /// - /// Exception class for webcmdlets to enable returning HTTP error response. - /// - public sealed class HttpResponseException : HttpRequestException - { /// - /// Initializes a new instance of the class. + /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. /// - /// Message for the exception. - /// Response from the HTTP server. - public HttpResponseException(string message, HttpResponseMessage response) : base(message, inner: null, response.StatusCode) + /// The WebRequest who's content is to be set. + /// A String object containing the content data. + /// + /// Because this function sets the request's ContentLength property and writes content data into the request's stream, + /// it should be called one time maximum on a given request. + /// + internal void SetRequestContent(HttpRequestMessage request, string content) { - Response = response; - } + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(content); + + Encoding encoding = null; + if (ContentType is not null) + { + // If Content-Type contains the encoding format (as CharSet), use this encoding format + // to encode the Body of the WebRequest sent to the server. Default Encoding format + // would be used if Charset is not supplied in the Content-Type property. + try + { + var mediaTypeHeaderValue = MediaTypeHeaderValue.Parse(ContentType); + if (!string.IsNullOrEmpty(mediaTypeHeaderValue.CharSet)) + { + encoding = Encoding.GetEncoding(mediaTypeHeaderValue.CharSet); + } + } + catch (Exception ex) when (ex is FormatException || ex is ArgumentException) + { + if (!SkipHeaderValidation) + { + ValidationMetadataException outerEx = new(WebCmdletStrings.ContentTypeException, ex); + ErrorRecord er = new(outerEx, "WebCmdletContentTypeException", ErrorCategory.InvalidArgument, ContentType); + ThrowTerminatingError(er); + } + } + } - /// - /// HTTP error response. - /// - public HttpResponseMessage Response { get; } - } + byte[] bytes = StreamHelper.EncodeToBytes(content, encoding); + ByteArrayContent byteArrayContent = new(bytes); + request.Content = byteArrayContent; + } - /// - /// Base class for Invoke-RestMethod and Invoke-WebRequest commands. - /// - public abstract partial class WebRequestPSCmdlet : PSCmdlet - { + internal void SetRequestContent(HttpRequestMessage request, XmlNode xmlNode) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(xmlNode); - /// - /// Cancellation token source. - /// - internal CancellationTokenSource _cancelToken = null; + byte[] bytes = null; + XmlDocument doc = xmlNode as XmlDocument; + if (doc?.FirstChild is XmlDeclaration) + { + XmlDeclaration decl = doc.FirstChild as XmlDeclaration; + Encoding encoding = Encoding.GetEncoding(decl.Encoding); + bytes = StreamHelper.EncodeToBytes(doc.OuterXml, encoding); + } + else + { + bytes = StreamHelper.EncodeToBytes(xmlNode.OuterXml, encoding: null); + } - /// - /// Parse Rel Links. - /// - internal bool _parseRelLink = false; + ByteArrayContent byteArrayContent = new(bytes); - /// - /// Automatically follow Rel Links. - /// - internal bool _followRelLink = false; + request.Content = byteArrayContent; + } /// - /// Automatically follow Rel Links. + /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. /// - internal Dictionary _relationLink = null; + /// The WebRequest who's content is to be set. + /// A Stream object containing the content data. + /// + /// Because this function sets the request's ContentLength property and writes content data into the request's stream, + /// it should be called one time maximum on a given request. + /// + internal void SetRequestContent(HttpRequestMessage request, Stream contentStream) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(contentStream); - /// - /// Maximum number of Rel Links to follow. - /// - internal int _maximumFollowRelLink = int.MaxValue; + StreamContent streamContent = new(contentStream); + request.Content = streamContent; + } /// - /// The remote endpoint returned a 206 status code indicating successful resume. + /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. /// - private bool _resumeSuccess = false; + /// The WebRequest who's content is to be set. + /// A MultipartFormDataContent object containing multipart/form-data content. + /// + /// Because this function sets the request's ContentLength property and writes content data into the request's stream, + /// it should be called one time maximum on a given request. + /// + internal void SetRequestContent(HttpRequestMessage request, MultipartFormDataContent multipartContent) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(multipartContent); + + // Content headers will be set by MultipartFormDataContent which will throw unless we clear them first + WebSession.ContentHeaders.Clear(); - /// - /// The current size of the local file being resumed. - /// - private long _resumeFileSize = 0; + request.Content = multipartContent; + } - private static HttpMethod GetHttpMethod(WebRequestMethod method) => method switch + internal void SetRequestContent(HttpRequestMessage request, IDictionary content) { - WebRequestMethod.Default or WebRequestMethod.Get => HttpMethod.Get, - WebRequestMethod.Delete => HttpMethod.Delete, - WebRequestMethod.Head => HttpMethod.Head, - WebRequestMethod.Patch => HttpMethod.Patch, - WebRequestMethod.Post => HttpMethod.Post, - WebRequestMethod.Put => HttpMethod.Put, - WebRequestMethod.Options => HttpMethod.Options, - WebRequestMethod.Trace => HttpMethod.Trace, - _ => new HttpMethod(method.ToString().ToUpperInvariant()) - }; + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(content); - #region Virtual Methods + string body = FormatDictionary(content); + SetRequestContent(request, body); + } - // NOTE: Only pass true for handleRedirect if the original request has an authorization header - // and PreserveAuthorizationOnRedirect is NOT set. - internal virtual HttpClient GetHttpClient(bool handleRedirect) + internal void ParseLinkHeader(HttpResponseMessage response, System.Uri requestUri) { - HttpClientHandler handler = new(); - handler.CookieContainer = WebSession.Cookies; - handler.AutomaticDecompression = DecompressionMethods.All; - - // set the credentials used by this request - if (WebSession.UseDefaultCredentials) + if (_relationLink is null) { - // the UseDefaultCredentials flag overrides other supplied credentials - handler.UseDefaultCredentials = true; + // Must ignore the case of relation links. See RFC 8288 (https://tools.ietf.org/html/rfc8288) + _relationLink = new Dictionary(StringComparer.OrdinalIgnoreCase); } - else if (WebSession.Credentials is not null) + else { - handler.Credentials = WebSession.Credentials; + _relationLink.Clear(); } - if (NoProxy) - { - handler.UseProxy = false; - } - else if (WebSession.Proxy is not null) + // We only support the URL in angle brackets and `rel`, other attributes are ignored + // user can still parse it themselves via the Headers property + const string pattern = "<(?.*?)>;\\s*rel=(?\")?(?(?(quoted).*?|[^,;]*))(?(quoted)\")"; + if (response.Headers.TryGetValues("Link", out IEnumerable links)) { - handler.Proxy = WebSession.Proxy; + foreach (string linkHeader in links) + { + MatchCollection matchCollection = Regex.Matches(linkHeader, pattern); + foreach (Match match in matchCollection) + { + if (match.Success) + { + string url = match.Groups["url"].Value; + string rel = match.Groups["rel"].Value; + if (url != string.Empty && rel != string.Empty && !_relationLink.ContainsKey(rel)) + { + Uri absoluteUri = new(requestUri, url); + _relationLink.Add(rel, absoluteUri.AbsoluteUri); + } + } + } + } } + } - if (WebSession.Certificates is not null) + /// + /// Adds content to a . Object type detection is used to determine if the value is string, File, or Collection. + /// + /// The Field Name to use. + /// The Field Value to use. + /// The to update. + /// If true, collection types in will be enumerated. If false, collections will be treated as single value. + private void AddMultipartContent(object fieldName, object fieldValue, MultipartFormDataContent formData, bool enumerate) + { + ArgumentNullException.ThrowIfNull(formData); + + // It is possible that the dictionary keys or values are PSObject wrapped depending on how the dictionary is defined and assigned. + // Before processing the field name and value we need to ensure we are working with the base objects and not the PSObject wrappers. + + // Unwrap fieldName PSObjects + if (fieldName is PSObject namePSObject) { - handler.ClientCertificates.AddRange(WebSession.Certificates); + fieldName = namePSObject.BaseObject; } - if (SkipCertificateCheck) + // Unwrap fieldValue PSObjects + if (fieldValue is PSObject valuePSObject) { - handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; - handler.ClientCertificateOptions = ClientCertificateOption.Manual; + fieldValue = valuePSObject.BaseObject; } - // This indicates GetResponse will handle redirects. - if (handleRedirect || WebSession.MaximumRedirection == 0) + // Treat a single FileInfo as a FileContent + if (fieldValue is FileInfo file) { - handler.AllowAutoRedirect = false; + formData.Add(GetMultipartFileContent(fieldName: fieldName, file: file)); + return; } - else if (WebSession.MaximumRedirection > 0) + + // Treat Strings and other single values as a StringContent. + // If enumeration is false, also treat IEnumerables as StringContents. + // String implements IEnumerable so the explicit check is required. + if (!enumerate || fieldValue is string || fieldValue is not IEnumerable) { - handler.MaxAutomaticRedirections = WebSession.MaximumRedirection; + formData.Add(GetMultipartStringContent(fieldName: fieldName, fieldValue: fieldValue)); + return; } - handler.SslProtocols = (SslProtocols)SslProtocol; + // Treat the value as a collection and enumerate it if enumeration is true + if (enumerate && fieldValue is IEnumerable items) + { + foreach (var item in items) + { + // Recurse, but do not enumerate the next level. IEnumerables will be treated as single values. + AddMultipartContent(fieldName: fieldName, fieldValue: item, formData: formData, enumerate: false); + } + } + } - HttpClient httpClient = new(handler); + /// + /// Gets a from the supplied field name and field value. Uses to convert the objects to strings. + /// + /// The Field Name to use for the + /// The Field Value to use for the + private static StringContent GetMultipartStringContent(object fieldName, object fieldValue) + { + var contentDisposition = new ContentDispositionHeaderValue("form-data"); + // .NET does not enclose field names in quotes, however, modern browsers and curl do. + contentDisposition.Name = "\"" + LanguagePrimitives.ConvertTo(fieldName) + "\""; - // Check timeout setting (in seconds instead of milliseconds as in HttpWebRequest) - httpClient.Timeout = TimeoutSec is 0 ? TimeSpan.FromMilliseconds(Timeout.Infinite) : new TimeSpan(0, 0, TimeoutSec); + var result = new StringContent(LanguagePrimitives.ConvertTo(fieldValue)); + result.Headers.ContentDisposition = contentDisposition; - return httpClient; + return result; } - internal virtual HttpRequestMessage GetRequest(Uri uri) + /// + /// Gets a from the supplied field name and . Uses to convert the fieldname to a string. + /// + /// The Field Name to use for the + /// The to use for the + private static StreamContent GetMultipartStreamContent(object fieldName, Stream stream) { - Uri requestUri = PrepareUri(uri); - HttpMethod httpMethod = string.IsNullOrEmpty(CustomMethod) ? GetHttpMethod(Method) : new HttpMethod(CustomMethod); + var contentDisposition = new ContentDispositionHeaderValue("form-data"); + // .NET does not enclose field names in quotes, however, modern browsers and curl do. + contentDisposition.Name = "\"" + LanguagePrimitives.ConvertTo(fieldName) + "\""; - // create the base WebRequest object - var request = new HttpRequestMessage(httpMethod, requestUri); + var result = new StreamContent(stream); + result.Headers.ContentDisposition = contentDisposition; + result.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - if (HttpVersion is not null) - { - request.Version = HttpVersion; - } + return result; + } - // pull in session data - if (WebSession.Headers.Count > 0) + /// + /// Gets a from the supplied field name and file. Calls to create the and then sets the file name. + /// + /// The Field Name to use for the + /// The file to use for the + private static StreamContent GetMultipartFileContent(object fieldName, FileInfo file) + { + var result = GetMultipartStreamContent(fieldName: fieldName, stream: new FileStream(file.FullName, FileMode.Open)); + + // .NET does not enclose field names in quotes, however, modern browsers and curl do. + result.Headers.ContentDisposition.FileName = "\"" + file.Name + "\""; + + return result; + } + + private static string FormatErrorMessage(string error, string contentType) + { + string formattedError = null; + + try { - WebSession.ContentHeaders.Clear(); - foreach (var entry in WebSession.Headers) + if (ContentHelper.IsXml(contentType)) { - if (HttpKnownHeaderNames.ContentHeaders.Contains(entry.Key)) - { - WebSession.ContentHeaders.Add(entry.Key, entry.Value); - } - else + XmlDocument doc = new(); + doc.LoadXml(error); + + XmlWriterSettings settings = new XmlWriterSettings { + Indent = true, + NewLineOnAttributes = true, + OmitXmlDeclaration = true + }; + + if (doc.FirstChild is XmlDeclaration) { - if (SkipHeaderValidation) - { - request.Headers.TryAddWithoutValidation(entry.Key, entry.Value); - } - else - { - request.Headers.Add(entry.Key, entry.Value); - } + XmlDeclaration decl = doc.FirstChild as XmlDeclaration; + settings.Encoding = Encoding.GetEncoding(decl.Encoding); } - } - } - // Set 'Transfer-Encoding: chunked' if 'Transfer-Encoding' is specified - if (WebSession.Headers.ContainsKey(HttpKnownHeaderNames.TransferEncoding)) - { - request.Headers.TransferEncodingChunked = true; - } + StringBuilder stringBuilder = new(); + using XmlWriter xmlWriter = XmlWriter.Create(stringBuilder, settings); + doc.Save(xmlWriter); + string xmlString = stringBuilder.ToString(); - // Set 'User-Agent' if WebSession.Headers doesn't already contain it - if (WebSession.Headers.TryGetValue(HttpKnownHeaderNames.UserAgent, out string userAgent)) - { - WebSession.UserAgent = userAgent; - } - else - { - if (SkipHeaderValidation) - { - request.Headers.TryAddWithoutValidation(HttpKnownHeaderNames.UserAgent, WebSession.UserAgent); + formattedError = Environment.NewLine + xmlString; } - else + else if (ContentHelper.IsJson(contentType)) { - request.Headers.Add(HttpKnownHeaderNames.UserAgent, WebSession.UserAgent); - } - } + JsonNode jsonNode = JsonNode.Parse(error); + JsonSerializerOptions options = new JsonSerializerOptions { WriteIndented = true }; + string jsonString = jsonNode.ToJsonString(options); - // Set 'Keep-Alive' to false. This means set the Connection to 'Close'. - if (DisableKeepAlive) - { - request.Headers.Add(HttpKnownHeaderNames.Connection, "Close"); + formattedError = Environment.NewLine + jsonString; + } } - - // Set 'Transfer-Encoding' - if (TransferEncoding is not null) + catch { - request.Headers.TransferEncodingChunked = true; - var headerValue = new TransferCodingHeaderValue(TransferEncoding); - if (!request.Headers.TransferEncoding.Contains(headerValue)) - { - request.Headers.TransferEncoding.Add(headerValue); - } + // Ignore errors } - - // If the file to resume downloading exists, create the Range request header using the file size. - // If not, create a Range to request the entire file. - if (Resume.IsPresent) + + if (string.IsNullOrEmpty(formattedError)) { - var fileInfo = new FileInfo(QualifiedOutFile); - if (fileInfo.Exists) - { - request.Headers.Range = new RangeHeaderValue(fileInfo.Length, null); - _resumeFileSize = fileInfo.Length; - } - else - { - request.Headers.Range = new RangeHeaderValue(0, null); - } + // Remove HTML tags making it easier to read + formattedError = System.Text.RegularExpressions.Regex.Replace(error, "<[^>]*>", string.Empty); } - return request; + return formattedError; } - internal virtual void FillRequestStream(HttpRequestMessage request) - { - ArgumentNullException.ThrowIfNull(request); + #endregion Helper Methods - // set the content type - if (ContentType is not null) - { - WebSession.ContentHeaders[HttpKnownHeaderNames.ContentType] = ContentType; - // request - } - // ContentType is null - else if (request.Method == HttpMethod.Post) - { - // Win8:545310 Invoke-WebRequest does not properly set MIME type for POST - WebSession.ContentHeaders.TryGetValue(HttpKnownHeaderNames.ContentType, out string contentType); - if (string.IsNullOrEmpty(contentType)) - { - WebSession.ContentHeaders[HttpKnownHeaderNames.ContentType] = "application/x-www-form-urlencoded"; - } - } + #region Abstract Methods - if (Form is not null) - { - var formData = new MultipartFormDataContent(); - foreach (DictionaryEntry formEntry in Form) - { - // AddMultipartContent will handle PSObject unwrapping, Object type determination and enumerateing top level IEnumerables. - AddMultipartContent(fieldName: formEntry.Key, fieldValue: formEntry.Value, formData: formData, enumerate: true); - } + /// + /// Read the supplied WebResponse object and push the resulting output into the pipeline. + /// + /// Instance of a WebResponse object to be processed. + internal abstract void ProcessResponse(HttpResponseMessage response); - SetRequestContent(request, formData); - } - else if (Body is not null) - { - // Coerce body into a usable form - object content = Body; + #endregion Abstract Methods + } - // Make sure we're using the base object of the body, not the PSObject wrapper - if (Body is PSObject psBody) - { - content = psBody.BaseObject; - } + // TODO: Merge Partials - switch (content) - { - case FormObject form: - SetRequestContent(request, form.Fields); - break; - case IDictionary dictionary when request.Method != HttpMethod.Get: - SetRequestContent(request, dictionary); - break; - case XmlNode xmlNode: - SetRequestContent(request, xmlNode); - break; - case Stream stream: - SetRequestContent(request, stream); - break; - case byte[] bytes: - SetRequestContent(request, bytes); - break; - case MultipartFormDataContent multipartFormDataContent: - SetRequestContent(request, multipartFormDataContent); - break; - default: - SetRequestContent(request, (string)LanguagePrimitives.ConvertTo(content, typeof(string), CultureInfo.InvariantCulture)); - break; - } - } - else if (InFile is not null) - { - // Copy InFile data - try - { - // Open the input file - SetRequestContent(request, new FileStream(InFile, FileMode.Open, FileAccess.Read, FileShare.Read)); - } - catch (UnauthorizedAccessException) - { - string msg = string.Format(CultureInfo.InvariantCulture, WebCmdletStrings.AccessDenied, _originalFilePath); + /// + /// Exception class for webcmdlets to enable returning HTTP error response. + /// + public sealed class HttpResponseException : HttpRequestException + { + /// + /// Initializes a new instance of the class. + /// + /// Message for the exception. + /// Response from the HTTP server. + public HttpResponseException(string message, HttpResponseMessage response) : base(message, inner: null, response.StatusCode) + { + Response = response; + } - throw new UnauthorizedAccessException(msg); - } + /// + /// HTTP error response. + /// + public HttpResponseMessage Response { get; } + } + + /// + /// Base class for Invoke-RestMethod and Invoke-WebRequest commands. + /// + public abstract partial class WebRequestPSCmdlet : PSCmdlet + { + + /// + /// Cancellation token source. + /// + internal CancellationTokenSource _cancelToken = null; + + /// + /// Parse Rel Links. + /// + internal bool _parseRelLink = false; + + /// + /// Automatically follow Rel Links. + /// + internal bool _followRelLink = false; + + /// + /// Automatically follow Rel Links. + /// + internal Dictionary _relationLink = null; + + /// + /// Maximum number of Rel Links to follow. + /// + internal int _maximumFollowRelLink = int.MaxValue; + + /// + /// The remote endpoint returned a 206 status code indicating successful resume. + /// + private bool _resumeSuccess = false; + + /// + /// The current size of the local file being resumed. + /// + private long _resumeFileSize = 0; + + private static HttpMethod GetHttpMethod(WebRequestMethod method) => method switch + { + WebRequestMethod.Default or WebRequestMethod.Get => HttpMethod.Get, + WebRequestMethod.Delete => HttpMethod.Delete, + WebRequestMethod.Head => HttpMethod.Head, + WebRequestMethod.Patch => HttpMethod.Patch, + WebRequestMethod.Post => HttpMethod.Post, + WebRequestMethod.Put => HttpMethod.Put, + WebRequestMethod.Options => HttpMethod.Options, + WebRequestMethod.Trace => HttpMethod.Trace, + _ => new HttpMethod(method.ToString().ToUpperInvariant()) + }; + + #region Virtual Methods + + // NOTE: Only pass true for handleRedirect if the original request has an authorization header + // and PreserveAuthorizationOnRedirect is NOT set. + internal virtual HttpClient GetHttpClient(bool handleRedirect) + { + HttpClientHandler handler = new(); + handler.CookieContainer = WebSession.Cookies; + handler.AutomaticDecompression = DecompressionMethods.All; + + // set the credentials used by this request + if (WebSession.UseDefaultCredentials) + { + // the UseDefaultCredentials flag overrides other supplied credentials + handler.UseDefaultCredentials = true; + } + else if (WebSession.Credentials is not null) + { + handler.Credentials = WebSession.Credentials; } - // For other methods like Put where empty content has meaning, we need to fill in the content - if (request.Content is null) + if (NoProxy) { - // If this is a Get request and there is no content, then don't fill in the content as empty content gets rejected by some web services per RFC7230 - if (request.Method == HttpMethod.Get && ContentType is null) - { - return; - } + handler.UseProxy = false; + } + else if (WebSession.Proxy is not null) + { + handler.Proxy = WebSession.Proxy; + } - request.Content = new StringContent(string.Empty); - request.Content.Headers.Clear(); + if (WebSession.Certificates is not null) + { + handler.ClientCertificates.AddRange(WebSession.Certificates); } - foreach (var entry in WebSession.ContentHeaders) + if (SkipCertificateCheck) { - if (!string.IsNullOrWhiteSpace(entry.Value)) - { - if (SkipHeaderValidation) - { - request.Content.Headers.TryAddWithoutValidation(entry.Key, entry.Value); - } - else - { - try - { - request.Content.Headers.Add(entry.Key, entry.Value); - } - catch (FormatException ex) - { - var outerEx = new ValidationMetadataException(WebCmdletStrings.ContentTypeException, ex); - ErrorRecord er = new(outerEx, "WebCmdletContentTypeException", ErrorCategory.InvalidArgument, ContentType); - ThrowTerminatingError(er); - } - } - } + handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; + handler.ClientCertificateOptions = ClientCertificateOption.Manual; } - } - // Returns true if the status code is one of the supported redirection codes. - private static bool IsRedirectCode(HttpStatusCode code) - { - int intCode = (int)code; - return - ( - (intCode >= 300 && intCode < 304) || - intCode == 307 || - intCode == 308 - ); - } + // This indicates GetResponse will handle redirects. + if (handleRedirect || WebSession.MaximumRedirection == 0) + { + handler.AllowAutoRedirect = false; + } + else if (WebSession.MaximumRedirection > 0) + { + handler.MaxAutomaticRedirections = WebSession.MaximumRedirection; + } - // Returns true if the status code is a redirection code and the action requires switching from POST to GET on redirection. - // NOTE: Some of these status codes map to the same underlying value but spelling them out for completeness. - private static bool IsRedirectToGet(HttpStatusCode code) - { - return - ( - code == HttpStatusCode.Found || - code == HttpStatusCode.Moved || - code == HttpStatusCode.Redirect || - code == HttpStatusCode.RedirectMethod || - code == HttpStatusCode.SeeOther || - code == HttpStatusCode.Ambiguous || - code == HttpStatusCode.MultipleChoices - ); - } + handler.SslProtocols = (SslProtocols)SslProtocol; - // Returns true if the status code shows a server or client error and MaximumRetryCount > 0 - private bool ShouldRetry(HttpStatusCode code) - { - int intCode = (int)code; + HttpClient httpClient = new(handler); - return - ( - (intCode == 304 || (intCode >= 400 && intCode <= 599)) && WebSession.MaximumRetryCount > 0 - ); + // Check timeout setting (in seconds instead of milliseconds as in HttpWebRequest) + httpClient.Timeout = TimeoutSec is 0 ? TimeSpan.FromMilliseconds(Timeout.Infinite) : new TimeSpan(0, 0, TimeoutSec); + + return httpClient; } - internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestMessage request, bool handleRedirect) + internal virtual HttpRequestMessage GetRequest(Uri uri) { - ArgumentNullException.ThrowIfNull(client); - ArgumentNullException.ThrowIfNull(request); + Uri requestUri = PrepareUri(uri); + HttpMethod httpMethod = string.IsNullOrEmpty(CustomMethod) ? GetHttpMethod(Method) : new HttpMethod(CustomMethod); - // Add 1 to account for the first request. - int totalRequests = WebSession.MaximumRetryCount + 1; - HttpRequestMessage req = request; - HttpResponseMessage response = null; + // create the base WebRequest object + var request = new HttpRequestMessage(httpMethod, requestUri); - do + if (HttpVersion is not null) { - // Track the current URI being used by various requests and re-requests. - Uri currentUri = req.RequestUri; - - _cancelToken = new CancellationTokenSource(); - response = client.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult(); + request.Version = HttpVersion; + } - if (handleRedirect - && WebSession.MaximumRedirection is not 0 - && IsRedirectCode(response.StatusCode) - && response.Headers.Location is not null) + // pull in session data + if (WebSession.Headers.Count > 0) + { + WebSession.ContentHeaders.Clear(); + foreach (var entry in WebSession.Headers) { - _cancelToken.Cancel(); - _cancelToken = null; - - // If explicit count was provided, reduce it for this redirection. - if (WebSession.MaximumRedirection > 0) + if (HttpKnownHeaderNames.ContentHeaders.Contains(entry.Key)) { - WebSession.MaximumRedirection--; + WebSession.ContentHeaders.Add(entry.Key, entry.Value); } - - // For selected redirects that used POST, GET must be used with the - // redirected Location. - // Since GET is the default; POST only occurs when -Method POST is used. - if (Method == WebRequestMethod.Post && IsRedirectToGet(response.StatusCode)) + else { - // See https://msdn.microsoft.com/library/system.net.httpstatuscode(v=vs.110).aspx - Method = WebRequestMethod.Get; + if (SkipHeaderValidation) + { + request.Headers.TryAddWithoutValidation(entry.Key, entry.Value); + } + else + { + request.Headers.Add(entry.Key, entry.Value); + } } - - currentUri = new Uri(request.RequestUri, response.Headers.Location); - - // Continue to handle redirection - using HttpRequestMessage redirectRequest = GetRequest(currentUri); - response.Dispose(); - response = GetResponse(client, redirectRequest, handleRedirect); } + } - // Request again without the Range header because the server indicated the range was not satisfiable. - // This happens when the local file is larger than the remote file. - // If the size of the remote file is the same as the local file, there is nothing to resume. - if (Resume.IsPresent - && response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable - && (response.Content.Headers.ContentRange.HasLength - && response.Content.Headers.ContentRange.Length != _resumeFileSize)) - { - _cancelToken.Cancel(); - - WriteVerbose(WebCmdletStrings.WebMethodResumeFailedVerboseMsg); - - // Disable the Resume switch so the subsequent calls to GetResponse() and FillRequestStream() - // are treated as a standard -OutFile request. This also disables appending local file. - Resume = new SwitchParameter(false); - - using (HttpRequestMessage requestWithoutRange = GetRequest(currentUri)) - { - FillRequestStream(requestWithoutRange); - - long requestContentLength = requestWithoutRange.Content is null ? 0 : requestWithoutRange.Content.Headers.ContentLength.Value; - - string reqVerboseMsg = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.WebMethodInvocationVerboseMsg, - requestWithoutRange.Version, - requestWithoutRange.Method, - requestContentLength); - - WriteVerbose(reqVerboseMsg); + // Set 'Transfer-Encoding: chunked' if 'Transfer-Encoding' is specified + if (WebSession.Headers.ContainsKey(HttpKnownHeaderNames.TransferEncoding)) + { + request.Headers.TransferEncodingChunked = true; + } - response.Dispose(); - response = GetResponse(client, requestWithoutRange, handleRedirect); - } + // Set 'User-Agent' if WebSession.Headers doesn't already contain it + if (WebSession.Headers.TryGetValue(HttpKnownHeaderNames.UserAgent, out string userAgent)) + { + WebSession.UserAgent = userAgent; + } + else + { + if (SkipHeaderValidation) + { + request.Headers.TryAddWithoutValidation(HttpKnownHeaderNames.UserAgent, WebSession.UserAgent); } - - _resumeSuccess = response.StatusCode == HttpStatusCode.PartialContent; - - // When MaximumRetryCount is not specified, the totalRequests is 1. - if (totalRequests > 1 && ShouldRetry(response.StatusCode)) + else { - int retryIntervalInSeconds = WebSession.RetryIntervalInSeconds; - - // If the status code is 429 get the retry interval from the Headers. - // Ignore broken header and its value. - if (response.StatusCode is HttpStatusCode.Conflict && response.Headers.TryGetValues(HttpKnownHeaderNames.RetryAfter, out IEnumerable retryAfter)) - { - try - { - IEnumerator enumerator = retryAfter.GetEnumerator(); - if (enumerator.MoveNext()) - { - retryIntervalInSeconds = Convert.ToInt32(enumerator.Current); - } - } - catch - { - // Ignore broken header. - } - } - - string retryMessage = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.RetryVerboseMsg, - retryIntervalInSeconds, - response.StatusCode); - - WriteVerbose(retryMessage); + request.Headers.Add(HttpKnownHeaderNames.UserAgent, WebSession.UserAgent); + } + } - _cancelToken = new CancellationTokenSource(); - Task.Delay(retryIntervalInSeconds * 1000, _cancelToken.Token).GetAwaiter().GetResult(); - _cancelToken.Cancel(); - _cancelToken = null; + // Set 'Keep-Alive' to false. This means set the Connection to 'Close'. + if (DisableKeepAlive) + { + request.Headers.Add(HttpKnownHeaderNames.Connection, "Close"); + } - req.Dispose(); - req = GetRequest(currentUri); - FillRequestStream(req); + // Set 'Transfer-Encoding' + if (TransferEncoding is not null) + { + request.Headers.TransferEncodingChunked = true; + var headerValue = new TransferCodingHeaderValue(TransferEncoding); + if (!request.Headers.TransferEncoding.Contains(headerValue)) + { + request.Headers.TransferEncoding.Add(headerValue); } + } - totalRequests--; + // If the file to resume downloading exists, create the Range request header using the file size. + // If not, create a Range to request the entire file. + if (Resume.IsPresent) + { + var fileInfo = new FileInfo(QualifiedOutFile); + if (fileInfo.Exists) + { + request.Headers.Range = new RangeHeaderValue(fileInfo.Length, null); + _resumeFileSize = fileInfo.Length; + } + else + { + request.Headers.Range = new RangeHeaderValue(0, null); + } } - while (totalRequests > 0 && !response.IsSuccessStatusCode); - return response; + return request; } - internal virtual void UpdateSession(HttpResponseMessage response) + internal virtual void FillRequestStream(HttpRequestMessage request) { - ArgumentNullException.ThrowIfNull(response); - } - - #endregion Virtual Methods - - #region Overrides + ArgumentNullException.ThrowIfNull(request); - /// - /// The main execution method for cmdlets derived from WebRequestPSCmdlet. - /// - protected override void ProcessRecord() - { - try + // set the content type + if (ContentType is not null) { - // Set cmdlet context for write progress - ValidateParameters(); - PrepareSession(); - - // If the request contains an authorization header and PreserveAuthorizationOnRedirect is not set, - // it needs to be stripped on the first redirect. - bool keepAuthorizationOnRedirect = PreserveAuthorizationOnRedirect.IsPresent - && WebSession.Headers.ContainsKey(HttpKnownHeaderNames.Authorization); - - bool handleRedirect = keepAuthorizationOnRedirect || AllowInsecureRedirect; - - using (HttpClient client = GetHttpClient(handleRedirect)) + WebSession.ContentHeaders[HttpKnownHeaderNames.ContentType] = ContentType; + // request + } + // ContentType is null + else if (request.Method == HttpMethod.Post) + { + // Win8:545310 Invoke-WebRequest does not properly set MIME type for POST + WebSession.ContentHeaders.TryGetValue(HttpKnownHeaderNames.ContentType, out string contentType); + if (string.IsNullOrEmpty(contentType)) { - int followedRelLink = 0; - Uri uri = Uri; - do - { - if (followedRelLink > 0) - { - string linkVerboseMsg = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.FollowingRelLinkVerboseMsg, - uri.AbsoluteUri); - - WriteVerbose(linkVerboseMsg); - } - - using (HttpRequestMessage request = GetRequest(uri)) - { - FillRequestStream(request); - try - { - long requestContentLength = request.Content is null ? 0 : request.Content.Headers.ContentLength.Value; - - string reqVerboseMsg = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.WebMethodInvocationVerboseMsg, - request.Version, - request.Method, - requestContentLength); - - WriteVerbose(reqVerboseMsg); - - using HttpResponseMessage response = GetResponse(client, request, handleRedirect); - - string contentType = ContentHelper.GetContentType(response); - string respVerboseMsg = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.WebResponseVerboseMsg, - response.Content.Headers.ContentLength, - contentType); - - WriteVerbose(respVerboseMsg); - - bool _isSuccess = response.IsSuccessStatusCode; - - // Check if the Resume range was not satisfiable because the file already completed downloading. - // This happens when the local file is the same size as the remote file. - if (Resume.IsPresent - && response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable - && response.Content.Headers.ContentRange.HasLength - && response.Content.Headers.ContentRange.Length == _resumeFileSize) - { - _isSuccess = true; - WriteVerbose(string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.OutFileWritingSkipped, - OutFile)); - - // Disable writing to the OutFile. - OutFile = null; - } - - if (ShouldCheckHttpStatus && !_isSuccess) - { - string message = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.ResponseStatusCodeFailure, - (int)response.StatusCode, - response.ReasonPhrase); - - HttpResponseException httpEx = new(message, response); - ErrorRecord er = new(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); - string detailMsg = string.Empty; - StreamReader reader = null; - try - { - reader = new StreamReader(StreamHelper.GetResponseStream(response)); - detailMsg = FormatErrorMessage(reader.ReadToEnd(), contentType); - } - catch - { - // Catch all - } - finally - { - reader?.Dispose(); - } - - if (!string.IsNullOrEmpty(detailMsg)) - { - er.ErrorDetails = new ErrorDetails(detailMsg); - } - - ThrowTerminatingError(er); - } - - if (_parseRelLink || _followRelLink) - { - ParseLinkHeader(response, uri); - } - - ProcessResponse(response); - UpdateSession(response); - - // If we hit our maximum redirection count, generate an error. - // Errors with redirection counts of greater than 0 are handled automatically by .NET, but are - // impossible to detect programmatically when we hit this limit. By handling this ourselves - // (and still writing out the result), users can debug actual HTTP redirect problems. - if (WebSession.MaximumRedirection == 0 && IsRedirectCode(response.StatusCode)) // Indicate "HttpClientHandler.AllowAutoRedirect is false" - { - ErrorRecord er = new(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request); - er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded); - WriteError(er); - } - } - catch (HttpRequestException ex) - { - ErrorRecord er = new(ex, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); - if (ex.InnerException is not null) - { - er.ErrorDetails = new ErrorDetails(ex.InnerException.Message); - } - - ThrowTerminatingError(er); - } - - if (_followRelLink) - { - if (!_relationLink.ContainsKey("next")) - { - return; - } - - uri = new Uri(_relationLink["next"]); - followedRelLink++; - } - } - } - while (_followRelLink && (followedRelLink < _maximumFollowRelLink)); + WebSession.ContentHeaders[HttpKnownHeaderNames.ContentType] = "application/x-www-form-urlencoded"; } } - catch (CryptographicException ex) + + if (Form is not null) { - ErrorRecord er = new(ex, "WebCmdletCertificateException", ErrorCategory.SecurityError, null); - ThrowTerminatingError(er); + var formData = new MultipartFormDataContent(); + foreach (DictionaryEntry formEntry in Form) + { + // AddMultipartContent will handle PSObject unwrapping, Object type determination and enumerateing top level IEnumerables. + AddMultipartContent(fieldName: formEntry.Key, fieldValue: formEntry.Value, formData: formData, enumerate: true); + } + + SetRequestContent(request, formData); } - catch (NotSupportedException ex) + else if (Body is not null) { - ErrorRecord er = new(ex, "WebCmdletIEDomNotSupportedException", ErrorCategory.NotImplemented, null); - ThrowTerminatingError(er); - } - } + // Coerce body into a usable form + object content = Body; - /// - /// To implement ^C. - /// - protected override void StopProcessing() => _cancelToken?.Cancel(); + // Make sure we're using the base object of the body, not the PSObject wrapper + if (Body is PSObject psBody) + { + content = psBody.BaseObject; + } - #endregion Overrides + switch (content) + { + case FormObject form: + SetRequestContent(request, form.Fields); + break; + case IDictionary dictionary when request.Method != HttpMethod.Get: + SetRequestContent(request, dictionary); + break; + case XmlNode xmlNode: + SetRequestContent(request, xmlNode); + break; + case Stream stream: + SetRequestContent(request, stream); + break; + case byte[] bytes: + SetRequestContent(request, bytes); + break; + case MultipartFormDataContent multipartFormDataContent: + SetRequestContent(request, multipartFormDataContent); + break; + default: + SetRequestContent(request, (string)LanguagePrimitives.ConvertTo(content, typeof(string), CultureInfo.InvariantCulture)); + break; + } + } + else if (InFile is not null) + { + // Copy InFile data + try + { + // Open the input file + SetRequestContent(request, new FileStream(InFile, FileMode.Open, FileAccess.Read, FileShare.Read)); + } + catch (UnauthorizedAccessException) + { + string msg = string.Format(CultureInfo.InvariantCulture, WebCmdletStrings.AccessDenied, _originalFilePath); - #region Helper Methods + throw new UnauthorizedAccessException(msg); + } + } - /// - /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. - /// - /// The WebRequest who's content is to be set. - /// A byte array containing the content data. - /// - /// Because this function sets the request's ContentLength property and writes content data into the request's stream, - /// it should be called one time maximum on a given request. - /// - internal void SetRequestContent(HttpRequestMessage request, byte[] content) - { - ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(content); + // For other methods like Put where empty content has meaning, we need to fill in the content + if (request.Content is null) + { + // If this is a Get request and there is no content, then don't fill in the content as empty content gets rejected by some web services per RFC7230 + if (request.Method == HttpMethod.Get && ContentType is null) + { + return; + } - ByteArrayContent byteArrayContent = new(content); - request.Content = byteArrayContent; - } + request.Content = new StringContent(string.Empty); + request.Content.Headers.Clear(); + } - /// - /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. - /// - /// The WebRequest who's content is to be set. - /// A String object containing the content data. - /// - /// Because this function sets the request's ContentLength property and writes content data into the request's stream, - /// it should be called one time maximum on a given request. - /// - internal void SetRequestContent(HttpRequestMessage request, string content) - { - ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(content); - - Encoding encoding = null; - if (ContentType is not null) + foreach (var entry in WebSession.ContentHeaders) { - // If Content-Type contains the encoding format (as CharSet), use this encoding format - // to encode the Body of the WebRequest sent to the server. Default Encoding format - // would be used if Charset is not supplied in the Content-Type property. - try + if (!string.IsNullOrWhiteSpace(entry.Value)) { - var mediaTypeHeaderValue = MediaTypeHeaderValue.Parse(ContentType); - if (!string.IsNullOrEmpty(mediaTypeHeaderValue.CharSet)) + if (SkipHeaderValidation) { - encoding = Encoding.GetEncoding(mediaTypeHeaderValue.CharSet); + request.Content.Headers.TryAddWithoutValidation(entry.Key, entry.Value); } - } - catch (Exception ex) when (ex is FormatException || ex is ArgumentException) - { - if (!SkipHeaderValidation) + else { - ValidationMetadataException outerEx = new(WebCmdletStrings.ContentTypeException, ex); - ErrorRecord er = new(outerEx, "WebCmdletContentTypeException", ErrorCategory.InvalidArgument, ContentType); - ThrowTerminatingError(er); + try + { + request.Content.Headers.Add(entry.Key, entry.Value); + } + catch (FormatException ex) + { + var outerEx = new ValidationMetadataException(WebCmdletStrings.ContentTypeException, ex); + ErrorRecord er = new(outerEx, "WebCmdletContentTypeException", ErrorCategory.InvalidArgument, ContentType); + ThrowTerminatingError(er); + } } } } - - byte[] bytes = StreamHelper.EncodeToBytes(content, encoding); - ByteArrayContent byteArrayContent = new(bytes); - request.Content = byteArrayContent; } - internal void SetRequestContent(HttpRequestMessage request, XmlNode xmlNode) + // Returns true if the status code is one of the supported redirection codes. + private static bool IsRedirectCode(HttpStatusCode code) { - ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(xmlNode); - - byte[] bytes = null; - XmlDocument doc = xmlNode as XmlDocument; - if (doc?.FirstChild is XmlDeclaration) - { - XmlDeclaration decl = doc.FirstChild as XmlDeclaration; - Encoding encoding = Encoding.GetEncoding(decl.Encoding); - bytes = StreamHelper.EncodeToBytes(doc.OuterXml, encoding); - } - else - { - bytes = StreamHelper.EncodeToBytes(xmlNode.OuterXml, encoding: null); - } - - ByteArrayContent byteArrayContent = new(bytes); - - request.Content = byteArrayContent; + int intCode = (int)code; + return + ( + (intCode >= 300 && intCode < 304) || + intCode == 307 || + intCode == 308 + ); } - /// - /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. - /// - /// The WebRequest who's content is to be set. - /// A Stream object containing the content data. - /// - /// Because this function sets the request's ContentLength property and writes content data into the request's stream, - /// it should be called one time maximum on a given request. - /// - internal void SetRequestContent(HttpRequestMessage request, Stream contentStream) + // Returns true if the status code is a redirection code and the action requires switching from POST to GET on redirection. + // NOTE: Some of these status codes map to the same underlying value but spelling them out for completeness. + private static bool IsRedirectToGet(HttpStatusCode code) { - ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(contentStream); - - StreamContent streamContent = new(contentStream); - request.Content = streamContent; + return + ( + code == HttpStatusCode.Found || + code == HttpStatusCode.Moved || + code == HttpStatusCode.Redirect || + code == HttpStatusCode.RedirectMethod || + code == HttpStatusCode.SeeOther || + code == HttpStatusCode.Ambiguous || + code == HttpStatusCode.MultipleChoices + ); } - /// - /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. - /// - /// The WebRequest who's content is to be set. - /// A MultipartFormDataContent object containing multipart/form-data content. - /// - /// Because this function sets the request's ContentLength property and writes content data into the request's stream, - /// it should be called one time maximum on a given request. - /// - internal void SetRequestContent(HttpRequestMessage request, MultipartFormDataContent multipartContent) + // Returns true if the status code shows a server or client error and MaximumRetryCount > 0 + private bool ShouldRetry(HttpStatusCode code) { - ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(multipartContent); - - // Content headers will be set by MultipartFormDataContent which will throw unless we clear them first - WebSession.ContentHeaders.Clear(); + int intCode = (int)code; - request.Content = multipartContent; + return + ( + (intCode == 304 || (intCode >= 400 && intCode <= 599)) && WebSession.MaximumRetryCount > 0 + ); } - internal void SetRequestContent(HttpRequestMessage request, IDictionary content) + internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestMessage request, bool handleRedirect) { + ArgumentNullException.ThrowIfNull(client); ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(content); - string body = FormatDictionary(content); - SetRequestContent(request, body); - } + // Add 1 to account for the first request. + int totalRequests = WebSession.MaximumRetryCount + 1; + HttpRequestMessage req = request; + HttpResponseMessage response = null; - internal void ParseLinkHeader(HttpResponseMessage response, System.Uri requestUri) - { - if (_relationLink is null) - { - // Must ignore the case of relation links. See RFC 8288 (https://tools.ietf.org/html/rfc8288) - _relationLink = new Dictionary(StringComparer.OrdinalIgnoreCase); - } - else + do { - _relationLink.Clear(); - } + // Track the current URI being used by various requests and re-requests. + Uri currentUri = req.RequestUri; - // We only support the URL in angle brackets and `rel`, other attributes are ignored - // user can still parse it themselves via the Headers property - const string pattern = "<(?.*?)>;\\s*rel=(?\")?(?(?(quoted).*?|[^,;]*))(?(quoted)\")"; - if (response.Headers.TryGetValues("Link", out IEnumerable links)) - { - foreach (string linkHeader in links) + _cancelToken = new CancellationTokenSource(); + response = client.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult(); + + if (handleRedirect + && WebSession.MaximumRedirection is not 0 + && IsRedirectCode(response.StatusCode) + && response.Headers.Location is not null) { - MatchCollection matchCollection = Regex.Matches(linkHeader, pattern); - foreach (Match match in matchCollection) + _cancelToken.Cancel(); + _cancelToken = null; + + // If explicit count was provided, reduce it for this redirection. + if (WebSession.MaximumRedirection > 0) { - if (match.Success) - { - string url = match.Groups["url"].Value; - string rel = match.Groups["rel"].Value; - if (url != string.Empty && rel != string.Empty && !_relationLink.ContainsKey(rel)) - { - Uri absoluteUri = new(requestUri, url); - _relationLink.Add(rel, absoluteUri.AbsoluteUri); - } - } + WebSession.MaximumRedirection--; + } + + // For selected redirects that used POST, GET must be used with the + // redirected Location. + // Since GET is the default; POST only occurs when -Method POST is used. + if (Method == WebRequestMethod.Post && IsRedirectToGet(response.StatusCode)) + { + // See https://msdn.microsoft.com/library/system.net.httpstatuscode(v=vs.110).aspx + Method = WebRequestMethod.Get; } + + currentUri = new Uri(request.RequestUri, response.Headers.Location); + + // Continue to handle redirection + using HttpRequestMessage redirectRequest = GetRequest(currentUri); + response.Dispose(); + response = GetResponse(client, redirectRequest, handleRedirect); } - } - } - /// - /// Adds content to a . Object type detection is used to determine if the value is string, File, or Collection. - /// - /// The Field Name to use. - /// The Field Value to use. - /// The > to update. - /// If true, collection types in will be enumerated. If false, collections will be treated as single value. - private void AddMultipartContent(object fieldName, object fieldValue, MultipartFormDataContent formData, bool enumerate) - { - ArgumentNullException.ThrowIfNull(formData); + // Request again without the Range header because the server indicated the range was not satisfiable. + // This happens when the local file is larger than the remote file. + // If the size of the remote file is the same as the local file, there is nothing to resume. + if (Resume.IsPresent + && response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable + && (response.Content.Headers.ContentRange.HasLength + && response.Content.Headers.ContentRange.Length != _resumeFileSize)) + { + _cancelToken.Cancel(); - // It is possible that the dictionary keys or values are PSObject wrapped depending on how the dictionary is defined and assigned. - // Before processing the field name and value we need to ensure we are working with the base objects and not the PSObject wrappers. + WriteVerbose(WebCmdletStrings.WebMethodResumeFailedVerboseMsg); - // Unwrap fieldName PSObjects - if (fieldName is PSObject namePSObject) - { - fieldName = namePSObject.BaseObject; - } + // Disable the Resume switch so the subsequent calls to GetResponse() and FillRequestStream() + // are treated as a standard -OutFile request. This also disables appending local file. + Resume = new SwitchParameter(false); - // Unwrap fieldValue PSObjects - if (fieldValue is PSObject valuePSObject) - { - fieldValue = valuePSObject.BaseObject; - } + using (HttpRequestMessage requestWithoutRange = GetRequest(currentUri)) + { + FillRequestStream(requestWithoutRange); - // Treat a single FileInfo as a FileContent - if (fieldValue is FileInfo file) - { - formData.Add(GetMultipartFileContent(fieldName: fieldName, file: file)); - return; - } + long requestContentLength = requestWithoutRange.Content is null ? 0 : requestWithoutRange.Content.Headers.ContentLength.Value; - // Treat Strings and other single values as a StringContent. - // If enumeration is false, also treat IEnumerables as StringContents. - // String implements IEnumerable so the explicit check is required. - if (!enumerate || fieldValue is string || fieldValue is not IEnumerable) - { - formData.Add(GetMultipartStringContent(fieldName: fieldName, fieldValue: fieldValue)); - return; - } + string reqVerboseMsg = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.WebMethodInvocationVerboseMsg, + requestWithoutRange.Version, + requestWithoutRange.Method, + requestContentLength); + + WriteVerbose(reqVerboseMsg); + + response.Dispose(); + response = GetResponse(client, requestWithoutRange, handleRedirect); + } + } + + _resumeSuccess = response.StatusCode == HttpStatusCode.PartialContent; + + // When MaximumRetryCount is not specified, the totalRequests is 1. + if (totalRequests > 1 && ShouldRetry(response.StatusCode)) + { + int retryIntervalInSeconds = WebSession.RetryIntervalInSeconds; + + // If the status code is 429 get the retry interval from the Headers. + // Ignore broken header and its value. + if (response.StatusCode is HttpStatusCode.Conflict && response.Headers.TryGetValues(HttpKnownHeaderNames.RetryAfter, out IEnumerable retryAfter)) + { + try + { + IEnumerator enumerator = retryAfter.GetEnumerator(); + if (enumerator.MoveNext()) + { + retryIntervalInSeconds = Convert.ToInt32(enumerator.Current); + } + } + catch + { + // Ignore broken header. + } + } + + string retryMessage = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.RetryVerboseMsg, + retryIntervalInSeconds, + response.StatusCode); + + WriteVerbose(retryMessage); + + _cancelToken = new CancellationTokenSource(); + Task.Delay(retryIntervalInSeconds * 1000, _cancelToken.Token).GetAwaiter().GetResult(); + _cancelToken.Cancel(); + _cancelToken = null; - // Treat the value as a collection and enumerate it if enumeration is true - if (enumerate && fieldValue is IEnumerable items) - { - foreach (var item in items) - { - // Recurse, but do not enumerate the next level. IEnumerables will be treated as single values. - AddMultipartContent(fieldName: fieldName, fieldValue: item, formData: formData, enumerate: false); + req.Dispose(); + req = GetRequest(currentUri); + FillRequestStream(req); } + + totalRequests--; } + while (totalRequests > 0 && !response.IsSuccessStatusCode); + + return response; } - /// - /// Gets a from the supplied field name and field value. Uses to convert the objects to strings. - /// - /// The Field Name to use for the - /// The Field Value to use for the - private static StringContent GetMultipartStringContent(object fieldName, object fieldValue) + internal virtual void UpdateSession(HttpResponseMessage response) { - var contentDisposition = new ContentDispositionHeaderValue("form-data"); - // .NET does not enclose field names in quotes, however, modern browsers and curl do. - contentDisposition.Name = "\"" + LanguagePrimitives.ConvertTo(fieldName) + "\""; + ArgumentNullException.ThrowIfNull(response); + } - var result = new StringContent(LanguagePrimitives.ConvertTo(fieldValue)); - result.Headers.ContentDisposition = contentDisposition; + #endregion Virtual Methods - return result; - } + #region Overrides /// - /// Gets a from the supplied field name and . Uses to convert the fieldname to a string. + /// The main execution method for cmdlets derived from WebRequestPSCmdlet. /// - /// The Field Name to use for the - /// The to use for the - private static StreamContent GetMultipartStreamContent(object fieldName, Stream stream) + protected override void ProcessRecord() { - var contentDisposition = new ContentDispositionHeaderValue("form-data"); - // .NET does not enclose field names in quotes, however, modern browsers and curl do. - contentDisposition.Name = "\"" + LanguagePrimitives.ConvertTo(fieldName) + "\""; + try + { + // Set cmdlet context for write progress + ValidateParameters(); + PrepareSession(); - var result = new StreamContent(stream); - result.Headers.ContentDisposition = contentDisposition; - result.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + // If the request contains an authorization header and PreserveAuthorizationOnRedirect is not set, + // it needs to be stripped on the first redirect. + bool keepAuthorizationOnRedirect = PreserveAuthorizationOnRedirect.IsPresent + && WebSession.Headers.ContainsKey(HttpKnownHeaderNames.Authorization); - return result; - } + bool handleRedirect = keepAuthorizationOnRedirect || AllowInsecureRedirect; - /// - /// Gets a from the supplied field name and file. Calls to create the and then sets the file name. - /// - /// The Field Name to use for the - /// The file to use for the - private static StreamContent GetMultipartFileContent(object fieldName, FileInfo file) - { - var result = GetMultipartStreamContent(fieldName: fieldName, stream: new FileStream(file.FullName, FileMode.Open)); - // .NET does not enclose field names in quotes, however, modern browsers and curl do. - result.Headers.ContentDisposition.FileName = "\"" + file.Name + "\""; + using (HttpClient client = GetHttpClient(handleRedirect)) + { + int followedRelLink = 0; + Uri uri = Uri; + do + { + if (followedRelLink > 0) + { + string linkVerboseMsg = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.FollowingRelLinkVerboseMsg, + uri.AbsoluteUri); - return result; - } + WriteVerbose(linkVerboseMsg); + } - private static string FormatErrorMessage(string error, string contentType) - { - string formattedError = null; + using (HttpRequestMessage request = GetRequest(uri)) + { + FillRequestStream(request); + try + { + long requestContentLength = request.Content is null ? 0 : request.Content.Headers.ContentLength.Value; - try - { - if (ContentHelper.IsXml(contentType)) - { - XmlDocument doc = new(); - doc.LoadXml(error); + string reqVerboseMsg = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.WebMethodInvocationVerboseMsg, + request.Version, + request.Method, + requestContentLength); - XmlWriterSettings settings = new XmlWriterSettings { - Indent = true, - NewLineOnAttributes = true, - OmitXmlDeclaration = true - }; + WriteVerbose(reqVerboseMsg); - if (doc.FirstChild is XmlDeclaration) - { - XmlDeclaration decl = doc.FirstChild as XmlDeclaration; - settings.Encoding = Encoding.GetEncoding(decl.Encoding); - } + using HttpResponseMessage response = GetResponse(client, request, handleRedirect); - StringBuilder stringBuilder = new(); - using XmlWriter xmlWriter = XmlWriter.Create(stringBuilder, settings); - doc.Save(xmlWriter); - string xmlString = stringBuilder.ToString(); + string contentType = ContentHelper.GetContentType(response); + string respVerboseMsg = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.WebResponseVerboseMsg, + response.Content.Headers.ContentLength, + contentType); - formattedError = Environment.NewLine + xmlString; - } - else if (ContentHelper.IsJson(contentType)) - { - JsonNode jsonNode = JsonNode.Parse(error); - JsonSerializerOptions options = new JsonSerializerOptions { WriteIndented = true }; - string jsonString = jsonNode.ToJsonString(options); + WriteVerbose(respVerboseMsg); - formattedError = Environment.NewLine + jsonString; + bool _isSuccess = response.IsSuccessStatusCode; + + // Check if the Resume range was not satisfiable because the file already completed downloading. + // This happens when the local file is the same size as the remote file. + if (Resume.IsPresent + && response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable + && response.Content.Headers.ContentRange.HasLength + && response.Content.Headers.ContentRange.Length == _resumeFileSize) + { + _isSuccess = true; + WriteVerbose(string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.OutFileWritingSkipped, + OutFile)); + + // Disable writing to the OutFile. + OutFile = null; + } + + if (ShouldCheckHttpStatus && !_isSuccess) + { + string message = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.ResponseStatusCodeFailure, + (int)response.StatusCode, + response.ReasonPhrase); + + HttpResponseException httpEx = new(message, response); + ErrorRecord er = new(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); + string detailMsg = string.Empty; + StreamReader reader = null; + try + { + reader = new StreamReader(StreamHelper.GetResponseStream(response)); + detailMsg = FormatErrorMessage(reader.ReadToEnd(), contentType); + } + catch + { + // Catch all + } + finally + { + reader?.Dispose(); + } + + if (!string.IsNullOrEmpty(detailMsg)) + { + er.ErrorDetails = new ErrorDetails(detailMsg); + } + + ThrowTerminatingError(er); + } + + if (_parseRelLink || _followRelLink) + { + ParseLinkHeader(response, uri); + } + + ProcessResponse(response); + UpdateSession(response); + + // If we hit our maximum redirection count, generate an error. + // Errors with redirection counts of greater than 0 are handled automatically by .NET, but are + // impossible to detect programmatically when we hit this limit. By handling this ourselves + // (and still writing out the result), users can debug actual HTTP redirect problems. + if (WebSession.MaximumRedirection == 0 && IsRedirectCode(response.StatusCode)) // Indicate "HttpClientHandler.AllowAutoRedirect is false" + { + ErrorRecord er = new(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request); + er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded); + WriteError(er); + } + } + catch (HttpRequestException ex) + { + ErrorRecord er = new(ex, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); + if (ex.InnerException is not null) + { + er.ErrorDetails = new ErrorDetails(ex.InnerException.Message); + } + + ThrowTerminatingError(er); + } + + if (_followRelLink) + { + if (!_relationLink.ContainsKey("next")) + { + return; + } + + uri = new Uri(_relationLink["next"]); + followedRelLink++; + } + } + } + while (_followRelLink && (followedRelLink < _maximumFollowRelLink)); } } - catch + catch (CryptographicException ex) { - // Ignore errors + ErrorRecord er = new(ex, "WebCmdletCertificateException", ErrorCategory.SecurityError, null); + ThrowTerminatingError(er); } - - if (string.IsNullOrEmpty(formattedError)) + catch (NotSupportedException ex) { - // Remove HTML tags making it easier to read - formattedError = System.Text.RegularExpressions.Regex.Replace(error, "<[^>]*>", string.Empty); + ErrorRecord er = new(ex, "WebCmdletIEDomNotSupportedException", ErrorCategory.NotImplemented, null); + ThrowTerminatingError(er); } - - return formattedError; } - #endregion Helper Methods + /// + /// To implement ^C. + /// + protected override void StopProcessing() => _cancelToken?.Cancel(); + + #endregion Overrides } } From 85e9cccc6931d055a5e8bff072c60f81807250fc Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:03:39 +0100 Subject: [PATCH 03/16] merge virtual methods --- .../Common/WebRequestPSCmdlet.Common.cs | 1581 ++++++++--------- 1 file changed, 789 insertions(+), 792 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 269d549b91c..6bb541d8ad0 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -685,1014 +685,1011 @@ internal virtual void PrepareSession() } } - #endregion Virtual Methods + + internal virtual HttpClient GetHttpClient(bool handleRedirect) + { + HttpClientHandler handler = new(); + handler.CookieContainer = WebSession.Cookies; + handler.AutomaticDecompression = DecompressionMethods.All; - #region Helper Properties + // Set the credentials used by this request + if (WebSession.UseDefaultCredentials) + { + // The UseDefaultCredentials flag overrides other supplied credentials + handler.UseDefaultCredentials = true; + } + else if (WebSession.Credentials is not null) + { + handler.Credentials = WebSession.Credentials; + } - internal string QualifiedOutFile => QualifyFilePath(OutFile); + if (NoProxy) + { + handler.UseProxy = false; + } + else if (WebSession.Proxy is not null) + { + handler.Proxy = WebSession.Proxy; + } - internal bool ShouldSaveToOutFile => !string.IsNullOrEmpty(OutFile); + if (WebSession.Certificates is not null) + { + handler.ClientCertificates.AddRange(WebSession.Certificates); + } - internal bool ShouldWriteToPipeline => !ShouldSaveToOutFile || PassThru; + if (SkipCertificateCheck) + { + handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; + handler.ClientCertificateOptions = ClientCertificateOption.Manual; + } - internal bool ShouldCheckHttpStatus => !SkipHttpErrorCheck; + // This indicates GetResponse will handle redirects. + if (handleRedirect || WebSession.MaximumRedirection == 0) + { + handler.AllowAutoRedirect = false; + } + else if (WebSession.MaximumRedirection > 0) + { + handler.MaxAutomaticRedirections = WebSession.MaximumRedirection; + } - /// - /// Determines whether writing to a file should Resume and append rather than overwrite. - /// - internal bool ShouldResume => Resume.IsPresent && _resumeSuccess; + handler.SslProtocols = (SslProtocols)SslProtocol; - #endregion Helper Properties + HttpClient httpClient = new(handler); - #region Helper Methods - private Uri PrepareUri(Uri uri) + // Check timeout setting (in seconds instead of milliseconds as in HttpWebRequest) + httpClient.Timeout = TimeoutSec is 0 ? TimeSpan.FromMilliseconds(Timeout.Infinite) : new TimeSpan(0, 0, TimeoutSec); + + return httpClient; + } + + internal virtual HttpRequestMessage GetRequest(Uri uri) { - uri = CheckProtocol(uri); + Uri requestUri = PrepareUri(uri); + HttpMethod httpMethod = string.IsNullOrEmpty(CustomMethod) ? GetHttpMethod(Method) : new HttpMethod(CustomMethod); - // Before creating the web request, - // preprocess Body if content is a dictionary and method is GET (set as query) - LanguagePrimitives.TryConvertTo(Body, out IDictionary bodyAsDictionary); - if (bodyAsDictionary is not null && (Method == WebRequestMethod.Default || Method == WebRequestMethod.Get || CustomMethod == "GET")) + // Create the base WebRequest object + var request = new HttpRequestMessage(httpMethod, requestUri); + + if (HttpVersion is not null) { - UriBuilder uriBuilder = new(uri); - if (uriBuilder.Query is not null && uriBuilder.Query.Length > 1) + request.Version = HttpVersion; + } + + // Pull in session data + if (WebSession.Headers.Count > 0) + { + WebSession.ContentHeaders.Clear(); + foreach (var entry in WebSession.Headers) { - uriBuilder.Query = string.Concat(uriBuilder.Query.AsSpan(1), "&", FormatDictionary(bodyAsDictionary)); + if (HttpKnownHeaderNames.ContentHeaders.Contains(entry.Key)) + { + WebSession.ContentHeaders.Add(entry.Key, entry.Value); + } + else + { + if (SkipHeaderValidation) + { + request.Headers.TryAddWithoutValidation(entry.Key, entry.Value); + } + else + { + request.Headers.Add(entry.Key, entry.Value); + } + } + } + } + + // Set 'Transfer-Encoding: chunked' if 'Transfer-Encoding' is specified + if (WebSession.Headers.ContainsKey(HttpKnownHeaderNames.TransferEncoding)) + { + request.Headers.TransferEncodingChunked = true; + } + + // Set 'User-Agent' if WebSession.Headers doesn't already contain it + if (WebSession.Headers.TryGetValue(HttpKnownHeaderNames.UserAgent, out string userAgent)) + { + WebSession.UserAgent = userAgent; + } + else + { + if (SkipHeaderValidation) + { + request.Headers.TryAddWithoutValidation(HttpKnownHeaderNames.UserAgent, WebSession.UserAgent); } else { - uriBuilder.Query = FormatDictionary(bodyAsDictionary); + request.Headers.Add(HttpKnownHeaderNames.UserAgent, WebSession.UserAgent); } - - uri = uriBuilder.Uri; - - // Set body to null to prevent later FillRequestStream - Body = null; } - return uri; - } - - private static Uri CheckProtocol(Uri uri) - { - ArgumentNullException.ThrowIfNull(uri); - - if (!uri.IsAbsoluteUri) + // Set 'Keep-Alive' to false. This means set the Connection to 'Close'. + if (DisableKeepAlive) { - uri = new Uri("http://" + uri.OriginalString); + request.Headers.Add(HttpKnownHeaderNames.Connection, "Close"); } - return uri; - } - - private string QualifyFilePath(string path) - { - string resolvedFilePath = PathUtils.ResolveFilePath(filePath: path, command: this, isLiteralPath: true); - return resolvedFilePath; - } - - private static string FormatDictionary(IDictionary content) - { - ArgumentNullException.ThrowIfNull(content); - - StringBuilder bodyBuilder = new(); - foreach (string key in content.Keys) + // Set 'Transfer-Encoding' + if (TransferEncoding is not null) { - if (bodyBuilder.Length > 0) + request.Headers.TransferEncodingChunked = true; + var headerValue = new TransferCodingHeaderValue(TransferEncoding); + if (!request.Headers.TransferEncoding.Contains(headerValue)) { - bodyBuilder.Append('&'); + request.Headers.TransferEncoding.Add(headerValue); } + } - object value = content[key]; - - // URLEncode the key and value - string encodedKey = WebUtility.UrlEncode(key); - string encodedValue = string.Empty; - if (value is not null) + // If the file to resume downloading exists, create the Range request header using the file size. + // If not, create a Range to request the entire file. + if (Resume.IsPresent) + { + var fileInfo = new FileInfo(QualifiedOutFile); + if (fileInfo.Exists) { - encodedValue = WebUtility.UrlEncode(value.ToString()); + request.Headers.Range = new RangeHeaderValue(fileInfo.Length, null); + _resumeFileSize = fileInfo.Length; + } + else + { + request.Headers.Range = new RangeHeaderValue(0, null); } - - bodyBuilder.Append($"{encodedKey}={encodedValue}"); } - return bodyBuilder.ToString(); - } - - private ErrorRecord GetValidationError(string msg, string errorId) - { - var ex = new ValidationMetadataException(msg); - var error = new ErrorRecord(ex, errorId, ErrorCategory.InvalidArgument, this); - return error; - } - - private ErrorRecord GetValidationError(string msg, string errorId, params object[] args) - { - msg = string.Format(CultureInfo.InvariantCulture, msg, args); - var ex = new ValidationMetadataException(msg); - var error = new ErrorRecord(ex, errorId, ErrorCategory.InvalidArgument, this); - return error; - } - - private string GetBasicAuthorizationHeader() - { - var password = new NetworkCredential(null, Credential.Password).Password; - string unencoded = string.Create(CultureInfo.InvariantCulture, $"{Credential.UserName}:{password}"); - byte[] bytes = Encoding.UTF8.GetBytes(unencoded); - return string.Create(CultureInfo.InvariantCulture, $"Basic {Convert.ToBase64String(bytes)}"); + return request; } - private string GetBearerAuthorizationHeader() + internal virtual void FillRequestStream(HttpRequestMessage request) { - return string.Create(CultureInfo.InvariantCulture, $"Bearer {new NetworkCredential(string.Empty, Token).Password}"); - } + ArgumentNullException.ThrowIfNull(request); - private void ProcessAuthentication() - { - if (Authentication == WebAuthenticationType.Basic) + // Set the request content type + if (ContentType is not null) { - WebSession.Headers["Authorization"] = GetBasicAuthorizationHeader(); + WebSession.ContentHeaders[HttpKnownHeaderNames.ContentType] = ContentType; } - else if (Authentication == WebAuthenticationType.Bearer || Authentication == WebAuthenticationType.OAuth) + else if (request.Method == HttpMethod.Post) { - WebSession.Headers["Authorization"] = GetBearerAuthorizationHeader(); + // Win8:545310 Invoke-WebRequest does not properly set MIME type for POST + WebSession.ContentHeaders.TryGetValue(HttpKnownHeaderNames.ContentType, out string contentType); + if (string.IsNullOrEmpty(contentType)) + { + WebSession.ContentHeaders[HttpKnownHeaderNames.ContentType] = "application/x-www-form-urlencoded"; + } } - else + + if (Form is not null) { - Diagnostics.Assert(false, string.Create(CultureInfo.InvariantCulture, $"Unrecognized Authentication value: {Authentication}")); - } - } - - /// - /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. - /// - /// The WebRequest who's content is to be set. - /// A byte array containing the content data. - /// - /// Because this function sets the request's ContentLength property and writes content data into the request's stream, - /// it should be called one time maximum on a given request. - /// - internal void SetRequestContent(HttpRequestMessage request, byte[] content) - { - ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(content); + var formData = new MultipartFormDataContent(); + foreach (DictionaryEntry formEntry in Form) + { + // AddMultipartContent will handle PSObject unwrapping, Object type determination and enumerateing top level IEnumerables. + AddMultipartContent(fieldName: formEntry.Key, fieldValue: formEntry.Value, formData: formData, enumerate: true); + } - ByteArrayContent byteArrayContent = new(content); - request.Content = byteArrayContent; - } + SetRequestContent(request, formData); + } + else if (Body is not null) + { + // Coerce body into a usable form + object content = Body; - /// - /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. - /// - /// The WebRequest who's content is to be set. - /// A String object containing the content data. - /// - /// Because this function sets the request's ContentLength property and writes content data into the request's stream, - /// it should be called one time maximum on a given request. - /// - internal void SetRequestContent(HttpRequestMessage request, string content) - { - ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(content); - - Encoding encoding = null; - if (ContentType is not null) + // Make sure we're using the base object of the body, not the PSObject wrapper + if (Body is PSObject psBody) + { + content = psBody.BaseObject; + } + + switch (content) + { + case FormObject form: + SetRequestContent(request, form.Fields); + break; + case IDictionary dictionary when request.Method != HttpMethod.Get: + SetRequestContent(request, dictionary); + break; + case XmlNode xmlNode: + SetRequestContent(request, xmlNode); + break; + case Stream stream: + SetRequestContent(request, stream); + break; + case byte[] bytes: + SetRequestContent(request, bytes); + break; + case MultipartFormDataContent multipartFormDataContent: + SetRequestContent(request, multipartFormDataContent); + break; + default: + SetRequestContent(request, (string)LanguagePrimitives.ConvertTo(content, typeof(string), CultureInfo.InvariantCulture)); + break; + } + } + else if (InFile is not null) { - // If Content-Type contains the encoding format (as CharSet), use this encoding format - // to encode the Body of the WebRequest sent to the server. Default Encoding format - // would be used if Charset is not supplied in the Content-Type property. + // Copy InFile data try { - var mediaTypeHeaderValue = MediaTypeHeaderValue.Parse(ContentType); - if (!string.IsNullOrEmpty(mediaTypeHeaderValue.CharSet)) - { - encoding = Encoding.GetEncoding(mediaTypeHeaderValue.CharSet); - } + // Open the input file + SetRequestContent(request, new FileStream(InFile, FileMode.Open, FileAccess.Read, FileShare.Read)); } - catch (Exception ex) when (ex is FormatException || ex is ArgumentException) + catch (UnauthorizedAccessException) { - if (!SkipHeaderValidation) - { - ValidationMetadataException outerEx = new(WebCmdletStrings.ContentTypeException, ex); - ErrorRecord er = new(outerEx, "WebCmdletContentTypeException", ErrorCategory.InvalidArgument, ContentType); - ThrowTerminatingError(er); - } + string msg = string.Format(CultureInfo.InvariantCulture, WebCmdletStrings.AccessDenied, _originalFilePath); + + throw new UnauthorizedAccessException(msg); } } - byte[] bytes = StreamHelper.EncodeToBytes(content, encoding); - ByteArrayContent byteArrayContent = new(bytes); - request.Content = byteArrayContent; - } - - internal void SetRequestContent(HttpRequestMessage request, XmlNode xmlNode) - { - ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(xmlNode); - - byte[] bytes = null; - XmlDocument doc = xmlNode as XmlDocument; - if (doc?.FirstChild is XmlDeclaration) + // For other methods like Put where empty content has meaning, we need to fill in the content + if (request.Content is null) { - XmlDeclaration decl = doc.FirstChild as XmlDeclaration; - Encoding encoding = Encoding.GetEncoding(decl.Encoding); - bytes = StreamHelper.EncodeToBytes(doc.OuterXml, encoding); + // If this is a Get request and there is no content, then don't fill in the content as empty content gets rejected by some web services per RFC7230 + if (request.Method == HttpMethod.Get && ContentType is null) + { + return; + } + + request.Content = new StringContent(string.Empty); + request.Content.Headers.Clear(); } - else + + foreach (var entry in WebSession.ContentHeaders) { - bytes = StreamHelper.EncodeToBytes(xmlNode.OuterXml, encoding: null); + if (!string.IsNullOrWhiteSpace(entry.Value)) + { + if (SkipHeaderValidation) + { + request.Content.Headers.TryAddWithoutValidation(entry.Key, entry.Value); + } + else + { + try + { + request.Content.Headers.Add(entry.Key, entry.Value); + } + catch (FormatException ex) + { + var outerEx = new ValidationMetadataException(WebCmdletStrings.ContentTypeException, ex); + ErrorRecord er = new(outerEx, "WebCmdletContentTypeException", ErrorCategory.InvalidArgument, ContentType); + ThrowTerminatingError(er); + } + } + } } - - ByteArrayContent byteArrayContent = new(bytes); - - request.Content = byteArrayContent; } - /// - /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. - /// - /// The WebRequest who's content is to be set. - /// A Stream object containing the content data. - /// - /// Because this function sets the request's ContentLength property and writes content data into the request's stream, - /// it should be called one time maximum on a given request. - /// - internal void SetRequestContent(HttpRequestMessage request, Stream contentStream) + // Returns true if the status code is one of the supported redirection codes. + private static bool IsRedirectCode(HttpStatusCode code) { - ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(contentStream); + int intCode = (int)code; + return + ( + (intCode >= 300 && intCode < 304) || + intCode == 307 || + intCode == 308 + ); + } - StreamContent streamContent = new(contentStream); - request.Content = streamContent; + // Returns true if the status code is a redirection code and the action requires switching from POST to GET on redirection. + // NOTE: Some of these status codes map to the same underlying value but spelling them out for completeness. + private static bool IsRedirectToGet(HttpStatusCode code) + { + return + ( + code == HttpStatusCode.Found || + code == HttpStatusCode.Moved || + code == HttpStatusCode.Redirect || + code == HttpStatusCode.RedirectMethod || + code == HttpStatusCode.SeeOther || + code == HttpStatusCode.Ambiguous || + code == HttpStatusCode.MultipleChoices + ); } - /// - /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. - /// - /// The WebRequest who's content is to be set. - /// A MultipartFormDataContent object containing multipart/form-data content. - /// - /// Because this function sets the request's ContentLength property and writes content data into the request's stream, - /// it should be called one time maximum on a given request. - /// - internal void SetRequestContent(HttpRequestMessage request, MultipartFormDataContent multipartContent) + // Returns true if the status code shows a server or client error and MaximumRetryCount > 0 + private bool ShouldRetry(HttpStatusCode code) { - ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(multipartContent); - - // Content headers will be set by MultipartFormDataContent which will throw unless we clear them first - WebSession.ContentHeaders.Clear(); + int intCode = (int)code; - request.Content = multipartContent; + return + ( + (intCode == 304 || (intCode >= 400 && intCode <= 599)) && WebSession.MaximumRetryCount > 0 + ); } - internal void SetRequestContent(HttpRequestMessage request, IDictionary content) + internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestMessage request, bool handleRedirect) { + ArgumentNullException.ThrowIfNull(client); ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(content); - string body = FormatDictionary(content); - SetRequestContent(request, body); - } + // Add 1 to account for the first request. + int totalRequests = WebSession.MaximumRetryCount + 1; + HttpRequestMessage req = request; + HttpResponseMessage response = null; - internal void ParseLinkHeader(HttpResponseMessage response, System.Uri requestUri) - { - if (_relationLink is null) - { - // Must ignore the case of relation links. See RFC 8288 (https://tools.ietf.org/html/rfc8288) - _relationLink = new Dictionary(StringComparer.OrdinalIgnoreCase); - } - else + do { - _relationLink.Clear(); - } + // Track the current URI being used by various requests and re-requests. + Uri currentUri = req.RequestUri; - // We only support the URL in angle brackets and `rel`, other attributes are ignored - // user can still parse it themselves via the Headers property - const string pattern = "<(?.*?)>;\\s*rel=(?\")?(?(?(quoted).*?|[^,;]*))(?(quoted)\")"; - if (response.Headers.TryGetValues("Link", out IEnumerable links)) - { - foreach (string linkHeader in links) + _cancelToken = new CancellationTokenSource(); + response = client.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult(); + + if (handleRedirect + && WebSession.MaximumRedirection is not 0 + && IsRedirectCode(response.StatusCode) + && response.Headers.Location is not null) { - MatchCollection matchCollection = Regex.Matches(linkHeader, pattern); - foreach (Match match in matchCollection) + _cancelToken.Cancel(); + _cancelToken = null; + + // If explicit count was provided, reduce it for this redirection. + if (WebSession.MaximumRedirection > 0) + { + WebSession.MaximumRedirection--; + } + + // For selected redirects that used POST, GET must be used with the + // redirected Location. + // Since GET is the default; POST only occurs when -Method POST is used. + if (Method == WebRequestMethod.Post && IsRedirectToGet(response.StatusCode)) + { + // See https://msdn.microsoft.com/library/system.net.httpstatuscode(v=vs.110).aspx + Method = WebRequestMethod.Get; + } + + currentUri = new Uri(request.RequestUri, response.Headers.Location); + + // Continue to handle redirection + using HttpRequestMessage redirectRequest = GetRequest(currentUri); + response.Dispose(); + response = GetResponse(client, redirectRequest, handleRedirect); + } + + // Request again without the Range header because the server indicated the range was not satisfiable. + // This happens when the local file is larger than the remote file. + // If the size of the remote file is the same as the local file, there is nothing to resume. + if (Resume.IsPresent + && response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable + && (response.Content.Headers.ContentRange.HasLength + && response.Content.Headers.ContentRange.Length != _resumeFileSize)) + { + _cancelToken.Cancel(); + + WriteVerbose(WebCmdletStrings.WebMethodResumeFailedVerboseMsg); + + // Disable the Resume switch so the subsequent calls to GetResponse() and FillRequestStream() + // are treated as a standard -OutFile request. This also disables appending local file. + Resume = new SwitchParameter(false); + + using (HttpRequestMessage requestWithoutRange = GetRequest(currentUri)) + { + FillRequestStream(requestWithoutRange); + + long requestContentLength = requestWithoutRange.Content is null ? 0 : requestWithoutRange.Content.Headers.ContentLength.Value; + + string reqVerboseMsg = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.WebMethodInvocationVerboseMsg, + requestWithoutRange.Version, + requestWithoutRange.Method, + requestContentLength); + + WriteVerbose(reqVerboseMsg); + + response.Dispose(); + response = GetResponse(client, requestWithoutRange, handleRedirect); + } + } + + _resumeSuccess = response.StatusCode == HttpStatusCode.PartialContent; + + // When MaximumRetryCount is not specified, the totalRequests is 1. + if (totalRequests > 1 && ShouldRetry(response.StatusCode)) + { + int retryIntervalInSeconds = WebSession.RetryIntervalInSeconds; + + // If the status code is 429 get the retry interval from the Headers. + // Ignore broken header and its value. + if (response.StatusCode is HttpStatusCode.Conflict && response.Headers.TryGetValues(HttpKnownHeaderNames.RetryAfter, out IEnumerable retryAfter)) { - if (match.Success) + try { - string url = match.Groups["url"].Value; - string rel = match.Groups["rel"].Value; - if (url != string.Empty && rel != string.Empty && !_relationLink.ContainsKey(rel)) + IEnumerator enumerator = retryAfter.GetEnumerator(); + if (enumerator.MoveNext()) { - Uri absoluteUri = new(requestUri, url); - _relationLink.Add(rel, absoluteUri.AbsoluteUri); + retryIntervalInSeconds = Convert.ToInt32(enumerator.Current); } } + catch + { + // Ignore broken header. + } } - } - } - } - - /// - /// Adds content to a . Object type detection is used to determine if the value is string, File, or Collection. - /// - /// The Field Name to use. - /// The Field Value to use. - /// The to update. - /// If true, collection types in will be enumerated. If false, collections will be treated as single value. - private void AddMultipartContent(object fieldName, object fieldValue, MultipartFormDataContent formData, bool enumerate) - { - ArgumentNullException.ThrowIfNull(formData); - - // It is possible that the dictionary keys or values are PSObject wrapped depending on how the dictionary is defined and assigned. - // Before processing the field name and value we need to ensure we are working with the base objects and not the PSObject wrappers. + + string retryMessage = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.RetryVerboseMsg, + retryIntervalInSeconds, + response.StatusCode); - // Unwrap fieldName PSObjects - if (fieldName is PSObject namePSObject) - { - fieldName = namePSObject.BaseObject; - } + WriteVerbose(retryMessage); - // Unwrap fieldValue PSObjects - if (fieldValue is PSObject valuePSObject) - { - fieldValue = valuePSObject.BaseObject; - } + _cancelToken = new CancellationTokenSource(); + Task.Delay(retryIntervalInSeconds * 1000, _cancelToken.Token).GetAwaiter().GetResult(); + _cancelToken.Cancel(); + _cancelToken = null; - // Treat a single FileInfo as a FileContent - if (fieldValue is FileInfo file) - { - formData.Add(GetMultipartFileContent(fieldName: fieldName, file: file)); - return; - } + req.Dispose(); + req = GetRequest(currentUri); + FillRequestStream(req); + } - // Treat Strings and other single values as a StringContent. - // If enumeration is false, also treat IEnumerables as StringContents. - // String implements IEnumerable so the explicit check is required. - if (!enumerate || fieldValue is string || fieldValue is not IEnumerable) - { - formData.Add(GetMultipartStringContent(fieldName: fieldName, fieldValue: fieldValue)); - return; + totalRequests--; } + while (totalRequests > 0 && !response.IsSuccessStatusCode); - // Treat the value as a collection and enumerate it if enumeration is true - if (enumerate && fieldValue is IEnumerable items) - { - foreach (var item in items) - { - // Recurse, but do not enumerate the next level. IEnumerables will be treated as single values. - AddMultipartContent(fieldName: fieldName, fieldValue: item, formData: formData, enumerate: false); - } - } + return response; } - /// - /// Gets a from the supplied field name and field value. Uses to convert the objects to strings. - /// - /// The Field Name to use for the - /// The Field Value to use for the - private static StringContent GetMultipartStringContent(object fieldName, object fieldValue) + internal virtual void UpdateSession(HttpResponseMessage response) { - var contentDisposition = new ContentDispositionHeaderValue("form-data"); - // .NET does not enclose field names in quotes, however, modern browsers and curl do. - contentDisposition.Name = "\"" + LanguagePrimitives.ConvertTo(fieldName) + "\""; + ArgumentNullException.ThrowIfNull(response); + } - var result = new StringContent(LanguagePrimitives.ConvertTo(fieldValue)); - result.Headers.ContentDisposition = contentDisposition; + #endregion Virtual Methods - return result; - } + #region Helper Properties - /// - /// Gets a from the supplied field name and . Uses to convert the fieldname to a string. - /// - /// The Field Name to use for the - /// The to use for the - private static StreamContent GetMultipartStreamContent(object fieldName, Stream stream) - { - var contentDisposition = new ContentDispositionHeaderValue("form-data"); - // .NET does not enclose field names in quotes, however, modern browsers and curl do. - contentDisposition.Name = "\"" + LanguagePrimitives.ConvertTo(fieldName) + "\""; + internal string QualifiedOutFile => QualifyFilePath(OutFile); - var result = new StreamContent(stream); - result.Headers.ContentDisposition = contentDisposition; - result.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + internal bool ShouldSaveToOutFile => !string.IsNullOrEmpty(OutFile); - return result; - } + internal bool ShouldWriteToPipeline => !ShouldSaveToOutFile || PassThru; + + internal bool ShouldCheckHttpStatus => !SkipHttpErrorCheck; /// - /// Gets a from the supplied field name and file. Calls to create the and then sets the file name. + /// Determines whether writing to a file should Resume and append rather than overwrite. /// - /// The Field Name to use for the - /// The file to use for the - private static StreamContent GetMultipartFileContent(object fieldName, FileInfo file) - { - var result = GetMultipartStreamContent(fieldName: fieldName, stream: new FileStream(file.FullName, FileMode.Open)); - - // .NET does not enclose field names in quotes, however, modern browsers and curl do. - result.Headers.ContentDisposition.FileName = "\"" + file.Name + "\""; + internal bool ShouldResume => Resume.IsPresent && _resumeSuccess; - return result; - } + #endregion Helper Properties - private static string FormatErrorMessage(string error, string contentType) + #region Helper Methods + private Uri PrepareUri(Uri uri) { - string formattedError = null; + uri = CheckProtocol(uri); - try + // Before creating the web request, + // preprocess Body if content is a dictionary and method is GET (set as query) + LanguagePrimitives.TryConvertTo(Body, out IDictionary bodyAsDictionary); + if (bodyAsDictionary is not null && (Method == WebRequestMethod.Default || Method == WebRequestMethod.Get || CustomMethod == "GET")) { - if (ContentHelper.IsXml(contentType)) + UriBuilder uriBuilder = new(uri); + if (uriBuilder.Query is not null && uriBuilder.Query.Length > 1) { - XmlDocument doc = new(); - doc.LoadXml(error); - - XmlWriterSettings settings = new XmlWriterSettings { - Indent = true, - NewLineOnAttributes = true, - OmitXmlDeclaration = true - }; - - if (doc.FirstChild is XmlDeclaration) - { - XmlDeclaration decl = doc.FirstChild as XmlDeclaration; - settings.Encoding = Encoding.GetEncoding(decl.Encoding); - } - - StringBuilder stringBuilder = new(); - using XmlWriter xmlWriter = XmlWriter.Create(stringBuilder, settings); - doc.Save(xmlWriter); - string xmlString = stringBuilder.ToString(); - - formattedError = Environment.NewLine + xmlString; + uriBuilder.Query = string.Concat(uriBuilder.Query.AsSpan(1), "&", FormatDictionary(bodyAsDictionary)); } - else if (ContentHelper.IsJson(contentType)) + else { - JsonNode jsonNode = JsonNode.Parse(error); - JsonSerializerOptions options = new JsonSerializerOptions { WriteIndented = true }; - string jsonString = jsonNode.ToJsonString(options); - - formattedError = Environment.NewLine + jsonString; + uriBuilder.Query = FormatDictionary(bodyAsDictionary); } - } - catch - { - // Ignore errors - } - - if (string.IsNullOrEmpty(formattedError)) - { - // Remove HTML tags making it easier to read - formattedError = System.Text.RegularExpressions.Regex.Replace(error, "<[^>]*>", string.Empty); - } - return formattedError; - } + uri = uriBuilder.Uri; - #endregion Helper Methods + // Set body to null to prevent later FillRequestStream + Body = null; + } - #region Abstract Methods + return uri; + } - /// - /// Read the supplied WebResponse object and push the resulting output into the pipeline. - /// - /// Instance of a WebResponse object to be processed. - internal abstract void ProcessResponse(HttpResponseMessage response); + private static Uri CheckProtocol(Uri uri) + { + ArgumentNullException.ThrowIfNull(uri); - #endregion Abstract Methods - } + if (!uri.IsAbsoluteUri) + { + uri = new Uri("http://" + uri.OriginalString); + } - // TODO: Merge Partials + return uri; + } - /// - /// Exception class for webcmdlets to enable returning HTTP error response. - /// - public sealed class HttpResponseException : HttpRequestException - { - /// - /// Initializes a new instance of the class. - /// - /// Message for the exception. - /// Response from the HTTP server. - public HttpResponseException(string message, HttpResponseMessage response) : base(message, inner: null, response.StatusCode) + private string QualifyFilePath(string path) { - Response = response; + string resolvedFilePath = PathUtils.ResolveFilePath(filePath: path, command: this, isLiteralPath: true); + return resolvedFilePath; } - /// - /// HTTP error response. - /// - public HttpResponseMessage Response { get; } - } - - /// - /// Base class for Invoke-RestMethod and Invoke-WebRequest commands. - /// - public abstract partial class WebRequestPSCmdlet : PSCmdlet - { - - /// - /// Cancellation token source. - /// - internal CancellationTokenSource _cancelToken = null; - - /// - /// Parse Rel Links. - /// - internal bool _parseRelLink = false; + private static string FormatDictionary(IDictionary content) + { + ArgumentNullException.ThrowIfNull(content); - /// - /// Automatically follow Rel Links. - /// - internal bool _followRelLink = false; + StringBuilder bodyBuilder = new(); + foreach (string key in content.Keys) + { + if (bodyBuilder.Length > 0) + { + bodyBuilder.Append('&'); + } - /// - /// Automatically follow Rel Links. - /// - internal Dictionary _relationLink = null; + object value = content[key]; - /// - /// Maximum number of Rel Links to follow. - /// - internal int _maximumFollowRelLink = int.MaxValue; + // URLEncode the key and value + string encodedKey = WebUtility.UrlEncode(key); + string encodedValue = string.Empty; + if (value is not null) + { + encodedValue = WebUtility.UrlEncode(value.ToString()); + } - /// - /// The remote endpoint returned a 206 status code indicating successful resume. - /// - private bool _resumeSuccess = false; + bodyBuilder.Append($"{encodedKey}={encodedValue}"); + } - /// - /// The current size of the local file being resumed. - /// - private long _resumeFileSize = 0; + return bodyBuilder.ToString(); + } - private static HttpMethod GetHttpMethod(WebRequestMethod method) => method switch + private ErrorRecord GetValidationError(string msg, string errorId) { - WebRequestMethod.Default or WebRequestMethod.Get => HttpMethod.Get, - WebRequestMethod.Delete => HttpMethod.Delete, - WebRequestMethod.Head => HttpMethod.Head, - WebRequestMethod.Patch => HttpMethod.Patch, - WebRequestMethod.Post => HttpMethod.Post, - WebRequestMethod.Put => HttpMethod.Put, - WebRequestMethod.Options => HttpMethod.Options, - WebRequestMethod.Trace => HttpMethod.Trace, - _ => new HttpMethod(method.ToString().ToUpperInvariant()) - }; + var ex = new ValidationMetadataException(msg); + var error = new ErrorRecord(ex, errorId, ErrorCategory.InvalidArgument, this); + return error; + } - #region Virtual Methods + private ErrorRecord GetValidationError(string msg, string errorId, params object[] args) + { + msg = string.Format(CultureInfo.InvariantCulture, msg, args); + var ex = new ValidationMetadataException(msg); + var error = new ErrorRecord(ex, errorId, ErrorCategory.InvalidArgument, this); + return error; + } - internal virtual HttpClient GetHttpClient(bool handleRedirect) + private string GetBasicAuthorizationHeader() { - HttpClientHandler handler = new(); - handler.CookieContainer = WebSession.Cookies; - handler.AutomaticDecompression = DecompressionMethods.All; + var password = new NetworkCredential(null, Credential.Password).Password; + string unencoded = string.Create(CultureInfo.InvariantCulture, $"{Credential.UserName}:{password}"); + byte[] bytes = Encoding.UTF8.GetBytes(unencoded); + return string.Create(CultureInfo.InvariantCulture, $"Basic {Convert.ToBase64String(bytes)}"); + } - // Set the credentials used by this request - if (WebSession.UseDefaultCredentials) - { - // The UseDefaultCredentials flag overrides other supplied credentials - handler.UseDefaultCredentials = true; - } - else if (WebSession.Credentials is not null) - { - handler.Credentials = WebSession.Credentials; - } + private string GetBearerAuthorizationHeader() + { + return string.Create(CultureInfo.InvariantCulture, $"Bearer {new NetworkCredential(string.Empty, Token).Password}"); + } - if (NoProxy) + private void ProcessAuthentication() + { + if (Authentication == WebAuthenticationType.Basic) { - handler.UseProxy = false; + WebSession.Headers["Authorization"] = GetBasicAuthorizationHeader(); } - else if (WebSession.Proxy is not null) + else if (Authentication == WebAuthenticationType.Bearer || Authentication == WebAuthenticationType.OAuth) { - handler.Proxy = WebSession.Proxy; + WebSession.Headers["Authorization"] = GetBearerAuthorizationHeader(); } - - if (WebSession.Certificates is not null) + else { - handler.ClientCertificates.AddRange(WebSession.Certificates); + Diagnostics.Assert(false, string.Create(CultureInfo.InvariantCulture, $"Unrecognized Authentication value: {Authentication}")); } + } + + /// + /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. + /// + /// The WebRequest who's content is to be set. + /// A byte array containing the content data. + /// + /// Because this function sets the request's ContentLength property and writes content data into the request's stream, + /// it should be called one time maximum on a given request. + /// + internal void SetRequestContent(HttpRequestMessage request, byte[] content) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(content); - if (SkipCertificateCheck) + ByteArrayContent byteArrayContent = new(content); + request.Content = byteArrayContent; + } + + /// + /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. + /// + /// The WebRequest who's content is to be set. + /// A String object containing the content data. + /// + /// Because this function sets the request's ContentLength property and writes content data into the request's stream, + /// it should be called one time maximum on a given request. + /// + internal void SetRequestContent(HttpRequestMessage request, string content) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(content); + + Encoding encoding = null; + if (ContentType is not null) { - handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; - handler.ClientCertificateOptions = ClientCertificateOption.Manual; + // If Content-Type contains the encoding format (as CharSet), use this encoding format + // to encode the Body of the WebRequest sent to the server. Default Encoding format + // would be used if Charset is not supplied in the Content-Type property. + try + { + var mediaTypeHeaderValue = MediaTypeHeaderValue.Parse(ContentType); + if (!string.IsNullOrEmpty(mediaTypeHeaderValue.CharSet)) + { + encoding = Encoding.GetEncoding(mediaTypeHeaderValue.CharSet); + } + } + catch (Exception ex) when (ex is FormatException || ex is ArgumentException) + { + if (!SkipHeaderValidation) + { + ValidationMetadataException outerEx = new(WebCmdletStrings.ContentTypeException, ex); + ErrorRecord er = new(outerEx, "WebCmdletContentTypeException", ErrorCategory.InvalidArgument, ContentType); + ThrowTerminatingError(er); + } + } } - // This indicates GetResponse will handle redirects. - if (handleRedirect || WebSession.MaximumRedirection == 0) + byte[] bytes = StreamHelper.EncodeToBytes(content, encoding); + ByteArrayContent byteArrayContent = new(bytes); + request.Content = byteArrayContent; + } + + internal void SetRequestContent(HttpRequestMessage request, XmlNode xmlNode) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(xmlNode); + + byte[] bytes = null; + XmlDocument doc = xmlNode as XmlDocument; + if (doc?.FirstChild is XmlDeclaration) { - handler.AllowAutoRedirect = false; + XmlDeclaration decl = doc.FirstChild as XmlDeclaration; + Encoding encoding = Encoding.GetEncoding(decl.Encoding); + bytes = StreamHelper.EncodeToBytes(doc.OuterXml, encoding); } - else if (WebSession.MaximumRedirection > 0) + else { - handler.MaxAutomaticRedirections = WebSession.MaximumRedirection; + bytes = StreamHelper.EncodeToBytes(xmlNode.OuterXml, encoding: null); } - handler.SslProtocols = (SslProtocols)SslProtocol; - - HttpClient httpClient = new(handler); - - // Check timeout setting (in seconds instead of milliseconds as in HttpWebRequest) - httpClient.Timeout = TimeoutSec is 0 ? TimeSpan.FromMilliseconds(Timeout.Infinite) : new TimeSpan(0, 0, TimeoutSec); + ByteArrayContent byteArrayContent = new(bytes); - return httpClient; + request.Content = byteArrayContent; } - internal virtual HttpRequestMessage GetRequest(Uri uri) + /// + /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. + /// + /// The WebRequest who's content is to be set. + /// A Stream object containing the content data. + /// + /// Because this function sets the request's ContentLength property and writes content data into the request's stream, + /// it should be called one time maximum on a given request. + /// + internal void SetRequestContent(HttpRequestMessage request, Stream contentStream) { - Uri requestUri = PrepareUri(uri); - HttpMethod httpMethod = string.IsNullOrEmpty(CustomMethod) ? GetHttpMethod(Method) : new HttpMethod(CustomMethod); + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(contentStream); - // Create the base WebRequest object - var request = new HttpRequestMessage(httpMethod, requestUri); + StreamContent streamContent = new(contentStream); + request.Content = streamContent; + } - if (HttpVersion is not null) - { - request.Version = HttpVersion; - } + /// + /// Sets the ContentLength property of the request and writes the specified content to the request's RequestStream. + /// + /// The WebRequest who's content is to be set. + /// A MultipartFormDataContent object containing multipart/form-data content. + /// + /// Because this function sets the request's ContentLength property and writes content data into the request's stream, + /// it should be called one time maximum on a given request. + /// + internal void SetRequestContent(HttpRequestMessage request, MultipartFormDataContent multipartContent) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(multipartContent); + + // Content headers will be set by MultipartFormDataContent which will throw unless we clear them first + WebSession.ContentHeaders.Clear(); - // Pull in session data - if (WebSession.Headers.Count > 0) - { - WebSession.ContentHeaders.Clear(); - foreach (var entry in WebSession.Headers) - { - if (HttpKnownHeaderNames.ContentHeaders.Contains(entry.Key)) - { - WebSession.ContentHeaders.Add(entry.Key, entry.Value); - } - else - { - if (SkipHeaderValidation) - { - request.Headers.TryAddWithoutValidation(entry.Key, entry.Value); - } - else - { - request.Headers.Add(entry.Key, entry.Value); - } - } - } - } + request.Content = multipartContent; + } - // Set 'Transfer-Encoding: chunked' if 'Transfer-Encoding' is specified - if (WebSession.Headers.ContainsKey(HttpKnownHeaderNames.TransferEncoding)) - { - request.Headers.TransferEncodingChunked = true; - } + internal void SetRequestContent(HttpRequestMessage request, IDictionary content) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(content); - // Set 'User-Agent' if WebSession.Headers doesn't already contain it - if (WebSession.Headers.TryGetValue(HttpKnownHeaderNames.UserAgent, out string userAgent)) - { - WebSession.UserAgent = userAgent; - } - else - { - if (SkipHeaderValidation) - { - request.Headers.TryAddWithoutValidation(HttpKnownHeaderNames.UserAgent, WebSession.UserAgent); - } - else - { - request.Headers.Add(HttpKnownHeaderNames.UserAgent, WebSession.UserAgent); - } - } + string body = FormatDictionary(content); + SetRequestContent(request, body); + } - // Set 'Keep-Alive' to false. This means set the Connection to 'Close'. - if (DisableKeepAlive) + internal void ParseLinkHeader(HttpResponseMessage response, System.Uri requestUri) + { + if (_relationLink is null) { - request.Headers.Add(HttpKnownHeaderNames.Connection, "Close"); + // Must ignore the case of relation links. See RFC 8288 (https://tools.ietf.org/html/rfc8288) + _relationLink = new Dictionary(StringComparer.OrdinalIgnoreCase); } - - // Set 'Transfer-Encoding' - if (TransferEncoding is not null) + else { - request.Headers.TransferEncodingChunked = true; - var headerValue = new TransferCodingHeaderValue(TransferEncoding); - if (!request.Headers.TransferEncoding.Contains(headerValue)) - { - request.Headers.TransferEncoding.Add(headerValue); - } + _relationLink.Clear(); } - // If the file to resume downloading exists, create the Range request header using the file size. - // If not, create a Range to request the entire file. - if (Resume.IsPresent) + // We only support the URL in angle brackets and `rel`, other attributes are ignored + // user can still parse it themselves via the Headers property + const string pattern = "<(?.*?)>;\\s*rel=(?\")?(?(?(quoted).*?|[^,;]*))(?(quoted)\")"; + if (response.Headers.TryGetValues("Link", out IEnumerable links)) { - var fileInfo = new FileInfo(QualifiedOutFile); - if (fileInfo.Exists) - { - request.Headers.Range = new RangeHeaderValue(fileInfo.Length, null); - _resumeFileSize = fileInfo.Length; - } - else + foreach (string linkHeader in links) { - request.Headers.Range = new RangeHeaderValue(0, null); + MatchCollection matchCollection = Regex.Matches(linkHeader, pattern); + foreach (Match match in matchCollection) + { + if (match.Success) + { + string url = match.Groups["url"].Value; + string rel = match.Groups["rel"].Value; + if (url != string.Empty && rel != string.Empty && !_relationLink.ContainsKey(rel)) + { + Uri absoluteUri = new(requestUri, url); + _relationLink.Add(rel, absoluteUri.AbsoluteUri); + } + } + } } } - - return request; } - internal virtual void FillRequestStream(HttpRequestMessage request) + /// + /// Adds content to a . Object type detection is used to determine if the value is string, File, or Collection. + /// + /// The Field Name to use. + /// The Field Value to use. + /// The to update. + /// If true, collection types in will be enumerated. If false, collections will be treated as single value. + private void AddMultipartContent(object fieldName, object fieldValue, MultipartFormDataContent formData, bool enumerate) { - ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(formData); - // Set the request content type - if (ContentType is not null) - { - WebSession.ContentHeaders[HttpKnownHeaderNames.ContentType] = ContentType; - } - else if (request.Method == HttpMethod.Post) + // It is possible that the dictionary keys or values are PSObject wrapped depending on how the dictionary is defined and assigned. + // Before processing the field name and value we need to ensure we are working with the base objects and not the PSObject wrappers. + + // Unwrap fieldName PSObjects + if (fieldName is PSObject namePSObject) { - // Win8:545310 Invoke-WebRequest does not properly set MIME type for POST - WebSession.ContentHeaders.TryGetValue(HttpKnownHeaderNames.ContentType, out string contentType); - if (string.IsNullOrEmpty(contentType)) - { - WebSession.ContentHeaders[HttpKnownHeaderNames.ContentType] = "application/x-www-form-urlencoded"; - } + fieldName = namePSObject.BaseObject; } - if (Form is not null) + // Unwrap fieldValue PSObjects + if (fieldValue is PSObject valuePSObject) { - var formData = new MultipartFormDataContent(); - foreach (DictionaryEntry formEntry in Form) - { - // AddMultipartContent will handle PSObject unwrapping, Object type determination and enumerateing top level IEnumerables. - AddMultipartContent(fieldName: formEntry.Key, fieldValue: formEntry.Value, formData: formData, enumerate: true); - } - - SetRequestContent(request, formData); + fieldValue = valuePSObject.BaseObject; } - else if (Body is not null) - { - // Coerce body into a usable form - object content = Body; - - // Make sure we're using the base object of the body, not the PSObject wrapper - if (Body is PSObject psBody) - { - content = psBody.BaseObject; - } - switch (content) - { - case FormObject form: - SetRequestContent(request, form.Fields); - break; - case IDictionary dictionary when request.Method != HttpMethod.Get: - SetRequestContent(request, dictionary); - break; - case XmlNode xmlNode: - SetRequestContent(request, xmlNode); - break; - case Stream stream: - SetRequestContent(request, stream); - break; - case byte[] bytes: - SetRequestContent(request, bytes); - break; - case MultipartFormDataContent multipartFormDataContent: - SetRequestContent(request, multipartFormDataContent); - break; - default: - SetRequestContent(request, (string)LanguagePrimitives.ConvertTo(content, typeof(string), CultureInfo.InvariantCulture)); - break; - } - } - else if (InFile is not null) + // Treat a single FileInfo as a FileContent + if (fieldValue is FileInfo file) { - // Copy InFile data - try - { - // Open the input file - SetRequestContent(request, new FileStream(InFile, FileMode.Open, FileAccess.Read, FileShare.Read)); - } - catch (UnauthorizedAccessException) - { - string msg = string.Format(CultureInfo.InvariantCulture, WebCmdletStrings.AccessDenied, _originalFilePath); - - throw new UnauthorizedAccessException(msg); - } + formData.Add(GetMultipartFileContent(fieldName: fieldName, file: file)); + return; } - // For other methods like Put where empty content has meaning, we need to fill in the content - if (request.Content is null) + // Treat Strings and other single values as a StringContent. + // If enumeration is false, also treat IEnumerables as StringContents. + // String implements IEnumerable so the explicit check is required. + if (!enumerate || fieldValue is string || fieldValue is not IEnumerable) { - // If this is a Get request and there is no content, then don't fill in the content as empty content gets rejected by some web services per RFC7230 - if (request.Method == HttpMethod.Get && ContentType is null) - { - return; - } - - request.Content = new StringContent(string.Empty); - request.Content.Headers.Clear(); + formData.Add(GetMultipartStringContent(fieldName: fieldName, fieldValue: fieldValue)); + return; } - foreach (var entry in WebSession.ContentHeaders) + // Treat the value as a collection and enumerate it if enumeration is true + if (enumerate && fieldValue is IEnumerable items) { - if (!string.IsNullOrWhiteSpace(entry.Value)) + foreach (var item in items) { - if (SkipHeaderValidation) - { - request.Content.Headers.TryAddWithoutValidation(entry.Key, entry.Value); - } - else - { - try - { - request.Content.Headers.Add(entry.Key, entry.Value); - } - catch (FormatException ex) - { - var outerEx = new ValidationMetadataException(WebCmdletStrings.ContentTypeException, ex); - ErrorRecord er = new(outerEx, "WebCmdletContentTypeException", ErrorCategory.InvalidArgument, ContentType); - ThrowTerminatingError(er); - } - } + // Recurse, but do not enumerate the next level. IEnumerables will be treated as single values. + AddMultipartContent(fieldName: fieldName, fieldValue: item, formData: formData, enumerate: false); } } } - // Returns true if the status code is one of the supported redirection codes. - private static bool IsRedirectCode(HttpStatusCode code) + /// + /// Gets a from the supplied field name and field value. Uses to convert the objects to strings. + /// + /// The Field Name to use for the + /// The Field Value to use for the + private static StringContent GetMultipartStringContent(object fieldName, object fieldValue) { - int intCode = (int)code; - return - ( - (intCode >= 300 && intCode < 304) || - intCode == 307 || - intCode == 308 - ); + var contentDisposition = new ContentDispositionHeaderValue("form-data"); + // .NET does not enclose field names in quotes, however, modern browsers and curl do. + contentDisposition.Name = "\"" + LanguagePrimitives.ConvertTo(fieldName) + "\""; + + var result = new StringContent(LanguagePrimitives.ConvertTo(fieldValue)); + result.Headers.ContentDisposition = contentDisposition; + + return result; } - // Returns true if the status code is a redirection code and the action requires switching from POST to GET on redirection. - // NOTE: Some of these status codes map to the same underlying value but spelling them out for completeness. - private static bool IsRedirectToGet(HttpStatusCode code) - { - return - ( - code == HttpStatusCode.Found || - code == HttpStatusCode.Moved || - code == HttpStatusCode.Redirect || - code == HttpStatusCode.RedirectMethod || - code == HttpStatusCode.SeeOther || - code == HttpStatusCode.Ambiguous || - code == HttpStatusCode.MultipleChoices - ); + /// + /// Gets a from the supplied field name and . Uses to convert the fieldname to a string. + /// + /// The Field Name to use for the + /// The to use for the + private static StreamContent GetMultipartStreamContent(object fieldName, Stream stream) + { + var contentDisposition = new ContentDispositionHeaderValue("form-data"); + // .NET does not enclose field names in quotes, however, modern browsers and curl do. + contentDisposition.Name = "\"" + LanguagePrimitives.ConvertTo(fieldName) + "\""; + + var result = new StreamContent(stream); + result.Headers.ContentDisposition = contentDisposition; + result.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + + return result; } - // Returns true if the status code shows a server or client error and MaximumRetryCount > 0 - private bool ShouldRetry(HttpStatusCode code) + /// + /// Gets a from the supplied field name and file. Calls to create the and then sets the file name. + /// + /// The Field Name to use for the + /// The file to use for the + private static StreamContent GetMultipartFileContent(object fieldName, FileInfo file) { - int intCode = (int)code; + var result = GetMultipartStreamContent(fieldName: fieldName, stream: new FileStream(file.FullName, FileMode.Open)); + + // .NET does not enclose field names in quotes, however, modern browsers and curl do. + result.Headers.ContentDisposition.FileName = "\"" + file.Name + "\""; - return - ( - (intCode == 304 || (intCode >= 400 && intCode <= 599)) && WebSession.MaximumRetryCount > 0 - ); + return result; } - internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestMessage request, bool handleRedirect) + private static string FormatErrorMessage(string error, string contentType) { - ArgumentNullException.ThrowIfNull(client); - ArgumentNullException.ThrowIfNull(request); - - // Add 1 to account for the first request. - int totalRequests = WebSession.MaximumRetryCount + 1; - HttpRequestMessage req = request; - HttpResponseMessage response = null; + string formattedError = null; - do + try { - // Track the current URI being used by various requests and re-requests. - Uri currentUri = req.RequestUri; - - _cancelToken = new CancellationTokenSource(); - response = client.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult(); - - if (handleRedirect - && WebSession.MaximumRedirection is not 0 - && IsRedirectCode(response.StatusCode) - && response.Headers.Location is not null) + if (ContentHelper.IsXml(contentType)) { - _cancelToken.Cancel(); - _cancelToken = null; + XmlDocument doc = new(); + doc.LoadXml(error); - // If explicit count was provided, reduce it for this redirection. - if (WebSession.MaximumRedirection > 0) - { - WebSession.MaximumRedirection--; - } + XmlWriterSettings settings = new XmlWriterSettings { + Indent = true, + NewLineOnAttributes = true, + OmitXmlDeclaration = true + }; - // For selected redirects that used POST, GET must be used with the - // redirected Location. - // Since GET is the default; POST only occurs when -Method POST is used. - if (Method == WebRequestMethod.Post && IsRedirectToGet(response.StatusCode)) + if (doc.FirstChild is XmlDeclaration) { - // See https://msdn.microsoft.com/library/system.net.httpstatuscode(v=vs.110).aspx - Method = WebRequestMethod.Get; + XmlDeclaration decl = doc.FirstChild as XmlDeclaration; + settings.Encoding = Encoding.GetEncoding(decl.Encoding); } - currentUri = new Uri(request.RequestUri, response.Headers.Location); + StringBuilder stringBuilder = new(); + using XmlWriter xmlWriter = XmlWriter.Create(stringBuilder, settings); + doc.Save(xmlWriter); + string xmlString = stringBuilder.ToString(); - // Continue to handle redirection - using HttpRequestMessage redirectRequest = GetRequest(currentUri); - response.Dispose(); - response = GetResponse(client, redirectRequest, handleRedirect); + formattedError = Environment.NewLine + xmlString; } - - // Request again without the Range header because the server indicated the range was not satisfiable. - // This happens when the local file is larger than the remote file. - // If the size of the remote file is the same as the local file, there is nothing to resume. - if (Resume.IsPresent - && response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable - && (response.Content.Headers.ContentRange.HasLength - && response.Content.Headers.ContentRange.Length != _resumeFileSize)) + else if (ContentHelper.IsJson(contentType)) { - _cancelToken.Cancel(); + JsonNode jsonNode = JsonNode.Parse(error); + JsonSerializerOptions options = new JsonSerializerOptions { WriteIndented = true }; + string jsonString = jsonNode.ToJsonString(options); - WriteVerbose(WebCmdletStrings.WebMethodResumeFailedVerboseMsg); + formattedError = Environment.NewLine + jsonString; + } + } + catch + { + // Ignore errors + } + + if (string.IsNullOrEmpty(formattedError)) + { + // Remove HTML tags making it easier to read + formattedError = System.Text.RegularExpressions.Regex.Replace(error, "<[^>]*>", string.Empty); + } - // Disable the Resume switch so the subsequent calls to GetResponse() and FillRequestStream() - // are treated as a standard -OutFile request. This also disables appending local file. - Resume = new SwitchParameter(false); + return formattedError; + } - using (HttpRequestMessage requestWithoutRange = GetRequest(currentUri)) - { - FillRequestStream(requestWithoutRange); + #endregion Helper Methods - long requestContentLength = requestWithoutRange.Content is null ? 0 : requestWithoutRange.Content.Headers.ContentLength.Value; + #region Abstract Methods - string reqVerboseMsg = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.WebMethodInvocationVerboseMsg, - requestWithoutRange.Version, - requestWithoutRange.Method, - requestContentLength); - - WriteVerbose(reqVerboseMsg); + /// + /// Read the supplied WebResponse object and push the resulting output into the pipeline. + /// + /// Instance of a WebResponse object to be processed. + internal abstract void ProcessResponse(HttpResponseMessage response); - response.Dispose(); - response = GetResponse(client, requestWithoutRange, handleRedirect); - } - } + #endregion Abstract Methods + } - _resumeSuccess = response.StatusCode == HttpStatusCode.PartialContent; + // TODO: Merge Partials - // When MaximumRetryCount is not specified, the totalRequests is 1. - if (totalRequests > 1 && ShouldRetry(response.StatusCode)) - { - int retryIntervalInSeconds = WebSession.RetryIntervalInSeconds; + /// + /// Exception class for webcmdlets to enable returning HTTP error response. + /// + public sealed class HttpResponseException : HttpRequestException + { + /// + /// Initializes a new instance of the class. + /// + /// Message for the exception. + /// Response from the HTTP server. + public HttpResponseException(string message, HttpResponseMessage response) : base(message, inner: null, response.StatusCode) + { + Response = response; + } - // If the status code is 429 get the retry interval from the Headers. - // Ignore broken header and its value. - if (response.StatusCode is HttpStatusCode.Conflict && response.Headers.TryGetValues(HttpKnownHeaderNames.RetryAfter, out IEnumerable retryAfter)) - { - try - { - IEnumerator enumerator = retryAfter.GetEnumerator(); - if (enumerator.MoveNext()) - { - retryIntervalInSeconds = Convert.ToInt32(enumerator.Current); - } - } - catch - { - // Ignore broken header. - } - } - - string retryMessage = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.RetryVerboseMsg, - retryIntervalInSeconds, - response.StatusCode); + /// + /// HTTP error response. + /// + public HttpResponseMessage Response { get; } + } - WriteVerbose(retryMessage); + /// + /// Base class for Invoke-RestMethod and Invoke-WebRequest commands. + /// + public abstract partial class WebRequestPSCmdlet : PSCmdlet + { - _cancelToken = new CancellationTokenSource(); - Task.Delay(retryIntervalInSeconds * 1000, _cancelToken.Token).GetAwaiter().GetResult(); - _cancelToken.Cancel(); - _cancelToken = null; + /// + /// Cancellation token source. + /// + internal CancellationTokenSource _cancelToken = null; - req.Dispose(); - req = GetRequest(currentUri); - FillRequestStream(req); - } + /// + /// Parse Rel Links. + /// + internal bool _parseRelLink = false; - totalRequests--; - } - while (totalRequests > 0 && !response.IsSuccessStatusCode); + /// + /// Automatically follow Rel Links. + /// + internal bool _followRelLink = false; - return response; - } + /// + /// Automatically follow Rel Links. + /// + internal Dictionary _relationLink = null; - internal virtual void UpdateSession(HttpResponseMessage response) - { - ArgumentNullException.ThrowIfNull(response); - } + /// + /// Maximum number of Rel Links to follow. + /// + internal int _maximumFollowRelLink = int.MaxValue; - #endregion Virtual Methods + /// + /// The remote endpoint returned a 206 status code indicating successful resume. + /// + private bool _resumeSuccess = false; + + /// + /// The current size of the local file being resumed. + /// + private long _resumeFileSize = 0; + + private static HttpMethod GetHttpMethod(WebRequestMethod method) => method switch + { + WebRequestMethod.Default or WebRequestMethod.Get => HttpMethod.Get, + WebRequestMethod.Delete => HttpMethod.Delete, + WebRequestMethod.Head => HttpMethod.Head, + WebRequestMethod.Patch => HttpMethod.Patch, + WebRequestMethod.Post => HttpMethod.Post, + WebRequestMethod.Put => HttpMethod.Put, + WebRequestMethod.Options => HttpMethod.Options, + WebRequestMethod.Trace => HttpMethod.Trace, + _ => new HttpMethod(method.ToString().ToUpperInvariant()) + }; #region Overrides From 35b5ce6fb335778f44fc9d83bb8e0f8f203e25ba Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:10:42 +0100 Subject: [PATCH 04/16] move overrides --- .../Common/WebRequestPSCmdlet.Common.cs | 158 +++++++++--------- 1 file changed, 80 insertions(+), 78 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 6bb541d8ad0..caae55fbd19 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -1612,84 +1612,6 @@ private static string FormatErrorMessage(string error, string contentType) internal abstract void ProcessResponse(HttpResponseMessage response); #endregion Abstract Methods - } - - // TODO: Merge Partials - - /// - /// Exception class for webcmdlets to enable returning HTTP error response. - /// - public sealed class HttpResponseException : HttpRequestException - { - /// - /// Initializes a new instance of the class. - /// - /// Message for the exception. - /// Response from the HTTP server. - public HttpResponseException(string message, HttpResponseMessage response) : base(message, inner: null, response.StatusCode) - { - Response = response; - } - - /// - /// HTTP error response. - /// - public HttpResponseMessage Response { get; } - } - - /// - /// Base class for Invoke-RestMethod and Invoke-WebRequest commands. - /// - public abstract partial class WebRequestPSCmdlet : PSCmdlet - { - - /// - /// Cancellation token source. - /// - internal CancellationTokenSource _cancelToken = null; - - /// - /// Parse Rel Links. - /// - internal bool _parseRelLink = false; - - /// - /// Automatically follow Rel Links. - /// - internal bool _followRelLink = false; - - /// - /// Automatically follow Rel Links. - /// - internal Dictionary _relationLink = null; - - /// - /// Maximum number of Rel Links to follow. - /// - internal int _maximumFollowRelLink = int.MaxValue; - - /// - /// The remote endpoint returned a 206 status code indicating successful resume. - /// - private bool _resumeSuccess = false; - - /// - /// The current size of the local file being resumed. - /// - private long _resumeFileSize = 0; - - private static HttpMethod GetHttpMethod(WebRequestMethod method) => method switch - { - WebRequestMethod.Default or WebRequestMethod.Get => HttpMethod.Get, - WebRequestMethod.Delete => HttpMethod.Delete, - WebRequestMethod.Head => HttpMethod.Head, - WebRequestMethod.Patch => HttpMethod.Patch, - WebRequestMethod.Post => HttpMethod.Post, - WebRequestMethod.Put => HttpMethod.Put, - WebRequestMethod.Options => HttpMethod.Options, - WebRequestMethod.Trace => HttpMethod.Trace, - _ => new HttpMethod(method.ToString().ToUpperInvariant()) - }; #region Overrides @@ -1871,4 +1793,84 @@ protected override void ProcessRecord() #endregion Overrides } + + // TODO: Merge Partials + + /// + /// Exception class for webcmdlets to enable returning HTTP error response. + /// + public sealed class HttpResponseException : HttpRequestException + { + /// + /// Initializes a new instance of the class. + /// + /// Message for the exception. + /// Response from the HTTP server. + public HttpResponseException(string message, HttpResponseMessage response) : base(message, inner: null, response.StatusCode) + { + Response = response; + } + + /// + /// HTTP error response. + /// + public HttpResponseMessage Response { get; } + } + + /// + /// Base class for Invoke-RestMethod and Invoke-WebRequest commands. + /// + public abstract partial class WebRequestPSCmdlet : PSCmdlet + { + + /// + /// Cancellation token source. + /// + internal CancellationTokenSource _cancelToken = null; + + /// + /// Parse Rel Links. + /// + internal bool _parseRelLink = false; + + /// + /// Automatically follow Rel Links. + /// + internal bool _followRelLink = false; + + /// + /// Automatically follow Rel Links. + /// + internal Dictionary _relationLink = null; + + /// + /// Maximum number of Rel Links to follow. + /// + internal int _maximumFollowRelLink = int.MaxValue; + + /// + /// The remote endpoint returned a 206 status code indicating successful resume. + /// + private bool _resumeSuccess = false; + + /// + /// The current size of the local file being resumed. + /// + private long _resumeFileSize = 0; + + private static HttpMethod GetHttpMethod(WebRequestMethod method) => method switch + { + WebRequestMethod.Default or WebRequestMethod.Get => HttpMethod.Get, + WebRequestMethod.Delete => HttpMethod.Delete, + WebRequestMethod.Head => HttpMethod.Head, + WebRequestMethod.Patch => HttpMethod.Patch, + WebRequestMethod.Post => HttpMethod.Post, + WebRequestMethod.Put => HttpMethod.Put, + WebRequestMethod.Options => HttpMethod.Options, + WebRequestMethod.Trace => HttpMethod.Trace, + _ => new HttpMethod(method.ToString().ToUpperInvariant()) + }; + + + } } From 3fc2395270fc615771d64f9e6e399097e0997115 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:11:32 +0100 Subject: [PATCH 05/16] remove extra spaces --- .../utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index caae55fbd19..2149a6a1ba3 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -1870,7 +1870,5 @@ public abstract partial class WebRequestPSCmdlet : PSCmdlet WebRequestMethod.Trace => HttpMethod.Trace, _ => new HttpMethod(method.ToString().ToUpperInvariant()) }; - - } } From 4f53e685f6a75eea12d259d375d38d2d9dce7acb Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:14:29 +0100 Subject: [PATCH 06/16] create region fields --- .../Common/WebRequestPSCmdlet.Common.cs | 75 ++++++++++--------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 2149a6a1ba3..c67db6adff3 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -90,6 +90,45 @@ public enum WebSslProtocol /// public abstract partial class WebRequestPSCmdlet : PSCmdlet { + #region Fields + + /// + /// Cancellation token source. + /// + internal CancellationTokenSource _cancelToken = null; + + /// + /// Parse Rel Links. + /// + internal bool _parseRelLink = false; + + /// + /// Automatically follow Rel Links. + /// + internal bool _followRelLink = false; + + /// + /// Automatically follow Rel Links. + /// + internal Dictionary _relationLink = null; + + /// + /// Maximum number of Rel Links to follow. + /// + internal int _maximumFollowRelLink = int.MaxValue; + + /// + /// The remote endpoint returned a 206 status code indicating successful resume. + /// + private bool _resumeSuccess = false; + + /// + /// The current size of the local file being resumed. + /// + private long _resumeFileSize = 0; + + #endregion Fields + #region Virtual Properties #region URI @@ -1822,42 +1861,6 @@ public HttpResponseException(string message, HttpResponseMessage response) : bas /// public abstract partial class WebRequestPSCmdlet : PSCmdlet { - - /// - /// Cancellation token source. - /// - internal CancellationTokenSource _cancelToken = null; - - /// - /// Parse Rel Links. - /// - internal bool _parseRelLink = false; - - /// - /// Automatically follow Rel Links. - /// - internal bool _followRelLink = false; - - /// - /// Automatically follow Rel Links. - /// - internal Dictionary _relationLink = null; - - /// - /// Maximum number of Rel Links to follow. - /// - internal int _maximumFollowRelLink = int.MaxValue; - - /// - /// The remote endpoint returned a 206 status code indicating successful resume. - /// - private bool _resumeSuccess = false; - - /// - /// The current size of the local file being resumed. - /// - private long _resumeFileSize = 0; - private static HttpMethod GetHttpMethod(WebRequestMethod method) => method switch { WebRequestMethod.Default or WebRequestMethod.Get => HttpMethod.Get, From 900c86e6644146340e537e76e5c46596563e332c Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:17:39 +0100 Subject: [PATCH 07/16] reorder fields --- .../Common/WebRequestPSCmdlet.Common.cs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index c67db6adff3..ab1dc2012d4 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -98,14 +98,19 @@ public abstract partial class WebRequestPSCmdlet : PSCmdlet internal CancellationTokenSource _cancelToken = null; /// - /// Parse Rel Links. + /// Automatically follow Rel Links. /// - internal bool _parseRelLink = false; + internal bool _followRelLink = false; /// - /// Automatically follow Rel Links. + /// Maximum number of Rel Links to follow. /// - internal bool _followRelLink = false; + internal int _maximumFollowRelLink = int.MaxValue; + + /// + /// Parse Rel Links. + /// + internal bool _parseRelLink = false; /// /// Automatically follow Rel Links. @@ -113,20 +118,15 @@ public abstract partial class WebRequestPSCmdlet : PSCmdlet internal Dictionary _relationLink = null; /// - /// Maximum number of Rel Links to follow. + /// The current size of the local file being resumed. /// - internal int _maximumFollowRelLink = int.MaxValue; + private long _resumeFileSize = 0; /// /// The remote endpoint returned a 206 status code indicating successful resume. /// private bool _resumeSuccess = false; - /// - /// The current size of the local file being resumed. - /// - private long _resumeFileSize = 0; - #endregion Fields #region Virtual Properties From 3b56f3cdf37011aa435d599e752f48699736b765 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:23:09 +0100 Subject: [PATCH 08/16] move Helper properties --- .../Common/WebRequestPSCmdlet.Common.cs | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index ab1dc2012d4..63234a10004 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -464,6 +464,23 @@ public virtual string CustomMethod #endregion Virtual Properties + #region Helper Properties + + internal string QualifiedOutFile => QualifyFilePath(OutFile); + + internal bool ShouldSaveToOutFile => !string.IsNullOrEmpty(OutFile); + + internal bool ShouldWriteToPipeline => !ShouldSaveToOutFile || PassThru; + + internal bool ShouldCheckHttpStatus => !SkipHttpErrorCheck; + + /// + /// Determines whether writing to a file should Resume and append rather than overwrite. + /// + internal bool ShouldResume => Resume.IsPresent && _resumeSuccess; + + #endregion Helper Properties + #region Virtual Methods internal virtual void ValidateParameters() @@ -1179,23 +1196,6 @@ internal virtual void UpdateSession(HttpResponseMessage response) #endregion Virtual Methods - #region Helper Properties - - internal string QualifiedOutFile => QualifyFilePath(OutFile); - - internal bool ShouldSaveToOutFile => !string.IsNullOrEmpty(OutFile); - - internal bool ShouldWriteToPipeline => !ShouldSaveToOutFile || PassThru; - - internal bool ShouldCheckHttpStatus => !SkipHttpErrorCheck; - - /// - /// Determines whether writing to a file should Resume and append rather than overwrite. - /// - internal bool ShouldResume => Resume.IsPresent && _resumeSuccess; - - #endregion Helper Properties - #region Helper Methods private Uri PrepareUri(Uri uri) { From 83fe8520f0626358cec262e834fab7e6ca7d5b4b Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:25:29 +0100 Subject: [PATCH 09/16] reorder Helper Properties --- .../utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 63234a10004..2e4854683da 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -468,10 +468,6 @@ public virtual string CustomMethod internal string QualifiedOutFile => QualifyFilePath(OutFile); - internal bool ShouldSaveToOutFile => !string.IsNullOrEmpty(OutFile); - - internal bool ShouldWriteToPipeline => !ShouldSaveToOutFile || PassThru; - internal bool ShouldCheckHttpStatus => !SkipHttpErrorCheck; /// @@ -479,6 +475,10 @@ public virtual string CustomMethod /// internal bool ShouldResume => Resume.IsPresent && _resumeSuccess; + internal bool ShouldSaveToOutFile => !string.IsNullOrEmpty(OutFile); + + internal bool ShouldWriteToPipeline => !ShouldSaveToOutFile || PassThru; + #endregion Helper Properties #region Virtual Methods From bdd4c0642970f2941359ea9aea3d9534a931c3f0 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:27:18 +0100 Subject: [PATCH 10/16] move abstract methods --- .../Common/WebRequestPSCmdlet.Common.cs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 2e4854683da..f7ccea320a5 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -481,6 +481,16 @@ public virtual string CustomMethod #endregion Helper Properties + #region Abstract Methods + + /// + /// Read the supplied WebResponse object and push the resulting output into the pipeline. + /// + /// Instance of a WebResponse object to be processed. + internal abstract void ProcessResponse(HttpResponseMessage response); + + #endregion Abstract Methods + #region Virtual Methods internal virtual void ValidateParameters() @@ -1642,16 +1652,6 @@ private static string FormatErrorMessage(string error, string contentType) #endregion Helper Methods - #region Abstract Methods - - /// - /// Read the supplied WebResponse object and push the resulting output into the pipeline. - /// - /// Instance of a WebResponse object to be processed. - internal abstract void ProcessResponse(HttpResponseMessage response); - - #endregion Abstract Methods - #region Overrides /// From 5dcc6bdab098a3720b4b7dd9d5e12429ca2b54b9 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:29:10 +0100 Subject: [PATCH 11/16] move overrides --- .../Common/WebRequestPSCmdlet.Common.cs | 360 +++++++++--------- 1 file changed, 180 insertions(+), 180 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index f7ccea320a5..a5ae22deecc 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -491,6 +491,186 @@ public virtual string CustomMethod #endregion Abstract Methods + #region Overrides + + /// + /// The main execution method for cmdlets derived from WebRequestPSCmdlet. + /// + protected override void ProcessRecord() + { + try + { + // Set cmdlet context for write progress + ValidateParameters(); + PrepareSession(); + + // If the request contains an authorization header and PreserveAuthorizationOnRedirect is not set, + // it needs to be stripped on the first redirect. + bool keepAuthorizationOnRedirect = PreserveAuthorizationOnRedirect.IsPresent + && WebSession.Headers.ContainsKey(HttpKnownHeaderNames.Authorization); + + bool handleRedirect = keepAuthorizationOnRedirect || AllowInsecureRedirect; + + using (HttpClient client = GetHttpClient(handleRedirect)) + { + int followedRelLink = 0; + Uri uri = Uri; + do + { + if (followedRelLink > 0) + { + string linkVerboseMsg = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.FollowingRelLinkVerboseMsg, + uri.AbsoluteUri); + + WriteVerbose(linkVerboseMsg); + } + + using (HttpRequestMessage request = GetRequest(uri)) + { + FillRequestStream(request); + try + { + long requestContentLength = request.Content is null ? 0 : request.Content.Headers.ContentLength.Value; + + string reqVerboseMsg = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.WebMethodInvocationVerboseMsg, + request.Version, + request.Method, + requestContentLength); + + WriteVerbose(reqVerboseMsg); + + using HttpResponseMessage response = GetResponse(client, request, handleRedirect); + + string contentType = ContentHelper.GetContentType(response); + string respVerboseMsg = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.WebResponseVerboseMsg, + response.Content.Headers.ContentLength, + contentType); + + WriteVerbose(respVerboseMsg); + + bool _isSuccess = response.IsSuccessStatusCode; + + // Check if the Resume range was not satisfiable because the file already completed downloading. + // This happens when the local file is the same size as the remote file. + if (Resume.IsPresent + && response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable + && response.Content.Headers.ContentRange.HasLength + && response.Content.Headers.ContentRange.Length == _resumeFileSize) + { + _isSuccess = true; + WriteVerbose(string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.OutFileWritingSkipped, + OutFile)); + + // Disable writing to the OutFile. + OutFile = null; + } + + if (ShouldCheckHttpStatus && !_isSuccess) + { + string message = string.Format( + CultureInfo.CurrentCulture, + WebCmdletStrings.ResponseStatusCodeFailure, + (int)response.StatusCode, + response.ReasonPhrase); + + HttpResponseException httpEx = new(message, response); + ErrorRecord er = new(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); + string detailMsg = string.Empty; + StreamReader reader = null; + try + { + reader = new StreamReader(StreamHelper.GetResponseStream(response)); + detailMsg = FormatErrorMessage(reader.ReadToEnd(), contentType); + } + catch + { + // Catch all + } + finally + { + reader?.Dispose(); + } + + if (!string.IsNullOrEmpty(detailMsg)) + { + er.ErrorDetails = new ErrorDetails(detailMsg); + } + + ThrowTerminatingError(er); + } + + if (_parseRelLink || _followRelLink) + { + ParseLinkHeader(response, uri); + } + + ProcessResponse(response); + UpdateSession(response); + + // If we hit our maximum redirection count, generate an error. + // Errors with redirection counts of greater than 0 are handled automatically by .NET, but are + // impossible to detect programmatically when we hit this limit. By handling this ourselves + // (and still writing out the result), users can debug actual HTTP redirect problems. + if (WebSession.MaximumRedirection == 0 && IsRedirectCode(response.StatusCode)) + { + ErrorRecord er = new(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request); + er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded); + WriteError(er); + } + } + catch (HttpRequestException ex) + { + ErrorRecord er = new(ex, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); + if (ex.InnerException is not null) + { + er.ErrorDetails = new ErrorDetails(ex.InnerException.Message); + } + + ThrowTerminatingError(er); + } + + if (_followRelLink) + { + if (!_relationLink.ContainsKey("next")) + { + return; + } + + uri = new Uri(_relationLink["next"]); + followedRelLink++; + } + } + } + while (_followRelLink && (followedRelLink < _maximumFollowRelLink)); + } + } + catch (CryptographicException ex) + { + ErrorRecord er = new(ex, "WebCmdletCertificateException", ErrorCategory.SecurityError, null); + ThrowTerminatingError(er); + } + catch (NotSupportedException ex) + { + ErrorRecord er = new(ex, "WebCmdletIEDomNotSupportedException", ErrorCategory.NotImplemented, null); + ThrowTerminatingError(er); + } + } + + /// + /// To implement ^C. + /// + protected override void StopProcessing() => _cancelToken?.Cancel(); + + #endregion Overrides + #region Virtual Methods internal virtual void ValidateParameters() @@ -1651,186 +1831,6 @@ private static string FormatErrorMessage(string error, string contentType) } #endregion Helper Methods - - #region Overrides - - /// - /// The main execution method for cmdlets derived from WebRequestPSCmdlet. - /// - protected override void ProcessRecord() - { - try - { - // Set cmdlet context for write progress - ValidateParameters(); - PrepareSession(); - - // If the request contains an authorization header and PreserveAuthorizationOnRedirect is not set, - // it needs to be stripped on the first redirect. - bool keepAuthorizationOnRedirect = PreserveAuthorizationOnRedirect.IsPresent - && WebSession.Headers.ContainsKey(HttpKnownHeaderNames.Authorization); - - bool handleRedirect = keepAuthorizationOnRedirect || AllowInsecureRedirect; - - using (HttpClient client = GetHttpClient(handleRedirect)) - { - int followedRelLink = 0; - Uri uri = Uri; - do - { - if (followedRelLink > 0) - { - string linkVerboseMsg = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.FollowingRelLinkVerboseMsg, - uri.AbsoluteUri); - - WriteVerbose(linkVerboseMsg); - } - - using (HttpRequestMessage request = GetRequest(uri)) - { - FillRequestStream(request); - try - { - long requestContentLength = request.Content is null ? 0 : request.Content.Headers.ContentLength.Value; - - string reqVerboseMsg = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.WebMethodInvocationVerboseMsg, - request.Version, - request.Method, - requestContentLength); - - WriteVerbose(reqVerboseMsg); - - using HttpResponseMessage response = GetResponse(client, request, handleRedirect); - - string contentType = ContentHelper.GetContentType(response); - string respVerboseMsg = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.WebResponseVerboseMsg, - response.Content.Headers.ContentLength, - contentType); - - WriteVerbose(respVerboseMsg); - - bool _isSuccess = response.IsSuccessStatusCode; - - // Check if the Resume range was not satisfiable because the file already completed downloading. - // This happens when the local file is the same size as the remote file. - if (Resume.IsPresent - && response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable - && response.Content.Headers.ContentRange.HasLength - && response.Content.Headers.ContentRange.Length == _resumeFileSize) - { - _isSuccess = true; - WriteVerbose(string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.OutFileWritingSkipped, - OutFile)); - - // Disable writing to the OutFile. - OutFile = null; - } - - if (ShouldCheckHttpStatus && !_isSuccess) - { - string message = string.Format( - CultureInfo.CurrentCulture, - WebCmdletStrings.ResponseStatusCodeFailure, - (int)response.StatusCode, - response.ReasonPhrase); - - HttpResponseException httpEx = new(message, response); - ErrorRecord er = new(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); - string detailMsg = string.Empty; - StreamReader reader = null; - try - { - reader = new StreamReader(StreamHelper.GetResponseStream(response)); - detailMsg = FormatErrorMessage(reader.ReadToEnd(), contentType); - } - catch - { - // Catch all - } - finally - { - reader?.Dispose(); - } - - if (!string.IsNullOrEmpty(detailMsg)) - { - er.ErrorDetails = new ErrorDetails(detailMsg); - } - - ThrowTerminatingError(er); - } - - if (_parseRelLink || _followRelLink) - { - ParseLinkHeader(response, uri); - } - - ProcessResponse(response); - UpdateSession(response); - - // If we hit our maximum redirection count, generate an error. - // Errors with redirection counts of greater than 0 are handled automatically by .NET, but are - // impossible to detect programmatically when we hit this limit. By handling this ourselves - // (and still writing out the result), users can debug actual HTTP redirect problems. - if (WebSession.MaximumRedirection == 0 && IsRedirectCode(response.StatusCode)) - { - ErrorRecord er = new(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request); - er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded); - WriteError(er); - } - } - catch (HttpRequestException ex) - { - ErrorRecord er = new(ex, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); - if (ex.InnerException is not null) - { - er.ErrorDetails = new ErrorDetails(ex.InnerException.Message); - } - - ThrowTerminatingError(er); - } - - if (_followRelLink) - { - if (!_relationLink.ContainsKey("next")) - { - return; - } - - uri = new Uri(_relationLink["next"]); - followedRelLink++; - } - } - } - while (_followRelLink && (followedRelLink < _maximumFollowRelLink)); - } - } - catch (CryptographicException ex) - { - ErrorRecord er = new(ex, "WebCmdletCertificateException", ErrorCategory.SecurityError, null); - ThrowTerminatingError(er); - } - catch (NotSupportedException ex) - { - ErrorRecord er = new(ex, "WebCmdletIEDomNotSupportedException", ErrorCategory.NotImplemented, null); - ThrowTerminatingError(er); - } - } - - /// - /// To implement ^C. - /// - protected override void StopProcessing() => _cancelToken?.Cancel(); - - #endregion Overrides } // TODO: Merge Partials From 586680ab107b308e54751d265aafb5a00e02b91e Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:41:15 +0100 Subject: [PATCH 12/16] move 3 to helper methods --- .../Common/WebRequestPSCmdlet.Common.cs | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index a5ae22deecc..58883040926 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -1205,45 +1205,6 @@ internal virtual void FillRequestStream(HttpRequestMessage request) } } - // Returns true if the status code is one of the supported redirection codes. - private static bool IsRedirectCode(HttpStatusCode code) - { - int intCode = (int)code; - return - ( - (intCode >= 300 && intCode < 304) || - intCode == 307 || - intCode == 308 - ); - } - - // Returns true if the status code is a redirection code and the action requires switching from POST to GET on redirection. - // NOTE: Some of these status codes map to the same underlying value but spelling them out for completeness. - private static bool IsRedirectToGet(HttpStatusCode code) - { - return - ( - code == HttpStatusCode.Found || - code == HttpStatusCode.Moved || - code == HttpStatusCode.Redirect || - code == HttpStatusCode.RedirectMethod || - code == HttpStatusCode.SeeOther || - code == HttpStatusCode.Ambiguous || - code == HttpStatusCode.MultipleChoices - ); - } - - // Returns true if the status code shows a server or client error and MaximumRetryCount > 0 - private bool ShouldRetry(HttpStatusCode code) - { - int intCode = (int)code; - - return - ( - (intCode == 304 || (intCode >= 400 && intCode <= 599)) && WebSession.MaximumRetryCount > 0 - ); - } - internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestMessage request, bool handleRedirect) { ArgumentNullException.ThrowIfNull(client); @@ -1830,6 +1791,45 @@ private static string FormatErrorMessage(string error, string contentType) return formattedError; } + // Returns true if the status code is one of the supported redirection codes. + private static bool IsRedirectCode(HttpStatusCode code) + { + int intCode = (int)code; + return + ( + (intCode >= 300 && intCode < 304) || + intCode == 307 || + intCode == 308 + ); + } + + // Returns true if the status code is a redirection code and the action requires switching from POST to GET on redirection. + // NOTE: Some of these status codes map to the same underlying value but spelling them out for completeness. + private static bool IsRedirectToGet(HttpStatusCode code) + { + return + ( + code == HttpStatusCode.Found || + code == HttpStatusCode.Moved || + code == HttpStatusCode.Redirect || + code == HttpStatusCode.RedirectMethod || + code == HttpStatusCode.SeeOther || + code == HttpStatusCode.Ambiguous || + code == HttpStatusCode.MultipleChoices + ); + } + + // Returns true if the status code shows a server or client error and MaximumRetryCount > 0 + private bool ShouldRetry(HttpStatusCode code) + { + int intCode = (int)code; + + return + ( + (intCode == 304 || (intCode >= 400 && intCode <= 599)) && WebSession.MaximumRetryCount > 0 + ); + } + #endregion Helper Methods } From 9451e0992e61045484c6c18b31ea96d8da943d7b Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:42:18 +0100 Subject: [PATCH 13/16] move 1 to helper methods --- .../Common/WebRequestPSCmdlet.Common.cs | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 58883040926..25d1df60192 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -1829,7 +1829,20 @@ private bool ShouldRetry(HttpStatusCode code) (intCode == 304 || (intCode >= 400 && intCode <= 599)) && WebSession.MaximumRetryCount > 0 ); } - + + private static HttpMethod GetHttpMethod(WebRequestMethod method) => method switch + { + WebRequestMethod.Default or WebRequestMethod.Get => HttpMethod.Get, + WebRequestMethod.Delete => HttpMethod.Delete, + WebRequestMethod.Head => HttpMethod.Head, + WebRequestMethod.Patch => HttpMethod.Patch, + WebRequestMethod.Post => HttpMethod.Post, + WebRequestMethod.Put => HttpMethod.Put, + WebRequestMethod.Options => HttpMethod.Options, + WebRequestMethod.Trace => HttpMethod.Trace, + _ => new HttpMethod(method.ToString().ToUpperInvariant()) + }; + #endregion Helper Methods } @@ -1861,17 +1874,6 @@ public HttpResponseException(string message, HttpResponseMessage response) : bas /// public abstract partial class WebRequestPSCmdlet : PSCmdlet { - private static HttpMethod GetHttpMethod(WebRequestMethod method) => method switch - { - WebRequestMethod.Default or WebRequestMethod.Get => HttpMethod.Get, - WebRequestMethod.Delete => HttpMethod.Delete, - WebRequestMethod.Head => HttpMethod.Head, - WebRequestMethod.Patch => HttpMethod.Patch, - WebRequestMethod.Post => HttpMethod.Post, - WebRequestMethod.Put => HttpMethod.Put, - WebRequestMethod.Options => HttpMethod.Options, - WebRequestMethod.Trace => HttpMethod.Trace, - _ => new HttpMethod(method.ToString().ToUpperInvariant()) - }; + } } From ea35c7b40bf2fda085cb1092a2e9cf5938206f82 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:43:44 +0100 Subject: [PATCH 14/16] remove partial --- .../WebCmdlet/Common/WebRequestPSCmdlet.Common.cs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 25d1df60192..c19681b40bd 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -88,7 +88,7 @@ public enum WebSslProtocol /// /// Base class for Invoke-RestMethod and Invoke-WebRequest commands. /// - public abstract partial class WebRequestPSCmdlet : PSCmdlet + public abstract class WebRequestPSCmdlet : PSCmdlet { #region Fields @@ -1846,8 +1846,6 @@ private bool ShouldRetry(HttpStatusCode code) #endregion Helper Methods } - // TODO: Merge Partials - /// /// Exception class for webcmdlets to enable returning HTTP error response. /// @@ -1868,12 +1866,4 @@ public HttpResponseException(string message, HttpResponseMessage response) : bas /// public HttpResponseMessage Response { get; } } - - /// - /// Base class for Invoke-RestMethod and Invoke-WebRequest commands. - /// - public abstract partial class WebRequestPSCmdlet : PSCmdlet - { - - } } From 24c6e0e0f682a5110ff39e5bb9010d2c03939f66 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 17:52:04 +0100 Subject: [PATCH 15/16] remove double new line --- .../utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index c19681b40bd..f046d9812c9 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -930,7 +930,6 @@ internal virtual void PrepareSession() WebSession.RetryIntervalInSeconds = RetryIntervalSec; } } - internal virtual HttpClient GetHttpClient(bool handleRedirect) { From c231e65229423bd54d434fac8d266818393f08f9 Mon Sep 17 00:00:00 2001 From: CarloToso <105941898+CarloToso@users.noreply.github.com> Date: Thu, 9 Feb 2023 18:35:32 +0100 Subject: [PATCH 16/16] remove extra spaces --- .../utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index f046d9812c9..a9c5934ecd3 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -491,7 +491,7 @@ public virtual string CustomMethod #endregion Abstract Methods - #region Overrides + #region Overrides /// /// The main execution method for cmdlets derived from WebRequestPSCmdlet.