diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs index da58abfc054..0559e35daa5 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/InvokeRestMethodCommand.Common.cs @@ -37,6 +37,29 @@ public override string CustomMethod set { base.CustomMethod = value; } } + /// + /// enable automatic following of rel links + /// + [Parameter] + [Alias("FL")] + public SwitchParameter FollowRelLink + { + get { return base._followRelLink; } + set { base._followRelLink = value; } + } + + /// + /// gets or sets the maximum number of rel links to follow + /// + [Parameter] + [Alias("ML")] + [ValidateRange(1, Int32.MaxValue)] + public int MaximumFollowRelLink + { + get { return base._maximumFollowRelLink; } + set { base._maximumFollowRelLink = value; } + } + #endregion Parameters #region Helper Methods diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs index 3dbfbeaae80..ebd8a659c3d 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/InvokeWebRequestCommand.CoreClr.cs @@ -20,6 +20,14 @@ public class InvokeWebRequestCommand : WebRequestPSCmdlet { #region Virtual Method Overrides + /// + /// Default constructor for InvokeWebRequestCommand + /// + public InvokeWebRequestCommand() : base() + { + this._parseRelLink = true; + } + /// /// Process the web response and output corresponding objects. /// @@ -46,6 +54,7 @@ internal override void ProcessResponse(HttpResponseMessage response) // creating a MemoryStream wrapper to response stream here to support IsStopping. responseStream = new WebResponseContentMemoryStream(responseStream, StreamHelper.ChunkSize, this); WebResponseObject ro = WebResponseObjectFactory.GetResponseObject(response, responseStream, this.Context, UseBasicParsing); + ro.RelationLink = _relationLink; WriteObject(ro); // use the rawcontent stream from WebResponseObject for further diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebRequestPSCmdlet.CoreClr.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebRequestPSCmdlet.CoreClr.cs index 47e83f69775..9a74abe7b00 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebRequestPSCmdlet.CoreClr.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebRequestPSCmdlet.CoreClr.cs @@ -16,6 +16,9 @@ using System.Security.Cryptography; using System.Threading; using System.Xml; +using System.Collections.Generic; +using System.Text.RegularExpressions; +using System.Linq; namespace Microsoft.PowerShell.Commands { @@ -61,6 +64,26 @@ public abstract partial class WebRequestPSCmdlet : PSCmdlet /// private CancellationTokenSource _cancelToken = null; + /// + /// Parse Rel Links + /// + internal bool _parseRelLink = false; + + /// + /// Automatically follow Rel Links + /// + internal bool _followRelLink = false; + + /// + /// Automatically follow Rel Links + /// + internal Dictionary _relationLink = null; + + /// + /// Maximum number of Rel Links to follow + /// + internal int _maximumFollowRelLink = Int32.MaxValue; + private HttpMethod GetHttpMethod(WebRequestMethod method) { switch (Method) @@ -234,7 +257,7 @@ internal virtual HttpRequestMessage GetRequest(Uri uri) } // Some web sites (e.g. Twitter) will return exception on POST when Expect100 is sent - // Default behaviour is continue to send body content anyway after a short period + // Default behavior is continue to send body content anyway after a short period // Here it send the two part as a whole. request.Headers.ExpectContinue = false; @@ -374,91 +397,121 @@ protected override void ProcessRecord() PrepareSession(); using (HttpClient client = GetHttpClient()) - using (HttpRequestMessage request = GetRequest(Uri)) { - FillRequestStream(request); - try + int followedRelLink = 0; + Uri uri = Uri; + do { - long requestContentLength = 0; - if (request.Content != null) - requestContentLength = request.Content.Headers.ContentLength.Value; - - string reqVerboseMsg = String.Format(CultureInfo.CurrentCulture, - "{0} {1} with {2}-byte payload", - request.Method, - request.RequestUri, - requestContentLength); - WriteVerbose(reqVerboseMsg); - - HttpResponseMessage response = GetResponse(client, request); - - string contentType = ContentHelper.GetContentType(response); - string respVerboseMsg = string.Format(CultureInfo.CurrentCulture, - "received {0}-byte response of content type {1}", - response.Content.Headers.ContentLength, - contentType); - WriteVerbose(respVerboseMsg); - - if (!response.IsSuccessStatusCode) + if (followedRelLink > 0) { - string message = String.Format(CultureInfo.CurrentCulture, WebCmdletStrings.ResponseStatusCodeFailure, - (int)response.StatusCode, response.ReasonPhrase); - HttpResponseException httpEx = new HttpResponseException(message, response); - ErrorRecord er = new ErrorRecord(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); - string detailMsg = ""; - StreamReader reader = null; + string linkVerboseMsg = string.Format(CultureInfo.CurrentCulture, + WebCmdletStrings.FollowingRelLinkVerboseMsg, + uri.AbsoluteUri); + WriteVerbose(linkVerboseMsg); + } + + using (HttpRequestMessage request = GetRequest(uri)) + { + FillRequestStream(request); try { - reader = new StreamReader(StreamHelper.GetResponseStream(response)); - // remove HTML tags making it easier to read - detailMsg = System.Text.RegularExpressions.Regex.Replace(reader.ReadToEnd(), "<[^>]*>",""); - } - catch (Exception) - { - // catch all - } - finally - { - if (reader != null) + long requestContentLength = 0; + if (request.Content != null) + requestContentLength = request.Content.Headers.ContentLength.Value; + + string reqVerboseMsg = String.Format(CultureInfo.CurrentCulture, + WebCmdletStrings.WebMethodInvocationVerboseMsg, + request.Method, + request.RequestUri, + requestContentLength); + WriteVerbose(reqVerboseMsg); + + HttpResponseMessage response = GetResponse(client, request); + + string contentType = ContentHelper.GetContentType(response); + string respVerboseMsg = string.Format(CultureInfo.CurrentCulture, + WebCmdletStrings.WebResponseVerboseMsg, + response.Content.Headers.ContentLength, + contentType); + WriteVerbose(respVerboseMsg); + + if (!response.IsSuccessStatusCode) { - reader.Dispose(); + string message = String.Format(CultureInfo.CurrentCulture, WebCmdletStrings.ResponseStatusCodeFailure, + (int)response.StatusCode, response.ReasonPhrase); + HttpResponseException httpEx = new HttpResponseException(message, response); + ErrorRecord er = new ErrorRecord(httpEx, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); + string detailMsg = ""; + StreamReader reader = null; + try + { + reader = new StreamReader(StreamHelper.GetResponseStream(response)); + // remove HTML tags making it easier to read + detailMsg = System.Text.RegularExpressions.Regex.Replace(reader.ReadToEnd(), "<[^>]*>",""); + } + catch (Exception) + { + // catch all + } + finally + { + if (reader != null) + { + reader.Dispose(); + } + } + if (!String.IsNullOrEmpty(detailMsg)) + { + er.ErrorDetails = new ErrorDetails(detailMsg); + } + ThrowTerminatingError(er); + } + + if (_parseRelLink || _followRelLink) + { + ParseLinkHeader(response, uri); + } + ProcessResponse(response); + UpdateSession(response); + + // If we hit our maximum redirection count, generate an error. + // Errors with redirection counts of greater than 0 are handled automatically by .NET, but are + // impossible to detect programmatically when we hit this limit. By handling this ourselves + // (and still writing out the result), users can debug actual HTTP redirect problems. + if (WebSession.MaximumRedirection == 0) // Indicate "HttpClientHandler.AllowAutoRedirect == false" + { + if (response.StatusCode == HttpStatusCode.Found || + response.StatusCode == HttpStatusCode.Moved || + response.StatusCode == HttpStatusCode.MovedPermanently) + { + ErrorRecord er = new ErrorRecord(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request); + er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded); + WriteError(er); + } } } - if (!String.IsNullOrEmpty(detailMsg)) + catch (HttpRequestException ex) { - er.ErrorDetails = new ErrorDetails(detailMsg); + ErrorRecord er = new ErrorRecord(ex, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); + if (ex.InnerException != null) + { + er.ErrorDetails = new ErrorDetails(ex.InnerException.Message); + } + ThrowTerminatingError(er); } - ThrowTerminatingError(er); - } - - ProcessResponse(response); - UpdateSession(response); - // If we hit our maximum redirection count, generate an error. - // Errors with redirection counts of greater than 0 are handled automatically by .NET, but are - // impossible to detect programmatically when we hit this limit. By handling this ourselves - // (and still writing out the result), users can debug actual HTTP redirect problems. - if (WebSession.MaximumRedirection == 0) // Indicate "HttpClientHandler.AllowAutoRedirect == false" - { - if (response.StatusCode == HttpStatusCode.Found || - response.StatusCode == HttpStatusCode.Moved || - response.StatusCode == HttpStatusCode.MovedPermanently) + if (_followRelLink) { - ErrorRecord er = new ErrorRecord(new InvalidOperationException(), "MaximumRedirectExceeded", ErrorCategory.InvalidOperation, request); - er.ErrorDetails = new ErrorDetails(WebCmdletStrings.MaximumRedirectionCountExceeded); - WriteError(er); + if (!_relationLink.ContainsKey("next")) + { + return; + } + uri = new Uri(_relationLink["next"]); + followedRelLink++; } } } - catch (HttpRequestException ex) - { - ErrorRecord er = new ErrorRecord(ex, "WebCmdletWebResponseException", ErrorCategory.InvalidOperation, request); - if (ex.InnerException != null) - { - er.ErrorDetails = new ErrorDetails(ex.InnerException.Message); - } - ThrowTerminatingError(er); - } + while (_followRelLink && (followedRelLink < _maximumFollowRelLink)); } } catch (CryptographicException ex) @@ -619,6 +672,40 @@ internal long SetRequestContent(HttpRequestMessage request, IDictionary content) } + internal void ParseLinkHeader(HttpResponseMessage response, System.Uri requestUri) + { + if (_relationLink == null) + { + _relationLink = new Dictionary(); + } + else + { + _relationLink.Clear(); + } + + // we only support the URL in angle brackets and `rel`, other attributes are ignored + // user can still parse it themselves via the Headers property + string pattern = "<(?.*?)>;\\srel=\"(?.*?)\""; + IEnumerable links; + if (response.Headers.TryGetValues("Link", out links)) + { + foreach(string link in links.FirstOrDefault().Split(",")) + { + Match match = Regex.Match(link, pattern); + if (match.Success) + { + string url = match.Groups["url"].Value; + string rel = match.Groups["rel"].Value; + if (url != String.Empty && rel != String.Empty && !_relationLink.ContainsKey(rel)) + { + Uri absoluteUri = new Uri(requestUri, url); + _relationLink.Add(rel, absoluteUri.AbsoluteUri.ToString()); + } + } + } + } + } + #endregion Helper Methods } } diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebResponseObject.CoreClr.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebResponseObject.CoreClr.cs index 4413c6d73ec..79fc400af7f 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebResponseObject.CoreClr.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/CoreCLR/WebResponseObject.CoreClr.cs @@ -41,6 +41,11 @@ public Dictionary> Headers } } + /// + /// gets the RelationLink property + /// + public Dictionary RelationLink { get; internal set; } + #endregion #region Constructors diff --git a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx index 5fe4258353a..f39548a3cce 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx +++ b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx @@ -213,4 +213,13 @@ Response status code does not indicate success: {0} ({1}). - \ No newline at end of file + + Following rel link {0} + + + {0} {1} with {2}-byte payload + + + received {0}-byte response of content type {1} + + diff --git a/src/System.Management.Automation/commands/utility/FormatAndOutput/common/DefaultFormatters/PowerShellCore_format_ps1xml.cs b/src/System.Management.Automation/commands/utility/FormatAndOutput/common/DefaultFormatters/PowerShellCore_format_ps1xml.cs index 5e4e1893ff3..c2fb419b8ff 100644 --- a/src/System.Management.Automation/commands/utility/FormatAndOutput/common/DefaultFormatters/PowerShellCore_format_ps1xml.cs +++ b/src/System.Management.Automation/commands/utility/FormatAndOutput/common/DefaultFormatters/PowerShellCore_format_ps1xml.cs @@ -1271,6 +1271,7 @@ private static IEnumerable ViewsOf_Microsoft_PowerShell_Co .AddItemProperty(@"Links") .AddItemProperty(@"ParsedHtml") .AddItemProperty(@"RawContentLength") + .AddItemProperty(@"RelationLink") .EndEntry() .EndList()); } @@ -1291,6 +1292,7 @@ private static IEnumerable ViewsOf_Microsoft_PowerShell_Co ", label: "RawContent") .AddItemProperty(@"Headers") .AddItemProperty(@"RawContentLength") + .AddItemProperty(@"RelationLink") .EndEntry() .EndList()); } diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 index d4c2d524d74..978dce66d7c 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 @@ -145,6 +145,14 @@ function GetTestData Describe "Invoke-WebRequest tests" -Tags "Feature" { + BeforeAll { + $null = Start-HttpListener -AsJob + } + + AfterAll { + $null = Stop-HttpListener + } + # Validate the output of Invoke-WebRequest # function ValidateResponse @@ -509,6 +517,35 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { $result.Error.FullyQualifiedErrorId | Should Be "WebCmdletWebResponseException,Microsoft.PowerShell.Commands.InvokeWebRequestCommand" } + It "Validate Invoke-WebRequest returns empty RelationLink property if there is no Link Header" { + + $command = "Invoke-WebRequest -Uri http://localhost:8080/PowerShell?test=response" + $result = ExecuteWebCommand -command $command + + $result.Output.RelationLink.Count | Should Be 0 + } + + It "Validate Invoke-WebRequest returns valid RelationLink property with absolute uris if Link Header is present" { + + $command = "Invoke-WebRequest -Uri 'http://localhost:8080/PowerShell?test=linkheader&maxlinks=5'" + $result = ExecuteWebCommand -command $command + $result.Output.RelationLink.Count | Should BeExactly 2 + $result.Output.RelationLink["next"] | Should BeExactly "http://localhost:8080/PowerShell?test=linkheader&maxlinks=5&linknumber=2" + $result.Output.RelationLink["last"] | Should BeExactly "http://localhost:8080/PowerShell?test=linkheader&maxlinks=5&linknumber=5" + } + + It "Validate Invoke-WebRequest quietly ignores invalid Link Headers in RelationLink property" -TestCases @( + @{ type = "noUrl" } + @{ type = "malformed" } + @{ type = "noRel" } + ) { + param($type) + $command = "Invoke-WebRequest -Uri 'http://localhost:8080/PowerShell?test=linkheader&type=$type'" + $result = ExecuteWebCommand -command $command + $result.Output.RelationLink.Count | Should BeExactly 1 + $result.Output.RelationLink["last"] | Should BeExactly "http://localhost:8080/PowerShell?test=linkheader&maxlinks=3&linknumber=3" + } + BeforeEach { if ($env:http_proxy) { $savedHttpProxy = $env:http_proxy @@ -538,6 +575,14 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { Describe "Invoke-RestMethod tests" -Tags "Feature" { + BeforeAll { + $null = Start-HttpListener -AsJob + } + + AfterAll { + $null = Stop-HttpListener + } + It "Invoke-RestMethod returns User-Agent" { $command = "Invoke-RestMethod -Uri http://httpbin.org/user-agent -TimeoutSec 5" @@ -872,6 +917,46 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { $result.Error.FullyQualifiedErrorId | Should Be "WebCmdletWebResponseException,Microsoft.PowerShell.Commands.InvokeRestMethodCommand" } + It "Validate Invoke-RestMethod -FollowRelLink doesn't fail if no Link Header is present" { + + $command = "Invoke-RestMethod -Uri 'http://localhost:8080/PowerShell?test=response&output=foo' -FollowRelLink" + $result = ExecuteWebCommand -command $command + + $result.Output | Should BeExactly "foo" + } + + It "Validate Invoke-RestMethod -FollowRelLink correctly follows all the available relation links" { + $maxLinks = 5 + + $command = "Invoke-RestMethod -Uri 'http://localhost:8080/PowerShell?test=linkheader&maxlinks=$maxlinks' -FollowRelLink" + $result = ExecuteWebCommand -command $command + + $result.Output.output.Count | Should BeExactly $maxLinks + 1..$maxLinks | ForEach-Object { $result.Output.output[$_ - 1] | Should BeExactly $_ } + } + + It "Validate Invoke-RestMethod -FollowRelLink correctly limits to -MaximumRelLink" { + $maxLinks = 10 + $maxLinksToFollow = 6 + + $command = "Invoke-RestMethod -Uri 'http://localhost:8080/PowerShell?test=linkheader&maxlinks=$maxlinks' -FollowRelLink -MaximumFollowRelLink $maxLinksToFollow" + $result = ExecuteWebCommand -command $command + + $result.Output.output.Count | Should BeExactly $maxLinksToFollow + 1..$maxLinksToFollow | ForEach-Object { $result.Output.output[$_ - 1] | Should BeExactly $_ } + } + + It "Validate Invoke-RestMethod quietly ignores invalid Link Headers if -FollowRelLink is specified" -TestCases @( + @{ type = "noUrl" } + @{ type = "malformed" } + @{ type = "noRel" } + ) { + param($type) + $command = "Invoke-RestMethod -Uri 'http://localhost:8080/PowerShell?test=linkheader&type=$type' -FollowRelLink" + $result = ExecuteWebCommand -command $command + $result.Output.output | Should BeExactly 1 + } + BeforeEach { if ($env:http_proxy) { $savedHttpProxy = $env:http_proxy diff --git a/test/tools/Modules/HttpListener/HttpListener.psm1 b/test/tools/Modules/HttpListener/HttpListener.psm1 index 45157b2fa80..1d6a795d26d 100644 --- a/test/tools/Modules/HttpListener/HttpListener.psm1 +++ b/test/tools/Modules/HttpListener/HttpListener.psm1 @@ -136,6 +136,51 @@ Function Start-HTTPListener { $output = $request | ConvertTo-Json } } + "linkheader" + { + $maxLinks = $queryItems["maxlinks"] + if ($maxlinks -eq $null) + { + $maxLinks = 3 + } + $linkNumber = [int]$queryItems["linknumber"] + $prev = "" + if ($linkNumber -eq 0) + { + $linkNumber = 1 + } + else + { + # use $urlPrefix to ensure output is resolved to absolute uri + $prev = ", <$($urlPrefix)?test=linkheader&maxlinks=$maxlinks&linknumber=$($linkNumber-1); rel=`"prev`"" + } + $links = "" + if ($linkNumber -lt $maxLinks) + { + switch ($queryItems["type"]) + { + "noUrl" + { + $links = "<>; rel=`"next`"," + } + "malformed" + { + $links = "{url}; foo," + } + "noRel" + { + $links = "; foo=`"bar`"," + } + default + { + $links = "<$($urlPrefix)?test=linkheader&maxlinks=$maxlinks&linknumber=$($linkNumber+1)>; rel=`"next`", " + } + } + } + $links = "$links<$($urlPrefix)?test=linkheader&maxlinks=$maxlinks&linknumber=$maxlinks>; rel=`"last`"$prev" + $outputHeader.Add("Link", $links) + $output = "{ `"output`": `"$linkNumber`"}" + } default { $statusCode = [System.Net.HttpStatusCode]::NotFound