From c5a2e2e8eb96a6454f35d4a9b005bf441571b15e Mon Sep 17 00:00:00 2001 From: Mark Kraus Date: Wed, 21 Mar 2018 03:43:12 -0500 Subject: [PATCH 1/4] [Feature] Add -Resume Feature to Web Cmdlets --- .spelling | 2 + .../Common/WebRequestPSCmdlet.Common.cs | 106 +++++++- .../utility/WebCmdlet/StreamHelper.cs | 15 +- .../resources/WebCmdletStrings.resx | 6 + .../WebCmdlets.Tests.ps1 | 230 ++++++++++++++++++ .../Modules/WebListener/WebListener.psm1 | 1 + .../Controllers/ResumeController.cs | 120 +++++++++ test/tools/WebListener/README.md | 69 ++++++ test/tools/WebListener/Startup.cs | 4 + 9 files changed, 549 insertions(+), 4 deletions(-) create mode 100644 test/tools/WebListener/Controllers/ResumeController.cs diff --git a/.spelling b/.spelling index 77cc97a19fd..64e31b6317d 100644 --- a/.spelling +++ b/.spelling @@ -1104,6 +1104,8 @@ v6.0. #region test/tools/WebListener/README.md Overrides - test/tools/WebListener/README.md Auth +NoResume NTLM +NumberBytes ResponseHeaders #endregion diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index f5c3cd3d322..1229cb6d432 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -354,6 +354,12 @@ public virtual string CustomMethod [Parameter] public virtual SwitchParameter PassThru { get; set; } + /// + /// Resumes downloading a partial or incomplete file. OutFile is required. + /// + [Parameter] + public virtual SwitchParameter Resume { get; set; } + #endregion #endregion Virtual Properties @@ -515,6 +521,14 @@ internal virtual void ValidateParameters() "WebCmdletOutFileMissingException"); ThrowTerminatingError(error); } + + // Resume requires OutFile. + if (Resume.IsPresent && OutFile == null) + { + ErrorRecord error = GetValidationError(WebCmdletStrings.ResumeOutFileMissing, + "WebCmdletOutFileMissingException"); + ThrowTerminatingError(error); + } } internal virtual void PrepareSession() @@ -637,6 +651,14 @@ internal bool ShouldWriteToPipeline get { return (!ShouldSaveToOutFile || PassThru); } } + /// + /// Determines whether writing to a file should Resume and append rather than overwrite. + /// + internal bool ShouldResume + { + get { return (Resume.IsPresent && _resumeSuccess); } + } + #endregion Helper Properties #region Helper Methods @@ -860,6 +882,16 @@ public abstract partial class WebRequestPSCmdlet : PSCmdlet /// internal int _maximumFollowRelLink = Int32.MaxValue; + /// + /// The remote endpoint returned a 206 status code indicating successful resume. + /// + private bool _resumeSuccess = false; + + /// + /// The current size of the local file being resumed. + /// + private long _resumeFileSize = 0; + private HttpMethod GetHttpMethod(WebRequestMethod method) { switch (Method) @@ -1062,6 +1094,22 @@ internal virtual HttpRequestMessage GetRequest(Uri uri, bool stripAuthorization) } } + // If the file to resume downloading exists, create the Range request header using the file size. + // If not, create a Range to request the entire file. + if (Resume.IsPresent) + { + var fileInfo = new FileInfo(QualifiedOutFile); + if (fileInfo.Exists) + { + request.Headers.Range = new RangeHeaderValue(fileInfo.Length, null); + _resumeFileSize = fileInfo.Length; + } + else + { + request.Headers.Range = new RangeHeaderValue(0, null); + } + } + // Some web sites (e.g. Twitter) will return exception on POST when Expect100 is sent // Default behavior is continue to send body content anyway after a short period // Here it send the two part as a whole. @@ -1233,6 +1281,9 @@ internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestM if (client == null) { throw new ArgumentNullException("client"); } if (request == null) { throw new ArgumentNullException("request"); } + // Track the current URI being used by various requests and re-requests. + var currentUri = request.RequestUri; + _cancelToken = new CancellationTokenSource(); HttpResponseMessage response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult(); @@ -1256,14 +1307,51 @@ internal virtual HttpResponseMessage GetResponse(HttpClient client, HttpRequestM } // recreate the HttpClient with redirection enabled since the first call suppressed redirection + currentUri = new Uri(request.RequestUri, response.Headers.Location); using (client = GetHttpClient(false)) - using (HttpRequestMessage redirectRequest = GetRequest(new Uri(request.RequestUri, response.Headers.Location), stripAuthorization:true)) + using (HttpRequestMessage redirectRequest = GetRequest(currentUri, stripAuthorization:true)) { FillRequestStream(redirectRequest); _cancelToken = new CancellationTokenSource(); response = client.SendAsync(redirectRequest, HttpCompletionOption.ResponseHeadersRead, _cancelToken.Token).GetAwaiter().GetResult(); } } + + // Request again without the Range header because the server indicated the range was not satisfiable. + // This happens when the local file is larger than the remote file. + // If the size of the remote file is the same as the local file, there is nothing to resume. + if (Resume.IsPresent && + response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable && + (response.Content.Headers.ContentRange.HasLength && + response.Content.Headers.ContentRange.Length != _resumeFileSize)) + { + _cancelToken.Cancel(); + + WriteVerbose(WebCmdletStrings.WebMethodResumeFailedVerboseMsg); + + // Disable the Resume switch so the subsequent calls to GetResponse() and FillRequestStream() + // are treated as a standard -OutFile request. This also disables appending local file. + Resume = new SwitchParameter(false); + + using (HttpRequestMessage requestWithoutRange = GetRequest(currentUri, stripAuthorization:false)) + { + FillRequestStream(requestWithoutRange); + long requestContentLength = 0; + if (requestWithoutRange.Content != null) + requestContentLength = requestWithoutRange.Content.Headers.ContentLength.Value; + + string reqVerboseMsg = String.Format(CultureInfo.CurrentCulture, + WebCmdletStrings.WebMethodInvocationVerboseMsg, + requestWithoutRange.Method, + requestWithoutRange.RequestUri, + requestContentLength); + WriteVerbose(reqVerboseMsg); + + return GetResponse(client, requestWithoutRange, stripAuthorization); + } + } + + _resumeSuccess = response.StatusCode == HttpStatusCode.PartialContent; return response; } @@ -1336,7 +1424,21 @@ protected override void ProcessRecord() contentType); WriteVerbose(respVerboseMsg); - if (!response.IsSuccessStatusCode) + bool _isSuccess = response.IsSuccessStatusCode; + + // Check if the Resume range was not satisfiable because the file already completed downloading. + // This happens when the local file is the same size as the remote file. + if (Resume.IsPresent && + response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable && + response.Content.Headers.ContentRange.HasLength && + response.Content.Headers.ContentRange.Length == _resumeFileSize) + { + _isSuccess = true; + // Disable writing to the OutFile. + OutFile = null; + } + + if (!_isSuccess) { string message = String.Format(CultureInfo.CurrentCulture, WebCmdletStrings.ResponseStatusCodeFailure, (int)response.StatusCode, response.ReasonPhrase); diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs index cd02e225332..5c60c4058a1 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/StreamHelper.cs @@ -342,9 +342,20 @@ internal static void WriteToStream(byte[] input, Stream output) /// internal static void SaveStreamToFile(Stream stream, string filePath, PSCmdlet cmdlet) { - using (FileStream output = File.Create(filePath)) + // If the web cmdlet should resume, append the file instead of overwriting. + if(cmdlet is WebRequestPSCmdlet webCmdlet && webCmdlet.ShouldResume) { - WriteToStream(stream, output, cmdlet); + using (FileStream output = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.Read)) + { + WriteToStream(stream, output, cmdlet); + } + } + else + { + using (FileStream output = File.Create(filePath)) + { + WriteToStream(stream, output, cmdlet); + } } } diff --git a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx index 82958bbe76a..f3e3bc585d7 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx +++ b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx @@ -189,6 +189,9 @@ The cmdlet cannot run because the following parameter is missing: OutFile. Provide a valid OutFile parameter value when using the PassThru parameter, then retry. + + The cmdlet cannot run because the following parameter is missing: OutFile. Provide a valid OutFile parameter value when using the Resume parameter, then retry. + The cmdlet cannot run because the following conflicting parameters are specified: ProxyCredential and ProxyUseDefaultCredentials. Specify either ProxyCredential or ProxyUseDefaultCredentials, then retry. @@ -249,6 +252,9 @@ {0} {1} with {2}-byte payload + + The remote server indicated it could not resume downloading. This often occurs when the local file and remote file are different sizes. The local file will be overwritten. + received {0}-byte response of content type {1} diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 index 350cc1f07cd..a08c8b29de6 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 @@ -1618,6 +1618,122 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { } } + Context "Invoke-WebRequest File Resume Feature" { + BeforeAll { + $outFile = Join-Path $TestDrive "resume.txt" + + # Download the entire file to reference in tests + $referenceFile = Join-Path $TestDrive "reference.txt" + $uri = Get-WebListenerUrl -Test 'Resume' + Invoke-WebRequest -uri $uri -OutFile $referenceFile -ErrorAction Stop + $referenceFileHash = Get-FileHash -Algorithm SHA256 -Path $referenceFile + $referenceFileSize = Get-Item $referenceFile | Select-Object -ExpandProperty Length + } + + AfterEach { + Remove-Item -Force -ErrorAction 'SilentlyContinue' -Path $outFile + } + + It "Invoke-WebRequest -Resume requires -OutFile" { + $uri = Get-WebListenerUrl -Test 'Resume' + + { Invoke-WebRequest -Resume -Uri $uri -ErrorAction Stop } | + Should -Throw -ErrorId 'WebCmdletOutFileMissingException,Microsoft.PowerShell.Commands.InvokeWebRequestCommand' + } + + It "Invoke-WebRequest -Resume Downloads the whole file when the file does not exist" { + $uri = Get-WebListenerUrl -Test 'Resume' + $response = Invoke-WebRequest -uri $uri -OutFile $outFile -Resume -PassThru + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'true' + $response.Headers.'X-Request-Range'[0] | Should -BeExactly 'bytes=0-' + $response.StatusCode | Should -BeExactly 206 + $response.Headers.'Content-Range'[0] | Should -BeExactly "bytes 0-$($referenceFileSize-1)/$referenceFileSize" + } + + It "Invoke-WebRequest -Resume overwrites an existing file that is larger than the remote file" { + # Create a file larger than the download file + $largerFileSize = $referenceFileSize + 20 + 1..$largerFileSize | ForEach-Object { [Byte]$_ } | Set-Content -AsByteStream $outFile + $largerFileSize = Get-Item $outFile | Select-Object -ExpandProperty Length + + $uri = Get-WebListenerUrl -Test 'Resume' + $response = Invoke-WebRequest -uri $uri -OutFile $outFile -Resume -PassThru + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -BeLessThan $largerFileSize + $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'false' + $response.StatusCode | Should -BeExactly 200 + $response.Headers.ContainsKey('Content-Range') | Should -BeFalse + } + + It "Invoke-WebRequest -Resume overwrites existing file when remote server does not support resume" { + # Create a file larger than the download file + $largerFileSize = $referenceFileSize + 20 + 1..$largerFileSize | ForEach-Object { [Byte]$_ } | Set-Content -AsByteStream $outFile + $largerFileSize = Get-Item $outFile | Select-Object -ExpandProperty Length + + $uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'NoResume' + $response = Invoke-WebRequest -Uri $uri -OutFile $outFile -Resume -PassThru + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'true' + $response.Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$largerFileSize-" + $response.StatusCode | Should -BeExactly 200 + $response.Headers.ContainsKey('Content-Range') | Should -BeFalse + } + + It "Invoke-WebRequest -Resume resumes downloading from bytes" -TestCases @( + @{bytes = 4} + @{bytes = 8} + @{bytes = 12} + @{bytes = 16} + ) { + param($bytes, $statuscode) + # Simulate partial download + $uri = Get-WebListenerUrl -Test 'Resume' -TestValue "Bytes/$bytes" + $null = Invoke-WebRequest -uri $uri -OutFile $outFile + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $bytes + + $uri = Get-WebListenerUrl -Test 'Resume' + $response = Invoke-WebRequest -uri $uri -OutFile $outFile -Resume -PassThru + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'true' + $response.Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$bytes-" + $response.StatusCode | Should -BeExactly 206 + $response.Headers.'Content-Range'[0] | Should -BeExactly "bytes $bytes-$($referenceFileSize-1)/$referenceFileSize" + } + + It "Invoke-WebRequest -Resume assumes the file was successfully completed when the local and remote file are the same size." { + # Download the entire file + $uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'NoResume' + $null = Invoke-WebRequest -uri $uri -OutFile $outFile + $fileSize = Get-Item $outFile | Select-Object -ExpandProperty Length + + $uri = Get-WebListenerUrl -Test 'Resume' + $response = Invoke-WebRequest -uri $uri -OutFile $outFile -Resume -PassThru + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'true' + $response.Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$fileSize-" + # The web cmdlets special case 416 as a success code when the local file and remote file are the same size + $response.StatusCode | Should -BeExactly 416 + $response.Headers.'Content-Range'[0] | Should -BeExactly "bytes */$referenceFileSize" + } + } + BeforeEach { if ($env:http_proxy) { $savedHttpProxy = $env:http_proxy @@ -2779,6 +2895,120 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { } } + Context "Invoke-RestMethod File Resume Feature" { + BeforeAll { + $outFile = Join-Path $TestDrive "resume.txt" + + # Download the entire file to reference in tests + $referenceFile = Join-Path $TestDrive "reference.txt" + $uri = Get-WebListenerUrl -Test 'Resume' + Invoke-RestMethod -uri $uri -OutFile $referenceFile -ErrorAction Stop + $referenceFileHash = Get-FileHash -Algorithm SHA256 -Path $referenceFile + $referenceFileSize = Get-Item $referenceFile | Select-Object -ExpandProperty Length + } + + AfterEach { + Remove-Item -Force -ErrorAction 'SilentlyContinue' -Path $outFile + } + + It "Invoke-RestMethod -Resume requires -OutFile" { + $uri = Get-WebListenerUrl -Test 'Resume' + + { Invoke-RestMethod -Resume -Uri $uri -ErrorAction Stop } | + Should -Throw -ErrorId 'WebCmdletOutFileMissingException,Microsoft.PowerShell.Commands.InvokeRestMethodCommand' + } + + It "Invoke-RestMethod -Resume Downloads the whole file when the file does not exist" { + # ensure the file does not exist + Remove-Item -Force -ErrorAction 'SilentlyContinue' -Path $outFile + + $uri = Get-WebListenerUrl -Test 'Resume' + Invoke-RestMethod -uri $uri -OutFile $outFile -RHV 'Headers' -Resume + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $Headers.'X-Has-Range'[0] | Should -BeExactly 'true' + $Headers.'X-Request-Range'[0] | Should -BeExactly 'bytes=0-' + $Headers.'Content-Range'[0] | Should -BeExactly "bytes 0-$($referenceFileSize-1)/$referenceFileSize" + } + + It "Invoke-RestMethod -Resume overwrites an existing file that is larger than the remote file" { + # Create a file larger than the download file + $largerFileSize = $referenceFileSize + 20 + 1..$largerFileSize | ForEach-Object { [Byte]$_ } | Set-Content -AsByteStream $outFile + $largerFileSize = Get-Item $outFile | Select-Object -ExpandProperty Length + + $uri = Get-WebListenerUrl -Test 'Resume' + $response = Invoke-RestMethod -uri $uri -OutFile $outFile -RHV 'Headers' -Resume + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -BeLessThan $largerFileSize + $Headers.'X-Has-Range'[0] | Should -BeExactly 'false' + $Headers.ContainsKey('Content-Range') | Should -BeFalse + } + + It "Invoke-RestMethod -Resume overwrites existing file when remote server does not support resume" { + # Create a file larger than the download file + $largerFileSize = $referenceFileSize + 20 + 1..$largerFileSize | ForEach-Object { [Byte]$_ } | Set-Content -AsByteStream $outFile + $largerFileSize = Get-Item $outFile | Select-Object -ExpandProperty Length + + $uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'NoResume' + $response = Invoke-RestMethod -uri $uri -OutFile $outFile -RHV 'Headers' -Resume + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -BeLessThan $largerFileSize + $Headers.'X-Has-Range'[0] | Should -BeExactly 'true' + $Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$largerFileSize-" + $Headers.ContainsKey('Content-Range') | Should -BeFalse + } + + It "Invoke-RestMethod -Resume resumes downloading from bytes" -TestCases @( + @{bytes = 4} + @{bytes = 8} + @{bytes = 12} + @{bytes = 16} + ) { + param($bytes) + # Simulate partial download + $uri = Get-WebListenerUrl -Test 'Resume' -TestValue "Bytes/$bytes" + $null = Invoke-RestMethod -uri $uri -OutFile $outFile + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $bytes + + $uri = Get-WebListenerUrl -Test 'Resume' + $response = Invoke-RestMethod -uri $uri -OutFile $outFile -RHV 'Headers' -Resume + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $Headers.'X-Has-Range'[0] | Should -BeExactly 'true' + $Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$bytes-" + $Headers.'Content-Range'[0] | Should -BeExactly "bytes $bytes-$($referenceFileSize-1)/$referenceFileSize" + } + + It "Invoke-RestMethod -Resume assumes the file was successfully completed when the local and remote file are the same size." { + # Download the entire file + $uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'NoResume' + $null = Invoke-RestMethod -uri $uri -OutFile $outFile + $fileSize = Get-Item $outFile | Select-Object -ExpandProperty Length + + $uri = Get-WebListenerUrl -Test 'Resume' + $response = Invoke-RestMethod -uri $uri -OutFile $outFile -RHV 'Headers' -Resume + + $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile + $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash + Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize + $Headers.'X-Has-Range'[0] | Should -BeExactly 'true' + $Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$fileSize-" + $Headers.'Content-Range'[0] | Should -BeExactly "bytes */$referenceFileSize" + } + } + BeforeEach { if ($env:http_proxy) { $savedHttpProxy = $env:http_proxy diff --git a/test/tools/Modules/WebListener/WebListener.psm1 b/test/tools/Modules/WebListener/WebListener.psm1 index 95856f0e208..c477067ee29 100644 --- a/test/tools/Modules/WebListener/WebListener.psm1 +++ b/test/tools/Modules/WebListener/WebListener.psm1 @@ -161,6 +161,7 @@ function Get-WebListenerUrl { 'Redirect', 'Response', 'ResponseHeaders', + 'Resume', '/' )] [String]$Test, diff --git a/test/tools/WebListener/Controllers/ResumeController.cs b/test/tools/WebListener/Controllers/ResumeController.cs new file mode 100644 index 00000000000..7e101753f13 --- /dev/null +++ b/test/tools/WebListener/Controllers/ResumeController.cs @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +using System; +using System.Diagnostics; +using System.Linq; +using System.Net.Http.Headers; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Primitives; +using mvc.Models; + + +namespace mvc.Controllers +{ + public class ResumeController : Controller + { + private static Byte[] FileBytes = new Byte[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20}; + + public async void Index() + { + SetResumeResponseHeaders(); + string rangeHeader; + int from = 0; + int to = FileBytes.Length - 1; + if (TryGetRangeHeader(out rangeHeader)) + { + var range = GetRange(rangeHeader); + if(range.From != null) + { + from = (int)range.From; + } + if(range.To != null) + { + to = (int)range.To; + } + + } + else + { + Response.ContentType = "application/octet-stream"; + Response.StatusCode = 200; + await Response.Body.WriteAsync(FileBytes, 0, FileBytes.Length); + return; + } + + if(to >= FileBytes.Length || from >= FileBytes.Length) + { + Response.StatusCode = 416; + Response.Headers["Content-Range"] = $"bytes */{FileBytes.Length}"; + return; + } + else + { + Response.ContentType = "application/octet-stream"; + Response.ContentLength = to - from + 1; + Response.Headers["Content-Range"] = $"bytes {from}-{to}/{FileBytes.Length}"; + Response.StatusCode = 206; + await Response.Body.WriteAsync(FileBytes, from, (int)Response.ContentLength); + } + } + + public async void NoResume() + { + SetResumeResponseHeaders(); + Response.ContentType = "application/octet-stream"; + Response.ContentLength = FileBytes.Length; + Response.StatusCode = 200; + await Response.Body.WriteAsync(FileBytes, 0, FileBytes.Length); + } + + public async void Bytes(int NumberBytes) + { + if (NumberBytes > FileBytes.Length || NumberBytes < 0) + { + NumberBytes = FileBytes.Length; + } + Response.ContentType = "application/octet-stream"; + Response.ContentLength = NumberBytes; + await Response.Body.WriteAsync(FileBytes, 0, NumberBytes); + } + + public IActionResult Error() + { + return View(new ErrorViewModel { RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier }); + } + + private RangeItemHeaderValue GetRange(string rangeHeader) + { + return RangeHeaderValue.Parse(rangeHeader).Ranges.FirstOrDefault(); + } + + private void SetResumeResponseHeaders() + { + string rangeHeader; + if (TryGetRangeHeader(out rangeHeader)) + { + Response.Headers["X-Has-Range"] = "true"; + Response.Headers["X-Request-Range"] = rangeHeader; + } + else + { + Response.Headers["X-Has-Range"] = "false"; + } + } + + private bool TryGetRangeHeader(out string rangeHeader) + { + var rangeHeaderSv = new StringValues(); + if(Request.Headers.TryGetValue("Range", out rangeHeaderSv)) + { + rangeHeader = rangeHeaderSv.FirstOrDefault(); + return true; + } + else + { + rangeHeader = string.Empty; + return false; + } + } + } +} diff --git a/test/tools/WebListener/README.md b/test/tools/WebListener/README.md index bc5cfc98e33..a6266628e03 100644 --- a/test/tools/WebListener/README.md +++ b/test/tools/WebListener/README.md @@ -613,3 +613,72 @@ Body: "x-header-01": "value01" } ``` + +### /Resume/ + +This endpoint simulates the download of a 20 byte file with support for resuming with the use of the `Range` HTTP request header. +The bytes returned are numbered 1 to 20 inclusive. +If the `Range` header is greater than 20, the endpoint will return a `416 Requested Range Not Satisfiable` response. +The endpoint also returns an `X-Has-Range` response header containing `true` or `false` if the HTTP Request contains a `Range` request header. +The endpoint will also return an `X-Request-Range` response header which contains the `Range` header value if one was present. + +```powershell +$uri = Get-WebListenerUrl -Test 'Resume' +$response = Invoke-WebRequest -Uri $uri -Headers @{"Range" = "bytes=0-"} +``` + +Response Headers: + +```none +HTTP/1.1 206 PartialContent +Date: Tue, 20 Mar 2018 08:45:42 GMT +Server: Kestrel +X-Has-Range: true +X-Request-Range: bytes=0- +Content-Length: 20 +Content-Type: application/octet-stream +Content-Range: bytes 0-19/20 +``` + +### /Resume/Bytes/{NumberBytes} + +This endpoint emulates a partial download of the same 20 bytes provided by the `/Resume/` endpoint. +The endpoint will return `{NumberBytes}` bytes of the 20 bytes. +For example `/Resume/Bytes/5` will return bytes 1 through 5 inclusive of the 20 byte file. + +```powershell +$uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'Bytes/5' +$response = Invoke-WebRequest -Uri $uri +``` + +Response Headers: + +```none +HTTP/1.1 200 OK +Date: Tue, 20 Mar 2018 08:50:57 GMT +Server: Kestrel +Content-Length: 5 +Content-Type: application/octet-stream +``` + +### /Resume/NoResume + +This endpoint is the same as `/Resume/` with the exception that it ignores the `Range` HTTP request header. +This endpoint always returns the full 20 bytes and a `200` status. +The `X-Has-Range` and `X-Request-Range` headers are also returned the same as the `/Resume/` endpoint. + +```powershell +$uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'NoResume' +$response = Invoke-WebRequest -Uri $uri +``` + +Response Headers: + +```none +HTTP/1.1 200 OK +Date: Tue, 20 Mar 2018 08:48:21 GMT +Server: Kestrel +X-Has-Range: false +Content-Length: 20 +Content-Type: application/octet-stream +``` diff --git a/test/tools/WebListener/Startup.cs b/test/tools/WebListener/Startup.cs index 9cf4844d578..775357d1814 100644 --- a/test/tools/WebListener/Startup.cs +++ b/test/tools/WebListener/Startup.cs @@ -44,6 +44,10 @@ public void Configure(IApplicationBuilder app, IHostingEnvironment env) app.UseMvc(routes => { + routes.MapRoute( + name: "resume_bytes", + template: "Resume/Bytes/{NumberBytes?}", + defaults: new {controller = "Resume", action = "Bytes"}); routes.MapRoute( name: "default", template: "{controller=Home}/{action=Index}/{id?}"); From 7a3f0eed65a72c7e391bea047a5ca33cdc86ebca Mon Sep 17 00:00:00 2001 From: Mark Kraus Date: Wed, 21 Mar 2018 16:45:23 -0500 Subject: [PATCH 2/4] [Feature] Address PR Feedback from Steve and Ilya --- .../Common/WebRequestPSCmdlet.Common.cs | 7 ++--- .../resources/WebCmdletStrings.resx | 8 +++--- test/tools/WebListener/Constants.cs | 5 ---- .../Controllers/ResponseController.cs | 2 +- .../Controllers/ResumeController.cs | 26 ++++++++++++------- 5 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 1229cb6d432..02d1d43d600 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -518,15 +518,15 @@ internal virtual void ValidateParameters() if (PassThru && (OutFile == null)) { ErrorRecord error = GetValidationError(WebCmdletStrings.OutFileMissing, - "WebCmdletOutFileMissingException"); + "WebCmdletOutFileMissingException", nameof(PassThru)); ThrowTerminatingError(error); } // Resume requires OutFile. if (Resume.IsPresent && OutFile == null) { - ErrorRecord error = GetValidationError(WebCmdletStrings.ResumeOutFileMissing, - "WebCmdletOutFileMissingException"); + ErrorRecord error = GetValidationError(WebCmdletStrings.OutFileMissing, + "WebCmdletOutFileMissingException", nameof(Resume)); ThrowTerminatingError(error); } } @@ -1436,6 +1436,7 @@ protected override void ProcessRecord() _isSuccess = true; // Disable writing to the OutFile. OutFile = null; + WriteVerbose(WebCmdletStrings.OutFileWritingSkipped); } if (!_isSuccess) diff --git a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx index f3e3bc585d7..4fb0c6490be 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx +++ b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx @@ -187,10 +187,10 @@ Path '{0}' is not a file system path. Please specify the path to a file in the file system. - The cmdlet cannot run because the following parameter is missing: OutFile. Provide a valid OutFile parameter value when using the PassThru parameter, then retry. + The cmdlet cannot run because the following parameter is missing: OutFile. Provide a valid OutFile parameter value when using the {0} parameter, then retry. - - The cmdlet cannot run because the following parameter is missing: OutFile. Provide a valid OutFile parameter value when using the Resume parameter, then retry. + + The remote server indicates the file is already complete. OutFile will not be written to. The cmdlet cannot run because the following conflicting parameters are specified: ProxyCredential and ProxyUseDefaultCredentials. Specify either ProxyCredential or ProxyUseDefaultCredentials, then retry. @@ -253,7 +253,7 @@ {0} {1} with {2}-byte payload - The remote server indicated it could not resume downloading. This often occurs when the local file and remote file are different sizes. The local file will be overwritten. + The remote server indicated it could not resume downloading. The local file will be overwritten. received {0}-byte response of content type {1} diff --git a/test/tools/WebListener/Constants.cs b/test/tools/WebListener/Constants.cs index ea4276318da..db4585163a0 100644 --- a/test/tools/WebListener/Constants.cs +++ b/test/tools/WebListener/Constants.cs @@ -14,9 +14,4 @@ internal static class Constants public const string NoUrlLinkHeader = "<>; rel=\"next\""; } - - internal static class StatusCodes - { - public const Int32 ApplicationError = 500; - } } diff --git a/test/tools/WebListener/Controllers/ResponseController.cs b/test/tools/WebListener/Controllers/ResponseController.cs index 40bf06a2cc6..23c0a461ff9 100644 --- a/test/tools/WebListener/Controllers/ResponseController.cs +++ b/test/tools/WebListener/Controllers/ResponseController.cs @@ -74,7 +74,7 @@ public String Index() catch (Exception ex) { output = JsonConvert.SerializeObject(ex); - Response.StatusCode = StatusCodes.ApplicationError; + Response.StatusCode = StatusCodes.Status500InternalServerError; contentType = Constants.ApplicationJson; } } diff --git a/test/tools/WebListener/Controllers/ResumeController.cs b/test/tools/WebListener/Controllers/ResumeController.cs index 7e101753f13..32d62cea2a8 100644 --- a/test/tools/WebListener/Controllers/ResumeController.cs +++ b/test/tools/WebListener/Controllers/ResumeController.cs @@ -1,13 +1,19 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. + using System; using System.Diagnostics; using System.Linq; using System.Net.Http.Headers; +using System.Net.Mime; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Primitives; +using Microsoft.Net.Http.Headers; using mvc.Models; +using RangeItemHeaderValue = System.Net.Http.Headers.RangeItemHeaderValue; +using RangeHeaderValue = System.Net.Http.Headers.RangeHeaderValue; namespace mvc.Controllers { @@ -36,24 +42,24 @@ public async void Index() } else { - Response.ContentType = "application/octet-stream"; - Response.StatusCode = 200; + Response.ContentType = MediaTypeNames.Application.Octet; + Response.StatusCode = StatusCodes.Status200OK; await Response.Body.WriteAsync(FileBytes, 0, FileBytes.Length); return; } if(to >= FileBytes.Length || from >= FileBytes.Length) { - Response.StatusCode = 416; - Response.Headers["Content-Range"] = $"bytes */{FileBytes.Length}"; + Response.StatusCode = StatusCodes.Status416RequestedRangeNotSatisfiable; + Response.Headers[HeaderNames.ContentRange] = $"bytes */{FileBytes.Length}"; return; } else { - Response.ContentType = "application/octet-stream"; + Response.ContentType = MediaTypeNames.Application.Octet; Response.ContentLength = to - from + 1; - Response.Headers["Content-Range"] = $"bytes {from}-{to}/{FileBytes.Length}"; - Response.StatusCode = 206; + Response.Headers[HeaderNames.ContentRange] = $"bytes {from}-{to}/{FileBytes.Length}"; + Response.StatusCode = StatusCodes.Status206PartialContent; await Response.Body.WriteAsync(FileBytes, from, (int)Response.ContentLength); } } @@ -61,9 +67,9 @@ public async void Index() public async void NoResume() { SetResumeResponseHeaders(); - Response.ContentType = "application/octet-stream"; + Response.ContentType = MediaTypeNames.Application.Octet; Response.ContentLength = FileBytes.Length; - Response.StatusCode = 200; + Response.StatusCode = StatusCodes.Status200OK; await Response.Body.WriteAsync(FileBytes, 0, FileBytes.Length); } @@ -73,7 +79,7 @@ public async void Bytes(int NumberBytes) { NumberBytes = FileBytes.Length; } - Response.ContentType = "application/octet-stream"; + Response.ContentType = MediaTypeNames.Application.Octet; Response.ContentLength = NumberBytes; await Response.Body.WriteAsync(FileBytes, 0, NumberBytes); } From 8d214c63db37daa64a6f06460d9c239869d2f5e6 Mon Sep 17 00:00:00 2001 From: Mark Kraus Date: Wed, 21 Mar 2018 17:13:16 -0500 Subject: [PATCH 3/4] [feature] Address PR Feedback --- .../Common/WebRequestPSCmdlet.Common.cs | 2 +- .../resources/WebCmdletStrings.resx | 2 +- .../WebCmdlets.Tests.ps1 | 52 +++++++------------ 3 files changed, 22 insertions(+), 34 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs index 02d1d43d600..ba03ffe2468 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/WebCmdlet/Common/WebRequestPSCmdlet.Common.cs @@ -1434,9 +1434,9 @@ protected override void ProcessRecord() response.Content.Headers.ContentRange.Length == _resumeFileSize) { _isSuccess = true; + WriteVerbose(String.Format(CultureInfo.CurrentCulture, WebCmdletStrings.OutFileWritingSkipped, OutFile)); // Disable writing to the OutFile. OutFile = null; - WriteVerbose(WebCmdletStrings.OutFileWritingSkipped); } if (!_isSuccess) diff --git a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx index 4fb0c6490be..71b1005673a 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx +++ b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx @@ -190,7 +190,7 @@ The cmdlet cannot run because the following parameter is missing: OutFile. Provide a valid OutFile parameter value when using the {0} parameter, then retry. - The remote server indicates the file is already complete. OutFile will not be written to. + The OutFile "{0}" is the same size as the remote file. The file will not be re-downloaded. The cmdlet cannot run because the following conflicting parameters are specified: ProxyCredential and ProxyUseDefaultCredentials. Specify either ProxyCredential or ProxyUseDefaultCredentials, then retry. diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 index a08c8b29de6..46f0a1136c8 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 @@ -1624,8 +1624,8 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { # Download the entire file to reference in tests $referenceFile = Join-Path $TestDrive "reference.txt" - $uri = Get-WebListenerUrl -Test 'Resume' - Invoke-WebRequest -uri $uri -OutFile $referenceFile -ErrorAction Stop + $resumeUri = Get-WebListenerUrl -Test 'Resume' + Invoke-WebRequest -uri $resumeUri -OutFile $referenceFile -ErrorAction Stop $referenceFileHash = Get-FileHash -Algorithm SHA256 -Path $referenceFile $referenceFileSize = Get-Item $referenceFile | Select-Object -ExpandProperty Length } @@ -1635,22 +1635,19 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { } It "Invoke-WebRequest -Resume requires -OutFile" { - $uri = Get-WebListenerUrl -Test 'Resume' - - { Invoke-WebRequest -Resume -Uri $uri -ErrorAction Stop } | + { Invoke-WebRequest -Resume -Uri $resumeUri -ErrorAction Stop } | Should -Throw -ErrorId 'WebCmdletOutFileMissingException,Microsoft.PowerShell.Commands.InvokeWebRequestCommand' } It "Invoke-WebRequest -Resume Downloads the whole file when the file does not exist" { - $uri = Get-WebListenerUrl -Test 'Resume' - $response = Invoke-WebRequest -uri $uri -OutFile $outFile -Resume -PassThru + $response = Invoke-WebRequest -uri $resumeUri -OutFile $outFile -Resume -PassThru $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'true' $response.Headers.'X-Request-Range'[0] | Should -BeExactly 'bytes=0-' - $response.StatusCode | Should -BeExactly 206 + $response.StatusCode | Should -Be 206 $response.Headers.'Content-Range'[0] | Should -BeExactly "bytes 0-$($referenceFileSize-1)/$referenceFileSize" } @@ -1660,15 +1657,14 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { 1..$largerFileSize | ForEach-Object { [Byte]$_ } | Set-Content -AsByteStream $outFile $largerFileSize = Get-Item $outFile | Select-Object -ExpandProperty Length - $uri = Get-WebListenerUrl -Test 'Resume' - $response = Invoke-WebRequest -uri $uri -OutFile $outFile -Resume -PassThru + $response = Invoke-WebRequest -uri $resumeUri -OutFile $outFile -Resume -PassThru $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize Get-Item $outFile | Select-Object -ExpandProperty Length | Should -BeLessThan $largerFileSize $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'false' - $response.StatusCode | Should -BeExactly 200 + $response.StatusCode | Should -Be 200 $response.Headers.ContainsKey('Content-Range') | Should -BeFalse } @@ -1686,7 +1682,7 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'true' $response.Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$largerFileSize-" - $response.StatusCode | Should -BeExactly 200 + $response.StatusCode | Should -Be 200 $response.Headers.ContainsKey('Content-Range') | Should -BeFalse } @@ -1702,15 +1698,14 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { $null = Invoke-WebRequest -uri $uri -OutFile $outFile Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $bytes - $uri = Get-WebListenerUrl -Test 'Resume' - $response = Invoke-WebRequest -uri $uri -OutFile $outFile -Resume -PassThru + $response = Invoke-WebRequest -uri $resumeUri -OutFile $outFile -Resume -PassThru $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'true' $response.Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$bytes-" - $response.StatusCode | Should -BeExactly 206 + $response.StatusCode | Should -Be 206 $response.Headers.'Content-Range'[0] | Should -BeExactly "bytes $bytes-$($referenceFileSize-1)/$referenceFileSize" } @@ -1720,8 +1715,7 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { $null = Invoke-WebRequest -uri $uri -OutFile $outFile $fileSize = Get-Item $outFile | Select-Object -ExpandProperty Length - $uri = Get-WebListenerUrl -Test 'Resume' - $response = Invoke-WebRequest -uri $uri -OutFile $outFile -Resume -PassThru + $response = Invoke-WebRequest -uri $resumeUri -OutFile $outFile -Resume -PassThru $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash @@ -1729,7 +1723,7 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'true' $response.Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$fileSize-" # The web cmdlets special case 416 as a success code when the local file and remote file are the same size - $response.StatusCode | Should -BeExactly 416 + $response.StatusCode | Should -Be 416 $response.Headers.'Content-Range'[0] | Should -BeExactly "bytes */$referenceFileSize" } } @@ -2901,8 +2895,8 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { # Download the entire file to reference in tests $referenceFile = Join-Path $TestDrive "reference.txt" - $uri = Get-WebListenerUrl -Test 'Resume' - Invoke-RestMethod -uri $uri -OutFile $referenceFile -ErrorAction Stop + $resumeUri = Get-WebListenerUrl -Test 'Resume' + Invoke-RestMethod -uri $resumeUri -OutFile $referenceFile -ErrorAction Stop $referenceFileHash = Get-FileHash -Algorithm SHA256 -Path $referenceFile $referenceFileSize = Get-Item $referenceFile | Select-Object -ExpandProperty Length } @@ -2912,9 +2906,7 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { } It "Invoke-RestMethod -Resume requires -OutFile" { - $uri = Get-WebListenerUrl -Test 'Resume' - - { Invoke-RestMethod -Resume -Uri $uri -ErrorAction Stop } | + { Invoke-RestMethod -Resume -Uri $resumeUri -ErrorAction Stop } | Should -Throw -ErrorId 'WebCmdletOutFileMissingException,Microsoft.PowerShell.Commands.InvokeRestMethodCommand' } @@ -2922,8 +2914,7 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { # ensure the file does not exist Remove-Item -Force -ErrorAction 'SilentlyContinue' -Path $outFile - $uri = Get-WebListenerUrl -Test 'Resume' - Invoke-RestMethod -uri $uri -OutFile $outFile -RHV 'Headers' -Resume + Invoke-RestMethod -uri $resumeUri -OutFile $outFile -ResponseHeadersVariable 'Headers' -Resume $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash @@ -2939,8 +2930,7 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { 1..$largerFileSize | ForEach-Object { [Byte]$_ } | Set-Content -AsByteStream $outFile $largerFileSize = Get-Item $outFile | Select-Object -ExpandProperty Length - $uri = Get-WebListenerUrl -Test 'Resume' - $response = Invoke-RestMethod -uri $uri -OutFile $outFile -RHV 'Headers' -Resume + $response = Invoke-RestMethod -uri $resumeUri -OutFile $outFile -ResponseHeadersVariable 'Headers' -Resume $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash @@ -2957,7 +2947,7 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { $largerFileSize = Get-Item $outFile | Select-Object -ExpandProperty Length $uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'NoResume' - $response = Invoke-RestMethod -uri $uri -OutFile $outFile -RHV 'Headers' -Resume + $response = Invoke-RestMethod -uri $uri -OutFile $outFile -ResponseHeadersVariable 'Headers' -Resume $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash @@ -2980,8 +2970,7 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { $null = Invoke-RestMethod -uri $uri -OutFile $outFile Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $bytes - $uri = Get-WebListenerUrl -Test 'Resume' - $response = Invoke-RestMethod -uri $uri -OutFile $outFile -RHV 'Headers' -Resume + $response = Invoke-RestMethod -uri $resumeUri -OutFile $outFile -ResponseHeadersVariable 'Headers' -Resume $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should BeExactly $referenceFileHash.Hash @@ -2997,8 +2986,7 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { $null = Invoke-RestMethod -uri $uri -OutFile $outFile $fileSize = Get-Item $outFile | Select-Object -ExpandProperty Length - $uri = Get-WebListenerUrl -Test 'Resume' - $response = Invoke-RestMethod -uri $uri -OutFile $outFile -RHV 'Headers' -Resume + $response = Invoke-RestMethod -uri $resumeUri -OutFile $outFile -ResponseHeadersVariable 'Headers' -Resume $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash From 939772cae8525476b085dbc535fb937d445113ee Mon Sep 17 00:00:00 2001 From: Mark Kraus Date: Thu, 22 Mar 2018 04:12:33 -0500 Subject: [PATCH 4/4] [feature] Address PR Feedback --- .../resources/WebCmdletStrings.resx | 2 +- .../WebCmdlets.Tests.ps1 | 36 +++++++++---------- .../Controllers/ResumeController.cs | 6 ++-- test/tools/WebListener/README.md | 12 +++---- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx index 71b1005673a..f22655aaf56 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx +++ b/src/Microsoft.PowerShell.Commands.Utility/resources/WebCmdletStrings.resx @@ -190,7 +190,7 @@ The cmdlet cannot run because the following parameter is missing: OutFile. Provide a valid OutFile parameter value when using the {0} parameter, then retry. - The OutFile "{0}" is the same size as the remote file. The file will not be re-downloaded. + The file will not be re-downloaded because the remote file is the same size as the OutFile: {0} The cmdlet cannot run because the following conflicting parameters are specified: ProxyCredential and ProxyUseDefaultCredentials. Specify either ProxyCredential or ProxyUseDefaultCredentials, then retry. diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 index 46f0a1136c8..d60a2187f82 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/WebCmdlets.Tests.ps1 @@ -1645,8 +1645,8 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize - $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'true' - $response.Headers.'X-Request-Range'[0] | Should -BeExactly 'bytes=0-' + $response.Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $response.Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly 'bytes=0-' $response.StatusCode | Should -Be 206 $response.Headers.'Content-Range'[0] | Should -BeExactly "bytes 0-$($referenceFileSize-1)/$referenceFileSize" } @@ -1663,7 +1663,7 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize Get-Item $outFile | Select-Object -ExpandProperty Length | Should -BeLessThan $largerFileSize - $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'false' + $response.Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'false' $response.StatusCode | Should -Be 200 $response.Headers.ContainsKey('Content-Range') | Should -BeFalse } @@ -1680,8 +1680,8 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize - $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'true' - $response.Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$largerFileSize-" + $response.Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $response.Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly "bytes=$largerFileSize-" $response.StatusCode | Should -Be 200 $response.Headers.ContainsKey('Content-Range') | Should -BeFalse } @@ -1703,8 +1703,8 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize - $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'true' - $response.Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$bytes-" + $response.Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $response.Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly "bytes=$bytes-" $response.StatusCode | Should -Be 206 $response.Headers.'Content-Range'[0] | Should -BeExactly "bytes $bytes-$($referenceFileSize-1)/$referenceFileSize" } @@ -1720,8 +1720,8 @@ Describe "Invoke-WebRequest tests" -Tags "Feature" { $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize - $response.Headers.'X-Has-Range'[0] | Should -BeExactly 'true' - $response.Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$fileSize-" + $response.Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $response.Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly "bytes=$fileSize-" # The web cmdlets special case 416 as a success code when the local file and remote file are the same size $response.StatusCode | Should -Be 416 $response.Headers.'Content-Range'[0] | Should -BeExactly "bytes */$referenceFileSize" @@ -2919,8 +2919,8 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize - $Headers.'X-Has-Range'[0] | Should -BeExactly 'true' - $Headers.'X-Request-Range'[0] | Should -BeExactly 'bytes=0-' + $Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly 'bytes=0-' $Headers.'Content-Range'[0] | Should -BeExactly "bytes 0-$($referenceFileSize-1)/$referenceFileSize" } @@ -2936,7 +2936,7 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize Get-Item $outFile | Select-Object -ExpandProperty Length | Should -BeLessThan $largerFileSize - $Headers.'X-Has-Range'[0] | Should -BeExactly 'false' + $Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'false' $Headers.ContainsKey('Content-Range') | Should -BeFalse } @@ -2953,8 +2953,8 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize Get-Item $outFile | Select-Object -ExpandProperty Length | Should -BeLessThan $largerFileSize - $Headers.'X-Has-Range'[0] | Should -BeExactly 'true' - $Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$largerFileSize-" + $Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly "bytes=$largerFileSize-" $Headers.ContainsKey('Content-Range') | Should -BeFalse } @@ -2975,8 +2975,8 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize - $Headers.'X-Has-Range'[0] | Should -BeExactly 'true' - $Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$bytes-" + $Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly "bytes=$bytes-" $Headers.'Content-Range'[0] | Should -BeExactly "bytes $bytes-$($referenceFileSize-1)/$referenceFileSize" } @@ -2991,8 +2991,8 @@ Describe "Invoke-RestMethod tests" -Tags "Feature" { $outFileHash = Get-FileHash -Algorithm SHA256 -Path $outFile $outFileHash.Hash | Should -BeExactly $referenceFileHash.Hash Get-Item $outFile | Select-Object -ExpandProperty Length | Should -Be $referenceFileSize - $Headers.'X-Has-Range'[0] | Should -BeExactly 'true' - $Headers.'X-Request-Range'[0] | Should -BeExactly "bytes=$fileSize-" + $Headers.'X-WebListener-Has-Range'[0] | Should -BeExactly 'true' + $Headers.'X-WebListener-Request-Range'[0] | Should -BeExactly "bytes=$fileSize-" $Headers.'Content-Range'[0] | Should -BeExactly "bytes */$referenceFileSize" } } diff --git a/test/tools/WebListener/Controllers/ResumeController.cs b/test/tools/WebListener/Controllers/ResumeController.cs index 32d62cea2a8..3d0de50bd91 100644 --- a/test/tools/WebListener/Controllers/ResumeController.cs +++ b/test/tools/WebListener/Controllers/ResumeController.cs @@ -99,12 +99,12 @@ private void SetResumeResponseHeaders() string rangeHeader; if (TryGetRangeHeader(out rangeHeader)) { - Response.Headers["X-Has-Range"] = "true"; - Response.Headers["X-Request-Range"] = rangeHeader; + Response.Headers["X-WebListener-Has-Range"] = "true"; + Response.Headers["X-WebListener-Request-Range"] = rangeHeader; } else { - Response.Headers["X-Has-Range"] = "false"; + Response.Headers["X-WebListener-Has-Range"] = "false"; } } diff --git a/test/tools/WebListener/README.md b/test/tools/WebListener/README.md index a6266628e03..1e961542eda 100644 --- a/test/tools/WebListener/README.md +++ b/test/tools/WebListener/README.md @@ -619,8 +619,8 @@ Body: This endpoint simulates the download of a 20 byte file with support for resuming with the use of the `Range` HTTP request header. The bytes returned are numbered 1 to 20 inclusive. If the `Range` header is greater than 20, the endpoint will return a `416 Requested Range Not Satisfiable` response. -The endpoint also returns an `X-Has-Range` response header containing `true` or `false` if the HTTP Request contains a `Range` request header. -The endpoint will also return an `X-Request-Range` response header which contains the `Range` header value if one was present. +The endpoint also returns an `X-WebListener-Has-Range` response header containing `true` or `false` if the HTTP Request contains a `Range` request header. +The endpoint will also return an `X-WebListener-Request-Range` response header which contains the `Range` header value if one was present. ```powershell $uri = Get-WebListenerUrl -Test 'Resume' @@ -633,8 +633,8 @@ Response Headers: HTTP/1.1 206 PartialContent Date: Tue, 20 Mar 2018 08:45:42 GMT Server: Kestrel -X-Has-Range: true -X-Request-Range: bytes=0- +X-WebListener-Has-Range: true +X-WebListener-Request-Range: bytes=0- Content-Length: 20 Content-Type: application/octet-stream Content-Range: bytes 0-19/20 @@ -665,7 +665,7 @@ Content-Type: application/octet-stream This endpoint is the same as `/Resume/` with the exception that it ignores the `Range` HTTP request header. This endpoint always returns the full 20 bytes and a `200` status. -The `X-Has-Range` and `X-Request-Range` headers are also returned the same as the `/Resume/` endpoint. +The `X-WebListener-Has-Range` and `X-WebListener-Request-Range` headers are also returned the same as the `/Resume/` endpoint. ```powershell $uri = Get-WebListenerUrl -Test 'Resume' -TestValue 'NoResume' @@ -678,7 +678,7 @@ Response Headers: HTTP/1.1 200 OK Date: Tue, 20 Mar 2018 08:48:21 GMT Server: Kestrel -X-Has-Range: false +X-WebListener-Has-Range: false Content-Length: 20 Content-Type: application/octet-stream ```