From 62d534b60903d4d2a4166905782b528690d9c978 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Wed, 6 Dec 2017 10:18:53 -0800 Subject: [PATCH 1/2] Update flaky test --- .../engine/Basic/ValidateAttributes.Tests.ps1 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/powershell/engine/Basic/ValidateAttributes.Tests.ps1 b/test/powershell/engine/Basic/ValidateAttributes.Tests.ps1 index d95af1cd2b7..79d6a0c3873 100644 --- a/test/powershell/engine/Basic/ValidateAttributes.Tests.ps1 +++ b/test/powershell/engine/Basic/ValidateAttributes.Tests.ps1 @@ -273,10 +273,7 @@ Describe 'Validate Attributes Tests' -Tags 'CI' { [System.Collections.Generic.List[byte]] $ByteList, [Parameter(Mandatory, ParameterSetName = "ByteCollection")] - [System.Collections.ObjectModel.Collection[byte]] $ByteCollection, - - [Parameter(ParameterSetName = "Default")] - $Value + [System.Collections.ObjectModel.Collection[byte]] $ByteCollection ) } @@ -316,10 +313,13 @@ Describe 'Validate Attributes Tests' -Tags 'CI' { $byteArray = [System.IO.File]::ReadAllBytes($filePath) $byteList = [System.Collections.Generic.List[byte]] $byteArray $byteCollection = [System.Collections.ObjectModel.Collection[byte]] $byteArray - ## Use the running time of 'MandatoryFunc -Value $byteArray' as the baseline time - $baseline = (Measure-Command { MandatoryFunc -Value $byteArray }).Milliseconds - ## Running time should be less than 'expected' - $expected = $baseline + 20 + ## Running time should be less than 'UpperBoundTime' + ## This is not really a performance test (perf test cannot run reliably in our CI), but a test + ## to make sure we don't check the elements of a value-type collection. + ## The crossgen'ed 'S.M.A.dll' is about 28mb in size, and it would take over 2000ms if we check + ## each byte of the array, list or collection. We use 200ms as the upper bound value in tests to + ## prove that we don't check each byte. + $UpperBoundTime = 200 if ($IsWindows) { $null = New-Item -Path $TESTDRIVE/file1 @@ -340,7 +340,7 @@ Describe 'Validate Attributes Tests' -Tags 'CI' { It "Validate running time ''" -TestCases $testCases { param ($ScriptBlock) - (Measure-Command $ScriptBlock).Milliseconds | Should BeLessThan $expected + (Measure-Command $ScriptBlock).Milliseconds | Should BeLessThan $UpperBoundTime } It "COM enumerable argument should work with 'ValidateNotNull' and 'ValidateNotNullOrEmpty'" -Skip:(!$IsWindows) { From da9cd9ef09a353fab780f0757f754c4e683ea1f2 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Wed, 6 Dec 2017 11:54:26 -0800 Subject: [PATCH 2/2] Address comment --- .../engine/Basic/ValidateAttributes.Tests.ps1 | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/powershell/engine/Basic/ValidateAttributes.Tests.ps1 b/test/powershell/engine/Basic/ValidateAttributes.Tests.ps1 index 79d6a0c3873..217197d7281 100644 --- a/test/powershell/engine/Basic/ValidateAttributes.Tests.ps1 +++ b/test/powershell/engine/Basic/ValidateAttributes.Tests.ps1 @@ -273,7 +273,10 @@ Describe 'Validate Attributes Tests' -Tags 'CI' { [System.Collections.Generic.List[byte]] $ByteList, [Parameter(Mandatory, ParameterSetName = "ByteCollection")] - [System.Collections.ObjectModel.Collection[byte]] $ByteCollection + [System.Collections.ObjectModel.Collection[byte]] $ByteCollection, + + [Parameter(ParameterSetName = "Default")] + $Value ) } @@ -313,13 +316,16 @@ Describe 'Validate Attributes Tests' -Tags 'CI' { $byteArray = [System.IO.File]::ReadAllBytes($filePath) $byteList = [System.Collections.Generic.List[byte]] $byteArray $byteCollection = [System.Collections.ObjectModel.Collection[byte]] $byteArray + ## Use the running time of 'MandatoryFunc -Value $byteArray' as the baseline time + ## because it does no check on the argument. + $baseline = (Measure-Command { MandatoryFunc -Value $byteArray }).Milliseconds ## Running time should be less than 'UpperBoundTime' ## This is not really a performance test (perf test cannot run reliably in our CI), but a test ## to make sure we don't check the elements of a value-type collection. - ## The crossgen'ed 'S.M.A.dll' is about 28mb in size, and it would take over 2000ms if we check - ## each byte of the array, list or collection. We use 200ms as the upper bound value in tests to - ## prove that we don't check each byte. - $UpperBoundTime = 200 + ## The crossgen'ed 'S.M.A.dll' is about 28mb in size, and it would take more than 2000ms if we + ## check each byte of the array, list or collection. We use ($baseline + 200)ms as the upper + ## bound value in tests to prove that we don't check each byte. + $UpperBoundTime = $baseline + 200 if ($IsWindows) { $null = New-Item -Path $TESTDRIVE/file1