From efadce537e3fff4d263631d5cc8a9720a1077efc Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 7 Aug 2018 16:59:20 -0700 Subject: [PATCH 01/19] add mac CI definition --- vsts-macos-ci.yml | 81 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 vsts-macos-ci.yml diff --git a/vsts-macos-ci.yml b/vsts-macos-ci.yml new file mode 100644 index 00000000000..bf1e5d9485d --- /dev/null +++ b/vsts-macos-ci.yml @@ -0,0 +1,81 @@ +name: PR-$(System.PullRequest.PullRequestNumber)-$(Date:yyyyMMdd)$(Rev:.rr) +variables: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + POWERSHELL_TELEMETRY_OPTOUT: 1 + # Avoid expensive initialization of dotnet cli, see: http://donovanbrown.com/post/Stop-wasting-time-during-NET-Core-builds + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + +resources: +- repo: self + clean: true +phases: +- phase: macOS_CI + + queue: + name: Hosted macOS Preview + steps: + - powershell: Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhss"))" + displayName: Set Build Name for Non-PR + condition: ne(variables['Build.Reason'], 'PullRequest') + +# Several scripts require TMPDIR to exist +# Set it to AGENT_TEMPDIRECTORY so that we know it exists + - powershell: | + Write-Host "##vso[task.setvariable variable=TMPDIR]$env:AGENT_TEMPDIRECTORY" + displayName: Set TMPDIR to AGENT_TEMPDIRECTORY + condition: succeededOrFailed() + + - powershell: | + git submodule update --init + displayName: SubModule Init + condition: succeededOrFailed() + + - powershell: | + rvm install ruby-2.3.3; + rvm --default use 2.3.3; + displayName: Install Ruby + condition: succeededOrFailed() + enabled: false + + - powershell: | + tools/travis.ps1 -Stage Bootstrap + displayName: Bootstrap + condition: succeeded() + + - powershell: | + tools/travis.ps1 + displayName: Build and test + condition: succeeded() + + - powershell: | + tools/travis.ps1 -Stage Failure + displayName: After Failure + condition: failed() + + - powershell: | + tools/travis.ps1 -Stage Success + displayName: After Success + condition: succeeded() + + - powershell: | + Get-ChildItem -Path *.pkg, *.tar.gz -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object { + Write-Host "##vso[artifact.upload containerfolder=artifacts;artifactname=artifacts]$_" + } + displayName: Publish Artifacts + condition: succeeded() + + - powershell: | + Get-ChildItem -Path Test*.xml -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object { + Write-Host "##vso[artifact.upload containerfolder=testResults;artifactname=testResults]$_" + } + displayName: Publish Test Results + condition: succeeded() + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: Publish Test Results **\Test*.xml + inputs: + testRunner: NUnit + testResultsFiles: '**\Test*.xml' + testRunTitle: Windows + mergeTestResults: true From 00c0d487c5065ad301e63eeea83a3bee77c67e66 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 7 Aug 2018 16:36:52 -0700 Subject: [PATCH 02/19] Increase the max time on the sleep as macOS is taking longer in VSTS --- .../Start-Sleep.Tests.ps1 | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/powershell/Modules/Microsoft.PowerShell.Utility/Start-Sleep.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Utility/Start-Sleep.Tests.ps1 index f427e7f923c..b0d398340ff 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Utility/Start-Sleep.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Utility/Start-Sleep.Tests.ps1 @@ -3,31 +3,33 @@ Describe "Start-Sleep DRT Unit Tests" -Tags "CI" { # WaitHandle.WaitOne(milliseconds, exitContext) is not accurate. - # The wait time varies from 980ms to 1020ms from observation, so + # The wait time varies from 980ms to 1150ms from observation, so # the tests here are changed to be greater than 950ms. - + $minTime = 950 + $maxTime = 1200 + It "Should work properly when sleeping with Second" { $watch = [System.Diagnostics.Stopwatch]::StartNew() Start-Sleep -Seconds 1 $watch.Stop() - $watch.ElapsedMilliseconds | Should -BeGreaterThan 950 - $watch.ElapsedMilliseconds | Should -BeLessThan 1100 + $watch.ElapsedMilliseconds | Should -BeGreaterThan $minTime + $watch.ElapsedMilliseconds | Should -BeLessThan $maxTime } It "Should work properly when sleeping with Milliseconds" { $watch = [System.Diagnostics.Stopwatch]::StartNew() Start-Sleep -Milliseconds 1000 $watch.Stop() - $watch.ElapsedMilliseconds | Should -BeGreaterThan 950 - $watch.ElapsedMilliseconds | Should -BeLessThan 1100 + $watch.ElapsedMilliseconds | Should -BeGreaterThan $minTime + $watch.ElapsedMilliseconds | Should -BeLessThan $maxTime } It "Should work properly when sleeping with ms alias" { $watch = [System.Diagnostics.Stopwatch]::StartNew() Start-Sleep -ms 1000 $watch.Stop() - $watch.ElapsedMilliseconds | Should -BeGreaterThan 950 - $watch.ElapsedMilliseconds | Should -BeLessThan 1100 + $watch.ElapsedMilliseconds | Should -BeGreaterThan $minTime + $watch.ElapsedMilliseconds | Should -BeLessThan $maxTime } } From 514ba2e60ba46154c7e6642283f5e91fd713c358 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 8 Aug 2018 13:21:57 -0700 Subject: [PATCH 03/19] export new function --- test/tools/Modules/HelpersCommon/HelpersCommon.psd1 | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/tools/Modules/HelpersCommon/HelpersCommon.psd1 b/test/tools/Modules/HelpersCommon/HelpersCommon.psd1 index f811668d6ae..f54fa638bad 100644 --- a/test/tools/Modules/HelpersCommon/HelpersCommon.psd1 +++ b/test/tools/Modules/HelpersCommon/HelpersCommon.psd1 @@ -16,5 +16,16 @@ Copyright = 'Copyright (c) Microsoft Corporation. All rights reserved.' Description = 'Temporary module contains functions for using in tests' -FunctionsToExport = 'Wait-UntilTrue', 'Test-IsElevated', 'Wait-FileToBePresent', 'Get-RandomFileName', 'Enable-Testhook', 'Disable-Testhook', 'Set-TesthookResult', 'Test-TesthookIsSet', 'Add-TestDynamicType' +FunctionsToExport = @( + 'Add-TestDynamicType' + 'Disable-Testhook' + 'Enable-Testhook' + 'Get-RandomFileName' + 'Send-VstsLogFile' + 'Set-TesthookResult' + 'Test-IsElevated' + 'Test-TesthookIsSet' + 'Wait-FileToBePresent' + 'Wait-UntilTrue' + ) } From 1157c3a3aa90a6ae06458798d1a6b9af22c5d6c5 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 8 Aug 2018 16:08:29 -0700 Subject: [PATCH 04/19] add function to upload a log file on VSTS --- .../Modules/HelpersCommon/HelpersCommon.psm1 | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 b/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 index 659f28e8e00..f104e0a2eeb 100644 --- a/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 +++ b/test/tools/Modules/HelpersCommon/HelpersCommon.psm1 @@ -192,3 +192,47 @@ public class TestDynamic : DynamicObject } '@ } + +# Upload an artifact in VSTS +# On other systems will just log where the file was placed +function Send-VstsLogFile { + param ( + [parameter(Mandatory,ParameterSetName='contents')] + [string[]] + $Contents, + [parameter(Mandatory,ParameterSetName='contents')] + [string] + $LogName, + [parameter(Mandatory,ParameterSetName='path')] + [ValidateScript({Test-Path -Path $_})] + [string] + $Path + ) + + $logFolder = Join-Path -path $pwd -ChildPath 'logfile' + if(!(Test-Path -Path $logFolder)) + { + $null = New-Item -Path $logFolder -ItemType Directory + if($IsMacOS -or $IsLinux) + { + $null = chmod a+rw $logFolder + } + } + + if($Contents) + { + $logFile = Join-Path -Path $logFolder -ChildPath ([System.Io.Path]::GetRandomFileName() + "-$LogName.txt") + $name = Split-Path -leaf -Path $logFile + + $Contents | out-file -path $logFile -Encoding ascii + } + else + { + $name = Split-Path -leaf -Path $path + $logFile = Join-Path -Path $logFolder -ChildPath ([System.Io.Path]::GetRandomFileName() + '-' + $name) + Copy-Item -Path $Path -Destination $logFile + } + + Write-Host "##vso[artifact.upload containerfolder=$name;artifactname=$name]$logFile" + Write-Verbose "Log file captured as $name" -Verbose +} From ecdad5b2c4c92f51e5f14908fb305e29deed889f Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 8 Aug 2018 15:24:25 -0700 Subject: [PATCH 05/19] Allow 'SkipInVSTS' tag in pester tests --- build.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index 44ddfa639d1..6c63007ae43 100644 --- a/build.psm1 +++ b/build.psm1 @@ -909,7 +909,7 @@ function Get-PesterTag { } $values = $vAst.FindAll({$args[0] -is "System.Management.Automation.Language.StringConstantExpressionAst"},$true).Value $values | ForEach-Object { - if (@('REQUIREADMINONWINDOWS', 'REQUIRESUDOONUNIX', 'SLOW') -contains $_) { + if (@('REQUIREADMINONWINDOWS', 'REQUIRESUDOONUNIX', 'SLOW', 'SKIPINVSTS') -contains $_) { # These are valid tags also, but they are not the priority tags } elseif (@('CI', 'FEATURE', 'SCENARIO') -contains $_) { From 12b84e397d242dc2d21e11cecfade28f1b740dbc Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 8 Aug 2018 15:46:17 -0700 Subject: [PATCH 06/19] exclude tests that fail in VSTS by tag --- tools/travis.ps1 | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/travis.ps1 b/tools/travis.ps1 index ef43a8c8a2c..a5845112ec5 100644 --- a/tools/travis.ps1 +++ b/tools/travis.ps1 @@ -204,12 +204,20 @@ elseif($Stage -eq 'Build') $testResultsNoSudo = "$pwd/TestResultsNoSudo.xml" $testResultsSudo = "$pwd/TestResultsSudo.xml" + $excludeTag = @('RequireSudoOnUnix') + + if($env:TF_BUILD) + { + # Used to skip tests that fail in VSTS + $excludeTag += 'SkipInVSTS' + } + $noSudoPesterParam = @{ 'BinDir' = $output 'PassThru' = $true 'Terse' = $true 'Tag' = @() - 'ExcludeTag' = @('RequireSudoOnUnix') + 'ExcludeTag' = $excludeTag 'OutputFile' = $testResultsNoSudo } @@ -256,6 +264,12 @@ elseif($Stage -eq 'Build') $sudoPesterParam.Remove('Path') $sudoPesterParam['Tag'] = @('RequireSudoOnUnix') $sudoPesterParam['ExcludeTag'] = @() + if($env:TF_BUILD) + { + # Used to skip tests that fail in VSTS + $sudoPesterParam['ExcludeTag'] = @('SkipInVSTS') + } + $sudoPesterParam['Sudo'] = $true $sudoPesterParam['OutputFile'] = $testResultsSudo $pesterPassThruSudoObject = Start-PSPester @sudoPesterParam From 56e32bd01ad5a45e1395798e5e6f29fe2e877d11 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 8 Aug 2018 15:25:28 -0700 Subject: [PATCH 07/19] Tag mac logging tests as failing in VSTS --- test/powershell/Host/Logging.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/powershell/Host/Logging.Tests.ps1 b/test/powershell/Host/Logging.Tests.ps1 index a53e21f664d..da8c6574950 100644 --- a/test/powershell/Host/Logging.Tests.ps1 +++ b/test/powershell/Host/Logging.Tests.ps1 @@ -158,7 +158,7 @@ Describe 'Basic SysLog tests on Linux' -Tag @('CI','RequireSudoOnUnix') { } } -Describe 'Basic os_log tests on MacOS' -Tag @('Feature','RequireSudoOnUnix') { +Describe 'Basic os_log tests on MacOS' -Tag @('Feature','RequireSudoOnUnix','SkipInVsts') { BeforeAll { [bool] $IsSupportedEnvironment = $IsMacOS [bool] $persistenceEnabled = $false From dac65b46ff03ba92f31d6908180f8610d2bebb85 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Wed, 8 Aug 2018 17:55:51 -0700 Subject: [PATCH 08/19] set build to continue on error --- vsts-macos-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/vsts-macos-ci.yml b/vsts-macos-ci.yml index bf1e5d9485d..34f675b74a5 100644 --- a/vsts-macos-ci.yml +++ b/vsts-macos-ci.yml @@ -43,6 +43,7 @@ phases: condition: succeeded() - powershell: | + $ErrorActionPreference = 'continue' tools/travis.ps1 displayName: Build and test condition: succeeded() From 6c2a67910f2b4cacc03ed150cd991a70e50ff408 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 9 Aug 2018 15:06:47 -0700 Subject: [PATCH 09/19] Add ability for the mac Logging tests to wait for a count of events --- test/tools/Modules/PSSysLog/PSSysLog.psm1 | 64 +++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/test/tools/Modules/PSSysLog/PSSysLog.psm1 b/test/tools/Modules/PSSysLog/PSSysLog.psm1 index dcddcf7c02d..27f4b5e4ce3 100644 --- a/test/tools/Modules/PSSysLog/PSSysLog.psm1 +++ b/test/tools/Modules/PSSysLog/PSSysLog.psm1 @@ -822,7 +822,7 @@ function Get-PSOsLog #> function Export-PSOsLog { - [CmdletBinding()] + [CmdletBinding(DefaultParameterSetName='default')] param ( [Parameter(Mandatory)] @@ -831,7 +831,16 @@ function Export-PSOsLog [string] $LogId = "powershell", - [int] $LogPid + [int] $LogPid, + + [Parameter(Mandatory, ParameterSetName='WaitUntil')] + [int] $TimeoutInMilliseconds, + + [Parameter(Mandatory, ParameterSetName='WaitUntil')] + [int] $IntervalInMilliseconds, + + [Parameter(Mandatory, ParameterSetName='WaitUntil')] + [string] $MinimumCount ) Test-MacOS @@ -863,7 +872,56 @@ function Export-PSOsLog ) } - Start-NativeExecution -command {log show --info @extraParams} + Wait-UntilSuccess { + # Leaving this in an turned on by default until the tests are stabilized. + Write-Verbose "Exporting macOS logs..." -Verbose + $log = @(Start-NativeExecution -command {log show --info @extraParams} | Select-String -SimpleMatch 'com.microsoft.powershell') + + if($log.Count -ge $MinimumCount){ + Write-Output $log + } + else { + throw "did not recieve at least $MinimumCount records but $($log.Count) instead." + } + } -TimeoutInMilliseconds $TimeoutInMilliseconds -IntervalInMilliseconds $IntervalInMilliseconds -LogErrorSb { + $log = Start-NativeExecution -command {log show --info @extraParams} | Select-String -SimpleMatch 'com.microsoft.powershell' + Send-VstsLogFile -Contents $log -LogName 'Export-PSOsLog-Failure' + } +} + +function Wait-UntilSuccess +{ + [CmdletBinding()] + param ( + [ScriptBlock]$sb, + [ScriptBlock]$LogErrorSb, + [int]$TimeoutInMilliseconds = 10000, + [int]$IntervalInMilliseconds = 10000 + ) + # Get the current time + $startTime = [DateTime]::Now + + # Loop until the script block returns + while ($true) { + try{ + return & $sb + } + catch{ + # If the timeout period has passed, return false + $msPassed = ([DateTime]::Now - $startTime).TotalMilliseconds + if ($msPassed -gt $timeoutInMilliseconds) { + if($LogErrorSb) + { + try { & $LogErrorSb } catch {Write-Verbose "Logging of Error details failed with: $_" -Verbose} + } + throw + } + } + + # Sleep for the specified interval + Start-Sleep -Milliseconds $intervalInMilliseconds + } + return $true } <# From e32e6faf05c93ff316e6d8e82c5ddd5230603957 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 9 Aug 2018 14:53:27 -0700 Subject: [PATCH 10/19] update the mac logging tests to wait based on a count Also, to send log files to VSTS on failure --- test/powershell/Host/Logging.Tests.ps1 | 71 ++++++++++++++++---------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/test/powershell/Host/Logging.Tests.ps1 b/test/powershell/Host/Logging.Tests.ps1 index da8c6574950..863e0272ccd 100644 --- a/test/powershell/Host/Logging.Tests.ps1 +++ b/test/powershell/Host/Logging.Tests.ps1 @@ -5,6 +5,7 @@ using namespace System.Text Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +Import-Module HelpersCommon Import-Module PSSysLog <# @@ -202,35 +203,51 @@ Describe 'Basic os_log tests on MacOS' -Tag @('Feature','RequireSudoOnUnix','Ski } It 'Verifies basic logging with no customizations' -Skip:(!$IsSupportedEnvironment) { - $configFile = WriteLogSettings -LogId $logId - $testPid = & $powershell -NoProfile -SettingsFile $configFile -Command '$PID' - - # Made tests more reliable - Start-Sleep -Milliseconds 500 - - Export-PSOsLog -After $after -LogPid $testPid -Verbose | Set-Content -Path $contentFile - $items = @(Get-PSOsLog -Path $contentFile -Id $logId -After $after -TotalCount 3 -Verbose) - - $items | Should -Not -Be $null - $items.Count | Should -BeGreaterThan 1 - $items[0].EventId | Should -BeExactly 'Perftrack_ConsoleStartupStart:PowershellConsoleStartup.WinStart.Informational' - $items[1].EventId | Should -BeExactly 'Perftrack_ConsoleStartupStop:PowershellConsoleStartup.WinStop.Informational' - # if there are more items than expected... - if ($items.Count -gt 2) - { - # Force reporting of the first unexpected item to help diagnosis - $items[2] | Should -Be $null + try { + $configFile = WriteLogSettings -LogId $logId + $testPid = & $powershell -NoProfile -SettingsFile $configFile -Command '$PID' + + Export-PSOsLog -After $after -LogPid $testPid -TimeoutInMilliseconds 30000 -IntervalInMilliseconds 3000 -MinimumCount 2 | + Set-Content -Path $contentFile + $items = @(Get-PSOsLog -Path $contentFile -Id $logId -After $after -TotalCount 3 -Verbose) + + $items | Should -Not -Be $null + $items.Count | Should -BeGreaterThan 1 + $items[0].EventId | Should -BeExactly 'Perftrack_ConsoleStartupStart:PowershellConsoleStartup.WinStart.Informational' + $items[1].EventId | Should -BeExactly 'Perftrack_ConsoleStartupStop:PowershellConsoleStartup.WinStop.Informational' + # if there are more items than expected... + if ($items.Count -gt 2) + { + # Force reporting of the first unexpected item to help diagnosis + $items[2] | Should -Be $null + } + } + catch { + if (Test-Path $contentFile) { + Send-VstsLogFile -Path $contentFile + } + throw } } - It 'Verifies logging level filtering works' -Skip:(!$IsSupportedEnvironment) { - $configFile = WriteLogSettings -LogId $logId -LogLevel Warning - $testPid = & $powershell -NoLogo -NoProfile -SettingsFile $configFile -Command '$PID' - - Export-PSOsLog -After $after -LogPid $testPid -Verbose | Set-Content -Path $contentFile - # by default, powershell startup should only logs informational events. - # With Level = Warning, nothing should be logged. - $items = Get-PSOsLog -Path $contentFile -Id $logId -After $after -TotalCount 3 - $items | Should -Be $null + # This is pending because it results in false postitives (-Skip:(!$IsSupportedEnvironment) ) + It 'Verifies logging level filtering works' -Pending { + try { + $configFile = WriteLogSettings -LogId $logId -LogLevel Warning + $testPid = & $powershell -NoLogo -NoProfile -SettingsFile $configFile -Command '$PID' + + Export-PSOsLog -After $after -LogPid $testPid | + Set-Content -Path $contentFile + # by default, powershell startup should only logs informational events. + # With Level = Warning, nothing should be logged. + $items = Get-PSOsLog -Path $contentFile -Id $logId -After $after -TotalCount 3 + $items | Should -Be $null + } + catch { + if (Test-Path $contentFile) { + Send-VstsLogFile -Path $contentFile + } + throw + } } } From 564ecda1ec4fe330495e3a44086dab38a17c62b3 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 9 Aug 2018 17:18:55 -0700 Subject: [PATCH 11/19] publish different results separately because failures stop the whole publish --- vsts-macos-ci.yml | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/vsts-macos-ci.yml b/vsts-macos-ci.yml index 34f675b74a5..9343652a27d 100644 --- a/vsts-macos-ci.yml +++ b/vsts-macos-ci.yml @@ -74,9 +74,45 @@ phases: - task: PublishTestResults@2 condition: succeededOrFailed() - displayName: Publish Test Results **\Test*.xml + displayName: Publish Test Results TestResultsSudo.xml inputs: testRunner: NUnit - testResultsFiles: '**\Test*.xml' - testRunTitle: Windows + testResultsFiles: '**\TestResultsSudo.xml' + testRunTitle: Sudo + mergeTestResults: true + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: Publish Test Results TestResultsNoSudo.xml + inputs: + testRunner: NUnit + testResultsFiles: '**\TestResultsNoSudo.xml' + testRunTitle: NoSudo + mergeTestResults: true + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: Publish Test Results TestResultsSudo.ExpTest.*.xml + inputs: + testRunner: NUnit + testResultsFiles: '**\TestResultsSudo.ExpTest.*.xml' + testRunTitle: Sudo Experimental Features + mergeTestResults: true + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: Publish Test Results TestResultsNoSudo.ExpTest.*.xml + inputs: + testRunner: NUnit + testResultsFiles: '**\TestResultsNoSudo.ExpTest.*.xml' + testRunTitle: Sudo Experimental Features + mergeTestResults: true + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: Publish Test Results *XUnitTestResults.xml + inputs: + testRunner: XUnit + testResultsFiles: '**\*XUnitTestResults.xml' + testRunTitle: XUnit mergeTestResults: true From 47eabc31c4dc5b86caf19d9dc49088390b06727d Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 9 Aug 2018 18:24:18 -0700 Subject: [PATCH 12/19] move yml to .vsts-ci folder and rename mac.yml --- vsts-macos-ci.yml => .vsts-ci/mac.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename vsts-macos-ci.yml => .vsts-ci/mac.yml (100%) diff --git a/vsts-macos-ci.yml b/.vsts-ci/mac.yml similarity index 100% rename from vsts-macos-ci.yml rename to .vsts-ci/mac.yml From fabc011221b8faa921df32feb63282263983d5f2 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 9 Aug 2018 18:33:12 -0700 Subject: [PATCH 13/19] remove trailing whitespace --- .vsts-ci/mac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml index 9343652a27d..3595d347628 100644 --- a/.vsts-ci/mac.yml +++ b/.vsts-ci/mac.yml @@ -24,7 +24,7 @@ phases: Write-Host "##vso[task.setvariable variable=TMPDIR]$env:AGENT_TEMPDIRECTORY" displayName: Set TMPDIR to AGENT_TEMPDIRECTORY condition: succeededOrFailed() - + - powershell: | git submodule update --init displayName: SubModule Init From 5546e006b5d9de0ed2b29a0bf84c378983e4c5b8 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 9 Aug 2018 18:40:26 -0700 Subject: [PATCH 14/19] Add comments about artifact uploading vs publishing test results --- .vsts-ci/mac.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml index 3595d347628..bc505ef812e 100644 --- a/.vsts-ci/mac.yml +++ b/.vsts-ci/mac.yml @@ -58,20 +58,24 @@ phases: displayName: After Success condition: succeeded() + # Uploads any packages as an artifact - powershell: | Get-ChildItem -Path *.pkg, *.tar.gz -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object { Write-Host "##vso[artifact.upload containerfolder=artifacts;artifactname=artifacts]$_" } displayName: Publish Artifacts - condition: succeeded() + condition: succeededOrFailed() + # Uploads any Test results as an artifact - powershell: | Get-ChildItem -Path Test*.xml -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object { Write-Host "##vso[artifact.upload containerfolder=testResults;artifactname=testResults]$_" } displayName: Publish Test Results - condition: succeeded() + condition: succeededOrFailed() + # Publish the various Test results as Test results to VSTS + # Separate tasks are used because a failure will stop the task - task: PublishTestResults@2 condition: succeededOrFailed() displayName: Publish Test Results TestResultsSudo.xml @@ -110,9 +114,18 @@ phases: - task: PublishTestResults@2 condition: succeededOrFailed() - displayName: Publish Test Results *XUnitTestResults.xml + displayName: Publish Test Results ParallelXUnitTestResults.xml + inputs: + testRunner: Parallel XUnit + testResultsFiles: '**\ParallelXUnitTestResults.xml' + testRunTitle: XUnit + mergeTestResults: true + + - task: PublishTestResults@2 + condition: succeededOrFailed() + displayName: Publish Test Results SequentialXUnitTestResults.xml inputs: - testRunner: XUnit - testResultsFiles: '**\*XUnitTestResults.xml' + testRunner: Sequential XUnit + testResultsFiles: '**\SequentialXUnitTestResults.xml' testRunTitle: XUnit mergeTestResults: true From 2c6a4b1dbad0f1b55890435a8a89a89ad2d0606e Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 9 Aug 2018 18:21:18 -0700 Subject: [PATCH 15/19] remove SkipInVSTS --- test/powershell/Host/Logging.Tests.ps1 | 2 +- tools/travis.ps1 | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/test/powershell/Host/Logging.Tests.ps1 b/test/powershell/Host/Logging.Tests.ps1 index 863e0272ccd..fcad56fc59d 100644 --- a/test/powershell/Host/Logging.Tests.ps1 +++ b/test/powershell/Host/Logging.Tests.ps1 @@ -159,7 +159,7 @@ Describe 'Basic SysLog tests on Linux' -Tag @('CI','RequireSudoOnUnix') { } } -Describe 'Basic os_log tests on MacOS' -Tag @('Feature','RequireSudoOnUnix','SkipInVsts') { +Describe 'Basic os_log tests on MacOS' -Tag @('Feature','RequireSudoOnUnix') { BeforeAll { [bool] $IsSupportedEnvironment = $IsMacOS [bool] $persistenceEnabled = $false diff --git a/tools/travis.ps1 b/tools/travis.ps1 index a5845112ec5..e0dcb95c438 100644 --- a/tools/travis.ps1 +++ b/tools/travis.ps1 @@ -206,12 +206,6 @@ elseif($Stage -eq 'Build') $excludeTag = @('RequireSudoOnUnix') - if($env:TF_BUILD) - { - # Used to skip tests that fail in VSTS - $excludeTag += 'SkipInVSTS' - } - $noSudoPesterParam = @{ 'BinDir' = $output 'PassThru' = $true @@ -264,12 +258,6 @@ elseif($Stage -eq 'Build') $sudoPesterParam.Remove('Path') $sudoPesterParam['Tag'] = @('RequireSudoOnUnix') $sudoPesterParam['ExcludeTag'] = @() - if($env:TF_BUILD) - { - # Used to skip tests that fail in VSTS - $sudoPesterParam['ExcludeTag'] = @('SkipInVSTS') - } - $sudoPesterParam['Sudo'] = $true $sudoPesterParam['OutputFile'] = $testResultsSudo $pesterPassThruSudoObject = Start-PSPester @sudoPesterParam From bf55cdda70d75e4cd0d130670737945048f5047d Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 9 Aug 2018 19:18:06 -0700 Subject: [PATCH 16/19] fix test runner for xunit tests --- .vsts-ci/mac.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml index bc505ef812e..e943b1aabc3 100644 --- a/.vsts-ci/mac.yml +++ b/.vsts-ci/mac.yml @@ -116,16 +116,16 @@ phases: condition: succeededOrFailed() displayName: Publish Test Results ParallelXUnitTestResults.xml inputs: - testRunner: Parallel XUnit + testRunner: XUnit testResultsFiles: '**\ParallelXUnitTestResults.xml' - testRunTitle: XUnit + testRunTitle: Parallel XUnit mergeTestResults: true - task: PublishTestResults@2 condition: succeededOrFailed() displayName: Publish Test Results SequentialXUnitTestResults.xml inputs: - testRunner: Sequential XUnit + testRunner: XUnit testResultsFiles: '**\SequentialXUnitTestResults.xml' - testRunTitle: XUnit + testRunTitle: Sequential XUnit mergeTestResults: true From a82840d57a72906f2e93bc4552c77e8fd57997cd Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Thu, 9 Aug 2018 19:19:40 -0700 Subject: [PATCH 17/19] also publish xunit test results --- .vsts-ci/mac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml index e943b1aabc3..273f2f89ecf 100644 --- a/.vsts-ci/mac.yml +++ b/.vsts-ci/mac.yml @@ -68,7 +68,7 @@ phases: # Uploads any Test results as an artifact - powershell: | - Get-ChildItem -Path Test*.xml -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object { + Get-ChildItem -Path Test*.xml, *XUnitTestResults.xml -Recurse | Select-Object -ExpandProperty FullName | ForEach-Object { Write-Host "##vso[artifact.upload containerfolder=testResults;artifactname=testResults]$_" } displayName: Publish Test Results From 5f49180b076c031dc18c78e4ccad4aca17034fd5 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Fri, 10 Aug 2018 08:44:48 -0700 Subject: [PATCH 18/19] remove SkipInVsts tag from build.psm1 --- build.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.psm1 b/build.psm1 index 6c63007ae43..44ddfa639d1 100644 --- a/build.psm1 +++ b/build.psm1 @@ -909,7 +909,7 @@ function Get-PesterTag { } $values = $vAst.FindAll({$args[0] -is "System.Management.Automation.Language.StringConstantExpressionAst"},$true).Value $values | ForEach-Object { - if (@('REQUIREADMINONWINDOWS', 'REQUIRESUDOONUNIX', 'SLOW', 'SKIPINVSTS') -contains $_) { + if (@('REQUIREADMINONWINDOWS', 'REQUIRESUDOONUNIX', 'SLOW') -contains $_) { # These are valid tags also, but they are not the priority tags } elseif (@('CI', 'FEATURE', 'SCENARIO') -contains $_) { From 6c3d918a06f4d224e7f5b42b4ddb118d1ad70b11 Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Fri, 10 Aug 2018 14:10:59 -0700 Subject: [PATCH 19/19] enable debug mode --- .vsts-ci/mac.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml index 273f2f89ecf..019b4e027d3 100644 --- a/.vsts-ci/mac.yml +++ b/.vsts-ci/mac.yml @@ -4,6 +4,8 @@ variables: POWERSHELL_TELEMETRY_OPTOUT: 1 # Avoid expensive initialization of dotnet cli, see: http://donovanbrown.com/post/Stop-wasting-time-during-NET-Core-builds DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + # Enable VSTS debug mode until stabilitized + system.debug: 'true' resources: - repo: self