From 38a6bda223856261aa7e5cd8ef1d2bed26619656 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Tue, 7 Mar 2023 13:42:50 -0800 Subject: [PATCH 01/13] Use dotnet to generate reference assemblies --- tools/packaging/packaging.psm1 | 14 +++--- .../azureDevOps/templates/nuget-pkg-sbom.yml | 50 ++++++++++++++++++- .../azureDevOps/templates/nuget.yml | 9 ---- 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 6eae9550503..af1a21c0b72 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -2183,10 +2183,11 @@ function New-ILNugetPackageSource [string] $LinuxFxdBinPath, [Parameter(Mandatory = $true)] - [string] $GenAPIToolPath, + [string] $RefAssemblyPath, [Parameter(Mandatory = $true)] [string] $CGManifestPath + ) if (! $Environment.IsWindows) @@ -2219,11 +2220,8 @@ function New-ILNugetPackageSource "Microsoft.WSMan.Management.dll", "Microsoft.WSMan.Runtime.dll") - $refBinPath = New-TempFolder $SnkFilePath = "$RepoRoot\src\signing\visualstudiopublic.snk" - New-ReferenceAssembly -linux64BinPath $LinuxFxdBinPath -RefAssemblyDestinationPath $refBinPath -RefAssemblyVersion $PackageVersion -SnkFilePath $SnkFilePath -GenAPIToolPath $GenAPIToolPath - if (! (Test-Path $PackagePath)) { $null = New-Item -Path $PackagePath -ItemType Directory } @@ -2237,7 +2235,7 @@ function New-ILNugetPackageSource #region ref $refFolder = New-Item (Join-Path $filePackageFolder.FullName "ref/$script:netCoreRuntime") -ItemType Directory -Force - CopyReferenceAssemblies -assemblyName $fileBaseName -refBinPath $refBinPath -refNugetPath $refFolder -assemblyFileList $fileList -winBinPath $WinFxdBinPath + CopyReferenceAssemblies -assemblyName $fileBaseName -refBinPath $RefAssemblyPath -refNugetPath $refFolder -assemblyFileList $fileList -winBinPath $WinFxdBinPath #endregion ref $packageRuntimesFolderPath = $packageRuntimesFolder.FullName @@ -2392,7 +2390,11 @@ function CopyReferenceAssemblies $supportedRefList = @( "Microsoft.PowerShell.Commands.Utility", - "Microsoft.PowerShell.ConsoleHost") + "Microsoft.PowerShell.ConsoleHost", + "Microsoft.PowerShell.Commands.Management", + "Microsoft.PowerShell.Commands.Diagnostics", + "Microsoft.PowerShell.Commands.Security" + ) switch ($assemblyName) { { $_ -in $supportedRefList } { diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml index fe8c1a872ff..92c3daa8d6a 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml @@ -3,7 +3,6 @@ parameters: - name: PackagePath - name: WinFxdPath - name: LinuxFxdPath - - name: GenAPIToolPath - name: ListOfFiles type: object default: @@ -20,6 +19,53 @@ parameters: - System.Management.Automation.dll steps: + +- pwsh: | + Import-Module "$env:REPOROOT/build.psm1" -Force + Start-PSBootstrap + + $sharedModules = @('Microsoft.PowerShell.Commands.Management', + 'Microsoft.PowerShell.Commands.Utility', + 'Microsoft.PowerShell.ConsoleHost', + 'Microsoft.PowerShell.Security', + 'System.Management.Automation' + ) + + $winOnlyModules = @('Microsoft.Management.Infrastructure.CimCmdlets', + 'Microsoft.PowerShell.Commands.Diagnostics', + 'Microsoft.PowerShell.CoreCLR.Eventing', + 'Microsoft.WSMan.Management', + 'Microsoft.WSMan.Runtime' + ) + + $refAssemblyFolder = Join-Path '$(System.ArtifactsDirectory)' 'RefAssembly' + $null = New-Item -Path $refAssemblyFolder -Force -Verbose + + Start-PSBuild -Clean -Runtime linux-x64 -Configuration Release + + $sharedModules | Foreach-Object { + $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" + Write-Verbose -Verbose 'RefAssembly: $refFile' + Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose + } + + Start-PSBuild -Clean -Runtime win7-x64 -Configuration Release + + $winOnlyModules | Foreach-Object { + $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" + Write-Verbose -Verbose 'RefAssembly: $refFile' + Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose + } + + Get-ChildItem $refAssemblyFolder | Out-String | Write-Verbose -Verbose + + # Set RefAssemblyPath path variable + $vstsCommandString = "vso[task.setvariable variable=RefAssemblyPath]${refAssemblyFolder}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + + displayName: Build reference assemblies + - ${{ each value in parameters.ListOfFiles }}: - pwsh: | $FileName = '${{ value }}' @@ -45,7 +91,7 @@ steps: Import-Module -Name $env:REPOROOT\build.psm1 Import-Module -Name $env:REPOROOT\tools\packaging Find-DotNet - New-ILNugetPackageSource -File $FileName -PackagePath '${{ parameters.PackagePath }}' -PackageVersion '${{ parameters.PackageVersion }}' -WinFxdBinPath '${{ parameters.WinFxdPath }}' -LinuxFxdBinPath '${{ parameters.LinuxFxdPath }}' -GenAPIToolPath '${{ parameters.GenAPIToolPath }}' -CGManifestPath $CGManifestPath + New-ILNugetPackageSource -File $FileName -PackagePath '${{ parameters.PackagePath }}' -PackageVersion '${{ parameters.PackageVersion }}' -WinFxdBinPath '${{ parameters.WinFxdPath }}' -LinuxFxdBinPath '${{ parameters.LinuxFxdPath }}' -CGManifestPath $CGManifestPath -RefAssemblyPath $(RefAssemblyPath) displayName: 'Create NuGet Package source for single file' - template: Sbom.yml@ComplianceRepo diff --git a/tools/releaseBuild/azureDevOps/templates/nuget.yml b/tools/releaseBuild/azureDevOps/templates/nuget.yml index f38e1a718c6..269b6149645 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget.yml @@ -106,14 +106,6 @@ jobs: archiveFilePatterns: '$(System.ArtifactsDirectory)/packages/powershell-*-linux-x64-fxdependent.tar.gz' destinationFolder: '$(linuxFxdPath)' - - task: PkgESInstallNuGetToolsPackage@10 - displayName: 'Install package Microsoft.DotNet.BuildTools.GenAPI' - inputs: - packageName: Microsoft.DotNet.BuildTools.GenAPI - packageVersion: '1.0.0-beta-00081' - packageSources: 'https://nuget.org/api/v2' - installRoot: '$(GenAPIToolPath)' - - template: SetVersionVariables.yml parameters: ReleaseTagVar: $(ReleaseTagVar) @@ -130,7 +122,6 @@ jobs: PackagePath: $(PackagePath) WinFxdPath: $(winFxdPath) LinuxFxdPath: $(linuxFxdPath) - GenAPIToolPath: $(GenAPIToolPath) - pwsh: | Get-ChildItem $(linuxFxdPath) From 2e7939549c74ddeeb947ccd21a3024cd026203e8 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Tue, 7 Mar 2023 17:13:35 -0800 Subject: [PATCH 02/13] Add xml files and more logging --- tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml index 92c3daa8d6a..73121e131db 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml @@ -47,6 +47,8 @@ steps: $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" Write-Verbose -Verbose 'RefAssembly: $refFile' Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose + $refDoc = Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\publish\$_.xml" + Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose } Start-PSBuild -Clean -Runtime win7-x64 -Configuration Release @@ -55,9 +57,11 @@ steps: $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" Write-Verbose -Verbose 'RefAssembly: $refFile' Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose + $refDoc = Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\publish\$_.xml" + Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose } - Get-ChildItem $refAssemblyFolder | Out-String | Write-Verbose -Verbose + Get-ChildItem $refAssemblyFolder -Recurse | Out-String | Write-Verbose -Verbose # Set RefAssemblyPath path variable $vstsCommandString = "vso[task.setvariable variable=RefAssemblyPath]${refAssemblyFolder}" From 504762996fbea0eb2d5ee4a6a7c47ec95fe4d83b Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Tue, 7 Mar 2023 17:47:35 -0800 Subject: [PATCH 03/13] Fix refdoc path --- tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml index 73121e131db..5921c15fe13 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml @@ -47,7 +47,7 @@ steps: $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" Write-Verbose -Verbose 'RefAssembly: $refFile' Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose - $refDoc = Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\publish\$_.xml" + $refDoc = Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\linux-x64\publish\$_.xml" Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose } @@ -57,7 +57,7 @@ steps: $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" Write-Verbose -Verbose 'RefAssembly: $refFile' Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose - $refDoc = Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\publish\$_.xml" + $refDoc = Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\win7-x64\publish\$_.xml" Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose } From 1877162803dbb63783c320b040d3f6d94371ac9d Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 8 Mar 2023 10:10:18 -0800 Subject: [PATCH 04/13] Add warning --- .../azureDevOps/templates/nuget-pkg-sbom.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml index 5921c15fe13..28e7e54e329 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml @@ -48,7 +48,12 @@ steps: Write-Verbose -Verbose 'RefAssembly: $refFile' Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose $refDoc = Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\linux-x64\publish\$_.xml" - Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose + if (-not (Test-Path $refDoc)) { + Write-Warning "$env:REPOROOT\src\$_\bin\Release\net8.0\linux-x64\publish\$_.xml not found" + } + else { + Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose + } } Start-PSBuild -Clean -Runtime win7-x64 -Configuration Release @@ -57,8 +62,12 @@ steps: $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" Write-Verbose -Verbose 'RefAssembly: $refFile' Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose - $refDoc = Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\win7-x64\publish\$_.xml" - Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose + if (-not (Test-Path $refDoc)) { + $refDoc = Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\win7-x64\publish\$_.xml" + } + else { + Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose + } } Get-ChildItem $refAssemblyFolder -Recurse | Out-String | Write-Verbose -Verbose From 7d97982cd21bf38187b9d54af6f414f8d051cf7b Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 8 Mar 2023 10:35:30 -0800 Subject: [PATCH 05/13] Fix paths --- .../azureDevOps/templates/nuget-pkg-sbom.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml index 28e7e54e329..f1b8204cff2 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml @@ -45,11 +45,12 @@ steps: $sharedModules | Foreach-Object { $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" - Write-Verbose -Verbose 'RefAssembly: $refFile' + Write-Verbose -Verbose "RefAssembly: $refFile" Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose - $refDoc = Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\linux-x64\publish\$_.xml" + $refDoc = "$env:REPOROOT\src\$_\bin\Release\net8.0\linux-x64\publish\$_.xml" if (-not (Test-Path $refDoc)) { - Write-Warning "$env:REPOROOT\src\$_\bin\Release\net8.0\linux-x64\publish\$_.xml not found" + Write-Warning "$refDoc not found" + Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\linux-x64\publish" | Out-String | Write-Verbose -Verbose } else { Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose @@ -62,8 +63,10 @@ steps: $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" Write-Verbose -Verbose 'RefAssembly: $refFile' Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose + $refDoc = "$env:REPOROOT\src\$_\bin\Release\net8.0\win7-x64\publish\$_.xml" if (-not (Test-Path $refDoc)) { - $refDoc = Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\win7-x64\publish\$_.xml" + Write-Warning "$refDoc not found" + Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\win7-x64\publish" | Out-String | Write-Verbose -Verbose } else { Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose From 0707b481051364ec8c8b7f746739f46313f0f8ec Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 8 Mar 2023 11:10:25 -0800 Subject: [PATCH 06/13] Use the xml from build folder --- .../releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml index f1b8204cff2..9f80618b8a1 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml @@ -47,10 +47,10 @@ steps: $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" Write-Verbose -Verbose "RefAssembly: $refFile" Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose - $refDoc = "$env:REPOROOT\src\$_\bin\Release\net8.0\linux-x64\publish\$_.xml" + $refDoc = "$env:REPOROOT\src\$_\bin\Release\net8.0\$_.xml" if (-not (Test-Path $refDoc)) { Write-Warning "$refDoc not found" - Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\linux-x64\publish" | Out-String | Write-Verbose -Verbose + Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\" | Out-String | Write-Verbose -Verbose } else { Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose @@ -63,10 +63,10 @@ steps: $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" Write-Verbose -Verbose 'RefAssembly: $refFile' Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose - $refDoc = "$env:REPOROOT\src\$_\bin\Release\net8.0\win7-x64\publish\$_.xml" + $refDoc = "$env:REPOROOT\src\$_\bin\Release\net8.0\$_.xml" if (-not (Test-Path $refDoc)) { Write-Warning "$refDoc not found" - Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\win7-x64\publish" | Out-String | Write-Verbose -Verbose + Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0" | Out-String | Write-Verbose -Verbose } else { Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose From a7b75c63546bbcfc55959bc3f6df1f73e6929202 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 8 Mar 2023 11:33:09 -0800 Subject: [PATCH 07/13] Fix copy --- .../azureDevOps/templates/nuget-pkg-sbom.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml index 9f80618b8a1..fad2473242f 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml @@ -44,16 +44,16 @@ steps: Start-PSBuild -Clean -Runtime linux-x64 -Configuration Release $sharedModules | Foreach-Object { - $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" + $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\$_.dll" Write-Verbose -Verbose "RefAssembly: $refFile" - Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose + Copy-Item -Path $refFile -Destination "$refAssemblyFolder\$_.dll" -Verbose $refDoc = "$env:REPOROOT\src\$_\bin\Release\net8.0\$_.xml" if (-not (Test-Path $refDoc)) { Write-Warning "$refDoc not found" Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0\" | Out-String | Write-Verbose -Verbose } else { - Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose + Copy-Item -Path $refDoc -Destination "$refAssemblyFolder\$_.xml" -Verbose } } @@ -62,14 +62,14 @@ steps: $winOnlyModules | Foreach-Object { $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" Write-Verbose -Verbose 'RefAssembly: $refFile' - Copy-Item -Path $refFile -Destination $refAssemblyFolder -Verbose + Copy-Item -Path $refFile -Destination"$refAssemblyFolder\$_.dll" -Verbose $refDoc = "$env:REPOROOT\src\$_\bin\Release\net8.0\$_.xml" if (-not (Test-Path $refDoc)) { Write-Warning "$refDoc not found" Get-ChildItem -Path "$env:REPOROOT\src\$_\bin\Release\net8.0" | Out-String | Write-Verbose -Verbose } else { - Copy-Item -Path $refDoc -Destination $refAssemblyFolder -Verbose + Copy-Item -Path $refDoc -Destination "$refAssemblyFolder\$_.xml" -Verbose } } From 9aa992594ede1b5fdcbd0c7b8fe1e186c0220e9f Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 8 Mar 2023 11:49:22 -0800 Subject: [PATCH 08/13] Create folder --- tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml index fad2473242f..0808eabc4b2 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml @@ -39,7 +39,7 @@ steps: ) $refAssemblyFolder = Join-Path '$(System.ArtifactsDirectory)' 'RefAssembly' - $null = New-Item -Path $refAssemblyFolder -Force -Verbose + $null = New-Item -Path $refAssemblyFolder -Force -Verbose -Type Directory Start-PSBuild -Clean -Runtime linux-x64 -Configuration Release From 6d86fed0e1f410698eff3c68a0155e606fe1e585 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 8 Mar 2023 13:00:41 -0800 Subject: [PATCH 09/13] fix typo --- tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml index 0808eabc4b2..f0a033fd9e4 100644 --- a/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml +++ b/tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml @@ -62,7 +62,7 @@ steps: $winOnlyModules | Foreach-Object { $refFile = Get-ChildItem -Path "$env:REPOROOT\src\$_\obj\Release\net8.0\refint\*.dll" Write-Verbose -Verbose 'RefAssembly: $refFile' - Copy-Item -Path $refFile -Destination"$refAssemblyFolder\$_.dll" -Verbose + Copy-Item -Path $refFile -Destination "$refAssemblyFolder\$_.dll" -Verbose $refDoc = "$env:REPOROOT\src\$_\bin\Release\net8.0\$_.xml" if (-not (Test-Path $refDoc)) { Write-Warning "$refDoc not found" From 0ef2de71d936d9103de7255dbb3ee9c2920007db Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 8 Mar 2023 13:40:07 -0800 Subject: [PATCH 10/13] Remove old variable --- tools/packaging/packaging.psm1 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index af1a21c0b72..b8bbd7ca423 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -2300,10 +2300,6 @@ function New-ILNugetPackageSource } $deps = New-FileDependencies -FileBaseName $fileBaseName -PackageVersion $PackageVersion New-CGManifest -FilePath (Join-Path -Path $CGManifestPath -ChildPath "CGManifest.json") -Dependencies $deps - - if (Test-Path $refBinPath) { - Remove-Item $refBinPath -Recurse -Force -ErrorAction SilentlyContinue - } } <# From d401ceb52745e6444a0219df45fc573ece3641d8 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 8 Mar 2023 15:19:06 -0800 Subject: [PATCH 11/13] fix ref files for sdk --- tools/packaging/packaging.psm1 | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index b8bbd7ca423..b49ac903c62 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -2393,30 +2393,26 @@ function CopyReferenceAssemblies ) switch ($assemblyName) { - { $_ -in $supportedRefList } { - $refDll = Join-Path -Path $refBinPath -ChildPath "$assemblyName.dll" - $refDoc = Join-Path -Path $refBinPath -ChildPath "$assemblyName.xml" - Copy-Item $refDll, $refDoc -Destination $refNugetPath -Force - Write-Log "Copied file '$refDll' and '$refDoc' to '$refNugetPath'" - } - "Microsoft.PowerShell.SDK" { foreach ($asmFileName in $assemblyFileList) { - $refFile = Join-Path -Path $refBinPath -ChildPath $asmFileName - if (Test-Path -Path $refFile) { - $refDoc = Join-Path -Path $refBinPath -ChildPath ([System.IO.Path]::ChangeExtension($asmFileName, "xml")) - Copy-Item $refFile, $refDoc -Destination $refNugetPath -Force - Write-Log "Copied file '$refFile' and '$refDoc' to '$refNugetPath'" + $fileName = [System.IO.Path]::GetFileNameWithoutExtension($asmFileName) + + if ($fileName -in $supportedRefList) { + $refFile = Join-Path -Path $refBinPath -ChildPath $asmFileName + if (Test-Path -Path $refFile) { + $refDoc = Join-Path -Path $refBinPath -ChildPath ([System.IO.Path]::ChangeExtension($asmFileName, "xml")) + Copy-Item $refFile, $refDoc -Destination $refNugetPath -Force + Write-Log "Copied file '$refFile' and '$refDoc' to '$refNugetPath'" + } } } } default { - $ref_SMA = Join-Path -Path $refBinPath -ChildPath System.Management.Automation.dll - $ref_doc = Join-Path -Path $refBinPath -ChildPath System.Management.Automation.xml - $self_ref_doc = Join-Path -Path $winBinPath -ChildPath "$assemblyName.xml" - Copy-Item $ref_SMA, $ref_doc, $self_ref_doc -Destination $refNugetPath -Force - Write-Log "Copied file '$ref_SMA' and '$ref_doc' to '$refNugetPath'" + $refDll = Join-Path -Path $refBinPath -ChildPath "$assemblyName.dll" + $refDoc = Join-Path -Path $refBinPath -ChildPath "$assemblyName.xml" + Copy-Item $refDll, $refDoc -Destination $refNugetPath -Force + Write-Log "Copied file '$refDll' and '$refDoc' to '$refNugetPath'" } } } From c8224ed7b6ac5c458e194a955e1b0e309195e819 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 8 Mar 2023 16:18:42 -0800 Subject: [PATCH 12/13] Add SMa --- tools/packaging/packaging.psm1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index b49ac903c62..432d36f818b 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -2388,8 +2388,8 @@ function CopyReferenceAssemblies "Microsoft.PowerShell.Commands.Utility", "Microsoft.PowerShell.ConsoleHost", "Microsoft.PowerShell.Commands.Management", - "Microsoft.PowerShell.Commands.Diagnostics", - "Microsoft.PowerShell.Commands.Security" + "Microsoft.PowerShell.Commands.Security", + "System.Management.Automation" ) switch ($assemblyName) { From 0d30ccfcccf23a246cfc88a5e9ab5f68d640de77 Mon Sep 17 00:00:00 2001 From: Aditya Patwardhan Date: Wed, 8 Mar 2023 18:56:07 -0800 Subject: [PATCH 13/13] Remove dead code --- tools/packaging/packaging.psm1 | 326 +-------------------------------- 1 file changed, 5 insertions(+), 321 deletions(-) diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1 index 432d36f818b..17f4031429e 100644 --- a/tools/packaging/packaging.psm1 +++ b/tools/packaging/packaging.psm1 @@ -2160,8 +2160,11 @@ Path to source folder containing Windows framework dependent assemblies. .PARAMETER LinuxFxdBinPath Path to source folder containing Linux framework dependent assemblies. -.PARAMETER GenAPIToolPath -Path to the GenAPI.exe tool. +.PARAMETER RefAssemblyPath +Path to the reference assemblies. + +.PARAMETER CGManifestPath +Path to the CGManifest.json file. #> function New-ILNugetPackageSource { @@ -2513,325 +2516,6 @@ function New-NuSpec { $nuspecObj.Save($filePath) } -<# -.SYNOPSIS -Create a reference assembly from System.Management.Automation.dll - -.DESCRIPTION -A unix variant of System.Management.Automation.dll is converted to a reference assembly. -GenAPI.exe generated the CS file containing the APIs. -This file is cleaned up and then compiled into a dll. - -.PARAMETER Unix64BinPath -Path to the folder containing unix 64 bit assemblies. - -.PARAMETER RefAssemblyDestinationPath -Path to the folder where the reference assembly is created. - -.PARAMETER RefAssemblyVersion -Version of the reference assembly. - -.PARAMETER GenAPIToolPath -Path to GenAPI.exe. Tool from https://www.nuget.org/packages/Microsoft.DotNet.BuildTools.GenAPI/ - -.PARAMETER SnkFilePath -Path to the snk file for strong name signing. -#> - -function New-ReferenceAssembly -{ - param( - [Parameter(Mandatory = $true)] - [string] $Linux64BinPath, - - [Parameter(Mandatory = $true)] - [string] $RefAssemblyDestinationPath, - - [Parameter(Mandatory = $true)] - [string] $RefAssemblyVersion, - - [Parameter(Mandatory = $true)] - [string] $GenAPIToolPath, - - [Parameter(Mandatory = $true)] - [string] $SnkFilePath - ) - - if (-not $Environment.IsWindows) - { - throw "New-ReferenceAssembly can be only executed on Windows platform." - } - - $genAPIExe = Get-ChildItem -Path "$GenAPIToolPath/*GenAPI.exe" -Recurse - - if (-not (Test-Path $genAPIExe)) - { - throw "GenAPI.exe was not found at: $GenAPIToolPath" - } - - Write-Log "GenAPI nuget package saved and expanded." - - $genAPIFolder = New-TempFolder - Write-Log "Working directory: $genAPIFolder." - - $SMAReferenceAssembly = $null - $assemblyNames = @( - "System.Management.Automation", - "Microsoft.PowerShell.Commands.Utility", - "Microsoft.PowerShell.ConsoleHost" - ) - - # Ensure needed dotNet version is available. Find-DotNet does this, and is part of build.psm1 which should already be imported. - Find-DotNet -Verbose - - foreach ($assemblyName in $assemblyNames) { - - Write-Log "Building reference assembly for '$assemblyName'" - $projectFolder = New-Item -Path "$genAPIFolder/$assemblyName" -ItemType Directory -Force - $generatedSource = Join-Path $projectFolder "$assemblyName.cs" - $filteredSource = Join-Path $projectFolder "${assemblyName}_Filtered.cs" - - $linuxDllPath = Join-Path $Linux64BinPath "$assemblyName.dll" - if (-not (Test-Path $linuxDllPath)) { - throw "$assemblyName.dll was not found at: $Linux64BinPath" - } - - $dllXmlDoc = Join-Path $Linux64BinPath "$assemblyName.xml" - if (-not (Test-Path $dllXmlDoc)) { - throw "$assemblyName.xml was not found at: $Linux64BinPath" - } - - $genAPIArgs = "$linuxDllPath","-libPath:$Linux64BinPath,$Linux64BinPath\ref" - Write-Log "GenAPI cmd: $genAPIExe $genAPIArgs" - - Start-NativeExecution { & $genAPIExe $genAPIArgs } | Out-File $generatedSource -Force - Write-Log "Reference assembly file generated at: $generatedSource" - - CleanupGeneratedSourceCode -assemblyName $assemblyName -generatedSource $generatedSource -filteredSource $filteredSource - - try - { - Push-Location $projectFolder - - $sourceProjectRoot = Join-Path $PSScriptRoot "projects/reference/$assemblyName" - $sourceProjectFile = Join-Path $sourceProjectRoot "$assemblyName.csproj" - - $destProjectFile = Join-Path $projectFolder "$assemblyName.csproj" - $nugetConfigFile = Join-Path $PSScriptRoot "../../nuget.config" - - Copy-Item -Path $sourceProjectFile -Destination $destProjectFile -Force -Verbose - Copy-Item -Path $nugetConfigFile -Destination $projectFolder -Verbose - - Write-Host "##vso[artifact.upload containerfolder=artifact;artifactname=artifact]$destProjectFile" - Write-Host "##vso[artifact.upload containerfolder=artifact;artifactname=artifact]$generatedSource" - - $arguments = GenerateBuildArguments -AssemblyName $assemblyName -RefAssemblyVersion $RefAssemblyVersion -SnkFilePath $SnkFilePath -SMAReferencePath $SMAReferenceAssembly - - Write-Log "Running: dotnet $arguments" - Start-NativeExecution -sb {dotnet $arguments} - - $refBinPath = Join-Path $projectFolder "bin/Release/$script:netCoreRuntime/$assemblyName.dll" - if ($null -eq $refBinPath) { - throw "Reference assembly was not built." - } - - Copy-Item $refBinPath $RefAssemblyDestinationPath -Force - Write-Log "Reference assembly '$assemblyName.dll' built and copied to $RefAssemblyDestinationPath" - - Copy-Item $dllXmlDoc $RefAssemblyDestinationPath -Force - Write-Log "Xml document '$assemblyName.xml' copied to $RefAssemblyDestinationPath" - - if ($assemblyName -eq "System.Management.Automation") { - $SMAReferenceAssembly = $refBinPath - } - } - finally - { - Pop-Location - } - } - - if (Test-Path $genAPIFolder) - { - Remove-Item $genAPIFolder -Recurse -Force -ErrorAction SilentlyContinue - } -} - -<# - Helper function for New-ReferenceAssembly to further clean up the - C# source code generated from GenApi.exe. -#> -function CleanupGeneratedSourceCode -{ - param( - [string] $assemblyName, - [string] $generatedSource, - [string] $filteredSource - ) - - $patternsToRemove = @( - '[System.Management.Automation.ArgumentToEncodingTransformationAttribute]' - 'typeof(System.Security.AccessControl.FileSecurity)' - '[System.Management.Automation.ArgumentTypeConverterAttribute' - '[System.Runtime.CompilerServices.IteratorStateMachineAttribute' - '[Microsoft.PowerShell.Commands.ArgumentToModuleTransformationAttribute]' - '[Microsoft.PowerShell.Commands.SetStrictModeCommand.ArgumentToVersionTransformationAttribute]' - '[Microsoft.PowerShell.Commands.SetStrictModeCommand.ValidateVersionAttribute]' - '[System.Management.Automation.OutputTypeAttribute(typeof(System.Management.Automation.PSRemotingJob))]' - 'typeof(System.Management.Automation.LanguagePrimitives.EnumMultipleTypeConverter)' - '[System.Management.Automation.Internal.CommonParameters.ValidateVariableName]' - '[System.Management.Automation.ArgumentEncodingCompletionsAttribute]' - '[Microsoft.PowerShell.Commands.AddMemberCommand' - '[System.Management.Automation.ArgumentCompleterAttribute(typeof(Microsoft.PowerShell.Commands.Utility.JoinItemCompleter))]' - '[System.Management.Automation.ArgumentCompleterAttribute(typeof(System.Management.Automation.PropertyNameCompleter))]' - '[Microsoft.PowerShell.Commands.ArgumentToTypeNameTransformationAttribute]' - '[System.Management.Automation.Internal.ArchitectureSensitiveAttribute]' - '[Microsoft.PowerShell.Commands.SelectStringCommand.FileinfoToStringAttribute]' - '[System.Runtime.CompilerServices.IsReadOnlyAttribute]' - '[System.Runtime.CompilerServices.NullableContextAttribute(' - '[System.Runtime.CompilerServices.NullableAttribute((byte)0)]' - '[System.Runtime.CompilerServices.NullableAttribute(new byte[]{ (byte)2, (byte)1, (byte)1})]' - '[System.Runtime.CompilerServices.AsyncStateMachineAttribute' - '[Microsoft.PowerShell.Commands.SetStrictModeCommand.ArgumentToPSVersionTransformationAttribute]' - '[Microsoft.PowerShell.Commands.HttpVersionCompletionsAttribute]' - '[System.Management.Automation.ArgumentToVersionTransformationAttribute]' - '[Microsoft.PowerShell.Commands.InvokeCommandCommand.ArgumentToPSVersionTransformationAttribute]' - '[Microsoft.PowerShell.Commands.InvokeCommandCommand.ValidateVersionAttribute]', - '[System.Management.Automation.OutputTypeAttribute(new System.Type[]{ typeof(Microsoft.PowerShell.Commands.Internal.Format.FormatStartData), typeof(Microsoft.PowerShell.Commands.Internal.Format.FormatEntryData), typeof(Microsoft.PowerShell.Commands.Internal.Format.FormatEndData), typeof(Microsoft.PowerShell.Commands.Internal.Format.GroupStartData), typeof(Microsoft.PowerShell.Commands.Internal.Format.GroupEndData)})]' - ) - - $patternsToReplace = @( - @{ - ApplyTo = @("Microsoft.PowerShell.Commands.Utility") - Pattern = "[System.Runtime.CompilerServices.IsReadOnlyAttribute]ref Microsoft.PowerShell.Commands.JsonObject.ConvertToJsonContext" - Replacement = "in Microsoft.PowerShell.Commands.JsonObject.ConvertToJsonContext" - }, - @{ - ApplyTo = @("Microsoft.PowerShell.Commands.Utility") - Pattern = "public partial struct ConvertToJsonContext" - Replacement = "public readonly struct ConvertToJsonContext" - }, - @{ - ApplyTo = @("Microsoft.PowerShell.Commands.Utility") - Pattern = "Unable to resolve assembly 'Assembly(Name=Newtonsoft.Json" - Replacement = "// Unable to resolve assembly 'Assembly(Name=Newtonsoft.Json" - }, - @{ - ApplyTo = @("System.Management.Automation") - Pattern = "Unable to resolve assembly 'Assembly(Name=System.Security.Principal.Windows" - Replacement = "// Unable to resolve assembly 'Assembly(Name=System.Security.Principal.Windows" - }, - @{ - ApplyTo = @("System.Management.Automation") - Pattern = "Unable to resolve assembly 'Assembly(Name=Microsoft.Management.Infrastructure" - Replacement = "// Unable to resolve assembly 'Assembly(Name=Microsoft.Management.Infrastructure" - }, - @{ - ApplyTo = @("System.Management.Automation") - Pattern = "Unable to resolve assembly 'Assembly(Name=System.Security.AccessControl" - Replacement = "// Unable to resolve assembly 'Assembly(Name=System.Security.AccessControl" - }, - @{ - ApplyTo = @("System.Management.Automation") - Pattern = "[System.Runtime.CompilerServices.NullableAttribute(new byte[]{ (byte)1, (byte)2, (byte)1})]" - Replacement = "/* [System.Runtime.CompilerServices.NullableAttribute(new byte[]{ (byte)1, (byte)2, (byte)1})] */ " - }, - @{ - ApplyTo = @("System.Management.Automation") - Pattern = "[System.Runtime.CompilerServices.NullableAttribute(new byte[]{ (byte)2, (byte)1})]" - Replacement = "/* [System.Runtime.CompilerServices.NullableAttribute(new byte[]{ (byte)2, (byte)1})] */ " - }, - @{ - ApplyTo = @("System.Management.Automation") - Pattern = "[System.Runtime.CompilerServices.CompilerGeneratedAttribute, System.Runtime.CompilerServices.NullableContextAttribute((byte)2)]" - Replacement = "/* [System.Runtime.CompilerServices.CompilerGeneratedAttribute, System.Runtime.CompilerServices.NullableContextAttribute((byte)2)] */ " - }, - @{ - ApplyTo = @("System.Management.Automation") - Pattern = "[System.Runtime.CompilerServices.CompilerGeneratedAttribute, System.Runtime.CompilerServices.IsReadOnlyAttribute]" - Replacement = "/* [System.Runtime.CompilerServices.CompilerGeneratedAttribute, System.Runtime.CompilerServices.IsReadOnlyAttribute] */ " - }, - @{ - ApplyTo = @("System.Management.Automation") - Pattern = "[System.Runtime.CompilerServices.CompilerGeneratedAttribute, System.Runtime.CompilerServices.NullableContextAttribute((byte)1)]" - Replacement = "/* [System.Runtime.CompilerServices.CompilerGeneratedAttribute, System.Runtime.CompilerServices.NullableContextAttribute((byte)1)] */ " - }, - @{ - ApplyTo = @("System.Management.Automation", "Microsoft.PowerShell.ConsoleHost") - Pattern = "[System.Runtime.CompilerServices.NullableAttribute((byte)2)]" - Replacement = "/* [System.Runtime.CompilerServices.NullableAttribute((byte)2)] */" - }, - @{ - ApplyTo = @("System.Management.Automation", "Microsoft.PowerShell.ConsoleHost") - Pattern = "[System.Runtime.CompilerServices.NullableAttribute((byte)1)]" - Replacement = "/* [System.Runtime.CompilerServices.NullableAttribute((byte)1)] */" - } - ) - - $reader = [System.IO.File]::OpenText($generatedSource) - $writer = [System.IO.File]::CreateText($filteredSource) - - while($null -ne ($line = $reader.ReadLine())) - { - $lineWasProcessed = $false - foreach ($patternToReplace in $patternsToReplace) - { - if ($assemblyName -in $patternToReplace.ApplyTo -and $line.Contains($patternToReplace.Pattern)) { - $line = $line.Replace($patternToReplace.Pattern, $patternToReplace.Replacement) - $lineWasProcessed = $true - } - } - - if (!$lineWasProcessed) { - $match = Select-String -InputObject $line -Pattern $patternsToRemove -SimpleMatch - if ($null -ne $match) - { - $line = "//$line" - } - } - - $writer.WriteLine($line) - } - - if ($null -ne $reader) - { - $reader.Close() - } - - if ($null -ne $writer) - { - $writer.Close() - } - - Move-Item $filteredSource $generatedSource -Force - Write-Log "Code cleanup complete for reference assembly '$assemblyName'." -} - -<# - Helper function for New-ReferenceAssembly to get the arguments - for building reference assemblies. -#> -function GenerateBuildArguments -{ - param( - [string] $AssemblyName, - [string] $RefAssemblyVersion, - [string] $SnkFilePath, - [string] $SMAReferencePath - ) - - $arguments = @('build') - $arguments += @('-c','Release') - $arguments += "/p:RefAsmVersion=$RefAssemblyVersion" - $arguments += "/p:SnkFile=$SnkFilePath" - - if ($AssemblyName -ne "System.Management.Automation") { - $arguments += "/p:SmaRefFile=$SMAReferencePath" - } - - return $arguments -} - <# .SYNOPSIS Create a NuGet package from a nuspec.