Skip to content

Clean up crossgen related build scripts also generate native symbols for R2R images #16297

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
TravisEz13 merged 3 commits into from
Oct 27, 2021
Merged

Clean up crossgen related build scripts also generate native symbols for R2R images #16297

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0d9f9c4
Clean up crossgen related build scripts
daxian-dbw Oct 22, 2021
04c6c5c
Fix ci.psm1
daxian-dbw Oct 23, 2021
7613beb
Clean up '-CrossGen' use in a few other files
daxian-dbw Oct 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions PowerShell.Common.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@
<TargetFramework>net6.0</TargetFramework>
<LangVersion>10.0</LangVersion>
<PublishReadyToRun Condition=" '$(Configuration)' != 'Debug' ">true</PublishReadyToRun>
<PublishReadyToRunEmitSymbols>true</PublishReadyToRunEmitSymbols>

<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
Expand Down
227 changes: 0 additions & 227 deletions build.psm1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,8 +314,6 @@ function Start-PSBuild {
[ValidateSet('Debug', 'Release', 'CodeCoverage', '')] # We might need "Checked" as well
[string]$Configuration,

[switch]$CrossGen,

[ValidatePattern("^v\d+\.\d+\.\d+(-\w+(\.\d{1,2})?)?$")]
[ValidateNotNullOrEmpty()]
[string]$ReleaseTag,
Expand Down Expand Up @@ -343,10 +341,6 @@ function Start-PSBuild {
}

if ($ForMinimalSize) {
if ($CrossGen) {
throw "Build for the minimal size requires the minimal disk footprint, so `CrossGen` is not allowed"
}

if ($Runtime -and "linux-x64", "win7-x64", "osx-x64" -notcontains $Runtime) {
throw "Build for the minimal size is enabled only for following runtimes: 'linux-x64', 'win7-x64', 'osx-x64'"
}
Expand Down Expand Up @@ -414,7 +408,6 @@ Fix steps:

# set output options
$OptionsArguments = @{
CrossGen=$CrossGen
Output=$Output
Runtime=$Runtime
Configuration=$Configuration
Expand Down Expand Up @@ -525,12 +518,6 @@ Fix steps:
Write-Log -message "Run dotnet $Arguments from $PWD"
Start-NativeExecution { dotnet $Arguments }
Write-Log -message "PowerShell output: $($Options.Output)"

if ($CrossGen) {
# fxdependent package cannot be CrossGen'ed
Start-CrossGen -PublishPath $publishPath -Runtime $script:Options.Runtime
Write-Log -message "pwsh.exe with ngen binaries is available at: $($Options.Output)"
}
} else {
$globalToolSrcFolder = Resolve-Path (Join-Path $Options.Top "../Microsoft.PowerShell.GlobalTool.Shim") | Select-Object -ExpandProperty Path

Expand Down Expand Up @@ -833,8 +820,6 @@ function New-PSOptions {
"win7-x86")]
[string]$Runtime,

[switch]$CrossGen,

# Accept a path to the output directory
# If not null or empty, name of the executable will be appended to
# this path, otherwise, to the default path, and then the full path
Expand Down Expand Up @@ -940,7 +925,6 @@ function New-PSOptions {
-RootInfo ([PSCustomObject]$RootInfo) `
-Top $Top `
-Runtime $Runtime `
-Crossgen $Crossgen.IsPresent `
-Configuration $Configuration `
-PSModuleRestore $PSModuleRestore.IsPresent `
-Framework $Framework `
Expand Down Expand Up @@ -2353,211 +2337,6 @@ function script:Start-NativeExecution
}
}

function Start-CrossGen {
[CmdletBinding()]
param(
[Parameter(Mandatory= $true)]
[ValidateNotNullOrEmpty()]
[String]
$PublishPath,

[Parameter(Mandatory=$true)]
[ValidateSet("alpine-x64",
"linux-arm",
"linux-arm64",
"linux-x64",
"osx-arm64",
"osx-x64",
"win-arm",
"win-arm64",
"win7-x64",
"win7-x86")]
[string]
$Runtime
)

function New-CrossGenAssembly {
param (
[Parameter(Mandatory = $true)]
[ValidateNotNullOrEmpty()]
[String[]]
$AssemblyPath,

[Parameter(Mandatory = $true)]
[ValidateNotNullOrEmpty()]
[String]
$CrossgenPath,

[Parameter(Mandatory = $true)]
[ValidateSet("alpine-x64",
"linux-arm",
"linux-arm64",
"linux-x64",
"osx-arm64",
"osx-x64",
"win-arm",
"win-arm64",
"win7-x64",
"win7-x86")]
[string]
$Runtime
)

$platformAssembliesPath = Split-Path $AssemblyPath[0] -Parent

$targetOS, $targetArch = $Runtime -split '-'

# Special cases where OS / Arch does not conform with runtime names
switch ($Runtime) {
'alpine-x64' {
$targetOS = 'linux'
$targetArch = 'x64'
}
'win-arm' {
$targetOS = 'windows'
$targetArch = 'arm'
}
'win-arm64' {
$targetOS = 'windows'
$targetArch = 'arm64'
}
'win7-x64' {
$targetOS = 'windows'
$targetArch = 'x64'
}
'win7-x86' {
$targetOS = 'windows'
$targetArch = 'x86'
}
}

$generatePdb = $targetos -eq 'windows'

# The path to folder must end with directory separator
$dirSep = [System.IO.Path]::DirectorySeparatorChar
$platformAssembliesPath = if (-not $platformAssembliesPath.EndsWith($dirSep)) { $platformAssembliesPath + $dirSep }

Start-NativeExecution {
$crossgen2Params = @(
"-r"
$platformAssembliesPath
"--out-near-input"
"--single-file-compilation"
"-O"
"--targetos"
$targetOS
"--targetarch"
$targetArch
)

if ($generatePdb) {
$crossgen2Params += "--pdb"
}

$crossgen2Params += $AssemblyPath

& $CrossgenPath $crossgen2Params
}
}

if (-not (Test-Path $PublishPath)) {
throw "Path '$PublishPath' does not exist."
}

# Get the path to crossgen
$crossGenExe = if ($environment.IsWindows) { "crossgen2.exe" } else { "crossgen2" }

# The crossgen tool is only published for these particular runtimes
$crossGenRuntime = if ($environment.IsWindows) {
# for windows the tool architecture is the host machine architecture, so it is always x64.
# we can cross compile for x86, arm and arm64
"win-x64"
} else {
$Runtime
}

if (-not $crossGenRuntime) {
throw "crossgen is not available for this platform"
}

$dotnetRuntimeVersion = $script:Options.Framework -replace 'net'

# Get the CrossGen.exe for the correct runtime with the latest version
$crossGenPath = Get-ChildItem $script:Environment.nugetPackagesRoot $crossGenExe -Recurse | `
Where-Object { $_.FullName -match $crossGenRuntime } | `
Where-Object { $_.FullName -match $dotnetRuntimeVersion } | `
Where-Object { (Split-Path $_.FullName -Parent).EndsWith('tools') } | `
Sort-Object -Property FullName -Descending | `
Select-Object -First 1 | `
ForEach-Object { $_.FullName }
if (-not $crossGenPath) {
throw "Unable to find latest version of crossgen2.exe. 'Please run Start-PSBuild -Clean' first, and then try again."
}
Write-Verbose "Matched CrossGen2.exe: $crossGenPath" -Verbose

# Common assemblies used by Add-Type or assemblies with high JIT and no pdbs to crossgen
$commonAssembliesForAddType = @(
"Microsoft.CodeAnalysis.CSharp.dll"
"Microsoft.CodeAnalysis.dll"
"System.Linq.Expressions.dll"
"Microsoft.CSharp.dll"
"System.Runtime.Extensions.dll"
"System.Linq.dll"
"System.Collections.Concurrent.dll"
"System.Collections.dll"
"Newtonsoft.Json.dll"
"System.IO.FileSystem.dll"
"System.Diagnostics.Process.dll"
"System.Threading.Tasks.Parallel.dll"
"System.Security.AccessControl.dll"
"System.Text.Encoding.CodePages.dll"
"System.Private.Uri.dll"
"System.Threading.dll"
"System.Security.Principal.Windows.dll"
"System.Console.dll"
"Microsoft.Win32.Registry.dll"
"System.IO.Pipes.dll"
"System.Diagnostics.FileVersionInfo.dll"
"System.Collections.Specialized.dll"
"Microsoft.ApplicationInsights.dll"
)

$fullAssemblyList = $commonAssembliesForAddType

$assemblyFullPaths = @()
$assemblyFullPaths += foreach ($assemblyName in $fullAssemblyList) {
Join-Path $PublishPath $assemblyName
}

New-CrossGenAssembly -CrossgenPath $crossGenPath -AssemblyPath $assemblyFullPaths -Runtime $Runtime

#
# With the latest dotnet.exe, the default load context is only able to load TPAs, and TPA
# only contains IL assembly names. In order to make the default load context able to load
# the NI PS assemblies, we need to replace the IL PS assemblies with the corresponding NI
# PS assemblies, but with the same IL assembly names.
#
Write-Verbose "PowerShell Ngen assemblies have been generated. Deploying ..." -Verbose
foreach ($assemblyName in $fullAssemblyList) {

# Remove the IL assembly and its symbols.
$assemblyPath = Join-Path $PublishPath $assemblyName
$symbolsPath = [System.IO.Path]::ChangeExtension($assemblyPath, ".pdb")

Remove-Item $assemblyPath -Force -ErrorAction Stop

# Rename the corresponding ni.dll assembly to be the same as the IL assembly
$niAssemblyPath = [System.IO.Path]::ChangeExtension($assemblyPath, "ni.dll")
Rename-Item $niAssemblyPath $assemblyPath -Force -ErrorAction Stop

# No symbols are available for Microsoft.CodeAnalysis.CSharp.dll, Microsoft.CodeAnalysis.dll,
# Microsoft.CodeAnalysis.VisualBasic.dll, and Microsoft.CSharp.dll.
if ($commonAssembliesForAddType -notcontains $assemblyName) {
Remove-Item $symbolsPath -Force -ErrorAction Stop
}
}
}

# Cleans the PowerShell repo - everything but the root folder
function Clear-PSRepo
{
Expand Down Expand Up @@ -3080,7 +2859,6 @@ function Restore-PSOptions {
-RootInfo $options.RootInfo `
-Top $options.Top `
-Runtime $options.Runtime `
-Crossgen $options.Crossgen `
-Configuration $options.Configuration `
-PSModuleRestore $options.PSModuleRestore `
-Framework $options.Framework `
Expand All @@ -3104,10 +2882,6 @@ function New-PSOptionsObject
[String]
$Runtime,

[Parameter(Mandatory)]
[Bool]
$CrossGen,

[Parameter(Mandatory)]
[String]
$Configuration,
Expand Down Expand Up @@ -3136,7 +2910,6 @@ function New-PSOptionsObject
Framework = $Framework
Runtime = $Runtime
Output = $Output
CrossGen = $CrossGen
PSModuleRestore = $PSModuleRestore
ForMinimalSize = $ForMinimalSize
}
Expand Down
6 changes: 3 additions & 3 deletions tools/ci.psm1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ function Invoke-CIBuild
Start-PSBuild -Configuration 'CodeCoverage' -PSModuleRestore -CI -ReleaseTag $releaseTag
}

Start-PSBuild -CrossGen -PSModuleRestore -Configuration 'Release' -CI -ReleaseTag $releaseTag
Start-PSBuild -PSModuleRestore -Configuration 'Release' -CI -ReleaseTag $releaseTag
Save-PSOptions

$options = (Get-PSOptions)
Expand Down Expand Up @@ -470,7 +470,7 @@ function Invoke-CIFinish
$prereleaseIteration = (get-date).Day
$preReleaseVersion = "$previewPrefix-$previewLabel.$prereleaseIteration"
# Build clean before backing to remove files from testing
Start-PSBuild -CrossGen -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean -Runtime $Runtime -output $buildFolder -PSOptionsPath "${buildFolder}/psoptions.json"
Start-PSBuild -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean -Runtime $Runtime -output $buildFolder -PSOptionsPath "${buildFolder}/psoptions.json"
$options = Get-PSOptions
# Remove symbol files.
$filter = Join-Path -Path (Split-Path $options.Output) -ChildPath '*.pdb'
Expand All @@ -481,7 +481,7 @@ function Invoke-CIFinish
$releaseTagParts = $releaseTag.split('.')
$preReleaseVersion = $releaseTagParts[0]+ ".9.9"
Write-Verbose "newPSReleaseTag: $preReleaseVersion" -Verbose
Start-PSBuild -CrossGen -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean -Runtime $Runtime -output $buildFolder -PSOptionsPath "${buildFolder}/psoptions.json"
Start-PSBuild -PSModuleRestore -Configuration 'Release' -ReleaseTag $preReleaseVersion -Clean -Runtime $Runtime -output $buildFolder -PSOptionsPath "${buildFolder}/psoptions.json"
$options = Get-PSOptions
# Remove symbol files.
$filter = Join-Path -Path (Split-Path $options.Output) -ChildPath '*.pdb'
Expand Down
24 changes: 1 addition & 23 deletions tools/packaging/packaging.psm1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,17 +117,6 @@ function Start-PSPackage {
$Script:Options = Get-PSOptions
$actualParams = @()

$crossGenCorrect = $false
if ($Runtime -match "arm" -or $Type -eq 'min-size') {
## crossgen doesn't support arm32/64;
## For the min-size package, we intentionally avoid crossgen.
$crossGenCorrect = $true
}
elseif ($Script:Options.CrossGen) {
$actualParams += '-CrossGen'
$crossGenCorrect = $true
}

$PSModuleRestoreCorrect = $false

# Require PSModuleRestore for packaging without symbols
Expand All @@ -144,14 +133,13 @@ function Start-PSPackage {
}

$precheckFailed = if ($Type -like 'fxdependent*' -or $Type -eq 'tar-alpine') {
## We do not check for runtime and crossgen for framework dependent package.
## We do not check on runtime for framework dependent package.
-not $Script:Options -or ## Start-PSBuild hasn't been executed yet
-not $PSModuleRestoreCorrect -or ## Last build didn't specify '-PSModuleRestore' correctly
$Script:Options.Configuration -ne $Configuration -or ## Last build was with configuration other than 'Release'
$Script:Options.Framework -ne $script:netCoreRuntime ## Last build wasn't for CoreCLR
} else {
-not $Script:Options -or ## Start-PSBuild hasn't been executed yet
-not $crossGenCorrect -or ## Last build didn't specify '-CrossGen' correctly
-not $PSModuleRestoreCorrect -or ## Last build didn't specify '-PSModuleRestore' correctly
$Script:Options.Runtime -ne $Runtime -or ## Last build wasn't for the required RID
$Script:Options.Configuration -ne $Configuration -or ## Last build was with configuration other than 'Release'
Expand All @@ -171,14 +159,8 @@ function Start-PSPackage {
# also ensure `Start-PSPackage` does what the user asks/expects, because once packages
# are generated, it'll be hard to verify if they were built from the correct content.


$params = @('-Clean')

# CrossGen cannot be done for framework dependent package as it is runtime agnostic.
if ($Type -notlike 'fxdependent*') {
$params += '-CrossGen'
}

if (!$IncludeSymbols.IsPresent) {
$params += '-PSModuleRestore'
}
Expand Down Expand Up @@ -4147,9 +4129,6 @@ function Invoke-AzDevOpsLinuxPackageBuild {
# We are cross compiling, so we can't generate experimental features
$buildParams.Add("SkipExperimentalFeatureGeneration", $true)
}
default {
$buildParams.Add("Crossgen", $true)
}
}

$buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${mainLinuxBuildFolder}"
Expand All @@ -4169,7 +4148,6 @@ function Invoke-AzDevOpsLinuxPackageBuild {
Remove-Item -Path $binDir -Recurse -Force
}

$buildParams['Crossgen'] = $false
$buildParams['ForMinimalSize'] = $true
$buildFolder = "${env:SYSTEM_ARTIFACTSDIRECTORY}/${minSizeLinuxBuildFolder}"
Start-PSBuild -Clean @buildParams @releaseTagParam -Output $buildFolder -PSOptionsPath "${buildFolder}-meta/psoptions.json"
Expand Down
Loading