Skip to content

Commit f41b6b4

Browse files
TravisEz13rjmholt
TravisEz13
and
rjmholt
authored
Add GitHub Workflow to keep notices up to date (#16284)
Co-authored-by: Robert Holt <rjmholt_msft@outlook.com>
1 parent 52b6d89 commit f41b6b4

4 files changed

Lines changed: 243 additions & 93 deletions

File tree

‎.github/workflows/update-cgmanifest.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Copyright (c) Microsoft Corporation.
2+
# Licensed under the MIT license.
3+
4+
name: Update cgmanifest
5+
on:
6+
workflow_dispatch:
7+
schedule:
8+
# At 13:00 UTC every day.
9+
- cron: '0 13 * * *'
10+
11+
defaults:
12+
run:
13+
shell: pwsh
14+
15+
env:
16+
DOTNET_CLI_TELEMETRY_OPTOUT: 1
17+
POWERSHELL_TELEMETRY_OPTOUT: 1
18+
19+
jobs:
20+
update-cgmanifest:
21+
name: Update cgmanifest
22+
timeout-minutes: 15
23+
runs-on: windows-latest
24+
if: github.repository == 'PowerShell/PowerShell'
25+
steps:
26+
- name: Checkout
27+
uses: actions/checkout@v2
28+
- name: Sync tags
29+
run: |
30+
git fetch --prune --unshallow --tags
31+
- name: Install Ships provider to deal with project.assets.json
32+
run: |
33+
Install-Module -Name dotnet.project.assets -force
34+
- name: Bootstrap
35+
run: |
36+
Import-Module ./build.psm1
37+
Start-PSBootStrap
38+
- name: Update Notices file
39+
run: |
40+
Invoke-WebRequest -Uri https://aka.ms/pwsh-daily-tpn -OutFile ./ThirdPartyNotices.txt
41+
- name: Execute script to update cgmanifest
42+
run: |
43+
Import-Module ./build.psm1
44+
Find-Dotnet
45+
./tools/findMissingNotices.ps1
46+
- name: Microsoft Teams Notifier
47+
uses: skitionek/notify-microsoft-teams@master
48+
if: failure()
49+
with:
50+
webhook_url: ${{ secrets.PS_BUILD_TEAMS_CHANNEL }}
51+
overwrite: "{title: `Failure in updating cgmanifest. Look at ${workflow_link}`}"
52+
- name: Create Pull Request
53+
uses: peter-evans/create-pull-request@v3
54+
id: cpr
55+
if: env.CREATE_PR == 'true'
56+
with:
57+
commit-message: "Update the cgmanifest with missing or updated components"
58+
committer: GitHub <noreply@github.com>
59+
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
60+
title: "Update ${{ env.FORMULA_NAME }} formula to version ${{ env.NEW_FORMULA_VERSION }}"
61+
reviewers: travisez13
62+
base: master
63+
draft: false
64+
branch: update-cgmanifest

‎build.psm1‎

Lines changed: 2 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ param(
66
[parameter(Mandatory = $false)][switch]$SkipLinuxDistroCheck = $false
77
)
88

9+
. "$PSScriptRoot\tools\buildCommon\startNativeExecution.ps1"
10+
911
Set-StrictMode -Version 3.0
1012

1113
# On Unix paths is separated by colon
@@ -2357,53 +2359,6 @@ function script:precheck([string]$command, [string]$missedMessage) {
23572359
}
23582360
}
23592361

2360-
# this function wraps native command Execution
2361-
# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/
2362-
function script:Start-NativeExecution
2363-
{
2364-
param(
2365-
[scriptblock]$sb,
2366-
[switch]$IgnoreExitcode,
2367-
[switch]$VerboseOutputOnError
2368-
)
2369-
$backupEAP = $ErrorActionPreference
2370-
$ErrorActionPreference = "Continue"
2371-
try {
2372-
if($VerboseOutputOnError.IsPresent)
2373-
{
2374-
$output = & $sb 2>&1
2375-
}
2376-
else
2377-
{
2378-
& $sb
2379-
}
2380-
2381-
# note, if $sb doesn't have a native invocation, $LASTEXITCODE will
2382-
# point to the obsolete value
2383-
if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) {
2384-
if($VerboseOutputOnError.IsPresent -and $output)
2385-
{
2386-
$output | Out-String | Write-Verbose -Verbose
2387-
}
2388-
2389-
# Get caller location for easier debugging
2390-
$caller = Get-PSCallStack -ErrorAction SilentlyContinue
2391-
if($caller)
2392-
{
2393-
$callerLocationParts = $caller[1].Location -split ":\s*line\s*"
2394-
$callerFile = $callerLocationParts[0]
2395-
$callerLine = $callerLocationParts[1]
2396-
2397-
$errorMessage = "Execution of {$sb} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE"
2398-
throw $errorMessage
2399-
}
2400-
throw "Execution of {$sb} failed with exit code $LASTEXITCODE"
2401-
}
2402-
} finally {
2403-
$ErrorActionPreference = $backupEAP
2404-
}
2405-
}
2406-
24072362
# Cleans the PowerShell repo - everything but the root folder
24082363
function Clear-PSRepo
24092364
{

‎tools/buildCommon/startNativeExecution.ps1‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Copyright (c) Microsoft Corporation.
2+
# Licensed under the MIT License.
3+
4+
# this function wraps native command Execution
5+
# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/
6+
function script:Start-NativeExecution {
7+
param(
8+
[Alias('sb')]
9+
[Parameter(Mandatory=$true)]
10+
[scriptblock]$ScriptBlock,
11+
[switch]$IgnoreExitcode,
12+
[switch]$VerboseOutputOnError
13+
)
14+
15+
$backupEAP = $ErrorActionPreference
16+
$ErrorActionPreference = "Continue"
17+
Write-Verbose "Executing: $ScriptBlock"
18+
try {
19+
if ($VerboseOutputOnError.IsPresent) {
20+
$output = & $ScriptBlock 2>&1
21+
} else {
22+
& $ScriptBlock
23+
}
24+
25+
# note, if $ScriptBlock doesn't have a native invocation, $LASTEXITCODE will
26+
# point to the obsolete value
27+
if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) {
28+
if ($VerboseOutputOnError.IsPresent -and $output) {
29+
$output | Out-String | Write-Verbose -Verbose
30+
}
31+
32+
# Get caller location for easier debugging
33+
$caller = Get-PSCallStack -ErrorAction SilentlyContinue
34+
if ($caller) {
35+
$callerLocationParts = $caller[1].Location -split ":\s*line\s*"
36+
$callerFile = $callerLocationParts[0]
37+
$callerLine = $callerLocationParts[1]
38+
39+
$errorMessage = "Execution of {$ScriptBlock} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE"
40+
throw $errorMessage
41+
}
42+
throw "Execution of {$ScriptBlock} failed with exit code $LASTEXITCODE"
43+
}
44+
} finally {
45+
$ErrorActionPreference = $backupEAP
46+
}
47+
}

‎tools/findMissingNotices.ps1‎

Lines changed: 130 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,23 @@
11
# Copyright (c) Microsoft Corporation.
22
# Licensed under the MIT License.
33

4+
# This script is used to completely rebuild the cgmanifgest.json file,
5+
# which is used to generate the notice file.
46
# Requires the module dotnet.project.assets from the PowerShell Gallery authored by @TravisEz13
57

6-
import-module dotnet.project.assets
8+
Import-Module dotnet.project.assets
9+
. "$PSScriptRoot\..\tools\buildCommon\startNativeExecution.ps1"
10+
11+
$existingRegistrationTable = @{}
12+
$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable
13+
$existingRegistrationsJson.Registrations | ForEach-Object {
14+
$registration = [Registration]$_
15+
if($registration.Component) {
16+
$name = $registration.Component.Name()
17+
$existingRegistrationTable.Add($name, $registration)
18+
}
19+
}
20+
721
Class Registration {
822
[Component]$Component
923
[bool]$DevelopmentDependency
@@ -79,58 +93,128 @@ function New-NugetComponent {
7993
return $registration
8094
}
8195

82-
$existingRegistrationTable = @{}
83-
$newRegistrations = @()
84-
$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable
85-
$existingRegistrationsJson.Registrations | ForEach-Object {
86-
$registration = [Registration]$_
87-
$existingRegistrationTable.Add($registration.Component.Name(), $registration)
88-
$newRegistrations += $registration
96+
$winDesktopSdk = 'Microsoft.NET.Sdk.WindowsDesktop'
97+
if (!$IsWindows) {
98+
$winDesktopSdk = 'Microsoft.NET.Sdk'
99+
Write-Warning "Always using $winDesktopSdk since this is not windows!!!"
89100
}
90101

91-
Get-PSDrive -Name pwsh-win-core -ErrorAction Ignore | Remove-PSDrive
92-
Push-Location $PSScriptRoot\..\src\powershell-win-core
93-
$null = dotnet restore
94-
$null = New-PADrive -Path $PSScriptRoot\..\src\powershell-win-core\obj\project.assets.json -Name pwsh-win-core
95-
$targets = Get-ChildItem -Path 'pwsh-win-core:/targets/net6.0-windows7.0|win7-x64' | Where-Object {
96-
$_.Type -eq 'package' -and
97-
$_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and
98-
$_.Name -notlike 'StyleCop*' -and
99-
$_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and
100-
$_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*'
101-
} | select-object -ExpandProperty name
102-
Pop-Location
103-
Get-PSDrive -Name pwsh-win-core | Remove-PSDrive
104-
105-
$updateRegistrations = @()
106-
$targets | ForEach-Object {
107-
$target = $_
108-
$parts = ($target -split '\|')
109-
$name = $parts[0]
110-
$targetVersion = $parts[1]
111-
$pattern = [regex]::Escape($name) + " "
112-
$tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern
113-
if (!$tpnMatch) {
114-
if ($existingRegistrationTable.ContainsKey($name)) {
115-
$registrationVersion = $existingRegistrationTable.$name.Component.Version()
116-
if ($registrationVersion -ne $targetVersion) {
117-
$registration = New-NugetComponent -Name $name -Version $targetVersion
118-
$updateRegistrations += $registration
119-
} else {
120-
Write-Verbose "$target already registered: $registrationVersion" -Verbose
121-
}
122-
} else {
102+
Function Get-CGRegistrations {
103+
param(
104+
[Parameter(Mandatory)]
105+
[ValidateSet(
106+
"alpine-x64",
107+
"linux-arm",
108+
"linux-arm64",
109+
"linux-x64",
110+
"osx-arm64",
111+
"osx-x64",
112+
"win-arm",
113+
"win-arm64",
114+
"win7-x64",
115+
"win7-x86",
116+
"modules")]
117+
[string]$Runtime,
118+
119+
[Parameter(Mandatory)]
120+
[System.Collections.Generic.Dictionary[string, Registration]] $RegistrationTable
121+
)
122+
123+
$newRegistrations = $Registrations
124+
125+
$dotnetTargetName = 'net6.0'
126+
$dotnetTargetNameWin7 = 'net6.0-windows7.0'
127+
$unixProjectName = 'powershell-unix'
128+
$windowsProjectName = 'powershell-win-core'
129+
$actualRuntime = $Runtime
130+
131+
switch -regex ($Runtime) {
132+
"alpine-.*" {
133+
$folder = $unixProjectName
134+
$target = "$dotnetTargetName|$Runtime"
135+
}
136+
"linux-.*" {
137+
$folder = $unixProjectName
138+
$target = "$dotnetTargetName|$Runtime"
139+
}
140+
"osx-.*" {
141+
$folder = $unixProjectName
142+
$target = "$dotnetTargetName|$Runtime"
143+
}
144+
"win7-.*" {
145+
$sdkToUse = $winDesktopSdk
146+
$folder = $windowsProjectName
147+
$target = "$dotnetTargetNameWin7|$Runtime"
148+
}
149+
"win-.*" {
150+
$folder = $windowsProjectName
151+
$target = "$dotnetTargetNameWin7|$Runtime"
152+
}
153+
"modules" {
154+
$folder = "modules"
155+
$actualRuntime = 'linux-x64'
156+
$target = "$dotnetTargetName|$actualRuntime"
157+
}
158+
Default {
159+
throw "Invalid runtime name: $Runtime"
160+
}
161+
}
162+
163+
Write-Verbose "Getting registrations for $folder - $actualRuntime ..." -Verbose
164+
Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive
165+
Push-Location $PSScriptRoot\..\src\$folder
166+
try {
167+
Start-NativeExecution -VerboseOutputOnError -sb {
168+
dotnet restore --runtime $actualRuntime "/property:SDKToUse=$sdkToUse"
169+
}
170+
$null = New-PADrive -Path $PSScriptRoot\..\src\$folder\obj\project.assets.json -Name $folder
171+
try {
172+
$targets = Get-ChildItem -Path "${folder}:/targets/$target" -ErrorAction Stop | Where-Object {
173+
$_.Type -eq 'package' -and
174+
$_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and
175+
$_.Name -notlike 'StyleCop*' -and
176+
$_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and
177+
$_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*'
178+
} | select-object -ExpandProperty name
179+
} catch {
180+
Get-ChildItem -Path "${folder}:/targets" | Out-String | Write-Verbose -Verbose
181+
throw
182+
}
183+
} finally {
184+
Pop-Location
185+
Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive
186+
}
187+
188+
$targets | ForEach-Object {
189+
$target = $_
190+
$parts = ($target -split '\|')
191+
$name = $parts[0]
192+
$targetVersion = $parts[1]
193+
$pattern = [regex]::Escape($name) + " "
194+
$tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern
195+
196+
# Add the registration to the cgmanifest if the TPN does not contain the name of the target OR
197+
# the exisitng CG contains the registration, because if the existing CG contains the registration,
198+
# that might be the only reason it is in the TPN.
199+
if ((!$tpnMatch -or $existingRegistrationTable.ContainsKey($name)) -and !$RegistrationTable.ContainsKey($target)) {
123200
$registration = New-NugetComponent -Name $name -Version $targetVersion
124-
$newRegistrations += $registration
201+
$RegistrationTable.Add($target, $registration)
125202
}
126203
}
127204
}
128205

129-
if ($updateRegistrations.count -gt 0) {
130-
#TODO delete existing and add new registration
131-
throw "updating registrations is not implemented"
206+
$registrations = [System.Collections.Generic.Dictionary[string, Registration]]::new()
207+
$lastCount = 0
208+
foreach ($runtime in "win7-x64", "linux-x64", "osx-x64", "alpine-x64", "win-arm", "linux-arm", "linux-arm64", "osx-arm64", "win-arm64", "win7-x86") {
209+
Get-CGRegistrations -Runtime $runtime -RegistrationTable $registrations
210+
$count = $registrations.Count
211+
$newCount = $count - $lastCount
212+
$lastCount = $count
213+
Write-Verbose "$newCount new registrations, $count total..." -Verbose
132214
}
133215

134-
$newCount = $newRegistrations.count - $existingRegistrationTable.count
216+
$newRegistrations = $registrations.Keys | Sort-Object | ForEach-Object { $registrations[$_] }
217+
218+
$count = $newRegistrations.Count
135219
@{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 | Set-Content $PSScriptRoot\..\cgmanifest.json
136-
Write-Verbose "$newCount registrations added" -Verbose
220+
Write-Verbose "$count registrations created!" -Verbose

0 commit comments

Comments
 (0)