Skip to content

Commit e0f8e5d

Browse files
MSLaguanaMSLaguana
committed
[1.2>1.3] [MERGE chakra-core#2887 @MSLaguana] Updating CI script
Merge pull request chakra-core#2887 from MSLaguana:fixCIPRTriggers Prior to this change, we were building some configurations for all branches when a PR came in. This is just backporting the netci.groovy script from `release/1.4` to be more consistent. Fixes chakra-core#2877
2 parents 85fa291 + 00c44f3 commit e0f8e5d

1 file changed

Lines changed: 144 additions & 131 deletions

File tree

‎netci.groovy‎

Lines changed: 144 additions & 131 deletions
Original file line numberDiff line numberDiff line change
@@ -27,153 +27,155 @@ def machineTypeToOSTagMap = [
2727

2828
def dailyRegex = 'dailies'
2929

30-
// Only generate PR check triggers for the version of netci.groovy in the master branch
31-
// since those PR checks will apply for all branches.
32-
def jobTypesToGenerate = [false]
33-
if (branch.startsWith('master')) {
34-
// OK to generate PR checks (this ensures we only generate one set of them)
35-
jobTypesToGenerate += true
36-
}
37-
3830
// ---------------
3931
// HELPER CLOSURES
4032
// ---------------
4133

34+
def CreateBuildTask = { isPR, buildArch, buildType, machine, configTag, buildExtra, testExtra, runCodeAnalysis, excludeConfigIf, nonDefaultTaskSetup ->
35+
if (excludeConfigIf && excludeConfigIf(isPR, buildArch, buildType)) {
36+
return // early exit: we don't want to create a job for this configuration
37+
}
38+
39+
def config = "${buildArch}_${buildType}"
40+
config = (configTag == null) ? config : "${configTag}_${config}"
41+
42+
// params: Project, BaseTaskName, IsPullRequest (appends '_prtest')
43+
def jobName = Utilities.getFullJobName(project, config, isPR)
44+
45+
def testableConfig = buildType in ['debug', 'test'] && buildArch != 'arm'
46+
def analysisConfig = buildType in ['release'] && runCodeAnalysis
47+
48+
def buildScript = "call .\\jenkins\\buildone.cmd ${buildArch} ${buildType} "
49+
buildScript += buildExtra ?: ''
50+
buildScript += analysisConfig ? ' "/p:runcodeanalysis=true"' : ''
51+
def testScript = "call .\\jenkins\\testone.cmd ${buildArch} ${buildType} "
52+
testScript += testExtra ?: ''
53+
def analysisScript = '.\\Build\\scripts\\check_prefast_error.ps1 . CodeAnalysis.err'
54+
55+
def newJob = job(jobName) {
56+
// This opens the set of build steps that will be run.
57+
// This looks strange, but it is actually a method call, with a
58+
// closure as a param, since Groovy allows method calls without parens.
59+
// (Compare with '.each' method used above.)
60+
steps {
61+
batchFile(buildScript) // run the parameter as if it were a batch file
62+
if (testableConfig) {
63+
batchFile(testScript)
64+
}
65+
if (analysisConfig) {
66+
powerShell(analysisScript)
67+
}
68+
}
69+
}
70+
71+
def msbuildType = msbuildTypeMap.get(buildType)
72+
def msbuildFlavor = "build_${buildArch}${msbuildType}"
73+
def archivalString = "test/${msbuildFlavor}.*,test/logs/**"
74+
archivalString += analysisConfig ? ',CodeAnalysis.err' : ''
75+
Utilities.addArchival(newJob, archivalString,
76+
'', // no exclusions from archival
77+
false, // doNotFailIfNothingArchived=false ~= failIfNothingArchived
78+
false) // archiveOnlyIfSuccessful=false ~= archiveAlways
79+
80+
Utilities.setMachineAffinity(newJob, machine, 'latest-or-auto')
81+
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
82+
83+
if (nonDefaultTaskSetup == null) {
84+
if (isPR) {
85+
def osTag = machineTypeToOSTagMap.get(machine)
86+
Utilities.addGithubPRTriggerForBranch(newJob, branch, "${osTag} ${config}")
87+
} else {
88+
Utilities.addGithubPushTrigger(newJob)
89+
}
90+
} else {
91+
// nonDefaultTaskSetup is e.g. DailyBuildTaskSetup (which sets up daily builds)
92+
// These jobs will only be configured for the branch specified below,
93+
// which is the name of the branch netci.groovy was processed for.
94+
// See list of such branches at:
95+
// https://github.com/dotnet/dotnet-ci/blob/master/jobs/data/repolist.txt
96+
nonDefaultTaskSetup(newJob, isPR, config)
97+
}
98+
}
99+
42100
def CreateBuildTasks = { machine, configTag, buildExtra, testExtra, runCodeAnalysis, excludeConfigIf, nonDefaultTaskSetup ->
43-
jobTypesToGenerate.each { isPR ->
101+
[true, false].each { isPR ->
44102
['x86', 'x64', 'arm'].each { buildArch ->
45103
['debug', 'test', 'release'].each { buildType ->
46-
if (excludeConfigIf && excludeConfigIf(isPR, buildArch, buildType)) {
47-
return // early exit: we don't want to create a job for this configuration
48-
}
49-
50-
def config = "${buildArch}_${buildType}"
51-
config = (configTag == null) ? config : "${configTag}_${config}"
52-
53-
// params: Project, BaseTaskName, IsPullRequest (appends '_prtest')
54-
def jobName = Utilities.getFullJobName(project, config, isPR)
55-
56-
def testableConfig = buildType in ['debug', 'test'] && buildArch != 'arm'
57-
def analysisConfig = buildType in ['release'] && runCodeAnalysis
58-
59-
def buildScript = "call .\\jenkins\\buildone.cmd ${buildArch} ${buildType} "
60-
buildScript += buildExtra ?: ''
61-
buildScript += analysisConfig ? ' "/p:runcodeanalysis=true"' : ''
62-
def testScript = "call .\\jenkins\\testone.cmd ${buildArch} ${buildType} "
63-
testScript += testExtra ?: ''
64-
def analysisScript = '.\\Build\\scripts\\check_prefast_error.ps1 . CodeAnalysis.err'
65-
66-
def newJob = job(jobName) {
67-
// This opens the set of build steps that will be run.
68-
// This looks strange, but it is actually a method call, with a
69-
// closure as a param, since Groovy allows method calls without parens.
70-
// (Compare with '.each' method used above.)
71-
steps {
72-
batchFile(buildScript) // run the parameter as if it were a batch file
73-
if (testableConfig) {
74-
batchFile(testScript)
75-
}
76-
if (analysisConfig) {
77-
powerShell(analysisScript)
78-
}
79-
}
80-
}
81-
82-
def msbuildType = msbuildTypeMap.get(buildType)
83-
def msbuildFlavor = "build_${buildArch}${msbuildType}"
84-
def archivalString = "test/${msbuildFlavor}.*,test/logs/**"
85-
archivalString += analysisConfig ? ',CodeAnalysis.err' : ''
86-
Utilities.addArchival(newJob, archivalString,
87-
'', // no exclusions from archival
88-
false, // doNotFailIfNothingArchived=false ~= failIfNothingArchived
89-
false) // archiveOnlyIfSuccessful=false ~= archiveAlways
90-
91-
Utilities.setMachineAffinity(newJob, machine, 'latest-or-auto')
92-
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
93-
94-
if (nonDefaultTaskSetup == null) {
95-
if (isPR) {
96-
def osTag = machineTypeToOSTagMap.get(machine)
97-
// Set up checks which apply to PRs targeting any branch
98-
Utilities.addGithubPRTrigger(newJob, "${osTag} ${config}")
99-
// To enable PR checks only for specific target branches, use the following instead:
100-
// Utilities.addGithubPRTriggerForBranch(newJob, branch, checkName)
101-
} else {
102-
Utilities.addGithubPushTrigger(newJob)
103-
}
104-
} else {
105-
// nonDefaultTaskSetup is e.g. DailyBuildTaskSetup (which sets up daily builds)
106-
// These jobs will only be configured for the branch specified below,
107-
// which is the name of the branch netci.groovy was processed for.
108-
// See list of such branches at:
109-
// https://github.com/dotnet/dotnet-ci/blob/master/jobs/data/repolist.txt
110-
nonDefaultTaskSetup(newJob, isPR, config)
111-
}
104+
CreateBuildTask(isPR, buildArch, buildType, machine, configTag, buildExtra, testExtra, runCodeAnalysis, excludeConfigIf, nonDefaultTaskSetup)
112105
}
113106
}
114107
}
115108
}
116109

110+
def CreateXPlatBuildTask = { isPR, buildType, staticBuild, machine, platform, configTag,
111+
xplatBranch, nonDefaultTaskSetup, customOption, testVariant ->
112+
113+
def config = (platform == "osx" ? "osx_${buildType}" : "linux_${buildType}")
114+
def numConcurrentCommand = (platform == "osx" ? "sysctl -n hw.logicalcpu" : "nproc")
115+
116+
config = (configTag == null) ? config : "${configTag}_${config}"
117+
config = staticBuild ? "${config}_static" : config
118+
119+
// params: Project, BaseTaskName, IsPullRequest (appends '_prtest')
120+
def jobName = Utilities.getFullJobName(project, config, isPR) + customOption.replaceAll(/[-]+/, "_")
121+
122+
def infoScript = "bash jenkins/get_system_info.sh --${platform}"
123+
def buildFlag = buildType == "release" ? "" : (buildType == "debug" ? "--debug" : "--test-build")
124+
def staticFlag = staticBuild ? "--static" : ""
125+
def icuFlag = (platform == "osx" ? "--icu=/usr/local/opt/icu4c/include" : "")
126+
def compilerPaths = (platform == "osx") ? "" : "--cxx=/usr/bin/clang++-3.8 --cc=/usr/bin/clang-3.8"
127+
def buildScript = "bash ./build.sh ${staticFlag} -j=`${numConcurrentCommand}` ${buildFlag} ${compilerPaths} ${icuFlag} ${customOption}"
128+
def testScript = "bash test/runtests.sh \"${testVariant}\""
129+
130+
def newJob = job(jobName) {
131+
steps {
132+
shell(infoScript)
133+
shell(buildScript)
134+
shell(testScript)
135+
}
136+
}
137+
138+
def archivalString = "BuildLinux/build.log"
139+
Utilities.addArchival(newJob, archivalString,
140+
'', // no exclusions from archival
141+
true, // doNotFailIfNothingArchived=false ~= failIfNothingArchived (true ~= doNotFail)
142+
false) // archiveOnlyIfSuccessful=false ~= archiveAlways
143+
144+
Utilities.setMachineAffinity(newJob, machine, 'latest-or-auto')
145+
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
146+
147+
if (nonDefaultTaskSetup == null) {
148+
if (isPR) {
149+
def osTag = machineTypeToOSTagMap.get(machine)
150+
Utilities.addGithubPRTriggerForBranch(newJob, xplatBranch, "${osTag} ${config}")
151+
} else {
152+
Utilities.addGithubPushTrigger(newJob)
153+
}
154+
} else {
155+
// nonDefaultTaskSetup is e.g. DailyBuildTaskSetup (which sets up daily builds)
156+
// These jobs will only be configured for the branch specified below,
157+
// which is the name of the branch netci.groovy was processed for.
158+
// See list of such branches at:
159+
// https://github.com/dotnet/dotnet-ci/blob/master/jobs/data/repolist.txt
160+
nonDefaultTaskSetup(newJob, isPR, config)
161+
}
162+
}
163+
117164
// Generic task to trigger clang-based cross-plat build tasks
118165
def CreateXPlatBuildTasks = { machine, platform, configTag, xplatBranch, nonDefaultTaskSetup ->
119166
[true, false].each { isPR ->
167+
CreateXPlatBuildTask(isPR, "test", "", machine, platform,
168+
configTag, xplatBranch, nonDefaultTaskSetup, "--no-jit", "--variants disable_jit")
169+
120170
['debug', 'test', 'release'].each { buildType ->
121171
def staticBuildConfigs = [true, false]
122172
if (platform == "osx") {
123173
staticBuildConfigs = [true]
124174
}
125175

126176
staticBuildConfigs.each { staticBuild ->
127-
def config = (platform == "osx" ? "osx_${buildType}" : "linux_${buildType}")
128-
def numConcurrentCommand = (platform == "osx" ? "sysctl -n hw.logicalcpu" : "nproc")
129-
130-
config = (configTag == null) ? config : "${configTag}_${config}"
131-
config = staticBuild ? "${config}_static" : config
132-
133-
// params: Project, BaseTaskName, IsPullRequest (appends '_prtest')
134-
def jobName = Utilities.getFullJobName(project, config, isPR)
135-
136-
def infoScript = "bash jenkins/get_system_info.sh --${platform}"
137-
def buildFlag = buildType == "release" ? "" : (buildType == "debug" ? "--debug" : "--test-build")
138-
def staticFlag = staticBuild ? "--static" : ""
139-
def icuFlag = (platform == "osx" ? "--icu=/usr/local/opt/icu4c/include" : "")
140-
def compilerPaths = (platform == "osx") ? "" : "--cxx=/usr/bin/clang++-3.8 --cc=/usr/bin/clang-3.8"
141-
def buildScript = "bash ./build.sh ${staticFlag} -j=`${numConcurrentCommand}` ${buildFlag} ${compilerPaths} ${icuFlag}"
142-
def testScript = "bash test/runtests.sh"
143-
144-
def newJob = job(jobName) {
145-
steps {
146-
shell(infoScript)
147-
shell(buildScript)
148-
shell(testScript)
149-
}
150-
}
151-
152-
def archivalString = "BuildLinux/build.log"
153-
Utilities.addArchival(newJob, archivalString,
154-
'', // no exclusions from archival
155-
true, // doNotFailIfNothingArchived=false ~= failIfNothingArchived (true ~= doNotFail)
156-
false) // archiveOnlyIfSuccessful=false ~= archiveAlways
157-
158-
Utilities.setMachineAffinity(newJob, machine, 'latest-or-auto')
159-
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
160-
161-
if (nonDefaultTaskSetup == null) {
162-
if (isPR) {
163-
def osTag = machineTypeToOSTagMap.get(machine)
164-
// Set up checks which apply to PRs targeting any branch
165-
Utilities.addGithubPRTriggerForBranch(newJob, xplatBranch, "${osTag} ${config}")
166-
} else {
167-
Utilities.addGithubPushTrigger(newJob)
168-
}
169-
} else {
170-
// nonDefaultTaskSetup is e.g. DailyBuildTaskSetup (which sets up daily builds)
171-
// These jobs will only be configured for the branch specified below,
172-
// which is the name of the branch netci.groovy was processed for.
173-
// See list of such branches at:
174-
// https://github.com/dotnet/dotnet-ci/blob/master/jobs/data/repolist.txt
175-
nonDefaultTaskSetup(newJob, isPR, config)
176-
}
177+
CreateXPlatBuildTask(isPR, buildType, staticBuild, machine, platform,
178+
configTag, xplatBranch, nonDefaultTaskSetup, "", "")
177179
}
178180
}
179181
}
@@ -183,7 +185,7 @@ def DailyBuildTaskSetup = { newJob, isPR, triggerName, groupRegex ->
183185
// The addition of triggers makes the job non-default in GitHub.
184186
if (isPR) {
185187
def triggerRegex = "(${dailyRegex}|${groupRegex}|${triggerName})"
186-
Utilities.addGithubPRTrigger(newJob,
188+
Utilities.addGithubPRTriggerForBranch(newJob, branch,
187189
triggerName, // GitHub task name
188190
"(?i).*test\\W+${triggerRegex}.*")
189191
} else {
@@ -204,7 +206,7 @@ def CreateStyleCheckTasks = { taskString, taskName, checkName ->
204206
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
205207
if (isPR) {
206208
// Set PR trigger.
207-
Utilities.addGithubPRTrigger(newJob, checkName)
209+
Utilities.addGithubPRTriggerForBranch(newJob, branch, checkName)
208210
} else {
209211
// Set a push trigger
210212
Utilities.addGithubPushTrigger(newJob)
@@ -220,6 +222,17 @@ def CreateStyleCheckTasks = { taskString, taskName, checkName ->
220222

221223
CreateBuildTasks('Windows_NT', null, null, null, true, null, null)
222224

225+
// Add some additional daily configs to trigger per-PR as a quality gate:
226+
// x64_debug Slow Tests
227+
CreateBuildTask(true, 'x64', 'debug',
228+
'Windows_NT', 'ci_slow', null, '-includeSlow', false, null, null)
229+
// x64_debug DisableJIT
230+
CreateBuildTask(true, 'x64', 'debug',
231+
'Windows_NT', 'ci_disablejit', '"/p:BuildJIT=false"', '-disablejit', false, null, null)
232+
// x64_debug Legacy
233+
CreateBuildTask(true, 'x64', 'debug',
234+
'Windows 7', 'ci_dev12', 'msbuild12', '-win7 -includeSlow', false, null, null)
235+
223236
// -----------------
224237
// DAILY BUILD TASKS
225238
// -----------------
@@ -267,11 +280,11 @@ def isXPlatCompatibleBranch = !(branch in ['release/1.1', 'release/1.1-ci', 'rel
267280

268281
// Include these explicitly-named branches
269282
def isXPlatDailyBranch = branch in ['master', 'linux', 'xplat']
270-
// Include some release/* branches
271-
if (branch.startsWith('release')) {
283+
// Include some release/* branches (ignore branches ending in '-ci')
284+
if (branch.startsWith('release') && !branch.endsWith('-ci')) {
272285
// Allows all current and future release/* branches on which we should run daily builds of XPlat configs.
273-
// RegEx matches e.g. release/1.1, release/1.2, release/1.3-ci, but not e.g. release/2.0, release/1.3, or release/1.10
274-
includeReleaseBranch = !(branch =~ /^release\/((1\.[12](\D.*)?)|(\d+\.\d+\D.*))$/)
286+
// RegEx matches branch names we should ignore (e.g. release/1.1, release/1.2, release/1.2-pre)
287+
includeReleaseBranch = !(branch =~ /^release\/(1\.[12](\D.*)?)$/)
275288
isXPlatDailyBranch |= includeReleaseBranch
276289
}
277290

0 commit comments

Comments
 (0)