Skip to content

Commit 3f98986

Browse files
ctruedenctrueden
committed
Add initial "mega melt" infrastructure
The mega melt is a script intended to test all components managed by pom-scijava together in a consistent runtime environment, using the melting-pot script from the scijava-scripts repository. This initial version is working quite well after some manual tweaks. Once it reaches a state where it can execute successfully in a fully automated way, we can hook it up to CI infrastructure to continually check for regressions as component versions continue rolling forward.
1 parent d1bdfac commit 3f98986

4 files changed

Lines changed: 289 additions & 0 deletions

File tree

‎tests/filter-build-log.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#
2+
# filter-build-log.py - Cuts out cruft to focus on build failure details.
3+
#
4+
5+
# This script filters build logs down to only [WARNING] and [ERROR] lines,
6+
# and consolidates lengthy duplicate class listings down to packages only.
7+
8+
import sys
9+
10+
def print_filtered_log(log):
11+
dups = []
12+
parsingdups = False
13+
atbeginning = True
14+
for line in log:
15+
line = line.rstrip('\n')
16+
if line.startswith('[INFO]'):
17+
# Filter out non-error build messages.
18+
continue
19+
if line.startswith('Download') or line.startswith('Progress'):
20+
# Filter out details of remote resource queries.
21+
continue
22+
if atbeginning and not line.strip():
23+
# Filter out leading blank lines.
24+
continue
25+
atbeginning = False
26+
if parsingdups:
27+
if line.startswith(' '):
28+
if line.index('/') >= 0:
29+
# Strip to containing package only.
30+
line = line[:line.rindex('/')]
31+
dups.append(line)
32+
else:
33+
parsingdups = False
34+
for dup in sorted(set(dups)):
35+
print(dup)
36+
print('')
37+
dups = []
38+
else:
39+
if line == ' Duplicate classes:':
40+
print(' Duplicate packages:')
41+
parsingdups = True
42+
else:
43+
print(line)
44+
45+
for arg in sys.argv[1:]:
46+
with open(arg) as f:
47+
print_filtered_log(f)
48+

‎tests/generate-mega-melt.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
#
2+
# generate-mega-melt.py - Make a POM depending on everything in pom-scijava.
3+
#
4+
5+
import os, re, sys
6+
from xml.dom import minidom
7+
8+
def child(node, tag):
9+
nodes = node.getElementsByTagName(tag)
10+
return None if len(nodes) == 0 else nodes[0]
11+
12+
script_dir = os.path.dirname(os.path.realpath(__file__)) if __file__ else '.'
13+
out = minidom.parse(os.path.join(script_dir, 'mega-melt-template.xml'))
14+
out.getElementsByTagName('project')[0].appendChild(out.createElement('dependencies'))
15+
outDeps = out.getElementsByTagName('dependencies')[0]
16+
17+
psj = minidom.parse(os.path.join(script_dir, '..', 'pom.xml'))
18+
depMgmt = psj.getElementsByTagName('dependencyManagement')[0]
19+
deps = depMgmt.getElementsByTagName('dependencies')[0]
20+
depList = deps.getElementsByTagName('dependency')
21+
22+
# Artifacts to exclude from the mega melt.
23+
ignoredArtifacts = [
24+
# TEMP: Until scenerygraphics/scenery#314 is addressed.
25+
'scenery',
26+
# NB: The following artifacts have messy dependency trees.
27+
# Too many problems to test as part of the mega-melt.
28+
# See WARNING block in pom-scijava's pom.xml for details.
29+
'imagej-server',
30+
'spark-core_2.11',
31+
# NB: Skip scijava forks of third-party projects.
32+
# These are very stable, with few/no dependencies, and
33+
# don't need to be retested as pom-scijava evolves.
34+
'j3dcore',
35+
'j3dutils',
36+
'jep',
37+
'junit-benchmarks',
38+
'vecmath',
39+
# NB: The following artifacts require native libraries to
40+
# build/test successfully, which might not be installed.
41+
'n5-blosc',
42+
'n5-zarr',
43+
# NB: All the SWT platform JARs have the same classes.
44+
# The current platform will be brought in transitively.
45+
'org.eclipse.swt.cocoa.macosx',
46+
'org.eclipse.swt.cocoa.macosx.x86_64',
47+
'org.eclipse.swt.gtk.aix.ppc',
48+
'org.eclipse.swt.gtk.aix.ppc64',
49+
'org.eclipse.swt.gtk.hpux.ia64',
50+
'org.eclipse.swt.gtk.linux.ppc',
51+
'org.eclipse.swt.gtk.linux.ppc64',
52+
'org.eclipse.swt.gtk.linux.s390',
53+
'org.eclipse.swt.gtk.linux.s390x',
54+
'org.eclipse.swt.gtk.linux.x86',
55+
'org.eclipse.swt.gtk.linux.x86_64',
56+
'org.eclipse.swt.gtk.solaris.sparc',
57+
'org.eclipse.swt.gtk.solaris.x86',
58+
'org.eclipse.swt.win32.win32.x86',
59+
'org.eclipse.swt.win32.win32.x86_64',
60+
# NB: All SLF4J bindings have the same classes.
61+
# We'll rely on logback-classic being present here.
62+
'slf4j-jcl',
63+
'slf4j-jdk14',
64+
'slf4j-nop',
65+
'slf4j-simple',
66+
# NB: Cannot include both commons-logging and jcl-over-slf4j;
67+
# see: http://www.slf4j.org/codes.html#jclDelegationLoop
68+
'jcl-over-slf4j'
69+
]
70+
71+
for dep in depList:
72+
# Harvest relevant information (ignore exclusions and version).
73+
groupId = child(dep, 'groupId')
74+
artifactId = child(dep, 'artifactId')
75+
classifier = child(dep, 'classifier')
76+
scope = child(dep, 'scope')
77+
78+
if artifactId.firstChild.data in ignoredArtifacts:
79+
continue
80+
81+
outDep = out.createElement('dependency')
82+
outDep.appendChild(groupId)
83+
outDep.appendChild(artifactId)
84+
if classifier:
85+
outDep.appendChild(classifier)
86+
if scope:
87+
outDep.appendChild(scope)
88+
outDeps.appendChild(outDep)
89+
90+
# Filter XML through a hacky substitution to avoid unwanted whitespace.
91+
xml = re.sub('\n[\n\r\t]*\n', '\n', out.toprettyxml())
92+
93+
outDir = sys.argv[1] if len(sys.argv) > 1 else 'mega-melt'
94+
try:
95+
os.mkdir(outDir)
96+
except:
97+
pass
98+
99+
with open(os.path.join(outDir, 'pom.xml'), 'w') as outPOM:
100+
outPOM.write(xml)

‎tests/mega-melt-template.xml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<parent>
5+
<groupId>org.scijava</groupId>
6+
<artifactId>pom-scijava</artifactId>
7+
<version>999-mega-melt</version>
8+
<relativePath />
9+
</parent>
10+
<artifactId>mega-melt</artifactId>
11+
<version>0-SNAPSHOT</version>
12+
<packaging>pom</packaging>
13+
<name>Mega Melt</name>
14+
<description>The whole BOM in one project</description>
15+
<url>https://scijava.org/</url>
16+
<inceptionYear>now</inceptionYear>
17+
<organization>
18+
<name>SciJava</name>
19+
<url>https://scijava.org/</url>
20+
</organization>
21+
<licenses>
22+
<license>
23+
<name>CC0 1.0 Universal License</name>
24+
<url>https://creativecommons.org/publicdomain/zero/1.0/</url>
25+
<distribution>repo</distribution>
26+
</license>
27+
</licenses>
28+
<developers>
29+
<developer>
30+
<id>ctrueden</id>
31+
<name>Curtis Rueden</name>
32+
</developer>
33+
</developers>
34+
<contributors>
35+
<contributor>
36+
<name>None</name>
37+
</contributor>
38+
</contributors>
39+
<mailingLists>
40+
<mailingList>
41+
<name>None</name>
42+
</mailingList>
43+
</mailingLists>
44+
<scm>
45+
<connection>scm:git:git://github.com/scijava/pom-scijava</connection>
46+
<developerConnection>scm:git:git@github.com:scijava/pom-scijava</developerConnection>
47+
<tag>HEAD</tag>
48+
<url>https://github.com/scijava/pom-scijava</url>
49+
</scm>
50+
<issueManagement>
51+
<system>None</system>
52+
</issueManagement>
53+
<ciManagement>
54+
<system>None</system>
55+
</ciManagement>
56+
<properties>
57+
<license.licenseName>N/A</license.licenseName>
58+
<license.copyrightOwners>N/A</license.copyrightOwners>
59+
</properties>
60+
<repositories>
61+
<repository>
62+
<id>scijava.public</id>
63+
<url>https://maven.scijava.org/content/groups/public</url>
64+
</repository>
65+
</repositories>
66+
</project>

‎tests/run.sh‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/bin/sh
2+
3+
#
4+
# run.sh - Tests correctness of the pom-scijava BOM.
5+
#
6+
7+
die () { echo "$*" >&2; exit 1; }
8+
9+
echo &&
10+
printf 'Generating mega-melt project... ' &&
11+
12+
dir=$(dirname "$0")
13+
pom="$dir/../pom.xml"
14+
test -f "$pom" || die 'Where is pom.xml?!'
15+
16+
generateMegaMeltScript="$dir/generate-mega-melt.py"
17+
filterBuildLogScript="$dir/filter-build-log.py"
18+
19+
megaMeltDir="$dir/../target/mega-melt"
20+
versionSwapLog="$megaMeltDir/version-swap.log"
21+
dependencyTreeLog="$megaMeltDir/dependency-tree.log"
22+
validationLog="$megaMeltDir/validation.log"
23+
validationErrorsLog="$megaMeltDir/validation-errors.log"
24+
megaMeltPOM="$megaMeltDir/pom.xml"
25+
meltingPotURL=https://raw.githubusercontent.com/scijava/scijava-scripts/master/melting-pot.sh
26+
meltingPotScript="$megaMeltDir/melting-pot.sh"
27+
28+
rm -rf "$megaMeltDir" && mkdir -p "$megaMeltDir" || die "Creation of $megaMeltDir failed!"
29+
mvn -f "$pom" versions:set -DnewVersion=999-mega-melt > "$versionSwapLog" &&
30+
mvn -f "$pom" install >> "$versionSwapLog" ||
31+
die "pom-scijava version swap failed:\n$(cat "$versionSwapLog")"
32+
mv -f "$pom.versionsBackup" "$pom" || die 'POM restoration failed!'
33+
python "$generateMegaMeltScript" "$megaMeltDir" || die 'Generation failed!'
34+
echo 'Done!'
35+
36+
# Ensure the mega-melt dependency structure validates.
37+
# In particular, this runs our enforcer rules: the build
38+
# will fail if there are any snapshots dependencies, or
39+
# any duplicate classes across artifacts on the classpath.
40+
echo &&
41+
printf 'Validating mega-melt project... ' &&
42+
mvn -f "$megaMeltPOM" dependency:tree > "$dependencyTreeLog" ||
43+
die "Invalid dependency tree:\n$(cat "$dependencyTreeLog")"
44+
mvn -f "$megaMeltPOM" -U clean package > "$validationLog" || {
45+
python "$filterBuildLogScript" "$validationLog" > "$validationErrorsLog"
46+
die "Validation build failed!\n\nDependency tree:\n$(cat "$dependencyTreeLog")\n\nBuild log:\n$(cat "$validationErrorsLog")"
47+
}
48+
echo 'Done!'
49+
50+
# Run mega-melt through the melting pot,
51+
# with all SciJava-affiliated groupIds.
52+
echo &&
53+
echo 'Executing melting pot...' &&
54+
curl -fsL "$meltingPotURL" > "$meltingPotScript" &&
55+
chmod +x "$meltingPotScript" &&
56+
"$meltingPotScript" "$megaMeltDir" \
57+
-o "$megaMeltDir/melting-pot" \
58+
-i 'ca.mcgill:*' \
59+
-i 'graphics.scenery:*' \
60+
-i 'io.scif:*' \
61+
-i 'jitk:*' \
62+
-i 'mpicbg:*' \
63+
-i 'net.imagej:*' \
64+
-i 'net.imglib2:*' \
65+
-i 'net.preibisch:*' \
66+
-i 'org.bonej:*' \
67+
-i 'org.janelia.saalfeldlab:*' \
68+
-i 'org.janelia:*' \
69+
-i 'org.scijava:*' \
70+
-i 'sc.fiji:*' \
71+
-i 'sc.iview:*' \
72+
-f -v $@ || die 'Melting pot build failed!'
73+
74+
echo
75+
echo 'All checks succeeded! :-D'

0 commit comments

Comments
 (0)