Skip to content

Commit 2ad5306

Browse files
jshajsha
committed
Speed up builds.
Only build / validate rulesets when they have changed. Also break out validation steps into their own script, shared by makecrx/makexpi.
1 parent dac336f commit 2ad5306

File tree

5 files changed

+75
-99
lines changed

5 files changed

+75
-99
lines changed

‎makecrx.sh‎

Lines changed: 1 addition & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -44,59 +44,10 @@ if [ "$1" != "--fast" -o ! -f "$RULESETS_SQLITE" ] ; then
4444
python2.7 ./utils/make-sqlite.py
4545
fi
4646

47-
# =============== BEGIN VALIDATION ================
4847
# Unless we're in a hurry, validate the ruleset library & locales
49-
50-
die() {
51-
echo >&2 "ERROR:" "$@"
52-
exit 1
53-
}
54-
5548
if [ "$1" != "--fast" ] ; then
56-
if python2.7 ./utils/trivial-validate.py --quiet --db $RULESETS_SQLITE >&2
57-
then
58-
echo Validation of included rulesets completed. >&2
59-
echo >&2
60-
else
61-
die "Validation of rulesets failed."
62-
fi
63-
64-
# Check for xmllint.
65-
type xmllint >/dev/null || die "xmllint not available"
66-
67-
GRAMMAR="utils/relaxng.xml"
68-
if [ -f "$GRAMMAR" ]
69-
then
70-
# xmllint spams stderr with "<FILENAME> validates, even with the --noout
71-
# flag. We can't grep -v for that line, because the pipeline will mask error
72-
# status from xmllint. Instead we run it once going to /dev/null, and if
73-
# there's an error run it again, showing only error output.
74-
validate_grammar() {
75-
find src/chrome/content/rules -name "*.xml" | \
76-
xargs xmllint --noout --relaxng utils/relaxng.xml
77-
}
78-
if validate_grammar 2>/dev/null
79-
then
80-
echo Validation of rulesets against $GRAMMAR succeeded. >&2
81-
else
82-
validate_grammar 2>&1 | grep -v validates
83-
die "Validation of rulesets against $GRAMMAR failed."
84-
fi
85-
else
86-
echo Validation of rulesets against $GRAMMAR SKIPPED. >&2
87-
fi
88-
89-
if [ -x ./utils/compare-locales.sh ] >&2
90-
then
91-
if ./utils/compare-locales.sh >&2
92-
then
93-
echo Validation of included locales completed. >&2
94-
else
95-
die "Validation of locales failed."
96-
fi
97-
fi
49+
bash utils/validate.sh
9850
fi
99-
# =============== END VALIDATION ================
10051

10152
sed -e "s/VERSION/$VERSION/g" chromium/updates-master.xml > chromium/updates.xml
10253

‎makexpi.sh‎

Lines changed: 12 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ APP_NAME=https-everywhere
1616
# ./makexpi.sh 0.2.3.development.2
1717

1818
cd "`dirname $0`"
19+
RULESETS_UNVALIDATED="$PWD/pkg/rulesets.unvalidated.sqlite"
1920
RULESETS_SQLITE="$PWD/src/defaults/rulesets.sqlite"
2021
ANDROID_APP_ID=org.mozilla.firefox
2122

@@ -51,15 +52,20 @@ if [ -n "$1" ] && [ "$2" != "--no-recurse" ] && [ "$1" != "--fast" ] ; then
5152
exit 0
5253
fi
5354

54-
if [ "$1" != "--fast" -o ! -f "$RULESETS_SQLITE" ] ; then
55+
# Only generate the sqlite database if any rulesets have changed. Tried
56+
# implementing this with make, but make is very slow with 11k+ input files.
57+
needs_update() {
58+
find src/chrome/content/rules/ -newer $RULESETS_UNVALIDATED |\
59+
grep -q .
60+
}
61+
if [ ! -f "$RULESETS_UNVALIDATED" ] || needs_update ; then
5562
# This is an optimization to get the OS reading the rulesets into RAM ASAP;
5663
# it's useful on machines with slow disk seek times; doing several of these
5764
# at once allows the IO subsystem to seek more efficiently.
5865
for firstchar in `echo {a..z} {A..Z} {0..9}` ; do
5966
# Those cover everything but it wouldn't matter if they didn't
6067
nohup cat src/chrome/content/rules/"$firstchar"*.xml >/dev/null 2>/dev/null &
6168
done
62-
6369
echo "Generating sqlite DB"
6470
python2.7 ./utils/make-sqlite.py
6571
fi
@@ -72,51 +78,11 @@ die() {
7278
exit 1
7379
}
7480

75-
if [ "$1" != "--fast" -a -z "$FAST" ] ; then
76-
if python2.7 ./utils/trivial-validate.py --quiet --db $RULESETS_SQLITE >&2
77-
then
78-
echo Validation of included rulesets completed. >&2
79-
echo >&2
80-
else
81-
die "Validation of rulesets failed."
82-
fi
83-
84-
# Check for xmllint.
85-
type xmllint >/dev/null || die "xmllint not available"
86-
87-
GRAMMAR="utils/relaxng.xml"
88-
if [ -f "$GRAMMAR" ]
89-
then
90-
# xmllint spams stderr with "<FILENAME> validates, even with the --noout
91-
# flag. We can't grep -v for that line, because the pipeline will mask error
92-
# status from xmllint. Instead we run it once going to /dev/null, and if
93-
# there's an error run it again, showing only error output.
94-
validate_grammar() {
95-
find src/chrome/content/rules -name "*.xml" | \
96-
xargs xmllint --noout --relaxng utils/relaxng.xml
97-
}
98-
if validate_grammar 2>/dev/null
99-
then
100-
echo Validation of rulesets against $GRAMMAR succeeded. >&2
101-
else
102-
validate_grammar 2>&1 | grep -v validates
103-
die "Validation of rulesets against $GRAMMAR failed."
104-
fi
105-
else
106-
echo Validation of rulesets against $GRAMMAR SKIPPED. >&2
107-
fi
108-
109-
if [ -x ./utils/compare-locales.sh ] >&2
110-
then
111-
if sh ./utils/compare-locales.sh >&2
112-
then
113-
echo Validation of included locales completed. >&2
114-
else
115-
die "Validation of locales failed."
116-
fi
117-
fi
81+
# If the unvalidated rulesets have changed, validate and copy to the validated
82+
# rulesets file.
83+
if [ "$RULESETS_UNVALIDATED" -nt "$RULESETS_SQLITE" ] ; then
84+
bash utils/validate.sh
11885
fi
119-
# =============== END VALIDATION ================
12086

12187
# The name/version of the XPI we're building comes from src/install.rdf
12288
XPI_NAME="pkg/$APP_NAME-`grep em:version src/install.rdf | sed -e 's/[<>]/ /g' | cut -f3`"

‎utils/compare-locales.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
#!/bin/sh
2-
2+
cd $(dirname $0)
33
set -eu;
44

55
# Check whether all included rules include every entity defined in the
66
# English locale DTDs. (Missing an entity is a fatal error.)
77

88
status=0;
99

10-
cd src/chrome/locale;
10+
cd ../src/chrome/locale;
1111

1212
grep '^<!ENTITY' en/* | cut -d' ' -f2 | sort -u > ../en_entities;
1313

‎utils/make-sqlite.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
locale.setlocale(locale.LC_ALL, 'C')
2121

2222
# Removing the file before we create it avoids some non-determinism.
23-
db_path = os.path.join(os.path.dirname(__file__), '../src/defaults/rulesets.sqlite')
23+
db_path = os.path.join(os.path.dirname(__file__), '../pkg/rulesets.unvalidated.sqlite')
2424
if os.path.isfile(db_path):
2525
os.remove(db_path)
2626
conn = sqlite3.connect(db_path)

‎utils/validate.sh‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/bin/bash
2+
# Validates rulesets, using both trivial-validate.py (slower, can make more
3+
# complicated assertions) and xmllint with our custom RELAX NG grammar that
4+
# specifies the format of ruleset XML (faster, more limited).
5+
#
6+
# The rulesets sqlite must already have been generated at
7+
# https-everywhere/pkg/rulesets.unvalidated.sqlite.
8+
#
9+
# If validation is successful, that file will be copied to
10+
# https-everywhere/src/defaults/rulesets.sqlite for inclusion in the XPI.
11+
set -o errexit -o pipefail
12+
cd $(dirname $0)
13+
14+
# =============== BEGIN VALIDATION ================
15+
# Unless we're in a hurry, validate the ruleset library & locales
16+
17+
die() {
18+
echo >&2 "ERROR:" "$@"
19+
exit 1
20+
}
21+
22+
INPUT="../pkg/rulesets.unvalidated.sqlite"
23+
24+
if python2.7 trivial-validate.py --quiet --db "$INPUT" >&2
25+
then
26+
echo Validation of included rulesets completed. >&2
27+
echo >&2
28+
else
29+
die "Validation of rulesets failed."
30+
fi
31+
32+
# Check for xmllint.
33+
type xmllint >/dev/null || die "xmllint not available"
34+
35+
GRAMMAR="relaxng.xml"
36+
# xmllint spams stderr with "<FILENAME> validates, even with the --noout
37+
# flag. We can't grep -v for that line, because the pipeline will mask error
38+
# status from xmllint. Instead we run it once going to /dev/null, and if
39+
# there's an error run it again, showing only error output.
40+
validate_grammar() {
41+
find ../src/chrome/content/rules -name "*.xml" | \
42+
xargs xmllint --noout --relaxng $GRAMMAR
43+
}
44+
if validate_grammar 2>/dev/null
45+
then
46+
echo Validation of rulesets against $GRAMMAR succeeded. >&2
47+
else
48+
validate_grammar 2>&1 | grep -v validates
49+
die "Validation of rulesets against $GRAMMAR failed."
50+
fi
51+
52+
if bash compare-locales.sh >&2
53+
then
54+
echo Validation of included locales completed. >&2
55+
else
56+
die "Validation of locales failed."
57+
fi
58+
59+
cp "$INPUT" ../src/defaults/rulesets.sqlite

0 commit comments

Comments
 (0)