Skip to content

Commit c64f422

Browse files
jshajsha
committed
Merge pull request EFForg#515 from jsha/faster-build
Speed up builds.
2 parents 27fdbb4 + 6743ee4 commit c64f422

File tree

5 files changed

+89
-111
lines changed

5 files changed

+89
-111
lines changed

‎makecrx.sh‎

Lines changed: 13 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@
1818
# "dummy-chromium.pem" private key for you to sign your own local releases,
1919
# but these .crx files won't detect and upgrade to official HTTPS Everywhere
2020
# releases signed by EFF :/. We should find a more elegant arrangement.
21+
22+
cd $(dirname $0)
23+
RULESETS_UNVALIDATED="$PWD/pkg/rulesets.unvalidated.sqlite"
2124
RULESETS_SQLITE="$PWD/src/defaults/rulesets.sqlite"
2225

2326
if [ -n "$1" ]; then
@@ -37,67 +40,20 @@ VERSION=`python -c "import json ; print(json.loads(open('chromium/manifest.json'
3740

3841
echo "Building chrome version" $VERSION
3942

40-
# Build the SQLite DB even though we don't yet use it in the Chrome extension,
41-
# because trivial-validate.py depends on it.
42-
if [ "$1" != "--fast" -o ! -f "$RULESETS_SQLITE" ] ; then
43+
# Only generate the sqlite database if any rulesets have changed. Tried
44+
# implementing this with make, but make is very slow with 15k+ input files.
45+
needs_update() {
46+
find src/chrome/content/rules/ -newer $RULESETS_UNVALIDATED |\
47+
grep -q .
48+
}
49+
if [ ! -f "$RULESETS_UNVALIDATED" ] || needs_update ; then
50+
# Build the SQLite DB even though we don't yet use it in the Chrome extension,
51+
# because trivial-validate.py depends on it.
4352
echo "Generating sqlite DB"
4453
python2.7 ./utils/make-sqlite.py
54+
bash utils/validate.sh
4555
fi
4656

47-
# =============== BEGIN VALIDATION ================
48-
# Unless we're in a hurry, validate the ruleset library & locales
49-
50-
die() {
51-
echo >&2 "ERROR:" "$@"
52-
exit 1
53-
}
54-
55-
if [ "$1" != "--fast" ] ; then
56-
if python2.7 ./utils/trivial-validate.py --quiet --db $RULESETS_SQLITE >&2
57-
then
58-
echo Validation of included rulesets completed. >&2
59-
echo >&2
60-
else
61-
die "Validation of rulesets failed."
62-
fi
63-
64-
# Check for xmllint.
65-
type xmllint >/dev/null || die "xmllint not available"
66-
67-
GRAMMAR="utils/relaxng.xml"
68-
if [ -f "$GRAMMAR" ]
69-
then
70-
# xmllint spams stderr with "<FILENAME> validates, even with the --noout
71-
# flag. We can't grep -v for that line, because the pipeline will mask error
72-
# status from xmllint. Instead we run it once going to /dev/null, and if
73-
# there's an error run it again, showing only error output.
74-
validate_grammar() {
75-
find src/chrome/content/rules -name "*.xml" | \
76-
xargs xmllint --noout --relaxng utils/relaxng.xml
77-
}
78-
if validate_grammar 2>/dev/null
79-
then
80-
echo Validation of rulesets against $GRAMMAR succeeded. >&2
81-
else
82-
validate_grammar 2>&1 | grep -v validates
83-
die "Validation of rulesets against $GRAMMAR failed."
84-
fi
85-
else
86-
echo Validation of rulesets against $GRAMMAR SKIPPED. >&2
87-
fi
88-
89-
if [ -x ./utils/compare-locales.sh ] >&2
90-
then
91-
if ./utils/compare-locales.sh >&2
92-
then
93-
echo Validation of included locales completed. >&2
94-
else
95-
die "Validation of locales failed."
96-
fi
97-
fi
98-
fi
99-
# =============== END VALIDATION ================
100-
10157
sed -e "s/VERSION/$VERSION/g" chromium/updates-master.xml > chromium/updates.xml
10258

10359
[ -d pkg ] || mkdir -p pkg

‎makexpi.sh‎

Lines changed: 14 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,14 @@ APP_NAME=https-everywhere
1616
# ./makexpi.sh 0.2.3.development.2
1717

1818
cd "`dirname $0`"
19+
RULESETS_UNVALIDATED="$PWD/pkg/rulesets.unvalidated.sqlite"
1920
RULESETS_SQLITE="$PWD/src/defaults/rulesets.sqlite"
2021
ANDROID_APP_ID=org.mozilla.firefox
21-
if [ "$1" == "--fast" ]; then
22-
FAST=true
23-
fi
2422

2523
[ -d pkg ] || mkdir pkg
2624

2725
# If the command line argument is a tag name, check that out and build it
28-
if [ -n "$1" ] && [ "$2" != "--no-recurse" ] && [ -z "$FAST" ] ; then
26+
if [ -n "$1" ] && [ "$2" != "--no-recurse" ] ; then
2927
BRANCH=`git branch | head -n 1 | cut -d \ -f 2-`
3028
SUBDIR=checkout
3129
[ -d $SUBDIR ] || mkdir $SUBDIR
@@ -54,15 +52,20 @@ if [ -n "$1" ] && [ "$2" != "--no-recurse" ] && [ -z "$FAST" ] ; then
5452
exit 0
5553
fi
5654

57-
if [ -z "$FAST" -o ! -f "$RULESETS_SQLITE" ] ; then
55+
# Only generate the sqlite database if any rulesets have changed. Tried
56+
# implementing this with make, but make is very slow with 15k+ input files.
57+
needs_update() {
58+
find src/chrome/content/rules/ -newer $RULESETS_UNVALIDATED |\
59+
grep -q .
60+
}
61+
if [ ! -f "$RULESETS_UNVALIDATED" ] || needs_update ; then
5862
# This is an optimization to get the OS reading the rulesets into RAM ASAP;
5963
# it's useful on machines with slow disk seek times; doing several of these
6064
# at once allows the IO subsystem to seek more efficiently.
6165
for firstchar in `echo {a..z} {A..Z} {0..9}` ; do
6266
# Those cover everything but it wouldn't matter if they didn't
6367
nohup cat src/chrome/content/rules/"$firstchar"*.xml >/dev/null 2>/dev/null &
6468
done
65-
6669
echo "Generating sqlite DB"
6770
python2.7 ./utils/make-sqlite.py
6871
fi
@@ -75,55 +78,15 @@ die() {
7578
exit 1
7679
}
7780

78-
if [ -z "$FAST" ] ; then
79-
if python2.7 ./utils/trivial-validate.py --quiet --db $RULESETS_SQLITE >&2
80-
then
81-
echo Validation of included rulesets completed. >&2
82-
echo >&2
83-
else
84-
die "Validation of rulesets failed."
85-
fi
86-
87-
# Check for xmllint.
88-
type xmllint >/dev/null || die "xmllint not available"
89-
90-
GRAMMAR="utils/relaxng.xml"
91-
if [ -f "$GRAMMAR" ]
92-
then
93-
# xmllint spams stderr with "<FILENAME> validates, even with the --noout
94-
# flag. We can't grep -v for that line, because the pipeline will mask error
95-
# status from xmllint. Instead we run it once going to /dev/null, and if
96-
# there's an error run it again, showing only error output.
97-
validate_grammar() {
98-
find src/chrome/content/rules -name "*.xml" | \
99-
xargs xmllint --noout --relaxng utils/relaxng.xml
100-
}
101-
if validate_grammar 2>/dev/null
102-
then
103-
echo Validation of rulesets against $GRAMMAR succeeded. >&2
104-
else
105-
validate_grammar 2>&1 | grep -v validates
106-
die "Validation of rulesets against $GRAMMAR failed."
107-
fi
108-
else
109-
echo Validation of rulesets against $GRAMMAR SKIPPED. >&2
110-
fi
111-
112-
if [ -x ./utils/compare-locales.sh ] >&2
113-
then
114-
if sh ./utils/compare-locales.sh >&2
115-
then
116-
echo Validation of included locales completed. >&2
117-
else
118-
die "Validation of locales failed."
119-
fi
120-
fi
81+
# If the unvalidated rulesets have changed, validate and copy to the validated
82+
# rulesets file.
83+
if [ "$RULESETS_UNVALIDATED" -nt "$RULESETS_SQLITE" ] ; then
84+
bash utils/validate.sh
12185
fi
122-
# =============== END VALIDATION ================
12386

12487
# The name/version of the XPI we're building comes from src/install.rdf
12588
XPI_NAME="pkg/$APP_NAME-`grep em:version src/install.rdf | sed -e 's/[<>]/ /g' | cut -f3`"
126-
if [ "$1" -a -z "$FAST" ] ; then
89+
if [ "$1" ]; then
12790
XPI_NAME="$XPI_NAME"
12891
else
12992
# During development, generate packages named with the short hash of HEAD.

‎utils/compare-locales.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
#!/bin/sh
2-
2+
cd $(dirname $0)
33
set -eu;
44

55
# Check whether all included rules include every entity defined in the
66
# English locale DTDs. (Missing an entity is a fatal error.)
77

88
status=0;
99

10-
cd src/chrome/locale;
10+
cd ../src/chrome/locale;
1111

1212
grep '^<!ENTITY' en/* | cut -d' ' -f2 | sort -u > ../en_entities;
1313

‎utils/make-sqlite.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
locale.setlocale(locale.LC_ALL, 'C')
2121

2222
# Removing the file before we create it avoids some non-determinism.
23-
db_path = os.path.join(os.path.dirname(__file__), '../src/defaults/rulesets.sqlite')
23+
db_path = os.path.join(os.path.dirname(__file__), '../pkg/rulesets.unvalidated.sqlite')
2424
if os.path.isfile(db_path):
2525
os.remove(db_path)
2626
conn = sqlite3.connect(db_path)

‎utils/validate.sh‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/bin/bash
2+
# Validates rulesets, using both trivial-validate.py (slower, can make more
3+
# complicated assertions) and xmllint with our custom RELAX NG grammar that
4+
# specifies the format of ruleset XML (faster, more limited).
5+
#
6+
# The rulesets sqlite must already have been generated at
7+
# https-everywhere/pkg/rulesets.unvalidated.sqlite.
8+
#
9+
# If validation is successful, that file will be copied to
10+
# https-everywhere/src/defaults/rulesets.sqlite for inclusion in the XPI.
11+
set -o errexit -o pipefail
12+
cd $(dirname $0)
13+
14+
# =============== BEGIN VALIDATION ================
15+
# Unless we're in a hurry, validate the ruleset library & locales
16+
17+
die() {
18+
echo >&2 "ERROR:" "$@"
19+
exit 1
20+
}
21+
22+
INPUT="../pkg/rulesets.unvalidated.sqlite"
23+
24+
if python2.7 trivial-validate.py --quiet --db "$INPUT" >&2
25+
then
26+
echo Validation of included rulesets completed. >&2
27+
echo >&2
28+
else
29+
die "Validation of rulesets failed."
30+
fi
31+
32+
# Check for xmllint.
33+
type xmllint >/dev/null || die "xmllint not available"
34+
35+
GRAMMAR="relaxng.xml"
36+
# xmllint spams stderr with "<FILENAME> validates, even with the --noout
37+
# flag. We can't grep -v for that line, because the pipeline will mask error
38+
# status from xmllint. Instead we run it once going to /dev/null, and if
39+
# there's an error run it again, showing only error output.
40+
validate_grammar() {
41+
find ../src/chrome/content/rules -name "*.xml" | \
42+
xargs xmllint --noout --relaxng $GRAMMAR
43+
}
44+
if validate_grammar 2>/dev/null
45+
then
46+
echo Validation of rulesets against $GRAMMAR succeeded. >&2
47+
else
48+
validate_grammar 2>&1 | grep -v validates
49+
die "Validation of rulesets against $GRAMMAR failed."
50+
fi
51+
52+
if bash compare-locales.sh >&2
53+
then
54+
echo Validation of included locales completed. >&2
55+
else
56+
die "Validation of locales failed."
57+
fi
58+
59+
cp "$INPUT" ../src/defaults/rulesets.sqlite

0 commit comments

Comments
 (0)