Skip to content

Commit a3268fb

Browse files
committed
First stab at faster validation.
1 parent afb03a0 commit a3268fb

1 file changed

Lines changed: 26 additions & 43 deletions

File tree

‎utils/trivial-validate.py‎

Lines changed: 26 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import argparse
44
import sys, re, os
5+
import sqlite3
56

67
try:
78
from lxml import etree
@@ -41,15 +42,15 @@ def warn(s):
4142
def fail(s):
4243
sys.stdout.write("failure: %s\n" % s)
4344

44-
def test_not_anchored(tree):
45+
def test_not_anchored(tree, fi):
4546
# Rules not anchored to the beginning of a line.
4647
"""The 'from' rule is not anchored to beginning of line using the ^ symbol."""
4748
for f in tree.xpath("/ruleset/rule/@from"):
4849
if not f or f[0] != "^":
4950
return False
5051
return True
5152

52-
def test_bad_regexp(tree):
53+
def test_bad_regexp(tree, fi):
5354
# Rules with invalid regular expressions.
5455
"""The 'from' rule contains an invalid extended regular expression."""
5556
for f in tree.xpath("/ruleset/rule/@from") + \
@@ -61,20 +62,19 @@ def test_bad_regexp(tree):
6162
return False
6263
return True
6364

64-
def test_missing_to(tree):
65-
65+
def test_missing_to(tree, fi):
6666
# Rules that are terminated before setting 'to'.
6767
# These cases are probably either due to a misplaced
6868
# rule end or intended to be different elements.
6969
"""Rule is missing a 'to' value."""
7070
for rule in tree.xpath("/ruleset/rule"):
71-
if not rule.get("to"):
72-
warn("'to' attribute missing in %s. " %fi)
71+
if not rule.get("to"):
72+
warn("'to' attribute missing in %s. " % fi)
7373
warn("Misplaced end or misnamed element?")
7474
return False
7575
return True
7676

77-
def test_unescaped_dots(tree):
77+
def test_unescaped_dots(tree, fi):
7878
# Rules containing unescaped dots outside of brackets and before slash.
7979
# Note: this is meant to require example\.com instead of example.com,
8080
# but it also forbids things like .* which usually ought to be replaced
@@ -99,15 +99,15 @@ def test_unescaped_dots(tree):
9999
escaped = False
100100
return True
101101

102-
def test_space_in_to(tree):
102+
def test_space_in_to(tree, fi):
103103
# Rules where the to pattern contains a space.
104104
"""The 'to' rule contains a space."""
105105
for t in tree.xpath("/ruleset/rule/@to"):
106106
if ' ' in t:
107107
return False
108108
return True
109109

110-
def test_unencrypted_to(tree):
110+
def test_unencrypted_to(tree, fi):
111111
# Rules that redirect to something other than https or http.
112112
# This used to test for http: but testing for lack of https: will
113113
# catch more kinds of mistakes.
@@ -125,15 +125,15 @@ def test_unencrypted_to(tree):
125125
return False
126126
return True
127127

128-
def test_backslash_in_to(tree):
128+
def test_backslash_in_to(tree, fi):
129129
# Rules containing backslashes in to pattern.
130130
"""The 'to' rule contains a backslash."""
131131
for t in tree.xpath("/ruleset/rule/@to"):
132132
if '\\' in t:
133133
return False
134134
return True
135135

136-
def test_no_trailing_slash(tree):
136+
def test_no_trailing_slash(tree, fi):
137137
# Rules not containing trailing slash in from or to pattern.
138138
"""Rule omits forward slash after host name."""
139139
for r in tree.xpath("/ruleset/rule"):
@@ -144,12 +144,12 @@ def test_no_trailing_slash(tree):
144144
return False
145145
return True
146146

147-
def test_lacks_target_host(tree):
147+
def test_lacks_target_host(tree, fi):
148148
# Rules that lack at least one target host (target tag with host attr).
149149
"""Rule fails to specify at least one target host."""
150150
return not not tree.xpath("/ruleset/target/@host")
151151

152-
def test_bad_target_host(tree):
152+
def test_bad_target_host(tree, fi):
153153
# Rules where a target host contains multiple wildcards or a slash.
154154
"""The target host must be a hostname, not URL, and must use at most one wildcard."""
155155
for target in tree.xpath("/ruleset/target/@host"):
@@ -159,15 +159,15 @@ def test_bad_target_host(tree):
159159
return False
160160
return True
161161

162-
def test_duplicated_target_host(tree):
162+
def test_duplicated_target_host(tree, fi):
163163
# Rules where a single target host appears more than once.
164164
"""Rule contains the same target host more than once."""
165165
targets = tree.xpath("/ruleset/target/@host")
166166
return len(set(targets)) == len(targets)
167167

168168
printable_characters = set(map(chr, list(range(32, 127))))
169169

170-
def test_non_ascii(tree):
170+
def test_non_ascii(tree, fi):
171171
# Rules containing non-printable characters.
172172
"""Rule contains non-printable character in 'to' pattern."""
173173
for t in tree.xpath("/ruleset/rule/@to"):
@@ -214,53 +214,36 @@ def nomes_all(where=sys.argv[1:]):
214214

215215
tests = [test_not_anchored, test_bad_regexp, test_unescaped_dots, test_missing_to,
216216
test_space_in_to, test_unencrypted_to, test_backslash_in_to,
217-
test_no_trailing_slash, test_lacks_target_host, test_bad_target_host,
217+
test_no_trailing_slash, test_bad_target_host,
218218
test_duplicated_target_host, test_non_ascii]
219219

220220
failure = 0
221221
seen_file = False
222-
all_names, all_targets = get_all_names_and_targets(dupdir)
223222

224-
for fi in nomes_all():
223+
conn = sqlite3.connect(os.path.join(os.path.dirname(__file__), '../src/defaults/rulesets.sqlite'))
224+
c = conn.cursor()
225+
for row in c.execute('''SELECT contents from rulesets'''):
225226
try:
226-
tree = etree.parse(fi)
227-
if fi[-4:] != ".xml":
228-
if tree.xpath("/ruleset"):
229-
warn("ruleset in file without .xml extension: %s" % fi)
230-
else:
231-
continue
232-
seen_file = True
227+
tree = etree.fromstring(row[0])
233228
except Exception as oops:
234-
if fi[-4:] != ".xml":
235-
continue
236229
failure = 1
237-
fail("%s failed XML validity: %s\n" % (fi, oops))
230+
print("failed XML validity: %s\n" % (oops))
238231
if failure or not tree.xpath("/ruleset"):
239232
continue
240233
if not test_ruleset_name(tree):
241234
failure = 1
242-
fail("unnamed ruleset: %s" % fi)
235+
fail("unnamed ruleset")
243236
continue
244237
ruleset_name = tree.xpath("/ruleset/@name")[0]
245-
if ruleset_name in all_names:
246-
failure = 1
247-
fail("duplicate ruleset name %s" % ruleset_name)
248-
all_names.add(ruleset_name)
238+
ruleset_file = tree.xpath("/ruleset/@f")[0]
249239
for test in tests:
250-
if not test(tree):
251-
failure = 1
252-
fail("%s failed test: %s" % (fi, test.__doc__))
240+
if not test(tree, ruleset_file):
241+
print("%s failed test: %s" % (ruleset_file, test.__doc__))
253242
for target in tree.xpath("/ruleset/target/@host"):
254243
if target in all_targets and not any(ign.search(target) for ign in ignoredups):
255244
# suppress warning about duplicate targets if an --ignoredups
256245
# pattern matches target
257-
warn("%s has duplicate target: %s" % (fi, target))
246+
warn("has duplicate target: %s" % (target))
258247
all_targets.add(target)
259248

260-
if not seen_file:
261-
which = "specified" if args else "current"
262-
sys.stdout.write("There were no valid XML files in the %s " % which)
263-
sys.stdout.write("directory.\n")
264-
failure = 3
265-
266249
sys.exit(failure)

0 commit comments

Comments
 (0)