Restore duplicate host testing to trvial-validate.

jsha · jsha · commit b9b193d05d40 · 2014-08-16T09:49:41.000-04:00
diff --git a/utils/make-sqlite.py b/utils/make-sqlite.py
@@ -49,6 +49,10 @@ def nomes_all(where=sys.argv[1:]):
     etree.strip_tags(tree,etree.Comment)
 
     targets = xpath_host(tree)
+    if not targets:
+      print 'File %s has no targets' % fi
+      sys.exit(1)
+
     # Strip out the target tags. These aren't necessary in the DB because
     # targets are looked up in the target table, which has a foreign key
     # pointing into the ruleset table.
diff --git a/utils/trivial-validate.py b/utils/trivial-validate.py
@@ -35,7 +35,7 @@ def warn(s):
 def fail(s):
     sys.stdout.write("failure: %s\n" % s)
 
-def test_not_anchored(tree, fi, host, fm, to):
+def test_not_anchored(tree, fi, fm, to):
     # Rules not anchored to the beginning of a line.
     """The 'from' rule is not anchored to beginning of line using the ^ symbol."""
     for f in fm:
@@ -47,7 +47,7 @@ def test_not_anchored(tree, fi, host, fm, to):
 xpath_exlusion_pattern = etree.XPath("/ruleset/exclusion/@pattern")
 xpath_cookie_pattern = etree.XPath("/ruleset/securecookie/@host")
 
-def test_bad_regexp(tree, fi, host, fm, to):
+def test_bad_regexp(tree, fi, fm, to):
     # Rules with invalid regular expressions.
     """The 'from' rule contains an invalid extended regular expression."""
     patterns = fm + xpath_exlusion_pattern(tree) + xpath_cookie_pattern(tree)
@@ -59,7 +59,7 @@ def test_bad_regexp(tree, fi, host, fm, to):
     return True
 
 xpath_rule = etree.XPath("/ruleset/rule")
-def test_missing_to(tree, fi, host, fm, to):
+def test_missing_to(tree, fi, fm, to):
     # Rules that are terminated before setting 'to'.
     # These cases are probably either due to a misplaced
     # rule end or intended to be different elements.
@@ -71,7 +71,7 @@ def test_missing_to(tree, fi, host, fm, to):
             return False
     return True
 
-def test_unescaped_dots(tree, fi, host, fm, to):
+def test_unescaped_dots(tree, fi, fm, to):
     # Rules containing unescaped dots outside of brackets and before slash.
     # Note: this is meant to require example\.com instead of example.com,
     # but it also forbids things like .* which usually ought to be replaced
@@ -96,15 +96,15 @@ def test_unescaped_dots(tree, fi, host, fm, to):
                escaped = False
     return True
 
-def test_space_in_to(tree, fi, host, fm, to):
+def test_space_in_to(tree, fi, fm, to):
     # Rules where the to pattern contains a space.
     """The 'to' rule contains a space."""
     for t in to:
         if ' ' in t:
             return False
     return True
 
-def test_unencrypted_to(tree, fi, host, fm, to):
+def test_unencrypted_to(tree, fi, fm, to):
     # Rules that redirect to something other than https or http.
     # This used to test for http: but testing for lack of https: will
     # catch more kinds of mistakes.
@@ -122,7 +122,7 @@ def test_unencrypted_to(tree, fi, host, fm, to):
             return False
     return True
 
-def test_backslash_in_to(tree, fi, host, fm, to):
+def test_backslash_in_to(tree, fi, fm, to):
     # Rules containing backslashes in to pattern.
     """The 'to' rule contains a backslash."""
     for t in to:
@@ -132,7 +132,7 @@ def test_backslash_in_to(tree, fi, host, fm, to):
 
 RE_TRAILING_SLASH = re.compile("//.*/")
 
-def test_no_trailing_slash(tree, fi, host, fm, to):
+def test_no_trailing_slash(tree, fi, fm, to):
     # Rules not containing trailing slash in from or to pattern.
     """Rule omits forward slash after host name."""
     for r in xpath_rule(tree):
@@ -143,29 +143,9 @@ def test_no_trailing_slash(tree, fi, host, fm, to):
             return False
     return True
 
-def test_lacks_target_host(tree, fi, host, fm, to):
-    # Rules that lack at least one target host (target tag with host attr).
-    """Rule fails to specify at least one target host."""
-    return not not host
-
-def test_bad_target_host(tree, fi, host, fm, to):
-    # Rules where a target host contains multiple wildcards or a slash.
-    """The target host must be a hostname, not URL, and must use at most one wildcard."""
-    for target in host:
-        if "/" in target:
-            return False
-        if target.count("*") > 1:
-            return False
-    return True
-
-def test_duplicated_target_host(tree, fi, host, fm, to):
-    # Rules where a single target host appears more than once.
-    """Rule contains the same target host more than once."""
-    return len(set(host)) == len(host)
-
 printable_characters = set(map(chr, list(range(32, 127))))
 
-def test_non_ascii(tree, fi, host, fm, to):
+def test_non_ascii(tree, fi, fm, to):
     # Rules containing non-printable characters.
     """Rule contains non-printable character in 'to' pattern."""
     for t in to:
@@ -174,6 +154,15 @@ def test_non_ascii(tree, fi, host, fm, to):
                 return False
     return True
 
+def is_valid_target_host(host):
+    # Rules where a target host contains multiple wildcards or a slash.
+    """The target host must be a hostname, not URL, and must use at most one wildcard."""
+    if "/" in host:
+        return False
+    if host.count("*") > 1:
+        return False
+    return True
+
 def nomes_all(where=sys.argv[1:]):
     """Returns generator to extract all files from a list of files/dirs"""
     if not where: where=['.']
@@ -193,8 +182,6 @@ def nomes_all(where=sys.argv[1:]):
          test_unencrypted_to,
          test_backslash_in_to,
          test_no_trailing_slash,
-         test_bad_target_host,
-         test_duplicated_target_host,
          test_non_ascii]
 
 failure = 0
@@ -224,18 +211,19 @@ def nomes_all(where=sys.argv[1:]):
         fail("unnamed ruleset")
         continue
     rf = xpath_ruleset_name(tree)[0]
-    host = xpath_host(tree)
     fm = xpath_from(tree)
     to = xpath_to(tree)
     for test in tests:
-        if not test(tree, rf, host=host, fm=fm, to=to):
-            print("%s failed test: %s" % (rf, test.__doc__))
-    for target in xpath_host(tree):
-        print target
-        if target in all_targets and not any(ign.search(target) for ign in ignoredups):
-            # suppress warning about duplicate targets if an --ignoredups
-            # pattern matches target
-            warn("has duplicate target: %s" % (target))
-        all_targets.add(target)
+        if not test(tree, rf, fm=fm, to=to):
+            failure = 1
+            fail("%s failed test: %s" % (rf, test.__doc__))
+
+for (host, count) in c.execute('''
+  select host, count(host) as c from targets group by host;'''):
+    if count > 1:
+      warn("Hostname %s shows up in %d different rulesets." % (host, count))
+    if not is_valid_target_host(host):
+      failure = 1
+      fail("%s failed: %s" % (host, is_valid_target_host.__doc__))
 
 sys.exit(failure)
