Update https_everywhere_checker/check_rules.py, Fix EFForg#10877 (EFForg#11187)

Chan Chak Shing · Hainish · commit 4964511b068b · 2017-08-22T12:31:53.000-07:00
diff --git a/test/rules/coverage.checker.config b/test/rules/coverage.checker.config
@@ -4,6 +4,7 @@
 [rulesets]
 rulesdir = src/chrome/content/rules
 check_coverage = true
+check_target_validity = true
 check_nonmatch_groups = true
 check_test_formatting = true
 include_default_off = false
diff --git a/test/rules/requirements.txt b/test/rules/requirements.txt
@@ -2,3 +2,4 @@ pycurl
 regex
 bsdiff4
 python-Levenshtein
+tldextract
diff --git a/test/rules/src/https_everywhere_checker/check_rules.py b/test/rules/src/https_everywhere_checker/check_rules.py
@@ -130,11 +130,18 @@ def fetchUrl(self, plainUrl, transformedUrl, fetcherPlain, fetcherRewriting, rul
 		logging.debug("Fetching plain page %s", plainUrl)
 		# If we get an exception (e.g. connection refused,
 		# connection timeout) on the plain page, don't treat
-		# that as a failure.
+		# that as a failure (except DNS resolution errors)
 		plainRcode, plainPage = None, None
 		try:
 			plainRcode, plainPage = fetcherPlain.fetchHtml(plainUrl)
 		except Exception, e:
+			errno, message = e
+			if errno == 6:
+				message = "Fetch error: %s => %s: %s" % (
+					plainUrl, transformedUrl, e)
+				self.queue_result("error", "fetch-error %s"% e, ruleFname, plainUrl, https_url=transformedUrl)
+				return message
+
 			logging.debug("Non-fatal fetch error for plain page %s: %s" % (plainUrl, e))
 
 		# Compare HTTP return codes - if original page returned 2xx,
@@ -290,6 +297,9 @@ def cli():
 	checkCoverage = False
 	if config.has_option("rulesets", "check_coverage"):
 		checkCoverage = config.getboolean("rulesets", "check_coverage")
+	checkTargetValidity = False
+	if config.has_option("rulesets", "check_target_validity"):
+		checkTargetValidity = config.getboolean("rulesets", "check_target_validity")
 	checkNonmatchGroups = False
 	if config.has_option("rulesets", "check_nonmatch_groups"):
 		checkNonmatchGroups = config.getboolean("rulesets", "check_nonmatch_groups")
@@ -341,6 +351,7 @@ def cli():
 	
 	rulesets = []
 	coverageProblemsExist = False
+	targetValidityProblemExist = False
 	nonmatchGroupProblemsExist = False
 	testFormattingProblemsExist = False
 	for xmlFname in xmlFnames:
@@ -363,6 +374,12 @@ def cli():
 			for problem in problems:
 				coverageProblemsExist = True
 				logging.error(problem)
+		if checkTargetValidity:
+			logging.debug("Checking target validity for '%s'." % ruleset.name)
+			problems = ruleset.getTargetValidityProblems()
+			for problem in problems:
+				targetValidityProblemExist = True
+				logging.error(problem)
 		if checkNonmatchGroups:
 			logging.debug("Checking non-match groups for '%s'." % ruleset.name)
 			problems = ruleset.getNonmatchGroupProblems()
@@ -445,6 +462,9 @@ def cli():
 	if checkCoverage:
 		if coverageProblemsExist:
 			return 1 # exit with error code
+	if checkTargetValidity:
+		if targetValidityProblemExist:
+			return 1 # exit with error code
 	if checkNonmatchGroups:
 		if nonmatchGroupProblemsExist:
 			return 1 # exit with error code
diff --git a/test/rules/src/https_everywhere_checker/rules.py b/test/rules/src/https_everywhere_checker/rules.py
@@ -1,6 +1,8 @@
+from tldextract import tldextract
 from urlparse import urlparse
 
 import regex
+import socket
 
 class Rule(object):
 	"""Represents one from->to rule element."""
@@ -179,6 +181,44 @@ def _determineTestApplication(self):
 				self.determine_test_application_run = True
 		return self.test_application_problems
 
+	def getTargetValidityProblems(self):
+		"""Verify that each target has a valid TLD in order to prevent problematic rewrite
+			 as stated in EFForg/https-everywhere/issues/10877. In particular, 
+			 right-wildcard target are ignored from this test.
+
+			 Returns an array of strings reporting any coverage problems if they exist,
+			 or empty list if coverage is sufficient.
+			 """
+		problems = self._determineTestApplication()
+		# Next, make sure each target has a valid TLD
+		for target in self.targets:
+			# Ignore right-wildcard targets
+			if target.endswith(".*"):
+				continue
+
+			# Ignore if target is an ipv4 address
+			try:
+				socket.inet_aton(target)
+				continue
+			except:
+				pass
+
+			# Ignore if target is an ipv6 address
+			try:
+				socket.inet_pton(socket.AF_INET6, target)
+				continue
+			except:
+				pass
+				
+			# Extract TLD from target if possible
+			res = tldextract.extract(target)
+			if res.suffix == "":
+				problems.append("%s: Target '%s' missing eTLD" % (self.filename, target))
+			elif res.domain == "":
+				problems.append("%s: Target '%s' containing entire eTLD" % (self.filename, target))
+				
+		return problems
+
 	def getCoverageProblems(self):
 		"""Verify that each rule and each exclusion has the right number of tests
 			 that applies to it. TODO: Also check that each target has the right
