Get encoding tests running again under nose and Py3

gsnedders · gsnedders · commit 2816de7bf872 · 2013-01-09T16:46:10.000Z
diff --git a/html5lib/tests/support.py b/html5lib/tests/support.py
@@ -74,8 +74,12 @@ def __getitem__(self, key):
         return dict.get(self, key, self.default)
 
 class TestData(object):
-    def __init__(self, filename, newTestHeading="data"):
-        self.f = codecs.open(filename, encoding="utf8")
+    def __init__(self, filename, newTestHeading="data", encoding="utf8"):
+        if encoding == None:
+            self.f = open(filename, mode="rb")
+        else:
+            self.f = codecs.open(filename, encoding=encoding)
+        self.encoding = encoding
         self.newTestHeading = newTestHeading
 
     def __del__(self):
@@ -93,7 +97,7 @@ def __iter__(self):
                     yield self.normaliseOutput(data)
                     data = DefaultDict(None)
                 key = heading
-                data[key]=""
+                data[key]="" if self.encoding else b""
             elif key is not None:
                 data[key] += line
         if data:
@@ -102,15 +106,16 @@ def __iter__(self):
     def isSectionHeading(self, line):
         """If the current heading is a test section heading return the heading,
         otherwise return False"""
-        if line.startswith("#"):
+        #print(line)
+        if line.startswith("#" if self.encoding else b"#"):
             return line[1:].strip()
         else:
             return False
     
     def normaliseOutput(self, data):
         #Remove trailing newlines
         for key,value in data.items():
-            if value.endswith("\n"):
+            if value.endswith("\n" if self.encoding else b"\n"):
                 data[key] = value[:-1]
         return data
 
diff --git a/html5lib/tests/test_encoding.py b/html5lib/tests/test_encoding.py
@@ -23,42 +23,30 @@ def test_codec_name_c(self):
     def test_codec_name_d(self):
         self.assertEqual(inputstream.codecName("ISO_8859--1"), "windows-1252")
 
-def buildTestSuite():
+def runEncodingTest(data, encoding):
+    p = HTMLParser()
+    t = p.parse(data, useChardet=False)
+    encoding = encoding.lower().decode("ascii")
+
+    errorMessage = ("Input:\n%s\nExpected:\n%s\nRecieved\n%s\n"%
+                    (data, repr(encoding), 
+                     repr(p.tokenizer.stream.charEncoding[0])))
+    assert encoding == p.tokenizer.stream.charEncoding[0], errorMessage
+
+def test_encoding():
     for filename in get_data_files("encoding"):
         test_name = os.path.basename(filename).replace('.dat',''). \
             replace('-','')
-        tests = TestData(filename, "data")
+        tests = TestData(filename, b"data", encoding=None)
         for idx, test in enumerate(tests):
-            def encodingTest(self, data=test['data'], 
-                             encoding=test['encoding']):
-                p = HTMLParser()
-                t = p.parse(data, useChardet=False)
-                
-                errorMessage = ("Input:\n%s\nExpected:\n%s\nRecieved\n%s\n"%
-                                (data, repr(encoding.lower()), 
-                                 repr(p.tokenizer.stream.charEncoding)))
-                self.assertEquals(encoding.lower(),
-                                  p.tokenizer.stream.charEncoding[0], 
-                                  errorMessage)
-            setattr(Html5EncodingTestCase, 'test_%s_%d' % (test_name, idx+1),
-                encodingTest)
-
-    try:
-        import chardet
-        def test_chardet(self):
-            data = open(os.path.join(test_dir, "encoding" , "chardet", "test_big5.txt")).read()
-            encoding = inputstream.HTMLInputStream(data).charEncoding
-            assert encoding[0].lower() == "big5"
-        setattr(Html5EncodingTestCase, 'test_chardet', test_chardet)
-    except ImportError:
-        print("chardet not found, skipping chardet tests")
-        
+            yield (runEncodingTest, test[b'data'], test[b'encoding'])
 
-    return unittest.defaultTestLoader.loadTestsFromName(__name__)
-
-def main():
-    buildTestSuite()
-    unittest.main()
-
-if __name__ == "__main__":
-    main()
+try:
+    import chardet
+    def test_chardet(self):
+        data = open(os.path.join(test_dir, "encoding" , "chardet", "test_big5.txt")).read()
+        encoding = inputstream.HTMLInputStream(data).charEncoding
+        assert encoding[0].lower() == "big5"
+    setattr(Html5EncodingTestCase, 'test_chardet', test_chardet)
+except ImportError:
+    print("chardet not found, skipping chardet tests")
