Set the right character encoding when a unicode object is passed to the inputstream. We now pass all tests again!

jgraham · jgraham · commit e97b0689f697 · 2007-07-16T20:56:42.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40909
diff --git a/src/html5lib/inputstream.py b/src/html5lib/inputstream.py
@@ -36,7 +36,10 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
         # List of where new lines occur
         self.newLines = [0]
 
-        # Raw Stream
+        self.charEncoding = encoding
+
+        # Raw Stream - for unicode objects this will encode to utf-8 and set
+        #              self.charEncoding as appropriate
         self.rawStream = self.openStream(source)
 
         # Encoding Information
@@ -49,9 +52,8 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
         self.defaultEncoding = "windows-1252"
         
         #Detect encoding iff no explicit "transport level" encoding is supplied
-        if encoding is None or not isValidEncoding(encoding):
-            encoding = self.detectEncoding(parseMeta, chardet)
-        self.charEncoding = encoding
+        if self.charEncoding is None or not isValidEncoding(self.charEncoding):
+            self.charEncoding = self.detectEncoding(parseMeta, chardet)
 
         self.dataStream = codecs.getreader(self.charEncoding)(self.rawStream, 'replace')
 
@@ -74,6 +76,7 @@ def openStream(self, source):
             # Otherwise treat source as a string and convert to a file object
             if isinstance(source, unicode):
                 source = source.encode('utf-8')
+                self.charEncoding = "utf-8"
             import cStringIO
             stream = cStringIO.StringIO(str(source))
         return stream
diff --git a/tests/test_sanitizer.py b/tests/test_sanitizer.py
@@ -8,6 +8,7 @@ def addTest(cls, name, expected, input):
     def test(self, expected=expected, input=input):
         expected = ''.join([token.toxml() for token in html5parser.HTMLParser().
           parseFragment(expected).childNodes])
+        expected = simplejson.loads(simplejson.dumps(expected))
         self.assertEqual(expected, self.sanitize_html(input))
     setattr(cls, name, test)
   addTest = classmethod(addTest)
@@ -73,7 +74,7 @@ def test_should_handle_astral_plane_characters(self):
 def buildTestSuite():
     for filename in html5lib_test_files("sanitizer"):
         for test in simplejson.load(file(filename)):
-            SanitizeTest.addTest('test_' + test['name'], test['output'], test['input'])
+          SanitizeTest.addTest('test_' + test['name'], test['output'], test['input'])
 
     return unittest.TestLoader().loadTestsFromTestCase(SanitizeTest)
 
