Get html5lib passing tests again after 964568c175 on UCS2 Python. This checks explicitly for lone surrogates in the UCS2 case.

gsnedders · gsnedders · commit 329cae600c60 · 2010-07-31T22:18:04.000+02:00
diff --git a/html5lib/inputstream.py b/html5lib/inputstream.py
@@ -134,10 +134,10 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
         #Craziness
         if len(u"\U0010FFFF") == 1:
             self.reportCharacterErrors = self.characterErrorsUCS4
+            self.replaceCharactersRegexp = re.compile(u"[\uD800-\uDFFF]")
         else:
             self.reportCharacterErrors = self.characterErrorsUCS2
-
-        self.replaceCharactersRegexp = re.compile(u"[\u0000\uD800-\uDFFF]")
+            self.replaceCharactersRegexp = re.compile(u"([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])")
 
         # List of where new lines occur
         self.newLines = [0]
@@ -349,8 +349,11 @@ def readChunk(self, chunkSize=None):
             return False
         
         self.reportCharacterErrors(data)
-
-        data = self.replaceCharactersRegexp.subn(u"\ufffd", data)[0]
+        
+        # Replace invalid characters
+        data = data.replace(u"\u0000", u"\ufffd")
+        data = self.replaceCharactersRegexp.sub(u"\ufffd", data)
+        
         #Check for CR LF broken across chunks
         if (self._lastChunkEndsWithCR and data[0] == u"\n"):
             data = data[1:]
