Replace unpaired surrogates with replacement characters (unlikely to work on UCS2 python)

James Graham · James Graham · commit 55bfda0fcff0 · 2010-07-22T14:28:34.000+02:00
diff --git a/html5lib/inputstream.py b/html5lib/inputstream.py
@@ -137,6 +137,8 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
         else:
             self.reportCharacterErrors = self.characterErrorsUCS2
 
+        self.replaceCharactersRegexp = re.compile(u"[\u0000\uD800-\uDFFF]")
+
         # List of where new lines occur
         self.newLines = [0]
 
@@ -159,6 +161,7 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
         if (self.charEncoding[0] is None):
             self.charEncoding = self.detectEncoding(parseMeta, chardet)
 
+
         self.reset()
 
     def reset(self):
@@ -347,7 +350,7 @@ def readChunk(self, chunkSize=None):
         
         self.reportCharacterErrors(data)
 
-        data = data.replace(u"\u0000", u"\ufffd")
+        data = self.replaceCharactersRegexp.subn(u"\ufffd", data)[0]
         #Check for CR LF broken across chunks
         if (self._lastChunkEndsWithCR and data[0] == u"\n"):
             data = data[1:]
