Deal with surrogates broken over chunk boundaries.

gsnedders · gsnedders · commit 15d9b0e71ea4 · 2010-08-01T13:08:19.000+02:00
diff --git a/html5lib/inputstream.py b/html5lib/inputstream.py
@@ -178,8 +178,8 @@ def reset(self):
         # number of columns in the last line of the previous chunk
         self.prevNumCols = 0
         
-        #Flag to indicate we may have a CR LF broken across a data chunk
-        self._lastChunkEndsWithCR = False
+        #Deal with CR LF and surrogates split over chunk boundaries
+        self._bufferedCharacter = None
 
     def openStream(self, source):
         """Produces a file object from source.
@@ -344,23 +344,30 @@ def readChunk(self, chunkSize=None):
         self.chunkOffset = 0
 
         data = self.dataStream.read(chunkSize)
-
-        if not data:
+        
+        #Deal with CR LF and surrogates broken across chunks
+        if self._bufferedCharacter:
+            if data:
+                data = data + self._bufferedCharacter
+            else:
+                data = self._bufferedCharacter
+            self._bufferedCharacter = None
+        elif not data:
+            # We have no more data, bye-bye stream
             return False
         
+        if len(data) > 1:
+            lastv = ord(data[-1])
+            if lastv == 0x0D or 0xD800 <= lastv <= 0xDBFF:
+                self._bufferedCharacter = data[-1]
+                data = data[:-1]
+        
         self.reportCharacterErrors(data)
         
         # Replace invalid characters
         data = data.replace(u"\u0000", u"\ufffd")
         data = self.replaceCharactersRegexp.sub(u"\ufffd", data)
-        
-        #Check for CR LF broken across chunks
-        if (self._lastChunkEndsWithCR and data[0] == u"\n"):
-            data = data[1:]
-            # Stop if the chunk is now empty
-            if not data:
-                return False
-        self._lastChunkEndsWithCR = data[-1] == u"\r"
+                    
         data = data.replace(u"\r\n", u"\n")
         data = data.replace(u"\r", u"\n")
 
@@ -400,8 +407,6 @@ def characterErrorsUCS2(self, data):
             else:
                 skip = False
                 self.errors.append("invalid-codepoint")
-        #This is still wrong if it is possible for a surrogate pair to break a
-        #chunk boundary
 
     def charsUntil(self, characters, opposite = False):
         """ Returns a string of characters from the stream up to but not
