Read files in chunks rather than one character at a time

jgraham · jgraham · commit 1478d54e052a · 2007-07-17T20:03:29.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40910
diff --git a/src/html5lib/inputstream.py b/src/html5lib/inputstream.py
@@ -55,13 +55,17 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
         if self.charEncoding is None or not isValidEncoding(self.charEncoding):
             self.charEncoding = self.detectEncoding(parseMeta, chardet)
 
-        self.dataStream = codecs.getreader(self.charEncoding)(self.rawStream, 'replace')
+        self.dataStream = codecs.getreader(self.charEncoding)(self.rawStream,
+                                                              'replace')
 
         self.queue = []
         self.errors = []
 
         self.line = self.col = 0
         self.lineLengths = []
+        
+        #Flag to indicate we may have a CR LF broken across a data chunk
+        self._lastChunkEndsWithCR = False
 
     def openStream(self, source):
         """Produces a file object from source.
@@ -199,64 +203,47 @@ def detectEncodingMeta(self):
     def position(self):
         """Returns (line, col) of the current position in the stream."""
         line, col = self.line, self.col
-        for c in self.queue[::-1]:
-            if c == '\n':
-                line -= 1
-                assert col == 0
-                col = self.lineLengths[line]
-            else:
-                col -= 1
         return (line + 1, col)
 
     def char(self):
         """ Read one character from the stream or queue if available. Return
             EOF when EOF is reached.
         """
-        if self.queue:
-            char = self.queue.pop(0)
-            if char == "\n":
-                self.lineLengths.append(self.col)
-                self.line += 1
-                self.col = 0
-            return char
+        if not self.queue:
+            self.readChunk()
+        #If we still don't have a character we have reached EOF
+        if not self.queue:
+            return EOF
+        
+        char = self.queue.pop(0)
+        
+        # update position in stream
+        if char == '\n':
+            self.lineLengths.append(self.col)
+            self.line += 1
+            self.col = 0
         else:
-            c = self.readChar()
-            if c is EOF:
-                return c
-            
-            if c == '\r':
-                #XXX This isn't right in the case with multiple CR in a row
-                #also recursing here isn't ideal + not sure what happens to input position
-                c = self.readChar()
-                if c is not EOF and c not in ('\n', '\r'):
-                    self.queue.insert(0, unicode(c))
-                elif c == '\r':
-                    self.queue.insert(0, u'\n')
-                c = '\n'
-
-            # update position in stream
-            if c == '\n':
-                self.lineLengths.append(self.col)
-                self.line += 1
-                self.col = 0
-            else:
-                self.col += 1
-            return unicode(c)
-
-    def readChar(self):
-        """Read the next character from the datastream and normalize for null
-        but not for CR"""
-        c = self.dataStream.read(1, 1)
-        if not c:
             self.col += 1
-            return EOF
-
-        # Normalize newlines and null characters
-        if c == '\x00':
+        return char
+
+    def readChunk(self, chunkSize=1024):
+        data = self.dataStream.read(1024)
+        if not data:
+            return
+        #Replace null characters
+        for i in xrange(data.count(u"\u0000")):
             self.errors.append(_('null character found in input stream, '
-                'replaced with U+FFFD'))
-            c = u'\uFFFD'
-        return c
+                                 'replaced with U+FFFD'))
+        data = data.replace(u"\u0000", u"\ufffd")
+        #Check for CR LF broken across chunks
+        if (self._lastChunkEndsWithCR and data[0] == "\n"):
+            data = data[1:]
+        self._lastChunkEndsWithCR = data[-1] == "\r"
+        data = data.replace("\r\n", "\n")
+        data = data.replace("\r", "\n")
+        
+        data = unicode(data)
+        self.queue.extend([char for char in data])
 
     def charsUntil(self, characters, opposite = False):
         """ Returns a string of characters from the stream up to but not
@@ -272,13 +259,20 @@ def charsUntil(self, characters, opposite = False):
         # from where it came.
         c = charStack.pop()
         if c != EOF:
-            self.queue.insert(0, c)
+            self.unget(c)
         
         return u"".join(charStack)
 
     def unget(self, chars):
         if chars:
             self.queue = list(chars) + self.queue
+            #Alter the current line, col position
+            for c in chars[::-1]:
+                if c == '\n':
+                    self.line -= 1
+                    self.col = self.lineLengths[self.line]
+                else:
+                    self.col -= 1
 
 class EncodingBytes(str):
     """String-like object with an assosiated position and various extra methods
