Progress towards streaming

rubys · rubys · commit 3391e3e6a996 · 2007-06-07T01:25:41.000Z
--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%40674
diff --git a/src/inputstream.py b/src/inputstream.py
@@ -31,9 +31,9 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
 
         """
         # List of where new lines occur
-        self.newLines = []
+        self.newLines = [0]
 
-      # Raw Stream
+        # Raw Stream
         self.rawStream = self.openStream(source)
 
         # Encoding Information
@@ -47,15 +47,18 @@ def __init__(self, source, encoding=None, parseMeta=True, chardet=True):
         if encoding is None or not isValidEncoding(encoding):
             encoding = self.detectEncoding(parseMeta, chardet)
         self.charEncoding = encoding
+        self.win1252 = False
 
         # Read bytes from stream decoding them into Unicode
-        uString = self.rawStream.read().decode(self.charEncoding, 'replace')
-
-        # Normalize new ipythonlines and null characters
-        uString = re.sub('\r\n?', '\n', uString)
-        uString = re.sub('\x00', u'\uFFFD', uString)
+        uString = self.rawStream.read()
 
         # Convert the unicode string into a list to be used as the data stream
+        if self.charEncoding == 'windows-1252':
+            self.win1252 = True
+        else:
+            self.win1252 = False
+            uString = uString.decode(self.charEncoding, 'replace')
+
         self.dataStream = uString
 
         self.queue = []
@@ -148,20 +151,8 @@ def detectEncodingMeta(self):
         self.rawStream.seek(0)
         return parser.getEncoding()
 
-    def determineNewLines(self):
-        # Looks through the stream to find where new lines occur so
-        # the position method can tell where it is.
-        self.newLines.append(0)
-        for i in xrange(len(self.dataStream)):
-            if self.dataStream[i] == u"\n":
-                self.newLines.append(i)
-
     def position(self):
         """Returns (line, col) of the current position in the stream."""
-        # Generate list of new lines first time around
-        if not self.newLines:
-            self.determineNewLines()
-
         line = 0
         tell = self.tell
         for pos in self.newLines:
@@ -184,8 +175,22 @@ def char(self):
             return self.queue.pop(0)
         else:
             try:
+                c = self.dataStream[self.tell]
                 self.tell += 1
-                return self.dataStream[self.tell - 1]
+                if self.win1252 and c >= '\x80': c=c.decode('windows-1252')
+
+                # Normalize newlines and null characters
+                if c == '\x00': c = u'\uFFFD'
+                if c == '\r':
+                    if self.tell < len(self.dataStream) and \
+                      self.dataStream[self.tell] == '\n':
+                        self.tell += 1
+                    c = '\n'
+
+                # record where newlines occur so that the position method
+                # can tell where it is
+                if c == '\n': self.newLines.append(self.tell - 1)
+                return c
             except:
                 return EOF
 
@@ -196,22 +201,17 @@ def charsUntil(self, characters, opposite = False):
         """
         charStack = [self.char()]
 
-        # First from the queue
-        while charStack[-1] and (charStack[-1] in characters) == opposite \
-          and self.queue:
-            charStack.append(self.queue.pop(0))
-
-        # Then the rest
         while charStack[-1] and (charStack[-1] in characters) == opposite:
-            try:
-                self.tell += 1
-                charStack.append(self.dataStream[self.tell - 1])
-            except:
-                charStack.append(EOF)
+            charStack.append(self.char())
 
         # Put the character stopped on back to the front of the queue
         # from where it came.
-        self.queue.insert(0, charStack.pop())
+        c = charStack.pop()
+        if c != EOF and self.tell <= len(self.dataStream) and \
+          self.dataStream[self.tell - 1] == c[0]:
+            self.tell -= 1
+        else:
+            self.queue.insert(0, c)
         return "".join(charStack)
 
 class EncodingBytes(str):
diff --git a/tests/test_stream.py b/tests/test_stream.py
@@ -31,9 +31,11 @@ def test_char_utf8(self):
         self.assertEquals(stream.char(), u'\u2018')
 
     def test_char_win1252(self):
-        stream = HTMLInputStream(u'\u2018'.encode('windows-1252'))
+        stream = HTMLInputStream(u"\xa9\xf1\u2019".encode('windows-1252'))
         self.assertEquals(stream.charEncoding, 'windows-1252')
-        self.assertEquals(stream.char(), u'\u2018')
+        self.assertEquals(stream.char(), u"\xa9")
+        self.assertEquals(stream.char(), u"\xf1")
+        self.assertEquals(stream.char(), u"\u2019")
 
     def test_bom(self):
         stream = HTMLInputStream(codecs.BOM_UTF8 + "'")
@@ -50,11 +52,11 @@ def test_newlines(self):
         self.assertEquals(stream.tell, 0)
         self.assertEquals(stream.charsUntil('c'),u"a\nbb\n")
         self.assertEquals(stream.tell, 6)
-        self.assertEquals(stream.position(), (3,1))
+        self.assertEquals(stream.position(), (3,0))
         self.assertEquals(stream.charsUntil('x'),u"ccc\ndddd")
         self.assertEquals(stream.tell, 14)
-        self.assertEquals(stream.position(), (4,5))
-        self.assertEquals(stream.newLines, [0,1,4,8])
+        self.assertEquals(stream.position(), (4,4))
+        self.assertEquals(stream.newLines, [0,1,5,9])
 
 def buildTestSuite():
     return unittest.defaultTestLoader.loadTestsFromName(__name__)
