Oops; doesnt handle UTF-8 correctly when reading from stream

quest · quest · commit 1f8dfd8723dd · 2012-04-22T00:27:15.000+02:00
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
@@ -271,8 +271,9 @@ def get_tokens_unprocessed(self, stream, stack=('root',)):
         statetokens = tokendefs[statestack[-1]]
         known_names = {}
 
-        text = self._decode(stream.read(self.bufsize))
+        text = stream.read(self.bufsize)
         hasmore = len(text) == self.bufsize
+        text = self._decode(text)
 
         while 1:
             for rexmatch, action, new_state in statetokens:
@@ -317,9 +318,9 @@ def get_tokens_unprocessed(self, stream, stack=('root',)):
                     break
             else:
                 if hasmore:
-                    buf = self._decode(stream.read(self.bufsize))
+                    buf = stream.read(self.bufsize)
                     hasmore = len(buf) == self.bufsize
-                    text = text[pos:] + buf
+                    text = text[pos:] + self._decode(buf)
                     pos = 0
                     continue
 
