Simplify handling of encoding in lexer.py

vmuriart · vmuriart · commit faacd60c2769 · 2016-05-29T11:34:37.000-07:00
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
@@ -13,7 +13,6 @@
 # and to allow some customizations.
 
 import re
-import sys
 
 from sqlparse import tokens
 from sqlparse.keywords import SQL_REGEX
@@ -42,14 +41,6 @@ def __init__(self):
                         new_state = (tdef[2],)
                 self._tokens[state].append((rex, tdef[1], new_state))
 
-    def _decode(self, text):
-        if not isinstance(text, text_type):
-            try:
-                text = text.decode(self.encoding)
-            except UnicodeDecodeError:
-                text = text.decode('unicode-escape')
-        return text
-
     def get_tokens(self, text):
         """
         Return an iterable of (tokentype, value) pairs generated from
@@ -58,27 +49,24 @@ def get_tokens(self, text):
 
         Also preprocess the text, i.e. expand tabs and strip it if
         wanted and applies registered filters.
-        """
-        if isinstance(text, string_types):
-            if sys.version_info[0] < 3 and isinstance(text, text_type):
-                text = StringIO(text.encode('utf-8'))
-                self.encoding = 'utf-8'
-            else:
-                text = StringIO(text)
-
-        return self.get_tokens_unprocessed(text)
 
-    def get_tokens_unprocessed(self, stream):
-        """
         Split ``text`` into (tokentype, text) pairs.
 
         ``stack`` is the inital stack (default: ``['root']``)
         """
         statestack = ['root', ]
         statetokens = self._tokens['root']
 
-        text = stream.read()
-        text = self._decode(text)
+        if isinstance(text, string_types):
+            text = StringIO(text)
+
+        text = text.read()
+        if not isinstance(text, text_type):
+            try:
+                text = text.decode(self.encoding)
+            except UnicodeDecodeError:
+                text = text.decode('unicode-escape')
+
         iterable = iter(range(len(text)))
 
         for pos in iterable:
