Merge python#23

bors[bot] · nanjekyejoannah · web-flow · commit f28eb8695fd0 · 2023-02-16T17:05:02.000Z
23: warn for tokenize() r=ltratt a=nanjekyejoannah

The constructor for `tokenize()` has changed in Python 3, to behave like the `tokenize.generate_token(`) function on Python 2.

Co-authored-by: Joannah Nanjekye &lt;jnanjeky@unb.ca&gt;
diff --git a/Lib/test/test_py3kwarn.py b/Lib/test/test_py3kwarn.py
@@ -331,6 +331,17 @@ def test_file_open(self):
             with check_py3k_warnings() as w:
                 self.assertWarning(f.read(), w, expected)
 
+    def test_tokenize(self):
+        import tokenize
+        import io
+        expected = "tokenize() changed in 3.x: use generate_tokens() instead."
+        def helper_tok():
+            for tok in tokenize.tokenize(io.BytesIO('1 + 2').readline):
+                print tok
+        with check_py3k_warnings() as w:
+            self.assertWarning(helper_tok(), w, expected)
+             
+
     def test_file(self):
         expected = ("The builtin 'file()'/'open()' function is not supported in 3.x, "
                     "use the 'io.open()' function instead with the encoding keyword argument")
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
@@ -29,6 +29,7 @@
 from itertools import chain
 import string, re
 from token import *
+import warnings
 
 import token
 __all__ = [x for x in dir(token) if not x.startswith("_")]
@@ -166,6 +167,8 @@ def tokenize(readline, tokeneater=printtoken):
     called once for each token, with five arguments, corresponding to the
     tuples generated by generate_tokens().
     """
+    warnings.warnpy3k_with_fix("tokenize() changed in 3.x", "use generate_tokens() instead.",
+                      stacklevel=2)
     try:
         tokenize_loop(readline, tokeneater)
     except StopTokenizing:
