Fix _tokenize CI failures and address code review

youknowone · youknowone · commit 042aed488d38 · 2026-03-10T01:04:43.000+09:00
- Rename tokenize.rs to _tokenize.rs
- Skip IndentationError when source contains tabs, as ruff and
  CPython handle tab indentation differently (CPython uses tabsize=8)
- Raise SyntaxError for unclosed brackets before ENDMARKER
- Remove expectedFailure from test_with_errored_file (now passes)
diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py
@@ -316,7 +316,6 @@ def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failur
             self.assertListEqual(out.splitlines(), stdout.splitlines())
             self.assertListEqual(err.splitlines(), stderr.splitlines())
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Should displays error when errored python file is given.
     def test_with_errored_file(self):
         """Should displays error when errored python file is given."""
         with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path:
diff --git a/crates/stdlib/src/_tokenize.rs b/crates/stdlib/src/_tokenize.rs
@@ -206,18 +206,23 @@ mod _tokenize {
             let kind = token.kind();
             let range = token.range();
 
-            // Check for lexical indentation errors
-            for err in errors.iter() {
-                if !matches!(
-                    err.error,
-                    ParseErrorType::Lexical(LexicalErrorType::IndentationError)
-                ) {
-                    continue;
-                }
-                if err.location.start() <= range.start()
-                    && range.start() < err.location.end()
-                {
-                    return Err(raise_indentation_error(vm, err, source, line_index));
+            // Check for lexical indentation errors.
+            // Skip when source has tabs — ruff and CPython handle tab
+            // indentation differently (CPython uses tabsize=8), so ruff may
+            // report false IndentationErrors for valid mixed-tab code.
+            if !source.contains('\t') {
+                for err in errors.iter() {
+                    if !matches!(
+                        err.error,
+                        ParseErrorType::Lexical(LexicalErrorType::IndentationError)
+                    ) {
+                        continue;
+                    }
+                    if err.location.start() <= range.start()
+                        && range.start() < err.location.end()
+                    {
+                        return Err(raise_indentation_error(vm, err, source, line_index));
+                    }
                 }
             }
 
@@ -348,6 +353,28 @@ mod _tokenize {
             }
         }
 
+        // Check for unclosed brackets before ENDMARKER — CPython's tokenizer
+        // raises SyntaxError("EOF in multi-line statement") in this case.
+        {
+            let bracket_count: i32 = tokens
+                .iter()
+                .map(|t| match t.kind() {
+                    TokenKind::Lpar | TokenKind::Lsqb | TokenKind::Lbrace => 1,
+                    TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace => -1,
+                    _ => 0,
+                })
+                .sum();
+            if bracket_count > 0 {
+                let last_line = source.lines().count();
+                return Err(raise_syntax_error(
+                    vm,
+                    "EOF in multi-line statement",
+                    last_line + 1,
+                    0,
+                ));
+            }
+        }
+
         // All tokens consumed — emit ENDMARKER
         let last_line = source.lines().count();
         let (em_line, em_col, em_line_str): (usize, isize, &str) = if extra_tokens {
@@ -414,6 +441,26 @@ mod _tokenize {
         (default_pos, "")
     }
 
+    /// Raise a SyntaxError with the given message and position.
+    fn raise_syntax_error(
+        vm: &VirtualMachine,
+        msg: &str,
+        lineno: usize,
+        offset: usize,
+    ) -> rustpython_vm::builtins::PyBaseExceptionRef {
+        let exc = vm.new_exception_msg(
+            vm.ctx.exceptions.syntax_error.to_owned(),
+            msg.into(),
+        );
+        let obj = exc.as_object();
+        let _ = obj.set_attr("msg", vm.ctx.new_str(msg), vm);
+        let _ = obj.set_attr("lineno", vm.ctx.new_int(lineno), vm);
+        let _ = obj.set_attr("offset", vm.ctx.new_int(offset), vm);
+        let _ = obj.set_attr("filename", vm.ctx.new_str("<string>"), vm);
+        let _ = obj.set_attr("text", vm.ctx.none(), vm);
+        exc
+    }
+
     /// Raise an IndentationError from a parse error.
     fn raise_indentation_error(
         vm: &VirtualMachine,
diff --git a/crates/stdlib/src/lib.rs b/crates/stdlib/src/lib.rs
@@ -49,7 +49,8 @@ mod pystruct;
 mod random;
 mod statistics;
 mod suggestions;
-mod tokenize;
+#[path = "_tokenize.rs"]
+mod _tokenize;
 // TODO: maybe make this an extension module, if we ever get those
 // mod re;
 #[cfg(all(feature = "host_env", not(target_arch = "wasm32")))]
@@ -226,7 +227,7 @@ pub fn stdlib_module_defs(ctx: &Context) -> Vec<&'static builtins::PyModuleDef>
         ssl::module_def(ctx),
         statistics::module_def(ctx),
         suggestions::module_def(ctx),
-        tokenize::module_def(ctx),
+        _tokenize::module_def(ctx),
         #[cfg(all(feature = "host_env", unix, not(target_os = "redox")))]
         syslog::module_def(ctx),
         #[cfg(all(
