Fix syntax_non_utf8 test to not depend on locale encoding

youknowone · youknowone · commit a6f53a3cb5bb · 2026-02-22T16:39:23.000+09:00
Use explicit encoding='latin-1' so the test works regardless of
the system locale (e.g. C/POSIX locale uses ASCII by default).
diff --git a/.cspell.json b/.cspell.json
@@ -147,6 +147,7 @@
     // "stat"
     "FIRMLINK",
     // CPython internal names
+    "PYTHONUTF",
     "sysdict",
     "settraceallthreads",
     "setprofileallthreads"
diff --git a/crates/vm/src/stdlib/_winapi.rs b/crates/vm/src/stdlib/_winapi.rs
@@ -279,9 +279,9 @@ mod _winapi {
         }
         si_attr!(dwFlags);
         si_attr!(wShowWindow);
-        si_attr!(hStdInput, usize);
-        si_attr!(hStdOutput, usize);
-        si_attr!(hStdError, usize);
+        si_attr!(hStdInput, isize);
+        si_attr!(hStdOutput, isize);
+        si_attr!(hStdError, isize);
 
         let mut env = args
             .env_mapping
@@ -1177,12 +1177,8 @@ mod _winapi {
             )
         };
 
-        if handle == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE {
-            return Err(vm.new_last_os_error());
-        }
-
         if handle.is_null() {
-            return Ok(None);
+            return Err(vm.new_last_os_error());
         }
 
         Ok(Some(WinHandle(handle)))
@@ -1944,9 +1940,7 @@ mod _winapi {
             } else {
                 hr as u32
             };
-            return Err(
-                std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm),
-            );
+            return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm));
         }
         Ok(())
     }
diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs
@@ -1588,7 +1588,7 @@ mod sys {
                 hash_randomization: settings.hash_seed.is_none() as u8,
                 isolated: settings.isolated as u8,
                 dev_mode: settings.dev_mode,
-                utf8_mode: settings.utf8_mode,
+                utf8_mode: if settings.utf8_mode < 0 { 1 } else { settings.utf8_mode as u8 },
                 int_max_str_digits: settings.int_max_str_digits,
                 safe_path: settings.safe_path,
                 warn_default_encoding: settings.warn_default_encoding as u8,
diff --git a/crates/vm/src/vm/setting.rs b/crates/vm/src/vm/setting.rs
@@ -124,7 +124,7 @@ pub struct Settings {
     pub stdio_encoding: Option<String>,
     /// PYTHONIOENCODING - stdio error handler
     pub stdio_errors: Option<String>,
-    pub utf8_mode: u8,
+    pub utf8_mode: i8,
     /// --check-hash-based-pycs
     pub check_hash_pycs_mode: CheckHashPycsMode,
 
@@ -211,7 +211,7 @@ impl Default for Settings {
             allow_external_library: cfg!(feature = "importlib"),
             stdio_encoding: None,
             stdio_errors: None,
-            utf8_mode: 0,
+            utf8_mode: -1,
             int_max_str_digits: 4300,
             #[cfg(feature = "flame-it")]
             profile_output: None,
diff --git a/extra_tests/snippets/syntax_non_utf8.py b/extra_tests/snippets/syntax_non_utf8.py
@@ -5,5 +5,5 @@
 dir_path = os.path.dirname(os.path.realpath(__file__))
 
 with assert_raises(SyntaxError):
-    with open(os.path.join(dir_path , "non_utf8.txt")) as f:
+    with open(os.path.join(dir_path , "non_utf8.txt"), encoding="latin-1") as f:
         eval(f.read())
diff --git a/src/settings.rs b/src/settings.rs
@@ -335,6 +335,13 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> {
     });
     settings.xoptions.extend(xopts);
 
+    // Resolve utf8_mode if not explicitly set by PYTHONUTF8 or -X utf8.
+    // Default to UTF-8 mode since RustPython's locale encoding detection
+    // is incomplete. Users can set PYTHONUTF8=0 or -X utf8=0 to disable.
+    if settings.utf8_mode < 0 {
+        settings.utf8_mode = 1;
+    }
+
     settings.warn_default_encoding =
         settings.warn_default_encoding || env_bool("PYTHONWARNDEFAULTENCODING");
     settings.faulthandler = settings.faulthandler || env_bool("PYTHONFAULTHANDLER");
