Fix syntax_non_utf8 test to not depend on locale encoding

youknowone · youknowone · commit 4ef242fd55bb · 2026-02-22T19:53:05.000+09:00
Use explicit encoding='latin-1' so the test works regardless of
the system locale (e.g. C/POSIX locale uses ASCII by default).
diff --git a/.cspell.json b/.cspell.json
@@ -147,6 +147,7 @@
     // "stat"
     "FIRMLINK",
     // CPython internal names
+    "PYTHONUTF",
     "sysdict",
     "settraceallthreads",
     "setprofileallthreads"
diff --git a/crates/vm/src/stdlib/_winapi.rs b/crates/vm/src/stdlib/_winapi.rs
@@ -279,9 +279,9 @@ mod _winapi {
         }
         si_attr!(dwFlags);
         si_attr!(wShowWindow);
-        si_attr!(hStdInput, usize);
-        si_attr!(hStdOutput, usize);
-        si_attr!(hStdError, usize);
+        si_attr!(hStdInput, isize);
+        si_attr!(hStdOutput, isize);
+        si_attr!(hStdError, isize);
 
         let mut env = args
             .env_mapping
@@ -1160,7 +1160,7 @@ mod _winapi {
         initial_state: bool,
         name: Option<PyStrRef>,
         vm: &VirtualMachine,
-    ) -> PyResult<Option<WinHandle>> {
+    ) -> PyResult<WinHandle> {
         use windows_sys::Win32::System::Threading::CreateEventW as WinCreateEventW;
 
         let _ = security_attributes; // Ignored, always NULL
@@ -1177,15 +1177,11 @@ mod _winapi {
             )
         };
 
-        if handle == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE {
-            return Err(vm.new_last_os_error());
-        }
-
         if handle.is_null() {
-            return Ok(None);
+            return Err(vm.new_last_os_error());
         }
 
-        Ok(Some(WinHandle(handle)))
+        Ok(WinHandle(handle))
     }
 
     /// SetEvent - Set the specified event object to the signaled state.
@@ -1944,9 +1940,7 @@ mod _winapi {
             } else {
                 hr as u32
             };
-            return Err(
-                std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm),
-            );
+            return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm));
         }
         Ok(())
     }
@@ -1967,6 +1961,7 @@ mod _winapi {
         if err != 0 {
             return Err(vm.new_os_error(err as i32));
         }
+        scopeguard::defer! { unsafe { RegCloseKey(hkcr) }; }
 
         let mut i: u32 = 0;
         let mut entries: Vec<(String, String)> = Vec::new();
@@ -1993,7 +1988,6 @@ mod _winapi {
                 break;
             }
             if err != 0 && err != windows_sys::Win32::Foundation::ERROR_MORE_DATA {
-                unsafe { RegCloseKey(hkcr) };
                 return Err(vm.new_os_error(err as i32));
             }
 
@@ -2013,7 +2007,6 @@ mod _winapi {
                 continue;
             }
             if err != 0 {
-                unsafe { RegCloseKey(hkcr) };
                 return Err(vm.new_os_error(err as i32));
             }
 
@@ -2057,8 +2050,6 @@ mod _winapi {
             }
         }
 
-        unsafe { RegCloseKey(hkcr) };
-
         // Process remaining entries
         for (mime_type, ext) in entries {
             on_type_read.call((vm.ctx.new_str(mime_type), vm.ctx.new_str(ext)), vm)?;
diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs
@@ -1588,7 +1588,11 @@ mod sys {
                 hash_randomization: settings.hash_seed.is_none() as u8,
                 isolated: settings.isolated as u8,
                 dev_mode: settings.dev_mode,
-                utf8_mode: settings.utf8_mode,
+                utf8_mode: if settings.utf8_mode < 0 {
+                    1
+                } else {
+                    settings.utf8_mode as u8
+                },
                 int_max_str_digits: settings.int_max_str_digits,
                 safe_path: settings.safe_path,
                 warn_default_encoding: settings.warn_default_encoding as u8,
diff --git a/crates/vm/src/vm/setting.rs b/crates/vm/src/vm/setting.rs
@@ -124,7 +124,7 @@ pub struct Settings {
     pub stdio_encoding: Option<String>,
     /// PYTHONIOENCODING - stdio error handler
     pub stdio_errors: Option<String>,
-    pub utf8_mode: u8,
+    pub utf8_mode: i8,
     /// --check-hash-based-pycs
     pub check_hash_pycs_mode: CheckHashPycsMode,
 
@@ -211,7 +211,7 @@ impl Default for Settings {
             allow_external_library: cfg!(feature = "importlib"),
             stdio_encoding: None,
             stdio_errors: None,
-            utf8_mode: 0,
+            utf8_mode: -1,
             int_max_str_digits: 4300,
             #[cfg(feature = "flame-it")]
             profile_output: None,
diff --git a/extra_tests/snippets/syntax_non_utf8.py b/extra_tests/snippets/syntax_non_utf8.py
@@ -5,5 +5,5 @@
 dir_path = os.path.dirname(os.path.realpath(__file__))
 
 with assert_raises(SyntaxError):
-    with open(os.path.join(dir_path , "non_utf8.txt")) as f:
+    with open(os.path.join(dir_path , "non_utf8.txt"), encoding="latin-1") as f:
         eval(f.read())
diff --git a/src/settings.rs b/src/settings.rs
@@ -260,10 +260,13 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> {
 
     settings.check_hash_pycs_mode = args.check_hash_based_pycs;
 
-    if let Some(val) = get_env("PYTHONUTF8") {
-        settings.utf8_mode = match val.to_str() {
-            Some("1") | Some("") => 1,
-            Some("0") => 0,
+    if let Some(val) = get_env("PYTHONUTF8")
+        && let Some(val_str) = val.to_str()
+        && !val_str.is_empty()
+    {
+        settings.utf8_mode = match val_str {
+            "1" => 1,
+            "0" => 0,
             _ => {
                 error!(
                     "Fatal Python error: config_init_utf8_mode: \
@@ -335,6 +338,13 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> {
     });
     settings.xoptions.extend(xopts);
 
+    // Resolve utf8_mode if not explicitly set by PYTHONUTF8 or -X utf8.
+    // Default to UTF-8 mode since RustPython's locale encoding detection
+    // is incomplete. Users can set PYTHONUTF8=0 or -X utf8=0 to disable.
+    if settings.utf8_mode < 0 {
+        settings.utf8_mode = 1;
+    }
+
     settings.warn_default_encoding =
         settings.warn_default_encoding || env_bool("PYTHONWARNDEFAULTENCODING");
     settings.faulthandler = settings.faulthandler || env_bool("PYTHONFAULTHANDLER");

Original file line number	Diff line number	Diff line change
`@@ -279,9 +279,9 @@ mod _winapi {`
`279`	`279`	`}`
`280`	`280`	`si_attr!(dwFlags);`
`281`	`281`	`si_attr!(wShowWindow);`
`282`		`- si_attr!(hStdInput, usize);`
`283`		`- si_attr!(hStdOutput, usize);`
`284`		`- si_attr!(hStdError, usize);`
	`282`	`+ si_attr!(hStdInput, isize);`
	`283`	`+ si_attr!(hStdOutput, isize);`
	`284`	`+ si_attr!(hStdError, isize);`
`285`	`285`
`286`	`286`	`let mut env = args`
`287`	`287`	`.env_mapping`
`@@ -1160,7 +1160,7 @@ mod _winapi {`
`1160`	`1160`	`initial_state: bool,`
`1161`	`1161`	`name: Option<PyStrRef>,`
`1162`	`1162`	`vm: &VirtualMachine,`
`1163`		`- ) -> PyResult<Option<WinHandle>> {`
	`1163`	`+ ) -> PyResult<WinHandle> {`
`1164`	`1164`	`use windows_sys::Win32::System::Threading::CreateEventW as WinCreateEventW;`
`1165`	`1165`
`1166`	`1166`	`let _ = security_attributes; // Ignored, always NULL`
`@@ -1177,15 +1177,11 @@ mod _winapi {`
`1177`	`1177`	`)`
`1178`	`1178`	`};`
`1179`	`1179`
`1180`		`- if handle == windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE {`
`1181`		`- return Err(vm.new_last_os_error());`
`1182`		`- }`
`1183`		`-`
`1184`	`1180`	`if handle.is_null() {`
`1185`		`- return Ok(None);`
	`1181`	`+ return Err(vm.new_last_os_error());`
`1186`	`1182`	`}`
`1187`	`1183`
`1188`		`- Ok(Some(WinHandle(handle)))`
	`1184`	`+ Ok(WinHandle(handle))`
`1189`	`1185`	`}`
`1190`	`1186`
`1191`	`1187`	`/// SetEvent - Set the specified event object to the signaled state.`
`@@ -1944,9 +1940,7 @@ mod _winapi {`
`1944`	`1940`	`} else {`
`1945`	`1941`	`hr as u32`
`1946`	`1942`	`};`
`1947`		`- return Err(`
`1948`		`- std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm),`
`1949`		`- );`
	`1943`	`+ return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm));`
`1950`	`1944`	`}`
`1951`	`1945`	`Ok(())`
`1952`	`1946`	`}`
`@@ -1967,6 +1961,7 @@ mod _winapi {`
`1967`	`1961`	`if err != 0 {`
`1968`	`1962`	`return Err(vm.new_os_error(err as i32));`
`1969`	`1963`	`}`
	`1964`	`+ scopeguard::defer! { unsafe { RegCloseKey(hkcr) }; }`
`1970`	`1965`
`1971`	`1966`	`let mut i: u32 = 0;`
`1972`	`1967`	`let mut entries: Vec<(String, String)> = Vec::new();`
`@@ -1993,7 +1988,6 @@ mod _winapi {`
`1993`	`1988`	`break;`
`1994`	`1989`	`}`
`1995`	`1990`	`if err != 0 && err != windows_sys::Win32::Foundation::ERROR_MORE_DATA {`
`1996`		`- unsafe { RegCloseKey(hkcr) };`
`1997`	`1991`	`return Err(vm.new_os_error(err as i32));`
`1998`	`1992`	`}`
`1999`	`1993`
`@@ -2013,7 +2007,6 @@ mod _winapi {`
`2013`	`2007`	`continue;`
`2014`	`2008`	`}`
`2015`	`2009`	`if err != 0 {`
`2016`		`- unsafe { RegCloseKey(hkcr) };`
`2017`	`2010`	`return Err(vm.new_os_error(err as i32));`
`2018`	`2011`	`}`
`2019`	`2012`
`@@ -2057,8 +2050,6 @@ mod _winapi {`
`2057`	`2050`	`}`
`2058`	`2051`	`}`
`2059`	`2052`
`2060`		`- unsafe { RegCloseKey(hkcr) };`
`2061`		`-`
`2062`	`2053`	`// Process remaining entries`
`2063`	`2054`	`for (mime_type, ext) in entries {`
`2064`	`2055`	`on_type_read.call((vm.ctx.new_str(mime_type), vm.ctx.new_str(ext)), vm)?;`