Fix syntax_non_utf8 test to not depend on locale encoding

youknowone · youknowone · commit d1643accd492 · 2026-02-22T00:29:55.000+09:00
Use explicit encoding='latin-1' so the test works regardless of
the system locale (e.g. C/POSIX locale uses ASCII by default).
diff --git a/.cspell.json b/.cspell.json
@@ -147,6 +147,7 @@
     // "stat"
     "FIRMLINK",
     // CPython internal names
+    "PYTHONUTF",
     "sysdict",
     "settraceallthreads",
     "setprofileallthreads"
diff --git a/crates/vm/src/stdlib/_winapi.rs b/crates/vm/src/stdlib/_winapi.rs
@@ -1944,9 +1944,7 @@ mod _winapi {
             } else {
                 hr as u32
             };
-            return Err(
-                std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm),
-            );
+            return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm));
         }
         Ok(())
     }
diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs
@@ -1588,7 +1588,7 @@ mod sys {
                 hash_randomization: settings.hash_seed.is_none() as u8,
                 isolated: settings.isolated as u8,
                 dev_mode: settings.dev_mode,
-                utf8_mode: settings.utf8_mode,
+                utf8_mode: settings.utf8_mode as u8,
                 int_max_str_digits: settings.int_max_str_digits,
                 safe_path: settings.safe_path,
                 warn_default_encoding: settings.warn_default_encoding as u8,
diff --git a/crates/vm/src/vm/setting.rs b/crates/vm/src/vm/setting.rs
@@ -124,7 +124,7 @@ pub struct Settings {
     pub stdio_encoding: Option<String>,
     /// PYTHONIOENCODING - stdio error handler
     pub stdio_errors: Option<String>,
-    pub utf8_mode: u8,
+    pub utf8_mode: i8,
     /// --check-hash-based-pycs
     pub check_hash_pycs_mode: CheckHashPycsMode,
 
@@ -211,7 +211,7 @@ impl Default for Settings {
             allow_external_library: cfg!(feature = "importlib"),
             stdio_encoding: None,
             stdio_errors: None,
-            utf8_mode: 0,
+            utf8_mode: -1,
             int_max_str_digits: 4300,
             #[cfg(feature = "flame-it")]
             profile_output: None,
diff --git a/extra_tests/snippets/syntax_non_utf8.py b/extra_tests/snippets/syntax_non_utf8.py
@@ -5,5 +5,5 @@
 dir_path = os.path.dirname(os.path.realpath(__file__))
 
 with assert_raises(SyntaxError):
-    with open(os.path.join(dir_path , "non_utf8.txt")) as f:
+    with open(os.path.join(dir_path , "non_utf8.txt"), encoding="latin-1") as f:
         eval(f.read())
diff --git a/src/settings.rs b/src/settings.rs
@@ -335,6 +335,13 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> {
     });
     settings.xoptions.extend(xopts);
 
+    // Resolve utf8_mode if not explicitly set by PYTHONUTF8 or -X utf8.
+    // Default to UTF-8 mode since RustPython's locale encoding detection
+    // is incomplete. Users can set PYTHONUTF8=0 or -X utf8=0 to disable.
+    if settings.utf8_mode < 0 {
+        settings.utf8_mode = 1;
+    }
+
     settings.warn_default_encoding =
         settings.warn_default_encoding || env_bool("PYTHONWARNDEFAULTENCODING");
     settings.faulthandler = settings.faulthandler || env_bool("PYTHONFAULTHANDLER");

Original file line number	Diff line number	Diff line change
`@@ -1944,9 +1944,7 @@ mod _winapi {`
`1944`	`1944`	`} else {`
`1945`	`1945`	`hr as u32`
`1946`	`1946`	`};`
`1947`		`- return Err(`
`1948`		`- std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm),`
`1949`		`- );`
	`1947`	`+ return Err(std::io::Error::from_raw_os_error(err as i32).to_pyexception(vm));`
`1950`	`1948`	`}`
`1951`	`1949`	`Ok(())`
`1952`	`1950`	`}`