RustPython · youknowone · Aug 11, 2026 · Aug 11, 2026
diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py
@@ -1067,8 +1067,7 @@ def test_decodehelper_bug36819(self):
                 decoded = input.decode(enc, "test.bug36819")
                 self.assertEqual(decoded, 'abcdx' * 51)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
+    @unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON")
     def test_encodehelper_bug36819(self):
         handler = RepeatedPosReturn()
         codecs.register_error("test.bug36819", handler.handle)

diff --git a/crates/capi/src/pyerrors.rs b/crates/capi/src/pyerrors.rs
@@ -347,7 +347,7 @@ pub unsafe extern "C" fn PyUnicodeDecodeError_Create(
             unsafe { slice::from_raw_parts(object.cast::<u8>(), length) }.to_vec()
         };
 
-        let exc = vm.new_unicode_decode_error_real(
+        let exc = vm.new_unicode_decode_error(
             vm.ctx.new_str(encoding),
             vm.ctx.new_bytes(bytes),
             start,

diff --git a/crates/host_env/src/posix.rs b/crates/host_env/src/posix.rs
@@ -1,5 +1,4 @@
 use alloc::ffi::CString;
-#[cfg(all(unix, not(target_os = "redox")))]
 use alloc::vec::Vec;
 use core::ffi::CStr;
 #[cfg(all(unix, not(target_os = "redox")))]
@@ -22,6 +21,12 @@ pub struct UnameInfo {
     pub machine: String,
 }
 
+#[derive(Debug)]
+pub struct UnameDecodeError {
+    pub bytes: Vec<u8>,
+    pub error: core::str::Utf8Error,
+}
+
 #[cfg(all(unix, not(target_os = "redox")))]
 #[derive(Clone, Copy, Debug)]
 pub struct StatVfsInfo {
@@ -354,14 +359,23 @@ pub fn fchownat(
     .map_err(std::io::Error::from)
 }
 
-pub fn uname_info() -> Result<UnameInfo, core::str::Utf8Error> {
+pub fn uname_info() -> Result<UnameInfo, UnameDecodeError> {
+    fn decode(value: &CStr) -> Result<String, UnameDecodeError> {
+        core::str::from_utf8(value.to_bytes())
+            .map(str::to_owned)
+            .map_err(|error| UnameDecodeError {
+                bytes: value.to_bytes().to_vec(),
+                error,
+            })
+    }
+
     let info = rustix::system::uname();
     Ok(UnameInfo {
-        sysname: info.sysname().to_str()?.into(),
-        nodename: info.nodename().to_str()?.into(),
-        release: info.release().to_str()?.into(),
-        version: info.version().to_str()?.into(),
-        machine: info.machine().to_str()?.into(),
+        sysname: decode(info.sysname())?,
+        nodename: decode(info.nodename())?,
+        release: decode(info.release())?,
+        version: decode(info.version())?,
+        machine: decode(info.machine())?,
     })
 }
 

diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs
@@ -638,7 +638,7 @@ pub mod array {
     impl ToPyResult for WideChar {
         fn to_pyresult(self, vm: &VirtualMachine) -> PyResult {
             Ok(CodePoint::try_from(self)
-                .map_err(|e| vm.new_unicode_encode_error(e))?
+                .map_err(|e| vm.new_value_error(e))?
                 .to_pyobject(vm))
         }
     }
@@ -1696,8 +1696,17 @@ pub mod array {
             })?,
             MachineFormatCode::Utf16 { big_endian } => {
                 let utf16: Vec<_> = chunks.map(|b| chunk_to_obj!(b, u16, big_endian)).collect();
-                let s = String::from_utf16(&utf16)
-                    .map_err(|_| vm.new_unicode_encode_error("items cannot decode as utf16"))?;
+                let s = String::from_utf16(&utf16).map_err(|_| {
+                    let (index, reason) = invalid_utf16(&utf16).unwrap();
+                    vm.new_unicode_decode_error(
+                        vm.ctx
+                            .new_str(if big_endian { "utf-16-be" } else { "utf-16-le" }),
+                        args.items.clone(),
+                        index * 2,
+                        index * 2 + 2,
+                        vm.ctx.new_str(reason),
+                    )
+                })?;
                 let bytes = PyArray::_unicode_to_wchar_bytes((*s).as_ref(), array.itemsize());
                 array.frombytes_move(bytes);
             }
@@ -1713,6 +1722,25 @@ pub mod array {
         PyArray::from(array).into_ref_with_type(vm, cls)
     }
 
+    fn invalid_utf16(units: &[u16]) -> Option<(usize, &'static str)> {
+        let mut index = 0;
+        while index < units.len() {
+            let unit = units[index];
+            if (0xd800..=0xdbff).contains(&unit) {
+                match units.get(index + 1) {
+                    Some(next) if (0xdc00..=0xdfff).contains(next) => index += 2,
+                    Some(_) => return Some((index, "illegal UTF-16 surrogate")),
+                    None => return Some((index, "unexpected end of data")),
+                }
+            } else if (0xdc00..=0xdfff).contains(&unit) {
+                return Some((index, "illegal encoding"));
+            } else {
+                index += 1;
+            }
+        }
+        None
+    }
+
     // Register array.array as collections.abc.MutableSequence
     pub(crate) fn module_exec(
         vm: &VirtualMachine,

diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs
@@ -64,7 +64,7 @@ mod _csv {
         bytes: &[u8],
         err: core::str::Utf8Error,
     ) -> PyBaseExceptionRef {
-        vm.new_unicode_decode_error_real(
+        vm.new_unicode_decode_error(
             vm.ctx.new_str("utf-8"),
             vm.ctx.new_bytes(bytes.to_vec()),
             err.valid_up_to(),

diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs
@@ -2601,7 +2601,7 @@ mod _socket {
             Some(ArgStrOrBytesLike::Buf(b)) => {
                 let bytes = b.borrow_buf();
                 let host_str = core::str::from_utf8(&bytes).map_err(|e| {
-                    vm.new_unicode_decode_error_real(
+                    vm.new_unicode_decode_error(
                         vm.ctx.new_str("utf-8"),
                         vm.ctx.new_bytes(bytes.to_vec()),
                         e.valid_up_to(),
@@ -2643,7 +2643,7 @@ mod _socket {
                         let bytes = b.borrow_buf();
                         core::str::from_utf8(&bytes)
                             .map_err(|e| {
-                                vm.new_unicode_decode_error_real(
+                                vm.new_unicode_decode_error(
                                     vm.ctx.new_str("utf-8"),
                                     vm.ctx.new_bytes(bytes.to_vec()),
                                     e.valid_up_to(),

diff --git a/crates/stdlib/src/tkinter.rs b/crates/stdlib/src/tkinter.rs
@@ -162,9 +162,18 @@ mod _tkinter {
 
         if let Some(tcl_obj) = obj.downcast_ref::<TclObject>() {
             let c_str = unsafe { tk_sys::Tcl_GetString(tcl_obj.value) };
-            let varname = unsafe { ffi::CStr::from_ptr(c_str as _) }
-                .to_str()
-                .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?
+            let bytes = unsafe { ffi::CStr::from_ptr(c_str as _) }.to_bytes();
+            let varname = core::str::from_utf8(bytes)
+                .map_err(|e| {
+                    vm.new_unicode_decode_error(
+                        vm.ctx.new_str("utf-8"),
+                        vm.ctx.new_bytes(bytes.to_vec()),
+                        e.valid_up_to(),
+                        e.error_len()
+                            .map_or(bytes.len(), |len| e.valid_up_to() + len),
+                        vm.ctx.new_str(e.to_string()),
+                    )
+                })?
                 .to_owned();
             return Ok(varname);
         }

diff --git a/crates/vm/src/codecs.rs b/crates/vm/src/codecs.rs
@@ -802,7 +802,7 @@ impl DecodeContext for PyDecodeContext<'_> {
                     } else {
                         vm.ctx.new_bytes(self.data.to_vec())
                     };
-                    vm.new_unicode_decode_error_real(
+                    vm.new_unicode_decode_error(
                         vm.ctx.new_str(self.encoding),
                         data,
                         byte_range.start,

diff --git a/crates/vm/src/function/fspath.rs b/crates/vm/src/function/fspath.rs
@@ -126,7 +126,7 @@ impl FsPath {
 
     pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
         rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| {
-            vm.new_unicode_decode_error_real(
+            vm.new_unicode_decode_error(
                 vm.ctx.new_str("utf-8"),
                 vm.ctx.new_bytes(b.to_vec()),
                 e.valid_up_to(),

diff --git a/crates/vm/src/stdlib/_codecs.rs b/crates/vm/src/stdlib/_codecs.rs
@@ -382,6 +382,23 @@ mod _codecs_windows {
     use crate::{builtins::PyStrRef, builtins::PyUtf8StrRef, function::ArgBytesLike};
     use rustpython_host_env::windows as host_windows;
 
+    fn string_from_utf16(
+        encoding: &str,
+        data: &[u8],
+        wide: &[u16],
+        vm: &VirtualMachine,
+    ) -> PyResult<String> {
+        String::from_utf16(wide).map_err(|err| {
+            vm.new_unicode_decode_error(
+                vm.ctx.new_str(encoding),
+                vm.ctx.new_bytes(data.to_vec()),
+                0,
+                data.len(),
+                vm.ctx.new_str(format!("{encoding}_decode failed: {err}")),
+            )
+        })
+    }
+
     #[derive(FromArgs)]
     struct MbcsEncodeArgs {
         #[pyarg(positional)]
@@ -399,9 +416,7 @@ mod _codecs_windows {
             Some(s) => s,
             None => {
                 // String contains surrogates - not encodable with mbcs
-                return Err(vm.new_unicode_encode_error(
-                    "'mbcs' codec can't encode character: surrogates not allowed",
-                ));
+                return encode_code_page_errors(host_windows::CP_ACP, &args.s, errors, "mbcs", vm);
             }
         };
         let char_len = args.s.char_len();
@@ -433,9 +448,7 @@ mod _codecs_windows {
         .map_err(|err| vm.new_os_error(format!("mbcs_encode failed: {err}")))?;
 
         if errors == "strict" && used_default_char {
-            return Err(vm.new_unicode_encode_error(
-                "'mbcs' codec can't encode characters: invalid character",
-            ));
+            return encode_code_page_errors(host_windows::CP_ACP, &args.s, errors, "mbcs", vm);
         }
 
         buffer.truncate(result);
@@ -484,8 +497,7 @@ mod _codecs_windows {
             )
             .map_err(|err| vm.new_os_error(format!("mbcs_decode failed: {err}")))?;
             buffer.truncate(result);
-            let s = String::from_utf16(&buffer)
-                .map_err(|e| vm.new_unicode_decode_error(format!("mbcs_decode failed: {e}")))?;
+            let s = string_from_utf16("mbcs", data.as_ref(), &buffer, vm)?;
             return Ok((s, len));
         }
 
@@ -500,8 +512,7 @@ mod _codecs_windows {
         )
         .map_err(|err| vm.new_os_error(format!("mbcs_decode failed: {err}")))?;
         buffer.truncate(result);
-        let s = String::from_utf16(&buffer)
-            .map_err(|e| vm.new_unicode_decode_error(format!("mbcs_decode failed: {e}")))?;
+        let s = string_from_utf16("mbcs", data.as_ref(), &buffer, vm)?;
 
         Ok((s, len))
     }
@@ -523,9 +534,7 @@ mod _codecs_windows {
             Some(s) => s,
             None => {
                 // String contains surrogates - not encodable with oem
-                return Err(vm.new_unicode_encode_error(
-                    "'oem' codec can't encode character: surrogates not allowed",
-                ));
+                return encode_code_page_errors(host_windows::CP_OEMCP, &args.s, errors, "oem", vm);
             }
         };
         let char_len = args.s.char_len();
@@ -557,9 +566,7 @@ mod _codecs_windows {
         .map_err(|err| vm.new_os_error(format!("oem_encode failed: {err}")))?;
 
         if errors == "strict" && used_default_char {
-            return Err(vm.new_unicode_encode_error(
-                "'oem' codec can't encode characters: invalid character",
-            ));
+            return encode_code_page_errors(host_windows::CP_OEMCP, &args.s, errors, "oem", vm);
         }
 
         buffer.truncate(result);
@@ -609,8 +616,7 @@ mod _codecs_windows {
             )
             .map_err(|err| vm.new_os_error(format!("oem_decode failed: {err}")))?;
             buffer.truncate(result);
-            let s = String::from_utf16(&buffer)
-                .map_err(|e| vm.new_unicode_decode_error(format!("oem_decode failed: {e}")))?;
+            let s = string_from_utf16("oem", data.as_ref(), &buffer, vm)?;
             return Ok((s, len));
         }
 
@@ -625,8 +631,7 @@ mod _codecs_windows {
         )
         .map_err(|err| vm.new_os_error(format!("oem_decode failed: {err}")))?;
         buffer.truncate(result);
-        let s = String::from_utf16(&buffer)
-            .map_err(|e| vm.new_unicode_decode_error(format!("oem_decode failed: {e}")))?;
+        let s = string_from_utf16("oem", data.as_ref(), &buffer, vm)?;
 
         Ok((s, len))
     }
@@ -1025,7 +1030,7 @@ mod _codecs_windows {
                 }
             }
             let object = vm.ctx.new_bytes(data.to_vec());
-            return Err(vm.new_unicode_decode_error_real(
+            return Err(vm.new_unicode_decode_error(
                 encoding_str,
                 object,
                 fail_pos,
@@ -1116,7 +1121,7 @@ mod _codecs_windows {
                     }
                     "strict" => {
                         let object = vm.ctx.new_bytes(data.to_vec());
-                        return Err(vm.new_unicode_decode_error_real(
+                        return Err(vm.new_unicode_decode_error(
                             encoding_str,
                             object,
                             pos,
@@ -1127,7 +1132,7 @@ mod _codecs_windows {
                     _ => {
                         // Custom error handler
                         let object = vm.ctx.new_bytes(data.to_vec());
-                        let exc = vm.new_unicode_decode_error_real(
+                        let exc = vm.new_unicode_decode_error(
                             encoding_str.clone(),
                             object,
                             pos,