Apply PyUtf8Str

RustPython · youknowone · Jul 30, 2025 · Jul 30, 2025 · Jul 30, 2025 · Jul 30, 2025
commit 9583af057b5789eea2e776b7fba29dd761ad9a18
diff --git a/vm/src/builtins/builtin_func.rs b/vm/src/builtins/builtin_func.rs
@@ -2,6 +2,7 @@ use super::{PyStrInterned, PyStrRef, PyType, type_};
 use crate::{
     AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
     class::PyClassImpl,
+    common::wtf8::Wtf8,
     convert::TryFromObject,
     function::{FuncArgs, PyComparisonValue, PyMethodDef, PyMethodFlags, PyNativeFn},
     types::{Callable, Comparable, PyComparisonOp, Representable, Unconstructible},
@@ -27,7 +28,7 @@ impl fmt::Debug for PyNativeFunction {
         write!(
             f,
             "builtin function {}.{} ({:?}) self as instance of {:?}",
-            self.module.map_or("<unknown>", |m| m.as_str()),
+            self.module.map_or(Wtf8::new("<unknown>"), |m| m.as_wtf8()),
             self.value.name,
             self.value.flags,
             self.zelf.as_ref().map(|z| z.class().name().to_owned())

diff --git a/vm/src/builtins/namespace.rs b/vm/src/builtins/namespace.rs
@@ -89,8 +89,8 @@ impl Representable for PyNamespace {
             let dict = zelf.as_object().dict().unwrap();
             let mut parts = Vec::with_capacity(dict.__len__());
             for (key, value) in dict {
-                let k = &key.repr(vm)?;
-                let key_str = k.as_str();
+                let k = key.repr(vm)?;
+                let key_str = k.as_wtf8();
                 let value_repr = value.repr(vm)?;
                 parts.push(format!("{}={}", &key_str[1..key_str.len() - 1], value_repr));
             }

diff --git a/vm/src/builtins/slice.rs b/vm/src/builtins/slice.rs
@@ -292,15 +292,10 @@ impl Representable for PySlice {
     #[inline]
     fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> {
         let start_repr = zelf.start_ref(vm).repr(vm)?;
-        let stop_repr = &zelf.stop.repr(vm)?;
+        let stop_repr = zelf.stop.repr(vm)?;
         let step_repr = zelf.step_ref(vm).repr(vm)?;
 
-        Ok(format!(
-            "slice({}, {}, {})",
-            start_repr.as_str(),
-            stop_repr.as_str(),
-            step_repr.as_str()
-        ))
+        Ok(format!("slice({start_repr}, {stop_repr}, {step_repr})"))
     }
 }
 

diff --git a/vm/src/builtins/super.rs b/vm/src/builtins/super.rs
@@ -104,7 +104,7 @@ impl Initializer for PySuper {
 
             let mut typ = None;
             for (i, var) in frame.code.freevars.iter().enumerate() {
-                if var.as_str() == "__class__" {
+                if var.as_bytes() == b"__class__" {
                     let i = frame.code.cellvars.len() + i;
                     let class = frame.cells_frees[i]
                         .get()
@@ -162,7 +162,7 @@ impl GetAttr for PySuper {
         // We want __class__ to return the class of the super object
         // (i.e. super, or a subclass), not the class of su->obj.
 
-        if name.as_str() == "__class__" {
+        if name.as_bytes() == b"__class__" {
             return skip(zelf, name);
         }
 

diff --git a/vm/src/builtins/type.rs b/vm/src/builtins/type.rs
@@ -340,13 +340,13 @@ impl PyType {
                 if name == identifier!(ctx, __new__) {
                     continue;
                 }
-                if name.as_str().starts_with("__") && name.as_str().ends_with("__") {
+                if name.as_bytes().starts_with(b"__") && name.as_bytes().ends_with(b"__") {
                     slot_name_set.insert(name);
                 }
             }
         }
         for &name in self.attributes.read().keys() {
-            if name.as_str().starts_with("__") && name.as_str().ends_with("__") {
+            if name.as_bytes().starts_with(b"__") && name.as_bytes().ends_with(b"__") {
                 slot_name_set.insert(name);
             }
         }

diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs
@@ -198,15 +198,11 @@ impl VirtualMachine {
         let mut filename_suffix = String::new();
 
         if let Some(lineno) = maybe_lineno {
-            writeln!(
-                output,
-                r##"  File "{}", line {}"##,
-                maybe_filename
-                    .as_ref()
-                    .map(|s| s.as_str())
-                    .unwrap_or("<string>"),
-                lineno
-            )?;
+            let filename = match maybe_filename {
+                Some(filename) => filename,
+                None => vm.ctx.new_str("<string>"),
+            };
+            writeln!(output, r##"  File "{filename}", line {lineno}"##,)?;
         } else if let Some(filename) = maybe_filename {
             filename_suffix = format!(" ({filename})");
         }
@@ -1498,7 +1494,7 @@ pub(super) mod types {
             let args = exc.args();
             let obj = exc.as_object().to_owned();
 
-            if args.len() == 2 {
+            let str = if args.len() == 2 {
                 // SAFETY: len() == 2 is checked so get_arg 1 or 2 won't panic
                 let errno = exc.get_arg(0).unwrap().str(vm)?;
                 let msg = exc.get_arg(1).unwrap().str(vm)?;
@@ -1518,10 +1514,11 @@ pub(super) mod types {
                         format!("[Errno {errno}] {msg}")
                     }
                 };
-                Ok(vm.ctx.new_str(s))
+                vm.ctx.new_str(s)
             } else {
-                Ok(exc.__str__(vm))
-            }
+                exc.__str__(vm)
+            };
+            Ok(str)
         }
 
         #[pymethod]

diff --git a/vm/src/format.rs b/vm/src/format.rs
@@ -112,7 +112,7 @@ fn format_internal(
 
                 // FIXME: compiler can intern specs using parser tree. Then this call can be interned_str
                 pystr = vm.format(&argument, vm.ctx.new_str(format_spec))?;
-                pystr.as_ref()
+                pystr.as_wtf8()
             }
             FormatPart::Literal(literal) => literal,
         };

diff --git a/vm/src/protocol/object.rs b/vm/src/protocol/object.rs
@@ -2,10 +2,10 @@
 //! <https://docs.python.org/3/c-api/object.html>
 
 use crate::{
-    AsObject, Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine,
+    AsObject, Py, PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine,
     builtins::{
-        PyAsyncGen, PyBytes, PyDict, PyDictRef, PyGenericAlias, PyInt, PyList, PyStr, PyStrRef,
-        PyTuple, PyTupleRef, PyType, PyTypeRef, pystr::AsPyStr,
+        PyAsyncGen, PyBytes, PyDict, PyDictRef, PyGenericAlias, PyInt, PyList, PyStr, PyTuple,
+        PyTupleRef, PyType, PyTypeRef, PyUtf8Str, pystr::AsPyStr,
     },
     bytes_inner::ByteInnerNewOptions,
     common::{hash::PyHash, str::to_ascii},
@@ -328,7 +328,11 @@ impl PyObject {
         }
     }
 
-    pub fn repr(&self, vm: &VirtualMachine) -> PyResult<PyStrRef> {
+    pub fn repr_utf8(&self, vm: &VirtualMachine) -> PyResult<PyRef<PyUtf8Str>> {
+        self.repr(vm)?.try_into_utf8(vm)
+    }
+
+    pub fn repr(&self, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> {
         vm.with_recursion("while getting the repr of an object", || {
             // TODO: RustPython does not implement type slots inheritance yet
             self.class()
@@ -346,13 +350,15 @@ impl PyObject {
     }
 
     pub fn ascii(&self, vm: &VirtualMachine) -> PyResult<ascii::AsciiString> {
-        let repr = self.repr(vm)?;
+        let repr = self.repr_utf8(vm)?;
         let ascii = to_ascii(repr.as_str());
         Ok(ascii)
     }
 
-    // Container of the virtual machine state:
-    pub fn str(&self, vm: &VirtualMachine) -> PyResult<PyStrRef> {
+    pub fn str_utf8(&self, vm: &VirtualMachine) -> PyResult<PyRef<PyUtf8Str>> {
+        self.str(vm)?.try_into_utf8(vm)
+    }
+    pub fn str(&self, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> {
         let obj = match self.to_owned().downcast_exact::<PyStr>(vm) {
             Ok(s) => return Ok(s.into_pyref()),
             Err(obj) => obj,

diff --git a/vm/src/stdlib/ctypes/structure.rs b/vm/src/stdlib/ctypes/structure.rs
@@ -39,7 +39,7 @@ impl Constructor for PyCStructure {
                 .downcast_ref::<PyStr>()
                 .ok_or_else(|| vm.new_type_error("Field name must be a string"))?;
             let typ = field.get(1).unwrap().clone();
-            field_data.insert(name.as_str().to_string(), typ);
+            field_data.insert(name.to_string(), typ);
         }
         todo!("Implement PyCStructure::py_new")
     }

diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs
@@ -1579,7 +1579,7 @@ mod _io {
         }
 
         #[pyslot]
-        fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> {
+        fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> {
             let name_repr = repr_file_obj_name(zelf, vm)?;
             let cls = zelf.class();
             let slot_name = cls.slot_name();
@@ -1592,7 +1592,7 @@ mod _io {
         }
 
         #[pymethod]
-        fn __repr__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> {
+        fn __repr__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> {
             Self::slot_repr(&zelf, vm)
         }
 

diff --git a/vm/src/stdlib/operator.rs b/vm/src/stdlib/operator.rs
@@ -5,8 +5,8 @@ mod _operator {
     use crate::{
         AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
         builtins::{PyInt, PyIntRef, PyStr, PyStrRef, PyTupleRef, PyTypeRef},
-        function::Either,
-        function::{ArgBytesLike, FuncArgs, KwArgs, OptionalArg},
+        common::wtf8::Wtf8,
+        function::{ArgBytesLike, Either, FuncArgs, KwArgs, OptionalArg},
         identifier,
         protocol::PyIter,
         recursion::ReprGuard,
@@ -324,7 +324,7 @@ mod _operator {
     ) -> PyResult<bool> {
         let res = match (a, b) {
             (Either::A(a), Either::A(b)) => {
-                if !a.as_str().is_ascii() || !b.as_str().is_ascii() {
+                if !a.is_ascii() || !b.is_ascii() {
                     return Err(vm.new_type_error(
                         "comparing strings with non-ASCII characters is not supported",
                     ));
@@ -371,13 +371,14 @@ mod _operator {
             attr: &Py<PyStr>,
             vm: &VirtualMachine,
         ) -> PyResult<PyObjectRef> {
-            let attr_str = attr.as_str();
-            let parts = attr_str.split('.').collect::<Vec<_>>();
+            let attr_str = attr.as_bytes();
+            let parts = attr_str.split(|&b| b == b'.').collect::<Vec<_>>();
             if parts.len() == 1 {
                 return obj.get_attr(attr, vm);
             }
             let mut obj = obj;
             for part in parts {
+                let part = Wtf8::from_bytes(part).expect("originally valid WTF-8");
                 obj = obj.get_attr(&vm.ctx.new_str(part), vm)?;
             }
             Ok(obj)

diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs
@@ -24,7 +24,7 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
 pub mod module {
     use crate::{
         AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine,
-        builtins::{PyDictRef, PyInt, PyListRef, PyStrRef, PyTupleRef, PyTypeRef},
+        builtins::{PyDictRef, PyInt, PyListRef, PyStrRef, PyTupleRef, PyTypeRef, PyUtf8StrRef},
         convert::{IntoPyException, ToPyObject, TryFromObject},
         function::{Either, KwArgs, OptionalArg},
         ospath::{IOErrorBuilder, OsPath, OsPathOrFd},
@@ -2242,7 +2242,7 @@ pub mod module {
             let i = match obj.downcast::<PyInt>() {
                 Ok(int) => int.try_to_primitive(vm)?,
                 Err(obj) => {
-                    let s = PyStrRef::try_from_object(vm, obj)?;
+                    let s = PyUtf8StrRef::try_from_object(vm, obj)?;
                     s.as_str().parse::<SysconfVar>().or_else(|_| {
                         if s.as_str() == "SC_PAGESIZE" {
                             Ok(SysconfVar::SC_PAGESIZE)

diff --git a/vm/src/stdlib/pwd.rs b/vm/src/stdlib/pwd.rs
@@ -6,7 +6,7 @@ pub(crate) use pwd::make_module;
 mod pwd {
     use crate::{
         PyObjectRef, PyResult, VirtualMachine,
-        builtins::{PyIntRef, PyStrRef},
+        builtins::{PyIntRef, PyUtf8StrRef},
         convert::{IntoPyException, ToPyObject},
         exceptions,
         types::PyStructSequence,
@@ -54,7 +54,7 @@ mod pwd {
     }
 
     #[pyfunction]
-    fn getpwnam(name: PyStrRef, vm: &VirtualMachine) -> PyResult<Passwd> {
+    fn getpwnam(name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<Passwd> {
         let pw_name = name.as_str();
         if pw_name.contains('\0') {
             return Err(exceptions::cstring_error(vm));

diff --git a/vm/src/stdlib/sys.rs b/vm/src/stdlib/sys.rs
@@ -712,7 +712,7 @@ mod sys {
         if !vm.is_none(&unraisable.exc_value) {
             write!(stderr, "{}: ", unraisable.exc_type);
             if let Ok(str) = unraisable.exc_value.str(vm) {
-                write!(stderr, "{}", str.as_str());
+                write!(stderr, "{}", str.to_str().unwrap_or("<str with surrogate>"));
             } else {
                 write!(stderr, "<exception str() failed>");
             }
@@ -734,7 +734,6 @@ mod sys {
                 e.as_object()
                     .repr(vm)
                     .unwrap_or_else(|_| vm.ctx.empty_str.to_owned())
-                    .as_str()
             );
         }
     }

diff --git a/vm/src/types/slot.rs b/vm/src/types/slot.rs
@@ -169,7 +169,7 @@ impl Default for PyTypeFlags {
 pub(crate) type GenericMethod = fn(&PyObject, FuncArgs, &VirtualMachine) -> PyResult;
 pub(crate) type HashFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyHash>;
 // CallFunc = GenericMethod
-pub(crate) type StringifyFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyStrRef>;
+pub(crate) type StringifyFunc = fn(&PyObject, &VirtualMachine) -> PyResult<PyRef<PyStr>>;
 pub(crate) type GetattroFunc = fn(&PyObject, &Py<PyStr>, &VirtualMachine) -> PyResult;
 pub(crate) type SetattroFunc =
     fn(&PyObject, &Py<PyStr>, PySetterValue, &VirtualMachine) -> PyResult<()>;
@@ -250,7 +250,7 @@ fn setitem_wrapper<K: ToPyObject>(
     .map(drop)
 }
 
-fn repr_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> {
+fn repr_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> {
     let ret = vm.call_special_method(zelf, identifier!(vm, __repr__), ())?;
     ret.downcast::<PyStr>().map_err(|obj| {
         vm.new_type_error(format!(
@@ -977,7 +977,7 @@ pub trait Hashable: PyPayload {
 pub trait Representable: PyPayload {
     #[inline]
     #[pyslot]
-    fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyStrRef> {
+    fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> {
         let zelf = zelf
             .downcast_ref()
             .ok_or_else(|| vm.new_type_error("unexpected payload for __repr__"))?;
@@ -986,12 +986,12 @@ pub trait Representable: PyPayload {
 
     #[inline]
     #[pymethod]
-    fn __repr__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyStrRef> {
+    fn __repr__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> {
         Self::slot_repr(&zelf, vm)
     }
 
     #[inline]
-    fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyStrRef> {
+    fn repr(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyRef<PyStr>> {
         let repr = Self::repr_str(zelf, vm)?;
         Ok(vm.ctx.new_str(repr))
     }

diff --git a/vm/src/utils.rs b/vm/src/utils.rs
@@ -2,7 +2,7 @@ use rustpython_common::wtf8::Wtf8;
 
 use crate::{
     PyObjectRef, PyResult, VirtualMachine,
-    builtins::PyStr,
+    builtins::{PyStr, PyUtf8Str},
     convert::{ToPyException, ToPyObject},
     exceptions::cstring_error,
 };
@@ -35,6 +35,7 @@ pub trait ToCString: AsRef<Wtf8> {
 
 impl ToCString for &str {}
 impl ToCString for PyStr {}
+impl ToCString for PyUtf8Str {}
 
 pub(crate) fn collection_repr<'a, I>(
     class_name: Option<&str>,

diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs
@@ -472,7 +472,7 @@ impl VirtualMachine {
             object,
         };
         if let Err(e) = unraisablehook.call((args,), self) {
-            println!("{}", e.as_object().repr(self).unwrap().as_str());
+            println!("{}", e.as_object().repr(self).unwrap());
         }
     }
 

diff --git a/vm/src/vm/vm_ops.rs b/vm/src/vm/vm_ops.rs
@@ -1,7 +1,8 @@
 use super::VirtualMachine;
 use crate::stdlib::warnings;
 use crate::{
-    builtins::{PyInt, PyIntRef, PyStr, PyStrRef},
+    PyRef,
+    builtins::{PyInt, PyIntRef, PyStr, PyStrRef, PyUtf8Str},
     object::{AsObject, PyObject, PyObjectRef, PyResult},
     protocol::{PyIterReturn, PyNumberBinaryOp, PyNumberTernaryOp, PySequence},
     types::PyComparisonOp,
@@ -517,6 +518,9 @@ impl VirtualMachine {
             ))
         })
     }
+    pub fn format_utf8(&self, obj: &PyObject, format_spec: PyStrRef) -> PyResult<PyRef<PyUtf8Str>> {
+        self.format(obj, format_spec)?.try_into_utf8(self)
+    }
 
     // https://docs.python.org/3/reference/expressions.html#membership-test-operations
     fn _membership_iter_search(
-Original file line number
+Diff line change
@@ Expand Up / @@ -472,7 +472,7 @@ impl VirtualMachine { @@
                 object,
             };
             if let Err(e) = unraisablehook.call((args,), self) {
-                println!("{}", e.as_object().repr(self).unwrap().as_str());
+                println!("{}", e.as_object().repr(self).unwrap());
             }
         }
@@ Expand Down @@