Skip to content

Commit d0baa1c

Browse files
HyoJongParkHyoJongPark
authored
Preserve lone surrogates across WASM conversions (#8508)
Assisted-by: Codex:GPT-5
1 parent 2690c16 commit d0baa1c

3 files changed

Lines changed: 86 additions & 28 deletions

File tree

‎crates/vm/src/py_serde.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,10 @@ impl serde::Serialize for PyObjectSerializer<'_> {
6363
seq.end()
6464
};
6565
if let Some(s) = self.pyobject.downcast_ref::<PyStr>() {
66-
serializer.serialize_str(s.as_ref())
66+
serializer.serialize_str(
67+
s.to_str()
68+
.ok_or_else(|| serde::ser::Error::custom("str contains surrogates"))?,
69+
)
6770
} else if self.pyobject.fast_isinstance(self.vm.ctx.types.float_type) {
6871
serializer.serialize_f64(float::get_value(self.pyobject))
6972
} else if self.pyobject.fast_isinstance(self.vm.ctx.types.bool_type) {

‎crates/wasm/src/convert.rs‎

Lines changed: 73 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,40 @@
22

33
use crate::js_module;
44
use crate::vm_class::{WASMVirtualMachine, stored_vm_from_wasm};
5-
use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array};
5+
use js_sys::{
6+
Array, ArrayBuffer, JsString, Map, Object, Promise, Reflect, SyntaxError, Uint8Array,
7+
};
8+
use rustpython_common::wtf8::{Wtf8, Wtf8Buf};
69
use rustpython_vm::{
710
AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine,
8-
builtins::{PyBaseException, PyBaseExceptionRef},
11+
builtins::{PyBaseException, PyBaseExceptionRef, PyDict, PyList, PyStr, PyTuple},
912
compiler::{CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType},
1013
exceptions,
1114
function::{ArgBytesLike, FuncArgs},
1215
py_serde,
1316
};
1417
use wasm_bindgen::{JsCast, closure::Closure, prelude::*};
1518

19+
pub(crate) fn js_string_to_wtf8(value: &JsString) -> Wtf8Buf {
20+
Wtf8Buf::from_wide(&value.iter().collect::<Vec<_>>())
21+
}
22+
23+
fn wtf8_to_js_string(value: &Wtf8) -> JsString {
24+
const CHUNK_SIZE: usize = 8192;
25+
26+
if let Ok(value) = value.as_str() {
27+
return value.into();
28+
}
29+
30+
value
31+
.encode_wide()
32+
.collect::<Vec<_>>()
33+
.chunks(CHUNK_SIZE)
34+
.map(JsString::from_char_code)
35+
.collect::<Array>()
36+
.join("")
37+
}
38+
1639
#[wasm_bindgen(inline_js = r"
1740
export class PyError extends Error {
1841
constructor(info) {
@@ -119,12 +142,9 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue {
119142
if let Some(ref kwargs) = kwargs {
120143
for pair in object_entries(kwargs) {
121144
let (key, val) = pair?;
122-
py_func_args.kwargs.insert(
123-
// JS strings coming in are UTF-16; go through Rust `String`
124-
// (kwargs keys are now WTF-8, so convert String -> Wtf8Buf).
125-
String::from(js_sys::JsString::from(key)).into(),
126-
js_to_py(vm, val),
127-
);
145+
py_func_args
146+
.kwargs
147+
.insert(js_string_to_wtf8(&key.into()), js_to_py(vm, val));
128148
}
129149
}
130150
let result = py_obj.call(py_func_args, vm);
@@ -151,17 +171,44 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue {
151171
}
152172

153173
if let Ok(bytes) = ArgBytesLike::try_from_borrowed_object(vm, &py_obj) {
154-
bytes.with_ref(|bytes| unsafe {
174+
return bytes.with_ref(|bytes| unsafe {
155175
// `Uint8Array::view` is an `unsafe fn` because it provides
156176
// a direct view into the WASM linear memory; if you were to allocate
157177
// something with Rust that view would probably become invalid. It's safe
158178
// because we then copy the array using `Uint8Array::slice`.
159179
let view = Uint8Array::view(bytes);
160180
view.slice(0, bytes.len() as u32).into()
161-
})
181+
});
182+
}
183+
py_serde_to_js(vm, &py_obj).unwrap_or(JsValue::UNDEFINED)
184+
}
185+
186+
fn py_serde_to_js(
187+
vm: &VirtualMachine,
188+
py_obj: &PyObjectRef,
189+
) -> Result<JsValue, serde_wasm_bindgen::Error> {
190+
if let Some(value) = py_obj.downcast_ref::<PyStr>() {
191+
Ok(wtf8_to_js_string(value.as_wtf8()).into())
192+
} else if let Some(value) = py_obj.downcast_ref::<PyList>() {
193+
let array = Array::new();
194+
for item in value.borrow_vec().iter() {
195+
array.push(&py_serde_to_js(vm, item)?);
196+
}
197+
Ok(array.into())
198+
} else if let Some(value) = py_obj.downcast_ref::<PyTuple>() {
199+
let array = Array::new();
200+
for item in value {
201+
array.push(&py_serde_to_js(vm, item)?);
202+
}
203+
Ok(array.into())
204+
} else if let Some(value) = py_obj.downcast_ref::<PyDict>() {
205+
let map = Map::new();
206+
for (key, value) in value {
207+
map.set(&py_serde_to_js(vm, &key)?, &py_serde_to_js(vm, &value)?);
208+
}
209+
Ok(map.into())
162210
} else {
163-
py_serde::serialize(vm, &py_obj, &serde_wasm_bindgen::Serializer::new())
164-
.unwrap_or(JsValue::UNDEFINED)
211+
py_serde::serialize(vm, py_obj, &serde_wasm_bindgen::Serializer::new())
165212
}
166213
}
167214

@@ -199,6 +246,15 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef {
199246
.map(|val| js_to_py(vm, val.expect("Iteration over array failed")))
200247
.collect();
201248
vm.ctx.new_list(elems).into()
249+
} else if let Some(map) = js_val.dyn_ref::<Map>() {
250+
let dict = vm.ctx.new_dict();
251+
for entry in map.entries() {
252+
let entry = Array::from(&entry.expect("Iteration over map failed"));
253+
let key = js_to_py(vm, entry.get(0));
254+
dict.set_item(&*key, js_to_py(vm, entry.get(1)), vm)
255+
.unwrap();
256+
}
257+
dict.into()
202258
} else if ArrayBuffer::is_view(&js_val) || js_val.is_instance_of::<ArrayBuffer>() {
203259
// unchecked_ref because if it's not an ArrayBuffer it could either be a TypedArray
204260
// or a DataView, but they all have a `buffer` property
@@ -216,12 +272,8 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef {
216272
for pair in object_entries(&Object::from(js_val)) {
217273
let (key, val) = pair.expect("iteration over object to not fail");
218274
let py_val = js_to_py(vm, val);
219-
dict.set_item(
220-
String::from(js_sys::JsString::from(key)).as_str(),
221-
py_val,
222-
vm,
223-
)
224-
.unwrap();
275+
dict.set_item(&*js_string_to_wtf8(&key.into()), py_val, vm)
276+
.unwrap();
225277
}
226278
dict.into()
227279
}
@@ -232,9 +284,7 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef {
232284
move |args: FuncArgs, vm: &VirtualMachine| -> PyResult {
233285
let this = Object::new();
234286
for (k, v) in args.kwargs {
235-
// WTF-8 -> JS string: lone surrogates in the key become U+FFFD
236-
// (wasm-bindgen only accepts Rust `String`); acceptable at this boundary.
237-
Reflect::set(&this, &k.to_string().into(), &py_to_js(vm, v))
287+
Reflect::set(&this, &wtf8_to_js_string(&k).into(), &py_to_js(vm, v))
238288
.expect("property to be settable");
239289
}
240290
let js_args = args
@@ -253,6 +303,8 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef {
253303
} else if js_val.is_undefined() {
254304
// Because `JSON.stringify(undefined)` returns undefined
255305
vm.ctx.none()
306+
} else if js_val.is_string() {
307+
vm.ctx.new_str(js_string_to_wtf8(&js_val.into())).into()
256308
} else {
257309
py_serde::deserialize(vm, serde_wasm_bindgen::Deserializer::from(js_val))
258310
.unwrap_or_else(|_| vm.ctx.none())

‎crates/wasm/src/vm_class.rs‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -325,9 +325,12 @@ impl WASMVirtualMachine {
325325
if let Some(imports) = imports {
326326
for entry in convert::object_entries(&imports) {
327327
let (key, value) = entry?;
328-
let key: String = Object::from(key).to_string().into();
329328
attrs
330-
.set_item(key.as_str(), convert::js_to_py(vm, value), vm)
329+
.set_item(
330+
&*convert::js_string_to_wtf8(&key.into()),
331+
convert::js_to_py(vm, value),
332+
vm,
333+
)
331334
.into_js(vm)?;
332335
}
333336
}
@@ -356,10 +359,10 @@ impl WASMVirtualMachine {
356359
let py_module = vm.new_module(&name, vm.ctx.new_dict(), None);
357360
for entry in convert::object_entries(&module) {
358361
let (key, value) = entry?;
359-
let key = Object::from(key).to_string();
360-
extend_module!(vm, &py_module, {
361-
String::from(key) => convert::js_to_py(vm, value),
362-
});
362+
let key = vm.ctx.new_str(convert::js_string_to_wtf8(&key.into()));
363+
py_module
364+
.set_attr(&key, convert::js_to_py(vm, value), vm)
365+
.into_js(vm)?;
363366
}
364367

365368
let sys_modules = vm.sys_module.get_attr("modules", vm).into_js(vm)?;

0 commit comments

Comments
 (0)