Report invalid \uXXXX escape position at the u character

changjoon-park · changjoon-park · commit 30c0bfc0f593 · 2026-04-25T04:38:50.000+09:00
CPython's json decoder reports the position of the `u` specifier
when a \uXXXX escape fails to parse, but RustPython was reporting
the preceding `\`. For surrogate-pair cases (\uXXXX\uYYYY) the
second call was passing char_offset + next_char_i + 1, which
lands on the first hex digit of the first escape -- unrelated to
the actual failure site.

Pass next_char_i (position of the primary `u`) to the primary
decode_unicode call, and capture the second `u`'s char index from
the next_tuple peek to pass to the surrogate-pair decode_unicode
call.

Verified: 13 targeted probes across invalid-hex, short, and pair
cases now all match CPython positions. test.test_json 214 tests
pass with no regressions.
diff --git a/crates/stdlib/src/json/machinery.rs b/crates/stdlib/src/json/machinery.rs
@@ -231,17 +231,20 @@ pub fn scanstring<'a>(
                     'r' => "\r",
                     't' => "\t",
                     'u' => {
-                        let mut uni = decode_unicode(&mut chars, char_offset + char_i)?;
+                        // Error position for an invalid \uXXXX escape points at the
+                        // `u`, not the `\` -- matches CPython's json.decoder.
+                        let mut uni = decode_unicode(&mut chars, char_offset + next_char_i)?;
                         chunk_start = byte_i + 6;
                         if let Some(lead) = uni.to_lead_surrogate() {
                             // uni is a surrogate -- try to find its pair
                             let mut chars2 = chars.clone();
-                            if let Some(((_, (byte_pos2, _)), (_, _))) = chars2
+                            if let Some(((_, (byte_pos2, _)), (u2_char_i, (_, _)))) = chars2
                                 .next_tuple()
                                 .filter(|((_, (_, c1)), (_, (_, c2)))| *c1 == '\\' && *c2 == 'u')
                             {
-                                let uni2 =
-                                    decode_unicode(&mut chars2, char_offset + next_char_i + 1)?;
+                                // u2_char_i is the `u` of the second \uXXXX; same
+                                // position convention as the primary call above.
+                                let uni2 = decode_unicode(&mut chars2, char_offset + u2_char_i)?;
                                 if let Some(trail) = uni2.to_trail_surrogate() {
                                     // ok, we found what we were looking for -- \uXXXX\uXXXX, both surrogates
                                     uni = lead.merge(trail).into();
