From 30c0bfc0f5939e203175008ee05935e4d8658eba Mon Sep 17 00:00:00 2001 From: changjoon-park Date: Sat, 25 Apr 2026 04:38:50 +0900 Subject: [PATCH 1/3] Report invalid \uXXXX escape position at the u character CPython's json decoder reports the position of the `u` specifier when a \uXXXX escape fails to parse, but RustPython was reporting the preceding `\`. For surrogate-pair cases (\uXXXX\uYYYY) the second call was passing char_offset + next_char_i + 1, which lands on the first hex digit of the first escape -- unrelated to the actual failure site. Pass next_char_i (position of the primary `u`) to the primary decode_unicode call, and capture the second `u`'s char index from the next_tuple peek to pass to the surrogate-pair decode_unicode call. Verified: 13 targeted probes across invalid-hex, short, and pair cases now all match CPython positions. test.test_json 214 tests pass with no regressions. --- crates/stdlib/src/json/machinery.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/stdlib/src/json/machinery.rs b/crates/stdlib/src/json/machinery.rs index 3c8b359c22a..6106acc8406 100644 --- a/crates/stdlib/src/json/machinery.rs +++ b/crates/stdlib/src/json/machinery.rs @@ -231,17 +231,20 @@ pub fn scanstring<'a>( 'r' => "\r", 't' => "\t", 'u' => { - let mut uni = decode_unicode(&mut chars, char_offset + char_i)?; + // Error position for an invalid \uXXXX escape points at the + // `u`, not the `\` -- matches CPython's json.decoder. + let mut uni = decode_unicode(&mut chars, char_offset + next_char_i)?; chunk_start = byte_i + 6; if let Some(lead) = uni.to_lead_surrogate() { // uni is a surrogate -- try to find its pair let mut chars2 = chars.clone(); - if let Some(((_, (byte_pos2, _)), (_, _))) = chars2 + if let Some(((_, (byte_pos2, _)), (u2_char_i, (_, _)))) = chars2 .next_tuple() .filter(|((_, (_, c1)), (_, (_, c2)))| *c1 == '\\' && *c2 == 'u') { - let uni2 = - decode_unicode(&mut chars2, char_offset + next_char_i + 1)?; + // u2_char_i is the `u` of the second \uXXXX; same + // position convention as the primary call above. + let uni2 = decode_unicode(&mut chars2, char_offset + u2_char_i)?; if let Some(trail) = uni2.to_trail_surrogate() { // ok, we found what we were looking for -- \uXXXX\uXXXX, both surrogates uni = lead.merge(trail).into(); From cf9c6b685b6e7efcab26044b122a7be0d98db477 Mon Sep 17 00:00:00 2001 From: changjoon-park Date: Sun, 26 Apr 2026 00:17:21 +0900 Subject: [PATCH 2/3] Add regression test for invalid \uXXXX escape position --- extra_tests/snippets/stdlib_json.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/extra_tests/snippets/stdlib_json.py b/extra_tests/snippets/stdlib_json.py index 53b3230a7ad..ece0331b518 100644 --- a/extra_tests/snippets/stdlib_json.py +++ b/extra_tests/snippets/stdlib_json.py @@ -239,3 +239,19 @@ class Dict(dict): RecursionError, lambda: json.loads(('[{"x":' * _deep) + "1" + ("}]" * _deep)), ) + + +# Invalid \uXXXX escape: error position points at the 'u', matching CPython. +try: + json.loads('"\\uXYZW"') +except json.JSONDecodeError as e: + assert e.pos == 2, f"expected pos=2, got {e.pos}" +else: + assert False, "expected JSONDecodeError" + +try: + json.loads('"abc\\uZZZZ"') +except json.JSONDecodeError as e: + assert e.pos == 5, f"expected pos=5, got {e.pos}" +else: + assert False, "expected JSONDecodeError" From 3ce5dda5809198212f0c209c304ee92869769853 Mon Sep 17 00:00:00 2001 From: changjoon-park Date: Sun, 26 Apr 2026 00:27:41 +0900 Subject: [PATCH 3/3] Use raise AssertionError instead of assert False (B011) --- extra_tests/snippets/stdlib_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extra_tests/snippets/stdlib_json.py b/extra_tests/snippets/stdlib_json.py index ece0331b518..9d6b47b9ac7 100644 --- a/extra_tests/snippets/stdlib_json.py +++ b/extra_tests/snippets/stdlib_json.py @@ -247,11 +247,11 @@ class Dict(dict): except json.JSONDecodeError as e: assert e.pos == 2, f"expected pos=2, got {e.pos}" else: - assert False, "expected JSONDecodeError" + raise AssertionError("expected JSONDecodeError") try: json.loads('"abc\\uZZZZ"') except json.JSONDecodeError as e: assert e.pos == 5, f"expected pos=5, got {e.pos}" else: - assert False, "expected JSONDecodeError" + raise AssertionError("expected JSONDecodeError")