Skip to content

Implement Unicode 17 grapheme iteration and properties #9145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 9 commits into from
Oct 16, 2026
Merged

Implement Unicode 17 grapheme iteration and properties #9145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
2b0ed2b
Implement native Unicode 17 grapheme iteration
youknowdot Oct 15, 2026
695e0d4
Bind grapheme Segment methods to the Python object
youknowdot Oct 15, 2026
c16f5e1
Use explicit object receivers for Segment descriptors
youknowdot Oct 15, 2026
1f0116d
Mark grapheme property lookups as must-use
youknowdot Oct 15, 2026
81470d0
Enable validated Unicode grapheme canonical tests
youknowdot Oct 15, 2026
85647ab
Accept platform line endings in grapheme oracle header
youknowdot Oct 15, 2026
08cc0b6
Keep grapheme property regression in test module
youknowdot Oct 15, 2026
52a10cf
Qualify grapheme iterator return type for generated bindings
youknowdot Oct 16, 2026
07e76cd
Fix deprecated PyO3 code input import in benchmarks
youknowdot Oct 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 9 additions & 16 deletions Lib/test/test_unicodedata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ def test_method_checksum(self):

class BaseUnicodeFunctionsTest:

@unittest.skip # TODO: RUSTPYTHON; AssertionError: '7bda75e48a961a01ab328358980cefc5c0a1666d' != '68cd01e2c680b851c1fcab012efb5635'
@unittest.skip("TODO: RUSTPYTHON; AssertionError: '7bda75e48a961a01ab328358980cefc5c0a1666d' != '68cd01e2c680b851c1fcab012efb5635'")
def test_function_checksum(self):
db = self.db
data = []
Expand Down Expand Up @@ -199,7 +199,7 @@ def test_no_names_in_pua(self):
char = chr(i)
self.assertRaises(ValueError, self.db.name, char)

@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: KeyError not raised by lookup
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: KeyError not raised by lookup
def test_lookup_nonexistant(self):
# just make sure that lookup can fail
for nonexistent in [
Expand Down Expand Up @@ -360,7 +360,7 @@ def test_bidirectional_unassigned(self):
self.assertEqual(self.db.bidirectional('\U0001FFFE'), '' if self.old else 'BN')
self.assertEqual(self.db.bidirectional('\U00010D01'), '' if self.old else 'AL')

@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '<compat> 03A3 != ''
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '<compat> 03A3 != ''
def test_decomposition(self):
self.assertEqual(self.db.decomposition('\uFFFE'),'')
self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034')
Expand Down Expand Up @@ -700,7 +700,7 @@ def test_east_asian_width(self):
# New in 17.0.0
self.assertEqual(eaw('\U00016FF2'), 'N' if self.old else 'W')

@unittest.skip # TODO: RUSTPYTHON; AssertionError: 'N' != 'W' (passed on latest, fails on 3.2)
@unittest.skip("TODO: RUSTPYTHON; AssertionError: 'N' != 'W' (passed on latest, fails on 3.2)")
def test_east_asian_width_unassigned(self):
eaw = self.db.east_asian_width
# unassigned
Expand Down Expand Up @@ -769,7 +769,7 @@ def parse_cp(s):
for cp in range(0, sys.maxunicode + 1):
self.assertEqual(self.db.name(chr(cp), None), names_ref.get(cp))

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'isxidstart'
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'isxidstart'
def test_isxidstart(self):
self.assertTrue(self.db.isxidstart('S'))
self.assertTrue(self.db.isxidstart('\u0AD0')) # GUJARATI OM
Expand All @@ -783,7 +783,7 @@ def test_isxidstart(self):
self.assertRaises(TypeError, self.db.isxidstart)
self.assertRaises(TypeError, self.db.isxidstart, 'xx')

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'isxidcontinue'
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'isxidcontinue'
def test_isxidcontinue(self):
self.assertTrue(self.db.isxidcontinue('S'))
self.assertTrue(self.db.isxidcontinue('_'))
Expand All @@ -798,7 +798,6 @@ def test_isxidcontinue(self):
self.assertRaises(TypeError, self.db.isxidcontinue)
self.assertRaises(TypeError, self.db.isxidcontinue, 'xx')

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'grapheme_cluster_break'
def test_grapheme_cluster_break(self):
gcb = self.db.grapheme_cluster_break
self.assertEqual(gcb(' '), 'Other')
Expand Down Expand Up @@ -896,7 +895,6 @@ def test_grapheme_cluster_break(self):
self.assertRaises(TypeError, gcb, '')
self.assertRaises(TypeError, gcb, 'xx')

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'indic_conjunct_break'
def test_indic_conjunct_break(self):
incb = self.db.indic_conjunct_break
self.assertEqual(incb(' '), 'None')
Expand Down Expand Up @@ -925,7 +923,6 @@ def test_indic_conjunct_break(self):
self.assertRaises(TypeError, incb, '')
self.assertRaises(TypeError, incb, 'xx')

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'extended_pictographic'
def test_extended_pictographic(self):
ext_pict = self.db.extended_pictographic
self.assertIs(ext_pict(' '), False)
Expand All @@ -946,7 +943,6 @@ def test_extended_pictographic(self):
self.assertRaises(TypeError, ext_pict, '')
self.assertRaises(TypeError, ext_pict, 'xx')

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'iter_graphemes'
def test_grapheme_break(self):
def graphemes(*args):
return list(map(str, self.db.iter_graphemes(*args)))
Expand Down Expand Up @@ -1017,7 +1013,7 @@ def graphemes(*args):
'a\U0001F1FA\U0001F1E6\U0001F1FA\U0001F1F3'),
['a', '\U0001F1FA\U0001F1E6', '\U0001F1FA\U0001F1F3'])

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'block'
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'block'
def test_block(self):
self.assertEqual(self.db.block('\u0000'), 'Basic Latin')
self.assertEqual(self.db.block('\u0041'), 'Basic Latin')
Expand Down Expand Up @@ -1102,7 +1098,7 @@ def test_block(self):
self.assertEqual(self.db.block('\U00100000'), 'Supplementary Private Use Area-B')
self.assertEqual(self.db.block('\U0010FFFF'), 'Supplementary Private Use Area-B')

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'block'
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'block'
def test_block_invalid_input(self):
self.assertRaises(TypeError, self.db.block)
self.assertRaises(TypeError, self.db.block, b'x')
Expand Down Expand Up @@ -1169,7 +1165,7 @@ def test_failed_import_during_compiling(self):
"(can't load unicodedata module)"
self.assertIn(error, result.err.decode("ascii"))

@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError Process return code is 1
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError Process return code is 1
def test_unicodedata_unload_reload(self):
# gh-149449: dropping unicodedata and running gc must not leave the
# cached _ucnhash_CAPI pointer dangling.
Expand Down Expand Up @@ -1262,7 +1258,6 @@ def test_linebreak_7643(self):
self.assertEqual(len(lines), 1,
r"%a should not be a linebreak" % c)

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'iter_graphemes'
def test_segment_object(self):
segments = list(unicodedata.iter_graphemes('spa\u0300m'))
self.assertEqual(len(segments), 4, segments)
Expand Down Expand Up @@ -1384,7 +1379,6 @@ class MyStr(str):

class GraphemeBreakTest(unittest.TestCase):
@requires_resource('network')
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError module 'unicodedata' has no attribute 'iter_graphemes'
def test_grapheme_break(self):
TESTDATAFILE = "GraphemeBreakTest.txt"
testdata = download_test_data_file(TESTDATAFILE)
Expand Down Expand Up @@ -1423,7 +1417,6 @@ def run_grapheme_break_tests(self, testdata):
self.assertEqual([x.start for x in result], breaks[i:-1], comment)
self.assertEqual([x.end for x in result], breaks[i+1:], comment)

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'unicodedata' has no attribute 'iter_graphemes'
def test_reference_loops(self):
# Test that reference loops involving GraphemeBreakIterator or
# Segment can be broken by the garbage collector.
Expand Down
2 changes: 1 addition & 1 deletion benches/execution.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use criterion::{
Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main,
measurement::WallTime,
};
use pyo3::types::{PyCode, PyCodeInput, PyCodeMethods, PyDict, PyDictMethods};
use pyo3::types::{PyCode, PyCodeMethods, PyDict, PyDictMethods, code::PyCodeInput};
use rustpython_compiler::Mode;
use rustpython_vm::{Interpreter, PyResult, Settings};
use std::{collections::HashMap, path::Path};
Expand Down
177 changes: 177 additions & 0 deletions crates/stdlib/src/unicodedata.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,187 @@ mod unicodedata {
use crate::vm::{
Py, PyObjectRef, PyPayload, PyResult, VirtualMachine,
builtins::{PyModule, PyStr, PyStrRef},
common::lock::PyMutex,
function::OptionalArg,
protocol::PyIterReturn,
types::{IterNext, Iterable, Representable, SelfIter},
};
use itertools::Itertools;
use rustpython_common::wtf8::{CodePoint, Wtf8Buf};
use unicode_core::grapheme::{self, GraphemeBreakState};

fn grapheme_char(character: &Py<PyStr>, vm: &VirtualMachine) -> PyResult<CodePoint> {
character
.as_wtf8()
.code_points()
.exactly_one()
.map_err(|_| vm.new_type_error("argument must be a unicode character, not str"))
}

#[pyfunction]
fn grapheme_cluster_break(chr: PyStrRef, vm: &VirtualMachine) -> PyResult<&'static str> {
Comment on lines +55 to +56

@chatgpt-codex-connector chatgpt-codex-connector Bot Oct 16, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Expose the grapheme APIs on UCD instances

With the default ucd_3_2_0 feature enabled, these APIs are registered only as module-level #[pyfunction]s; no corresponding methods are added to Ucd. Unicode_3_2_0_FunctionsTest sets db = unicodedata.ucd_3_2_0 and inherits the four newly unmarked tests from BaseUnicodeFunctionsTest, so those tests now raise AttributeError when they access self.db.grapheme_cluster_break, the other properties, or iter_graphemes, making the standard test_unicodedata suite fail and leaving the CPython UCD API incomplete. Add the methods to Ucd or retain expected-failure markers until they work.

AGENTS.md reference: AGENTS.md:L279-L282

Useful? React with 👍 / 👎.

grapheme_char(&chr, vm).map(grapheme::grapheme_cluster_break)
}

#[pyfunction]
fn indic_conjunct_break(chr: PyStrRef, vm: &VirtualMachine) -> PyResult<&'static str> {
grapheme_char(&chr, vm).map(grapheme::indic_conjunct_break)
}

#[pyfunction]
fn extended_pictographic(chr: PyStrRef, vm: &VirtualMachine) -> PyResult<bool> {
grapheme_char(&chr, vm).map(grapheme::extended_pictographic)
}

#[pyclass(no_attr, name = "Segment", traverse)]
#[derive(Debug, PyPayload)]
struct Segment {
string: PyStrRef,
#[pytraverse(skip)]
start: usize,
#[pytraverse(skip)]
end: usize,
#[pytraverse(skip)]
bytes: core::ops::Range<usize>,
}

#[pyclass(with(Representable), flags(IMMUTABLETYPE, DISALLOW_INSTANTIATION))]
impl Segment {
#[pygetset]
fn start(zelf: &Py<Self>) -> usize {
zelf.start
}

#[pygetset]
fn end(zelf: &Py<Self>) -> usize {
zelf.end
}

#[pymethod]
fn __str__(zelf: &Py<Self>, vm: &VirtualMachine) -> PyStrRef {
vm.ctx.new_str(&zelf.string.as_wtf8()[zelf.bytes.clone()])
}
}

impl Representable for Segment {
#[inline]
fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> {
Ok(format!("<Segment {}:{}>", zelf.start, zelf.end))
}
}

#[derive(Debug)]
struct GraphemePosition {
start: usize,
start_byte: usize,
pos: usize,
pos_byte: usize,
end: usize,
end_byte: usize,
state: GraphemeBreakState,
}

#[pyclass(no_attr, name = "GraphemeBreakIterator", traverse)]
#[derive(Debug, PyPayload)]
struct GraphemeBreakIterator {
string: PyStrRef,
#[pytraverse(skip)]
position: PyMutex<GraphemePosition>,
}

#[pyclass(with(IterNext, Iterable), flags(IMMUTABLETYPE, DISALLOW_INSTANTIATION))]
impl GraphemeBreakIterator {}

impl SelfIter for GraphemeBreakIterator {}

impl IterNext for GraphemeBreakIterator {
fn next(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> {
let mut position = zelf.position.lock();
if position.start >= position.end {
return Ok(PyIterReturn::StopIteration(None));
}
let start = position.start;
let start_byte = position.start_byte;
let scan_byte = position.pos_byte;
let mut points = zelf.string.as_wtf8()[scan_byte..position.end_byte]
.code_point_indices()
.peekable();
let (end, end_byte) = loop {
let Some((offset, point)) = points.next() else {
position.start = position.pos;
position.start_byte = position.end_byte;
break (position.pos, position.end_byte);
};
let byte = scan_byte + offset;
let index = position.pos;
let boundary = position.state.advance(point);
position.pos += 1;
position.pos_byte = points
.peek()
.map_or(position.end_byte, |(offset, _)| scan_byte + offset);
if boundary {
position.start = index;
position.start_byte = byte;
break (index, byte);
}
};
drop(position);
Ok(PyIterReturn::Return(
Segment {
string: zelf.string.clone(),
start,
end,
bytes: start_byte..end_byte,
}
.into_ref(&vm.ctx)
.into(),
))
}
}

#[pyfunction]
fn iter_graphemes(
unistr: PyStrRef,
start: OptionalArg<PyObjectRef>,
end: OptionalArg<PyObjectRef>,
vm: &VirtualMachine,
) -> PyResult<crate::vm::PyRef<GraphemeBreakIterator>> {
// Convert in argument order, including overflow, before adjusting bounds.
let index = |arg: OptionalArg<PyObjectRef>, default| -> PyResult<isize> {
match arg {
OptionalArg::Present(value) => value.try_index(vm)?.try_to_primitive(vm),
OptionalArg::Missing => Ok(default),
}
};
let mut start = index(start, 0)?;
let mut end = index(end, isize::MAX)?;
let len = unistr.char_len() as isize;
if end > len {
end = len;
} else if end < 0 {
end = (end + len).max(0);
}
if start < 0 {
start = (start + len).max(0);
}
let start = start as usize;
let end = end as usize;
let start_byte = unistr.char_index_to_byte(start);
let end_byte = unistr.char_index_to_byte(end);
Ok(GraphemeBreakIterator {
string: unistr,
position: PyMutex::new(GraphemePosition {
start,
start_byte,
pos: start,
pos_byte: start_byte,
end,
end_byte,
state: GraphemeBreakState::default(),
}),
}
.into_ref(&vm.ctx))
}

pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
__module_exec(vm, module);
Expand Down
Loading
Loading