Skip to content

Use current locale for byte regular-expression matching #9042

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 2 commits into from
Oct 10, 2026
Merged

Use current locale for byte regular-expression matching #9042

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
785b1be
Use current locale for byte regular-expression matching
youknowdot Oct 10, 2026
2b6cbc0
Enable the repaired locale caching regression
youknowdot Oct 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Lib/test/test_re.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2179,7 +2179,6 @@ def test_bug_20998(self):
# with ignore case.
self.assertEqual(re.fullmatch('[a-c]+', 'ABC', re.I).span(), (0, 3))

@unittest.expectedFailure # TODO: RUSTPYTHON; self.assertTrue(re.match(b'\xc5', b'\xe5', re.L|re.I))\n AssertionError: None is not true
@unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390")
@unittest.skipIf(sys.platform.startswith("sunos"),
"test doesn't work on Solaris, gh-91214")
Expand Down
47 changes: 47 additions & 0 deletions crates/host_env/src/locale.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,53 @@ use core::{ffi::CStr, ptr};

pub use libc::{LC_ALL, LC_COLLATE, LC_CTYPE, LC_MONETARY, LC_NUMERIC, LC_TIME};

// CPython uses the wide-character APIs after btowc() on Apple platforms.
// Darwin's wint_t is int; libc does not expose these declarations.
#[cfg(target_vendor = "apple")]
unsafe extern "C" {
fn btowc(ch: libc::c_int) -> libc::c_int;
fn iswalnum(ch: libc::c_int) -> libc::c_int;
fn towlower(ch: libc::c_int) -> libc::c_int;
fn towupper(ch: libc::c_int) -> libc::c_int;
}

/// Classify a byte using the current LC_CTYPE locale.
pub fn byte_isalnum(ch: u8) -> bool {
// ctype functions require EOF or an unsigned-char value, never a signed byte.
#[cfg(not(target_vendor = "apple"))]
unsafe {
libc::isalnum(ch.into()) != 0
}
#[cfg(target_vendor = "apple")]
unsafe {
iswalnum(btowc(ch.into())) != 0
}
}

/// Lowercase a byte using the current LC_CTYPE locale.
pub fn byte_tolower(ch: u8) -> u32 {
#[cfg(not(target_vendor = "apple"))]
unsafe {
libc::tolower(ch.into()) as u32
}
#[cfg(target_vendor = "apple")]
unsafe {
towlower(btowc(ch.into())) as u32
}
}

/// Uppercase a byte using the current LC_CTYPE locale.
pub fn byte_toupper(ch: u8) -> u32 {
#[cfg(not(target_vendor = "apple"))]
unsafe {
libc::toupper(ch.into()) as u32
}
#[cfg(target_vendor = "apple")]
unsafe {
towupper(btowc(ch.into())) as u32
}
}

#[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))]
pub use libc::LC_MESSAGES;

Expand Down
73 changes: 44 additions & 29 deletions crates/sre_engine/src/engine.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
// good luck to those that follow; here be dragons

use crate::string::{
is_digit, is_linebreak, is_loc_word, is_space, is_uni_digit, is_uni_linebreak, is_uni_space,
is_uni_word, is_word, lower_ascii, lower_locate, lower_unicode, upper_locate, upper_unicode,
is_digit, is_linebreak, is_space, is_uni_digit, is_uni_linebreak, is_uni_space, is_uni_word,
is_word, lower_ascii, lower_unicode, upper_unicode,
};

use super::{MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor};
use super::{Locale, MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor};
use alloc::{vec, vec::Vec};
use core::{convert::TryFrom, ptr::null};
use optional::Optioned;
Expand All @@ -18,6 +18,7 @@ pub struct Request<'a, S> {
pub pattern_codes: &'a [u32],
pub match_all: bool,
pub must_advance: bool,
pub locale: Locale,
}

impl<'a, S: StrDrive> Request<'a, S> {
Expand All @@ -38,6 +39,7 @@ impl<'a, S: StrDrive> Request<'a, S> {
pattern_codes,
match_all,
must_advance: false,
locale: Locale::default(),
}
}
}
Expand Down Expand Up @@ -674,20 +676,24 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
}
SreOpcode::CATEGORY => {
let cat_code = SreCatCode::try_from(ctx.peek_code(req, 1)).unwrap();
if ctx.at_end(req) || !category(cat_code, ctx.peek_char::<S>()) {
if ctx.at_end(req)
|| !category(cat_code, ctx.peek_char::<S>(), req.locale)
{
break 'result false;
}
ctx.skip_code(2);
ctx.advance_char::<S>();
}
SreOpcode::IN => general_op_in!(charset),
SreOpcode::IN => general_op_in!(|set, c| charset(set, c, req.locale)),
SreOpcode::IN_IGNORE => {
general_op_in!(|set, c| charset(set, lower_ascii(c)))
general_op_in!(|set, c| charset(set, lower_ascii(c), req.locale))
}
SreOpcode::IN_UNI_IGNORE => {
general_op_in!(|set, c| charset(set, lower_unicode(c)))
general_op_in!(|set, c| charset(set, lower_unicode(c), req.locale))
}
SreOpcode::IN_LOC_IGNORE => {
general_op_in!(|set, c| charset_loc_ignore(set, c, req.locale))
}
SreOpcode::IN_LOC_IGNORE => general_op_in!(charset_loc_ignore),
SreOpcode::MARK => {
state
.marks
Expand Down Expand Up @@ -837,13 +843,17 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
SreOpcode::NOT_LITERAL_UNI_IGNORE => {
general_op_literal!(|code, c| code != lower_unicode(c))
}
SreOpcode::LITERAL_LOC_IGNORE => general_op_literal!(char_loc_ignore),
SreOpcode::LITERAL_LOC_IGNORE => {
general_op_literal!(|code, c| char_loc_ignore(code, c, req.locale))
}
SreOpcode::NOT_LITERAL_LOC_IGNORE => {
general_op_literal!(|code, c| !char_loc_ignore(code, c))
general_op_literal!(|code, c| !char_loc_ignore(code, c, req.locale))
}
SreOpcode::GROUPREF => general_op_groupref!(|x| x),
SreOpcode::GROUPREF_IGNORE => general_op_groupref!(lower_ascii),
SreOpcode::GROUPREF_LOC_IGNORE => general_op_groupref!(lower_locate),
SreOpcode::GROUPREF_LOC_IGNORE => {
general_op_groupref!(|c| req.locale.lower(c))
}
SreOpcode::GROUPREF_UNI_IGNORE => general_op_groupref!(lower_unicode),
SreOpcode::GROUPREF_EXISTS => {
let (group_start, group_end) =
Expand Down Expand Up @@ -1042,7 +1052,7 @@ fn search_info_charset<S: StrDrive>(
req.must_advance = false;

loop {
while !ctx.at_end(req) && !charset(set, ctx.peek_char::<S>()) {
while !ctx.at_end(req) && !charset(set, ctx.peek_char::<S>(), req.locale) {
ctx.advance_char::<S>();
}
if ctx.at_end(req) {
Expand Down Expand Up @@ -1226,27 +1236,27 @@ fn at<S: StrDrive>(req: &Request<'_, S>, ctx: &MatchContext, at_code: SreAtCode)
}
SreAtCode::END_LINE => ctx.at_linebreak(req) || ctx.at_end(req),
SreAtCode::END_STRING => ctx.at_end(req),
SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, is_loc_word),
SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, is_loc_word),
SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, |ch| req.locale.is_word(ch)),
SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, |ch| req.locale.is_word(ch)),
SreAtCode::UNI_BOUNDARY => ctx.at_boundary(req, is_uni_word),
SreAtCode::UNI_NON_BOUNDARY => ctx.at_non_boundary(req, is_uni_word),
}
}

fn char_loc_ignore(code: u32, c: u32) -> bool {
code == c || code == lower_locate(c) || code == upper_locate(c)
fn char_loc_ignore(code: u32, c: u32, locale: Locale) -> bool {
code == c || code == locale.lower(c) || code == locale.upper(c)
}

fn charset_loc_ignore(set: &[u32], c: u32) -> bool {
let lo = lower_locate(c);
if charset(set, c) {
fn charset_loc_ignore(set: &[u32], c: u32, locale: Locale) -> bool {
let lo = locale.lower(c);
if charset(set, lo, locale) {
return true;
}
let up = upper_locate(c);
up != lo && charset(set, up)
let up = locale.upper(c);
up != lo && charset(set, up, locale)
}

fn category(cat_code: SreCatCode, c: u32) -> bool {
fn category(cat_code: SreCatCode, c: u32, locale: Locale) -> bool {
match cat_code {
SreCatCode::DIGIT => is_digit(c),
SreCatCode::NOT_DIGIT => !is_digit(c),
Expand All @@ -1256,8 +1266,8 @@ fn category(cat_code: SreCatCode, c: u32) -> bool {
SreCatCode::NOT_WORD => !is_word(c),
SreCatCode::LINEBREAK => is_linebreak(c),
SreCatCode::NOT_LINEBREAK => !is_linebreak(c),
SreCatCode::LOC_WORD => is_loc_word(c),
SreCatCode::LOC_NOT_WORD => !is_loc_word(c),
SreCatCode::LOC_WORD => locale.is_word(c),
SreCatCode::LOC_NOT_WORD => !locale.is_word(c),
SreCatCode::UNI_DIGIT => is_uni_digit(c),
SreCatCode::UNI_NOT_DIGIT => !is_uni_digit(c),
SreCatCode::UNI_SPACE => is_uni_space(c),
Expand All @@ -1269,7 +1279,7 @@ fn category(cat_code: SreCatCode, c: u32) -> bool {
}
}

fn charset(set: &[u32], ch: u32) -> bool {
fn charset(set: &[u32], ch: u32, locale: Locale) -> bool {
/* check if character is a member of the given set */
let mut ok = true;
let mut i = 0;
Expand All @@ -1292,7 +1302,7 @@ fn charset(set: &[u32], ch: u32) -> bool {
break;
}
};
if category(cat_code, ch) {
if category(cat_code, ch, locale) {
return ok;
}
i += 2;
Expand Down Expand Up @@ -1382,7 +1392,8 @@ fn _count<S: StrDrive>(
ctx.skip_char::<S>(max_count);
}
SreOpcode::IN => {
while ctx.cursor.position < end && charset(&ctx.pattern(req)[2..], ctx.peek_char::<S>())
while ctx.cursor.position < end
&& charset(&ctx.pattern(req)[2..], ctx.peek_char::<S>(), req.locale)
{
ctx.advance_char::<S>();
}
Expand All @@ -1400,10 +1411,14 @@ fn _count<S: StrDrive>(
general_count_literal(req, ctx, end, |code, c| code != lower_ascii(c));
}
SreOpcode::LITERAL_LOC_IGNORE => {
general_count_literal(req, ctx, end, char_loc_ignore);
general_count_literal(req, ctx, end, |code, c| {
char_loc_ignore(code, c, req.locale)
});
}
SreOpcode::NOT_LITERAL_LOC_IGNORE => {
general_count_literal(req, ctx, end, |code, c| !char_loc_ignore(code, c));
general_count_literal(req, ctx, end, |code, c| {
!char_loc_ignore(code, c, req.locale)
});
}
SreOpcode::LITERAL_UNI_IGNORE => {
general_count_literal(req, ctx, end, |code, c| code == lower_unicode(c));
Expand Down
2 changes: 1 addition & 1 deletion crates/sre_engine/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ pub mod string;

pub use constants::{SRE_MAGIC, SreAtCode, SreCatCode, SreFlag, SreInfo, SreOpcode};
pub use engine::{Request, SearchIter, State};
pub use string::{StrDrive, StringCursor};
pub use string::{Locale, StrDrive, StringCursor};

pub const CODESIZE: usize = 4;

Expand Down
55 changes: 36 additions & 19 deletions crates/sre_engine/src/string.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,15 +391,44 @@ pub(crate) fn is_digit(ch: u32) -> bool {
u8::try_from(ch).is_ok_and(|x| x.is_ascii_digit())
}

#[inline]
pub(crate) fn is_loc_alnum(ch: u32) -> bool {
// FIXME: Ignore the locales
u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric())
/// Byte classification and case conversion in the current locale.
///
/// Callbacks keep host APIs outside this `no_std` engine. They are called while
/// matching, rather than cached in a compiled pattern, so a locale change also
/// affects existing patterns and scanners. Without host callbacks, matching uses
/// the ASCII rules of the C locale.
#[derive(Debug, Clone, Copy)]
pub struct Locale {
pub is_alnum: fn(u8) -> bool,
pub to_lower: fn(u8) -> u32,
pub to_upper: fn(u8) -> u32,
}

#[inline]
pub(crate) fn is_loc_word(ch: u32) -> bool {
ch == UNDERSCORE || is_loc_alnum(ch)
impl Default for Locale {
fn default() -> Self {
Self {
is_alnum: |ch| ch.is_ascii_alphanumeric(),
to_lower: |ch| ch.to_ascii_lowercase() as u32,
to_upper: |ch| ch.to_ascii_uppercase() as u32,
}
}
}

impl Locale {
#[inline]
pub(crate) fn is_word(self, ch: u32) -> bool {
ch == UNDERSCORE || u8::try_from(ch).is_ok_and(self.is_alnum)
}

#[inline]
pub(crate) fn lower(self, ch: u32) -> u32 {
u8::try_from(ch).map_or(ch, self.to_lower)
}

#[inline]
pub(crate) fn upper(self, ch: u32) -> u32 {
u8::try_from(ch).map_or(ch, self.to_upper)
}
}

#[inline]
Expand All @@ -413,18 +442,6 @@ pub fn lower_ascii(ch: u32) -> u32 {
u8::try_from(ch).map_or(ch, |x| x.to_ascii_lowercase() as u32)
}

#[inline]
pub(crate) fn lower_locate(ch: u32) -> u32 {
// FIXME: Ignore the locales
lower_ascii(ch)
}

#[inline]
pub(crate) fn upper_locate(ch: u32) -> u32 {
// FIXME: Ignore the locales
u8::try_from(ch).map_or(ch, |x| x.to_ascii_uppercase() as u32)
}

#[inline]
pub(crate) fn is_uni_digit(ch: u32) -> bool {
// SRE_UNI_IS_DIGIT matches Unicode decimal digits (Py_UNICODE_ISDECIMAL),
Expand Down
12 changes: 11 additions & 1 deletion crates/vm/src/stdlib/_sre.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,17 @@ mod _sre {
fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef;

fn create_request(self, pattern: &Pattern, start: usize, end: usize) -> Request<'_, Self> {
Request::new(self, start, end, &pattern.code, false)
let request = Request::new(self, start, end, &pattern.code, false);
#[cfg(all(feature = "host_env", any(unix, windows)))]
let request = Request {
locale: rustpython_sre_engine::Locale {
is_alnum: rustpython_host_env::locale::byte_isalnum,
to_lower: rustpython_host_env::locale::byte_tolower,
to_upper: rustpython_host_env::locale::byte_toupper,
},
..request
};
request
}
}

Expand Down
Loading