Skip to content

Commit fd062ca

Browse files
youknowdotyouknowdot
authored
Use current locale for byte regular-expression matching (#9042)
* Use current locale for byte regular-expression matching Extract the Python 3.14-compatible LOCALE matching paths from #8954, keeping host callbacks outside the no_std engine and preserving the C-locale fallback. Existing canonical regex tests cover locale changes and compiled patterns. Keep expected-failure markers until patched native execution proves they are obsolete; omit duplicate snippet tests and Python 3.15 prefixmatch additions. Assisted-by: OpenAI Codex:model identifier unavailable * Enable the repaired locale caching regression The exact patched native interpreter passes the existing locale flag and compiled-pattern tests and unexpectedly passes locale caching. Remove only the obsolete expected-failure decorator; preserve all canonical assertions and platform skips. Assisted-by: OpenAI Codex:model identifier unavailable
1 parent c516ca9 commit fd062ca

6 files changed

Lines changed: 139 additions & 51 deletions

File tree

‎Lib/test/test_re.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2179,7 +2179,6 @@ def test_bug_20998(self):
21792179
# with ignore case.
21802180
self.assertEqual(re.fullmatch('[a-c]+', 'ABC', re.I).span(), (0, 3))
21812181

2182-
@unittest.expectedFailure # TODO: RUSTPYTHON; self.assertTrue(re.match(b'\xc5', b'\xe5', re.L|re.I))\n AssertionError: None is not true
21832182
@unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390")
21842183
@unittest.skipIf(sys.platform.startswith("sunos"),
21852184
"test doesn't work on Solaris, gh-91214")

‎crates/host_env/src/locale.rs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,53 @@ use core::{ffi::CStr, ptr};
33

44
pub use libc::{LC_ALL, LC_COLLATE, LC_CTYPE, LC_MONETARY, LC_NUMERIC, LC_TIME};
55

6+
// CPython uses the wide-character APIs after btowc() on Apple platforms.
7+
// Darwin's wint_t is int; libc does not expose these declarations.
8+
#[cfg(target_vendor = "apple")]
9+
unsafe extern "C" {
10+
fn btowc(ch: libc::c_int) -> libc::c_int;
11+
fn iswalnum(ch: libc::c_int) -> libc::c_int;
12+
fn towlower(ch: libc::c_int) -> libc::c_int;
13+
fn towupper(ch: libc::c_int) -> libc::c_int;
14+
}
15+
16+
/// Classify a byte using the current LC_CTYPE locale.
17+
pub fn byte_isalnum(ch: u8) -> bool {
18+
// ctype functions require EOF or an unsigned-char value, never a signed byte.
19+
#[cfg(not(target_vendor = "apple"))]
20+
unsafe {
21+
libc::isalnum(ch.into()) != 0
22+
}
23+
#[cfg(target_vendor = "apple")]
24+
unsafe {
25+
iswalnum(btowc(ch.into())) != 0
26+
}
27+
}
28+
29+
/// Lowercase a byte using the current LC_CTYPE locale.
30+
pub fn byte_tolower(ch: u8) -> u32 {
31+
#[cfg(not(target_vendor = "apple"))]
32+
unsafe {
33+
libc::tolower(ch.into()) as u32
34+
}
35+
#[cfg(target_vendor = "apple")]
36+
unsafe {
37+
towlower(btowc(ch.into())) as u32
38+
}
39+
}
40+
41+
/// Uppercase a byte using the current LC_CTYPE locale.
42+
pub fn byte_toupper(ch: u8) -> u32 {
43+
#[cfg(not(target_vendor = "apple"))]
44+
unsafe {
45+
libc::toupper(ch.into()) as u32
46+
}
47+
#[cfg(target_vendor = "apple")]
48+
unsafe {
49+
towupper(btowc(ch.into())) as u32
50+
}
51+
}
52+
653
#[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))]
754
pub use libc::LC_MESSAGES;
855

‎crates/sre_engine/src/engine.rs‎

Lines changed: 44 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
// good luck to those that follow; here be dragons
22

33
use crate::string::{
4-
is_digit, is_linebreak, is_loc_word, is_space, is_uni_digit, is_uni_linebreak, is_uni_space,
5-
is_uni_word, is_word, lower_ascii, lower_locate, lower_unicode, upper_locate, upper_unicode,
4+
is_digit, is_linebreak, is_space, is_uni_digit, is_uni_linebreak, is_uni_space, is_uni_word,
5+
is_word, lower_ascii, lower_unicode, upper_unicode,
66
};
77

8-
use super::{MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor};
8+
use super::{Locale, MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor};
99
use alloc::{vec, vec::Vec};
1010
use core::{convert::TryFrom, ptr::null};
1111
use optional::Optioned;
@@ -18,6 +18,7 @@ pub struct Request<'a, S> {
1818
pub pattern_codes: &'a [u32],
1919
pub match_all: bool,
2020
pub must_advance: bool,
21+
pub locale: Locale,
2122
}
2223

2324
impl<'a, S: StrDrive> Request<'a, S> {
@@ -38,6 +39,7 @@ impl<'a, S: StrDrive> Request<'a, S> {
3839
pattern_codes,
3940
match_all,
4041
must_advance: false,
42+
locale: Locale::default(),
4143
}
4244
}
4345
}
@@ -674,20 +676,24 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
674676
}
675677
SreOpcode::CATEGORY => {
676678
let cat_code = SreCatCode::try_from(ctx.peek_code(req, 1)).unwrap();
677-
if ctx.at_end(req) || !category(cat_code, ctx.peek_char::<S>()) {
679+
if ctx.at_end(req)
680+
|| !category(cat_code, ctx.peek_char::<S>(), req.locale)
681+
{
678682
break 'result false;
679683
}
680684
ctx.skip_code(2);
681685
ctx.advance_char::<S>();
682686
}
683-
SreOpcode::IN => general_op_in!(charset),
687+
SreOpcode::IN => general_op_in!(|set, c| charset(set, c, req.locale)),
684688
SreOpcode::IN_IGNORE => {
685-
general_op_in!(|set, c| charset(set, lower_ascii(c)))
689+
general_op_in!(|set, c| charset(set, lower_ascii(c), req.locale))
686690
}
687691
SreOpcode::IN_UNI_IGNORE => {
688-
general_op_in!(|set, c| charset(set, lower_unicode(c)))
692+
general_op_in!(|set, c| charset(set, lower_unicode(c), req.locale))
693+
}
694+
SreOpcode::IN_LOC_IGNORE => {
695+
general_op_in!(|set, c| charset_loc_ignore(set, c, req.locale))
689696
}
690-
SreOpcode::IN_LOC_IGNORE => general_op_in!(charset_loc_ignore),
691697
SreOpcode::MARK => {
692698
state
693699
.marks
@@ -837,13 +843,17 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
837843
SreOpcode::NOT_LITERAL_UNI_IGNORE => {
838844
general_op_literal!(|code, c| code != lower_unicode(c))
839845
}
840-
SreOpcode::LITERAL_LOC_IGNORE => general_op_literal!(char_loc_ignore),
846+
SreOpcode::LITERAL_LOC_IGNORE => {
847+
general_op_literal!(|code, c| char_loc_ignore(code, c, req.locale))
848+
}
841849
SreOpcode::NOT_LITERAL_LOC_IGNORE => {
842-
general_op_literal!(|code, c| !char_loc_ignore(code, c))
850+
general_op_literal!(|code, c| !char_loc_ignore(code, c, req.locale))
843851
}
844852
SreOpcode::GROUPREF => general_op_groupref!(|x| x),
845853
SreOpcode::GROUPREF_IGNORE => general_op_groupref!(lower_ascii),
846-
SreOpcode::GROUPREF_LOC_IGNORE => general_op_groupref!(lower_locate),
854+
SreOpcode::GROUPREF_LOC_IGNORE => {
855+
general_op_groupref!(|c| req.locale.lower(c))
856+
}
847857
SreOpcode::GROUPREF_UNI_IGNORE => general_op_groupref!(lower_unicode),
848858
SreOpcode::GROUPREF_EXISTS => {
849859
let (group_start, group_end) =
@@ -1042,7 +1052,7 @@ fn search_info_charset<S: StrDrive>(
10421052
req.must_advance = false;
10431053

10441054
loop {
1045-
while !ctx.at_end(req) && !charset(set, ctx.peek_char::<S>()) {
1055+
while !ctx.at_end(req) && !charset(set, ctx.peek_char::<S>(), req.locale) {
10461056
ctx.advance_char::<S>();
10471057
}
10481058
if ctx.at_end(req) {
@@ -1226,27 +1236,27 @@ fn at<S: StrDrive>(req: &Request<'_, S>, ctx: &MatchContext, at_code: SreAtCode)
12261236
}
12271237
SreAtCode::END_LINE => ctx.at_linebreak(req) || ctx.at_end(req),
12281238
SreAtCode::END_STRING => ctx.at_end(req),
1229-
SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, is_loc_word),
1230-
SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, is_loc_word),
1239+
SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, |ch| req.locale.is_word(ch)),
1240+
SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, |ch| req.locale.is_word(ch)),
12311241
SreAtCode::UNI_BOUNDARY => ctx.at_boundary(req, is_uni_word),
12321242
SreAtCode::UNI_NON_BOUNDARY => ctx.at_non_boundary(req, is_uni_word),
12331243
}
12341244
}
12351245

1236-
fn char_loc_ignore(code: u32, c: u32) -> bool {
1237-
code == c || code == lower_locate(c) || code == upper_locate(c)
1246+
fn char_loc_ignore(code: u32, c: u32, locale: Locale) -> bool {
1247+
code == c || code == locale.lower(c) || code == locale.upper(c)
12381248
}
12391249

1240-
fn charset_loc_ignore(set: &[u32], c: u32) -> bool {
1241-
let lo = lower_locate(c);
1242-
if charset(set, c) {
1250+
fn charset_loc_ignore(set: &[u32], c: u32, locale: Locale) -> bool {
1251+
let lo = locale.lower(c);
1252+
if charset(set, lo, locale) {
12431253
return true;
12441254
}
1245-
let up = upper_locate(c);
1246-
up != lo && charset(set, up)
1255+
let up = locale.upper(c);
1256+
up != lo && charset(set, up, locale)
12471257
}
12481258

1249-
fn category(cat_code: SreCatCode, c: u32) -> bool {
1259+
fn category(cat_code: SreCatCode, c: u32, locale: Locale) -> bool {
12501260
match cat_code {
12511261
SreCatCode::DIGIT => is_digit(c),
12521262
SreCatCode::NOT_DIGIT => !is_digit(c),
@@ -1256,8 +1266,8 @@ fn category(cat_code: SreCatCode, c: u32) -> bool {
12561266
SreCatCode::NOT_WORD => !is_word(c),
12571267
SreCatCode::LINEBREAK => is_linebreak(c),
12581268
SreCatCode::NOT_LINEBREAK => !is_linebreak(c),
1259-
SreCatCode::LOC_WORD => is_loc_word(c),
1260-
SreCatCode::LOC_NOT_WORD => !is_loc_word(c),
1269+
SreCatCode::LOC_WORD => locale.is_word(c),
1270+
SreCatCode::LOC_NOT_WORD => !locale.is_word(c),
12611271
SreCatCode::UNI_DIGIT => is_uni_digit(c),
12621272
SreCatCode::UNI_NOT_DIGIT => !is_uni_digit(c),
12631273
SreCatCode::UNI_SPACE => is_uni_space(c),
@@ -1269,7 +1279,7 @@ fn category(cat_code: SreCatCode, c: u32) -> bool {
12691279
}
12701280
}
12711281

1272-
fn charset(set: &[u32], ch: u32) -> bool {
1282+
fn charset(set: &[u32], ch: u32, locale: Locale) -> bool {
12731283
/* check if character is a member of the given set */
12741284
let mut ok = true;
12751285
let mut i = 0;
@@ -1292,7 +1302,7 @@ fn charset(set: &[u32], ch: u32) -> bool {
12921302
break;
12931303
}
12941304
};
1295-
if category(cat_code, ch) {
1305+
if category(cat_code, ch, locale) {
12961306
return ok;
12971307
}
12981308
i += 2;
@@ -1382,7 +1392,8 @@ fn _count<S: StrDrive>(
13821392
ctx.skip_char::<S>(max_count);
13831393
}
13841394
SreOpcode::IN => {
1385-
while ctx.cursor.position < end && charset(&ctx.pattern(req)[2..], ctx.peek_char::<S>())
1395+
while ctx.cursor.position < end
1396+
&& charset(&ctx.pattern(req)[2..], ctx.peek_char::<S>(), req.locale)
13861397
{
13871398
ctx.advance_char::<S>();
13881399
}
@@ -1400,10 +1411,14 @@ fn _count<S: StrDrive>(
14001411
general_count_literal(req, ctx, end, |code, c| code != lower_ascii(c));
14011412
}
14021413
SreOpcode::LITERAL_LOC_IGNORE => {
1403-
general_count_literal(req, ctx, end, char_loc_ignore);
1414+
general_count_literal(req, ctx, end, |code, c| {
1415+
char_loc_ignore(code, c, req.locale)
1416+
});
14041417
}
14051418
SreOpcode::NOT_LITERAL_LOC_IGNORE => {
1406-
general_count_literal(req, ctx, end, |code, c| !char_loc_ignore(code, c));
1419+
general_count_literal(req, ctx, end, |code, c| {
1420+
!char_loc_ignore(code, c, req.locale)
1421+
});
14071422
}
14081423
SreOpcode::LITERAL_UNI_IGNORE => {
14091424
general_count_literal(req, ctx, end, |code, c| code == lower_unicode(c));

‎crates/sre_engine/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ pub mod string;
88

99
pub use constants::{SRE_MAGIC, SreAtCode, SreCatCode, SreFlag, SreInfo, SreOpcode};
1010
pub use engine::{Request, SearchIter, State};
11-
pub use string::{StrDrive, StringCursor};
11+
pub use string::{Locale, StrDrive, StringCursor};
1212

1313
pub const CODESIZE: usize = 4;
1414

‎crates/sre_engine/src/string.rs‎

Lines changed: 36 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -391,15 +391,44 @@ pub(crate) fn is_digit(ch: u32) -> bool {
391391
u8::try_from(ch).is_ok_and(|x| x.is_ascii_digit())
392392
}
393393

394-
#[inline]
395-
pub(crate) fn is_loc_alnum(ch: u32) -> bool {
396-
// FIXME: Ignore the locales
397-
u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric())
394+
/// Byte classification and case conversion in the current locale.
395+
///
396+
/// Callbacks keep host APIs outside this `no_std` engine. They are called while
397+
/// matching, rather than cached in a compiled pattern, so a locale change also
398+
/// affects existing patterns and scanners. Without host callbacks, matching uses
399+
/// the ASCII rules of the C locale.
400+
#[derive(Debug, Clone, Copy)]
401+
pub struct Locale {
402+
pub is_alnum: fn(u8) -> bool,
403+
pub to_lower: fn(u8) -> u32,
404+
pub to_upper: fn(u8) -> u32,
398405
}
399406

400-
#[inline]
401-
pub(crate) fn is_loc_word(ch: u32) -> bool {
402-
ch == UNDERSCORE || is_loc_alnum(ch)
407+
impl Default for Locale {
408+
fn default() -> Self {
409+
Self {
410+
is_alnum: |ch| ch.is_ascii_alphanumeric(),
411+
to_lower: |ch| ch.to_ascii_lowercase() as u32,
412+
to_upper: |ch| ch.to_ascii_uppercase() as u32,
413+
}
414+
}
415+
}
416+
417+
impl Locale {
418+
#[inline]
419+
pub(crate) fn is_word(self, ch: u32) -> bool {
420+
ch == UNDERSCORE || u8::try_from(ch).is_ok_and(self.is_alnum)
421+
}
422+
423+
#[inline]
424+
pub(crate) fn lower(self, ch: u32) -> u32 {
425+
u8::try_from(ch).map_or(ch, self.to_lower)
426+
}
427+
428+
#[inline]
429+
pub(crate) fn upper(self, ch: u32) -> u32 {
430+
u8::try_from(ch).map_or(ch, self.to_upper)
431+
}
403432
}
404433

405434
#[inline]
@@ -413,18 +442,6 @@ pub fn lower_ascii(ch: u32) -> u32 {
413442
u8::try_from(ch).map_or(ch, |x| x.to_ascii_lowercase() as u32)
414443
}
415444

416-
#[inline]
417-
pub(crate) fn lower_locate(ch: u32) -> u32 {
418-
// FIXME: Ignore the locales
419-
lower_ascii(ch)
420-
}
421-
422-
#[inline]
423-
pub(crate) fn upper_locate(ch: u32) -> u32 {
424-
// FIXME: Ignore the locales
425-
u8::try_from(ch).map_or(ch, |x| x.to_ascii_uppercase() as u32)
426-
}
427-
428445
#[inline]
429446
pub(crate) fn is_uni_digit(ch: u32) -> bool {
430447
// SRE_UNI_IS_DIGIT matches Unicode decimal digits (Py_UNICODE_ISDECIMAL),

‎crates/vm/src/stdlib/_sre.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,17 @@ mod _sre {
5959
fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef;
6060

6161
fn create_request(self, pattern: &Pattern, start: usize, end: usize) -> Request<'_, Self> {
62-
Request::new(self, start, end, &pattern.code, false)
62+
let request = Request::new(self, start, end, &pattern.code, false);
63+
#[cfg(all(feature = "host_env", any(unix, windows)))]
64+
let request = Request {
65+
locale: rustpython_sre_engine::Locale {
66+
is_alnum: rustpython_host_env::locale::byte_isalnum,
67+
to_lower: rustpython_host_env::locale::byte_tolower,
68+
to_upper: rustpython_host_env::locale::byte_toupper,
69+
},
70+
..request
71+
};
72+
request
6373
}
6474
}
6575

0 commit comments

Comments
 (0)