diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 80c21f6b198..7a480a62b6c 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2179,7 +2179,6 @@ def test_bug_20998(self): # with ignore case. self.assertEqual(re.fullmatch('[a-c]+', 'ABC', re.I).span(), (0, 3)) - @unittest.expectedFailure # TODO: RUSTPYTHON; self.assertTrue(re.match(b'\xc5', b'\xe5', re.L|re.I))\n AssertionError: None is not true @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390") @unittest.skipIf(sys.platform.startswith("sunos"), "test doesn't work on Solaris, gh-91214") diff --git a/crates/host_env/src/locale.rs b/crates/host_env/src/locale.rs index e9df052c208..07dfa512d65 100644 --- a/crates/host_env/src/locale.rs +++ b/crates/host_env/src/locale.rs @@ -3,6 +3,53 @@ use core::{ffi::CStr, ptr}; pub use libc::{LC_ALL, LC_COLLATE, LC_CTYPE, LC_MONETARY, LC_NUMERIC, LC_TIME}; +// CPython uses the wide-character APIs after btowc() on Apple platforms. +// Darwin's wint_t is int; libc does not expose these declarations. +#[cfg(target_vendor = "apple")] +unsafe extern "C" { + fn btowc(ch: libc::c_int) -> libc::c_int; + fn iswalnum(ch: libc::c_int) -> libc::c_int; + fn towlower(ch: libc::c_int) -> libc::c_int; + fn towupper(ch: libc::c_int) -> libc::c_int; +} + +/// Classify a byte using the current LC_CTYPE locale. +pub fn byte_isalnum(ch: u8) -> bool { + // ctype functions require EOF or an unsigned-char value, never a signed byte. + #[cfg(not(target_vendor = "apple"))] + unsafe { + libc::isalnum(ch.into()) != 0 + } + #[cfg(target_vendor = "apple")] + unsafe { + iswalnum(btowc(ch.into())) != 0 + } +} + +/// Lowercase a byte using the current LC_CTYPE locale. +pub fn byte_tolower(ch: u8) -> u32 { + #[cfg(not(target_vendor = "apple"))] + unsafe { + libc::tolower(ch.into()) as u32 + } + #[cfg(target_vendor = "apple")] + unsafe { + towlower(btowc(ch.into())) as u32 + } +} + +/// Uppercase a byte using the current LC_CTYPE locale. +pub fn byte_toupper(ch: u8) -> u32 { + #[cfg(not(target_vendor = "apple"))] + unsafe { + libc::toupper(ch.into()) as u32 + } + #[cfg(target_vendor = "apple")] + unsafe { + towupper(btowc(ch.into())) as u32 + } +} + #[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))] pub use libc::LC_MESSAGES; diff --git a/crates/sre_engine/src/engine.rs b/crates/sre_engine/src/engine.rs index c2a6ac81975..7224b4e878f 100644 --- a/crates/sre_engine/src/engine.rs +++ b/crates/sre_engine/src/engine.rs @@ -1,11 +1,11 @@ // good luck to those that follow; here be dragons use crate::string::{ - is_digit, is_linebreak, is_loc_word, is_space, is_uni_digit, is_uni_linebreak, is_uni_space, - is_uni_word, is_word, lower_ascii, lower_locate, lower_unicode, upper_locate, upper_unicode, + is_digit, is_linebreak, is_space, is_uni_digit, is_uni_linebreak, is_uni_space, is_uni_word, + is_word, lower_ascii, lower_unicode, upper_unicode, }; -use super::{MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor}; +use super::{Locale, MAXREPEAT, SreAtCode, SreCatCode, SreInfo, SreOpcode, StrDrive, StringCursor}; use alloc::{vec, vec::Vec}; use core::{convert::TryFrom, ptr::null}; use optional::Optioned; @@ -18,6 +18,7 @@ pub struct Request<'a, S> { pub pattern_codes: &'a [u32], pub match_all: bool, pub must_advance: bool, + pub locale: Locale, } impl<'a, S: StrDrive> Request<'a, S> { @@ -38,6 +39,7 @@ impl<'a, S: StrDrive> Request<'a, S> { pattern_codes, match_all, must_advance: false, + locale: Locale::default(), } } } @@ -674,20 +676,24 @@ fn _match(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo } SreOpcode::CATEGORY => { let cat_code = SreCatCode::try_from(ctx.peek_code(req, 1)).unwrap(); - if ctx.at_end(req) || !category(cat_code, ctx.peek_char::()) { + if ctx.at_end(req) + || !category(cat_code, ctx.peek_char::(), req.locale) + { break 'result false; } ctx.skip_code(2); ctx.advance_char::(); } - SreOpcode::IN => general_op_in!(charset), + SreOpcode::IN => general_op_in!(|set, c| charset(set, c, req.locale)), SreOpcode::IN_IGNORE => { - general_op_in!(|set, c| charset(set, lower_ascii(c))) + general_op_in!(|set, c| charset(set, lower_ascii(c), req.locale)) } SreOpcode::IN_UNI_IGNORE => { - general_op_in!(|set, c| charset(set, lower_unicode(c))) + general_op_in!(|set, c| charset(set, lower_unicode(c), req.locale)) + } + SreOpcode::IN_LOC_IGNORE => { + general_op_in!(|set, c| charset_loc_ignore(set, c, req.locale)) } - SreOpcode::IN_LOC_IGNORE => general_op_in!(charset_loc_ignore), SreOpcode::MARK => { state .marks @@ -837,13 +843,17 @@ fn _match(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo SreOpcode::NOT_LITERAL_UNI_IGNORE => { general_op_literal!(|code, c| code != lower_unicode(c)) } - SreOpcode::LITERAL_LOC_IGNORE => general_op_literal!(char_loc_ignore), + SreOpcode::LITERAL_LOC_IGNORE => { + general_op_literal!(|code, c| char_loc_ignore(code, c, req.locale)) + } SreOpcode::NOT_LITERAL_LOC_IGNORE => { - general_op_literal!(|code, c| !char_loc_ignore(code, c)) + general_op_literal!(|code, c| !char_loc_ignore(code, c, req.locale)) } SreOpcode::GROUPREF => general_op_groupref!(|x| x), SreOpcode::GROUPREF_IGNORE => general_op_groupref!(lower_ascii), - SreOpcode::GROUPREF_LOC_IGNORE => general_op_groupref!(lower_locate), + SreOpcode::GROUPREF_LOC_IGNORE => { + general_op_groupref!(|c| req.locale.lower(c)) + } SreOpcode::GROUPREF_UNI_IGNORE => general_op_groupref!(lower_unicode), SreOpcode::GROUPREF_EXISTS => { let (group_start, group_end) = @@ -1042,7 +1052,7 @@ fn search_info_charset( req.must_advance = false; loop { - while !ctx.at_end(req) && !charset(set, ctx.peek_char::()) { + while !ctx.at_end(req) && !charset(set, ctx.peek_char::(), req.locale) { ctx.advance_char::(); } if ctx.at_end(req) { @@ -1226,27 +1236,27 @@ fn at(req: &Request<'_, S>, ctx: &MatchContext, at_code: SreAtCode) } SreAtCode::END_LINE => ctx.at_linebreak(req) || ctx.at_end(req), SreAtCode::END_STRING => ctx.at_end(req), - SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, is_loc_word), - SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, is_loc_word), + SreAtCode::LOC_BOUNDARY => ctx.at_boundary(req, |ch| req.locale.is_word(ch)), + SreAtCode::LOC_NON_BOUNDARY => ctx.at_non_boundary(req, |ch| req.locale.is_word(ch)), SreAtCode::UNI_BOUNDARY => ctx.at_boundary(req, is_uni_word), SreAtCode::UNI_NON_BOUNDARY => ctx.at_non_boundary(req, is_uni_word), } } -fn char_loc_ignore(code: u32, c: u32) -> bool { - code == c || code == lower_locate(c) || code == upper_locate(c) +fn char_loc_ignore(code: u32, c: u32, locale: Locale) -> bool { + code == c || code == locale.lower(c) || code == locale.upper(c) } -fn charset_loc_ignore(set: &[u32], c: u32) -> bool { - let lo = lower_locate(c); - if charset(set, c) { +fn charset_loc_ignore(set: &[u32], c: u32, locale: Locale) -> bool { + let lo = locale.lower(c); + if charset(set, lo, locale) { return true; } - let up = upper_locate(c); - up != lo && charset(set, up) + let up = locale.upper(c); + up != lo && charset(set, up, locale) } -fn category(cat_code: SreCatCode, c: u32) -> bool { +fn category(cat_code: SreCatCode, c: u32, locale: Locale) -> bool { match cat_code { SreCatCode::DIGIT => is_digit(c), SreCatCode::NOT_DIGIT => !is_digit(c), @@ -1256,8 +1266,8 @@ fn category(cat_code: SreCatCode, c: u32) -> bool { SreCatCode::NOT_WORD => !is_word(c), SreCatCode::LINEBREAK => is_linebreak(c), SreCatCode::NOT_LINEBREAK => !is_linebreak(c), - SreCatCode::LOC_WORD => is_loc_word(c), - SreCatCode::LOC_NOT_WORD => !is_loc_word(c), + SreCatCode::LOC_WORD => locale.is_word(c), + SreCatCode::LOC_NOT_WORD => !locale.is_word(c), SreCatCode::UNI_DIGIT => is_uni_digit(c), SreCatCode::UNI_NOT_DIGIT => !is_uni_digit(c), SreCatCode::UNI_SPACE => is_uni_space(c), @@ -1269,7 +1279,7 @@ fn category(cat_code: SreCatCode, c: u32) -> bool { } } -fn charset(set: &[u32], ch: u32) -> bool { +fn charset(set: &[u32], ch: u32, locale: Locale) -> bool { /* check if character is a member of the given set */ let mut ok = true; let mut i = 0; @@ -1292,7 +1302,7 @@ fn charset(set: &[u32], ch: u32) -> bool { break; } }; - if category(cat_code, ch) { + if category(cat_code, ch, locale) { return ok; } i += 2; @@ -1382,7 +1392,8 @@ fn _count( ctx.skip_char::(max_count); } SreOpcode::IN => { - while ctx.cursor.position < end && charset(&ctx.pattern(req)[2..], ctx.peek_char::()) + while ctx.cursor.position < end + && charset(&ctx.pattern(req)[2..], ctx.peek_char::(), req.locale) { ctx.advance_char::(); } @@ -1400,10 +1411,14 @@ fn _count( general_count_literal(req, ctx, end, |code, c| code != lower_ascii(c)); } SreOpcode::LITERAL_LOC_IGNORE => { - general_count_literal(req, ctx, end, char_loc_ignore); + general_count_literal(req, ctx, end, |code, c| { + char_loc_ignore(code, c, req.locale) + }); } SreOpcode::NOT_LITERAL_LOC_IGNORE => { - general_count_literal(req, ctx, end, |code, c| !char_loc_ignore(code, c)); + general_count_literal(req, ctx, end, |code, c| { + !char_loc_ignore(code, c, req.locale) + }); } SreOpcode::LITERAL_UNI_IGNORE => { general_count_literal(req, ctx, end, |code, c| code == lower_unicode(c)); diff --git a/crates/sre_engine/src/lib.rs b/crates/sre_engine/src/lib.rs index df598974ba9..7c31a6bae9d 100644 --- a/crates/sre_engine/src/lib.rs +++ b/crates/sre_engine/src/lib.rs @@ -8,7 +8,7 @@ pub mod string; pub use constants::{SRE_MAGIC, SreAtCode, SreCatCode, SreFlag, SreInfo, SreOpcode}; pub use engine::{Request, SearchIter, State}; -pub use string::{StrDrive, StringCursor}; +pub use string::{Locale, StrDrive, StringCursor}; pub const CODESIZE: usize = 4; diff --git a/crates/sre_engine/src/string.rs b/crates/sre_engine/src/string.rs index d16a3ee88af..52b427a9414 100644 --- a/crates/sre_engine/src/string.rs +++ b/crates/sre_engine/src/string.rs @@ -391,15 +391,44 @@ pub(crate) fn is_digit(ch: u32) -> bool { u8::try_from(ch).is_ok_and(|x| x.is_ascii_digit()) } -#[inline] -pub(crate) fn is_loc_alnum(ch: u32) -> bool { - // FIXME: Ignore the locales - u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric()) +/// Byte classification and case conversion in the current locale. +/// +/// Callbacks keep host APIs outside this `no_std` engine. They are called while +/// matching, rather than cached in a compiled pattern, so a locale change also +/// affects existing patterns and scanners. Without host callbacks, matching uses +/// the ASCII rules of the C locale. +#[derive(Debug, Clone, Copy)] +pub struct Locale { + pub is_alnum: fn(u8) -> bool, + pub to_lower: fn(u8) -> u32, + pub to_upper: fn(u8) -> u32, } -#[inline] -pub(crate) fn is_loc_word(ch: u32) -> bool { - ch == UNDERSCORE || is_loc_alnum(ch) +impl Default for Locale { + fn default() -> Self { + Self { + is_alnum: |ch| ch.is_ascii_alphanumeric(), + to_lower: |ch| ch.to_ascii_lowercase() as u32, + to_upper: |ch| ch.to_ascii_uppercase() as u32, + } + } +} + +impl Locale { + #[inline] + pub(crate) fn is_word(self, ch: u32) -> bool { + ch == UNDERSCORE || u8::try_from(ch).is_ok_and(self.is_alnum) + } + + #[inline] + pub(crate) fn lower(self, ch: u32) -> u32 { + u8::try_from(ch).map_or(ch, self.to_lower) + } + + #[inline] + pub(crate) fn upper(self, ch: u32) -> u32 { + u8::try_from(ch).map_or(ch, self.to_upper) + } } #[inline] @@ -413,18 +442,6 @@ pub fn lower_ascii(ch: u32) -> u32 { u8::try_from(ch).map_or(ch, |x| x.to_ascii_lowercase() as u32) } -#[inline] -pub(crate) fn lower_locate(ch: u32) -> u32 { - // FIXME: Ignore the locales - lower_ascii(ch) -} - -#[inline] -pub(crate) fn upper_locate(ch: u32) -> u32 { - // FIXME: Ignore the locales - u8::try_from(ch).map_or(ch, |x| x.to_ascii_uppercase() as u32) -} - #[inline] pub(crate) fn is_uni_digit(ch: u32) -> bool { // SRE_UNI_IS_DIGIT matches Unicode decimal digits (Py_UNICODE_ISDECIMAL), diff --git a/crates/vm/src/stdlib/_sre.rs b/crates/vm/src/stdlib/_sre.rs index 9fe57a86955..2e24b1477cc 100644 --- a/crates/vm/src/stdlib/_sre.rs +++ b/crates/vm/src/stdlib/_sre.rs @@ -59,7 +59,17 @@ mod _sre { fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef; fn create_request(self, pattern: &Pattern, start: usize, end: usize) -> Request<'_, Self> { - Request::new(self, start, end, &pattern.code, false) + let request = Request::new(self, start, end, &pattern.code, false); + #[cfg(all(feature = "host_env", any(unix, windows)))] + let request = Request { + locale: rustpython_sre_engine::Locale { + is_alnum: rustpython_host_env::locale::byte_isalnum, + to_lower: rustpython_host_env::locale::byte_tolower, + to_upper: rustpython_host_env::locale::byte_toupper, + }, + ..request + }; + request } }