Skip to content

Commit 9b50bd2

Browse files
youknowoneyouknowone
committed
fix sre
1 parent b85a764 commit 9b50bd2

3 files changed

Lines changed: 90 additions & 5 deletions

File tree

‎crates/sre_engine/src/engine.rs‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,21 @@ impl Marks {
110110
self.marks_stack.pop();
111111
}
112112

113+
fn stack_depth(&self) -> usize {
114+
self.marks_stack.len()
115+
}
116+
117+
fn discard_to(&mut self, depth: usize) {
118+
self.marks_stack.truncate(depth);
119+
}
120+
121+
fn restore_to(&mut self, depth: usize) {
122+
let (marks, last_index) = self.marks_stack[depth].clone();
123+
self.marks = marks;
124+
self.last_index = last_index;
125+
self.marks_stack.truncate(depth);
126+
}
127+
113128
fn clear(&mut self) {
114129
self.last_index = -1;
115130
self.marks.clear();
@@ -144,6 +159,7 @@ impl State {
144159
jump: Jump::OpCode,
145160
repeat_ctx_id: usize::MAX,
146161
count: -1,
162+
marks_stack_base: usize::MAX,
147163
};
148164
_match(req, self, ctx)
149165
}
@@ -165,6 +181,7 @@ impl State {
165181
jump: Jump::OpCode,
166182
repeat_ctx_id: usize::MAX,
167183
count: -1,
184+
marks_stack_base: usize::MAX,
168185
};
169186

170187
if ctx.peek_code(&req, 0) == SreOpcode::INFO as u32 {
@@ -483,6 +500,7 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
483500
}
484501
Jump::PossessiveRepeat2 => {
485502
if popped_result {
503+
ctx.cursor = state.cursor;
486504
ctx.count += 1;
487505
ctx.jump = Jump::PossessiveRepeat1;
488506
continue 'context;
@@ -495,6 +513,7 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
495513
if ((ctx.count as usize) < max_count || max_count == MAXREPEAT)
496514
&& ctx.cursor.position != state.cursor.position
497515
{
516+
ctx.marks_stack_base = state.marks.stack_depth();
498517
state.marks.push();
499518
ctx.cursor = state.cursor;
500519
let mut next = ctx.next_offset(4, Jump::PossessiveRepeat4);
@@ -507,12 +526,12 @@ fn _match<S: StrDrive>(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo
507526
}
508527
Jump::PossessiveRepeat4 => {
509528
if popped_result {
510-
state.marks.pop_discard();
529+
state.marks.discard_to(ctx.marks_stack_base);
511530
ctx.count += 1;
512531
ctx.jump = Jump::PossessiveRepeat3;
513532
continue 'context;
514533
}
515-
state.marks.pop();
534+
state.marks.restore_to(ctx.marks_stack_base);
516535
state.cursor = ctx.cursor;
517536
ctx.skip_code_from(req, 1);
518537
ctx.skip_code(1);
@@ -1057,6 +1076,7 @@ struct MatchContext {
10571076
jump: Jump,
10581077
repeat_ctx_id: usize,
10591078
count: isize,
1079+
marks_stack_base: usize,
10601080
}
10611081

10621082
impl MatchContext {
@@ -1147,7 +1167,9 @@ impl MatchContext {
11471167
mut word_checker: F,
11481168
) -> bool {
11491169
if self.at_beginning() && self.at_end(req) {
1150-
return false;
1170+
// Python 3.14 changed `\B` to match an empty input. Keep the
1171+
// boundary predicate false there, but its negation true.
1172+
return true;
11511173
}
11521174
let that = !self.at_beginning() && word_checker(self.back_peek_char::<S>());
11531175
let this = !self.at_end(req) && word_checker(self.peek_char::<S>());

‎crates/sre_engine/tests/tests.rs‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#[cfg(test)]
33
mod tests {
44
use rustpython_sre_engine::{Request, State, StrDrive};
5+
use rustpython_wtf8::Wtf8Buf;
56

67
struct Pattern {
78
#[expect(unused, reason = "Needed for automated script")]
@@ -44,7 +45,7 @@ mod tests {
4445
#[rustfmt::skip] let big_b = Pattern { pattern: "\\B", code: &[14, 4, 0, 0, 0, 6, 11, 1] };
4546
// END GENERATED
4647
let (req, mut state) = big_b.state("");
47-
assert!(!state.search(req));
48+
assert!(state.search(req));
4849
}
4950

5051
#[test]
@@ -169,6 +170,49 @@ mod tests {
169170
assert!(!state.py_match(&req));
170171
}
171172

173+
#[test]
174+
fn possessive_repeat_keeps_last_capture() {
175+
use optional::Optioned;
176+
177+
let single_code = &[17, 0, 24, 6, 0, 1, 16, 101, 1, 17, 1, 1];
178+
let req = Request::new("eeea", 3, usize::MAX, single_code, false);
179+
let mut single_state = State::default();
180+
assert!(single_state.py_match(&req));
181+
assert_eq!(
182+
single_state.marks.get(0),
183+
(Optioned::some(3), Optioned::some(3))
184+
);
185+
186+
// (e?){2,4}+a: the fourth successful iteration is empty, so group 1
187+
// must retain its final empty span rather than the previous "e".
188+
#[rustfmt::skip] let optional = Pattern {
189+
pattern: "(e?){2,4}+a",
190+
code: &[14, 4, 0, 1, 5, 28, 14, 2, 4, 17, 0, 24, 6, 0, 1, 16, 101, 1, 17, 1, 1, 16, 97, 1],
191+
};
192+
let (req, mut state) = optional.state("eeea");
193+
assert!(state.py_match(&req));
194+
assert_eq!(
195+
state.marks.get(0),
196+
(Optioned::some(3), Optioned::some(3))
197+
);
198+
199+
// ((x)|y|z){3}+: group 1 is the final "z"; group 2 retains "x".
200+
#[rustfmt::skip] let alternation = Pattern {
201+
pattern: "((x)|y|z){3}+",
202+
code: &[14, 4, 0, 3, 3, 28, 28, 3, 3, 17, 0, 7, 9, 17, 2, 16, 120, 17, 3, 15, 12, 5, 16, 121, 15, 7, 5, 16, 122, 15, 2, 0, 17, 1, 1, 1],
203+
};
204+
let (req, mut state) = alternation.state("xyz");
205+
assert!(state.py_match(&req));
206+
assert_eq!(
207+
state.marks.get(0),
208+
(Optioned::some(2), Optioned::some(3))
209+
);
210+
assert_eq!(
211+
state.marks.get(1),
212+
(Optioned::some(0), Optioned::some(1))
213+
);
214+
}
215+
172216
#[test]
173217
fn bug_20998() {
174218
// pattern p = re.compile('[a-c]+', re.I)
@@ -181,6 +225,23 @@ mod tests {
181225
assert_eq!(state.cursor.position, 3);
182226
}
183227

228+
#[test]
229+
fn ascii_ignore_keeps_nonascii_range_literal() {
230+
// pattern p = re.compile(r'[\u0430-\u045f]', re.I | re.A)
231+
//
232+
// ASCII-only case folding must not discard an exact non-ASCII range:
233+
// U+0450 lies in the compiled U+0430..U+045F interval.
234+
#[rustfmt::skip] let p = Pattern {
235+
pattern: "[\\u0430-\\u045f]",
236+
code: &[14, 8, 4, 1, 1, 22, 1072, 1119, 0, 13, 5, 22, 1072, 1119, 0, 1],
237+
};
238+
let (req, mut state) = p.state("\u{0450}");
239+
assert!(state.py_match(&req));
240+
let subject = Wtf8Buf::from("\u{0450}");
241+
let (req, mut state) = p.state(subject.as_ref());
242+
assert!(state.py_match(&req));
243+
}
244+
184245
#[test]
185246
fn bigcharset() {
186247
// pattern p = re.compile('[a-z]*', re.I)

‎crates/stdlib/src/_tokenize.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,9 @@ mod _tokenize {
237237
}
238238

239239
let raw_type = token_kind_value(kind);
240-
let token_type = if extra_tokens && raw_type > TOKEN_DEDENT && raw_type < TOKEN_OP {
240+
let token_type = if extra_tokens
241+
&& (kind == TokenKind::Unknown || (raw_type > TOKEN_DEDENT && raw_type < TOKEN_OP))
242+
{
241243
TOKEN_OP
242244
} else {
243245
raw_type

0 commit comments

Comments
 (0)