Skip to content

Commit ea5a6cd

Browse files
youknowoneyouknowone
authored
Bytecode parity (#7481)
* Bytecode parity Compiler changes: - Remove PUSH_NULL from decorator cal ls, use CALL 0 - Collect __static_attributes__ from self.xxx = patterns - Sort __static_attributes__ alphabetically - Move __classdict__ init before __doc__ in class prologue - Fold unary negative constants - Fold constant list/set literals (3+ elements) - Use BUILD_MAP 0 + MAP_ADD for 16+ dict pairs - Always run peephole optimizer for s uperinstructions - Emit RETURN_GENERATOR for generator functions - Add is_generator flag to SymbolTabl e * Fix formatting and collapsible_if clippy warnings in compile.rs * Fix clippy, fold_unary_negative chaining, and generator line tracing - Replace irrefutable if-let with let for ExceptHandler - Remove folded UNARY_NEGATIVE instead of replacing with NOP, enabling chained negation folding - Initialize prev_line to def line for generators/coroutines to suppress spurious LINE events from preamble instructions - Remove expectedFailure markers for now-passing tests * Fix JIT StoreFastStoreFast, format, and remove expectedFailure markers - Add StoreFastStoreFast handling in JIT instructions - Fix cargo fmt in frame.rs - Remove 11 expectedFailure markers for async jump tests in test_sys_settrace that now pass * Fix peephole optimizer: use NOP replacement instead of remove() Using remove() shifts instruction indices and corrupts subsequent references, causing "pop stackref but null found" panics at runtime. Replace folded/combined instructions with NOP instead, which are cleaned up by the existing remove_nops pass. * Revert peephole_optimize to use remove() for chaining support NOP replacement broke chaining of peephole optimizations (e.g. LOAD_CONST+TO_BOOL then LOAD_CONST+UNARY_NOT for 'not True'). The remove() approach is used by upstream and works correctly here; fold_unary_negative keeps NOP replacement since it doesn't need chaining. * Fix StoreFastStoreFast to handle NULL from LoadFastAndClear StoreFast uses pop_value_opt() to allow NULL values from LoadFastAndClear in inlined comprehension cleanup paths. StoreFastStoreFast must do the same, otherwise the peephole optimizer's fusion of two StoreFast instructions panics when restoring unbound locals after an inlined comprehension.
1 parent 6b5c5a9 commit ea5a6cd

12 files changed

+519
-209
lines changed

‎Lib/test/test_compile.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2486,7 +2486,6 @@ def f():
24862486

24872487
class TestStaticAttributes(unittest.TestCase):
24882488

2489-
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'C' has no attribute '__static_attributes__'
24902489
def test_basic(self):
24912490
class C:
24922491
def f(self):
@@ -2518,7 +2517,6 @@ def h(self, a):
25182517

25192518
self.assertEqual(sorted(C.__static_attributes__), ['u', 'v', 'x', 'y', 'z'])
25202519

2521-
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'C' has no attribute '__static_attributes__'
25222520
def test_nested_class(self):
25232521
class C:
25242522
def f(self):
@@ -2533,7 +2531,6 @@ def g(self):
25332531
self.assertEqual(sorted(C.__static_attributes__), ['x', 'y'])
25342532
self.assertEqual(sorted(C.D.__static_attributes__), ['y', 'z'])
25352533

2536-
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: type object 'C' has no attribute '__static_attributes__'
25372534
def test_subclass(self):
25382535
class C:
25392536
def f(self):
@@ -2593,7 +2590,6 @@ def test_tuple(self):
25932590
def test_set(self):
25942591
self.check_stack_size("{" + "x, " * self.N + "x}")
25952592

2596-
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 202 not less than or equal to 7
25972593
def test_dict(self):
25982594
self.check_stack_size("{" + "x:x, " * self.N + "x:x}")
25992595

‎Lib/test/test_peepholer.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -862,7 +862,6 @@ def setUp(self):
862862
self.addCleanup(sys.settrace, sys.gettrace())
863863
sys.settrace(None)
864864

865-
@unittest.expectedFailure # TODO: RUSTPYTHON; no LOAD_FAST_BORROW_LOAD_FAST_BORROW superinstruction
866865
def test_load_fast_known_simple(self):
867866
def f():
868867
x = 1

‎Lib/test/test_sys_settrace.py‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1420,8 +1420,6 @@ def test_jump_out_of_block_backwards(output):
14201420
output.append(6)
14211421
output.append(7)
14221422

1423-
# TODO: RUSTPYTHON
1424-
@unittest.expectedFailure
14251423
@async_jump_test(4, 5, [3, 5])
14261424
async def test_jump_out_of_async_for_block_forwards(output):
14271425
for i in [1]:
@@ -1430,8 +1428,6 @@ async def test_jump_out_of_async_for_block_forwards(output):
14301428
output.append(4)
14311429
output.append(5)
14321430

1433-
# TODO: RUSTPYTHON
1434-
@unittest.expectedFailure
14351431
@async_jump_test(5, 2, [2, 4, 2, 4, 5, 6])
14361432
async def test_jump_out_of_async_for_block_backwards(output):
14371433
for i in [1]:
@@ -1539,8 +1535,6 @@ def test_jump_forwards_out_of_with_block(output):
15391535
output.append(2)
15401536
output.append(3)
15411537

1542-
# TODO: RUSTPYTHON
1543-
@unittest.expectedFailure
15441538
@async_jump_test(2, 3, [1, 3])
15451539
async def test_jump_forwards_out_of_async_with_block(output):
15461540
async with asynctracecontext(output, 1):
@@ -1553,8 +1547,6 @@ def test_jump_backwards_out_of_with_block(output):
15531547
with tracecontext(output, 2):
15541548
output.append(3)
15551549

1556-
# TODO: RUSTPYTHON
1557-
@unittest.expectedFailure
15581550
@async_jump_test(3, 1, [1, 2, 1, 2, 3, -2])
15591551
async def test_jump_backwards_out_of_async_with_block(output):
15601552
output.append(1)
@@ -1624,8 +1616,6 @@ def test_jump_across_with(output):
16241616
with tracecontext(output, 4):
16251617
output.append(5)
16261618

1627-
# TODO: RUSTPYTHON
1628-
@unittest.expectedFailure
16291619
@async_jump_test(2, 4, [1, 4, 5, -4])
16301620
async def test_jump_across_async_with(output):
16311621
output.append(1)
@@ -1643,8 +1633,6 @@ def test_jump_out_of_with_block_within_for_block(output):
16431633
output.append(5)
16441634
output.append(6)
16451635

1646-
# TODO: RUSTPYTHON
1647-
@unittest.expectedFailure
16481636
@async_jump_test(4, 5, [1, 3, 5, 6])
16491637
async def test_jump_out_of_async_with_block_within_for_block(output):
16501638
output.append(1)
@@ -1663,8 +1651,6 @@ def test_jump_out_of_with_block_within_with_block(output):
16631651
output.append(5)
16641652
output.append(6)
16651653

1666-
# TODO: RUSTPYTHON
1667-
@unittest.expectedFailure
16681654
@async_jump_test(4, 5, [1, 2, 3, 5, -2, 6])
16691655
async def test_jump_out_of_async_with_block_within_with_block(output):
16701656
output.append(1)
@@ -1684,8 +1670,6 @@ def test_jump_out_of_with_block_within_finally_block(output):
16841670
output.append(6)
16851671
output.append(7)
16861672

1687-
# TODO: RUSTPYTHON
1688-
@unittest.expectedFailure
16891673
@async_jump_test(5, 6, [2, 4, 6, 7])
16901674
async def test_jump_out_of_async_with_block_within_finally_block(output):
16911675
try:
@@ -1719,8 +1703,6 @@ def test_jump_out_of_with_assignment(output):
17191703
output.append(4)
17201704
output.append(5)
17211705

1722-
# TODO: RUSTPYTHON
1723-
@unittest.expectedFailure
17241706
@async_jump_test(3, 5, [1, 2, 5])
17251707
async def test_jump_out_of_async_with_assignment(output):
17261708
output.append(1)
@@ -1768,8 +1750,6 @@ def test_jump_over_for_block_before_else(output):
17681750
output.append(7)
17691751
output.append(8)
17701752

1771-
# TODO: RUSTPYTHON
1772-
@unittest.expectedFailure
17731753
@async_jump_test(1, 7, [7, 8])
17741754
async def test_jump_over_async_for_block_before_else(output):
17751755
output.append(1)
@@ -2053,8 +2033,6 @@ def test_jump_between_with_blocks(output):
20532033
with tracecontext(output, 4):
20542034
output.append(5)
20552035

2056-
# TODO: RUSTPYTHON
2057-
@unittest.expectedFailure
20582036
@async_jump_test(3, 5, [1, 2, 5, -2])
20592037
async def test_jump_between_async_with_blocks(output):
20602038
output.append(1)

‎crates/codegen/src/compile.rs‎

Lines changed: 138 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1200,8 +1200,10 @@ impl Compiler {
12001200
/// Emit RESUME instruction with proper handling for async preamble and module lineno.
12011201
/// codegen_enter_scope equivalent for RESUME emission.
12021202
fn emit_resume_for_scope(&mut self, scope_type: CompilerScope, lineno: u32) {
1203-
// For async functions/coroutines, emit RETURN_GENERATOR + POP_TOP before RESUME
1204-
if scope_type == CompilerScope::AsyncFunction {
1203+
// For generators and async functions, emit RETURN_GENERATOR + POP_TOP before RESUME
1204+
let is_gen =
1205+
scope_type == CompilerScope::AsyncFunction || self.current_symbol_table().is_generator;
1206+
if is_gen {
12051207
emit!(self, Instruction::ReturnGenerator);
12061208
emit!(self, Instruction::PopTop);
12071209
}
@@ -2758,18 +2760,15 @@ impl Compiler {
27582760
fn prepare_decorators(&mut self, decorator_list: &[ast::Decorator]) -> CompileResult<()> {
27592761
for decorator in decorator_list {
27602762
self.compile_expression(&decorator.expression)?;
2761-
emit!(self, Instruction::PushNull);
27622763
}
27632764
Ok(())
27642765
}
27652766

2766-
/// Apply decorators in reverse order (LIFO from stack).
2767-
/// Stack [dec1, NULL, dec2, NULL, func] -> dec2(func) -> dec1(dec2(func))
2768-
/// The forward loop works because each Call pops from TOS, naturally
2769-
/// applying decorators bottom-up (innermost first).
2767+
/// Apply decorators: each decorator calls the function below it.
2768+
/// Stack: [dec1, dec2, func] → CALL 0 → [dec1, dec2(func)] → CALL 0 → [dec1(dec2(func))]
27702769
fn apply_decorators(&mut self, decorator_list: &[ast::Decorator]) {
27712770
for _ in decorator_list {
2772-
emit!(self, Instruction::Call { argc: 1 });
2771+
emit!(self, Instruction::Call { argc: 0 });
27732772
}
27742773
}
27752774

@@ -4510,6 +4509,93 @@ impl Compiler {
45104509
Ok(())
45114510
}
45124511

4512+
/// Collect attribute names assigned via `self.xxx = ...` in methods.
4513+
/// These are stored as __static_attributes__ in the class dict.
4514+
fn collect_static_attributes(body: &[ast::Stmt], attrs: Option<&mut IndexSet<String>>) {
4515+
let Some(attrs) = attrs else { return };
4516+
for stmt in body {
4517+
// Only scan def/async def at class body level
4518+
let (params, func_body) = match stmt {
4519+
ast::Stmt::FunctionDef(f) => (&f.parameters, &f.body),
4520+
_ => continue,
4521+
};
4522+
// Get first parameter name (usually "self" or "cls")
4523+
let first_param = params
4524+
.args
4525+
.first()
4526+
.or(params.posonlyargs.first())
4527+
.map(|p| &p.parameter.name);
4528+
let Some(self_name) = first_param else {
4529+
continue;
4530+
};
4531+
// Scan function body for self.xxx = ... (STORE_ATTR on first param)
4532+
Self::scan_store_attrs(func_body, self_name.as_str(), attrs);
4533+
}
4534+
}
4535+
4536+
/// Recursively scan statements for `name.attr = value` patterns.
4537+
fn scan_store_attrs(stmts: &[ast::Stmt], name: &str, attrs: &mut IndexSet<String>) {
4538+
for stmt in stmts {
4539+
match stmt {
4540+
ast::Stmt::Assign(a) => {
4541+
for target in &a.targets {
4542+
if let ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) = target
4543+
&& let ast::Expr::Name(n) = value.as_ref()
4544+
&& n.id.as_str() == name
4545+
{
4546+
attrs.insert(attr.to_string());
4547+
}
4548+
}
4549+
}
4550+
ast::Stmt::AnnAssign(a) => {
4551+
if let ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) =
4552+
a.target.as_ref()
4553+
&& let ast::Expr::Name(n) = value.as_ref()
4554+
&& n.id.as_str() == name
4555+
{
4556+
attrs.insert(attr.to_string());
4557+
}
4558+
}
4559+
ast::Stmt::AugAssign(a) => {
4560+
if let ast::Expr::Attribute(ast::ExprAttribute { value, attr, .. }) =
4561+
a.target.as_ref()
4562+
&& let ast::Expr::Name(n) = value.as_ref()
4563+
&& n.id.as_str() == name
4564+
{
4565+
attrs.insert(attr.to_string());
4566+
}
4567+
}
4568+
ast::Stmt::If(s) => {
4569+
Self::scan_store_attrs(&s.body, name, attrs);
4570+
for clause in &s.elif_else_clauses {
4571+
Self::scan_store_attrs(&clause.body, name, attrs);
4572+
}
4573+
}
4574+
ast::Stmt::For(s) => {
4575+
Self::scan_store_attrs(&s.body, name, attrs);
4576+
Self::scan_store_attrs(&s.orelse, name, attrs);
4577+
}
4578+
ast::Stmt::While(s) => {
4579+
Self::scan_store_attrs(&s.body, name, attrs);
4580+
Self::scan_store_attrs(&s.orelse, name, attrs);
4581+
}
4582+
ast::Stmt::Try(s) => {
4583+
Self::scan_store_attrs(&s.body, name, attrs);
4584+
for handler in &s.handlers {
4585+
let ast::ExceptHandler::ExceptHandler(h) = handler;
4586+
Self::scan_store_attrs(&h.body, name, attrs);
4587+
}
4588+
Self::scan_store_attrs(&s.orelse, name, attrs);
4589+
Self::scan_store_attrs(&s.finalbody, name, attrs);
4590+
}
4591+
ast::Stmt::With(s) => {
4592+
Self::scan_store_attrs(&s.body, name, attrs);
4593+
}
4594+
_ => {}
4595+
}
4596+
}
4597+
}
4598+
45134599
// Python/compile.c find_ann
45144600
fn find_ann(body: &[ast::Stmt]) -> bool {
45154601
for statement in body {
@@ -4617,6 +4703,13 @@ impl Compiler {
46174703
}
46184704
);
46194705

4706+
// PEP 649: Initialize __classdict__ cell (before __doc__)
4707+
if self.current_symbol_table().needs_classdict {
4708+
emit!(self, Instruction::LoadLocals);
4709+
let classdict_idx = self.get_cell_var_index("__classdict__")?;
4710+
emit!(self, Instruction::StoreDeref { i: classdict_idx });
4711+
}
4712+
46204713
// Store __doc__ only if there's an explicit docstring
46214714
if let Some(doc) = doc_str {
46224715
self.emit_load_const(ConstantData::Str { value: doc.into() });
@@ -4645,13 +4738,6 @@ impl Compiler {
46454738
);
46464739
}
46474740

4648-
// PEP 649: Initialize __classdict__ cell for class annotation scope
4649-
if self.current_symbol_table().needs_classdict {
4650-
emit!(self, Instruction::LoadLocals);
4651-
let classdict_idx = self.get_cell_var_index("__classdict__")?;
4652-
emit!(self, Instruction::StoreDeref { i: classdict_idx });
4653-
}
4654-
46554741
// Handle class annotations based on future_annotations flag
46564742
if Self::find_ann(body) {
46574743
if self.future_annotations {
@@ -4669,6 +4755,16 @@ impl Compiler {
46694755
}
46704756
}
46714757

4758+
// Collect __static_attributes__: scan methods for self.xxx = ... patterns
4759+
Self::collect_static_attributes(
4760+
body,
4761+
self.code_stack
4762+
.last_mut()
4763+
.unwrap()
4764+
.static_attributes
4765+
.as_mut(),
4766+
);
4767+
46724768
// 3. Compile the class body
46734769
self.compile_statements(body)?;
46744770

@@ -4684,14 +4780,15 @@ impl Compiler {
46844780

46854781
// Emit __static_attributes__ tuple
46864782
{
4687-
let attrs: Vec<String> = self
4783+
let mut attrs: Vec<String> = self
46884784
.code_stack
46894785
.last()
46904786
.unwrap()
46914787
.static_attributes
46924788
.as_ref()
46934789
.map(|s| s.iter().cloned().collect())
46944790
.unwrap_or_default();
4791+
attrs.sort();
46954792
self.emit_load_const(ConstantData::Tuple {
46964793
elements: attrs
46974794
.into_iter()
@@ -5091,8 +5188,7 @@ impl Compiler {
50915188
method: SpecialMethod::AEnter
50925189
}
50935190
); // [bound_aexit, bound_aenter]
5094-
// bound_aenter is already bound, call with NULL self_or_null
5095-
emit!(self, Instruction::PushNull); // [bound_aexit, bound_aenter, NULL]
5191+
emit!(self, Instruction::PushNull);
50965192
emit!(self, Instruction::Call { argc: 0 }); // [bound_aexit, awaitable]
50975193
emit!(self, Instruction::GetAwaitable { r#where: 1 });
50985194
self.emit_load_const(ConstantData::None);
@@ -5112,8 +5208,7 @@ impl Compiler {
51125208
method: SpecialMethod::Enter
51135209
}
51145210
); // [bound_exit, bound_enter]
5115-
// bound_enter is already bound, call with NULL self_or_null
5116-
emit!(self, Instruction::PushNull); // [bound_exit, bound_enter, NULL]
5211+
emit!(self, Instruction::PushNull);
51175212
emit!(self, Instruction::Call { argc: 0 }); // [bound_exit, enter_result]
51185213
}
51195214

@@ -5168,8 +5263,8 @@ impl Compiler {
51685263
});
51695264

51705265
// ===== Normal exit path =====
5171-
// Stack: [..., __exit__]
5172-
// Call __exit__(None, None, None)
5266+
// Stack: [..., bound_exit]
5267+
// Call bound_exit(None, None, None)
51735268
self.set_source_range(with_range);
51745269
emit!(self, Instruction::PushNull);
51755270
self.emit_load_const(ConstantData::None);
@@ -6894,17 +6989,28 @@ impl Compiler {
68946989
let has_unpacking = items.iter().any(|item| item.key.is_none());
68956990

68966991
if !has_unpacking {
6897-
// Simple case: no ** unpacking, build all pairs directly
6898-
for item in items {
6899-
self.compile_expression(item.key.as_ref().unwrap())?;
6900-
self.compile_expression(&item.value)?;
6901-
}
6902-
emit!(
6903-
self,
6904-
Instruction::BuildMap {
6905-
count: u32::try_from(items.len()).expect("too many dict items"),
6992+
// STACK_USE_GUIDELINE: for large dicts (16+ pairs), use
6993+
// BUILD_MAP 0 + MAP_ADD to avoid excessive stack usage
6994+
let big = items.len() * 2 > 30; // ~15 pairs threshold
6995+
if big {
6996+
emit!(self, Instruction::BuildMap { count: 0 });
6997+
for item in items {
6998+
self.compile_expression(item.key.as_ref().unwrap())?;
6999+
self.compile_expression(&item.value)?;
7000+
emit!(self, Instruction::MapAdd { i: 1 });
69067001
}
6907-
);
7002+
} else {
7003+
for item in items {
7004+
self.compile_expression(item.key.as_ref().unwrap())?;
7005+
self.compile_expression(&item.value)?;
7006+
}
7007+
emit!(
7008+
self,
7009+
Instruction::BuildMap {
7010+
count: u32::try_from(items.len()).expect("too many dict items"),
7011+
}
7012+
);
7013+
}
69087014
return Ok(());
69097015
}
69107016

0 commit comments

Comments
 (0)