Skip to content

Commit 8c73f3c

Browse files
youknowoneyouknowone
committed
emit RESUME
1 parent 9216500 commit 8c73f3c

8 files changed

Lines changed: 186 additions & 86 deletions

‎Lib/test/test_compile.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,8 +152,6 @@ def test_indentation(self):
152152
pass"""
153153
compile(s, "<string>", "exec")
154154

155-
# TODO: RUSTPYTHON
156-
@unittest.expectedFailure
157155
# This test is probably specific to CPython and may not generalize
158156
# to other implementations. We are trying to ensure that when
159157
# the first line of code starts after 256, correct line numbers

‎Lib/test/test_peepholer.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,6 @@ def test_pack_unpack(self):
147147
self.assertNotInBytecode(code, 'UNPACK_SEQUENCE')
148148
self.check_lnotab(code)
149149

150-
@unittest.expectedFailure # TODO: RUSTPYTHON
151150
def test_folding_of_tuples_of_constants(self):
152151
for line, elem in (
153152
('a = 1,2,3', (1, 2, 3)),

‎crates/codegen/src/compile.rs‎

Lines changed: 43 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -997,12 +997,6 @@ impl Compiler {
997997
key: usize, // In RustPython, we use the index in symbol_table_stack as key
998998
lineno: u32,
999999
) -> CompileResult<()> {
1000-
// Create location
1001-
let location = SourceLocation {
1002-
line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN),
1003-
character_offset: OneIndexed::MIN,
1004-
};
1005-
10061000
// Allocate a new compiler unit
10071001

10081002
// In Rust, we'll create the structure directly
@@ -1148,40 +1142,50 @@ impl Compiler {
11481142
self.set_qualname();
11491143
}
11501144

1151-
// Emit RESUME instruction
1152-
let _resume_loc = if scope_type == CompilerScope::Module {
1153-
// Module scope starts with lineno 0
1154-
SourceLocation {
1155-
line: OneIndexed::MIN,
1156-
character_offset: OneIndexed::MIN,
1157-
}
1158-
} else {
1159-
location
1160-
};
1145+
// Emit RESUME (handles async preamble and module lineno 0)
1146+
// CPython: LOCATION(lineno, lineno, 0, 0), then loc.lineno = 0 for module
1147+
self.emit_resume_for_scope(scope_type, lineno);
11611148

1162-
// Set the source range for the RESUME instruction
1163-
// For now, just use an empty range at the beginning
1164-
self.current_source_range = TextRange::default();
1149+
Ok(())
1150+
}
11651151

1152+
/// Emit RESUME instruction with proper handling for async preamble and module lineno.
1153+
/// codegen_enter_scope equivalent for RESUME emission.
1154+
fn emit_resume_for_scope(&mut self, scope_type: CompilerScope, lineno: u32) {
11661155
// For async functions/coroutines, emit RETURN_GENERATOR + POP_TOP before RESUME
11671156
if scope_type == CompilerScope::AsyncFunction {
11681157
emit!(self, Instruction::ReturnGenerator);
11691158
emit!(self, Instruction::PopTop);
11701159
}
11711160

1172-
emit!(
1173-
self,
1174-
Instruction::Resume {
1175-
arg: bytecode::ResumeType::AtFuncStart as u32
1176-
}
1177-
);
1161+
// CPython: LOCATION(lineno, lineno, 0, 0)
1162+
// Module scope: loc.lineno = 0 (before the first line)
1163+
let lineno_override = if scope_type == CompilerScope::Module {
1164+
Some(0)
1165+
} else {
1166+
None
1167+
};
11781168

1179-
if scope_type == CompilerScope::Module {
1180-
// This would be loc.lineno = -1 in CPython
1181-
// We handle this differently in RustPython
1182-
}
1169+
// Use lineno for location (col = 0 as in CPython)
1170+
let location = SourceLocation {
1171+
line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN),
1172+
character_offset: OneIndexed::MIN, // col = 0
1173+
};
1174+
let end_location = location; // end_lineno = lineno, end_col = 0
1175+
let except_handler = self.current_except_handler();
11831176

1184-
Ok(())
1177+
self.current_block().instructions.push(ir::InstructionInfo {
1178+
instr: Instruction::Resume {
1179+
arg: OpArgMarker::marker(),
1180+
}
1181+
.into(),
1182+
arg: OpArg(bytecode::ResumeType::AtFuncStart as u32),
1183+
target: BlockIdx::NULL,
1184+
location,
1185+
end_location,
1186+
except_handler,
1187+
lineno_override,
1188+
});
11851189
}
11861190

11871191
fn push_output(
@@ -1750,6 +1754,8 @@ impl Compiler {
17501754
self.future_annotations = symbol_table.future_annotations;
17511755
self.symbol_table_stack.push(symbol_table);
17521756

1757+
self.emit_resume_for_scope(CompilerScope::Module, 1);
1758+
17531759
let (doc, statements) = split_doc(&body.body, &self.opts);
17541760
if let Some(value) = doc {
17551761
self.emit_load_const(ConstantData::Str {
@@ -1795,6 +1801,8 @@ impl Compiler {
17951801
self.future_annotations = symbol_table.future_annotations;
17961802
self.symbol_table_stack.push(symbol_table);
17971803

1804+
self.emit_resume_for_scope(CompilerScope::Module, 1);
1805+
17981806
// Handle annotations based on future_annotations flag
17991807
if Self::find_ann(body) {
18001808
if self.future_annotations {
@@ -1858,6 +1866,7 @@ impl Compiler {
18581866
symbol_table: SymbolTable,
18591867
) -> CompileResult<()> {
18601868
self.symbol_table_stack.push(symbol_table);
1869+
self.emit_resume_for_scope(CompilerScope::Module, 1);
18611870

18621871
self.compile_statements(body)?;
18631872

@@ -1887,6 +1896,8 @@ impl Compiler {
18871896
symbol_table: SymbolTable,
18881897
) -> CompileResult<()> {
18891898
self.symbol_table_stack.push(symbol_table);
1899+
self.emit_resume_for_scope(CompilerScope::Module, 1);
1900+
18901901
self.compile_expression(&expression.body)?;
18911902
self.emit_return_value();
18921903
Ok(())
@@ -2631,7 +2642,7 @@ impl Compiler {
26312642

26322643
// Get the current symbol table
26332644
let key = self.symbol_table_stack.len() - 1;
2634-
let lineno = expr.range().start().to_u32();
2645+
let lineno = self.get_source_line_number().get().to_u32();
26352646

26362647
// Enter scope with the type parameter name
26372648
self.enter_scope(name, CompilerScope::TypeParams, key, lineno)?;
@@ -7764,6 +7775,7 @@ impl Compiler {
77647775
location,
77657776
end_location,
77667777
except_handler,
7778+
lineno_override: None,
77677779
});
77687780
}
77697781

@@ -7788,8 +7800,6 @@ impl Compiler {
77887800
}
77897801

77907802
fn emit_load_const(&mut self, constant: ConstantData) {
7791-
// Always add to co_consts and emit LOAD_CONST.
7792-
// Optimization pass converts LOAD_CONST to LOAD_SMALL_INT where appropriate.
77937803
let idx = self.arg_constant(constant);
77947804
self.emit_arg(idx, |idx| Instruction::LoadConst { idx })
77957805
}

‎crates/codegen/src/ir.rs‎

Lines changed: 107 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,15 @@ use rustpython_compiler_core::{
1414
varint::{write_signed_varint, write_varint},
1515
};
1616

17+
/// Location info for linetable generation (allows line 0 for RESUME)
18+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
19+
struct LineTableLocation {
20+
line: i32,
21+
end_line: i32,
22+
col: i32,
23+
end_col: i32,
24+
}
25+
1726
/// Metadata for a code unit
1827
// = _PyCompile_CodeUnitMetadata
1928
#[derive(Clone, Debug)]
@@ -94,6 +103,8 @@ pub struct InstructionInfo {
94103
pub location: SourceLocation,
95104
pub end_location: SourceLocation,
96105
pub except_handler: Option<ExceptHandlerInfo>,
106+
/// Override line number for linetable (e.g., line 0 for module RESUME)
107+
pub lineno_override: Option<i32>,
97108
}
98109

99110
/// Exception handler information for an instruction.
@@ -161,11 +172,16 @@ impl CodeInfo {
161172
mut self,
162173
opts: &crate::compile::CompileOpts,
163174
) -> crate::InternalResult<CodeObject> {
164-
// Always fold tuple constants and convert to LOAD_SMALL_INT
165-
// 1. fold_tuple_of_constants - fold constant sequences into tuples
166-
// 2. maybe_instr_make_load_smallint - convert LOAD_CONST to LOAD_SMALL_INT
175+
// Always fold tuple constants
167176
self.fold_tuple_constants();
168-
self.convert_to_load_small_int();
177+
// Python only applies LOAD_SMALL_INT conversion to module-level code
178+
// (not inside functions). Module code lacks OPTIMIZED flag.
179+
// Note: RustPython incorrectly sets NEWLOCALS on modules, so only check OPTIMIZED
180+
let is_module_level = !self.flags.contains(CodeFlags::OPTIMIZED);
181+
if is_module_level {
182+
self.convert_to_load_small_int();
183+
}
184+
self.remove_unused_consts();
169185
self.remove_nops();
170186

171187
if opts.optimize > 0 {
@@ -209,6 +225,7 @@ impl CodeInfo {
209225

210226
let mut instructions = Vec::new();
211227
let mut locations = Vec::new();
228+
let mut linetable_locations: Vec<LineTableLocation> = Vec::new();
212229

213230
// convert_pseudo_ops: instructions before the main loop
214231
for block in blocks
@@ -315,6 +332,16 @@ impl CodeInfo {
315332
(info.location, info.end_location),
316333
info.arg.instr_size(),
317334
));
335+
// Collect linetable locations with lineno_override support
336+
let lt_loc = LineTableLocation {
337+
line: info
338+
.lineno_override
339+
.unwrap_or_else(|| info.location.line.get() as i32),
340+
end_line: info.end_location.line.get() as i32,
341+
col: info.location.character_offset.to_zero_indexed() as i32,
342+
end_col: info.end_location.character_offset.to_zero_indexed() as i32,
343+
};
344+
linetable_locations.extend(core::iter::repeat_n(lt_loc, info.arg.instr_size()));
318345
instructions.extend(
319346
extras
320347
.map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte))
@@ -330,12 +357,13 @@ impl CodeInfo {
330357
}
331358

332359
instructions.clear();
333-
locations.clear()
360+
locations.clear();
361+
linetable_locations.clear();
334362
}
335363

336-
// Generate linetable from locations
364+
// Generate linetable from linetable_locations (supports line 0 for RESUME)
337365
let linetable = generate_linetable(
338-
&locations,
366+
&linetable_locations,
339367
first_line_number.get() as i32,
340368
opts.debug_ranges,
341369
);
@@ -578,12 +606,72 @@ impl CodeInfo {
578606
};
579607

580608
// Check if it's in small int range: -5 to 256 (_PY_IS_SMALL_INT)
581-
if let Some(small) = value.to_i32() {
582-
if (-5..=256).contains(&small) {
583-
// Convert to LOAD_CONST to LOAD_SMALL_INT
584-
instr.instr = Instruction::LoadSmallInt { idx: Arg::marker() }.into();
585-
// The arg is the i32 value stored as u32 (two's complement)
586-
instr.arg = OpArg(small as u32);
609+
if let Some(small) = value.to_i32().filter(|v| (-5..=256).contains(v)) {
610+
// Convert LOAD_CONST to LOAD_SMALL_INT
611+
instr.instr = Instruction::LoadSmallInt { idx: Arg::marker() }.into();
612+
// The arg is the i32 value stored as u32 (two's complement)
613+
instr.arg = OpArg(small as u32);
614+
}
615+
}
616+
}
617+
}
618+
619+
/// Remove constants that are no longer referenced by LOAD_CONST instructions.
620+
/// remove_unused_consts
621+
fn remove_unused_consts(&mut self) {
622+
let nconsts = self.metadata.consts.len();
623+
if nconsts == 0 {
624+
return;
625+
}
626+
627+
// Mark used constants
628+
// The first constant (index 0) is always kept (may be docstring)
629+
let mut used = vec![false; nconsts];
630+
used[0] = true;
631+
632+
for block in &self.blocks {
633+
for instr in &block.instructions {
634+
if let Some(Instruction::LoadConst { .. }) = instr.instr.real() {
635+
let idx = instr.arg.0 as usize;
636+
if idx < nconsts {
637+
used[idx] = true;
638+
}
639+
}
640+
}
641+
}
642+
643+
// Check if any constants can be removed
644+
let n_used: usize = used.iter().filter(|&&u| u).count();
645+
if n_used == nconsts {
646+
return; // Nothing to remove
647+
}
648+
649+
// Build old_to_new index mapping
650+
let mut old_to_new = vec![0usize; nconsts];
651+
let mut new_idx = 0usize;
652+
for (old_idx, &is_used) in used.iter().enumerate() {
653+
if is_used {
654+
old_to_new[old_idx] = new_idx;
655+
new_idx += 1;
656+
}
657+
}
658+
659+
// Build new consts list
660+
let old_consts: Vec<_> = self.metadata.consts.iter().cloned().collect();
661+
self.metadata.consts.clear();
662+
for (old_idx, constant) in old_consts.into_iter().enumerate() {
663+
if used[old_idx] {
664+
self.metadata.consts.insert(constant);
665+
}
666+
}
667+
668+
// Update LOAD_CONST instruction arguments
669+
for block in &mut self.blocks {
670+
for instr in &mut block.instructions {
671+
if let Some(Instruction::LoadConst { .. }) = instr.instr.real() {
672+
let old_idx = instr.arg.0 as usize;
673+
if old_idx < nconsts {
674+
instr.arg = OpArg(old_to_new[old_idx] as u32);
587675
}
588676
}
589677
}
@@ -759,7 +847,7 @@ fn iter_blocks(blocks: &[Block]) -> impl Iterator<Item = (BlockIdx, &Block)> + '
759847

760848
/// Generate Python 3.11+ format linetable from source locations
761849
fn generate_linetable(
762-
locations: &[(SourceLocation, SourceLocation)],
850+
locations: &[LineTableLocation],
763851
first_line: i32,
764852
debug_ranges: bool,
765853
) -> Box<[u8]> {
@@ -774,7 +862,7 @@ fn generate_linetable(
774862
let mut i = 0;
775863

776864
while i < locations.len() {
777-
let (loc, end_loc) = &locations[i];
865+
let loc = &locations[i];
778866

779867
// Count consecutive instructions with the same location
780868
let mut length = 1;
@@ -787,8 +875,8 @@ fn generate_linetable(
787875
let entry_length = length.min(8);
788876

789877
// Get line information
790-
let line = loc.line.get() as i32;
791-
let end_line = end_loc.line.get() as i32;
878+
let line = loc.line;
879+
let end_line = loc.end_line;
792880
let line_delta = line - prev_line;
793881
let end_line_delta = end_line - line;
794882

@@ -808,8 +896,8 @@ fn generate_linetable(
808896
}
809897

810898
// Get column information (only when debug_ranges is enabled)
811-
let col = loc.character_offset.to_zero_indexed() as i32;
812-
let end_col = end_loc.character_offset.to_zero_indexed() as i32;
899+
let col = loc.col;
900+
let end_col = loc.end_col;
813901

814902
// Choose the appropriate encoding based on line delta and column info
815903
if line_delta == 0 && end_line_delta == 0 {

‎crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap‎

Lines changed: 9 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)