Skip to content

Commit ed4557f

Browse files
windelbouwmanwindelbouwman
committed
Added cool script to test snippets in both implementations
1 parent 62cb3e2 commit ed4557f

File tree

6 files changed

+310
-87
lines changed

6 files changed

+310
-87
lines changed

‎parser/src/lexer.rs‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -180,21 +180,8 @@ impl<'input> Lexer<'input> {
180180
};
181181
c
182182
}
183-
}
184-
185-
/* Implement iterator pattern for the get_tok function.
186-
187-
Calling the next element in the iterator will yield the next lexical
188-
token.
189-
*/
190-
impl<'input> Iterator for Lexer<'input> {
191-
type Item = Spanned<Tok>;
192-
193-
fn next(&mut self) -> Option<Self::Item> {
194-
// Idea: create some sort of hash map for single char tokens:
195-
// let mut X = HashMap::new();
196-
// X.insert('=', Tok::Equal);
197183

184+
fn inner_next(&mut self) -> Option<Spanned<Tok>> {
198185
// Detect indentation levels
199186
loop {
200187
if self.at_begin_of_line {
@@ -225,7 +212,14 @@ impl<'input> Iterator for Lexer<'input> {
225212
return Some(Ok((0, Tok::Indent, 0)));
226213
} else if col < current_indentation {
227214
// One or more dedentations
215+
// Pop off other levels until col is found:
216+
217+
let l2 = self.indentation_stack.pop().unwrap();
228218
return Some(Ok((0, Tok::Dedent, 0)));
219+
if col == l2 {
220+
// TODO: handle wrong indentations
221+
}
222+
229223
}
230224
}
231225
}
@@ -504,6 +498,25 @@ impl<'input> Iterator for Lexer<'input> {
504498
}
505499
}
506500

501+
/* Implement iterator pattern for the get_tok function.
502+
503+
Calling the next element in the iterator will yield the next lexical
504+
token.
505+
*/
506+
impl<'input> Iterator for Lexer<'input> {
507+
type Item = Spanned<Tok>;
508+
509+
fn next(&mut self) -> Option<Self::Item> {
510+
// Idea: create some sort of hash map for single char tokens:
511+
// let mut X = HashMap::new();
512+
// X.insert('=', Tok::Equal);
513+
let token = self.inner_next();
514+
trace!("Lex token {:?}, nesting={:?}, indent stack: {:?}", token, self.nesting, self.indentation_stack);
515+
token
516+
}
517+
518+
}
519+
507520
#[cfg(test)]
508521
mod tests {
509522
use super::Tok;
@@ -538,7 +551,7 @@ mod tests {
538551

539552
#[test]
540553
fn test_indentation() {
541-
let source = String::from("def foo():\n return 99\n");
554+
let source = String::from("def foo():\n return 99\n\n");
542555
let tokens = lex_source(&source);
543556
assert_eq!(
544557
tokens,
@@ -556,6 +569,7 @@ mod tests {
556569
Tok::Number { value: 99 },
557570
Tok::Newline,
558571
Tok::Dedent,
572+
Tok::Newline,
559573
]
560574
);
561575
}

‎py_code_object/test_snippets.py‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@
1414

1515

1616
logger = logging.getLogger('tests')
17-
THIS_DIR = '.'
18-
TEST_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', 'tests'))
19-
CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(THIS_DIR))
17+
ROOT_DIR = '..'
18+
TEST_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'tests'))
19+
CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'py_code_object'))
20+
RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR))
2021

2122

2223
@contextlib.contextmanager
@@ -27,8 +28,17 @@ def pushd(path):
2728
os.chdir(old_dir)
2829

2930

30-
def perform_test(filename):
31-
logger.info('Running %s', filename)
31+
def perform_test(filename, method):
32+
logger.info('Running %s via %s', filename, method)
33+
if method == 'cpython':
34+
run_via_cpython(filename)
35+
elif method == 'rustpython':
36+
run_via_rustpython(filename)
37+
else:
38+
raise NotImplementedError(method)
39+
40+
41+
def run_via_cpython(filename):
3242
# Step1: Create bytecode file:
3343
bytecode_filename = filename + '.bytecode'
3444
with open(bytecode_filename, 'w') as f:
@@ -42,15 +52,23 @@ def perform_test(filename):
4252
subprocess.check_call(['cargo', 'run', bytecode_filename], env=env)
4353

4454

45-
def create_test_function(cls, filename):
55+
def run_via_rustpython(filename):
56+
env = os.environ.copy()
57+
env['RUST_LOG'] = 'debug'
58+
env['RUST_BACKTRACE'] = '1'
59+
with pushd(RUSTPYTHON_RUNNER_DIR):
60+
subprocess.check_call(['cargo', 'run', filename], env=env)
61+
62+
63+
def create_test_function(cls, filename, method):
4664
""" Create a test function for a single snippet """
4765
core_test_directory, snippet_filename = os.path.split(filename)
48-
test_function_name = 'test_' \
66+
test_function_name = 'test_{}_'.format(method) \
4967
+ os.path.splitext(snippet_filename)[0] \
5068
.replace('.', '_').replace('-', '_')
5169

5270
def test_function(self):
53-
perform_test(filename)
71+
perform_test(filename, method)
5472

5573
if hasattr(cls, test_function_name):
5674
raise ValueError('Duplicate test case {}'.format(test_function_name))
@@ -59,8 +77,9 @@ def test_function(self):
5977

6078
def populate(cls):
6179
""" Decorator function which can populate a unittest.TestCase class """
62-
for filename in get_test_files():
63-
create_test_function(cls, filename)
80+
for method in ['cpython', 'rustpython']:
81+
for filename in get_test_files():
82+
create_test_function(cls, filename, method)
6483
return cls
6584

6685

‎src/compile.rs‎

Lines changed: 44 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,28 +9,35 @@ use rustpython_parser::ast;
99
use rustpython_vm::bytecode::{self, CodeObject, Instruction};
1010

1111
struct Compiler {
12-
code_object: CodeObject,
12+
code_object_stack: Vec<CodeObject>,
1313
nxt_label: usize,
1414
}
1515

1616
pub fn compile(p: ast::Program) -> CodeObject {
1717
let mut compiler = Compiler::new();
18-
compiler.compile_program(p);
19-
compiler.code_object
18+
compiler.compile_program(p)
2019
}
2120

2221
type Label = usize;
2322

2423
impl Compiler {
2524
fn new() -> Self {
2625
Compiler {
27-
code_object: CodeObject::new(),
26+
code_object_stack: Vec::new(),
2827
nxt_label: 0,
2928
}
3029
}
3130

32-
fn compile_program(&mut self, program: ast::Program) {
31+
fn compile_program(&mut self, program: ast::Program) -> CodeObject {
32+
self.code_object_stack.push(CodeObject::new());
3333
self.compile_statements(program.statements);
34+
assert!(self.code_object_stack.len() == 1);
35+
36+
// Emit None at end:
37+
self.emit(Instruction::LoadConst { value: bytecode::Constant::None });
38+
self.emit(Instruction::ReturnValue);
39+
40+
self.code_object_stack.pop().unwrap()
3441
}
3542

3643
fn compile_statements(&mut self, statements: Vec<ast::Statement>) {
@@ -39,20 +46,6 @@ impl Compiler {
3946
}
4047
}
4148

42-
// Generate a new label
43-
fn new_label(&mut self) -> Label {
44-
let l = self.nxt_label;
45-
self.nxt_label += 1;
46-
l
47-
}
48-
49-
// Assign current position the given label
50-
fn set_label(&mut self, label: Label) {
51-
let position = self.code_object.instructions.len();
52-
// assert!(label not in self.label_map)
53-
self.code_object.label_map.insert(label, position);
54-
}
55-
5649
fn compile_statement(&mut self, statement: ast::Statement) {
5750
trace!("Compiling {:?}", statement);
5851
match statement {
@@ -125,7 +118,16 @@ impl Compiler {
125118
self.emit(Instruction::PopBlock);
126119
}
127120
ast::Statement::FunctionDef { name, body } => {
121+
// Create bytecode for this function:
122+
self.code_object_stack.push(CodeObject::new());
128123
self.compile_statements(body);
124+
let code = self.code_object_stack.pop().unwrap();
125+
self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: code } });
126+
self.emit(Instruction::LoadConst { value: bytecode::Constant::String { value: name.clone() } });
127+
128+
// Turn code object into function object:
129+
self.emit(Instruction::MakeFunction);
130+
self.emit(Instruction::StoreName { name: name });
129131
}
130132
ast::Statement::ClassDef { name } => {
131133
// TODO?
@@ -136,6 +138,8 @@ impl Compiler {
136138
self.compile_expression(test);
137139

138140
// if true, jump over raise:
141+
let end_label = self.new_label();
142+
self.emit(Instruction::JumpIf { target: end_label } );
139143

140144
self.emit(Instruction::LoadName {
141145
name: String::from("AssertionError"),
@@ -149,7 +153,7 @@ impl Compiler {
149153
self.emit(Instruction::CallFunction { count: 0 });
150154
}
151155
}
152-
// TODO?
156+
self.set_label(end_label);
153157
}
154158
ast::Statement::Break => {
155159
self.emit(Instruction::Break);
@@ -285,7 +289,26 @@ impl Compiler {
285289
}
286290
}
287291

292+
// Low level helper functions:
288293
fn emit(&mut self, instruction: Instruction) {
289-
self.code_object.instructions.push(instruction);
294+
self.current_code_object().instructions.push(instruction);
295+
}
296+
297+
fn current_code_object(&mut self) -> &mut CodeObject {
298+
self.code_object_stack.last_mut().unwrap()
299+
}
300+
301+
// Generate a new label
302+
fn new_label(&mut self) -> Label {
303+
let l = self.nxt_label;
304+
self.nxt_label += 1;
305+
l
306+
}
307+
308+
// Assign current position the given label
309+
fn set_label(&mut self, label: Label) {
310+
let position = self.current_code_object().instructions.len();
311+
// assert!(label not in self.label_map)
312+
self.current_code_object().label_map.insert(label, position);
290313
}
291314
}

‎vm/src/bytecode.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ pub enum Instruction {
4646
Break,
4747
Jump { target: Label },
4848
JumpIf { target: Label },
49-
MakeFunction { code: CodeObject },
49+
MakeFunction,
5050
CallFunction { count: usize },
5151
ForIter,
5252
ReturnValue,
@@ -64,6 +64,7 @@ pub enum Constant {
6464
Integer { value: i32 }, // TODO: replace by arbitrary big int math.
6565
// TODO: Float { value: f64 },
6666
String { value: String },
67+
Code { code: CodeObject },
6768
None,
6869
}
6970

‎vm/src/pyobject.rs‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ pub enum PyObject {
2828
Integer {
2929
value: i32,
3030
},
31+
Boolean {
32+
value: bool,
33+
},
3134
List {
3235
elements: Vec<PyObjectRef>,
3336
},
@@ -40,11 +43,16 @@ pub enum PyObject {
4043
iterated_obj: PyObjectRef,
4144
},
4245
Slice {
43-
elements: Vec<PyObjectRef>,
46+
start: Option<i32>,
47+
stop: Option<i32>,
48+
step: Option<i32>,
4449
},
4550
Code {
4651
code: bytecode::CodeObject,
4752
},
53+
Function {
54+
code: bytecode::CodeObject,
55+
},
4856
None,
4957
RustFunction {
5058
function: fn(Vec<PyObjectRef>),
@@ -59,10 +67,11 @@ impl PyObjectRef {
5967
}*/
6068

6169
impl PyObject {
62-
pub fn call(&self, args: Vec<PyObjectRef>) {
70+
pub fn call(&self, args: Vec<PyObjectRef>) -> PyObjectRef {
6371
match *self {
6472
PyObject::RustFunction { ref function } => {
6573
function(args);
74+
PyObject::None.into_ref()
6675
}
6776
_ => {
6877
println!("Not impl {:?}", self);
@@ -215,6 +224,32 @@ impl<'a> Mul<&'a PyObject> for &'a PyObject {
215224
}
216225
}
217226

227+
// impl<'a> PartialEq<&'a PyObject> for &'a PyObject {
228+
impl PartialEq for PyObject {
229+
fn eq(&self, other: &PyObject) -> bool {
230+
match (self, other) {
231+
(&PyObject::Integer { value: ref v1i }, &PyObject::Integer { value: ref v2i }) => {
232+
v2i == v1i
233+
},
234+
/*
235+
(&NativeType::Float(ref v1f), &NativeType::Float(ref v2f)) => {
236+
curr_frame.stack.push(Rc::new(NativeType::Boolean(v2f == v1f)));
237+
},
238+
(&NativeType::Str(ref v1s), &NativeType::Str(ref v2s)) => {
239+
curr_frame.stack.push(Rc::new(NativeType::Boolean(v2s == v1s)));
240+
},
241+
(&NativeType::Int(ref v1i), &NativeType::Float(ref v2f)) => {
242+
curr_frame.stack.push(Rc::new(NativeType::Boolean(v2f == &(*v1i as f64))));
243+
},
244+
(&NativeType::List(ref l1), &NativeType::List(ref l2)) => {
245+
curr_frame.stack.push(Rc::new(NativeType::Boolean(l2 == l1)));
246+
},
247+
*/
248+
_ => panic!("TypeError in COMPARE_OP: can't compare {:?} with {:?}", self, other)
249+
}
250+
}
251+
}
252+
218253
#[cfg(test)]
219254
mod tests {
220255
use super::PyObject;

0 commit comments

Comments
 (0)