Skip to content

Commit b3851c8

Browse files
coolreader18coolreader18
committed
Split the ast from the parser, remove compiler dep on parser
1 parent 17e0e75 commit b3851c8

File tree

25 files changed

+419
-285
lines changed

25 files changed

+419
-285
lines changed

‎Cargo.lock‎

Lines changed: 10 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,10 @@ license = "MIT"
99
include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
1010

1111
[workspace]
12-
members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "vm/pylib-crate", "common", "jit"]
12+
members = [
13+
".", "ast", "bytecode", "common", "compiler", "derive",
14+
"jit", "parser", "vm", "vm/pylib-crate", "wasm/lib",
15+
]
1316

1417
[[bench]]
1518
name = "bench"

‎ast/Cargo.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[package]
2+
name = "rustpython-ast"
3+
version = "0.1.0"
4+
authors = ["Noah <33094578+coolreader18@users.noreply.github.com>"]
5+
edition = "2018"
6+
7+
[dependencies]
8+
num-bigint = "0.3"

parser/src/ast.rs renamed to ast/src/ast.rs

File renamed without changes.

‎ast/src/lib.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
mod ast;
2+
mod location;
3+
4+
pub use ast::*;
5+
pub use location::Location;

parser/src/location.rs renamed to ast/src/location.rs

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,33 @@ impl fmt::Display for Location {
1616
}
1717

1818
impl Location {
19-
pub fn visualize(&self, line: &str, desc: &str) -> String {
20-
// desc.to_owned()
21-
format!("{}\n{}\n{}↑", desc, line, " ".repeat(self.column - 1))
19+
pub fn visualize<'a>(
20+
&self,
21+
line: &'a str,
22+
desc: impl fmt::Display + 'a,
23+
) -> impl fmt::Display + 'a {
24+
struct Visualize<'a, D: fmt::Display> {
25+
loc: Location,
26+
line: &'a str,
27+
desc: D,
28+
}
29+
impl<D: fmt::Display> fmt::Display for Visualize<'_, D> {
30+
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
31+
write!(
32+
f,
33+
"{}\n{}\n{arrow:>pad$}",
34+
self.desc,
35+
self.line,
36+
pad = self.loc.column,
37+
arrow = "↑",
38+
)
39+
}
40+
}
41+
Visualize {
42+
loc: *self,
43+
line,
44+
desc,
45+
}
2246
}
2347
}
2448

‎compiler/Cargo.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,11 @@ edition = "2018"
1111
indexmap = "1.0"
1212
itertools = "0.9"
1313
rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
14-
rustpython-parser = { path = "../parser", version = "0.1.1" }
14+
rustpython-ast = { path = "../ast" }
1515
num-complex = { version = "0.3", features = ["serde"] }
1616
log = "0.4"
1717
arrayvec = "0.5"
1818

1919
[dev-dependencies]
20+
rustpython-parser = { path = "../parser" }
2021
insta = "1.1"

‎compiler/src/compile.rs‎

Lines changed: 13 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ use crate::symboltable::{
1212
};
1313
use itertools::Itertools;
1414
use num_complex::Complex64;
15+
use rustpython_ast as ast;
1516
use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Label};
16-
use rustpython_parser::{ast, parser};
1717

1818
type CompileResult<T> = Result<T, CompileError>;
1919

@@ -61,31 +61,6 @@ impl CompileContext {
6161
}
6262
}
6363

64-
/// Compile a given sourcecode into a bytecode object.
65-
pub fn compile(
66-
source: &str,
67-
mode: Mode,
68-
source_path: String,
69-
opts: CompileOpts,
70-
) -> CompileResult<CodeObject> {
71-
let to_compile_error =
72-
|parse_error| CompileError::from_parse_error(parse_error, source_path.clone());
73-
match mode {
74-
Mode::Exec => {
75-
let ast = parser::parse_program(source).map_err(to_compile_error)?;
76-
compile_program(ast, source_path, opts)
77-
}
78-
Mode::Eval => {
79-
let statement = parser::parse_statement(source).map_err(to_compile_error)?;
80-
compile_statement_eval(statement, source_path, opts)
81-
}
82-
Mode::Single => {
83-
let ast = parser::parse_program(source).map_err(to_compile_error)?;
84-
compile_program_single(ast, source_path, opts)
85-
}
86-
}
87-
}
88-
8964
/// A helper function for the shared code of the different compile functions
9065
fn with_compiler(
9166
source_path: String,
@@ -106,8 +81,10 @@ pub fn compile_program(
10681
source_path: String,
10782
opts: CompileOpts,
10883
) -> CompileResult<CodeObject> {
109-
let symbol_table = make_symbol_table(&ast)
110-
.map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?;
84+
let symbol_table = match make_symbol_table(&ast) {
85+
Ok(x) => x,
86+
Err(e) => return Err(e.into_compile_error(source_path)),
87+
};
11188
with_compiler(source_path, opts, |compiler| {
11289
compiler.compile_program(&ast, symbol_table)
11390
})
@@ -119,8 +96,10 @@ pub fn compile_statement_eval(
11996
source_path: String,
12097
opts: CompileOpts,
12198
) -> CompileResult<CodeObject> {
122-
let symbol_table = statements_to_symbol_table(&statement)
123-
.map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?;
99+
let symbol_table = match statements_to_symbol_table(&statement) {
100+
Ok(x) => x,
101+
Err(e) => return Err(e.into_compile_error(source_path)),
102+
};
124103
with_compiler(source_path, opts, |compiler| {
125104
compiler.compile_statement_eval(&statement, symbol_table)
126105
})
@@ -132,8 +111,10 @@ pub fn compile_program_single(
132111
source_path: String,
133112
opts: CompileOpts,
134113
) -> CompileResult<CodeObject> {
135-
let symbol_table = make_symbol_table(&ast)
136-
.map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?;
114+
let symbol_table = match make_symbol_table(&ast) {
115+
Ok(x) => x,
116+
Err(e) => return Err(e.into_compile_error(source_path)),
117+
};
137118
with_compiler(source_path, opts, |compiler| {
138119
compiler.compile_program_single(&ast, symbol_table)
139120
})
@@ -165,7 +146,6 @@ impl Compiler {
165146
error,
166147
location,
167148
source_path: self.source_path.clone(),
168-
statement: None,
169149
}
170150
}
171151

‎compiler/src/error.rs‎

Lines changed: 30 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,24 @@
1-
use rustpython_parser::error::{LexicalErrorType, ParseError, ParseErrorType};
2-
use rustpython_parser::location::Location;
3-
use rustpython_parser::token::Tok;
1+
use rustpython_ast::Location;
42

53
use std::error::Error;
64
use std::fmt;
75

86
#[derive(Debug)]
97
pub struct CompileError {
10-
pub statement: Option<String>,
118
pub error: CompileErrorType,
129
pub location: Location,
1310
pub source_path: String,
1411
}
1512

16-
impl CompileError {
17-
pub fn from_parse_error(parse_error: ParseError, source_path: String) -> Self {
18-
Self {
19-
statement: None,
20-
error: CompileErrorType::Parse(parse_error.error),
21-
location: parse_error.location,
22-
source_path,
23-
}
24-
}
25-
26-
pub fn update_statement_info(&mut self, statement: String) {
27-
self.statement = Some(statement);
28-
}
29-
}
30-
3113
#[derive(Debug)]
14+
#[non_exhaustive]
3215
pub enum CompileErrorType {
3316
/// Invalid assignment, cannot store value in target.
3417
Assign(&'static str),
3518
/// Invalid delete
3619
Delete(&'static str),
3720
/// Expected an expression got a statement
3821
ExpectExpr,
39-
/// Parser error
40-
Parse(ParseErrorType),
4122
SyntaxError(String),
4223
/// Multiple `*` detected
4324
MultipleStarArgs,
@@ -57,74 +38,43 @@ pub enum CompileErrorType {
5738
InvalidFutureFeature(String),
5839
}
5940

60-
impl CompileError {
61-
pub fn is_indentation_error(&self) -> bool {
62-
if let CompileErrorType::Parse(parse) = &self.error {
63-
match parse {
64-
ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true,
65-
ParseErrorType::UnrecognizedToken(token, expected) => {
66-
*token == Tok::Indent || expected.clone() == Some("Indent".to_owned())
67-
}
68-
_ => false,
69-
}
70-
} else {
71-
false
72-
}
73-
}
74-
75-
pub fn is_tab_error(&self) -> bool {
76-
if let CompileErrorType::Parse(parse) = &self.error {
77-
if let ParseErrorType::Lexical(lex) = parse {
78-
if let LexicalErrorType::TabError = lex {
79-
return true;
80-
}
81-
}
82-
}
83-
false
84-
}
85-
}
86-
87-
impl fmt::Display for CompileError {
41+
impl fmt::Display for CompileErrorType {
8842
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
89-
let error_desc = match &self.error {
90-
CompileErrorType::Assign(target) => format!("can't assign to {}", target),
91-
CompileErrorType::Delete(target) => format!("can't delete {}", target),
92-
CompileErrorType::ExpectExpr => "Expecting expression, got statement".to_owned(),
93-
CompileErrorType::Parse(err) => err.to_string(),
94-
CompileErrorType::SyntaxError(err) => err.to_string(),
43+
match self {
44+
CompileErrorType::Assign(target) => write!(f, "can't assign to {}", target),
45+
CompileErrorType::Delete(target) => write!(f, "can't delete {}", target),
46+
CompileErrorType::ExpectExpr => write!(f, "Expecting expression, got statement"),
47+
CompileErrorType::SyntaxError(err) => write!(f, "{}", err.as_str()),
9548
CompileErrorType::MultipleStarArgs => {
96-
"two starred expressions in assignment".to_owned()
49+
write!(f, "two starred expressions in assignment")
9750
}
98-
CompileErrorType::InvalidStarExpr => "can't use starred expression here".to_owned(),
99-
CompileErrorType::InvalidBreak => "'break' outside loop".to_owned(),
100-
CompileErrorType::InvalidContinue => "'continue' outside loop".to_owned(),
101-
CompileErrorType::InvalidReturn => "'return' outside function".to_owned(),
102-
CompileErrorType::InvalidYield => "'yield' outside function".to_owned(),
103-
CompileErrorType::InvalidYieldFrom => "'yield from' outside function".to_owned(),
104-
CompileErrorType::InvalidAwait => "'await' outside async function".to_owned(),
105-
CompileErrorType::AsyncYieldFrom => "'yield from' inside async function".to_owned(),
51+
CompileErrorType::InvalidStarExpr => write!(f, "can't use starred expression here"),
52+
CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"),
53+
CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"),
54+
CompileErrorType::InvalidReturn => write!(f, "'return' outside function"),
55+
CompileErrorType::InvalidYield => write!(f, "'yield' outside function"),
56+
CompileErrorType::InvalidYieldFrom => write!(f, "'yield from' outside function"),
57+
CompileErrorType::InvalidAwait => write!(f, "'await' outside async function"),
58+
CompileErrorType::AsyncYieldFrom => write!(f, "'yield from' inside async function"),
10659
CompileErrorType::AsyncReturnValue => {
107-
"'return' with value inside async generator".to_owned()
108-
}
109-
CompileErrorType::InvalidFuturePlacement => {
110-
"from __future__ imports must occur at the beginning of the file".to_owned()
60+
write!(f, "'return' with value inside async generator")
11161
}
62+
CompileErrorType::InvalidFuturePlacement => write!(
63+
f,
64+
"from __future__ imports must occur at the beginning of the file"
65+
),
11266
CompileErrorType::InvalidFutureFeature(feat) => {
113-
format!("future feature {} is not defined", feat)
114-
}
115-
};
116-
117-
if let Some(statement) = &self.statement {
118-
if self.location.column() > 0 {
119-
if let Some(line) = statement.lines().nth(self.location.row() - 1) {
120-
// visualize the error, when location and statement are provided
121-
return write!(f, "{}", self.location.visualize(line, &error_desc));
122-
}
67+
write!(f, "future feature {} is not defined", feat)
12368
}
12469
}
70+
}
71+
}
12572

126-
// print line number
127-
write!(f, "{} at {}", error_desc, self.location)
73+
impl Error for CompileErrorType {}
74+
75+
impl fmt::Display for CompileError {
76+
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
77+
write!(f, "{} at {}", self.error, self.location)
12878
}
12979
}
13080

‎compiler/src/mode.rs‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
use rustpython_parser::parser;
2-
31
#[derive(Clone, Copy)]
42
pub enum Mode {
53
Exec,
@@ -19,15 +17,6 @@ impl std::str::FromStr for Mode {
1917
}
2018
}
2119

22-
impl Mode {
23-
pub fn to_parser_mode(self) -> parser::Mode {
24-
match self {
25-
Mode::Exec | Mode::Single => parser::Mode::Program,
26-
Mode::Eval => parser::Mode::Statement,
27-
}
28-
}
29-
}
30-
3120
#[derive(Debug)]
3221
pub struct ModeParseError {
3322
_priv: (),

0 commit comments

Comments
 (0)