Skip to content

Commit 4453c3e

Browse files
jordy25519jordy25519
committed
Create error types for parser and compiler
Pass through locations and tokens in ParseError
1 parent 233242c commit 4453c3e

File tree

14 files changed

+283
-136
lines changed

14 files changed

+283
-136
lines changed

‎parser/src/error.rs‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
//! Define internal parse error types
2+
//! The goal is to provide a matching and a safe error API, maksing errors from LALR
3+
extern crate lalrpop_util;
4+
use self::lalrpop_util::ParseError as InnerError;
5+
6+
use lexer::{Location, LexicalError};
7+
use token::Tok;
8+
9+
use std::error::Error;
10+
use std::fmt;
11+
12+
// A token of type `Tok` was observed, with a span given by the two Location values
13+
type TokSpan = (Location, Tok, Location);
14+
15+
/// Represents an error during parsing
16+
#[derive(Debug)]
17+
pub enum ParseError {
18+
/// Parser encountered an unexpected end of input
19+
EOF(Option<Location>),
20+
/// Parser encountered an extra token
21+
ExtraToken(TokSpan),
22+
/// Parser encountered an invalid token
23+
InvalidToken(Location),
24+
/// Parser encountered an unexpected token
25+
UnrecognizedToken(TokSpan, Vec<String>),
26+
/// Maps to `User` type from `lalrpop-util`
27+
Other,
28+
}
29+
30+
/// Convert `lalrpop_util::ParseError` to our internal type
31+
impl From<InnerError<Location, Tok, LexicalError>> for ParseError {
32+
fn from(err: InnerError<Location, Tok, LexicalError>) -> Self {
33+
match err {
34+
// TODO: Are there cases where this isn't an EOF?
35+
InnerError::InvalidToken { location } => ParseError::EOF(Some(location)),
36+
InnerError::ExtraToken { token } => ParseError::ExtraToken(token),
37+
// Inner field is a unit-like enum `LexicalError::StringError` with no useful info
38+
InnerError::User { ..} => ParseError::Other,
39+
InnerError::UnrecognizedToken { token, expected } => {
40+
match token {
41+
Some(tok) => ParseError::UnrecognizedToken(tok, expected),
42+
// EOF was observed when it was unexpected
43+
None => ParseError::EOF(None),
44+
}
45+
}
46+
}
47+
}
48+
}
49+
50+
impl fmt::Display for ParseError {
51+
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
52+
match *self {
53+
ParseError::EOF(ref location) => {
54+
if let Some(l) = location {
55+
write!(f, "Got unexpected EOF at: {:?}", l)
56+
} else {
57+
write!(f, "Got unexpected EOF")
58+
}
59+
}
60+
ParseError::ExtraToken(ref t_span) => {
61+
write!(f, "Got extraneous token: {:?} at: {:?}", t_span.1, t_span.0)
62+
},
63+
ParseError::InvalidToken(ref location) => write!(f, "Got invalid token at: {:?}", location),
64+
ParseError::UnrecognizedToken(ref t_span, _) => {
65+
write!(f, "Got unexpected token: {:?} at {:?}", t_span.1, t_span.0)
66+
}
67+
// This is user defined, it probably means a more useful error should have been given upstream.
68+
ParseError::Other => write!(f, "Got unsupported token(s)"),
69+
}
70+
}
71+
}
72+
73+
impl Error for ParseError {
74+
fn source(&self) -> Option<&(dyn Error + 'static)> {
75+
None
76+
}
77+
}

‎parser/src/lib.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,9 @@ extern crate num_bigint;
55
extern crate num_traits;
66

77
pub mod ast;
8+
pub mod error;
89
pub mod lexer;
910
pub mod parser;
1011
#[cfg_attr(rustfmt, rustfmt_skip)]
1112
mod python;
1213
pub mod token;
13-
14-
pub use self::parser::parse;

‎parser/src/parser.rs‎

Lines changed: 5 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,27 @@
11
extern crate lalrpop_util;
22

3-
use std::error::Error;
4-
use std::fs::File;
5-
use std::io::Read;
63
use std::iter;
7-
use std::path::Path;
84

95
use super::ast;
6+
use super::error::ParseError;
107
use super::lexer;
118
use super::python;
129
use super::token;
1310

14-
pub fn read_file(filename: &Path) -> Result<String, String> {
15-
info!("Loading file {:?}", filename);
16-
match File::open(&filename) {
17-
Ok(mut file) => {
18-
let mut s = String::new();
19-
20-
match file.read_to_string(&mut s) {
21-
Err(why) => Err(String::from("Reading file failed: ") + why.description()),
22-
Ok(_) => Ok(s),
23-
}
24-
}
25-
Err(why) => Err(String::from("Opening file failed: ") + why.description()),
26-
}
27-
}
28-
2911
/*
3012
* Parse python code.
3113
* Grammar may be inspired by antlr grammar for python:
3214
* https://github.com/antlr/grammars-v4/tree/master/python3
3315
*/
3416

35-
pub fn parse(filename: &Path) -> Result<ast::Program, String> {
36-
info!("Parsing: {}", filename.display());
37-
let txt = read_file(filename)?;
38-
debug!("Read contents of file: {}", txt);
39-
parse_program(&txt)
40-
}
41-
4217
macro_rules! do_lalr_parsing {
4318
($input: expr, $pat: ident, $tok: ident) => {{
4419
let lxr = lexer::make_tokenizer($input);
4520
let marker_token = (Default::default(), token::Tok::$tok, Default::default());
4621
let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
4722

4823
match python::TopParser::new().parse(tokenizer) {
49-
Err(why) => Err(format!("{:?}", why)),
24+
Err(err) => Err(ParseError::from(err)),
5025
Ok(top) => {
5126
if let ast::Top::$pat(x) = top {
5227
Ok(x)
@@ -58,11 +33,11 @@ macro_rules! do_lalr_parsing {
5833
}};
5934
}
6035

61-
pub fn parse_program(source: &str) -> Result<ast::Program, String> {
36+
pub fn parse_program(source: &str) -> Result<ast::Program, ParseError> {
6237
do_lalr_parsing!(source, Program, StartProgram)
6338
}
6439

65-
pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, String> {
40+
pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, ParseError> {
6641
do_lalr_parsing!(source, Statement, StartStatement)
6742
}
6843

@@ -88,7 +63,7 @@ pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, String> {
8863
/// expr);
8964
///
9065
/// ```
91-
pub fn parse_expression(source: &str) -> Result<ast::Expression, String> {
66+
pub fn parse_expression(source: &str) -> Result<ast::Expression, ParseError> {
9267
do_lalr_parsing!(source, Expression, StartExpression)
9368
}
9469

‎src/main.rs‎

Lines changed: 38 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
//extern crate rustpython_parser;
21
#[macro_use]
32
extern crate clap;
43
extern crate env_logger;
@@ -9,16 +8,21 @@ extern crate rustpython_vm;
98
extern crate rustyline;
109

1110
use clap::{App, Arg};
12-
use rustpython_parser::parser;
13-
use rustpython_vm::obj::objstr;
14-
use rustpython_vm::print_exception;
15-
use rustpython_vm::pyobject::{AttributeProtocol, PyObjectRef, PyResult};
16-
use rustpython_vm::VirtualMachine;
17-
use rustpython_vm::{compile, import};
18-
use rustyline::error::ReadlineError;
19-
use rustyline::Editor;
20-
use std::path::Path;
21-
use std::path::PathBuf;
11+
use rustpython_parser::error::ParseError;
12+
use rustpython_vm::{
13+
compile,
14+
error::CompileError,
15+
import,
16+
obj::objstr,
17+
print_exception,
18+
pyobject::{AttributeProtocol, PyObjectRef, PyResult},
19+
util, VirtualMachine,
20+
};
21+
use rustyline::{error::ReadlineError, Editor};
22+
use std::{
23+
error::Error,
24+
path::{Path, PathBuf},
25+
};
2226

2327
fn main() {
2428
env_logger::init();
@@ -69,7 +73,16 @@ fn main() {
6973
}
7074

7175
fn _run_string(vm: &mut VirtualMachine, source: &str, source_path: String) -> PyResult {
72-
let code_obj = compile::compile(vm, source, &compile::Mode::Exec, source_path)?;
76+
let code_obj = compile::compile(
77+
source,
78+
&compile::Mode::Exec,
79+
source_path,
80+
vm.ctx.code_type(),
81+
)
82+
.map_err(|err| {
83+
let syntax_error = vm.context().exceptions.syntax_error.clone();
84+
vm.new_exception(syntax_error, err.description().to_string())
85+
})?;
7386
// trace!("Code object: {:?}", code_obj.borrow());
7487
let builtins = vm.get_builtin_scope();
7588
let vars = vm.context().new_scope(Some(builtins)); // Keep track of local variables
@@ -100,35 +113,31 @@ fn run_module(vm: &mut VirtualMachine, module: &str) -> PyResult {
100113
fn run_script(vm: &mut VirtualMachine, script_file: &str) -> PyResult {
101114
debug!("Running file {}", script_file);
102115
// Parse an ast from it:
103-
let filepath = Path::new(script_file);
104-
match parser::read_file(filepath) {
105-
Ok(source) => _run_string(vm, &source, filepath.to_str().unwrap().to_string()),
106-
Err(msg) => {
107-
error!("Parsing went horribly wrong: {}", msg);
116+
let file_path = Path::new(script_file);
117+
match util::read_file(file_path) {
118+
Ok(source) => _run_string(vm, &source, file_path.to_str().unwrap().to_string()),
119+
Err(err) => {
120+
error!("Failed reading file: {:?}", err.kind());
108121
std::process::exit(1);
109122
}
110123
}
111124
}
112125

113126
fn shell_exec(vm: &mut VirtualMachine, source: &str, scope: PyObjectRef) -> bool {
114-
match compile::compile(vm, source, &compile::Mode::Single, "<stdin>".to_string()) {
127+
match compile::compile(source, &compile::Mode::Single, "<stdin>".to_string(), vm.ctx.code_type()) {
115128
Ok(code) => {
116129
if let Err(err) = vm.run_code_obj(code, scope) {
117130
print_exception(vm, &err);
118131
}
119132
}
133+
// Don't inject syntax errors for line continuation
134+
Err(CompileError::Parse(ParseError::EOF(_))) => {
135+
return false;
136+
}
120137
Err(err) => {
121-
// Enum rather than special string here.
122-
let name = vm.new_str("msg".to_string());
123-
let msg = match vm.get_attribute(err.clone(), name) {
124-
Ok(value) => objstr::get_value(&value),
125-
Err(_) => panic!("Expected msg attribute on exception object!"),
126-
};
127-
if msg == "Unexpected end of input." {
128-
return false;
129-
} else {
130-
print_exception(vm, &err);
131-
}
138+
let syntax_error = vm.context().exceptions.syntax_error.clone();
139+
let exc = vm.new_exception(syntax_error, format!("{}", err));
140+
print_exception(vm, &exc);
132141
}
133142
};
134143
true

‎src/util.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
use std::fs::File;
2+
use std::io::{Read, Result};
3+
use std::path::Path;
4+
5+
/// Read a file at `path` into a String
6+
pub fn read_file(path: &Path) -> Result<String> {
7+
info!("Loading file {:?}", path);
8+
let mut f = File::open(&path)?;
9+
let mut buffer = String::new();
10+
f.read_to_string(&mut buffer)?;
11+
12+
Ok(buffer)
13+
}

‎vm/src/builtins.rs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
55
// use std::ops::Deref;
66
use std::char;
7+
use std::error::Error;
78
use std::io::{self, Write};
89

910
use super::compile;
@@ -149,7 +150,10 @@ fn builtin_compile(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
149150

150151
let filename = objstr::get_value(filename);
151152

152-
compile::compile(vm, &source, &mode, filename)
153+
compile::compile(&source, &mode, filename, vm.ctx.code_type()).map_err(|err| {
154+
let syntax_error = vm.context().exceptions.syntax_error.clone();
155+
vm.new_exception(syntax_error, err.description().to_string())
156+
})
153157
}
154158

155159
fn builtin_delattr(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
@@ -199,7 +203,10 @@ fn builtin_eval(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
199203
let source = objstr::get_value(source);
200204
// TODO: fix this newline bug:
201205
let source = format!("{}\n", source);
202-
compile::compile(vm, &source, &mode, "<string>".to_string())?
206+
compile::compile(&source, &mode, "<string>".to_string(), vm.ctx.code_type()).map_err(|err| {
207+
let syntax_error = vm.context().exceptions.syntax_error.clone();
208+
vm.new_exception(syntax_error, err.description().to_string())
209+
})?
203210
} else {
204211
return Err(vm.new_type_error("code argument must be str or code object".to_string()));
205212
};
@@ -245,7 +252,10 @@ fn builtin_exec(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
245252
let source = objstr::get_value(source);
246253
// TODO: fix this newline bug:
247254
let source = format!("{}\n", source);
248-
compile::compile(vm, &source, &mode, "<string>".to_string())?
255+
compile::compile(&source, &mode, "<string>".to_string(), vm.ctx.code_type()).map_err(|err| {
256+
let syntax_error = vm.context().exceptions.syntax_error.clone();
257+
vm.new_exception(syntax_error, err.description().to_string())
258+
})?
249259
} else if objtype::isinstance(source, &vm.ctx.code_type()) {
250260
source.clone()
251261
} else {

0 commit comments

Comments
 (0)