Reduce lalrpop generated code size by about half. Relates to issue RustPython#173.

windelbouwman · windelbouwman · commit 940c41bda87d · 2018-10-21T16:01:39.000+02:00
diff --git a/parser/src/ast.rs b/parser/src/ast.rs
@@ -12,6 +12,13 @@ pub struct Node {
 }
 */
 
+#[derive(Debug, PartialEq)]
+pub enum Top {
+    Program(Program),
+    Statement(LocatedStatement),
+    Expression(Expression),
+}
+
 #[derive(Debug, PartialEq)]
 pub struct Program {
     pub statements: Vec<LocatedStatement>,
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
@@ -3,11 +3,13 @@ extern crate lalrpop_util;
 use std::error::Error;
 use std::fs::File;
 use std::io::Read;
+use std::iter;
 use std::path::Path;
 
 use super::ast;
 use super::lexer;
 use super::python;
+use super::token;
 
 pub fn read_file(filename: &Path) -> Result<String, String> {
     match File::open(&filename) {
@@ -40,32 +42,35 @@ pub fn parse(filename: &Path) -> Result<ast::Program, String> {
     }
 }
 
+macro_rules! do_lalr_parsing {
+    ($input: expr, $pat: ident, $tok: ident) => {{
+        let lxr = lexer::Lexer::new($input);
+        let marker_token = (Default::default(), token::Tok::$tok, Default::default());
+        let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
+
+        match python::TopParser::new().parse(tokenizer) {
+            Err(why) => Err(format!("{:?}", why)),
+            Ok(top) => {
+                if let ast::Top::$pat(x) = top {
+                    Ok(x)
+                } else {
+                    unreachable!()
+                }
+            }
+        }
+    }};
+}
+
 pub fn parse_program(source: &str) -> Result<ast::Program, String> {
-    let lxr = lexer::Lexer::new(&source);
-    match python::ProgramParser::new().parse(lxr) {
-        Err(lalrpop_util::ParseError::UnrecognizedToken {
-            token: None,
-            expected: _,
-        }) => Err(String::from("Unexpected end of input.")),
-        Err(why) => Err(String::from(format!("{:?}", why))),
-        Ok(p) => Ok(p),
-    }
+    do_lalr_parsing!(source, Program, StartProgram)
 }
 
 pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, String> {
-    let lxr = lexer::Lexer::new(&source);
-    match python::StatementParser::new().parse(lxr) {
-        Err(why) => Err(String::from(format!("{:?}", why))),
-        Ok(p) => Ok(p),
-    }
+    do_lalr_parsing!(source, Statement, StartStatement)
 }
 
 pub fn parse_expression(source: &str) -> Result<ast::Expression, String> {
-    let lxr = lexer::Lexer::new(&source);
-    match python::ExpressionParser::new().parse(lxr) {
-        Err(why) => Err(String::from(format!("{:?}", why))),
-        Ok(p) => Ok(p),
-    }
+    do_lalr_parsing!(source, Expression, StartExpression)
 }
 
 #[cfg(test)]
diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop
@@ -11,7 +11,16 @@ use std::str::FromStr;
 
 grammar;
 
-pub Program: ast::Program = {
+// This is a hack to reduce the amount of lalrpop tables generated:
+// For each public entry point, a full parse table is generated.
+// By having only a single pub function, we reduce this to one.
+pub Top: ast::Top = {
+    StartProgram <p:Program> => ast::Top::Program(p),
+    StartStatement <s:Statement> => ast::Top::Statement(s),
+    StartExpression <e:Expression> => ast::Top::Expression(e),
+};
+
+Program: ast::Program = {
   <lines:FileLine*> => ast::Program { statements: Vec::from_iter(lines.into_iter().filter_map(|e| e)) },
 };
 
@@ -26,7 +35,7 @@ Suite: Vec<ast::LocatedStatement> = {
   "\n" indent <s:Statement+> dedent => s,
 };
 
-pub Statement: ast::LocatedStatement = {
+Statement: ast::LocatedStatement = {
   SimpleStatement,
   CompoundStatement,
 };
@@ -608,7 +617,7 @@ CompOp: ast::Comparison = {
   "is" "not" => ast::Comparison::IsNot,
 };
 
-pub Expression: ast::Expression = {
+Expression: ast::Expression = {
   <e1:Expression> "|" <e2:XorExpression> => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) },
   <e:XorExpression> => e,
 };
@@ -914,6 +923,9 @@ extern {
     enum lexer::Tok {
         indent => lexer::Tok::Indent,
         dedent => lexer::Tok::Dedent,
+        StartProgram => lexer::Tok::StartProgram,
+        StartStatement => lexer::Tok::StartStatement,
+        StartExpression => lexer::Tok::StartExpression,
         "+" => lexer::Tok::Plus,
         "-" => lexer::Tok::Minus,
         ":" => lexer::Tok::Colon,
diff --git a/parser/src/token.rs b/parser/src/token.rs
@@ -7,6 +7,9 @@ pub enum Tok {
     Newline,
     Indent,
     Dedent,
+    StartProgram,
+    StartStatement,
+    StartExpression,
     Lpar,
     Rpar,
     Lsqb,
diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs
@@ -21,14 +21,16 @@ pub struct CodeObject {
     pub instructions: Vec<Instruction>,
     pub label_map: HashMap<Label, usize>,
     pub locations: Vec<ast::Location>,
-    pub arg_names: Vec<String>,
+    pub arg_names: Vec<String>, // Names of positional arguments
+    pub varargs: Option<String>,
     pub source_path: Option<String>,
     pub obj_name: String, // Name of the object that created this code object
 }
 
 impl CodeObject {
     pub fn new(
         arg_names: Vec<String>,
+        varargs: Option<String>,
         source_path: Option<String>,
         obj_name: String,
     ) -> CodeObject {
@@ -37,6 +39,7 @@ impl CodeObject {
             label_map: HashMap::new(),
             locations: Vec::new(),
             arg_names: arg_names,
+            varargs: varargs,
             source_path: source_path,
             obj_name: obj_name,
         }
diff --git a/vm/src/compile.rs b/vm/src/compile.rs
@@ -83,8 +83,12 @@ impl Compiler {
     }
 
     fn push_new_code_object(&mut self, source_path: Option<String>, obj_name: String) {
-        self.code_object_stack
-            .push(CodeObject::new(Vec::new(), source_path.clone(), obj_name));
+        self.code_object_stack.push(CodeObject::new(
+            Vec::new(),
+            None,
+            source_path.clone(),
+            obj_name,
+        ));
     }
 
     fn pop_code_object(&mut self) -> CodeObject {
@@ -384,30 +388,15 @@ impl Compiler {
                 decorator_list,
             } => {
                 // Create bytecode for this function:
-                let have_kwargs = args.defaults.len() > 0;
-                if have_kwargs {
-                    // Construct a tuple:
-                    let size = args.defaults.len();
-                    for element in &args.defaults {
-                        self.compile_expression(element)?;
-                    }
-                    self.emit(Instruction::BuildTuple { size });
-                }
-
-                self.code_object_stack.push(CodeObject::new(
-                    args.args.clone(),
-                    self.source_path.clone(),
-                    name.clone(),
-                ));
+                let flags = self.enter_function(name, args)?;
                 self.compile_statements(body)?;
 
                 // Emit None at end:
                 self.emit(Instruction::LoadConst {
                     value: bytecode::Constant::None,
                 });
                 self.emit(Instruction::ReturnValue);
-
-                let code = self.code_object_stack.pop().unwrap();
+                let code = self.pop_code_object();
 
                 self.prepare_decorators(decorator_list)?;
                 self.emit(Instruction::LoadConst {
@@ -420,10 +409,6 @@ impl Compiler {
                 });
 
                 // Turn code object into function object:
-                let mut flags = bytecode::FunctionOpArg::empty();
-                if have_kwargs {
-                    flags = flags | bytecode::FunctionOpArg::HAS_DEFAULTS;
-                }
                 self.emit(Instruction::MakeFunction { flags: flags });
                 self.apply_decorators(decorator_list);
 
@@ -442,6 +427,7 @@ impl Compiler {
                 self.emit(Instruction::LoadBuildClass);
                 self.code_object_stack.push(CodeObject::new(
                     vec![String::from("__locals__")],
+                    None,
                     self.source_path.clone(),
                     name.clone(),
                 ));
@@ -455,7 +441,7 @@ impl Compiler {
                 });
                 self.emit(Instruction::ReturnValue);
 
-                let code = self.code_object_stack.pop().unwrap();
+                let code = self.pop_code_object();
                 self.emit(Instruction::LoadConst {
                     value: bytecode::Constant::Code { code: code },
                 });
@@ -587,6 +573,36 @@ impl Compiler {
         Ok(())
     }
 
+    fn enter_function(
+        &mut self,
+        name: &String,
+        args: &ast::Parameters,
+    ) -> Result<bytecode::FunctionOpArg, String> {
+        let have_kwargs = args.defaults.len() > 0;
+        if have_kwargs {
+            // Construct a tuple:
+            let size = args.defaults.len();
+            for element in &args.defaults {
+                self.compile_expression(element)?;
+            }
+            self.emit(Instruction::BuildTuple { size });
+        }
+
+        self.code_object_stack.push(CodeObject::new(
+            args.args.clone(),
+            args.vararg.clone(),
+            self.source_path.clone(),
+            name.clone(),
+        ));
+
+        let mut flags = bytecode::FunctionOpArg::empty();
+        if have_kwargs {
+            flags = flags | bytecode::FunctionOpArg::HAS_DEFAULTS;
+        }
+
+        Ok(flags)
+    }
+
     fn prepare_decorators(&mut self, decorator_list: &Vec<ast::Expression>) -> Result<(), String> {
         for decorator in decorator_list {
             self.compile_expression(decorator)?;
@@ -886,26 +902,19 @@ impl Compiler {
                 });
             }
             ast::Expression::Lambda { args, body } => {
-                self.code_object_stack.push(CodeObject::new(
-                    args.args.clone(),
-                    self.source_path.clone(),
-                    "<lambda>".to_string(),
-                ));
+                let name = "<lambda>".to_string();
+                let flags = self.enter_function(&name, args)?;
                 self.compile_expression(body)?;
                 self.emit(Instruction::ReturnValue);
-                let code = self.code_object_stack.pop().unwrap();
+                let code = self.pop_code_object();
                 self.emit(Instruction::LoadConst {
                     value: bytecode::Constant::Code { code: code },
                 });
                 self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::String {
-                        value: String::from("<lambda>"),
-                    },
+                    value: bytecode::Constant::String { value: name },
                 });
                 // Turn code object into function object:
-                self.emit(Instruction::MakeFunction {
-                    flags: bytecode::FunctionOpArg::empty(),
-                });
+                self.emit(Instruction::MakeFunction { flags: flags });
             }
             ast::Expression::Comprehension { kind, generators } => {
                 self.compile_comprehension(kind, generators)?;
@@ -943,6 +952,7 @@ impl Compiler {
         // Create magnificent function <listcomp>:
         self.code_object_stack.push(CodeObject::new(
             vec![".0".to_string()],
+            None,
             self.source_path.clone(),
             name,
         ));
@@ -1033,7 +1043,7 @@ impl Compiler {
         self.emit(Instruction::ReturnValue);
 
         // Fetch code for listcomp function:
-        let code = self.code_object_stack.pop().unwrap();
+        let code = self.pop_code_object();
 
         // List comprehension code:
         self.emit(Instruction::LoadConst {
diff --git a/vm/src/vm.rs b/vm/src/vm.rs
