fix line continuation parsing

palaviv · palaviv · commit ad4beab6f67d · 2020-01-24T21:50:19.000+02:00
diff --git a/parser/src/error.rs b/parser/src/error.rs
@@ -27,6 +27,7 @@ pub enum LexicalErrorType {
     DuplicateKeywordArgumentError,
     UnrecognizedToken { tok: char },
     FStringError(FStringErrorType),
+    LineContinuationError,
     OtherError(String),
 }
 
@@ -55,6 +56,9 @@ impl fmt::Display for LexicalErrorType {
             LexicalErrorType::UnrecognizedToken { tok } => {
                 write!(f, "Got unexpected token {}", tok)
             }
+            LexicalErrorType::LineContinuationError => {
+                write!(f, "unexpected character after line continuation character")
+            }
             LexicalErrorType::OtherError(msg) => write!(f, "{}", msg),
         }
     }
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
@@ -117,8 +117,7 @@ pub type LexResult = Result<Spanned, LexicalError>;
 
 pub fn make_tokenizer<'a>(source: &'a str) -> impl Iterator<Item = LexResult> + 'a {
     let nlh = NewlineHandler::new(source.chars());
-    let lch = LineContinuationHandler::new(nlh);
-    Lexer::new(lch)
+    Lexer::new(nlh)
 }
 
 // The newline handler is an iterator which collapses different newline
@@ -178,61 +177,6 @@ where
     }
 }
 
-// Glues \ and \n into a single line:
-pub struct LineContinuationHandler<T: Iterator<Item = char>> {
-    source: T,
-    chr0: Option<char>,
-    chr1: Option<char>,
-}
-
-impl<T> LineContinuationHandler<T>
-where
-    T: Iterator<Item = char>,
-{
-    pub fn new(source: T) -> Self {
-        let mut nlh = LineContinuationHandler {
-            source,
-            chr0: None,
-            chr1: None,
-        };
-        nlh.shift();
-        nlh.shift();
-        nlh
-    }
-
-    fn shift(&mut self) -> Option<char> {
-        let result = self.chr0;
-        self.chr0 = self.chr1;
-        self.chr1 = self.source.next();
-        result
-    }
-}
-
-impl<T> Iterator for LineContinuationHandler<T>
-where
-    T: Iterator<Item = char>,
-{
-    type Item = char;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        // Collapse \r\n into \n
-        loop {
-            if self.chr0 == Some('\\') && self.chr1 == Some('\n') {
-                // Skip backslash and newline
-                self.shift();
-                self.shift();
-            // Idea: insert trailing newline here:
-            // } else if self.chr0 != Some('\n') && self.chr1.is_none() {
-            //     self.chr1 = Some('\n');
-            } else {
-                break;
-            }
-        }
-
-        self.shift()
-    }
-}
-
 impl<T> Lexer<T>
 where
     T: Iterator<Item = char>,
@@ -1201,6 +1145,18 @@ where
                     self.next_char();
                 }
             }
+            '\\' => {
+                self.next_char();
+                if let Some('\n') = self.chr0 {
+                    self.next_char();
+                } else {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::LineContinuationError,
+                        location: self.get_pos(),
+                    });
+                }
+            }
+
             _ => {
                 let c = self.next_char();
                 return Err(LexicalError {
diff --git a/tests/snippets/funky_syntax.py b/tests/snippets/funky_syntax.py
@@ -1,3 +1,4 @@
+import traceback
 
 a = 2
 b = 2 + 4 if a < 5 else 'boe'
@@ -12,3 +13,10 @@
 assert e(True) == 1
 assert e(False) == 0
 
+try:
+	a = "aaaa" + \
+		"bbbb"
+	1/0
+except ZeroDivisionError as ex:
+	tb = traceback.extract_tb(ex.__traceback__)
+	assert tb[0].lineno == 19
diff --git a/tests/snippets/invalid_syntax.py b/tests/snippets/invalid_syntax.py
@@ -51,3 +51,12 @@ def valid_func():
 
 with assert_raises(SyntaxError):
     compile('0xX', 'test.py', 'exec')
+
+
+src = """
+"aaaa" \a
+"bbbb"
+"""
+
+with assert_raises(SyntaxError):
+    compile(src, 'test.py', 'exec')

Original file line number	Diff line number	Diff line change
`@@ -27,6 +27,7 @@ pub enum LexicalErrorType {`
`27`	`27`	`DuplicateKeywordArgumentError,`
`28`	`28`	`UnrecognizedToken { tok: char },`
`29`	`29`	`FStringError(FStringErrorType),`
	`30`	`+ LineContinuationError,`
`30`	`31`	`OtherError(String),`
`31`	`32`	`}`
`32`	`33`
`@@ -55,6 +56,9 @@ impl fmt::Display for LexicalErrorType {`
`55`	`56`	`LexicalErrorType::UnrecognizedToken { tok } => {`
`56`	`57`	`write!(f, "Got unexpected token {}", tok)`
`57`	`58`	`}`
	`59`	`+ LexicalErrorType::LineContinuationError => {`
	`60`	`+ write!(f, "unexpected character after line continuation character")`
	`61`	`+ }`
`58`	`62`	`LexicalErrorType::OtherError(msg) => write!(f, "{}", msg),`
`59`	`63`	`}`
`60`	`64`	`}`