From 94372304cc1ef65bac283efbfefd68e87436fefb Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Fri, 17 May 2019 18:34:47 -0700 Subject: [PATCH 1/3] Treat line continuation at EOF as a `SyntaxError` This makes the parser consistent with the tokenize module (already the case in `pypy`). sample ------ ```python x = 5\ ``` before ------ ```console $ python3 t.py $ python3 -mtokenize t.py t.py:2:0: error: EOF in multi-line statement ``` after ----- ```console $ ./python t.py File "t.py", line 3 x = 5\ ^ SyntaxError: unexpected EOF while parsing $ ./python -m tokenize t.py t.py:2:0: error: EOF in multi-line statement ``` --- Lib/test/test_eof.py | 6 ++++++ .../2019-05-17-18-34-30.bpo-2180.aBqHeW.rst | 1 + Parser/tokenizer.c | 11 ++++++++++- 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-05-17-18-34-30.bpo-2180.aBqHeW.rst diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index 7baa7ae57c6ff4..7b2b2b66d04a9c 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -24,5 +24,11 @@ def test_EOFS(self): else: raise support.TestFailed + def test_line_continuation_EOF(self): + expect = 'unexpected EOF while parsing (, line 1)' + with self.assertRaises(SyntaxError) as excinfo: + exec('x = 5\\') + self.assertEqual(str(excinfo.exception), expect) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-17-18-34-30.bpo-2180.aBqHeW.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-17-18-34-30.bpo-2180.aBqHeW.rst new file mode 100644 index 00000000000000..a2207c17aea0a8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-17-18-34-30.bpo-2180.aBqHeW.rst @@ -0,0 +1 @@ +Treat line continuation at EOF as a ``SyntaxError`` by Anthony Sottile. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 5dc2ae65c42daf..e52d498d5542d5 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -983,7 +983,8 @@ tok_nextc(struct tok_state *tok) return EOF; /* Last line does not end in \n, fake one */ - strcpy(tok->inp, "\n"); + if (tok->inp[-1] != '\n') + strcpy(tok->inp, "\n"); } tok->inp = strchr(tok->inp, '\0'); done = tok->inp[-1] == '\n'; @@ -1674,6 +1675,14 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) tok->cur = tok->inp; return ERRORTOKEN; } + c = tok_nextc(tok); + if (c == EOF) { + tok->done = E_EOF; + tok->cur = tok->inp; + return ERRORTOKEN; + } else { + tok_backup(tok, c); + } tok->cont_line = 1; goto again; /* Read next line */ } From e054eaa2edec03c07fc6e27f17b9382db673bd66 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 18 May 2019 00:11:56 -0700 Subject: [PATCH 2/3] Add a file based test to exercise tok_nextc(). --- Lib/test/test_eof.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index 7b2b2b66d04a9c..b9328a4db9cd21 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -1,7 +1,10 @@ """test script for a few new invalid token catches""" -import unittest +import subprocess +import sys +import tempfile from test import support +import unittest class EOFTestCase(unittest.TestCase): def test_EOFC(self): @@ -25,10 +28,32 @@ def test_EOFS(self): raise support.TestFailed def test_line_continuation_EOF(self): + """A contination at the end of input must be an error; bpo2180.""" expect = 'unexpected EOF while parsing (, line 1)' with self.assertRaises(SyntaxError) as excinfo: exec('x = 5\\') self.assertEqual(str(excinfo.exception), expect) + with self.assertRaises(SyntaxError) as excinfo: + exec('\\') + self.assertEqual(str(excinfo.exception), expect) + + @unittest.skipIf(not sys.executable, "sys.executable required") + def test_line_continuation_EOF_from_file_bpo2180(self): + """Ensure tok_nextc() does not add too many ending newlines.""" + with tempfile.NamedTemporaryFile() as temp_f: + temp_f.write(b'\\') + temp_f.flush() + proc = subprocess.run([sys.executable, temp_f.name], + capture_output=True) + self.assertIn(b'unexpected EOF while parsing', proc.stderr) + self.assertGreater(proc.returncode, 0) + temp_f.seek(0) + temp_f.write(b'y = 6\\') + temp_f.flush() + proc = subprocess.run([sys.executable, temp_f.name], + capture_output=True) + self.assertIn(b'unexpected EOF while parsing', proc.stderr) + self.assertGreater(proc.returncode, 0) if __name__ == "__main__": unittest.main() From 657ca9fe3b37c48ca9f38145d66f9a359dc95dd3 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sat, 18 May 2019 10:52:45 -0700 Subject: [PATCH 3/3] Switch to script_helper to fix windows tests --- Lib/test/test_eof.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index b9328a4db9cd21..a091ceaa25bc4f 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -1,9 +1,8 @@ """test script for a few new invalid token catches""" -import subprocess import sys -import tempfile from test import support +from test.support import script_helper import unittest class EOFTestCase(unittest.TestCase): @@ -40,20 +39,14 @@ def test_line_continuation_EOF(self): @unittest.skipIf(not sys.executable, "sys.executable required") def test_line_continuation_EOF_from_file_bpo2180(self): """Ensure tok_nextc() does not add too many ending newlines.""" - with tempfile.NamedTemporaryFile() as temp_f: - temp_f.write(b'\\') - temp_f.flush() - proc = subprocess.run([sys.executable, temp_f.name], - capture_output=True) - self.assertIn(b'unexpected EOF while parsing', proc.stderr) - self.assertGreater(proc.returncode, 0) - temp_f.seek(0) - temp_f.write(b'y = 6\\') - temp_f.flush() - proc = subprocess.run([sys.executable, temp_f.name], - capture_output=True) - self.assertIn(b'unexpected EOF while parsing', proc.stderr) - self.assertGreater(proc.returncode, 0) + with support.temp_dir() as temp_dir: + file_name = script_helper.make_script(temp_dir, 'foo', '\\') + rc, out, err = script_helper.assert_python_failure(file_name) + self.assertIn(b'unexpected EOF while parsing', err) + + file_name = script_helper.make_script(temp_dir, 'foo', 'y = 6\\') + rc, out, err = script_helper.assert_python_failure(file_name) + self.assertIn(b'unexpected EOF while parsing', err) if __name__ == "__main__": unittest.main()