diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 9efd459a6f0..71f192f90d9 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -58,5 +58,41 @@ def test_no_memory(self): # Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr. self.assertIn(p.returncode, (1, 120)) + @cpython_only + def test_multiline_string_parsing(self): + # bpo-39209: Multiline string tokens need to be handled in the tokenizer + # in two places: the interactive path and the non-interactive path. + user_input = '''\ + x = """ + + + + + 0KiB + 0 + 1.3 + 0 + + + 16738211KiB + 237.15 + 1.3 + 0 + + never + none + + + """ + ''' + user_input = dedent(user_input) + user_input = user_input.encode() + p = spawn_repl() + with SuppressCrashReport(): + p.stdin.write(user_input) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst new file mode 100644 index 00000000000..c05b3f8dfa4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst @@ -0,0 +1,2 @@ +Correctly handle multi-line tokens in interactive mode. Patch by Pablo +Galindo. diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index f84093dae5b..f73c32684c7 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -886,6 +886,7 @@ tok_nextc(struct tok_state *tok) size_t start = tok->start - tok->buf; size_t oldlen = tok->cur - tok->buf; size_t newlen = oldlen + strlen(newtok); + Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf; char *buf = tok->buf; buf = (char *)PyMem_REALLOC(buf, newlen+1); tok->lineno++; @@ -898,6 +899,7 @@ tok_nextc(struct tok_state *tok) } tok->buf = buf; tok->cur = tok->buf + oldlen; + tok->multi_line_start = tok->buf + cur_multi_line_start; tok->line_start = tok->cur; strcpy(tok->buf + oldlen, newtok); PyMem_FREE(newtok);