Skip to content

gh-97997: Add col_offset field to tokenizer and use that for AST nodes #98000

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pablogsal merged 5 commits into from
Oct 7, 2022
Merged

gh-97997: Add col_offset field to tokenizer and use that for AST nodes #98000

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
faefec5
gh-97997: Add col_offset field to tokenizer and use that for AST nodes
lysnikolaou Oct 6, 2022
d3f852c
📜🤖 Added by blurb_it.
blurb-it[bot] Oct 6, 2022
5c47419
Remove unnecessary assignment of starting_col_offset
lysnikolaou Oct 7, 2022
4796a40
Add comment
lysnikolaou Oct 7, 2022
d128a37
Address feedback; add macro to advance new line
lysnikolaou Oct 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Address feedback; add macro to advance new line
  • Loading branch information
@lysnikolaou
lysnikolaou committed Oct 7, 2022
commit d128a37eae72ce5eafa3536f421668ee1b1bbbbc
15 changes: 7 additions & 8 deletions Parser/tokenizer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@
#define MAKE_TOKEN(token_type) token_setup(tok, token, token_type, p_start, p_end)
#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\
type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end))
#define ADVANCE_LINENO() \
tok->lineno++; \
tok->col_offset = 0;

/* Forward */
static struct tok_state *tok_new(void);
Expand Down Expand Up @@ -875,8 +878,7 @@ tok_underflow_string(struct tok_state *tok) {
tok->buf = tok->cur;
}
tok->line_start = tok->cur;
tok->lineno++;
tok->col_offset = 0;
ADVANCE_LINENO();
tok->inp = end;
return 1;
}
Expand Down Expand Up @@ -935,8 +937,7 @@ tok_underflow_interactive(struct tok_state *tok) {
else if (tok->start != NULL) {
Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf;
size_t size = strlen(newtok);
tok->lineno++;
tok->col_offset = 0;
ADVANCE_LINENO();
if (!tok_reserve_buf(tok, size + 1)) {
PyMem_Free(tok->buf);
tok->buf = NULL;
Expand All @@ -949,8 +950,7 @@ tok_underflow_interactive(struct tok_state *tok) {
tok->multi_line_start = tok->buf + cur_multi_line_start;
}
else {
tok->lineno++;
tok->col_offset = 0;
ADVANCE_LINENO();
PyMem_Free(tok->buf);
tok->buf = newtok;
tok->cur = tok->buf;
Expand Down Expand Up @@ -1005,8 +1005,7 @@ tok_underflow_file(struct tok_state *tok) {
*tok->inp = '\0';
}

tok->lineno++;
tok->col_offset = 0;
ADVANCE_LINENO();
if (tok->decoding_state != STATE_NORMAL) {
if (tok->lineno > 2) {
tok->decoding_state = STATE_NORMAL;
Expand Down