Skip to content

Commit 713bb19

Browse files
vstinnervstinner
authored
bpo-45434: Mark the PyTokenizer C API as private (pythonGH-28924)
Rename PyTokenize functions to mark them as private: * PyTokenizer_FindEncodingFilename() => _PyTokenizer_FindEncodingFilename() * PyTokenizer_FromString() => _PyTokenizer_FromString() * PyTokenizer_FromFile() => _PyTokenizer_FromFile() * PyTokenizer_FromUTF8() => _PyTokenizer_FromUTF8() * PyTokenizer_Free() => _PyTokenizer_Free() * PyTokenizer_Get() => _PyTokenizer_Get() Remove the unused PyTokenizer_FindEncoding() function. import.c: remove unused #include "errcode.h".
1 parent 3901c08 commit 713bb19

File tree

7 files changed

+40
-48
lines changed

7 files changed

+40
-48
lines changed

‎Parser/pegen.c‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -729,7 +729,7 @@ _PyPegen_fill_token(Parser *p)
729729
{
730730
const char *start;
731731
const char *end;
732-
int type = PyTokenizer_Get(p->tok, &start, &end);
732+
int type = _PyTokenizer_Get(p->tok, &start, &end);
733733

734734
// Record and skip '# type: ignore' comments
735735
while (type == TYPE_IGNORE) {
@@ -746,7 +746,7 @@ _PyPegen_fill_token(Parser *p)
746746
PyErr_NoMemory();
747747
return -1;
748748
}
749-
type = PyTokenizer_Get(p->tok, &start, &end);
749+
type = _PyTokenizer_Get(p->tok, &start, &end);
750750
}
751751

752752
// If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing
@@ -1306,7 +1306,7 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
13061306
for (;;) {
13071307
const char *start;
13081308
const char *end;
1309-
switch (PyTokenizer_Get(p->tok, &start, &end)) {
1309+
switch (_PyTokenizer_Get(p->tok, &start, &end)) {
13101310
case ERRORTOKEN:
13111311
if (p->tok->level != 0) {
13121312
int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
@@ -1411,7 +1411,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
14111411
const char *enc, const char *ps1, const char *ps2,
14121412
PyCompilerFlags *flags, int *errcode, PyArena *arena)
14131413
{
1414-
struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2);
1414+
struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2);
14151415
if (tok == NULL) {
14161416
if (PyErr_Occurred()) {
14171417
raise_tokenizer_init_error(filename_ob);
@@ -1441,7 +1441,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
14411441
_PyPegen_Parser_Free(p);
14421442

14431443
error:
1444-
PyTokenizer_Free(tok);
1444+
_PyTokenizer_Free(tok);
14451445
return result;
14461446
}
14471447

@@ -1453,9 +1453,9 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
14531453

14541454
struct tok_state *tok;
14551455
if (flags == NULL || flags->cf_flags & PyCF_IGNORE_COOKIE) {
1456-
tok = PyTokenizer_FromUTF8(str, exec_input);
1456+
tok = _PyTokenizer_FromUTF8(str, exec_input);
14571457
} else {
1458-
tok = PyTokenizer_FromString(str, exec_input);
1458+
tok = _PyTokenizer_FromString(str, exec_input);
14591459
}
14601460
if (tok == NULL) {
14611461
if (PyErr_Occurred()) {
@@ -1483,7 +1483,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
14831483
_PyPegen_Parser_Free(p);
14841484

14851485
error:
1486-
PyTokenizer_Free(tok);
1486+
_PyTokenizer_Free(tok);
14871487
return result;
14881488
}
14891489

‎Parser/string_parser.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -386,7 +386,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end,
386386
str[0] = '(';
387387
str[len+1] = ')';
388388

389-
struct tok_state* tok = PyTokenizer_FromString(str, 1);
389+
struct tok_state* tok = _PyTokenizer_FromString(str, 1);
390390
if (tok == NULL) {
391391
PyMem_Free(str);
392392
return NULL;
@@ -409,7 +409,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end,
409409
exit:
410410
PyMem_Free(str);
411411
_PyPegen_Parser_Free(p2);
412-
PyTokenizer_Free(tok);
412+
_PyTokenizer_Free(tok);
413413
return result;
414414
}
415415

‎Parser/tokenizer.c‎

Lines changed: 20 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ static char *
108108
error_ret(struct tok_state *tok) /* XXX */
109109
{
110110
tok->decoding_erred = 1;
111-
if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
111+
if (tok->fp != NULL && tok->buf != NULL) /* see _PyTokenizer_Free */
112112
PyMem_Free(tok->buf);
113113
tok->buf = tok->cur = tok->inp = NULL;
114114
tok->start = NULL;
@@ -702,7 +702,7 @@ decode_str(const char *input, int single, struct tok_state *tok)
702702
/* Set up tokenizer for string */
703703

704704
struct tok_state *
705-
PyTokenizer_FromString(const char *str, int exec_input)
705+
_PyTokenizer_FromString(const char *str, int exec_input)
706706
{
707707
struct tok_state *tok = tok_new();
708708
char *decoded;
@@ -711,7 +711,7 @@ PyTokenizer_FromString(const char *str, int exec_input)
711711
return NULL;
712712
decoded = decode_str(str, exec_input, tok);
713713
if (decoded == NULL) {
714-
PyTokenizer_Free(tok);
714+
_PyTokenizer_Free(tok);
715715
return NULL;
716716
}
717717

@@ -723,23 +723,23 @@ PyTokenizer_FromString(const char *str, int exec_input)
723723
/* Set up tokenizer for UTF-8 string */
724724

725725
struct tok_state *
726-
PyTokenizer_FromUTF8(const char *str, int exec_input)
726+
_PyTokenizer_FromUTF8(const char *str, int exec_input)
727727
{
728728
struct tok_state *tok = tok_new();
729729
char *translated;
730730
if (tok == NULL)
731731
return NULL;
732732
tok->input = translated = translate_newlines(str, exec_input, tok);
733733
if (translated == NULL) {
734-
PyTokenizer_Free(tok);
734+
_PyTokenizer_Free(tok);
735735
return NULL;
736736
}
737737
tok->decoding_state = STATE_NORMAL;
738738
tok->enc = NULL;
739739
tok->str = translated;
740740
tok->encoding = new_string("utf-8", 5, tok);
741741
if (!tok->encoding) {
742-
PyTokenizer_Free(tok);
742+
_PyTokenizer_Free(tok);
743743
return NULL;
744744
}
745745

@@ -751,14 +751,14 @@ PyTokenizer_FromUTF8(const char *str, int exec_input)
751751
/* Set up tokenizer for file */
752752

753753
struct tok_state *
754-
PyTokenizer_FromFile(FILE *fp, const char* enc,
755-
const char *ps1, const char *ps2)
754+
_PyTokenizer_FromFile(FILE *fp, const char* enc,
755+
const char *ps1, const char *ps2)
756756
{
757757
struct tok_state *tok = tok_new();
758758
if (tok == NULL)
759759
return NULL;
760760
if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) {
761-
PyTokenizer_Free(tok);
761+
_PyTokenizer_Free(tok);
762762
return NULL;
763763
}
764764
tok->cur = tok->inp = tok->buf;
@@ -771,7 +771,7 @@ PyTokenizer_FromFile(FILE *fp, const char* enc,
771771
gets copied into the parse tree. */
772772
tok->encoding = new_string(enc, strlen(enc), tok);
773773
if (!tok->encoding) {
774-
PyTokenizer_Free(tok);
774+
_PyTokenizer_Free(tok);
775775
return NULL;
776776
}
777777
tok->decoding_state = STATE_NORMAL;
@@ -782,7 +782,7 @@ PyTokenizer_FromFile(FILE *fp, const char* enc,
782782
/* Free a tok_state structure */
783783

784784
void
785-
PyTokenizer_Free(struct tok_state *tok)
785+
_PyTokenizer_Free(struct tok_state *tok)
786786
{
787787
if (tok->encoding != NULL) {
788788
PyMem_Free(tok->encoding);
@@ -2049,7 +2049,8 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
20492049
}
20502050

20512051
int
2052-
PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end)
2052+
_PyTokenizer_Get(struct tok_state *tok,
2053+
const char **p_start, const char **p_end)
20532054
{
20542055
int result = tok_get(tok, p_start, p_end);
20552056
if (tok->decoding_erred) {
@@ -2062,15 +2063,15 @@ PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end)
20622063
/* Get the encoding of a Python file. Check for the coding cookie and check if
20632064
the file starts with a BOM.
20642065
2065-
PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
2066+
_PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
20662067
encoding in the first or second line of the file (in which case the encoding
20672068
should be assumed to be UTF-8).
20682069
20692070
The char* returned is malloc'ed via PyMem_Malloc() and thus must be freed
20702071
by the caller. */
20712072

20722073
char *
2073-
PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
2074+
_PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
20742075
{
20752076
struct tok_state *tok;
20762077
FILE *fp;
@@ -2087,7 +2088,7 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
20872088
if (fp == NULL) {
20882089
return NULL;
20892090
}
2090-
tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
2091+
tok = _PyTokenizer_FromFile(fp, NULL, NULL, NULL);
20912092
if (tok == NULL) {
20922093
fclose(fp);
20932094
return NULL;
@@ -2100,12 +2101,12 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
21002101
tok->filename = PyUnicode_FromString("<string>");
21012102
if (tok->filename == NULL) {
21022103
fclose(fp);
2103-
PyTokenizer_Free(tok);
2104+
_PyTokenizer_Free(tok);
21042105
return encoding;
21052106
}
21062107
}
21072108
while (tok->lineno < 2 && tok->done == E_OK) {
2108-
PyTokenizer_Get(tok, &p_start, &p_end);
2109+
_PyTokenizer_Get(tok, &p_start, &p_end);
21092110
}
21102111
fclose(fp);
21112112
if (tok->encoding) {
@@ -2114,24 +2115,16 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
21142115
strcpy(encoding, tok->encoding);
21152116
}
21162117
}
2117-
PyTokenizer_Free(tok);
2118+
_PyTokenizer_Free(tok);
21182119
return encoding;
21192120
}
21202121

2121-
char *
2122-
PyTokenizer_FindEncoding(int fd)
2123-
{
2124-
return PyTokenizer_FindEncodingFilename(fd, NULL);
2125-
}
2126-
21272122
#ifdef Py_DEBUG
2128-
21292123
void
21302124
tok_dump(int type, char *start, char *end)
21312125
{
21322126
printf("%s", _PyParser_TokenNames[type]);
21332127
if (type == NAME || type == NUMBER || type == STRING || type == OP)
21342128
printf("(%.*s)", (int)(end - start), start);
21352129
}
2136-
2137-
#endif
2130+
#endif // Py_DEBUG

‎Parser/tokenizer.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -86,12 +86,12 @@ struct tok_state {
8686
enum interactive_underflow_t interactive_underflow;
8787
};
8888

89-
extern struct tok_state *PyTokenizer_FromString(const char *, int);
90-
extern struct tok_state *PyTokenizer_FromUTF8(const char *, int);
91-
extern struct tok_state *PyTokenizer_FromFile(FILE *, const char*,
89+
extern struct tok_state *_PyTokenizer_FromString(const char *, int);
90+
extern struct tok_state *_PyTokenizer_FromUTF8(const char *, int);
91+
extern struct tok_state *_PyTokenizer_FromFile(FILE *, const char*,
9292
const char *, const char *);
93-
extern void PyTokenizer_Free(struct tok_state *);
94-
extern int PyTokenizer_Get(struct tok_state *, const char **, const char **);
93+
extern void _PyTokenizer_Free(struct tok_state *);
94+
extern int _PyTokenizer_Get(struct tok_state *, const char **, const char **);
9595

9696
#define tok_dump _Py_tok_dump
9797

‎Python/Python-tokenize.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ tokenizeriter_new_impl(PyTypeObject *type, const char *source)
4747
if (filename == NULL) {
4848
return NULL;
4949
}
50-
self->tok = PyTokenizer_FromUTF8(source, 1);
50+
self->tok = _PyTokenizer_FromUTF8(source, 1);
5151
if (self->tok == NULL) {
5252
Py_DECREF(filename);
5353
return NULL;
@@ -61,7 +61,7 @@ tokenizeriter_next(tokenizeriterobject *it)
6161
{
6262
const char *start;
6363
const char *end;
64-
int type = PyTokenizer_Get(it->tok, &start, &end);
64+
int type = _PyTokenizer_Get(it->tok, &start, &end);
6565
if (type == ERRORTOKEN && PyErr_Occurred()) {
6666
return NULL;
6767
}
@@ -105,7 +105,7 @@ static void
105105
tokenizeriter_dealloc(tokenizeriterobject *it)
106106
{
107107
PyTypeObject *tp = Py_TYPE(it);
108-
PyTokenizer_Free(it->tok);
108+
_PyTokenizer_Free(it->tok);
109109
tp->tp_free(it);
110110
Py_DECREF(tp);
111111
}

‎Python/import.c‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
#include "pycore_interp.h" // _PyInterpreterState_ClearModules()
1212
#include "pycore_pystate.h" // _PyInterpreterState_GET()
1313
#include "pycore_sysmodule.h"
14-
#include "errcode.h"
1514
#include "marshal.h"
1615
#include "code.h"
1716
#include "importdl.h"

‎Python/traceback.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
#define MAX_NTHREADS 100
3030

3131
/* Function from Parser/tokenizer.c */
32-
extern char * PyTokenizer_FindEncodingFilename(int, PyObject *);
32+
extern char* _PyTokenizer_FindEncodingFilename(int, PyObject *);
3333

3434
_Py_IDENTIFIER(TextIOWrapper);
3535
_Py_IDENTIFIER(close);
@@ -431,7 +431,7 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, i
431431
Py_DECREF(binary);
432432
return 0;
433433
}
434-
found_encoding = PyTokenizer_FindEncodingFilename(fd, filename);
434+
found_encoding = _PyTokenizer_FindEncodingFilename(fd, filename);
435435
if (found_encoding == NULL)
436436
PyErr_Clear();
437437
encoding = (found_encoding != NULL) ? found_encoding : "utf-8";

0 commit comments

Comments
 (0)