Skip to content

Commit da5727a

Browse files
vstinnervstinner
authored
pythongh-92651: Remove the Include/token.h header file (python#92652)
Remove the token.h header file. There was never any public tokenizer C API. The token.h header file was only designed to be used by Python internals. Move Include/token.h to Include/internal/pycore_token.h. Including this header file now requires that the Py_BUILD_CORE macro is defined. It no longer checks for the Py_LIMITED_API macro. Rename functions: * PyToken_OneChar() => _PyToken_OneChar() * PyToken_TwoChars() => _PyToken_TwoChars() * PyToken_ThreeChars() => _PyToken_ThreeChars()
1 parent b69297e commit da5727a

13 files changed

Lines changed: 52 additions & 39 deletions

File tree

‎Doc/whatsnew/3.12.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,3 +139,8 @@ Deprecated
139139

140140
Removed
141141
-------
142+
143+
* Remove the ``token.h`` header file. There was never any public tokenizer C
144+
API. The ``token.h`` header file was only designed to be used by Python
145+
internals.
146+
(Contributed by Victor Stinner in :gh:`92651`.)

Include/token.h renamed to Include/internal/pycore_token.h

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
11
/* Auto-generated by Tools/scripts/generate_token.py */
22

33
/* Token types */
4-
#ifndef Py_LIMITED_API
5-
#ifndef Py_TOKEN_H
6-
#define Py_TOKEN_H
4+
#ifndef Py_INTERNAL_TOKEN_H
5+
#define Py_INTERNAL_TOKEN_H
76
#ifdef __cplusplus
87
extern "C" {
98
#endif
109

10+
#ifndef Py_BUILD_CORE
11+
# error "this header requires Py_BUILD_CORE define"
12+
#endif
13+
1114
#undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */
1215

1316
#define ENDMARKER 0
@@ -85,13 +88,13 @@ extern "C" {
8588
(x) == DEDENT)
8689

8790

91+
// Symbols exported for test_peg_generator
8892
PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */
89-
PyAPI_FUNC(int) PyToken_OneChar(int);
90-
PyAPI_FUNC(int) PyToken_TwoChars(int, int);
91-
PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);
93+
PyAPI_FUNC(int) _PyToken_OneChar(int);
94+
PyAPI_FUNC(int) _PyToken_TwoChars(int, int);
95+
PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int);
9296

9397
#ifdef __cplusplus
9498
}
9599
#endif
96-
#endif /* !Py_TOKEN_H */
97-
#endif /* Py_LIMITED_API */
100+
#endif // !Py_INTERNAL_TOKEN_H

‎Makefile.pre.in‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1325,11 +1325,11 @@ regen-token:
13251325
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py rst \
13261326
$(srcdir)/Grammar/Tokens \
13271327
$(srcdir)/Doc/library/token-list.inc
1328-
# Regenerate Include/token.h from Grammar/Tokens
1328+
# Regenerate Include/internal/pycore_token.h from Grammar/Tokens
13291329
# using Tools/scripts/generate_token.py
13301330
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py h \
13311331
$(srcdir)/Grammar/Tokens \
1332-
$(srcdir)/Include/token.h
1332+
$(srcdir)/Include/internal/pycore_token.h
13331333
# Regenerate Parser/token.c from Grammar/Tokens
13341334
# using Tools/scripts/generate_token.py
13351335
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py c \
@@ -1521,7 +1521,6 @@ PYTHON_HEADERS= \
15211521
$(srcdir)/Include/structmember.h \
15221522
$(srcdir)/Include/structseq.h \
15231523
$(srcdir)/Include/sysmodule.h \
1524-
$(srcdir)/Include/token.h \
15251524
$(srcdir)/Include/traceback.h \
15261525
$(srcdir)/Include/tracemalloc.h \
15271526
$(srcdir)/Include/tupleobject.h \
@@ -1632,6 +1631,7 @@ PYTHON_HEADERS= \
16321631
$(srcdir)/Include/internal/pycore_structseq.h \
16331632
$(srcdir)/Include/internal/pycore_symtable.h \
16341633
$(srcdir)/Include/internal/pycore_sysmodule.h \
1634+
$(srcdir)/Include/internal/pycore_token.h \
16351635
$(srcdir)/Include/internal/pycore_traceback.h \
16361636
$(srcdir)/Include/internal/pycore_tuple.h \
16371637
$(srcdir)/Include/internal/pycore_typeobject.h \

‎Misc/NEWS.d/next/C API/2022-05-11-02-33-10.gh-issue-92651.FIXLf0.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Remove the ``token.h`` header file. There was never any public tokenizer C
2+
API. The ``token.h`` header file was only designed to be used by Python
3+
internals. Patch by Victor Stinner.

‎PCbuild/pythoncore.vcxproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,7 @@
244244
<ClInclude Include="..\Include\internal\pycore_structseq.h" />
245245
<ClInclude Include="..\Include\internal\pycore_sysmodule.h" />
246246
<ClInclude Include="..\Include\internal\pycore_symtable.h" />
247+
<ClInclude Include="..\Include\internal\pycore_token.h" />
247248
<ClInclude Include="..\Include\internal\pycore_traceback.h" />
248249
<ClInclude Include="..\Include\internal\pycore_tuple.h" />
249250
<ClInclude Include="..\Include\internal\pycore_typeobject.h" />
@@ -291,7 +292,6 @@
291292
<ClInclude Include="..\Include\structseq.h" />
292293
<ClInclude Include="..\Include\symtable.h" />
293294
<ClInclude Include="..\Include\sysmodule.h" />
294-
<ClInclude Include="..\Include\token.h" />
295295
<ClInclude Include="..\Include\traceback.h" />
296296
<ClInclude Include="..\Include\tracemalloc.h" />
297297
<ClInclude Include="..\Include\tupleobject.h" />

‎PCbuild/pythoncore.vcxproj.filters‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -213,9 +213,6 @@
213213
<ClInclude Include="..\Include\sysmodule.h">
214214
<Filter>Include</Filter>
215215
</ClInclude>
216-
<ClInclude Include="..\Include\token.h">
217-
<Filter>Include</Filter>
218-
</ClInclude>
219216
<ClInclude Include="..\Include\traceback.h">
220217
<Filter>Include</Filter>
221218
</ClInclude>
@@ -633,6 +630,9 @@
633630
<ClInclude Include="..\Include\internal\pycore_symtable.h">
634631
<Filter>Include\internal</Filter>
635632
</ClInclude>
633+
<ClInclude Include="..\Include\internal\pycore_token.h">
634+
<Filter>Include\internal</Filter>
635+
</ClInclude>
636636
<ClInclude Include="..\Include\internal\pycore_traceback.h">
637637
<Filter>Include\internal</Filter>
638638
</ClInclude>

‎PCbuild/regen.targets‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
<_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc">
2020
<Format>rst</Format>
2121
</_TokenOutputs>
22-
<_TokenOutputs Include="$(PySourcePath)Include\token.h">
22+
<_TokenOutputs Include="$(PySourcePath)Include\internal\pycore_token.h">
2323
<Format>h</Format>
2424
</_TokenOutputs>
2525
<_TokenOutputs Include="$(PySourcePath)Parser\token.c">

‎Parser/pegen.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33

44
#define PY_SSIZE_T_CLEAN
55
#include <Python.h>
6-
#include <token.h>
76
#include <pycore_ast.h>
7+
#include <pycore_token.h>
88

99
#if 0
1010
#define PyPARSE_YIELD_IS_KEYWORD 0x0001

‎Parser/token.c‎

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Parser/tokenizer.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1992,10 +1992,10 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
19921992
/* Check for two-character token */
19931993
{
19941994
int c2 = tok_nextc(tok);
1995-
int token = PyToken_TwoChars(c, c2);
1995+
int token = _PyToken_TwoChars(c, c2);
19961996
if (token != OP) {
19971997
int c3 = tok_nextc(tok);
1998-
int token3 = PyToken_ThreeChars(c, c2, c3);
1998+
int token3 = _PyToken_ThreeChars(c, c2, c3);
19991999
if (token3 != OP) {
20002000
token = token3;
20012001
}
@@ -2059,7 +2059,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
20592059
/* Punctuation character */
20602060
*p_start = tok->start;
20612061
*p_end = tok->cur;
2062-
return PyToken_OneChar(c);
2062+
return _PyToken_OneChar(c);
20632063
}
20642064

20652065
int

0 commit comments

Comments
 (0)