Skip to content

Commit 442f209

Browse files
benjaminpbenjaminp
committed
create NameConstant AST class for None, True, and False literals (closes #16619)
1 parent 4b237e3 commit 442f209

15 files changed

Lines changed: 152 additions & 72 deletions

File tree

‎Include/Python-ast.h‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,8 +182,9 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
182182
SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
183183
Yield_kind=12, YieldFrom_kind=13, Compare_kind=14,
184184
Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18,
185-
Ellipsis_kind=19, Attribute_kind=20, Subscript_kind=21,
186-
Starred_kind=22, Name_kind=23, List_kind=24, Tuple_kind=25};
185+
NameConstant_kind=19, Ellipsis_kind=20, Attribute_kind=21,
186+
Subscript_kind=22, Starred_kind=23, Name_kind=24,
187+
List_kind=25, Tuple_kind=26};
187188
struct _expr {
188189
enum _expr_kind kind;
189190
union {
@@ -278,6 +279,10 @@ struct _expr {
278279
bytes s;
279280
} Bytes;
280281

282+
struct {
283+
singleton value;
284+
} NameConstant;
285+
281286
struct {
282287
expr_ty value;
283288
identifier attr;
@@ -509,6 +514,9 @@ expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena);
509514
expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena);
510515
#define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3)
511516
expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena);
517+
#define NameConstant(a0, a1, a2, a3) _Py_NameConstant(a0, a1, a2, a3)
518+
expr_ty _Py_NameConstant(singleton value, int lineno, int col_offset, PyArena
519+
*arena);
512520
#define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2)
513521
expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena);
514522
#define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)

‎Include/asdl.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ typedef PyObject * identifier;
55
typedef PyObject * string;
66
typedef PyObject * bytes;
77
typedef PyObject * object;
8+
typedef PyObject * singleton;
89

910
/* It would be nice if the code generated by asdl_c.py was completely
1011
independent of Python, but it is a goal the requires too much work

‎Lib/ast.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ def literal_eval(node_or_string):
4242
Python literal structures: strings, bytes, numbers, tuples, lists, dicts,
4343
sets, booleans, and None.
4444
"""
45-
_safe_names = {'None': None, 'True': True, 'False': False}
4645
if isinstance(node_or_string, str):
4746
node_or_string = parse(node_or_string, mode='eval')
4847
if isinstance(node_or_string, Expression):
@@ -61,9 +60,8 @@ def _convert(node):
6160
elif isinstance(node, Dict):
6261
return dict((_convert(k), _convert(v)) for k, v
6362
in zip(node.keys, node.values))
64-
elif isinstance(node, Name):
65-
if node.id in _safe_names:
66-
return _safe_names[node.id]
63+
elif isinstance(node, NameConstant):
64+
return node.value
6765
elif isinstance(node, UnaryOp) and \
6866
isinstance(node.op, (UAdd, USub)) and \
6967
isinstance(node.operand, (Num, UnaryOp, BinOp)):

‎Lib/test/test_ast.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -928,6 +928,9 @@ def test_list(self):
928928
def test_tuple(self):
929929
self._sequence(ast.Tuple)
930930

931+
def test_nameconstant(self):
932+
self.expr(ast.NameConstant(4), "singleton must be True, False, or None")
933+
931934
def test_stdlib_validates(self):
932935
stdlib = os.path.dirname(ast.__file__)
933936
tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")]
@@ -959,13 +962,13 @@ def main():
959962

960963
#### EVERYTHING BELOW IS GENERATED #####
961964
exec_results = [
962-
('Module', [('Expr', (1, 0), ('Name', (1, 0), 'None', ('Load',)))]),
965+
('Module', [('Expr', (1, 0), ('NameConstant', (1, 0), None))]),
963966
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Pass', (1, 9))], [], None)]),
964967
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None)], None, None, [], None, None, [], []), [('Pass', (1, 10))], [], None)]),
965968
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None)], None, None, [], None, None, [('Num', (1, 8), 0)], []), [('Pass', (1, 12))], [], None)]),
966969
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], 'args', None, [], None, None, [], []), [('Pass', (1, 14))], [], None)]),
967970
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], 'kwargs', None, [], []), [('Pass', (1, 17))], [], None)]),
968-
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None), ('arg', 'b', None), ('arg', 'c', None), ('arg', 'd', None), ('arg', 'e', None)], 'args', None, [], 'kwargs', None, [('Num', (1, 11), 1), ('Name', (1, 16), 'None', ('Load',)), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])], []), [('Pass', (1, 52))], [], None)]),
971+
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None), ('arg', 'b', None), ('arg', 'c', None), ('arg', 'd', None), ('arg', 'e', None)], 'args', None, [], 'kwargs', None, [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])], []), [('Pass', (1, 52))], [], None)]),
969972
('Module', [('ClassDef', (1, 0), 'C', [], [], None, None, [('Pass', (1, 8))], [])]),
970973
('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], None, None, [('Pass', (1, 17))], [])]),
971974
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
@@ -1002,14 +1005,14 @@ def main():
10021005
('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
10031006
]
10041007
eval_results = [
1005-
('Expression', ('Name', (1, 0), 'None', ('Load',))),
1008+
('Expression', ('NameConstant', (1, 0), None)),
10061009
('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
10071010
('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
10081011
('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
1009-
('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('Name', (1, 7), 'None', ('Load',)))),
1012+
('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('NameConstant', (1, 7), None))),
10101013
('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
10111014
('Expression', ('Dict', (1, 0), [], [])),
1012-
('Expression', ('Set', (1, 0), [('Name', (1, 1), 'None', ('Load',))])),
1015+
('Expression', ('Set', (1, 0), [('NameConstant', (1, 1), None)])),
10131016
('Expression', ('Dict', (1, 0), [('Num', (2, 6), 1)], [('Num', (4, 10), 2)])),
10141017
('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
10151018
('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),

‎Lib/test/test_syntax.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
3434
>>> None = 1
3535
Traceback (most recent call last):
36-
SyntaxError: assignment to keyword
36+
SyntaxError: can't assign to keyword
3737
3838
It's a syntax error to assign to the empty tuple. Why isn't it an
3939
error to assign to the empty list? It will always raise some error at
@@ -233,7 +233,7 @@
233233
SyntaxError: can't assign to generator expression
234234
>>> None += 1
235235
Traceback (most recent call last):
236-
SyntaxError: assignment to keyword
236+
SyntaxError: can't assign to keyword
237237
>>> f() += 1
238238
Traceback (most recent call last):
239239
SyntaxError: can't assign to function call

‎Misc/NEWS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ What's New in Python 3.4.0 Alpha 1?
1010
Core and Builtins
1111
-----------------
1212

13+
- Issue #16619: Create NameConstant AST class to represent None, True, and False
14+
literals. As a result, these constants are never loaded at runtime from
15+
builtins.
16+
1317
- Issue #16455: On FreeBSD and Solaris, if the locale is C, the
1418
ASCII/surrogateescape codec is now used, instead of the locale encoding, to
1519
decode the command line arguments. This change fixes inconsistencies with

‎Parser/Python.asdl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
-- ASDL's five builtin types are identifier, int, string, bytes, object
1+
-- ASDL's six builtin types are identifier, int, string, bytes, object, singleton
22

33
module Python
44
{
@@ -69,8 +69,8 @@ module Python
6969
| Num(object n) -- a number as a PyObject.
7070
| Str(string s) -- need to specify raw, unicode, etc?
7171
| Bytes(bytes s)
72+
| NameConstant(singleton value)
7273
| Ellipsis
73-
-- other literals? bools?
7474

7575
-- the following expression can appear in assignment context
7676
| Attribute(expr value, identifier attr, expr_context ctx)

‎Parser/asdl.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ def p_field_5(self, type_):
222222
" field ::= Id ? "
223223
return Field(type[0], opt=True)
224224

225-
builtin_types = ("identifier", "string", "bytes", "int", "object")
225+
builtin_types = ("identifier", "string", "bytes", "int", "object", "singleton")
226226

227227
# below is a collection of classes to capture the AST of an AST :-)
228228
# not sure if any of the methods are useful yet, but I'm adding them

‎Parser/asdl_c.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -820,6 +820,7 @@ def visitModule(self, mod):
820820
Py_INCREF((PyObject*)o);
821821
return (PyObject*)o;
822822
}
823+
#define ast2obj_singleton ast2obj_object
823824
#define ast2obj_identifier ast2obj_object
824825
#define ast2obj_string ast2obj_object
825826
#define ast2obj_bytes ast2obj_object
@@ -831,6 +832,17 @@ def visitModule(self, mod):
831832
832833
/* Conversion Python -> AST */
833834
835+
static int obj2ast_singleton(PyObject *obj, PyObject** out, PyArena* arena)
836+
{
837+
if (obj != Py_None && obj != Py_True && obj != Py_False) {
838+
PyErr_SetString(PyExc_ValueError,
839+
"AST singleton must be True, False, or None");
840+
return 1;
841+
}
842+
*out = obj;
843+
return 0;
844+
}
845+
834846
static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
835847
{
836848
if (obj == Py_None)

‎Python/Python-ast.c‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,10 @@ static PyTypeObject *Bytes_type;
271271
static char *Bytes_fields[]={
272272
"s",
273273
};
274+
static PyTypeObject *NameConstant_type;
275+
static char *NameConstant_fields[]={
276+
"value",
277+
};
274278
static PyTypeObject *Ellipsis_type;
275279
static PyTypeObject *Attribute_type;
276280
_Py_IDENTIFIER(attr);
@@ -673,6 +677,7 @@ static PyObject* ast2obj_object(void *o)
673677
Py_INCREF((PyObject*)o);
674678
return (PyObject*)o;
675679
}
680+
#define ast2obj_singleton ast2obj_object
676681
#define ast2obj_identifier ast2obj_object
677682
#define ast2obj_string ast2obj_object
678683
#define ast2obj_bytes ast2obj_object
@@ -684,6 +689,17 @@ static PyObject* ast2obj_int(long b)
684689

685690
/* Conversion Python -> AST */
686691

692+
static int obj2ast_singleton(PyObject *obj, PyObject** out, PyArena* arena)
693+
{
694+
if (obj != Py_None && obj != Py_True && obj != Py_False) {
695+
PyErr_SetString(PyExc_ValueError,
696+
"AST singleton must be True, False, or None");
697+
return 1;
698+
}
699+
*out = obj;
700+
return 0;
701+
}
702+
687703
static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
688704
{
689705
if (obj == Py_None)
@@ -860,6 +876,9 @@ static int init_types(void)
860876
if (!Str_type) return 0;
861877
Bytes_type = make_type("Bytes", expr_type, Bytes_fields, 1);
862878
if (!Bytes_type) return 0;
879+
NameConstant_type = make_type("NameConstant", expr_type,
880+
NameConstant_fields, 1);
881+
if (!NameConstant_type) return 0;
863882
Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0);
864883
if (!Ellipsis_type) return 0;
865884
Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3);
@@ -1920,6 +1939,25 @@ Bytes(bytes s, int lineno, int col_offset, PyArena *arena)
19201939
return p;
19211940
}
19221941

1942+
expr_ty
1943+
NameConstant(singleton value, int lineno, int col_offset, PyArena *arena)
1944+
{
1945+
expr_ty p;
1946+
if (!value) {
1947+
PyErr_SetString(PyExc_ValueError,
1948+
"field value is required for NameConstant");
1949+
return NULL;
1950+
}
1951+
p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
1952+
if (!p)
1953+
return NULL;
1954+
p->kind = NameConstant_kind;
1955+
p->v.NameConstant.value = value;
1956+
p->lineno = lineno;
1957+
p->col_offset = col_offset;
1958+
return p;
1959+
}
1960+
19231961
expr_ty
19241962
Ellipsis(int lineno, int col_offset, PyArena *arena)
19251963
{
@@ -2028,6 +2066,7 @@ Name(identifier id, expr_context_ty ctx, int lineno, int col_offset, PyArena
20282066
*arena)
20292067
{
20302068
expr_ty p;
2069+
assert(PyUnicode_CompareWithASCIIString(id, "True") && PyUnicode_CompareWithASCIIString(id, "False") && PyUnicode_CompareWithASCIIString(id, "None"));
20312070
if (!id) {
20322071
PyErr_SetString(PyExc_ValueError,
20332072
"field id is required for Name");
@@ -2948,6 +2987,15 @@ ast2obj_expr(void* _o)
29482987
goto failed;
29492988
Py_DECREF(value);
29502989
break;
2990+
case NameConstant_kind:
2991+
result = PyType_GenericNew(NameConstant_type, NULL, NULL);
2992+
if (!result) goto failed;
2993+
value = ast2obj_singleton(o->v.NameConstant.value);
2994+
if (!value) goto failed;
2995+
if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
2996+
goto failed;
2997+
Py_DECREF(value);
2998+
break;
29512999
case Ellipsis_kind:
29523000
result = PyType_GenericNew(Ellipsis_type, NULL, NULL);
29533001
if (!result) goto failed;
@@ -5688,6 +5736,29 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
56885736
if (*out == NULL) goto failed;
56895737
return 0;
56905738
}
5739+
isinstance = PyObject_IsInstance(obj, (PyObject*)NameConstant_type);
5740+
if (isinstance == -1) {
5741+
return 1;
5742+
}
5743+
if (isinstance) {
5744+
singleton value;
5745+
5746+
if (_PyObject_HasAttrId(obj, &PyId_value)) {
5747+
int res;
5748+
tmp = _PyObject_GetAttrId(obj, &PyId_value);
5749+
if (tmp == NULL) goto failed;
5750+
res = obj2ast_singleton(tmp, &value, arena);
5751+
if (res != 0) goto failed;
5752+
Py_XDECREF(tmp);
5753+
tmp = NULL;
5754+
} else {
5755+
PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from NameConstant");
5756+
return 1;
5757+
}
5758+
*out = NameConstant(value, lineno, col_offset, arena);
5759+
if (*out == NULL) goto failed;
5760+
return 0;
5761+
}
56915762
isinstance = PyObject_IsInstance(obj, (PyObject*)Ellipsis_type);
56925763
if (isinstance == -1) {
56935764
return 1;
@@ -7008,6 +7079,8 @@ PyInit__ast(void)
70087079
NULL;
70097080
if (PyDict_SetItemString(d, "Bytes", (PyObject*)Bytes_type) < 0) return
70107081
NULL;
7082+
if (PyDict_SetItemString(d, "NameConstant",
7083+
(PyObject*)NameConstant_type) < 0) return NULL;
70117084
if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0)
70127085
return NULL;
70137086
if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) <

0 commit comments

Comments
 (0)