Skip to content

Commit 64949cb

Browse files
jeremyhyltonjeremyhylton
committed
PEP 227 implementation
The majority of the changes are in the compiler. The mainloop changes primarily to implement the new opcodes and to pass a function's closure to eval_code2(). Frames and functions got new slots to hold the closure. Include/compile.h Add co_freevars and co_cellvars slots to code objects. Update PyCode_New() to take freevars and cellvars as arguments Include/funcobject.h Add func_closure slot to function objects. Add GetClosure()/SetClosure() functions (and corresponding macros) for getting at the closure. Include/frameobject.h PyFrame_New() now takes a closure. Include/opcode.h Add four new opcodes: MAKE_CLOSURE, LOAD_CLOSURE, LOAD_DEREF, STORE_DEREF. Remove comment about old requirement for opcodes to fit in 7 bits. compile.c Implement changes to code objects for co_freevars and co_cellvars. Modify symbol table to use st_cur_name (string object for the name of the current scope) and st_cur_children (list of nested blocks). Also define st_nested, which might more properly be called st_cur_nested. Add several DEF_XXX flags to track def-use information for free variables. New or modified functions of note: com_make_closure(struct compiling *, PyCodeObject *) Emit LOAD_CLOSURE opcodes as needed to pass cells for free variables into nested scope. com_addop_varname(struct compiling *, int, char *) Emits opcodes for LOAD_DEREF and STORE_DEREF. get_ref_type(struct compiling *, char *name) Return NAME_CLOSURE if ref type is FREE or CELL symtable_load_symbols(struct compiling *) Decides what variables are cell or free based on def-use info. Can now raise SyntaxError if nested scopes are mixed with exec or from blah import *. make_scope_info(PyObject *, PyObject *, int, int) Helper functions for symtable scope stack. symtable_update_free_vars(struct symtable *) After a code block has been analyzed, it must check each of its children for free variables that are not defined in the block. If a variable is free in a child and not defined in the parent, then it is defined by block the enclosing the current one or it is a global. This does the right logic. symtable_add_use() is now a macro for symtable_add_def() symtable_assign(struct symtable *, node *) Use goto instead of for (;;) Fixed bug in symtable where name of keyword argument in function call was treated as assignment in the scope of the call site. Ex: def f(): g(a=2) # a was considered a local of f ceval.c eval_code2() now take one more argument, a closure. Implement LOAD_CLOSURE, LOAD_DEREF, STORE_DEREF, MAKE_CLOSURE> Also: When name error occurs for global variable, report that the name was global in the error mesage. Objects/frameobject.c Initialize f_closure to be a tuple containing space for cellvars and freevars. f_closure is NULL if neither are present. Objects/funcobject.c Add support for func_closure. Python/import.c Change the magic number. Python/marshal.c Track changes to code objects.
1 parent fbd849f commit 64949cb

10 files changed

Lines changed: 929 additions & 338 deletions

File tree

‎Include/compile.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ typedef struct {
1818
PyObject *co_consts; /* list (constants used) */
1919
PyObject *co_names; /* list of strings (names used) */
2020
PyObject *co_varnames; /* tuple of strings (local variable names) */
21+
PyObject *co_freevars; /* tuple of strings (free variable names) */
22+
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
2123
/* The rest doesn't count for hash/cmp */
2224
PyObject *co_filename; /* string (where it was loaded from) */
2325
PyObject *co_name; /* string (name, for reference) */
@@ -42,7 +44,8 @@ struct _node; /* Declare the existence of this type */
4244
DL_IMPORT(PyCodeObject *) PyNode_Compile(struct _node *, char *);
4345
DL_IMPORT(PyCodeObject *) PyCode_New(
4446
int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *,
45-
PyObject *, PyObject *, int, PyObject *); /* same as struct above */
47+
PyObject *, PyObject *, PyObject *, PyObject *, int, PyObject *);
48+
/* same as struct above */
4649
DL_IMPORT(int) PyCode_Addr2Line(PyCodeObject *, int);
4750

4851
/* for internal use only */

‎Include/frameobject.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ typedef struct _frame {
2020
PyObject *f_builtins; /* builtin symbol table (PyDictObject) */
2121
PyObject *f_globals; /* global symbol table (PyDictObject) */
2222
PyObject *f_locals; /* local symbol table (PyDictObject) */
23+
PyObject *f_closure; /* environment for free variables */
2324
PyObject **f_valuestack; /* points after the last local */
2425
PyObject *f_trace; /* Trace function */
2526
PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
@@ -43,7 +44,8 @@ extern DL_IMPORT(PyTypeObject) PyFrame_Type;
4344
#define PyFrame_Check(op) ((op)->ob_type == &PyFrame_Type)
4445

4546
DL_IMPORT(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
46-
PyObject *, PyObject *);
47+
PyObject *, PyObject *,
48+
PyObject *);
4749

4850

4951
/* The rest of the interface is specific for frame objects */

‎Include/funcobject.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ typedef struct {
1212
PyObject *func_code;
1313
PyObject *func_globals;
1414
PyObject *func_defaults;
15+
PyObject *func_closure;
1516
PyObject *func_doc;
1617
PyObject *func_name;
1718
PyObject *func_dict;
@@ -26,6 +27,8 @@ extern DL_IMPORT(PyObject *) PyFunction_GetCode(PyObject *);
2627
extern DL_IMPORT(PyObject *) PyFunction_GetGlobals(PyObject *);
2728
extern DL_IMPORT(PyObject *) PyFunction_GetDefaults(PyObject *);
2829
extern DL_IMPORT(int) PyFunction_SetDefaults(PyObject *, PyObject *);
30+
extern DL_IMPORT(PyObject *) PyFunction_GetClosure(PyObject *);
31+
extern DL_IMPORT(int) PyFunction_SetClosure(PyObject *, PyObject *);
2932

3033
/* Macros for direct access to these values. Type checks are *not*
3134
done, so use with care. */
@@ -35,6 +38,8 @@ extern DL_IMPORT(int) PyFunction_SetDefaults(PyObject *, PyObject *);
3538
(((PyFunctionObject *)func) -> func_globals)
3639
#define PyFunction_GET_DEFAULTS(func) \
3740
(((PyFunctionObject *)func) -> func_defaults)
41+
#define PyFunction_GET_CLOSURE(func) \
42+
(((PyFunctionObject *)func) -> func_closure)
3843

3944
#ifdef __cplusplus
4045
}

‎Include/opcode.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -114,17 +114,17 @@ extern "C" {
114114

115115
#define SET_LINENO 127 /* Current line number */
116116

117-
/* It used to be the case that opcodes should fit in 7 bits. This is
118-
no longer the case -- 8 bits is fine (the instruction stream is now
119-
a sequence of unsigned characters). We gladly use the new space
120-
for new opcodes. */
121-
122117
#define RAISE_VARARGS 130 /* Number of raise arguments (1, 2 or 3) */
123118
/* CALL_FUNCTION_XXX opcodes defined below depend on this definition */
124119
#define CALL_FUNCTION 131 /* #args + (#kwargs<<8) */
125120
#define MAKE_FUNCTION 132 /* #defaults */
126121
#define BUILD_SLICE 133 /* Number of items */
127122

123+
#define MAKE_CLOSURE 134 /* #free vars */
124+
#define LOAD_CLOSURE 135 /* Load free variable from closure */
125+
#define LOAD_DEREF 136 /* Load and dereference from closure cell */
126+
#define STORE_DEREF 137 /* Store into cell */
127+
128128
/* The next 3 opcodes must be contiguous and satisfy
129129
(CALL_FUNCTION_VAR - CALL_FUNCTION) & 3 == 1 */
130130
#define CALL_FUNCTION_VAR 140 /* #args + (#kwargs<<8) */

‎Objects/frameobject.c‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ frame_dealloc(PyFrameObject *f)
7979
Py_XDECREF(f->f_builtins);
8080
Py_XDECREF(f->f_globals);
8181
Py_XDECREF(f->f_locals);
82+
Py_XDECREF(f->f_closure);
8283
Py_XDECREF(f->f_trace);
8384
Py_XDECREF(f->f_exc_type);
8485
Py_XDECREF(f->f_exc_value);
@@ -106,14 +107,14 @@ PyTypeObject PyFrame_Type = {
106107
};
107108

108109
PyFrameObject *
109-
PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
110-
PyObject *globals, PyObject *locals)
110+
PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals,
111+
PyObject *locals, PyObject *closure)
111112
{
112113
PyFrameObject *back = tstate->frame;
113114
static PyObject *builtin_object;
114115
PyFrameObject *f;
115116
PyObject *builtins;
116-
int extras;
117+
int extras, ncells;
117118

118119
if (builtin_object == NULL) {
119120
builtin_object = PyString_InternFromString("__builtins__");
@@ -128,6 +129,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
128129
return NULL;
129130
}
130131
extras = code->co_stacksize + code->co_nlocals;
132+
ncells = PyTuple_GET_SIZE(code->co_cellvars);
131133
if (back == NULL || back->f_globals != globals) {
132134
builtins = PyDict_GetItem(globals, builtin_object);
133135
if (builtins != NULL && PyModule_Check(builtins))
@@ -197,6 +199,22 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
197199
locals = globals;
198200
Py_INCREF(locals);
199201
}
202+
if (closure || ncells) {
203+
int i, size;
204+
size = ncells;
205+
if (closure)
206+
size += PyTuple_GET_SIZE(closure);
207+
f->f_closure = PyTuple_New(size);
208+
for (i = 0; i < ncells; ++i)
209+
PyTuple_SET_ITEM(f->f_closure, i, PyCell_New(NULL));
210+
for (i = ncells; i < size; ++i) {
211+
PyObject *o = PyTuple_GET_ITEM(closure, i - ncells);
212+
Py_INCREF(o);
213+
PyTuple_SET_ITEM(f->f_closure, i, o);
214+
}
215+
}
216+
else
217+
f->f_closure = NULL;
200218
f->f_locals = locals;
201219
f->f_trace = NULL;
202220
f->f_exc_type = f->f_exc_value = f->f_exc_traceback = NULL;

‎Objects/funcobject.c‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ PyFunction_New(PyObject *code, PyObject *globals)
2020
op->func_name = ((PyCodeObject *)code)->co_name;
2121
Py_INCREF(op->func_name);
2222
op->func_defaults = NULL; /* No default arguments */
23+
op->func_closure = NULL;
2324
consts = ((PyCodeObject *)code)->co_consts;
2425
if (PyTuple_Size(consts) >= 1) {
2526
doc = PyTuple_GetItem(consts, 0);
@@ -89,6 +90,37 @@ PyFunction_SetDefaults(PyObject *op, PyObject *defaults)
8990
return 0;
9091
}
9192

93+
PyObject *
94+
PyFunction_GetClosure(PyObject *op)
95+
{
96+
if (!PyFunction_Check(op)) {
97+
PyErr_BadInternalCall();
98+
return NULL;
99+
}
100+
return ((PyFunctionObject *) op) -> func_closure;
101+
}
102+
103+
int
104+
PyFunction_SetClosure(PyObject *op, PyObject *closure)
105+
{
106+
if (!PyFunction_Check(op)) {
107+
PyErr_BadInternalCall();
108+
return -1;
109+
}
110+
if (closure == Py_None)
111+
closure = NULL;
112+
else if (PyTuple_Check(closure)) {
113+
Py_XINCREF(closure);
114+
}
115+
else {
116+
PyErr_SetString(PyExc_SystemError, "non-tuple closure");
117+
return -1;
118+
}
119+
Py_XDECREF(((PyFunctionObject *) op) -> func_closure);
120+
((PyFunctionObject *) op) -> func_closure = closure;
121+
return 0;
122+
}
123+
92124
/* Methods */
93125

94126
#define OFF(x) offsetof(PyFunctionObject, x)
@@ -98,6 +130,7 @@ static struct memberlist func_memberlist[] = {
98130
{"func_globals", T_OBJECT, OFF(func_globals), READONLY},
99131
{"func_name", T_OBJECT, OFF(func_name), READONLY},
100132
{"__name__", T_OBJECT, OFF(func_name), READONLY},
133+
{"func_closure", T_OBJECT, OFF(func_closure)},
101134
{"func_defaults", T_OBJECT, OFF(func_defaults)},
102135
{"func_doc", T_OBJECT, OFF(func_doc)},
103136
{"__doc__", T_OBJECT, OFF(func_doc)},

‎Python/ceval.c‎

Lines changed: 77 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,16 @@
3333

3434
typedef PyObject *(*callproc)(PyObject *, PyObject *, PyObject *);
3535

36+
#define REPR(ob) PyString_AS_STRING(PyObject_Repr(ob))
37+
3638
/* Forward declarations */
3739

3840
static PyObject *eval_code2(PyCodeObject *,
3941
PyObject *, PyObject *,
4042
PyObject **, int,
4143
PyObject **, int,
42-
PyObject **, int);
44+
PyObject **, int,
45+
PyObject *);
4346

4447
static PyObject *call_object(PyObject *, PyObject *, PyObject *);
4548
static PyObject *call_cfunction(PyObject *, PyObject *, PyObject *);
@@ -78,6 +81,8 @@ static void format_exc_check_arg(PyObject *, char *, PyObject *);
7881

7982
#define NAME_ERROR_MSG \
8083
"name '%.200s' is not defined"
84+
#define GLOBAL_NAME_ERROR_MSG \
85+
"global name '%.200s' is not defined"
8186
#define UNBOUNDLOCAL_ERROR_MSG \
8287
"local variable '%.200s' referenced before assignment"
8388

@@ -335,7 +340,8 @@ PyEval_EvalCode(PyCodeObject *co, PyObject *globals, PyObject *locals)
335340
globals, locals,
336341
(PyObject **)NULL, 0,
337342
(PyObject **)NULL, 0,
338-
(PyObject **)NULL, 0);
343+
(PyObject **)NULL, 0,
344+
NULL);
339345
}
340346

341347

@@ -344,7 +350,7 @@ PyEval_EvalCode(PyCodeObject *co, PyObject *globals, PyObject *locals)
344350
static PyObject *
345351
eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
346352
PyObject **args, int argcount, PyObject **kws, int kwcount,
347-
PyObject **defs, int defcount)
353+
PyObject **defs, int defcount, PyObject *closure)
348354
{
349355
#ifdef DXPAIRS
350356
int lastopcode = 0;
@@ -425,11 +431,9 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
425431
lltrace = PyDict_GetItemString(globals, "__lltrace__") != NULL;
426432
#endif
427433

428-
f = PyFrame_New(
429-
tstate, /*back*/
434+
f = PyFrame_New(tstate, /*back*/
430435
co, /*code*/
431-
globals, /*globals*/
432-
locals); /*locals*/
436+
globals, locals, closure);
433437
if (f == NULL)
434438
return NULL;
435439

@@ -1535,7 +1539,7 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
15351539
w = GETNAMEV(oparg);
15361540
if ((err = PyDict_DelItem(f->f_globals, w)) != 0)
15371541
format_exc_check_arg(
1538-
PyExc_NameError, NAME_ERROR_MSG ,w);
1542+
PyExc_NameError, GLOBAL_NAME_ERROR_MSG, w);
15391543
break;
15401544

15411545
case LOAD_CONST:
@@ -1577,7 +1581,7 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
15771581
if (x == NULL) {
15781582
format_exc_check_arg(
15791583
PyExc_NameError,
1580-
NAME_ERROR_MSG ,w);
1584+
GLOBAL_NAME_ERROR_MSG ,w);
15811585
break;
15821586
}
15831587
}
@@ -1618,6 +1622,25 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
16181622
SETLOCAL(oparg, NULL);
16191623
continue;
16201624

1625+
case LOAD_CLOSURE:
1626+
x = PyTuple_GET_ITEM(f->f_closure, oparg);
1627+
Py_INCREF(x);
1628+
PUSH(x);
1629+
break;
1630+
1631+
case LOAD_DEREF:
1632+
x = PyTuple_GET_ITEM(f->f_closure, oparg);
1633+
w = PyCell_Get(x);
1634+
Py_INCREF(w);
1635+
PUSH(w);
1636+
break;
1637+
1638+
case STORE_DEREF:
1639+
w = POP();
1640+
x = PyTuple_GET_ITEM(f->f_closure, oparg);
1641+
PyCell_Set(x, w);
1642+
continue;
1643+
16211644
case BUILD_TUPLE:
16221645
x = PyTuple_New(oparg);
16231646
if (x != NULL) {
@@ -1939,6 +1962,46 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
19391962
PUSH(x);
19401963
break;
19411964

1965+
case MAKE_CLOSURE:
1966+
{
1967+
int nfree;
1968+
v = POP(); /* code object */
1969+
x = PyFunction_New(v, f->f_globals);
1970+
nfree = PyTuple_GET_SIZE(((PyCodeObject *)v)->co_freevars);
1971+
Py_DECREF(v);
1972+
/* XXX Maybe this should be a separate opcode? */
1973+
if (x != NULL && nfree > 0) {
1974+
v = PyTuple_New(nfree);
1975+
if (v == NULL) {
1976+
Py_DECREF(x);
1977+
x = NULL;
1978+
break;
1979+
}
1980+
while (--nfree >= 0) {
1981+
w = POP();
1982+
PyTuple_SET_ITEM(v, nfree, w);
1983+
}
1984+
err = PyFunction_SetClosure(x, v);
1985+
Py_DECREF(v);
1986+
}
1987+
if (x != NULL && oparg > 0) {
1988+
v = PyTuple_New(oparg);
1989+
if (v == NULL) {
1990+
Py_DECREF(x);
1991+
x = NULL;
1992+
break;
1993+
}
1994+
while (--oparg >= 0) {
1995+
w = POP();
1996+
PyTuple_SET_ITEM(v, oparg, w);
1997+
}
1998+
err = PyFunction_SetDefaults(x, v);
1999+
Py_DECREF(v);
2000+
}
2001+
PUSH(x);
2002+
break;
2003+
}
2004+
19422005
case BUILD_SLICE:
19432006
if (oparg == 3)
19442007
w = POP();
@@ -2761,8 +2824,8 @@ call_eval_code2(PyObject *func, PyObject *arg, PyObject *kw)
27612824
(PyCodeObject *)PyFunction_GET_CODE(func),
27622825
PyFunction_GET_GLOBALS(func), (PyObject *)NULL,
27632826
&PyTuple_GET_ITEM(arg, 0), PyTuple_Size(arg),
2764-
k, nk,
2765-
d, nd);
2827+
k, nk, d, nd,
2828+
PyFunction_GET_CLOSURE(func));
27662829

27672830
if (k != NULL)
27682831
PyMem_DEL(k);
@@ -2805,6 +2868,7 @@ fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk)
28052868
PyObject *co = PyFunction_GET_CODE(func);
28062869
PyObject *globals = PyFunction_GET_GLOBALS(func);
28072870
PyObject *argdefs = PyFunction_GET_DEFAULTS(func);
2871+
PyObject *closure = PyFunction_GET_CLOSURE(func);
28082872
PyObject **d = NULL;
28092873
int nd = 0;
28102874

@@ -2814,7 +2878,8 @@ fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk)
28142878
}
28152879
return eval_code2((PyCodeObject *)co, globals,
28162880
(PyObject *)NULL, (*pp_stack)-n, na,
2817-
(*pp_stack)-2*nk, nk, d, nd);
2881+
(*pp_stack)-2*nk, nk, d, nd,
2882+
closure);
28182883
}
28192884

28202885
static PyObject *

0 commit comments

Comments
 (0)