Skip to content

Commit 6baf76e

Browse files
dpgeorgedpgeorge
committed
py: make closures work.
1 parent 8cc96a3 commit 6baf76e

File tree

14 files changed

+213
-72
lines changed

14 files changed

+213
-72
lines changed

‎py/bc0.h‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,11 @@
1313
#define MP_BC_LOAD_FAST_2 (0x22)
1414
#define MP_BC_LOAD_FAST_N (0x23) // uint
1515
#define MP_BC_LOAD_DEREF (0x24) // uint
16-
#define MP_BC_LOAD_CLOSURE (0x25) // uint
17-
#define MP_BC_LOAD_NAME (0x26) // qstr
18-
#define MP_BC_LOAD_GLOBAL (0x27) // qstr
19-
#define MP_BC_LOAD_ATTR (0x28) // qstr
20-
#define MP_BC_LOAD_METHOD (0x29) // qstr
21-
#define MP_BC_LOAD_BUILD_CLASS (0x2a)
16+
#define MP_BC_LOAD_NAME (0x25) // qstr
17+
#define MP_BC_LOAD_GLOBAL (0x26) // qstr
18+
#define MP_BC_LOAD_ATTR (0x27) // qstr
19+
#define MP_BC_LOAD_METHOD (0x28) // qstr
20+
#define MP_BC_LOAD_BUILD_CLASS (0x29)
2221

2322
#define MP_BC_STORE_FAST_0 (0x30)
2423
#define MP_BC_STORE_FAST_1 (0x31)

‎py/compile.c‎

Lines changed: 46 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -767,7 +767,12 @@ void close_over_variables_etc(compiler_t *comp, scope_t *this_scope, int n_dict_
767767
for (int j = 0; j < this_scope->id_info_len; j++) {
768768
id_info_t *id2 = &this_scope->id_info[j];
769769
if (id2->kind == ID_INFO_KIND_FREE && id->qstr == id2->qstr) {
770+
#if MICROPY_EMIT_CPYTHON
770771
EMIT(load_closure, id->qstr, id->local_num);
772+
#else
773+
// in Micro Python we load closures using LOAD_FAST
774+
EMIT(load_fast, id->qstr, id->local_num);
775+
#endif
771776
nfree += 1;
772777
}
773778
}
@@ -2806,7 +2811,11 @@ void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) {
28062811
if (id->kind == ID_INFO_KIND_LOCAL) {
28072812
EMIT(load_const_tok, MP_TOKEN_KW_NONE);
28082813
} else {
2814+
#if MICROPY_EMIT_CPYTHON
28092815
EMIT(load_closure, comp->qstr___class__, 0); // XXX check this is the correct local num
2816+
#else
2817+
EMIT(load_fast, comp->qstr___class__, 0); // XXX check this is the correct local num
2818+
#endif
28102819
}
28112820
EMIT(return_value);
28122821
}
@@ -2894,7 +2903,7 @@ void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind_t pass
28942903

28952904
void compile_scope_compute_things(compiler_t *comp, scope_t *scope) {
28962905
// in functions, turn implicit globals into explicit globals
2897-
// compute num_locals, and the index of each local
2906+
// compute the index of each local
28982907
scope->num_locals = 0;
28992908
for (int i = 0; i < scope->id_info_len; i++) {
29002909
id_info_t *id = &scope->id_info[i];
@@ -2913,19 +2922,27 @@ void compile_scope_compute_things(compiler_t *comp, scope_t *scope) {
29132922
}
29142923

29152924
// compute the index of cell vars (freevars[idx] in CPython)
2916-
int num_closed = 0;
2925+
#if MICROPY_EMIT_CPYTHON
2926+
int num_cell = 0;
2927+
#endif
29172928
for (int i = 0; i < scope->id_info_len; i++) {
29182929
id_info_t *id = &scope->id_info[i];
2930+
#if MICROPY_EMIT_CPYTHON
2931+
// in CPython the cells are numbered starting from 0
29192932
if (id->kind == ID_INFO_KIND_CELL) {
2920-
id->local_num = num_closed;
2921-
#if !MICROPY_EMIT_CPYTHON
2922-
// the cells come right after the fast locals (CPython doesn't add this offset)
2923-
id->local_num += scope->num_locals;
2924-
#endif
2925-
num_closed += 1;
2933+
id->local_num = num_cell;
2934+
num_cell += 1;
2935+
}
2936+
#else
2937+
// in Micro Python the cells come right after the fast locals
2938+
// parameters are not counted here, since they remain at the start
2939+
// of the locals, even if they are cell vars
2940+
if (!id->param && id->kind == ID_INFO_KIND_CELL) {
2941+
id->local_num = scope->num_locals;
2942+
scope->num_locals += 1;
29262943
}
2944+
#endif
29272945
}
2928-
scope->num_cells = num_closed;
29292946

29302947
// compute the index of free vars (freevars[idx] in CPython)
29312948
// make sure they are in the order of the parent scope
@@ -2937,16 +2954,32 @@ void compile_scope_compute_things(compiler_t *comp, scope_t *scope) {
29372954
for (int j = 0; j < scope->id_info_len; j++) {
29382955
id_info_t *id2 = &scope->id_info[j];
29392956
if (id2->kind == ID_INFO_KIND_FREE && id->qstr == id2->qstr) {
2940-
id2->local_num = num_closed + num_free;
2941-
#if !MICROPY_EMIT_CPYTHON
2942-
// the frees come right after the cells (CPython doesn't add this offset)
2943-
id2->local_num += scope->num_locals;
2957+
assert(!id2->param); // free vars should not be params
2958+
#if MICROPY_EMIT_CPYTHON
2959+
// in CPython the frees are numbered after the cells
2960+
id2->local_num = num_cell + num_free;
2961+
#else
2962+
// in Micro Python the frees come first, before the params
2963+
id2->local_num = num_free;
29442964
#endif
29452965
num_free += 1;
29462966
}
29472967
}
29482968
}
29492969
}
2970+
#if !MICROPY_EMIT_CPYTHON
2971+
// in Micro Python shift all other locals after the free locals
2972+
if (num_free > 0) {
2973+
for (int i = 0; i < scope->id_info_len; i++) {
2974+
id_info_t *id = &scope->id_info[i];
2975+
if (id->param || id->kind != ID_INFO_KIND_FREE) {
2976+
id->local_num += num_free;
2977+
}
2978+
}
2979+
scope->num_params += num_free; // free vars are counted as params for passing them into the function
2980+
scope->num_locals += num_free;
2981+
}
2982+
#endif
29502983
}
29512984

29522985
// compute flags

‎py/emitbc.c‎

Lines changed: 49 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -49,36 +49,6 @@ void* emit_bc_get_code(emit_t* emit) {
4949
return emit->code_base;
5050
}
5151

52-
static void emit_bc_set_native_types(emit_t *emit, bool do_native_types) {
53-
}
54-
55-
static void emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) {
56-
emit->pass = pass;
57-
emit->stack_size = 0;
58-
emit->last_emit_was_return_value = false;
59-
emit->scope = scope;
60-
if (pass == PASS_2) {
61-
memset(emit->label_offsets, -1, emit->max_num_labels * sizeof(uint));
62-
}
63-
emit->code_offset = 0;
64-
}
65-
66-
static void emit_bc_end_pass(emit_t *emit) {
67-
// check stack is back to zero size
68-
if (emit->stack_size != 0) {
69-
printf("ERROR: stack size not back to zero; got %d\n", emit->stack_size);
70-
}
71-
72-
if (emit->pass == PASS_2) {
73-
// calculate size of code in bytes
74-
emit->code_size = emit->code_offset;
75-
emit->code_base = m_new(byte, emit->code_size);
76-
77-
} else if (emit->pass == PASS_3) {
78-
rt_assign_byte_code(emit->scope->unique_code_id, emit->code_base, emit->code_size, emit->scope->num_params, emit->scope->num_locals, emit->scope->num_cells, emit->scope->stack_size, (emit->scope->flags & SCOPE_FLAG_GENERATOR) != 0);
79-
}
80-
}
81-
8252
// all functions must go through this one to emit bytes
8353
static byte* emit_get_cur_to_write_bytes(emit_t* emit, int num_bytes_to_write) {
8454
//printf("emit %d\n", num_bytes_to_write);
@@ -166,6 +136,53 @@ static void emit_write_byte_1_signed_label(emit_t* emit, byte b1, int label) {
166136
c[2] = code_offset >> 8;
167137
}
168138

139+
static void emit_bc_set_native_types(emit_t *emit, bool do_native_types) {
140+
}
141+
142+
static void emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) {
143+
emit->pass = pass;
144+
emit->stack_size = 0;
145+
emit->last_emit_was_return_value = false;
146+
emit->scope = scope;
147+
if (pass == PASS_2) {
148+
memset(emit->label_offsets, -1, emit->max_num_labels * sizeof(uint));
149+
}
150+
emit->code_offset = 0;
151+
152+
// prelude for initialising closed over variables
153+
int num_cell = 0;
154+
for (int i = 0; i < scope->id_info_len; i++) {
155+
id_info_t *id = &scope->id_info[i];
156+
if (id->kind == ID_INFO_KIND_CELL) {
157+
num_cell += 1;
158+
}
159+
}
160+
assert(num_cell <= 255);
161+
emit_write_byte_1(emit, num_cell); // write number of locals that are cells
162+
for (int i = 0; i < scope->id_info_len; i++) {
163+
id_info_t *id = &scope->id_info[i];
164+
if (id->kind == ID_INFO_KIND_CELL) {
165+
emit_write_byte_1(emit, id->local_num); // write the local which should be converted to a cell
166+
}
167+
}
168+
}
169+
170+
static void emit_bc_end_pass(emit_t *emit) {
171+
// check stack is back to zero size
172+
if (emit->stack_size != 0) {
173+
printf("ERROR: stack size not back to zero; got %d\n", emit->stack_size);
174+
}
175+
176+
if (emit->pass == PASS_2) {
177+
// calculate size of code in bytes
178+
emit->code_size = emit->code_offset;
179+
emit->code_base = m_new(byte, emit->code_size);
180+
181+
} else if (emit->pass == PASS_3) {
182+
rt_assign_byte_code(emit->scope->unique_code_id, emit->code_base, emit->code_size, emit->scope->num_params, emit->scope->num_locals, emit->scope->stack_size, (emit->scope->flags & SCOPE_FLAG_GENERATOR) != 0);
183+
}
184+
}
185+
169186
bool emit_bc_last_emit_was_return_value(emit_t *emit) {
170187
return emit->last_emit_was_return_value;
171188
}
@@ -288,8 +305,8 @@ static void emit_bc_load_deref(emit_t *emit, qstr qstr, int local_num) {
288305
}
289306

290307
static void emit_bc_load_closure(emit_t *emit, qstr qstr, int local_num) {
291-
emit_pre(emit, 1);
292-
emit_write_byte_1_uint(emit, MP_BC_LOAD_CLOSURE, local_num);
308+
// not needed/supported for BC
309+
assert(0);
293310
}
294311

295312
static void emit_bc_load_name(emit_t *emit, qstr qstr) {

‎py/obj.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ typedef struct _mp_map_t mp_map_t;
120120

121121
mp_obj_t mp_obj_new_none(void);
122122
mp_obj_t mp_obj_new_bool(bool value);
123+
mp_obj_t mp_obj_new_cell(mp_obj_t obj);
123124
mp_obj_t mp_obj_new_int(machine_int_t value);
124125
mp_obj_t mp_obj_new_str(qstr qstr);
125126
#if MICROPY_ENABLE_FLOAT
@@ -134,7 +135,7 @@ mp_obj_t mp_obj_new_range(int start, int stop, int step);
134135
mp_obj_t mp_obj_new_range_iterator(int cur, int stop, int step);
135136
mp_obj_t mp_obj_new_fun_bc(int n_args, uint n_state, const byte *code);
136137
mp_obj_t mp_obj_new_fun_asm(uint n_args, void *fun);
137-
mp_obj_t mp_obj_new_gen_wrap(uint n_locals, uint n_cells, uint n_stack, mp_obj_t fun);
138+
mp_obj_t mp_obj_new_gen_wrap(uint n_locals, uint n_stack, mp_obj_t fun);
138139
mp_obj_t mp_obj_new_gen_instance(mp_obj_t state, const byte *ip, mp_obj_t *sp);
139140
mp_obj_t mp_obj_new_closure(mp_obj_t fun, mp_obj_t closure_tuple);
140141
mp_obj_t mp_obj_new_tuple(uint n, mp_obj_t *items);

‎py/objclosure.c‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#include <stdlib.h>
22
#include <stdint.h>
3+
#include <string.h>
34
#include <assert.h>
45

56
#include "nlr.h"
@@ -11,14 +12,31 @@
1112
typedef struct _mp_obj_closure_t {
1213
mp_obj_base_t base;
1314
mp_obj_t fun;
14-
mp_obj_t vars;
15+
uint n_closed;
16+
mp_obj_t *closed;
1517
} mp_obj_closure_t;
1618

19+
// args are in reverse order in the array
20+
mp_obj_t closure_call_n(mp_obj_t self_in, int n_args, const mp_obj_t *args) {
21+
mp_obj_closure_t *self = self_in;
22+
23+
// concatenate args and closed-over-vars, in reverse order
24+
// TODO perhaps cache this array so we don't need to create it each time we are called
25+
mp_obj_t *args2 = m_new(mp_obj_t, self->n_closed + n_args);
26+
memcpy(args2, args, n_args * sizeof(mp_obj_t));
27+
for (int i = 0; i < self->n_closed; i++) {
28+
args2[n_args + i] = self->closed[self->n_closed - 1 - i];
29+
}
30+
31+
// call the function with the new vars array
32+
return rt_call_function_n(self->fun, n_args + self->n_closed, args2);
33+
}
34+
1735
const mp_obj_type_t closure_type = {
1836
{ &mp_const_type },
1937
"closure",
2038
NULL, // print
21-
NULL, // call_n
39+
closure_call_n, // call_n
2240
NULL, // unary_op
2341
NULL, // binary_op
2442
NULL, // getiter
@@ -30,6 +48,6 @@ mp_obj_t mp_obj_new_closure(mp_obj_t fun, mp_obj_t closure_tuple) {
3048
mp_obj_closure_t *o = m_new_obj(mp_obj_closure_t);
3149
o->base.type = &closure_type;
3250
o->fun = fun;
33-
o->vars = closure_tuple;
51+
mp_obj_tuple_get(closure_tuple, &o->n_closed, &o->closed);
3452
return o;
3553
}

‎py/objgenerator.c‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,14 @@ mp_obj_t gen_wrap_call_n(mp_obj_t self_in, int n_args, const mp_obj_t *args) {
3838
for (int i = 0; i < n_args; i++) {
3939
state[1 + i] = args[n_args - 1 - i];
4040
}
41+
42+
// TODO
43+
// prelude for making cells (closed over variables)
44+
// for now we just make sure there are no cells variables
45+
// need to work out how to implement closed over variables in generators
46+
assert(bc_code[0] == 0);
47+
bc_code += 1;
48+
4149
return mp_obj_new_gen_instance(state, bc_code, state + self->n_state);
4250
}
4351

@@ -53,11 +61,11 @@ const mp_obj_type_t gen_wrap_type = {
5361
{{NULL, NULL},}, // method list
5462
};
5563

56-
mp_obj_t mp_obj_new_gen_wrap(uint n_locals, uint n_cells, uint n_stack, mp_obj_t fun) {
64+
mp_obj_t mp_obj_new_gen_wrap(uint n_locals, uint n_stack, mp_obj_t fun) {
5765
mp_obj_gen_wrap_t *o = m_new_obj(mp_obj_gen_wrap_t);
5866
o->base.type = &gen_wrap_type;
5967
// we have at least 3 locals so the bc can write back fast[0,1,2] safely; should improve how this is done
60-
o->n_state = ((n_locals + n_cells) < 3 ? 3 : (n_locals + n_cells)) + n_stack;
68+
o->n_state = (n_locals < 3 ? 3 : n_locals) + n_stack;
6169
o->fun = fun;
6270
return o;
6371
}

‎py/runtime.c‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ typedef struct _mp_code_t {
5959
mp_code_kind_t kind;
6060
int n_args;
6161
int n_locals;
62-
int n_cells;
6362
int n_stack;
6463
bool is_generator;
6564
union {
@@ -178,14 +177,13 @@ static void alloc_unique_codes(void) {
178177
}
179178
}
180179

181-
void rt_assign_byte_code(int unique_code_id, byte *code, uint len, int n_args, int n_locals, int n_cells, int n_stack, bool is_generator) {
180+
void rt_assign_byte_code(int unique_code_id, byte *code, uint len, int n_args, int n_locals, int n_stack, bool is_generator) {
182181
alloc_unique_codes();
183182

184183
assert(unique_code_id < next_unique_code_id);
185184
unique_codes[unique_code_id].kind = MP_CODE_BYTE;
186185
unique_codes[unique_code_id].n_args = n_args;
187186
unique_codes[unique_code_id].n_locals = n_locals;
188-
unique_codes[unique_code_id].n_cells = n_cells;
189187
unique_codes[unique_code_id].n_stack = n_stack;
190188
unique_codes[unique_code_id].is_generator = is_generator;
191189
unique_codes[unique_code_id].u_byte.code = code;
@@ -221,7 +219,6 @@ void rt_assign_native_code(int unique_code_id, void *fun, uint len, int n_args)
221219
unique_codes[unique_code_id].kind = MP_CODE_NATIVE;
222220
unique_codes[unique_code_id].n_args = n_args;
223221
unique_codes[unique_code_id].n_locals = 0;
224-
unique_codes[unique_code_id].n_cells = 0;
225222
unique_codes[unique_code_id].n_stack = 0;
226223
unique_codes[unique_code_id].is_generator = false;
227224
unique_codes[unique_code_id].u_native.fun = fun;
@@ -255,7 +252,6 @@ void rt_assign_inline_asm_code(int unique_code_id, void *fun, uint len, int n_ar
255252
unique_codes[unique_code_id].kind = MP_CODE_INLINE_ASM;
256253
unique_codes[unique_code_id].n_args = n_args;
257254
unique_codes[unique_code_id].n_locals = 0;
258-
unique_codes[unique_code_id].n_cells = 0;
259255
unique_codes[unique_code_id].n_stack = 0;
260256
unique_codes[unique_code_id].is_generator = false;
261257
unique_codes[unique_code_id].u_inline_asm.fun = fun;
@@ -632,7 +628,7 @@ mp_obj_t rt_make_function_from_id(int unique_code_id) {
632628
mp_obj_t fun;
633629
switch (c->kind) {
634630
case MP_CODE_BYTE:
635-
fun = mp_obj_new_fun_bc(c->n_args, c->n_locals + c->n_cells + c->n_stack, c->u_byte.code);
631+
fun = mp_obj_new_fun_bc(c->n_args, c->n_locals + c->n_stack, c->u_byte.code);
636632
break;
637633
case MP_CODE_NATIVE:
638634
switch (c->n_args) {
@@ -652,13 +648,14 @@ mp_obj_t rt_make_function_from_id(int unique_code_id) {
652648

653649
// check for generator functions and if so wrap in generator object
654650
if (c->is_generator) {
655-
fun = mp_obj_new_gen_wrap(c->n_locals, c->n_cells, c->n_stack, fun);
651+
fun = mp_obj_new_gen_wrap(c->n_locals, c->n_stack, fun);
656652
}
657653

658654
return fun;
659655
}
660656

661657
mp_obj_t rt_make_closure_from_id(int unique_code_id, mp_obj_t closure_tuple) {
658+
DEBUG_OP_printf("make_closure_from_id %d\n", unique_code_id);
662659
// make function object
663660
mp_obj_t ffun = rt_make_function_from_id(unique_code_id);
664661
// wrap function in closure object

‎py/runtime0.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,6 @@ extern void *const rt_fun_table[RT_F_NUMBER_OF];
8282
void rt_init(void);
8383
void rt_deinit(void);
8484
int rt_get_unique_code_id(bool is_main_module);
85-
void rt_assign_byte_code(int unique_code_id, byte *code, uint len, int n_args, int n_locals, int n_cells, int n_stack, bool is_generator);
85+
void rt_assign_byte_code(int unique_code_id, byte *code, uint len, int n_args, int n_locals, int n_stack, bool is_generator);
8686
void rt_assign_native_code(int unique_code_id, void *f, uint len, int n_args);
8787
void rt_assign_inline_asm_code(int unique_code_id, void *f, uint len, int n_args);

0 commit comments

Comments
 (0)