Skip to content

Commit 0a2e965

Browse files
dpgeorgedpgeorge
committed
py: Add ability to have frozen persistent bytecode from .mpy files.
The config variable MICROPY_MODULE_FROZEN is now made of two separate parts: MICROPY_MODULE_FROZEN_STR and MICROPY_MODULE_FROZEN_MPY. This allows to have none, either or both of frozen strings and frozen mpy files (aka frozen bytecode).
1 parent 0699c6b commit 0a2e965

16 files changed

Lines changed: 148 additions & 50 deletions

File tree

‎esp8266/mpconfigport.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
#define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_FLOAT)
6262
#define MICROPY_ERROR_REPORTING (MICROPY_ERROR_REPORTING_NORMAL)
6363
#define MICROPY_STREAMS_NON_BLOCK (1)
64-
#define MICROPY_MODULE_FROZEN (1)
64+
#define MICROPY_MODULE_FROZEN_STR (1)
6565
#define MICROPY_MODULE_FROZEN_LEXER mp_lexer_new_from_str32
6666

6767
#define MICROPY_FATFS_ENABLE_LFN (1)

‎lib/utils/pyexec.c‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -50,22 +50,33 @@ STATIC bool repl_display_debugging_info = 0;
5050
#define EXEC_FLAG_PRINT_EOF (1)
5151
#define EXEC_FLAG_ALLOW_DEBUGGING (2)
5252
#define EXEC_FLAG_IS_REPL (4)
53+
#define EXEC_FLAG_SOURCE_IS_RAW_CODE (8)
5354

5455
// parses, compiles and executes the code in the lexer
5556
// frees the lexer before returning
5657
// EXEC_FLAG_PRINT_EOF prints 2 EOF chars: 1 after normal output, 1 after exception output
5758
// EXEC_FLAG_ALLOW_DEBUGGING allows debugging info to be printed after executing the code
5859
// EXEC_FLAG_IS_REPL is used for REPL inputs (flag passed on to mp_compile)
59-
STATIC int parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, int exec_flags) {
60+
STATIC int parse_compile_execute(void *source, mp_parse_input_kind_t input_kind, int exec_flags) {
6061
int ret = 0;
6162
uint32_t start = 0;
6263

6364
nlr_buf_t nlr;
6465
if (nlr_push(&nlr) == 0) {
65-
// parse and compile the script
66-
qstr source_name = lex->source_name;
67-
mp_parse_tree_t parse_tree = mp_parse(lex, input_kind);
68-
mp_obj_t module_fun = mp_compile(&parse_tree, source_name, MP_EMIT_OPT_NONE, exec_flags & EXEC_FLAG_IS_REPL);
66+
mp_obj_t module_fun;
67+
#if MICROPY_MODULE_FROZEN_MPY
68+
if (exec_flags & EXEC_FLAG_SOURCE_IS_RAW_CODE) {
69+
// source is a raw_code object, create the function
70+
module_fun = mp_make_function_from_raw_code(source, MP_OBJ_NULL, MP_OBJ_NULL);
71+
} else
72+
#endif
73+
{
74+
// source is a lexer, parse and compile the script
75+
mp_lexer_t *lex = source;
76+
qstr source_name = lex->source_name;
77+
mp_parse_tree_t parse_tree = mp_parse(lex, input_kind);
78+
module_fun = mp_compile(&parse_tree, source_name, MP_EMIT_OPT_NONE, exec_flags & EXEC_FLAG_IS_REPL);
79+
}
6980

7081
// execute code
7182
mp_hal_set_interrupt_char(CHAR_CTRL_C); // allow ctrl-C to interrupt us
@@ -488,14 +499,24 @@ int pyexec_file(const char *filename) {
488499

489500
#if MICROPY_MODULE_FROZEN
490501
int pyexec_frozen_module(const char *name) {
491-
mp_lexer_t *lex = mp_find_frozen_module(name, strlen(name));
502+
void *frozen_data;
503+
int frozen_type = mp_find_frozen_module(name, strlen(name), &frozen_data);
492504

493-
if (lex == NULL) {
494-
printf("could not find module '%s'\n", name);
495-
return false;
496-
}
505+
switch (frozen_type) {
506+
#if MICROPY_MODULE_FROZEN_STR
507+
case MP_FROZEN_STR:
508+
return parse_compile_execute(frozen_data, MP_PARSE_FILE_INPUT, 0);
509+
#endif
497510

498-
return parse_compile_execute(lex, MP_PARSE_FILE_INPUT, 0);
511+
#if MICROPY_MODULE_FROZEN_MPY
512+
case MP_FROZEN_MPY:
513+
return parse_compile_execute(frozen_data, MP_PARSE_FILE_INPUT, EXEC_FLAG_SOURCE_IS_RAW_CODE);
514+
#endif
515+
516+
default:
517+
printf("could not find module '%s'\n", name);
518+
return false;
519+
}
499520
}
500521
#endif
501522

‎pic16bit/mpconfigport.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@
6060
#define MICROPY_PY_IO (0)
6161
#define MICROPY_PY_STRUCT (0)
6262
#define MICROPY_PY_SYS (0)
63-
#define MICROPY_MODULE_FROZEN (0)
6463
#define MICROPY_CPYTHON_COMPAT (0)
6564
#define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_MPZ)
6665
#define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_NONE)

‎py/builtinimport.c‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ STATIC void do_load_from_lexer(mp_obj_t module_obj, mp_lexer_t *lex, const char
144144
}
145145
#endif
146146

147-
#if MICROPY_PERSISTENT_CODE_LOAD
147+
#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_MODULE_FROZEN_MPY
148148
STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code) {
149149
#if MICROPY_PY___FILE__
150150
// TODO
@@ -182,8 +182,9 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code) {
182182
#endif
183183

184184
STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
185-
// create the lexer
185+
#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_ENABLE_COMPILER
186186
char *file_str = vstr_null_terminated_str(file);
187+
#endif
187188

188189
#if MICROPY_PERSISTENT_CODE_LOAD
189190
if (file_str[file->len - 3] == 'm') {
@@ -340,8 +341,9 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
340341
DEBUG_printf("Module not yet loaded\n");
341342

342343
#if MICROPY_MODULE_FROZEN
343-
mp_lexer_t *lex = mp_find_frozen_module(mod_str, mod_len);
344-
if (lex != NULL) {
344+
void *frozen_data;
345+
int frozen_type = mp_find_frozen_module(mod_str, mod_len, &frozen_data);
346+
if (frozen_type != MP_FROZEN_NONE) {
345347
module_obj = mp_obj_new_module(module_name_qstr);
346348
// if args[3] (fromtuple) has magic value False, set up
347349
// this module for command-line "-m" option (set module's
@@ -351,7 +353,16 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
351353
mp_obj_module_t *o = MP_OBJ_TO_PTR(module_obj);
352354
mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR___main__));
353355
}
354-
do_load_from_lexer(module_obj, lex, mod_str);
356+
#if MICROPY_MODULE_FROZEN_STR
357+
if (frozen_type == MP_FROZEN_STR) {
358+
do_load_from_lexer(module_obj, frozen_data, mod_str);
359+
}
360+
#endif
361+
#if MICROPY_MODULE_FROZEN_MPY
362+
if (frozen_type == MP_FROZEN_MPY) {
363+
do_execute_raw_code(module_obj, frozen_data);
364+
}
365+
#endif
355366
return module_obj;
356367
}
357368
#endif

‎py/emitglue.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, mp_uint_t n_clos
172172
return mp_obj_new_closure(ffun, n_closed_over & 0xff, args + ((n_closed_over >> 7) & 2));
173173
}
174174

175-
#if MICROPY_PERSISTENT_CODE
175+
#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
176176

177177
#include "py/smallint.h"
178178

@@ -229,7 +229,7 @@ STATIC void extract_prelude(const byte **ip, const byte **ip2, bytecode_prelude_
229229
}
230230
}
231231

232-
#endif // MICROPY_PERSISTENT_CODE
232+
#endif // MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
233233

234234
#if MICROPY_PERSISTENT_CODE_LOAD
235235

‎py/frozenmod.c‎

Lines changed: 54 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
* The MIT License (MIT)
55
*
66
* Copyright (c) 2015 Paul Sokolovsky
7+
* Copyright (c) 2016 Damien P. George
78
*
89
* Permission is hereby granted, free of charge, to any person obtaining a copy
910
* of this software and associated documentation files (the "Software"), to deal
@@ -30,32 +31,75 @@
3031
#include "py/lexer.h"
3132
#include "py/frozenmod.h"
3233

33-
#if MICROPY_MODULE_FROZEN
34+
#if MICROPY_MODULE_FROZEN_STR
3435

3536
#ifndef MICROPY_MODULE_FROZEN_LEXER
3637
#define MICROPY_MODULE_FROZEN_LEXER mp_lexer_new_from_str_len
3738
#else
3839
mp_lexer_t *MICROPY_MODULE_FROZEN_LEXER(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len);
3940
#endif
4041

41-
extern const char mp_frozen_names[];
42-
extern const uint32_t mp_frozen_sizes[];
43-
extern const char mp_frozen_content[];
42+
extern const char mp_frozen_str_names[];
43+
extern const uint32_t mp_frozen_str_sizes[];
44+
extern const char mp_frozen_str_content[];
4445

45-
mp_lexer_t *mp_find_frozen_module(const char *str, int len) {
46-
const char *name = mp_frozen_names;
46+
STATIC mp_lexer_t *mp_find_frozen_str(const char *str, size_t len) {
47+
const char *name = mp_frozen_str_names;
4748

4849
size_t offset = 0;
4950
for (int i = 0; *name != 0; i++) {
50-
int l = strlen(name);
51+
size_t l = strlen(name);
5152
if (l == len && !memcmp(str, name, l)) {
52-
mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(MP_QSTR_, mp_frozen_content + offset, mp_frozen_sizes[i], 0);
53+
mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(MP_QSTR_, mp_frozen_str_content + offset, mp_frozen_str_sizes[i], 0);
5354
return lex;
5455
}
5556
name += l + 1;
56-
offset += mp_frozen_sizes[i] + 1;
57+
offset += mp_frozen_str_sizes[i] + 1;
58+
}
59+
return NULL;
60+
}
61+
62+
#endif
63+
64+
#if MICROPY_MODULE_FROZEN_MPY
65+
66+
#include "py/emitglue.h"
67+
68+
extern const char mp_frozen_mpy_names[];
69+
extern const mp_raw_code_t *const mp_frozen_mpy_content[];
70+
71+
STATIC const mp_raw_code_t *mp_find_frozen_mpy(const char *str, size_t len) {
72+
const char *name = mp_frozen_mpy_names;
73+
for (size_t i = 0; *name != 0; i++) {
74+
size_t l = strlen(name);
75+
if (l == len && !memcmp(str, name, l)) {
76+
return mp_frozen_mpy_content[i];
77+
}
78+
name += l + 1;
5779
}
5880
return NULL;
5981
}
6082

61-
#endif // MICROPY_MODULE_FROZEN
83+
#endif
84+
85+
#if MICROPY_MODULE_FROZEN
86+
87+
int mp_find_frozen_module(const char *str, size_t len, void **data) {
88+
#if MICROPY_MODULE_FROZEN_STR
89+
mp_lexer_t *lex = mp_find_frozen_str(str, len);
90+
if (lex != NULL) {
91+
*data = lex;
92+
return MP_FROZEN_STR;
93+
}
94+
#endif
95+
#if MICROPY_MODULE_FROZEN_MPY
96+
const mp_raw_code_t *rc = mp_find_frozen_mpy(str, len);
97+
if (rc != NULL) {
98+
*data = (void*)rc;
99+
return MP_FROZEN_MPY;
100+
}
101+
#endif
102+
return MP_FROZEN_NONE;
103+
}
104+
105+
#endif

‎py/frozenmod.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,10 @@
2424
* THE SOFTWARE.
2525
*/
2626

27-
mp_lexer_t *mp_find_frozen_module(const char *str, int len);
27+
enum {
28+
MP_FROZEN_NONE,
29+
MP_FROZEN_STR,
30+
MP_FROZEN_MPY,
31+
};
32+
33+
int mp_find_frozen_module(const char *str, size_t len, void **data);

‎py/mpconfig.h‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@
234234
// Whether generated code can persist independently of the VM/runtime instance
235235
// This is enabled automatically when needed by other features
236236
#ifndef MICROPY_PERSISTENT_CODE
237-
#define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE)
237+
#define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE || MICROPY_MODULE_FROZEN_MPY)
238238
#endif
239239

240240
// Whether to emit x64 native code
@@ -526,9 +526,19 @@ typedef double mp_float_t;
526526
#define MICROPY_MODULE_WEAK_LINKS (0)
527527
#endif
528528

529-
// Whether frozen modules are supported
529+
// Whether frozen modules are supported in the form of strings
530+
#ifndef MICROPY_MODULE_FROZEN_STR
531+
#define MICROPY_MODULE_FROZEN_STR (0)
532+
#endif
533+
534+
// Whether frozen modules are supported in the form of .mpy files
535+
#ifndef MICROPY_MODULE_FROZEN_MPY
536+
#define MICROPY_MODULE_FROZEN_MPY (0)
537+
#endif
538+
539+
// Convenience macro for whether frozen modules are supported
530540
#ifndef MICROPY_MODULE_FROZEN
531-
#define MICROPY_MODULE_FROZEN (0)
541+
#define MICROPY_MODULE_FROZEN (MICROPY_MODULE_FROZEN_STR || MICROPY_MODULE_FROZEN_MPY)
532542
#endif
533543

534544
// Whether you can override builtins in the builtins module

‎py/qstr.c‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,20 +87,27 @@ mp_uint_t qstr_compute_hash(const byte *data, size_t len) {
8787
return hash;
8888
}
8989

90-
STATIC const qstr_pool_t const_pool = {
90+
const qstr_pool_t mp_qstr_const_pool = {
9191
NULL, // no previous pool
9292
0, // no previous pool
9393
10, // set so that the first dynamically allocated pool is twice this size; must be <= the len (just below)
94-
MP_QSTR_number_of, // corresponds to number of strings in array just below
94+
MP_QSTRnumber_of, // corresponds to number of strings in array just below
9595
{
9696
#define QDEF(id, str) str,
9797
#include "genhdr/qstrdefs.generated.h"
9898
#undef QDEF
9999
},
100100
};
101101

102+
#ifdef MICROPY_QSTR_EXTRA_POOL
103+
extern const qstr_pool_t MICROPY_QSTR_EXTRA_POOL;
104+
#define CONST_POOL MICROPY_QSTR_EXTRA_POOL
105+
#else
106+
#define CONST_POOL mp_qstr_const_pool
107+
#endif
108+
102109
void qstr_init(void) {
103-
MP_STATE_VM(last_pool) = (qstr_pool_t*)&const_pool; // we won't modify the const_pool since it has no allocated room left
110+
MP_STATE_VM(last_pool) = (qstr_pool_t*)&CONST_POOL; // we won't modify the const_pool since it has no allocated room left
104111
MP_STATE_VM(qstr_last_chunk) = NULL;
105112
}
106113

@@ -258,7 +265,7 @@ void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, si
258265
*n_qstr = 0;
259266
*n_str_data_bytes = 0;
260267
*n_total_bytes = 0;
261-
for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &const_pool; pool = pool->prev) {
268+
for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &CONST_POOL; pool = pool->prev) {
262269
*n_pool += 1;
263270
*n_qstr += pool->len;
264271
for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) {
@@ -275,7 +282,7 @@ void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, si
275282

276283
#if MICROPY_PY_MICROPYTHON_MEM_INFO
277284
void qstr_dump_data(void) {
278-
for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &const_pool; pool = pool->prev) {
285+
for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &CONST_POOL; pool = pool->prev) {
279286
for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) {
280287
mp_printf(&mp_plat_print, "Q(%s)\n", Q_GET_DATA(*q));
281288
}

‎py/qstr.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ enum {
4040
#define QDEF(id, str) id,
4141
#include "genhdr/qstrdefs.generated.h"
4242
#undef QDEF
43-
MP_QSTR_number_of,
43+
MP_QSTRnumber_of, // no underscore so it can't clash with any of the above
4444
};
4545

4646
typedef size_t qstr;

0 commit comments

Comments
 (0)