Skip to content

Commit 24c02c4

Browse files
jimmojimmo
committed
py/makemoduledefs.py: Add a way to register extensible built-in modules.
Signed-off-by: Jim Mussared <jim.mussared@gmail.com>
1 parent 45ac651 commit 24c02c4

8 files changed

Lines changed: 97 additions & 33 deletions

File tree

‎py/builtinhelp.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ STATIC void mp_help_print_modules(void) {
7979
mp_obj_t list = mp_obj_new_list(0, NULL);
8080

8181
mp_help_add_from_map(list, &mp_builtin_module_map);
82+
mp_help_add_from_map(list, &mp_builtin_extensible_module_map);
8283

8384
#if MICROPY_MODULE_FROZEN
8485
extern const char mp_frozen_names[];

‎py/builtinimport.c‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -384,28 +384,22 @@ STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name,
384384

385385
// An import of a non-extensible built-in will always bypass the
386386
// filesystem. e.g. `import micropython` or `import pyb`.
387-
module_obj = mp_module_get_builtin(level_mod_name);
387+
module_obj = mp_module_get_builtin(level_mod_name, false);
388388
if (module_obj != MP_OBJ_NULL) {
389389
return module_obj;
390390
}
391391

392-
#if MICROPY_PY_SYS
393-
// Never allow sys to be overridden from the filesystem. If weak links
394-
// are disabled, then this also provides a default weak link so that
395-
// `import sys` is treated like `import usys` (and therefore bypasses
396-
// the filesystem).
397-
if (level_mod_name == MP_QSTR_sys) {
398-
return MP_OBJ_FROM_PTR(&mp_module_sys);
399-
}
400-
#endif
401-
402392
// First module in the dotted-name; search for a directory or file
403393
// relative to all the locations in sys.path.
404394
stat = stat_top_level(level_mod_name, &path);
405395

406396
// TODO: If stat failed, now try extensible built-in modules.
407-
408-
// TODO: If importing `ufoo`, try `foo`.
397+
if (stat == MP_IMPORT_STAT_NO_EXIST) {
398+
module_obj = mp_module_get_builtin(level_mod_name, true);
399+
if (module_obj != MP_OBJ_NULL) {
400+
return module_obj;
401+
}
402+
}
409403
} else {
410404
DEBUG_printf("Searching for sub-module\n");
411405

@@ -637,7 +631,7 @@ mp_obj_t mp_builtin___import___default(size_t n_args, const mp_obj_t *args) {
637631

638632
// Try the name directly as a built-in.
639633
qstr module_name_qstr = mp_obj_str_get_qstr(args[0]);
640-
mp_obj_t module_obj = mp_module_get_builtin(module_name_qstr);
634+
mp_obj_t module_obj = mp_module_get_builtin(module_name_qstr, false);
641635
if (module_obj != MP_OBJ_NULL) {
642636
return module_obj;
643637
}

‎py/makemoduledefs.py‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,17 @@
11
"""
22
This pre-processor parses a single file containing a list of
3-
MP_REGISTER_MODULE(module_name, obj_module)
4-
These are used to generate a header with the required entries for
5-
"mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c
3+
`MP_REGISTER_MODULE(MP_QSTR_module_name, obj_module)` or
4+
`MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_module_name, obj_module)`
5+
(i.e. the output of `py/makeqstrdefs.py cat module`).
6+
7+
The output is a header (typically moduledefs.h) which is included by
8+
py/objmodule.c that contains entries to be included in the definition of
9+
- mp_rom_map_elem_t mp_builtin_module_table[]
10+
- mp_rom_map_elem_t mp_builtin_extensible_module_table[]
11+
12+
Extensible modules are modules that can be overridden from the filesystem, see
13+
py/builtinimnport.c:process_import_at_level. Regular modules will always use
14+
the built-in version.
615
"""
716

817
from __future__ import print_function
@@ -13,7 +22,10 @@
1322
import argparse
1423

1524

16-
pattern = re.compile(r"\s*MP_REGISTER_MODULE\((.*?),\s*(.*?)\);", flags=re.DOTALL)
25+
pattern = re.compile(
26+
r"\s*(MP_REGISTER_MODULE|MP_REGISTER_EXTENSIBLE_MODULE)\(MP_QSTR_(.*?),\s*(.*?)\);",
27+
flags=re.DOTALL,
28+
)
1729

1830

1931
def find_module_registrations(filename):
@@ -37,14 +49,23 @@ def generate_module_table_header(modules):
3749

3850
# Print header file for all external modules.
3951
mod_defs = set()
52+
extensible_mod_defs = set()
4053
print("// Automatically generated by makemoduledefs.py.\n")
41-
for module_name, obj_module in modules:
54+
for macro_name, module_name, obj_module in modules:
4255
mod_def = "MODULE_DEF_{}".format(module_name.upper())
43-
mod_defs.add(mod_def)
56+
if macro_name == "MP_REGISTER_MODULE":
57+
mod_defs.add(mod_def)
58+
elif macro_name == "MP_REGISTER_EXTENSIBLE_MODULE":
59+
extensible_mod_defs.add(mod_def)
4460
if "," in obj_module:
4561
print(
46-
"ERROR: Call to MP_REGISTER_MODULE({}, {}) should be MP_REGISTER_MODULE({}, {})\n".format(
47-
module_name, obj_module, module_name, obj_module.split(",")[0]
62+
"ERROR: Call to {}({}, {}) should be {}({}, {})\n".format(
63+
macro_name,
64+
module_name,
65+
obj_module,
66+
macro_name,
67+
module_name,
68+
obj_module.split(",")[0],
4869
),
4970
file=sys.stderr,
5071
)
@@ -53,7 +74,7 @@ def generate_module_table_header(modules):
5374
(
5475
"extern const struct _mp_obj_module_t {obj_module};\n"
5576
"#undef {mod_def}\n"
56-
"#define {mod_def} {{ MP_ROM_QSTR({module_name}), MP_ROM_PTR(&{obj_module}) }},\n"
77+
"#define {mod_def} {{ MP_ROM_QSTR(MP_QSTR_{module_name}), MP_ROM_PTR(&{obj_module}) }},\n"
5778
).format(
5879
module_name=module_name,
5980
obj_module=obj_module,
@@ -68,6 +89,13 @@ def generate_module_table_header(modules):
6889

6990
print("// MICROPY_REGISTERED_MODULES")
7091

92+
print("\n#define MICROPY_REGISTERED_EXTENSIBLE_MODULES \\")
93+
94+
for mod_def in sorted(extensible_mod_defs):
95+
print(" {mod_def} \\".format(mod_def=mod_def))
96+
97+
print("// MICROPY_REGISTERED_EXTENSIBLE_MODULES")
98+
7199

72100
def main():
73101
parser = argparse.ArgumentParser()

‎py/makeqstrdefs.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
# Extract MP_COMPRESSED_ROM_TEXT("") macros. (Which come from MP_ERROR_TEXT)
2222
_MODE_COMPRESS = "compress"
2323

24-
# Extract MP_REGISTER_MODULE(...) macros.
24+
# Extract MP_REGISTER_(EXTENSIBLE_)MODULE(...) macros.
2525
_MODE_MODULE = "module"
2626

2727
# Extract MP_REGISTER_ROOT_POINTER(...) macros.
@@ -93,7 +93,7 @@ def process_file(f):
9393
elif args.mode == _MODE_COMPRESS:
9494
re_match = re.compile(r'MP_COMPRESSED_ROM_TEXT\("([^"]*)"\)')
9595
elif args.mode == _MODE_MODULE:
96-
re_match = re.compile(r"MP_REGISTER_MODULE\(.*?,\s*.*?\);")
96+
re_match = re.compile(r"MP_REGISTER_(?:EXTENSIBLE_)?MODULE\(.*?,\s*.*?\);")
9797
elif args.mode == _MODE_ROOT_POINTER:
9898
re_match = re.compile(r"MP_REGISTER_ROOT_POINTER\(.*?\);")
9999
output = []

‎py/obj.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,9 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t;
434434
// param obj_module: mp_obj_module_t instance
435435
#define MP_REGISTER_MODULE(module_name, obj_module)
436436

437+
// As above, but allow this module to be extended from the filesystem.
438+
#define MP_REGISTER_EXTENSIBLE_MODULE(module_name, obj_module)
439+
437440
// Declare a root pointer (to avoid garbage collection of a global static variable).
438441
// param variable_declaration: a valid C variable declaration
439442
#define MP_REGISTER_ROOT_POINTER(variable_declaration)

‎py/objmodule.c‎

Lines changed: 42 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -166,17 +166,54 @@ mp_obj_t mp_obj_new_module(qstr module_name) {
166166
// Global module table and related functions
167167

168168
STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = {
169-
// builtin modules declared with MP_REGISTER_MODULE()
169+
// built-in modules declared with MP_REGISTER_MODULE()
170170
MICROPY_REGISTERED_MODULES
171171
};
172172
MP_DEFINE_CONST_MAP(mp_builtin_module_map, mp_builtin_module_table);
173173

174-
// Attempts to find (and initialise) a builtin, otherwise returns
174+
STATIC const mp_rom_map_elem_t mp_builtin_extensible_module_table[] = {
175+
// built-in modules declared with MP_REGISTER_EXTENSIBLE_MODULE()
176+
MICROPY_REGISTERED_EXTENSIBLE_MODULES
177+
};
178+
MP_DEFINE_CONST_MAP(mp_builtin_extensible_module_map, mp_builtin_extensible_module_table);
179+
180+
// Attempts to find (and initialise) a built-in, otherwise returns
175181
// MP_OBJ_NULL.
176-
mp_obj_t mp_module_get_builtin(qstr module_name) {
177-
mp_map_elem_t *elem = mp_map_lookup((mp_map_t *)&mp_builtin_module_map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP);
182+
mp_obj_t mp_module_get_builtin(qstr module_name, bool extensible) {
183+
mp_map_elem_t *elem = mp_map_lookup((mp_map_t *)(extensible ? &mp_builtin_extensible_module_map : &mp_builtin_module_map), MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP);
178184
if (!elem) {
179-
return MP_OBJ_NULL;
185+
#if MICROPY_PY_SYS
186+
// Special case for sys, which isn't extensible but can always be
187+
// imported with the alias `usys`.
188+
if (module_name == MP_QSTR_usys) {
189+
return MP_OBJ_FROM_PTR(&mp_module_sys);
190+
}
191+
#endif
192+
193+
if (extensible) {
194+
// At this point we've already tried non-extensible built-ins, the
195+
// filesystem, and now extensible built-ins. No match, so fail
196+
// the import.
197+
return MP_OBJ_NULL;
198+
}
199+
200+
// We're trying to match a non-extensible built-in (i.e. before trying
201+
// the filesystem), but if the user is importing `ufoo`, _and_ `foo`
202+
// is an extensible module, then allow it as a way of forcing the
203+
// built-in. Essentially, this makes it as if all the extensible
204+
// built-ins also had non-extensible aliases named `ufoo`. Newer code
205+
// should be using sys.path to force the built-in, but this retains
206+
// the old behaviour of the u-prefix being used to force a built-in
207+
// import.
208+
size_t module_name_len;
209+
const char *module_name_str = (const char *)qstr_data(module_name, &module_name_len);
210+
if (module_name_str[0] != 'u') {
211+
return MP_OBJ_NULL;
212+
}
213+
elem = mp_map_lookup((mp_map_t *)&mp_builtin_extensible_module_map, MP_OBJ_NEW_QSTR(qstr_from_strn(module_name_str + 1, module_name_len - 1)), MP_MAP_LOOKUP);
214+
if (!elem) {
215+
return MP_OBJ_NULL;
216+
}
180217
}
181218

182219
#if MICROPY_MODULE_BUILTIN_INIT

‎py/objmodule.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,9 @@
3232
#define MP_MODULE_ATTR_DELEGATION_ENTRY(ptr) { MP_ROM_QSTR(MP_QSTRnull), MP_ROM_PTR(ptr) }
3333

3434
extern const mp_map_t mp_builtin_module_map;
35+
extern const mp_map_t mp_builtin_extensible_module_map;
3536

36-
mp_obj_t mp_module_get_builtin(qstr module_name);
37+
mp_obj_t mp_module_get_builtin(qstr module_name, bool extensible);
3738

3839
void mp_module_generic_attr(qstr attr, mp_obj_t *dest, const uint16_t *keys, mp_obj_t *values);
3940

‎py/repl.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,8 +162,8 @@ STATIC bool test_qstr(mp_obj_t obj, qstr name) {
162162
return dest[0] != MP_OBJ_NULL;
163163
} else {
164164
// try builtin module
165-
return mp_map_lookup((mp_map_t *)&mp_builtin_module_map,
166-
MP_OBJ_NEW_QSTR(name), MP_MAP_LOOKUP);
165+
return mp_map_lookup((mp_map_t *)&mp_builtin_module_map, MP_OBJ_NEW_QSTR(name), MP_MAP_LOOKUP) ||
166+
mp_map_lookup((mp_map_t *)&mp_builtin_extensible_module_map, MP_OBJ_NEW_QSTR(name), MP_MAP_LOOKUP);
167167
}
168168
}
169169

0 commit comments

Comments
 (0)