Skip to content

Commit cf22f47

Browse files
pi-anldpgeorge
authored andcommitted
py: Allow registration of modules at their definition.
During make, makemoduledefs.py parses the current builds c files for MP_REGISTER_MODULE(module_name, obj_module, enabled_define) These are used to generate a header with the required entries for "mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c
1 parent e4ac104 commit cf22f47

6 files changed

Lines changed: 143 additions & 4 deletions

File tree

ports/windows/msvc/genhdr.targets

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/>
55

66
<!--Generate qstrdefs.generated.h and mpversion.h similar to what is done in py/mkrules.mk and py/py.mk-->
7-
<Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeQstrData">
7+
<Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeQstrData">
88
</Target>
99

1010
<PropertyGroup>
@@ -83,6 +83,20 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) {
8383
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py cat $(DestDir)qstr.i.last $(DestDir)qstr $(QstrDefsCollected)"/>
8484
</Target>
8585

86+
<Target Name="MakeModuleDefs" DependsOnTargets="MakeDestDir">
87+
<PropertyGroup>
88+
<DestFile>$(DestDir)moduledefs.h</DestFile>
89+
<TmpFile>$(DestFile).tmp</TmpFile>
90+
</PropertyGroup>
91+
<ItemGroup>
92+
<PyUserModuleFiles Include="@(ClCompile)">
93+
<Path>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', ''))</Path>
94+
</PyUserModuleFiles>
95+
</ItemGroup>
96+
<Exec Command="$(PyPython) $(PySrcDir)makemoduledefs.py --vpath=&quot;., $(PyBaseDir), $(PyUserCModules)&quot; @(PyUserModuleFiles->'%(Path)', ' ') > $(TmpFile)"/>
97+
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
98+
</Target>
99+
86100
<Target Name="MakeQstrData" DependsOnTargets="MakeQstrDefs" Inputs="$(QstrDefsCollected);$(PyQstrDefs);$(QstrDefs)" Outputs="$(QstrGen)">
87101
<PropertyGroup>
88102
<TmpFile>$(QstrGen).tmp</TmpFile>

py/makemoduledefs.py

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#!/usr/bin/env python
2+
3+
# This pre-processor parses provided objects' c files for
4+
# MP_REGISTER_MODULE(module_name, obj_module, enabled_define)
5+
# These are used to generate a header with the required entries for
6+
# "mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c
7+
8+
from __future__ import print_function
9+
10+
import re
11+
import os
12+
import argparse
13+
14+
15+
pattern = re.compile(
16+
r"[\n;]\s*MP_REGISTER_MODULE\((.*?),\s*(.*?),\s*(.*?)\);",
17+
flags=re.DOTALL
18+
)
19+
20+
21+
def find_c_file(obj_file, vpath):
22+
""" Search vpaths for the c file that matches the provided object_file.
23+
24+
:param str obj_file: object file to find the matching c file for
25+
:param List[str] vpath: List of base paths, similar to gcc vpath
26+
:return: str path to c file or None
27+
"""
28+
c_file = None
29+
relative_c_file = os.path.splitext(obj_file)[0] + ".c"
30+
relative_c_file = relative_c_file.lstrip('/\\')
31+
for p in vpath:
32+
possible_c_file = os.path.join(p, relative_c_file)
33+
if os.path.exists(possible_c_file):
34+
c_file = possible_c_file
35+
break
36+
37+
return c_file
38+
39+
40+
def find_module_registrations(c_file):
41+
""" Find any MP_REGISTER_MODULE definitions in the provided c file.
42+
43+
:param str c_file: path to c file to check
44+
:return: List[(module_name, obj_module, enabled_define)]
45+
"""
46+
global pattern
47+
48+
if c_file is None:
49+
# No c file to match the object file, skip
50+
return set()
51+
52+
with open(c_file) as c_file_obj:
53+
return set(re.findall(pattern, c_file_obj.read()))
54+
55+
56+
def generate_module_table_header(modules):
57+
""" Generate header with module table entries for builtin modules.
58+
59+
:param List[(module_name, obj_module, enabled_define)] modules: module defs
60+
:return: None
61+
"""
62+
63+
# Print header file for all external modules.
64+
mod_defs = []
65+
print("// Automatically generated by makemoduledefs.py.\n")
66+
for module_name, obj_module, enabled_define in modules:
67+
mod_def = "MODULE_DEF_{}".format(module_name.upper())
68+
mod_defs.append(mod_def)
69+
print((
70+
"#if ({enabled_define})\n"
71+
" extern const struct _mp_obj_module_t {obj_module};\n"
72+
" #define {mod_def} {{ MP_ROM_QSTR({module_name}), MP_ROM_PTR(&{obj_module}) }},\n"
73+
"#else\n"
74+
" #define {mod_def}\n"
75+
"#endif\n"
76+
).format(module_name=module_name, obj_module=obj_module,
77+
enabled_define=enabled_define, mod_def=mod_def)
78+
)
79+
80+
print("\n#define MICROPY_REGISTERED_MODULES \\")
81+
82+
for mod_def in mod_defs:
83+
print(" {mod_def} \\".format(mod_def=mod_def))
84+
85+
print("// MICROPY_REGISTERED_MODULES")
86+
87+
88+
def main():
89+
parser = argparse.ArgumentParser()
90+
parser.add_argument("--vpath", default=".",
91+
help="comma separated list of folders to search for c files in")
92+
parser.add_argument("files", nargs="*",
93+
help="list of c files to search")
94+
args = parser.parse_args()
95+
96+
vpath = [p.strip() for p in args.vpath.split(',')]
97+
98+
modules = set()
99+
for obj_file in args.files:
100+
c_file = find_c_file(obj_file, vpath)
101+
modules |= find_module_registrations(c_file)
102+
103+
generate_module_table_header(sorted(modules))
104+
105+
106+
if __name__ == '__main__':
107+
main()

py/modarray.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,6 @@ const mp_obj_module_t mp_module_array = {
4040
.globals = (mp_obj_dict_t*)&mp_module_array_globals,
4141
};
4242

43+
MP_REGISTER_MODULE(MP_QSTR_array, mp_module_array, MICROPY_PY_ARRAY);
44+
4345
#endif

py/obj.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,13 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t;
326326
#define MP_DEFINE_CONST_STATICMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_staticmethod}, fun_name}
327327
#define MP_DEFINE_CONST_CLASSMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_classmethod}, fun_name}
328328

329+
// Declare a module as a builtin, processed by makemoduledefs.py
330+
// param module_name: MP_QSTR_<module name>
331+
// param obj_module: mp_obj_module_t instance
332+
// prarm enabled_define: used as `#if (enabled_define) around entry`
333+
334+
#define MP_REGISTER_MODULE(module_name, obj_module, enabled_define)
335+
329336
// Underlying map/hash table implementation (not dict object or map function)
330337

331338
typedef struct _mp_map_elem_t {

py/objmodule.c

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
#include "py/runtime.h"
3232
#include "py/builtin.h"
3333

34+
#include "genhdr/moduledefs.h"
35+
3436
STATIC void module_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
3537
(void)kind;
3638
mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in);
@@ -136,9 +138,6 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = {
136138
{ MP_ROM_QSTR(MP_QSTR_builtins), MP_ROM_PTR(&mp_module_builtins) },
137139
{ MP_ROM_QSTR(MP_QSTR_micropython), MP_ROM_PTR(&mp_module_micropython) },
138140

139-
#if MICROPY_PY_ARRAY
140-
{ MP_ROM_QSTR(MP_QSTR_array), MP_ROM_PTR(&mp_module_array) },
141-
#endif
142141
#if MICROPY_PY_IO
143142
{ MP_ROM_QSTR(MP_QSTR_uio), MP_ROM_PTR(&mp_module_io) },
144143
#endif
@@ -226,6 +225,11 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = {
226225

227226
// extra builtin modules as defined by a port
228227
MICROPY_PORT_BUILTIN_MODULES
228+
229+
#ifdef MICROPY_REGISTERED_MODULES
230+
// builtin modules declared with MP_REGISTER_MODULE()
231+
MICROPY_REGISTERED_MODULES
232+
#endif
229233
};
230234

231235
MP_DEFINE_CONST_MAP(mp_builtin_module_map, mp_builtin_module_table);

py/py.mk

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,11 @@ $(HEADER_BUILD)/qstrdefs.generated.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_C
323323
$(Q)cat $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) | $(SED) 's/^Q(.*)/"&"/' | $(CPP) $(CFLAGS) - | $(SED) 's/^"\(Q(.*)\)"/\1/' > $(HEADER_BUILD)/qstrdefs.preprocessed.h
324324
$(Q)$(PYTHON) $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@
325325

326+
# build a list of registered modules for py/objmodule.c.
327+
$(HEADER_BUILD)/moduledefs.h: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h
328+
@$(ECHO) "GEN $@"
329+
$(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py --vpath="., $(TOP), $(USER_C_MODULES)" $(SRC_QSTR) > $@
330+
326331
# Force nlr code to always be compiled with space-saving optimisation so
327332
# that the function preludes are of a minimal and predictable form.
328333
$(PY_BUILD)/nlr%.o: CFLAGS += -Os

0 commit comments

Comments
 (0)