py: Allow registration of modules at their definition.

During make, makemoduledefs.py parses the current builds c files for
MP_REGISTER_MODULE(module_name, obj_module, enabled_define)

These are used to generate a header with the required entries for
"mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c

ports/windows/msvc/genhdr.targets

@@ -4,7 +4,7 @@
   <Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/>
 
   <!--Generate qstrdefs.generated.h and mpversion.h similar to what is done in py/mkrules.mk and py/py.mk-->
-  <Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeQstrData">
+  <Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeQstrData">
   </Target>
 
   <PropertyGroup>
@@ -83,6 +83,20 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) {
     <Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py cat $(DestDir)qstr.i.last $(DestDir)qstr $(QstrDefsCollected)"/>
   </Target>
 
+  <Target Name="MakeModuleDefs" DependsOnTargets="MakeDestDir">
+    <PropertyGroup>
+      <DestFile>$(DestDir)moduledefs.h</DestFile>
+      <TmpFile>$(DestFile).tmp</TmpFile>
+    </PropertyGroup>
+    <ItemGroup>
+      <PyUserModuleFiles Include="@(ClCompile)">
+        <Path>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', ''))</Path>
+      </PyUserModuleFiles>
+    </ItemGroup>
+    <Exec Command="$(PyPython) $(PySrcDir)makemoduledefs.py --vpath=&quot;., $(PyBaseDir), $(PyUserCModules)&quot; @(PyUserModuleFiles->'%(Path)', ' ') > $(TmpFile)"/>
+    <MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
+  </Target>
+
   <Target Name="MakeQstrData" DependsOnTargets="MakeQstrDefs" Inputs="$(QstrDefsCollected);$(PyQstrDefs);$(QstrDefs)" Outputs="$(QstrGen)">
     <PropertyGroup>
       <TmpFile>$(QstrGen).tmp</TmpFile>

py/makemoduledefs.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+
+# This pre-processor parses provided objects' c files for
+# MP_REGISTER_MODULE(module_name, obj_module, enabled_define)
+# These are used to generate a header with the required entries for
+# "mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c
+
+from __future__ import print_function
+
+import re
+import os
+import argparse
+
+
+pattern = re.compile(
+    r"[\n;]\s*MP_REGISTER_MODULE\((.*?),\s*(.*?),\s*(.*?)\);",
+    flags=re.DOTALL
+)
+
+
+def find_c_file(obj_file, vpath):
+    """ Search vpaths for the c file that matches the provided object_file.
+
+    :param str obj_file: object file to find the matching c file for
+    :param List[str] vpath: List of base paths, similar to gcc vpath
+    :return: str path to c file or None
+    """
+    c_file = None
+    relative_c_file = os.path.splitext(obj_file)[0] + ".c"
+    relative_c_file = relative_c_file.lstrip('/\\')
+    for p in vpath:
+        possible_c_file = os.path.join(p, relative_c_file)
+        if os.path.exists(possible_c_file):
+            c_file = possible_c_file
+            break
+
+    return c_file
+
+
+def find_module_registrations(c_file):
+    """ Find any MP_REGISTER_MODULE definitions in the provided c file.
+
+    :param str c_file: path to c file to check
+    :return: List[(module_name, obj_module, enabled_define)]
+    """
+    global pattern
+
+    if c_file is None:
+        # No c file to match the object file, skip
+        return set()
+
+    with open(c_file) as c_file_obj:
+        return set(re.findall(pattern, c_file_obj.read()))
+
+
+def generate_module_table_header(modules):
+    """ Generate header with module table entries for builtin modules.
+
+    :param List[(module_name, obj_module, enabled_define)] modules: module defs
+    :return: None
+    """
+
+    # Print header file for all external modules.
+    mod_defs = []
+    print("// Automatically generated by makemoduledefs.py.\n")
+    for module_name, obj_module, enabled_define in modules:
+        mod_def = "MODULE_DEF_{}".format(module_name.upper())
+        mod_defs.append(mod_def)
+        print((
+            "#if ({enabled_define})\n"
+            "    extern const struct _mp_obj_module_t {obj_module};\n"
+            "    #define {mod_def} {{ MP_ROM_QSTR({module_name}), MP_ROM_PTR(&{obj_module}) }},\n"
+            "#else\n"
+            "    #define {mod_def}\n"
+            "#endif\n"
+            ).format(module_name=module_name, obj_module=obj_module,
+                     enabled_define=enabled_define, mod_def=mod_def)
+        )
+
+    print("\n#define MICROPY_REGISTERED_MODULES \\")
+
+    for mod_def in mod_defs:
+        print("    {mod_def} \\".format(mod_def=mod_def))
+
+    print("// MICROPY_REGISTERED_MODULES")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vpath", default=".",
+                        help="comma separated list of folders to search for c files in")
+    parser.add_argument("files", nargs="*",
+                        help="list of c files to search")
+    args = parser.parse_args()
+
+    vpath = [p.strip() for p in args.vpath.split(',')]
+
+    modules = set()
+    for obj_file in args.files:
+        c_file = find_c_file(obj_file, vpath)
+        modules |= find_module_registrations(c_file)
+
+    generate_module_table_header(sorted(modules))
+
+
+if __name__ == '__main__':
+    main()

py/modarray.c

@@ -40,4 +40,6 @@ const mp_obj_module_t mp_module_array = {
     .globals = (mp_obj_dict_t*)&mp_module_array_globals,
 };
 
+MP_REGISTER_MODULE(MP_QSTR_array, mp_module_array, MICROPY_PY_ARRAY);
+
 #endif

py/obj.h

@@ -326,6 +326,13 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t;
 #define MP_DEFINE_CONST_STATICMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_staticmethod}, fun_name}
 #define MP_DEFINE_CONST_CLASSMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_classmethod}, fun_name}
 
+// Declare a module as a builtin, processed by makemoduledefs.py
+// param module_name: MP_QSTR_<module name>
+// param obj_module: mp_obj_module_t instance
+// prarm enabled_define: used as `#if (enabled_define) around entry`
+
+#define MP_REGISTER_MODULE(module_name, obj_module, enabled_define)
+
 // Underlying map/hash table implementation (not dict object or map function)
 
 typedef struct _mp_map_elem_t {

py/objmodule.c

@@ -31,6 +31,8 @@
 #include "py/runtime.h"
 #include "py/builtin.h"
 
+#include "genhdr/moduledefs.h"
+
 STATIC void module_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
     (void)kind;
     mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in);
@@ -136,9 +138,6 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = {
     { MP_ROM_QSTR(MP_QSTR_builtins), MP_ROM_PTR(&mp_module_builtins) },
     { MP_ROM_QSTR(MP_QSTR_micropython), MP_ROM_PTR(&mp_module_micropython) },
 
-#if MICROPY_PY_ARRAY
-    { MP_ROM_QSTR(MP_QSTR_array), MP_ROM_PTR(&mp_module_array) },
-#endif
 #if MICROPY_PY_IO
     { MP_ROM_QSTR(MP_QSTR_uio), MP_ROM_PTR(&mp_module_io) },
 #endif
@@ -226,6 +225,11 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = {
 
     // extra builtin modules as defined by a port
     MICROPY_PORT_BUILTIN_MODULES
+
+    #ifdef MICROPY_REGISTERED_MODULES
+    // builtin modules declared with MP_REGISTER_MODULE()
+    MICROPY_REGISTERED_MODULES
+    #endif
 };
 
 MP_DEFINE_CONST_MAP(mp_builtin_module_map, mp_builtin_module_table);

py/py.mk

@@ -323,6 +323,11 @@ $(HEADER_BUILD)/qstrdefs.generated.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_C
 	$(Q)cat $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) | $(SED) 's/^Q(.*)/"&"/' | $(CPP) $(CFLAGS) - | $(SED) 's/^"\(Q(.*)\)"/\1/' > $(HEADER_BUILD)/qstrdefs.preprocessed.h
 	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@
 
+# build a list of registered modules for py/objmodule.c.
+$(HEADER_BUILD)/moduledefs.h: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h
+	@$(ECHO) "GEN $@"
+	$(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py --vpath="., $(TOP), $(USER_C_MODULES)" $(SRC_QSTR) > $@
+
 # Force nlr code to always be compiled with space-saving optimisation so
 # that the function preludes are of a minimal and predictable form.
 $(PY_BUILD)/nlr%.o: CFLAGS += -Os