Support muti-module for AOT mode (#2482)

Support muti-module for AOT mode, currently only implement the
multi-module's function import feature for AOT, the memory/table/
global import are not implemented yet.

And update wamr-test-suites scripts, multi-module sample and some
CIs accordingly.

.github/workflows/compilation_on_android_ubuntu.yml

@@ -328,7 +328,12 @@ jobs:
         working-directory: samples/wasm-c-api
 
   build_samples_others:
-    needs: [build_iwasm]
+    needs:
+      [
+        build_iwasm,
+        build_llvm_libraries_on_ubuntu_2204,
+        build_wamrc,
+      ]    
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -341,6 +346,9 @@ jobs:
           [
             "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz",
           ]
+        include:
+          - os: ubuntu-22.04
+            llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -358,7 +366,23 @@ jobs:
           sudo wget ${{ matrix.wabt_release }}
           sudo tar -xzf wabt-1.0.31-*.tar.gz
           sudo mv wabt-1.0.31 wabt
-
+      - name: Get LLVM libraries
+        id: retrieve_llvm_libs
+        uses: actions/cache@v3
+        with:
+          path: |
+            ./core/deps/llvm/build/bin
+            ./core/deps/llvm/build/include
+            ./core/deps/llvm/build/lib
+            ./core/deps/llvm/build/libexec
+            ./core/deps/llvm/build/share
+          key: ${{ matrix.llvm_cache_key }}
+      - name: Build wamrc
+        run: |
+          mkdir build && cd build
+          cmake ..
+          cmake --build . --config Release --parallel 4
+        working-directory: wamr-compiler
       - name: Build Sample [basic]
         run: |
           cd samples/basic
@@ -385,9 +409,10 @@ jobs:
         run: |
           cd samples/multi-module
           mkdir build && cd build
-          cmake ..
+          cmake .. -DWAMR_BUILD_AOT=1
           cmake --build . --config Release --parallel 4
-          ./multi_module
+          ./multi_module mC.wasm
+          ./multi_module mC.aot
 
       - name: Build Sample [spawn-thread]
         run: |

.github/workflows/compilation_on_macos.yml

@@ -245,7 +245,7 @@ jobs:
         working-directory: samples/wasm-c-api
 
   build_samples_others:
-    needs: [build_iwasm]
+    needs: [build_iwasm, build_wamrc]
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -304,7 +304,7 @@ jobs:
           mkdir build && cd build
           cmake ..
           cmake --build . --config Release --parallel 4
-          ./multi_module
+          ./multi_module mC.wasm
 
       - name: Build Sample [spawn-thread]
         run: |

.github/workflows/nightly_run.yml

@@ -380,7 +380,12 @@ jobs:
         working-directory: samples/wasm-c-api
 
   build_samples_others:
-    needs: [build_iwasm]
+    needs:
+      [
+        build_iwasm,
+        build_llvm_libraries_on_ubuntu_2004,
+        build_wamrc,
+      ]
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -393,6 +398,9 @@ jobs:
           [
             "https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz",
           ]
+        include:
+          - os: ubuntu-20.04
+            llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -409,6 +417,26 @@ jobs:
           sudo wget ${{ matrix.wabt_release }}
           sudo tar -xzf wabt-1.0.31-*.tar.gz
           sudo mv wabt-1.0.31 wabt
+
+      - name: Get LLVM libraries
+        id: retrieve_llvm_libs
+        uses: actions/cache@v3
+        with:
+          path: |
+            ./core/deps/llvm/build/bin
+            ./core/deps/llvm/build/include
+            ./core/deps/llvm/build/lib
+            ./core/deps/llvm/build/libexec
+            ./core/deps/llvm/build/share
+          key: ${{ matrix.llvm_cache_key }}
+
+      - name: Build wamrc
+        run: |
+          mkdir build && cd build
+          cmake -D WAMR_BUILD_SANITIZER="${{matrix.sanitizer}}" ..
+          cmake --build . --config Release --parallel 4
+        working-directory: wamr-compiler
+
       - name: Build Sample [basic]
         run: |
           cd samples/basic
@@ -432,9 +460,10 @@ jobs:
         run: |
           cd samples/multi-module
           mkdir build && cd build
-          cmake ..
+          cmake .. -DWAMR_BUILD_AOT=1
           cmake --build . --config Release --parallel 4
-          ./multi_module
+          ./multi_module mC.wasm
+          ./multi_module mC.aot
       - name: Build Sample [spawn-thread]
         run: |
           cd samples/spawn-thread

core/iwasm/aot/aot_loader.c

@@ -558,6 +558,56 @@ str2uint32(const char *buf, uint32 *p_res);
 static bool
 str2uint64(const char *buf, uint64 *p_res);
 
+#if WASM_ENABLE_MULTI_MODULE != 0
+static void *
+aot_loader_resolve_function(const char *module_name, const char *function_name,
+                            const AOTFuncType *expected_function_type,
+                            char *error_buf, uint32 error_buf_size)
+{
+    WASMModuleCommon *module_reg;
+    void *function = NULL;
+    AOTExport *export = NULL;
+    AOTModule *module = NULL;
+    AOTFuncType *target_function_type = NULL;
+
+    module_reg = wasm_runtime_find_module_registered(module_name);
+    if (!module_reg || module_reg->module_type != Wasm_Module_AoT) {
+        LOG_DEBUG("can not find a module named %s for function %s", module_name,
+                  function_name);
+        set_error_buf(error_buf, error_buf_size, "unknown import");
+        return NULL;
+    }
+
+    module = (AOTModule *)module_reg;
+    export = loader_find_export(module_reg, module_name, function_name,
+                                EXPORT_KIND_FUNC, error_buf, error_buf_size);
+    if (!export) {
+        return NULL;
+    }
+
+    /* resolve function type and function */
+    if (export->index < module->import_func_count) {
+        target_function_type = module->import_funcs[export->index].func_type;
+        function = module->import_funcs[export->index].func_ptr_linked;
+    }
+    else {
+        target_function_type =
+            module->func_types[module->func_type_indexes
+                                   [export->index - module->import_func_count]];
+        function =
+            (module->func_ptrs[export->index - module->import_func_count]);
+    }
+    /* check function type */
+    if (!wasm_type_equal(expected_function_type, target_function_type)) {
+        LOG_DEBUG("%s.%s failed the type check", module_name, function_name);
+        set_error_buf(error_buf, error_buf_size, "incompatible import type");
+        return NULL;
+    }
+    return function;
+}
+
+#endif /* end of WASM_ENABLE_MULTI_MODULE */
+
 static bool
 load_native_symbol_section(const uint8 *buf, const uint8 *buf_end,
                            AOTModule *module, bool is_load_from_file_buf,
@@ -1357,11 +1407,16 @@ load_import_funcs(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module,
                   bool is_load_from_file_buf, char *error_buf,
                   uint32 error_buf_size)
 {
-    const char *module_name, *field_name;
+    char *module_name, *field_name;
     const uint8 *buf = *p_buf;
     AOTImportFunc *import_funcs;
     uint64 size;
     uint32 i;
+#if WASM_ENABLE_MULTI_MODULE != 0
+    AOTModule *sub_module = NULL;
+    AOTFunc *linked_func = NULL;
+    WASMType *declare_func_type = NULL;
+#endif
 
     /* Allocate memory */
     size = sizeof(AOTImportFunc) * (uint64)module->import_func_count;
@@ -1377,17 +1432,46 @@ load_import_funcs(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module,
             set_error_buf(error_buf, error_buf_size, "unknown type");
             return false;
         }
+
+#if WASM_ENABLE_MULTI_MODULE != 0
+        declare_func_type = module->func_types[import_funcs[i].func_type_index];
+        read_string(buf, buf_end, module_name);
+        read_string(buf, buf_end, field_name);
+
+        import_funcs[i].module_name = module_name;
+        import_funcs[i].func_name = field_name;
+        linked_func = wasm_native_resolve_symbol(
+            module_name, field_name, declare_func_type,
+            &import_funcs[i].signature, &import_funcs[i].attachment,
+            &import_funcs[i].call_conv_raw);
+        if (!linked_func) {
+            if (!wasm_runtime_is_built_in_module(module_name)) {
+                sub_module = (AOTModule *)wasm_runtime_load_depended_module(
+                    (WASMModuleCommon *)module, module_name, error_buf,
+                    error_buf_size);
+                if (!sub_module) {
+                    return false;
+                }
+            }
+            linked_func = aot_loader_resolve_function(
+                module_name, field_name, declare_func_type, error_buf,
+                error_buf_size);
+        }
+        import_funcs[i].func_ptr_linked = linked_func;
+        import_funcs[i].func_type = declare_func_type;
+
+#else
         import_funcs[i].func_type =
             module->func_types[import_funcs[i].func_type_index];
         read_string(buf, buf_end, import_funcs[i].module_name);
         read_string(buf, buf_end, import_funcs[i].func_name);
-
         module_name = import_funcs[i].module_name;
         field_name = import_funcs[i].func_name;
         import_funcs[i].func_ptr_linked = wasm_native_resolve_symbol(
             module_name, field_name, import_funcs[i].func_type,
             &import_funcs[i].signature, &import_funcs[i].attachment,
             &import_funcs[i].call_conv_raw);
+#endif
 
 #if WASM_ENABLE_LIBC_WASI != 0
         if (!strcmp(import_funcs[i].module_name, "wasi_unstable")
@@ -2872,12 +2956,17 @@ create_module(char *error_buf, uint32 error_buf_size)
     AOTModule *module =
         loader_malloc(sizeof(AOTModule), error_buf, error_buf_size);
 
+    bh_list_status ret;
     if (!module) {
         return NULL;
     }
-
     module->module_type = Wasm_Module_AoT;
-
+#if WASM_ENABLE_MULTI_MODULE != 0
+    module->import_module_list = &module->import_module_list_head;
+    ret = bh_list_init(module->import_module_list);
+    bh_assert(ret == BH_LIST_SUCCESS);
+#endif
+    (void)ret;
     return module;
 }
 
@@ -3201,6 +3290,19 @@ aot_unload(AOTModule *module)
 
     if (module->const_str_set)
         bh_hash_map_destroy(module->const_str_set);
+#if WASM_ENABLE_MULTI_MODULE != 0
+    /* just release the sub module list */
+    if (module->import_module_list) {
+        WASMRegisteredModule *node =
+            bh_list_first_elem(module->import_module_list);
+        while (node) {
+            WASMRegisteredModule *next = bh_list_elem_next(node);
+            bh_list_remove(module->import_module_list, node);
+            wasm_runtime_free(node);
+            node = next;
+        }
+    }
+#endif
 
     if (module->code && !module->is_indirect_mode) {
         /* The layout is: literal size + literal + code (with plt table) */

core/iwasm/aot/aot_runtime.c

@@ -1091,6 +1091,9 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent,
     uint8 *p;
     uint32 i, extra_info_offset;
     const bool is_sub_inst = parent != NULL;
+#if WASM_ENABLE_MULTI_MODULE != 0
+    bool ret = false;
+#endif
 
     /* Check heap size */
     heap_size = align_uint(heap_size, 8);
@@ -1134,6 +1137,18 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent,
     module_inst->e =
         (WASMModuleInstanceExtra *)((uint8 *)module_inst + extra_info_offset);
 
+#if WASM_ENABLE_MULTI_MODULE != 0
+    ((AOTModuleInstanceExtra *)module_inst->e)->sub_module_inst_list =
+        &((AOTModuleInstanceExtra *)module_inst->e)->sub_module_inst_list_head;
+    ret = wasm_runtime_sub_module_instantiate(
+        (WASMModuleCommon *)module, (WASMModuleInstanceCommon *)module_inst,
+        stack_size, heap_size, error_buf, error_buf_size);
+    if (!ret) {
+        LOG_DEBUG("build a sub module list failed");
+        goto fail;
+    }
+#endif
+
     /* Initialize global info */
     p = (uint8 *)module_inst + module_inst_struct_size
         + module_inst_mem_inst_size;
@@ -1256,6 +1271,11 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst)
     }
 #endif
 
+#if WASM_ENABLE_MULTI_MODULE != 0
+    wasm_runtime_sub_module_deinstantiate(
+        (WASMModuleInstanceCommon *)module_inst);
+#endif
+
     if (module_inst->tables)
         wasm_runtime_free(module_inst->tables);
 
@@ -1411,10 +1431,43 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function,
                   unsigned argc, uint32 argv[])
 {
     AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst;
-    AOTFuncType *func_type = function->u.func.func_type;
+    AOTFuncType *func_type = function->is_import_func
+                                 ? function->u.func_import->func_type
+                                 : function->u.func.func_type;
     uint32 result_count = func_type->result_count;
     uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0;
     bool ret;
+    void *func_ptr = function->is_import_func
+                         ? function->u.func_import->func_ptr_linked
+                         : function->u.func.func_ptr;
+#if WASM_ENABLE_MULTI_MODULE != 0
+    bh_list *sub_module_list_node = NULL;
+    const char *sub_inst_name = NULL;
+    const char *func_name = function->u.func_import->module_name;
+    if (function->is_import_func) {
+        sub_module_list_node =
+            ((AOTModuleInstanceExtra *)module_inst->e)->sub_module_inst_list;
+        sub_module_list_node = bh_list_first_elem(sub_module_list_node);
+        while (sub_module_list_node) {
+            sub_inst_name =
+                ((AOTSubModInstNode *)sub_module_list_node)->module_name;
+            if (strcmp(sub_inst_name, func_name) == 0) {
+                exec_env = wasm_runtime_get_exec_env_singleton(
+                    (WASMModuleInstanceCommon *)((AOTSubModInstNode *)
+                                                     sub_module_list_node)
+                        ->module_inst);
+                module_inst = (AOTModuleInstance *)exec_env->module_inst;
+                break;
+            }
+            sub_module_list_node = bh_list_elem_next(sub_module_list_node);
+        }
+        if (exec_env == NULL) {
+            wasm_runtime_set_exception((WASMModuleInstanceCommon *)module_inst,
+                                       "create singleton exec_env failed");
+            return false;
+        }
+    }
+#endif
 
     if (argc < func_type->param_cell_num) {
         char buf[108];
@@ -1436,8 +1489,7 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function,
 #endif
 
     /* func pointer was looked up previously */
-    bh_assert(function->u.func.func_ptr != NULL);
+    bh_assert(func_ptr != NULL);
-
     /* set thread handle and stack boundary */
     wasm_exec_env_set_thread_info(exec_env);
 
@@ -1546,8 +1598,8 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function,
         }
 #endif
 
-        ret = invoke_native_internal(exec_env, function->u.func.func_ptr,
+        ret = invoke_native_internal(exec_env, func_ptr, func_type, NULL, NULL,
-                                     func_type, NULL, NULL, argv, argc, argv);
+                                     argv, argc, argv);
 
 #if WASM_ENABLE_DUMP_CALL_STACK != 0
         if (aot_copy_exception(module_inst, NULL)) {
@@ -1939,7 +1991,10 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc,
     void *attachment;
     char buf[96];
     bool ret = false;
-
+#if WASM_ENABLE_MULTI_MODULE != 0
+    bh_list *sub_module_list_node = NULL;
+    const char *sub_inst_name = NULL;
+#endif
     bh_assert(func_idx < aot_module->import_func_count);
 
     import_func = aot_module->import_funcs + func_idx;
@@ -1963,6 +2018,28 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc,
     }
     else if (!import_func->call_conv_raw) {
         signature = import_func->signature;
+#if WASM_ENABLE_MULTI_MODULE != 0
+        sub_module_list_node =
+            ((AOTModuleInstanceExtra *)module_inst->e)->sub_module_inst_list;
+        sub_module_list_node = bh_list_first_elem(sub_module_list_node);
+        while (sub_module_list_node) {
+            sub_inst_name =
+                ((AOTSubModInstNode *)sub_module_list_node)->module_name;
+            if (strcmp(sub_inst_name, import_func->module_name) == 0) {
+                exec_env = wasm_runtime_get_exec_env_singleton(
+                    (WASMModuleInstanceCommon *)((AOTSubModInstNode *)
+                                                     sub_module_list_node)
+                        ->module_inst);
+                break;
+            }
+            sub_module_list_node = bh_list_elem_next(sub_module_list_node);
+        }
+        if (exec_env == NULL) {
+            wasm_runtime_set_exception((WASMModuleInstanceCommon *)module_inst,
+                                       "create singleton exec_env failed");
+            goto fail;
+        }
+#endif
         ret =
             wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature,
                                        attachment, argv, argc, argv);

core/iwasm/aot/aot_runtime.h

@@ -90,6 +90,10 @@ typedef struct AOTFunctionInstance {
 typedef struct AOTModuleInstanceExtra {
     DefPointer(const uint32 *, stack_sizes);
     WASMModuleInstanceExtraCommon common;
+#if WASM_ENABLE_MULTI_MODULE != 0
+    bh_list sub_module_inst_list_head;
+    bh_list *sub_module_inst_list;
+#endif
 } AOTModuleInstanceExtra;
 
 #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
@@ -229,6 +233,12 @@ typedef struct AOTModule {
     WASIArguments wasi_args;
     bool import_wasi_api;
 #endif
+
+#if WASM_ENABLE_MULTI_MODULE != 0
+    /* TODO: add mutex for mutli-thread? */
+    bh_list import_module_list_head;
+    bh_list *import_module_list;
+#endif
 #if WASM_ENABLE_DEBUG_AOT != 0
     void *elf_hdr;
     uint32 elf_size;
@@ -247,6 +257,10 @@ typedef struct AOTModule {
 #define AOTTableInstance WASMTableInstance
 #define AOTModuleInstance WASMModuleInstance
 
+#if WASM_ENABLE_MULTI_MODULE != 0
+#define AOTSubModInstNode WASMSubModInstNode
+#endif
+
 /* Target info, read from ELF header of object file */
 typedef struct AOTTargetInfo {
     /* Binary type, elf32l/elf32b/elf64l/elf64b */

core/iwasm/common/wasm_runtime_common.c

@@ -125,6 +125,36 @@ runtime_malloc(uint64 size, WASMModuleInstanceCommon *module_inst,
     return mem;
 }
 
+#if WASM_ENABLE_MULTI_MODULE != 0
+/*
+    TODO:
+    Let loader_malloc be a general API both for AOT and WASM.
+*/
+
+#define loader_malloc(size, error_buf, error_buf_size) \
+    runtime_malloc(size, NULL, error_buf, error_buf_size)
+static void
+set_error_buf_v(const WASMModuleCommon *module, char *error_buf,
+                uint32 error_buf_size, const char *format, ...)
+{
+    va_list args;
+    char buf[128];
+    if (error_buf != NULL) {
+        va_start(args, format);
+        vsnprintf(buf, sizeof(buf), format, args);
+        va_end(args);
+        if (module->module_type == Wasm_Module_AoT) {
+            snprintf(error_buf, error_buf_size, "AOT module load failed: %s",
+                     buf);
+        }
+        else if (module->module_type == Wasm_Module_Bytecode) {
+            snprintf(error_buf, error_buf_size, "WASM module load failed: %s",
+                     buf);
+        }
+    }
+}
+#endif
+
 #if WASM_ENABLE_FAST_JIT != 0
 static JitCompOptions jit_options = { 0 };
 #endif
@@ -1198,20 +1228,20 @@ wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf,
     if (get_package_type(buf, size) == Wasm_Module_Bytecode) {
 #if WASM_ENABLE_INTERP != 0
         module_common =
-            (WASMModuleCommon *)wasm_load(buf, size, error_buf, error_buf_size);
+            (WASMModuleCommon *)wasm_load(buf, size,
-        return register_module_with_null_name(module_common, error_buf,
+#if WASM_ENABLE_MULTI_MODULE != 0
-                                              error_buf_size);
+                                          true,
+#endif
+                                          error_buf, error_buf_size);
 #endif
     }
     else if (get_package_type(buf, size) == Wasm_Module_AoT) {
 #if WASM_ENABLE_AOT != 0
         module_common = (WASMModuleCommon *)aot_load_from_aot_file(
             buf, size, error_buf, error_buf_size);
-        return register_module_with_null_name(module_common, error_buf,
-                                              error_buf_size);
 #endif
     }
-
+    else {
         if (size < 4)
             set_error_buf(error_buf, error_buf_size,
                           "WASM module load failed: unexpected end");
@@ -1219,6 +1249,13 @@ wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf,
             set_error_buf(error_buf, error_buf_size,
                           "WASM module load failed: magic header not detected");
         return NULL;
+    }
+    if (!module_common) {
+        LOG_DEBUG("WASM module load failed");
+        return NULL;
+    }
+    return register_module_with_null_name(module_common, error_buf,
+                                          error_buf_size);
 }
 
 WASMModuleCommon *
@@ -1231,6 +1268,10 @@ wasm_runtime_load_from_sections(WASMSection *section_list, bool is_aot,
 #if WASM_ENABLE_INTERP != 0
         module_common = (WASMModuleCommon *)wasm_load_from_sections(
             section_list, error_buf, error_buf_size);
+        if (!module_common) {
+            LOG_DEBUG("WASM module load failed from sections");
+            return NULL;
+        }
         return register_module_with_null_name(module_common, error_buf,
                                               error_buf_size);
 #endif
@@ -1239,6 +1280,10 @@ wasm_runtime_load_from_sections(WASMSection *section_list, bool is_aot,
 #if WASM_ENABLE_AOT != 0
         module_common = (WASMModuleCommon *)aot_load_from_sections(
             section_list, error_buf, error_buf_size);
+        if (!module_common) {
+            LOG_DEBUG("WASM module load failed from sections");
+            return NULL;
+        }
         return register_module_with_null_name(module_common, error_buf,
                                               error_buf_size);
 #endif
@@ -5663,6 +5708,314 @@ wasm_runtime_is_import_global_linked(const char *module_name,
 #endif
 }
 
+#if WASM_ENABLE_LIBC_WASI != 0 || WASM_ENABLE_MULTI_MODULE != 0
+WASMExport *
+loader_find_export(const WASMModuleCommon *module, const char *module_name,
+                   const char *field_name, uint8 export_kind, char *error_buf,
+                   uint32 error_buf_size)
+{
+    WASMExport *exports = NULL, *result = NULL, *export;
+    uint32 export_count = 0, i;
+#if WASM_ENABLE_AOT != 0
+    if (module->module_type == Wasm_Module_AoT) {
+        AOTModule *aot_module = (AOTModule *)module;
+        exports = (WASMExport *)aot_module->exports;
+        export_count = aot_module->export_count;
+    }
+#endif
+#if WASM_ENABLE_INTERP != 0
+    if (module->module_type == Wasm_Module_Bytecode) {
+        WASMModule *wasm_module = (WASMModule *)module;
+        exports = wasm_module->exports;
+        export_count = wasm_module->export_count;
+    }
+#endif
+    for (i = 0, export = exports; i < export_count; ++i, ++export) {
+        if (export->kind == export_kind && !strcmp(field_name, export->name)) {
+            result = export;
+            goto exit;
+        }
+    }
+    if (i == export_count) {
+        LOG_DEBUG("can not find an export %d named %s in the module %s",
+                  export_kind, field_name, module_name);
+        set_error_buf(error_buf, error_buf_size,
+                      "unknown import or incompatible import type");
+    }
+exit:
+    return result;
+}
+#endif
+
+#if WASM_ENABLE_MULTI_MODULE != 0
+WASMModuleCommon *
+wasm_runtime_search_sub_module(const WASMModuleCommon *parent_module,
+                               const char *sub_module_name)
+{
+    WASMRegisteredModule *node = NULL;
+#if WASM_ENABLE_AOT != 0
+    if (parent_module->module_type == Wasm_Module_AoT) {
+        node = bh_list_first_elem(
+            ((AOTModule *)parent_module)->import_module_list);
+    }
+#endif
+#if WASM_ENABLE_INTERP != 0
+    if (parent_module->module_type == Wasm_Module_Bytecode) {
+        node = bh_list_first_elem(
+            ((WASMModule *)parent_module)->import_module_list);
+    }
+#endif
+    while (node && strcmp(sub_module_name, node->module_name)) {
+        node = bh_list_elem_next(node);
+    }
+    return node ? node->module : NULL;
+}
+
+bool
+wasm_runtime_register_sub_module(const WASMModuleCommon *parent_module,
+                                 const char *sub_module_name,
+                                 WASMModuleCommon *sub_module)
+{
+    /* register sub_module into its parent sub module list */
+    WASMRegisteredModule *node = NULL;
+    bh_list_status ret;
+
+    if (wasm_runtime_search_sub_module(parent_module, sub_module_name)) {
+        LOG_DEBUG("%s has been registered in its parent", sub_module_name);
+        return true;
+    }
+
+    node = loader_malloc(sizeof(WASMRegisteredModule), NULL, 0);
+    if (!node) {
+        return false;
+    }
+
+    node->module_name = sub_module_name;
+    node->module = sub_module;
+#if WASM_ENABLE_AOT != 0
+    if (parent_module->module_type == Wasm_Module_AoT) {
+        ret = bh_list_insert(((AOTModule *)parent_module)->import_module_list,
+                             node);
+    }
+#endif
+#if WASM_ENABLE_INTERP != 0
+    if (parent_module->module_type == Wasm_Module_Bytecode) {
+        ret = bh_list_insert(((WASMModule *)parent_module)->import_module_list,
+                             node);
+    }
+#endif
+    bh_assert(BH_LIST_SUCCESS == ret);
+    (void)ret;
+    return true;
+}
+
+WASMModuleCommon *
+wasm_runtime_load_depended_module(const WASMModuleCommon *parent_module,
+                                  const char *sub_module_name, char *error_buf,
+                                  uint32 error_buf_size)
+{
+    WASMModuleCommon *sub_module = NULL;
+    bool ret = false;
+    uint8 *buffer = NULL;
+    uint32 buffer_size = 0;
+
+    /* check the registered module list of the parent */
+    sub_module = wasm_runtime_search_sub_module(parent_module, sub_module_name);
+    if (sub_module) {
+        LOG_DEBUG("%s has been loaded before", sub_module_name);
+        return sub_module;
+    }
+
+    /* check the global registered module list */
+    sub_module = wasm_runtime_find_module_registered(sub_module_name);
+    if (sub_module) {
+        LOG_DEBUG("%s has been loaded", sub_module_name);
+        goto wasm_runtime_register_sub_module;
+    }
+    LOG_VERBOSE("loading %s", sub_module_name);
+    if (!reader) {
+        set_error_buf_v(parent_module, error_buf, error_buf_size,
+                        "no sub module reader to load %s", sub_module_name);
+        return NULL;
+    }
+    /* start to maintain a loading module list */
+    ret = wasm_runtime_is_loading_module(sub_module_name);
+    if (ret) {
+        set_error_buf_v(parent_module, error_buf, error_buf_size,
+                        "found circular dependency on %s", sub_module_name);
+        return NULL;
+    }
+    ret = wasm_runtime_add_loading_module(sub_module_name, error_buf,
+                                          error_buf_size);
+    if (!ret) {
+        LOG_DEBUG("can not add %s into loading module list\n", sub_module_name);
+        return NULL;
+    }
+
+    ret = reader(parent_module->module_type, sub_module_name, &buffer,
+                 &buffer_size);
+    if (!ret) {
+        LOG_DEBUG("read the file of %s failed", sub_module_name);
+        set_error_buf_v(parent_module, error_buf, error_buf_size,
+                        "unknown import", sub_module_name);
+        goto delete_loading_module;
+    }
+    if (get_package_type(buffer, buffer_size) != parent_module->module_type) {
+        LOG_DEBUG("moudle %s type error", sub_module_name);
+        goto delete_loading_module;
+    }
+    if (get_package_type(buffer, buffer_size) == Wasm_Module_Bytecode) {
+#if WASM_ENABLE_INTERP != 0
+        sub_module = (WASMModuleCommon *)wasm_load(buffer, buffer_size, false,
+                                                   error_buf, error_buf_size);
+#endif
+    }
+    else if (get_package_type(buffer, buffer_size) == Wasm_Module_AoT) {
+#if WASM_ENABLE_AOT != 0
+        sub_module = (WASMModuleCommon *)aot_load_from_aot_file(
+            buffer, buffer_size, error_buf, error_buf_size);
+#endif
+    }
+    if (!sub_module) {
+        LOG_DEBUG("error: can not load the sub_module %s", sub_module_name);
+        /* others will be destroyed in runtime_destroy() */
+        goto destroy_file_buffer;
+    }
+    wasm_runtime_delete_loading_module(sub_module_name);
+    /* register on a global list */
+    ret = wasm_runtime_register_module_internal(
+        sub_module_name, (WASMModuleCommon *)sub_module, buffer, buffer_size,
+        error_buf, error_buf_size);
+    if (!ret) {
+        LOG_DEBUG("error: can not register module %s globally\n",
+                  sub_module_name);
+        /* others will be unloaded in runtime_destroy() */
+        goto unload_module;
+    }
+
+    /* register into its parent list */
+wasm_runtime_register_sub_module:
+    ret = wasm_runtime_register_sub_module(parent_module, sub_module_name,
+                                           sub_module);
+    if (!ret) {
+        set_error_buf_v(parent_module, error_buf, error_buf_size,
+                        "failed to register sub module %s", sub_module_name);
+        /* since it is in the global module list, no need to
+         * unload the module. the runtime_destroy() will do it
+         */
+        return NULL;
+    }
+
+    return sub_module;
+
+unload_module:
+    wasm_runtime_unload(sub_module);
+
+destroy_file_buffer:
+    if (destroyer) {
+        destroyer(buffer, buffer_size);
+    }
+    else {
+        LOG_WARNING("need to release the reading buffer of %s manually",
+                    sub_module_name);
+    }
+
+delete_loading_module:
+    wasm_runtime_delete_loading_module(sub_module_name);
+    return NULL;
+}
+
+bool
+wasm_runtime_sub_module_instantiate(WASMModuleCommon *module,
+                                    WASMModuleInstanceCommon *module_inst,
+                                    uint32 stack_size, uint32 heap_size,
+                                    char *error_buf, uint32 error_buf_size)
+{
+    bh_list *sub_module_inst_list = NULL;
+    WASMRegisteredModule *sub_module_list_node = NULL;
+
+#if WASM_ENABLE_AOT != 0
+    if (module->module_type == Wasm_Module_AoT) {
+        sub_module_inst_list =
+            ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e)
+                ->sub_module_inst_list;
+        sub_module_list_node =
+            bh_list_first_elem(((AOTModule *)module)->import_module_list);
+    }
+#endif
+#if WASM_ENABLE_INTERP != 0
+    if (module->module_type == Wasm_Module_Bytecode) {
+        sub_module_inst_list =
+            ((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e)
+                ->sub_module_inst_list;
+        sub_module_list_node =
+            bh_list_first_elem(((WASMModule *)module)->import_module_list);
+    }
+#endif
+    while (sub_module_list_node) {
+        WASMSubModInstNode *sub_module_inst_list_node = NULL;
+        WASMModuleCommon *sub_module = sub_module_list_node->module;
+        WASMModuleInstanceCommon *sub_module_inst = NULL;
+        sub_module_inst = wasm_runtime_instantiate_internal(
+            sub_module, NULL, NULL, stack_size, heap_size, error_buf,
+            error_buf_size);
+        if (!sub_module_inst) {
+            LOG_DEBUG("instantiate %s failed",
+                      sub_module_list_node->module_name);
+            return false;
+        }
+        sub_module_inst_list_node = loader_malloc(sizeof(WASMSubModInstNode),
+                                                  error_buf, error_buf_size);
+        if (!sub_module_inst_list_node) {
+            LOG_DEBUG("Malloc WASMSubModInstNode failed, SZ:%d",
+                      sizeof(WASMSubModInstNode));
+            if (sub_module_inst)
+                wasm_runtime_deinstantiate_internal(sub_module_inst, false);
+            return false;
+        }
+        sub_module_inst_list_node->module_inst =
+            (WASMModuleInstance *)sub_module_inst;
+        sub_module_inst_list_node->module_name =
+            sub_module_list_node->module_name;
+        bh_list_status ret =
+            bh_list_insert(sub_module_inst_list, sub_module_inst_list_node);
+        bh_assert(BH_LIST_SUCCESS == ret);
+        (void)ret;
+        sub_module_list_node = bh_list_elem_next(sub_module_list_node);
+    }
+
+    return true;
+}
+
+void
+wasm_runtime_sub_module_deinstantiate(WASMModuleInstanceCommon *module_inst)
+{
+    bh_list *list = NULL;
+#if WASM_ENABLE_AOT != 0
+    if (module_inst->module_type == Wasm_Module_AoT) {
+        list = ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e)
+                   ->sub_module_inst_list;
+    }
+#endif
+#if WASM_ENABLE_INTERP != 0
+    if (module_inst->module_type == Wasm_Module_Bytecode) {
+        list =
+            ((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e)
+                ->sub_module_inst_list;
+    }
+#endif
+
+    WASMSubModInstNode *node = bh_list_first_elem(list);
+    while (node) {
+        WASMSubModInstNode *next_node = bh_list_elem_next(node);
+        bh_list_remove(list, node);
+        wasm_runtime_deinstantiate_internal(
+            (WASMModuleInstanceCommon *)node->module_inst, false);
+        wasm_runtime_free(node);
+        node = next_node;
+    }
+}
+#endif /* end of WASM_ENABLE_MULTI_MODULE */
 #if WASM_ENABLE_MODULE_INST_CONTEXT != 0
 void *
 wasm_runtime_create_context_key(void (*dtor)(WASMModuleInstanceCommon *inst,

core/iwasm/common/wasm_runtime_common.h

@@ -788,6 +788,9 @@ wasm_runtime_register_module_internal(const char *module_name,
 void
 wasm_runtime_unregister_module(const WASMModuleCommon *module);
 
+WASMModuleCommon *
+wasm_runtime_find_module_registered(const char *module_name);
+
 bool
 wasm_runtime_add_loading_module(const char *module_name, char *error_buf,
                                 uint32 error_buf_size);
@@ -800,6 +803,35 @@ wasm_runtime_is_loading_module(const char *module_name);
 
 void
 wasm_runtime_destroy_loading_module_list();
+
+WASMModuleCommon *
+wasm_runtime_search_sub_module(const WASMModuleCommon *parent_module,
+                               const char *sub_module_name);
+
+bool
+wasm_runtime_register_sub_module(const WASMModuleCommon *parent_module,
+                                 const char *sub_module_name,
+                                 WASMModuleCommon *sub_module);
+
+WASMModuleCommon *
+wasm_runtime_load_depended_module(const WASMModuleCommon *parent_module,
+                                  const char *sub_module_name, char *error_buf,
+                                  uint32 error_buf_size);
+
+bool
+wasm_runtime_sub_module_instantiate(WASMModuleCommon *module,
+                                    WASMModuleInstanceCommon *module_inst,
+                                    uint32 stack_size, uint32 heap_size,
+                                    char *error_buf, uint32 error_buf_size);
+void
+wasm_runtime_sub_module_deinstantiate(WASMModuleInstanceCommon *module_inst);
+#endif
+
+#if WASM_ENABLE_LIBC_WASI != 0 || WASM_ENABLE_MULTI_MODULE != 0
+WASMExport *
+loader_find_export(const WASMModuleCommon *module, const char *module_name,
+                   const char *field_name, uint8 export_kind, char *error_buf,
+                   uint32 error_buf_size);
 #endif /* WASM_ENALBE_MULTI_MODULE */
 
 bool

core/iwasm/include/wasm_export.h

@@ -311,7 +311,7 @@ wasm_runtime_is_xip_file(const uint8_t *buf, uint32_t size);
 /**
  * Callback to load a module file into a buffer in multi-module feature
  */
-typedef bool (*module_reader)(const char *module_name,
+typedef bool (*module_reader)(package_type_t module_type,const char *module_name,
                               uint8_t **p_buffer, uint32_t *p_size);
 
 /**

core/iwasm/interpreter/wasm_loader.c

@@ -694,34 +694,10 @@ wasm_loader_find_export(const WASMModule *module, const char *module_name,
                         const char *field_name, uint8 export_kind,
                         char *error_buf, uint32 error_buf_size)
 {
-    WASMExport *export;
+    WASMExport *export =
-    uint32 i;
+        loader_find_export((WASMModuleCommon *)module, module_name, field_name,
-
+                           export_kind, error_buf, error_buf_size);
-    for (i = 0, export = module->exports; i < module->export_count;
+    ;
-         ++i, ++export) {
-        /**
-         * need to consider a scenario that different kinds of exports
-         * may have the same name, like
-         * (table (export "m1" "exported") 10 funcref)
-         * (memory (export "m1" "exported") 10)
-         **/
-        if (export->kind == export_kind && !strcmp(field_name, export->name)) {
-            break;
-        }
-    }
-
-    if (i == module->export_count) {
-        LOG_DEBUG("can not find an export %d named %s in the module %s",
-                  export_kind, field_name, module_name);
-        set_error_buf(error_buf, error_buf_size,
-                      "unknown import or incompatible import type");
-        return NULL;
-    }
-
-    (void)module_name;
-
-    /* since there is a validation in load_export_section(), it is for sure
-     * export->index is valid*/
     return export;
 }
 #endif
@@ -912,152 +888,6 @@ wasm_loader_resolve_global(const char *module_name, const char *global_name,
     return global;
 }
 
-static WASMModule *
-search_sub_module(const WASMModule *parent_module, const char *sub_module_name)
-{
-    WASMRegisteredModule *node =
-        bh_list_first_elem(parent_module->import_module_list);
-    while (node && strcmp(sub_module_name, node->module_name)) {
-        node = bh_list_elem_next(node);
-    }
-    return node ? (WASMModule *)node->module : NULL;
-}
-
-static bool
-register_sub_module(const WASMModule *parent_module,
-                    const char *sub_module_name, WASMModule *sub_module)
-{
-    /* register sub_module into its parent sub module list */
-    WASMRegisteredModule *node = NULL;
-    bh_list_status ret;
-
-    if (search_sub_module(parent_module, sub_module_name)) {
-        LOG_DEBUG("%s has been registered in its parent", sub_module_name);
-        return true;
-    }
-
-    node = loader_malloc(sizeof(WASMRegisteredModule), NULL, 0);
-    if (!node) {
-        return false;
-    }
-
-    node->module_name = sub_module_name;
-    node->module = (WASMModuleCommon *)sub_module;
-    ret = bh_list_insert(parent_module->import_module_list, node);
-    bh_assert(BH_LIST_SUCCESS == ret);
-    (void)ret;
-    return true;
-}
-
-static WASMModule *
-load_depended_module(const WASMModule *parent_module,
-                     const char *sub_module_name, char *error_buf,
-                     uint32 error_buf_size)
-{
-    WASMModule *sub_module = NULL;
-    bool ret = false;
-    uint8 *buffer = NULL;
-    uint32 buffer_size = 0;
-    const module_reader reader = wasm_runtime_get_module_reader();
-    const module_destroyer destroyer = wasm_runtime_get_module_destroyer();
-
-    /* check the registered module list of the parent */
-    sub_module = search_sub_module(parent_module, sub_module_name);
-    if (sub_module) {
-        LOG_DEBUG("%s has been loaded before", sub_module_name);
-        return sub_module;
-    }
-
-    /* check the global registered module list */
-    sub_module =
-        (WASMModule *)wasm_runtime_find_module_registered(sub_module_name);
-    if (sub_module) {
-        LOG_DEBUG("%s has been loaded", sub_module_name);
-        goto register_sub_module;
-    }
-
-    LOG_VERBOSE("loading %s", sub_module_name);
-
-    if (!reader) {
-        set_error_buf_v(error_buf, error_buf_size,
-                        "no sub module reader to load %s", sub_module_name);
-        return NULL;
-    }
-
-    /* start to maintain a loading module list */
-    ret = wasm_runtime_is_loading_module(sub_module_name);
-    if (ret) {
-        set_error_buf_v(error_buf, error_buf_size,
-                        "found circular dependency on %s", sub_module_name);
-        return NULL;
-    }
-
-    ret = wasm_runtime_add_loading_module(sub_module_name, error_buf,
-                                          error_buf_size);
-    if (!ret) {
-        LOG_DEBUG("can not add %s into loading module list\n", sub_module_name);
-        return NULL;
-    }
-
-    ret = reader(sub_module_name, &buffer, &buffer_size);
-    if (!ret) {
-        LOG_DEBUG("read the file of %s failed", sub_module_name);
-        set_error_buf_v(error_buf, error_buf_size, "unknown import",
-                        sub_module_name);
-        goto delete_loading_module;
-    }
-
-    sub_module =
-        wasm_loader_load(buffer, buffer_size, false, error_buf, error_buf_size);
-    if (!sub_module) {
-        LOG_DEBUG("error: can not load the sub_module %s", sub_module_name);
-        /* others will be destroyed in runtime_destroy() */
-        goto destroy_file_buffer;
-    }
-
-    wasm_runtime_delete_loading_module(sub_module_name);
-
-    /* register on a global list */
-    ret = wasm_runtime_register_module_internal(
-        sub_module_name, (WASMModuleCommon *)sub_module, buffer, buffer_size,
-        error_buf, error_buf_size);
-    if (!ret) {
-        LOG_DEBUG("error: can not register module %s globally\n",
-                  sub_module_name);
-        /* others will be unloaded in runtime_destroy() */
-        goto unload_module;
-    }
-
-    /* register into its parent list */
-register_sub_module:
-    ret = register_sub_module(parent_module, sub_module_name, sub_module);
-    if (!ret) {
-        set_error_buf_v(error_buf, error_buf_size,
-                        "failed to register sub module %s", sub_module_name);
-        /* since it is in the global module list, no need to
-         * unload the module. the runtime_destroy() will do it
-         */
-        return NULL;
-    }
-
-    return sub_module;
-
-unload_module:
-    wasm_loader_unload(sub_module);
-
-destroy_file_buffer:
-    if (destroyer) {
-        destroyer(buffer, buffer_size);
-    }
-    else {
-        LOG_WARNING("need to release the reading buffer of %s manually",
-                    sub_module_name);
-    }
-
-delete_loading_module:
-    wasm_runtime_delete_loading_module(sub_module_name);
-    return NULL;
-}
 #endif /* end of WASM_ENABLE_MULTI_MODULE */
 
 static bool
@@ -1104,8 +934,9 @@ load_function_import(const uint8 **p_buf, const uint8 *buf_end,
 #if WASM_ENABLE_MULTI_MODULE != 0
     else {
         if (!wasm_runtime_is_built_in_module(sub_module_name)) {
-            sub_module = load_depended_module(parent_module, sub_module_name,
+            sub_module = (WASMModule *)wasm_runtime_load_depended_module(
-                                              error_buf, error_buf_size);
+                (WASMModuleCommon *)parent_module, sub_module_name, error_buf,
+                error_buf_size);
             if (!sub_module) {
                 return false;
             }
@@ -1193,8 +1024,9 @@ load_table_import(const uint8 **p_buf, const uint8 *buf_end,
 
 #if WASM_ENABLE_MULTI_MODULE != 0
     if (!wasm_runtime_is_built_in_module(sub_module_name)) {
-        sub_module = load_depended_module(parent_module, sub_module_name,
+        sub_module = (WASMModule *)wasm_runtime_load_depended_module(
-                                          error_buf, error_buf_size);
+            (WASMModuleCommon *)parent_module, sub_module_name, error_buf,
+            error_buf_size);
         if (!sub_module) {
             return false;
         }
@@ -1327,8 +1159,9 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end,
 
 #if WASM_ENABLE_MULTI_MODULE != 0
     if (!wasm_runtime_is_built_in_module(sub_module_name)) {
-        sub_module = load_depended_module(parent_module, sub_module_name,
+        sub_module = (WASMModule *)wasm_runtime_load_depended_module(
-                                          error_buf, error_buf_size);
+            (WASMModuleCommon *)parent_module, sub_module_name, error_buf,
+            error_buf_size);
         if (!sub_module) {
             return false;
         }
@@ -1427,8 +1260,9 @@ load_global_import(const uint8 **p_buf, const uint8 *buf_end,
 #if WASM_ENABLE_MULTI_MODULE != 0
     if (!global->is_linked
         && !wasm_runtime_is_built_in_module(sub_module_name)) {
-        sub_module = load_depended_module(parent_module, sub_module_name,
+        sub_module = (WASMModule *)wasm_runtime_load_depended_module(
-                                          error_buf, error_buf_size);
+            (WASMModuleCommon *)parent_module, sub_module_name, error_buf,
+            error_buf_size);
         if (!sub_module) {
             return false;
         }

core/iwasm/interpreter/wasm_runtime.c

@@ -51,11 +51,15 @@ set_error_buf_v(char *error_buf, uint32 error_buf_size, const char *format, ...)
 }
 
 WASMModule *
-wasm_load(uint8 *buf, uint32 size, char *error_buf, uint32 error_buf_size)
+wasm_load(uint8 *buf, uint32 size,
+#if WASM_ENABLE_MULTI_MODULE != 0
+          bool main_module,
+#endif
+          char *error_buf, uint32 error_buf_size)
 {
     return wasm_loader_load(buf, size,
 #if WASM_ENABLE_MULTI_MODULE != 0
-                            true,
+                            main_module,
 #endif
                             error_buf, error_buf_size);
 }
@@ -1265,78 +1269,6 @@ execute_free_function(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
     return ret;
 }
 
-#if WASM_ENABLE_MULTI_MODULE != 0
-static bool
-sub_module_instantiate(WASMModule *module, WASMModuleInstance *module_inst,
-                       uint32 stack_size, uint32 heap_size, char *error_buf,
-                       uint32 error_buf_size)
-{
-    bh_list *sub_module_inst_list = module_inst->e->sub_module_inst_list;
-    WASMRegisteredModule *sub_module_list_node =
-        bh_list_first_elem(module->import_module_list);
-
-    while (sub_module_list_node) {
-        WASMSubModInstNode *sub_module_inst_list_node = NULL;
-        WASMModule *sub_module = (WASMModule *)sub_module_list_node->module;
-        WASMModuleInstance *sub_module_inst = NULL;
-
-        sub_module_inst =
-            wasm_instantiate(sub_module, NULL, NULL, stack_size, heap_size,
-                             error_buf, error_buf_size);
-        if (!sub_module_inst) {
-            LOG_DEBUG("instantiate %s failed",
-                      sub_module_list_node->module_name);
-            goto failed;
-        }
-
-        sub_module_inst_list_node = runtime_malloc(sizeof(WASMSubModInstNode),
-                                                   error_buf, error_buf_size);
-        if (!sub_module_inst_list_node) {
-            LOG_DEBUG("Malloc WASMSubModInstNode failed, SZ:%d",
-                      sizeof(WASMSubModInstNode));
-            goto failed;
-        }
-
-        sub_module_inst_list_node->module_inst = sub_module_inst;
-        sub_module_inst_list_node->module_name =
-            sub_module_list_node->module_name;
-        bh_list_status ret =
-            bh_list_insert(sub_module_inst_list, sub_module_inst_list_node);
-        bh_assert(BH_LIST_SUCCESS == ret);
-        (void)ret;
-
-        sub_module_list_node = bh_list_elem_next(sub_module_list_node);
-
-        continue;
-    failed:
-        if (sub_module_inst_list_node) {
-            bh_list_remove(sub_module_inst_list, sub_module_inst_list_node);
-            wasm_runtime_free(sub_module_inst_list_node);
-        }
-
-        if (sub_module_inst)
-            wasm_deinstantiate(sub_module_inst, false);
-        return false;
-    }
-
-    return true;
-}
-
-static void
-sub_module_deinstantiate(WASMModuleInstance *module_inst)
-{
-    bh_list *list = module_inst->e->sub_module_inst_list;
-    WASMSubModInstNode *node = bh_list_first_elem(list);
-    while (node) {
-        WASMSubModInstNode *next_node = bh_list_elem_next(node);
-        bh_list_remove(list, node);
-        wasm_deinstantiate(node->module_inst, false);
-        wasm_runtime_free(node);
-        node = next_node;
-    }
-}
-#endif
-
 static bool
 check_linked_symbol(WASMModuleInstance *module_inst, char *error_buf,
                     uint32 error_buf_size)
@@ -1713,8 +1645,9 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent,
 #if WASM_ENABLE_MULTI_MODULE != 0
     module_inst->e->sub_module_inst_list =
         &module_inst->e->sub_module_inst_list_head;
-    ret = sub_module_instantiate(module, module_inst, stack_size, heap_size,
+    ret = wasm_runtime_sub_module_instantiate(
-                                 error_buf, error_buf_size);
+        (WASMModuleCommon *)module, (WASMModuleInstanceCommon *)module_inst,
+        stack_size, heap_size, error_buf, error_buf_size);
     if (!ret) {
         LOG_DEBUG("build a sub module list failed");
         goto fail;
@@ -2197,7 +2130,8 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst)
 #endif
 
 #if WASM_ENABLE_MULTI_MODULE != 0
-    sub_module_deinstantiate(module_inst);
+    wasm_runtime_sub_module_deinstantiate(
+        (WASMModuleInstanceCommon *)module_inst);
 #endif
 
     if (module_inst->memory_count > 0)

core/iwasm/interpreter/wasm_runtime.h

@@ -396,7 +396,11 @@ wasm_get_func_code_end(WASMFunctionInstance *func)
 }
 
 WASMModule *
-wasm_load(uint8 *buf, uint32 size, char *error_buf, uint32 error_buf_size);
+wasm_load(uint8 *buf, uint32 size,
+#if WASM_ENABLE_MULTI_MODULE != 0
+          bool main_module,
+#endif
+          char *error_buf, uint32 error_buf_size);
 
 WASMModule *
 wasm_load_from_sections(WASMSection *section_list, char *error_buf,

product-mini/platforms/posix/main.c

@@ -420,19 +420,28 @@ handle_module_path(const char *module_path)
 static char *module_search_path = ".";
 
 static bool
-module_reader_callback(const char *module_name, uint8 **p_buffer,
+module_reader_callback(package_type_t module_type, const char *module_name,
-                       uint32 *p_size)
+                       uint8 **p_buffer, uint32 *p_size)
 {
-    const char *format = "%s/%s.wasm";
+    char *file_format;
+#if WASM_ENABLE_INTERP != 0
+    if (module_type == Wasm_Module_Bytecode)
+        file_format = ".wasm";
+#endif
+#if WASM_ENABLE_AOT != 0
+    if (module_type == Wasm_Module_AoT)
+        file_format = ".aot";
+
+#endif
+    const char *format = "%s/%s%s";
     int sz = strlen(module_search_path) + strlen("/") + strlen(module_name)
-             + strlen(".wasm") + 1;
+             + strlen(file_format) + 1;
     char *wasm_file_name = BH_MALLOC(sz);
     if (!wasm_file_name) {
         return false;
     }
-
+    snprintf(wasm_file_name, sz, format, module_search_path, module_name,
-    snprintf(wasm_file_name, sz, format, module_search_path, module_name);
+             file_format);
-
     *p_buffer = (uint8_t *)bh_read_file_to_buffer(wasm_file_name, p_size);
 
     wasm_runtime_free(wasm_file_name);

product-mini/platforms/windows/main.c

@@ -204,20 +204,29 @@ handle_module_path(const char *module_path)
 
 static char *module_search_path = ".";
 static bool
-module_reader_callback(const char *module_name, uint8 **p_buffer,
+module_reader_callback(package_type_t module_type, const char *module_name,
-                       uint32 *p_size)
+                       uint8 **p_buffer, uint32 *p_size)
 {
-    const char *format = "%s/%s.wasm";
+    char *file_format;
-    uint32 sz = (uint32)(strlen(module_search_path) + strlen("/")
+#if WASM_ENABLE_INTERP != 0
-                         + strlen(module_name) + strlen(".wasm") + 1);
+    if (module_type == Wasm_Module_Bytecode)
+        file_format = ".wasm";
+#endif
+#if WASM_ENABLE_AOT != 0
+    if (module_type == Wasm_Module_AoT)
+        file_format = ".aot";
+
+#endif
+    const char *format = "%s/%s%s";
+    int sz = strlen(module_search_path) + strlen("/") + strlen(module_name)
+             + strlen(file_format) + 1;
     char *wasm_file_name = BH_MALLOC(sz);
     if (!wasm_file_name) {
         return false;
     }
-
+    snprintf(wasm_file_name, sz, format, module_search_path, module_name,
-    snprintf(wasm_file_name, sz, format, module_search_path, module_name);
+             file_format);
-
+    *p_buffer = (uint8_t *)bh_read_file_to_buffer(wasm_file_name, p_size);
-    *p_buffer = (uint8 *)bh_read_file_to_buffer(wasm_file_name, p_size);
 
     wasm_runtime_free(wasm_file_name);
     return *p_buffer != NULL;

samples/multi-module/CMakeLists.txt

@@ -43,8 +43,12 @@ if (NOT CMAKE_BUILD_TYPE)
 endif ()
 
 set(WAMR_BUILD_INTERP 1)
-set(WAMR_BUILD_AOT 0)
+if (NOT DEFINED WAMR_BUILD_AOT)
-set(WAMR_BUILD_JIT 0)
+  set(WAMR_BUILD_AOT 0)
+endif ()
+if (NOT DEFINED WAMR_BUILD_JIT)
+  set(WAMR_BUILD_JIT 0)
+endif ()
 set(WAMR_BUILD_LIBC_BUILTIN 1)
 set(WAMR_BUILD_LIBC_WASI 1)
 set(WAMR_BUILD_MULTI_MODULE 1)
@@ -144,6 +148,43 @@ ExternalProject_Add(WASM_MODULE
                        ./mE.wasm ${CMAKE_BINARY_DIR}
 )
 
+################ WASM MODULES TO AOT
+if (WAMR_BUILD_AOT EQUAL 1)
+  set(WAMR_COMPILER_DIR ${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build)
+  message(CHECK_START "Detecting WAMR_COMPILER at ${WAMR_COMPILER_DIR}")
+  find_file(WAMR_COMPILER
+    wamrc
+    PATHS "${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build"
+    NO_DEFAULT_PATH
+    NO_CMAKE_FIND_ROOT_PATH
+  )
+  if(WAMR_COMPILER)
+    message(CHECK_PASS "found")
+  else()
+    message(CHECK_FAIL "not found")
+  endif()
+  if((NOT EXISTS ${WAMR_COMPILER}) )
+    message(FATAL_ERROR "Please build wamrc under the path=${WAMR_ROOT_DIR}/wamr-compiler/ ")
+  else()
+    message(STATUS "WAMR_COMPILER is ${WAMR_COMPILER}")
+  endif()
+
+  add_custom_target(
+    wasm_to_aot
+    ALL
+    DEPENDS
+    WASM_MODULE ${WAMR_COMPILER}
+    COMMAND
+    ${WAMR_COMPILER} -o mA.aot ./mA.wasm
+    COMMAND
+    ${WAMR_COMPILER} -o mB.aot ./mB.wasm
+    COMMAND
+    ${WAMR_COMPILER} -o mC.aot ./mC.wasm
+    WORKING_DIRECTORY
+    ${CMAKE_BINARY_DIR}
+  )
+endif()
+
 ################ NATIVE
 include_directories(${CMAKE_CURRENT_LIST_DIR}/src)
 include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake)

samples/multi-module/README.md

@@ -0,0 +1,20 @@
+# WAMR MULTI-MODUEL SAMPLE
+**WAMR supports *multi-module* in both *interpreter* mode and *aot* mode.**
+
+Multi-modules will determine the running mode based on the type of the main module.
+
+
+``` shell
+$ mkdir build
+$ cd build
+$ cmake ..
+$ make
+$ # It will build multi-module runtime and 
+$ # wasm file under the ./build .
+$ # If you have built wamrc,
+$ # aot file will also genrate.
+$ ./multi-module mC.wasm
+$ ...
+$ ./multi-module mC.aot
+$ ...
+

samples/multi-module/src/main.c

@@ -1,59 +1,63 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-
 #include "bh_read_file.h"
 #include "platform_common.h"
 #include "wasm_export.h"
 
-static char *
+#if WASM_ENABLE_MULTI_MODULE != 0
-build_module_path(const char *module_name)
+static char *module_search_path = ".";
+static bool
+module_reader_callback(package_type_t module_type, const char *module_name,
+                       uint8 **p_buffer, uint32 *p_size)
 {
-    const char *module_search_path = ".";
+    char *file_format;
-    const char *format = "%s/%s.wasm";
+#if WASM_ENABLE_INTERP != 0
+    if (module_type == Wasm_Module_Bytecode)
+        file_format = ".wasm";
+#endif
+#if WASM_ENABLE_AOT != 0
+    if (module_type == Wasm_Module_AoT)
+        file_format = ".aot";
+
+#endif
+    const char *format = "%s/%s%s";
     int sz = strlen(module_search_path) + strlen("/") + strlen(module_name)
-             + strlen(".wasm") + 1;
+             + strlen(file_format) + 1;
     char *wasm_file_name = BH_MALLOC(sz);
     if (!wasm_file_name) {
-        return NULL;
-    }
-
-    snprintf(wasm_file_name, sz, format, module_search_path, module_name);
-    return wasm_file_name;
-}
-
-static bool
-module_reader_cb(const char *module_name, uint8 **p_buffer, uint32 *p_size)
-{
-    char *wasm_file_path = build_module_path(module_name);
-    if (!wasm_file_path) {
         return false;
     }
+    snprintf(wasm_file_name, sz, format, module_search_path, module_name,
+             file_format);
+    *p_buffer = (uint8_t *)bh_read_file_to_buffer(wasm_file_name, p_size);
 
-    printf("- bh_read_file_to_buffer %s\n", wasm_file_path);
+    wasm_runtime_free(wasm_file_name);
-    *p_buffer = (uint8_t *)bh_read_file_to_buffer(wasm_file_path, p_size);
-    BH_FREE(wasm_file_path);
     return *p_buffer != NULL;
 }
 
 static void
-module_destroyer_cb(uint8 *buffer, uint32 size)
+moudle_destroyer(uint8 *buffer, uint32 size)
 {
-    printf("- release the read file buffer\n");
     if (!buffer) {
         return;
     }
 
-    BH_FREE(buffer);
+    wasm_runtime_free(buffer);
     buffer = NULL;
 }
+#endif /* WASM_ENABLE_MULTI_MODULE */
 
 /* 10M */
 static char sandbox_memory_space[10 * 1024 * 1024] = { 0 };
 int
-main()
+main(int argc, char *argv[])
 {
     bool ret = false;
+    if (argc != 2) {
+        return -1;
+    }
+    char *wasm_file = argv[1];
     /* 16K */
     const uint32 stack_size = 16 * 1024;
     const uint32 heap_size = 16 * 1024;
@@ -84,16 +88,16 @@ main()
 #if WASM_ENABLE_MULTI_MODULE != 0
     printf("- wasm_runtime_set_module_reader\n");
     /* set module reader and destroyer */
-    wasm_runtime_set_module_reader(module_reader_cb, module_destroyer_cb);
+    wasm_runtime_set_module_reader(module_reader_callback, moudle_destroyer);
 #endif
 
     /* load WASM byte buffer from WASM bin file */
-    if (!module_reader_cb("mC", &file_buf, &file_buf_size)) {
+    if (!(file_buf =
+              (uint8 *)bh_read_file_to_buffer(wasm_file, &file_buf_size)))
         goto RELEASE_RUNTIME;
-    }
-
     /* load mC and let WAMR load mA and mB */
     printf("- wasm_runtime_load\n");
+
     if (!(module = wasm_runtime_load(file_buf, file_buf_size, error_buf,
                                      sizeof(error_buf)))) {
         printf("%s\n", error_buf);
@@ -158,7 +162,7 @@ UNLOAD_MODULE:
     printf("- wasm_runtime_unload\n");
     wasm_runtime_unload(module);
 RELEASE_BINARY:
-    module_destroyer_cb(file_buf, file_buf_size);
+    moudle_destroyer(file_buf, file_buf_size);
 RELEASE_RUNTIME:
     printf("- wasm_runtime_destroy\n");
     wasm_runtime_destroy();

tests/wamr-test-suites/spec-test-script/muti_module_aot_ignore_cases.patch

@@ -0,0 +1,174 @@
+diff --git a/test/core/linking.wast b/test/core/linking.wast
+index d0bfb5f..6617945 100644
+--- a/test/core/linking.wast
++++ b/test/core/linking.wast
+@@ -35,7 +35,7 @@
+ 
+ 
+ ;; Globals
+-
++(;
+ (module $Mg
+   (global $glob (export "glob") i32 (i32.const 42))
+   (func (export "get") (result i32) (global.get $glob))
+@@ -63,7 +63,7 @@
+   (export "Mg.get_mut" (func $get_mut))
+   (export "Mg.set_mut" (func $set_mut))
+ )
+-
++;)
+ (;
+ (assert_return (get $Mg "glob") (i32.const 42))
+ (assert_return (get $Ng "Mg.glob") (i32.const 42))
+@@ -84,7 +84,7 @@
+ (assert_return (invoke $Ng "Mg.get_mut") (i32.const 241))
+ ;)
+ 
+-
++(;
+ (assert_unlinkable
+   (module (import "Mg" "mut_glob" (global i32)))
+   "incompatible import type"
+@@ -166,7 +166,7 @@
+     (call_indirect (type 1) (local.get 0))
+   )
+ )
+-
++;)
+ (;
+ (assert_return (invoke $Mt "call" (i32.const 2)) (i32.const 4))
+ (assert_return (invoke $Nt "Mt.call" (i32.const 2)) (i32.const 4))
+@@ -191,7 +191,7 @@
+ (assert_return (invoke $Nt "call" (i32.const 3)) (i32.const -4))
+ (assert_trap (invoke $Nt "call" (i32.const 4)) "indirect call type mismatch")
+ ;)
+-
++(;
+ (module $Ot
+   (type (func (result i32)))
+ 
+@@ -204,7 +204,7 @@
+     (call_indirect (type 0) (local.get 0))
+   )
+ )
+-
++;)
+ (;
+ (assert_return (invoke $Mt "call" (i32.const 3)) (i32.const 4))
+ (assert_return (invoke $Nt "Mt.call" (i32.const 3)) (i32.const 4))
+@@ -231,7 +231,7 @@
+ 
+ (assert_trap (invoke $Ot "call" (i32.const 20)) "undefined element")
+ ;)
+-
++(;
+ (module
+   (table (import "Mt" "tab") 0 funcref)
+   (elem (i32.const 9) $f)
+@@ -266,7 +266,7 @@
+   "unknown import"
+ )
+ (assert_trap (invoke $Mt "call" (i32.const 7)) "uninitialized element")
+-
++;)
+ ;; Unlike in the v1 spec, active element segments stored before an
+ ;; out-of-bounds access persist after the instantiation failure.
+ (;
+@@ -297,7 +297,7 @@
+ (assert_return (invoke $Mt "call" (i32.const 7)) (i32.const 0))
+ ;)
+ 
+-
++(;
+ (module $Mtable_ex
+   (table $t1 (export "t-func") 1 funcref)
+   (table $t2 (export "t-extern") 1 externref)
+@@ -308,7 +308,7 @@
+   (table (import "Mtable_ex" "t-func") 1 funcref)
+   (table (import "Mtable_ex" "t-extern") 1 externref)
+ )
+-
++;)
+ (;
+ (assert_unlinkable
+   (module (table (import "Mtable_ex" "t-func") 1 externref))
+@@ -322,7 +322,7 @@
+ 
+ 
+ ;; Memories
+-
++(;
+ (module $Mm
+   (memory (export "mem") 1 5)
+   (data (i32.const 10) "\00\01\02\03\04\05\06\07\08\09")
+@@ -357,14 +357,14 @@
+     (i32.load8_u (local.get 0))
+   )
+ )
+-
++;)
+ (;
+ (assert_return (invoke $Mm "load" (i32.const 12)) (i32.const 0xa7))
+ (assert_return (invoke $Nm "Mm.load" (i32.const 12)) (i32.const 0xa7))
+ (assert_return (invoke $Nm "load" (i32.const 12)) (i32.const 0xf2))
+ (assert_return (invoke $Om "load" (i32.const 12)) (i32.const 0xa7))
+ ;)
+-
++(;
+ (module
+   (memory (import "Mm" "mem") 0)
+   (data (i32.const 0xffff) "a")
+@@ -385,7 +385,7 @@
+     (memory.grow (local.get 0))
+   )
+ )
+-
++;)
+ (;
+ (assert_return (invoke $Pm "grow" (i32.const 0)) (i32.const 1))
+ (assert_return (invoke $Pm "grow" (i32.const 2)) (i32.const 1))
+@@ -396,7 +396,7 @@
+ (assert_return (invoke $Pm "grow" (i32.const 1)) (i32.const -1))
+ (assert_return (invoke $Pm "grow" (i32.const 0)) (i32.const 5))
+ ;)
+-
++(;
+ (assert_unlinkable
+   (module
+     (func $host (import "spectest" "print"))
+@@ -419,11 +419,12 @@
+   )
+   "out of bounds memory access"
+ )
++;)
+ (;
+ (assert_return (invoke $Mm "load" (i32.const 0)) (i32.const 97))
+ (assert_return (invoke $Mm "load" (i32.const 327670)) (i32.const 0))
+ ;)
+-
++(;
+ (assert_trap
+   (module
+     (memory (import "Mm" "mem") 1)
+@@ -434,10 +435,11 @@
+   )
+   "out of bounds table access"
+ )
++;)
+ (;
+ (assert_return (invoke $Mm "load" (i32.const 0)) (i32.const 97))
+ ;)
+-
++(;
+ ;; Store is modified if the start function traps.
+ (module $Ms
+   (type $t (func (result i32)))
+@@ -451,7 +453,7 @@
+   )
+ )
+ (register "Ms" $Ms)
+-
++;)
+ (;
+ (assert_trap
+   (module

tests/wamr-test-suites/spec-test-script/runtest.py

@@ -1299,6 +1299,16 @@ if __name__ == "__main__":
 
                     # add new_module copied from the old into temp_file_repo[]
                     temp_file_repo.append(new_module)
+
+                    if test_aot:
+                        new_module_aot = os.path.join(tempfile.gettempdir(), name_new + ".aot")
+                        r = compile_wasm_to_aot(new_module, new_module_aot, True, opts, r)
+                        try:
+                            assert_prompt(r, ['Compile success'], opts.start_timeout, True)
+                        except:
+                            raise Exception("compile wasm to aot failed")
+                        # add aot module into temp_file_repo[]
+                        temp_file_repo.append(new_module_aot)
                 else:
                     # there is no name defined in register cmd
                     raise Exception("can not find module name from the register")
@@ -1341,3 +1351,4 @@ if __name__ == "__main__":
             log("Leaving tempfiles: %s" % ([wast_tempfile, wasm_tempfile]))
 
         sys.exit(ret_code)
+        

tests/wamr-test-suites/test_wamr.sh

@@ -333,6 +333,9 @@ function spec_test()
     if [[ ${ENABLE_SIMD} == 1 ]]; then
         git apply ../../spec-test-script/simd_ignore_cases.patch
     fi
+    if [[ ${ENABLE_MULTI_MODULE} == 1 && $1 == 'aot'  ]]; then
+        git apply ../../spec-test-script/muti_module_aot_ignore_cases.patch
+    fi
 
     # udpate thread cases
     if [ ${ENABLE_MULTI_THREAD} == 1 ]; then
@@ -424,7 +427,7 @@ function spec_test()
 
     # multi-module only enable in interp mode
     if [[ 1 == ${ENABLE_MULTI_MODULE} ]]; then
-        if [[ $1 == 'classic-interp' || $1 == 'fast-interp' ]]; then
+        if [[ $1 == 'classic-interp' || $1 == 'fast-interp' || $1 == 'aot' ]]; then
             ARGS_FOR_SPEC_TEST+="-M "
         fi
     fi