From 32b2943369bccce52fbbc86bc9239757d37cbf52 Mon Sep 17 00:00:00 2001 From: Xu Jun <693788454@qq.com> Date: Thu, 16 Jul 2020 20:35:04 +0800 Subject: [PATCH] enable pthread for AoT && update AOT current version to 2 (#311) --- core/config.h | 2 +- core/iwasm/aot/aot_loader.c | 245 ++++--- core/iwasm/aot/aot_runtime.c | 643 +++++++++++++----- core/iwasm/aot/aot_runtime.h | 94 ++- core/iwasm/common/wasm_exec_env.h | 11 +- core/iwasm/common/wasm_runtime_common.c | 44 +- core/iwasm/common/wasm_runtime_common.h | 8 + core/iwasm/common/wasm_shared_memory.c | 11 +- core/iwasm/common/wasm_shared_memory.h | 27 +- core/iwasm/compilation/aot.c | 159 +++-- core/iwasm/compilation/aot.h | 101 ++- core/iwasm/compilation/aot_emit_aot_file.c | 131 +++- core/iwasm/compilation/aot_emit_control.c | 125 ++++ core/iwasm/compilation/aot_emit_control.h | 5 + core/iwasm/compilation/aot_emit_function.c | 16 + core/iwasm/compilation/aot_emit_memory.c | 119 ++-- core/iwasm/compilation/aot_emit_variable.c | 5 +- core/iwasm/compilation/aot_llvm.c | 244 +++++-- core/iwasm/compilation/aot_llvm.h | 21 +- core/iwasm/include/aot_export.h | 1 + core/iwasm/interpreter/wasm_interp_classic.c | 4 +- core/iwasm/interpreter/wasm_interp_fast.c | 4 +- core/iwasm/interpreter/wasm_loader.c | 8 +- core/iwasm/interpreter/wasm_mini_loader.c | 25 +- core/iwasm/interpreter/wasm_runtime.c | 8 +- core/iwasm/interpreter/wasm_runtime.h | 6 +- .../lib-pthread/lib_pthread_wrapper.c | 12 +- .../libraries/thread-mgr/thread_manager.c | 30 +- doc/pthread_library.md | 11 +- product-mini/platforms/linux/CMakeLists.txt | 5 + wamr-compiler/CMakeLists.txt | 2 + wamr-compiler/main.c | 6 + 32 files changed, 1549 insertions(+), 584 deletions(-) diff --git a/core/config.h b/core/config.h index 104dfdf3d..9dd042a5c 100644 --- a/core/config.h +++ b/core/config.h @@ -62,7 +62,7 @@ enum { #endif #define AOT_MAGIC_NUMBER 0x746f6100 -#define AOT_CURRENT_VERSION 1 +#define AOT_CURRENT_VERSION 2 #ifndef WASM_ENABLE_JIT #define WASM_ENABLE_JIT 0 diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index af0d8fdb5..e6c48cadf 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -349,6 +349,14 @@ fail: return false; } +static void +destroy_import_memories(AOTImportMemory *import_memories, + bool is_jit_mode) +{ + if (!is_jit_mode) + wasm_runtime_free(import_memories); +} + static void destroy_mem_init_data_list(AOTMemInitData **data_list, uint32 count, bool is_jit_mode) @@ -420,11 +428,28 @@ load_memory_info(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, char *error_buf, uint32 error_buf_size) { + uint32 i; + uint64 total_size; const uint8 *buf = *p_buf; - read_uint32(buf, buf_end, module->num_bytes_per_page); - read_uint32(buf, buf_end, module->mem_init_page_count); - read_uint32(buf, buf_end, module->mem_max_page_count); + read_uint32(buf, buf_end, module->import_memory_count); + /* We don't support import_memory_count > 0 currently */ + bh_assert(module->import_memory_count == 0); + + read_uint32(buf, buf_end, module->memory_count); + total_size = sizeof(AOTMemory) * (uint64)module->memory_count; + if (!(module->memories = + loader_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < module->memory_count; i++) { + read_uint32(buf, buf_end, module->memories[i].memory_flags); + read_uint32(buf, buf_end, module->memories[i].num_bytes_per_page); + read_uint32(buf, buf_end, module->memories[i].mem_init_page_count); + read_uint32(buf, buf_end, module->memories[i].mem_max_page_count); + } + read_uint32(buf, buf_end, module->mem_init_data_count); /* load memory init data list */ @@ -439,6 +464,20 @@ fail: return false; } +static void +destroy_import_tables(AOTImportTable *import_tables, bool is_jit_mode) +{ + if (!is_jit_mode) + wasm_runtime_free(import_tables); +} + +static void +destroy_tables(AOTTable *tables, bool is_jit_mode) +{ + if (!is_jit_mode) + wasm_runtime_free(tables); +} + static void destroy_table_init_data_list(AOTTableInitData **data_list, uint32 count, bool is_jit_mode) @@ -452,6 +491,36 @@ destroy_table_init_data_list(AOTTableInitData **data_list, uint32 count, } } +static bool +load_table_list(const uint8 **p_buf, const uint8 *buf_end, + AOTModule *module, char *error_buf, uint32 error_buf_size) +{ + const uint8 *buf = *p_buf; + AOTTable *table; + uint64 size; + uint32 i; + + /* Allocate memory */ + size = sizeof(AOTTable) * (uint64)module->table_count; + if (!(module->tables = table = + loader_malloc(size, error_buf, error_buf_size))) { + return false; + } + + /* Create each table data segment */ + for (i = 0; i < module->table_count; i++, table++) { + read_uint32(buf, buf_end, table->elem_type); + read_uint32(buf, buf_end, table->table_flags); + read_uint32(buf, buf_end, table->table_init_size); + read_uint32(buf, buf_end, table->table_max_size); + } + + *p_buf = buf; + return true; +fail: + return false; +} + static bool load_table_init_data_list(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, @@ -471,9 +540,10 @@ load_table_init_data_list(const uint8 **p_buf, const uint8 *buf_end, /* Create each table data segment */ for (i = 0; i < module->table_init_data_count; i++) { - uint32 init_expr_type, func_index_count; + uint32 table_index, init_expr_type, func_index_count; uint64 init_expr_value, size1; + read_uint32(buf, buf_end, table_index); read_uint32(buf, buf_end, init_expr_type); read_uint64(buf, buf_end, init_expr_value); read_uint32(buf, buf_end, func_index_count); @@ -485,6 +555,7 @@ load_table_init_data_list(const uint8 **p_buf, const uint8 *buf_end, return false; } + data_list[i]->table_index = table_index; data_list[i]->offset.init_expr_type = (uint8)init_expr_type; data_list[i]->offset.u.i64 = (int64)init_expr_value; data_list[i]->func_index_count = func_index_count; @@ -504,7 +575,16 @@ load_table_info(const uint8 **p_buf, const uint8 *buf_end, { const uint8 *buf = *p_buf; - read_uint32(buf, buf_end, module->table_size); + read_uint32(buf, buf_end, module->import_table_count); + /* We don't support import_table_count > 0 currently */ + bh_assert(module->import_table_count == 0); + + read_uint32(buf, buf_end, module->table_count); + if (module->table_count > 0 + && !load_table_list(&buf, buf_end, module, + error_buf, error_buf_size)) + return false; + read_uint32(buf, buf_end, module->table_init_data_count); /* load table init data list */ @@ -1088,40 +1168,42 @@ fail: } static void -destroy_export_funcs(AOTExportFunc *export_funcs, bool is_jit_mode) +destroy_exports(AOTExport *exports, bool is_jit_mode) { if (!is_jit_mode) - wasm_runtime_free(export_funcs); + wasm_runtime_free(exports); } static bool -load_export_funcs(const uint8 **p_buf, const uint8 *buf_end, - AOTModule *module, - char *error_buf, uint32 error_buf_size) +load_exports(const uint8 **p_buf, const uint8 *buf_end, + AOTModule *module, char *error_buf, uint32 error_buf_size) { const uint8 *buf = *p_buf; - AOTExportFunc *export_funcs; + AOTExport *exports; uint64 size; uint32 i; /* Allocate memory */ - size = sizeof(AOTExportFunc) * (uint64)module->export_func_count; - if (!(module->export_funcs = export_funcs = + size = sizeof(AOTExport) * (uint64)module->export_count; + if (!(module->exports = exports = loader_malloc(size, error_buf, error_buf_size))) { return false; } - /* Create each export func */ - for (i = 0; i < module->export_func_count; i++) { - read_uint32(buf, buf_end, export_funcs[i].func_index); - if (export_funcs[i].func_index >= + /* Create each export */ + for (i = 0; i < module->export_count; i++) { + read_uint32(buf, buf_end, exports[i].index); + read_uint8(buf, buf_end, exports[i].kind); + read_string(buf, buf_end, exports[i].name); +#if 0 /* TODO: check kind and index */ + if (export_funcs[i].index >= module->func_count + module->import_func_count) { set_error_buf(error_buf, error_buf_size, "AOT module load failed: " "function index is out of range."); return false; } - read_string(buf, buf_end, export_funcs[i].func_name); +#endif } *p_buf = buf; @@ -1138,9 +1220,9 @@ load_export_section(const uint8 *buf, const uint8 *buf_end, const uint8 *p = buf, *p_end = buf_end; /* load export functions */ - read_uint32(p, p_end, module->export_func_count); - if (module->export_func_count > 0 - && !load_export_funcs(&p, p_end, module, error_buf, error_buf_size)) + read_uint32(p, p_end, module->export_count); + if (module->export_count > 0 + && !load_exports(&p, p_end, module, error_buf, error_buf_size)) return false; if (p != p_end) { @@ -1385,12 +1467,11 @@ load_relocation_section(const uint8 *buf, const uint8 *buf_end, { AOTRelocationGroup *groups = NULL, *group; uint32 symbol_count = 0; - uint32 group_count = 0, i, j, func_index, func_type_index; + uint32 group_count = 0, i, j; uint64 size; uint32 *symbol_offsets, total_string_len; uint8 *symbol_buf, *symbol_buf_end; bool ret = false; - AOTExportFunc *export_func; read_uint32(buf, buf_end, symbol_count); @@ -1510,21 +1591,6 @@ load_relocation_section(const uint8 *buf, const uint8 *buf_end, } } - export_func = module->export_funcs; - for (i = 0; i < module->export_func_count; i++, export_func++) { - func_index = export_func->func_index - module->import_func_count; - if (func_index >= module->func_count) { - set_error_buf(error_buf, error_buf_size, - "AOT module load failed: " - "invalid export function index."); - ret = false; - goto fail; - } - func_type_index = module->func_type_indexes[func_index]; - export_func->func_type = module->func_types[func_type_index]; - export_func->func_ptr = module->func_ptrs[func_index]; - } - ret = true; fail: @@ -1621,17 +1687,13 @@ static void aot_free(void *ptr) static AOTModule* create_module(char *error_buf, uint32 error_buf_size) { - AOTModule *module = wasm_runtime_malloc(sizeof(AOTModule)); + AOTModule *module = + loader_malloc(sizeof(AOTModule), error_buf, error_buf_size); if (!module) { - set_error_buf(error_buf, error_buf_size, - "AOT module load failed: " - "allocate memory failed."); return NULL; } - memset(module, 0, sizeof(AOTModule)); - module->module_type = Wasm_Module_AoT; if (!(module->const_str_set = @@ -1703,10 +1765,9 @@ create_sections(const uint8 *buf, uint32 size, read_uint32(p, p_end, section_size); CHECK_BUF(p, p_end, section_size); - if (!(section = wasm_runtime_malloc(sizeof(AOTSection)))) { - set_error_buf(error_buf, error_buf_size, - "AOT module load failed: " - "allocate memory failed."); + if (!(section = + loader_malloc(sizeof(AOTSection), + error_buf, error_buf_size))) { goto fail; } @@ -1858,25 +1919,38 @@ aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, AOTModule *module; /* Allocate memory for module */ - if (!(module = wasm_runtime_malloc(sizeof(AOTModule)))) { - set_error_buf(error_buf, error_buf_size, - "Allocate memory for AOT module failed."); + if (!(module = + loader_malloc(sizeof(AOTModule), error_buf, error_buf_size))) { return NULL; } - memset(module, 0, sizeof(AOTModule)); - module->module_type = Wasm_Module_AoT; - module->num_bytes_per_page = comp_data->num_bytes_per_page; - module->mem_init_page_count = comp_data->mem_init_page_count; - module->mem_max_page_count = comp_data->mem_max_page_count; + + module->import_memory_count = comp_data->import_memory_count; + module->import_memories = comp_data->import_memories; + + module->memory_count = comp_data->memory_count; + if (module->memory_count) { + size = sizeof(AOTMemory) * (uint64)module->memory_count; + if (!(module->memories = + loader_malloc(size, error_buf, error_buf_size))) { + goto fail1; + } + + bh_memcpy_s(module->memories, size, comp_data->memories, size); + } module->mem_init_data_list = comp_data->mem_init_data_list; module->mem_init_data_count = comp_data->mem_init_data_count; + module->import_table_count = comp_data->import_table_count; + module->import_tables = comp_data->import_tables; + + module->table_count = comp_data->table_count; + module->tables = comp_data->tables; + module->table_init_data_list = comp_data->table_init_data_list; module->table_init_data_count = comp_data->table_init_data_count; - module->table_size = comp_data->table_size; module->func_type_count = comp_data->func_type_count; module->func_types = comp_data->func_types; @@ -1899,15 +1973,13 @@ aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, /* Allocate memory for function pointers */ size = (uint64)module->func_count * sizeof(void *); - if (size >= UINT32_MAX - || !(module->func_ptrs = wasm_runtime_malloc((uint32)size))) { - set_error_buf(error_buf, error_buf_size, "Create func ptrs fail."); - goto fail1; + if (!(module->func_ptrs = + loader_malloc(size, error_buf, error_buf_size))) { + goto fail2; } /* Resolve function addresses */ bh_assert(comp_ctx->exec_engine); - memset(module->func_ptrs, 0, (uint32)size); for (i = 0; i < comp_data->func_count; i++) { snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); if (!(module->func_ptrs[i] = @@ -1915,30 +1987,21 @@ aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, func_name))) { set_error_buf(error_buf, error_buf_size, "Get function address fail."); - goto fail2; + goto fail3; } } /* Allocation memory for function type indexes */ size = (uint64)module->func_count * sizeof(uint32); - if (size >= UINT32_MAX - || !(module->func_type_indexes = wasm_runtime_malloc((uint32)size))) { - set_error_buf(error_buf, error_buf_size, "Create func type indexes fail."); - goto fail2; + if (!(module->func_type_indexes = + loader_malloc(size, error_buf, error_buf_size))) { + goto fail3; } - memset(module->func_type_indexes, 0, (uint32)size); for (i = 0; i < comp_data->func_count; i++) module->func_type_indexes[i] = comp_data->funcs[i]->func_type_index; - module->export_func_count = comp_data->export_func_count; - module->export_funcs = comp_data->export_funcs; - - /* Set export function pointers */ - for (i = 0; i < module->export_func_count; i++) { - module->export_funcs[i].func_ptr = - module->func_ptrs[module->export_funcs[i].func_index - - module->import_func_count]; - } + module->export_count = comp_data->wasm_module->export_count; + module->exports = comp_data->wasm_module->exports; module->start_func_index = comp_data->start_func_index; if (comp_data->start_func_index != (uint32)-1) { @@ -1975,8 +2038,11 @@ aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, return module; -fail2: +fail3: wasm_runtime_free(module->func_ptrs); +fail2: + if (module->memory_count > 0) + wasm_runtime_free(module->memories); fail1: wasm_runtime_free(module); return NULL; @@ -2001,6 +2067,9 @@ aot_convert_wasm_module(WASMModule *wasm_module, } option.is_jit_mode = true; +#if WASM_ENABLE_THREAD_MGR != 0 + option.enable_thread_mgr = true; +#endif comp_ctx = aot_create_comp_context(comp_data, &option); if (!comp_ctx) { aot_last_error = aot_get_last_error(); @@ -2046,11 +2115,25 @@ aot_unload(AOTModule *module) wasm_loader_unload(module->wasm_module); #endif + if (module->import_memories) + destroy_import_memories(module->import_memories, + module->is_jit_mode); + + if (module->memories) + wasm_runtime_free(module->memories); + if (module->mem_init_data_list) destroy_mem_init_data_list(module->mem_init_data_list, module->mem_init_data_count, module->is_jit_mode); + if (module->import_tables) + destroy_import_tables(module->import_tables, + module->is_jit_mode); + + if (module->tables) + destroy_tables(module->tables, module->is_jit_mode); + if (module->table_init_data_list) destroy_table_init_data_list(module->table_init_data_list, module->table_init_data_count, @@ -2073,9 +2156,9 @@ aot_unload(AOTModule *module) destroy_import_funcs(module->import_funcs, module->is_jit_mode); - if (module->export_funcs) - destroy_export_funcs(module->export_funcs, - module->is_jit_mode); + if (module->exports) + destroy_exports(module->exports, + module->is_jit_mode); if (module->func_type_indexes) wasm_runtime_free(module->func_type_indexes); diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 7805b1f9e..d3fe3f690 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -6,6 +6,9 @@ #include "aot_runtime.h" #include "bh_log.h" #include "mem_alloc.h" +#if WASM_ENABLE_SHARED_MEMORY != 0 +#include "../common/wasm_shared_memory.h" +#endif static void set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) @@ -38,7 +41,7 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, uint32 i; InitializerExpression *init_expr; uint8 *p = (uint8*)module_inst->global_data.ptr; - AOTImportGlobal *import_global = module->import_globals;; + AOTImportGlobal *import_global = module->import_globals; AOTGlobal *global = module->globals; /* Initialize import global data */ @@ -146,22 +149,93 @@ table_instantiate(AOTModuleInstance *module_inst, AOTModule *module, return true; } -static bool +static void +memories_deinstantiate(AOTModuleInstance *module_inst) +{ + uint32 i; + AOTMemoryInstance *memory_inst; + + for (i = 0; i < module_inst->memory_count; i++) { + memory_inst = ((AOTMemoryInstance **)module_inst->memories.ptr)[i]; + if (memory_inst) { +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (memory_inst->is_shared) { + int32 ref_count = + shared_memory_dec_reference( + (WASMModuleCommon *)module_inst->aot_module.ptr); + bh_assert(ref_count >= 0); + + /* if the reference count is not zero, + don't free the memory */ + if (ref_count > 0) + continue; + } +#endif + if (memory_inst->heap_handle.ptr) + mem_allocator_destroy(memory_inst->heap_handle.ptr); + + if (memory_inst->heap_data.ptr) { +#ifndef OS_ENABLE_HW_BOUND_CHECK + wasm_runtime_free(memory_inst->heap_data.ptr); +#else + os_munmap((uint8*)memory_inst->memory_data.ptr - 2 * (uint64)BH_GB, + 8 * (uint64)BH_GB); +#endif + } + } + } + wasm_runtime_free(module_inst->memories.ptr); +} + +static AOTMemoryInstance* memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, + AOTMemoryInstance *memory_inst, AOTMemory *memory, uint32 heap_size, char *error_buf, uint32 error_buf_size) { - uint32 i, global_index, global_data_offset, base_offset, length; - AOTMemInitData *data_seg; void *heap_handle; - uint64 memory_data_size = (uint64)module->num_bytes_per_page - * module->mem_init_page_count; + uint64 memory_data_size = (uint64)memory->num_bytes_per_page + * memory->mem_init_page_count; uint64 total_size = heap_size + memory_data_size; uint8 *p; +#if WASM_ENABLE_SHARED_MEMORY != 0 + AOTMemoryInstance *shared_memory_instance; + bool is_shared_memory = memory->memory_flags & 0x02 ? true : false; + uint64 max_memory_data_size = (uint64)memory->num_bytes_per_page + * memory->mem_max_page_count; + + /* Shared memory */ + if (is_shared_memory) { + WASMSharedMemNode *node = + wasm_module_get_shared_memory((WASMModuleCommon *)module); + /* If the memory of this module has been instantiated, + return the memory instance directly */ + if (node) { + uint32 ref_count; + ref_count = shared_memory_inc_reference( + (WASMModuleCommon *)module); + bh_assert(ref_count > 0); + shared_memory_instance = + (AOTMemoryInstance *)shared_memory_get_memory_inst(node); + bh_assert(shared_memory_instance); + + /* Set the shared memory flag, so the runtime will get the + actual memory inst through module_inst->memories array */ + memory_inst->is_shared = true; + (void)ref_count; + return shared_memory_instance; + } +#ifndef OS_ENABLE_HW_BOUND_CHECK + /* Allocate max page for shared memory */ + total_size = heap_size + max_memory_data_size; +#endif + } +#endif + #ifndef OS_ENABLE_HW_BOUND_CHECK /* Allocate memory */ if (!(p = runtime_malloc(total_size, error_buf, error_buf_size))) { - return false; + return NULL; } #else uint8 *mapped_mem; @@ -178,7 +252,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, MMAP_PROT_NONE, MMAP_MAP_NONE))) { set_error_buf(error_buf, error_buf_size, "AOT module instantiate failed: mmap memory failed."); - return false; + return NULL; } p = mapped_mem + 2 * (uint64)BH_GB - heap_size; @@ -186,40 +260,128 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, set_error_buf(error_buf, error_buf_size, "AOT module instantiate failed: mprotec memory failed."); os_munmap(mapped_mem, map_size); - return false; + return NULL; } memset(p, 0, (uint32)total_size); #endif + memory_inst->module_type = Wasm_Module_AoT; /* Initialize heap info */ - module_inst->heap_data.ptr = p; + memory_inst->heap_data.ptr = p; p += heap_size; - module_inst->heap_data_end.ptr = p; - module_inst->heap_data_size = heap_size; - module_inst->heap_base_offset = -(int32)heap_size; + memory_inst->heap_data_end.ptr = p; + memory_inst->heap_data_size = heap_size; + memory_inst->heap_base_offset = -(int32)heap_size; if (heap_size > 0) { - if (!(heap_handle = mem_allocator_create(module_inst->heap_data.ptr, + if (!(heap_handle = mem_allocator_create(memory_inst->heap_data.ptr, heap_size))) { set_error_buf(error_buf, error_buf_size, - "AOT module instantiate failed: init app heap failed."); + "AOT module instantiate failed:" + "init app heap failed."); goto fail1; } - module_inst->heap_handle.ptr = heap_handle; + memory_inst->heap_handle.ptr = heap_handle; } /* Init memory info */ - module_inst->memory_data.ptr = p; - p += (uint32)memory_data_size; - module_inst->memory_data_end.ptr = p; - module_inst->memory_data_size = (uint32)memory_data_size; - module_inst->mem_cur_page_count = module->mem_init_page_count; - module_inst->mem_max_page_count = module->mem_max_page_count; + memory_inst->memory_data.ptr = p; +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (is_shared_memory) { + p += (uint32)max_memory_data_size; + } + else +#endif + { + p += (uint32)memory_data_size; + } + memory_inst->memory_data_end.ptr = p; + memory_inst->memory_data_size = (uint32)memory_data_size; + memory_inst->mem_cur_page_count = memory->mem_init_page_count; + memory_inst->mem_max_page_count = memory->mem_max_page_count; - module_inst->mem_bound_check_heap_base = (int64)module_inst->heap_base_offset; - module_inst->mem_bound_check_1byte = (int64)module_inst->memory_data_size - 1; - module_inst->mem_bound_check_2bytes = (int64)module_inst->memory_data_size - 2; - module_inst->mem_bound_check_4bytes = (int64)module_inst->memory_data_size - 4; - module_inst->mem_bound_check_8bytes = (int64)module_inst->memory_data_size - 8; + memory_inst->mem_bound_check_heap_base = memory_inst->heap_base_offset; + memory_inst->mem_bound_check_1byte = + (int64)memory_inst->memory_data_size - 1; + memory_inst->mem_bound_check_2bytes = + (int64)memory_inst->memory_data_size - 2; + memory_inst->mem_bound_check_4bytes = + (int64)memory_inst->memory_data_size - 4; + memory_inst->mem_bound_check_8bytes = + (int64)memory_inst->memory_data_size - 8; + +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (is_shared_memory) { + memory_inst->is_shared = true; + if (!shared_memory_set_memory_inst((WASMModuleCommon *)module, + (WASMMemoryInstanceCommon *)memory_inst)) { + set_error_buf(error_buf, error_buf_size, + "Instantiate memory failed:" + "allocate memory failed."); + goto fail2; + } + } +#endif + + return memory_inst; + +#if WASM_ENABLE_SHARED_MEMORY != 0 +fail2: + if (heap_size > 0) { + mem_allocator_destroy(memory_inst->heap_handle.ptr); + memory_inst->heap_handle.ptr = NULL; + } +#endif +fail1: +#ifndef OS_ENABLE_HW_BOUND_CHECK + wasm_runtime_free(memory_inst->heap_data.ptr); +#else + os_munmap(mapped_mem, map_size); +#endif + memory_inst->heap_data.ptr = NULL; + return NULL; +} + +static AOTMemoryInstance* +aot_get_default_memory(AOTModuleInstance *module_inst) +{ + if (module_inst->memories.ptr) + return ((AOTMemoryInstance **)module_inst->memories.ptr)[0]; + else + return NULL; +} + +static bool +memories_instantiate(AOTModuleInstance *module_inst, AOTModule *module, + uint32 heap_size, char *error_buf, uint32 error_buf_size) +{ + uint32 global_index, global_data_offset, base_offset, length; + uint32 i, memory_count = module->memory_count; + AOTMemoryInstance *memories, *memory_inst; + AOTMemInitData *data_seg; + uint64 total_size; + + module_inst->memory_count = memory_count; + total_size = sizeof(AOTPointer) * (uint64)memory_count; + if (!(module_inst->memories.ptr = + runtime_malloc(total_size, error_buf, error_buf_size))) { + return false; + } + + memories = module_inst->global_table_data.memory_instances; + for (i = 0; i < memory_count; i++, memories++) { + memory_inst = + memory_instantiate(module_inst, module, + memories, &module->memories[i], + heap_size, error_buf, error_buf_size); + if (!memory_inst) { + return false; + } + + ((AOTMemoryInstance **)module_inst->memories.ptr)[i] = memory_inst; + } + + /* Get default memory instance */ + memory_inst = aot_get_default_memory(module_inst); for (i = 0; i < module->mem_init_data_count; i++) { data_seg = module->mem_init_data_list[i]; @@ -255,47 +417,34 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, } /* Copy memory data */ - bh_assert(module_inst->memory_data.ptr); + bh_assert(memory_inst->memory_data.ptr); /* Check memory data */ /* check offset since length might negative */ - if (base_offset > module_inst->memory_data_size) { + if (base_offset > memory_inst->memory_data_size) { LOG_DEBUG("base_offset(%d) > memory_data_size(%d)", base_offset, - module_inst->memory_data_size); + memory_inst->memory_data_size); set_error_buf(error_buf, error_buf_size, "data segment does not fit"); - goto fail2; + return false; } /* check offset + length(could be zero) */ length = data_seg->byte_count; - if (base_offset + length > module_inst->memory_data_size) { + if (base_offset + length > memory_inst->memory_data_size) { LOG_DEBUG("base_offset(%d) + length(%d) > memory_data_size(%d)", - base_offset, length, module_inst->memory_data_size); + base_offset, length, memory_inst->memory_data_size); set_error_buf(error_buf, error_buf_size, "data segment does not fit"); - goto fail2; + return false; } - memcpy((uint8*)module_inst->memory_data.ptr + base_offset, - data_seg->bytes, length); + bh_memcpy_s((uint8*)memory_inst->memory_data.ptr + base_offset, + memory_inst->memory_data_size - base_offset, + data_seg->bytes, length); } return true; - -fail2: - if (heap_size > 0) { - mem_allocator_destroy(module_inst->heap_handle.ptr); - module_inst->heap_handle.ptr = NULL; - } -fail1: -#ifndef OS_ENABLE_HW_BOUND_CHECK - wasm_runtime_free(module_inst->heap_data.ptr); -#else - os_munmap(mapped_mem, map_size); -#endif - module_inst->heap_data.ptr = NULL; - return false; } static bool @@ -349,6 +498,64 @@ init_func_type_indexes(AOTModuleInstance *module_inst, AOTModule *module, return true; } +static bool +create_export_funcs(AOTModuleInstance *module_inst, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + AOTExport *exports = module->exports; + AOTFunctionInstance *export_func; + uint64 size; + uint32 i, func_index, ftype_index; + + for (i = 0; i < module->export_count; i++) { + if (exports[i].kind == EXPORT_KIND_FUNC) + module_inst->export_func_count++; + } + + if (module_inst->export_func_count > 0) { + /* Allocate memory */ + size = sizeof(AOTFunctionInstance) + * (uint64)module_inst->export_func_count; + if (!(module_inst->export_funcs.ptr = export_func = + runtime_malloc(size, error_buf, error_buf_size))) { + return false; + } + + for (i = 0; i < module->export_count; i++) { + if (exports[i].kind == EXPORT_KIND_FUNC) { + export_func->func_name = exports[i].name; + export_func->func_index = exports[i].index; + if (export_func->func_index < module->import_func_count) { + export_func->is_import_func = true; + export_func->u.func_import = + &module->import_funcs[export_func->func_index]; + } + else { + export_func->is_import_func = false; + func_index = export_func->func_index + - module->import_func_count; + ftype_index = module->func_type_indexes[func_index]; + export_func->u.func.func_type = + module->func_types[ftype_index]; + export_func->u.func.func_ptr = + module->func_ptrs[func_index]; + } + export_func++; + } + } + } + + return true; +} + +static bool +create_exports(AOTModuleInstance *module_inst, AOTModule *module, + char *error_buf, uint32 error_buf_size) +{ + return create_export_funcs(module_inst, module, + error_buf, error_buf_size); +} + static bool execute_post_inst_function(AOTModuleInstance *module_inst) { @@ -386,6 +593,22 @@ execute_start_function(AOTModuleInstance *module_inst) return !aot_get_exception(module_inst); } +#if WASM_ENABLE_BULK_MEMORY != 0 +static bool +execute_memory_init_function(AOTModuleInstance *module_inst) +{ + AOTFunctionInstance *memory_init_func = + aot_lookup_function(module_inst, "__wasm_call_ctors", "()"); + + if (!memory_init_func) + /* Not found */ + return true; + + return aot_create_exec_env_and_call_function(module_inst, memory_init_func, + 0, NULL); +} +#endif + AOTModuleInstance* aot_instantiate(AOTModule *module, bool is_sub_inst, uint32 stack_size, uint32 heap_size, @@ -394,8 +617,13 @@ aot_instantiate(AOTModule *module, bool is_sub_inst, AOTModuleInstance *module_inst; uint32 module_inst_struct_size = offsetof(AOTModuleInstance, global_table_data.bytes); - uint64 table_data_size = (uint64)module->table_size * sizeof(uint32); + uint64 module_inst_mem_inst_size = + (uint64)module->memory_count * sizeof(AOTMemoryInstance); + uint32 table_size = module->table_count > 0 ? + module->tables[0].table_init_size : 0; + uint64 table_data_size = (uint64)table_size * sizeof(uint32); uint64 total_size = (uint64)module_inst_struct_size + + module_inst_mem_inst_size + module->global_data_size + table_data_size; uint8 *p; @@ -418,7 +646,8 @@ aot_instantiate(AOTModule *module, bool is_sub_inst, module_inst->aot_module.ptr = module; /* Initialize global info */ - p = (uint8*)module_inst + module_inst_struct_size; + p = (uint8*)module_inst + module_inst_struct_size + + module_inst_mem_inst_size; module_inst->global_data.ptr = p; module_inst->global_data_size = module->global_data_size; if (!global_instantiate(module_inst, module, error_buf, error_buf_size)) @@ -427,15 +656,15 @@ aot_instantiate(AOTModule *module, bool is_sub_inst, /* Initialize table info */ p += module->global_data_size; module_inst->table_data.ptr = p; - module_inst->table_size = module->table_size; + module_inst->table_size = table_size; /* Set all elements to -1 to mark them as uninitialized elements */ memset(module_inst->table_data.ptr, -1, (uint32)table_data_size); if (!table_instantiate(module_inst, module, error_buf, error_buf_size)) goto fail; /* Initialize memory space */ - if (!memory_instantiate(module_inst, module, heap_size, - error_buf, error_buf_size)) + if (!memories_instantiate(module_inst, module, heap_size, + error_buf, error_buf_size)) goto fail; /* Initialize function pointers */ @@ -446,19 +675,24 @@ aot_instantiate(AOTModule *module, bool is_sub_inst, if (!init_func_type_indexes(module_inst, module, error_buf, error_buf_size)) goto fail; -#if WASM_ENABLE_LIBC_WASI != 0 - if (heap_size > 0 - && !wasm_runtime_init_wasi((WASMModuleInstanceCommon*)module_inst, - module->wasi_args.dir_list, - module->wasi_args.dir_count, - module->wasi_args.map_dir_list, - module->wasi_args.map_dir_count, - module->wasi_args.env, - module->wasi_args.env_count, - module->wasi_args.argv, - module->wasi_args.argc, - error_buf, error_buf_size)) + if (!create_exports(module_inst, module, error_buf, error_buf_size)) goto fail; + +#if WASM_ENABLE_LIBC_WASI != 0 + if (!is_sub_inst) { + if (heap_size > 0 + && !wasm_runtime_init_wasi((WASMModuleInstanceCommon*)module_inst, + module->wasi_args.dir_list, + module->wasi_args.dir_count, + module->wasi_args.map_dir_list, + module->wasi_args.map_dir_count, + module->wasi_args.env, + module->wasi_args.env_count, + module->wasi_args.argv, + module->wasi_args.argc, + error_buf, error_buf_size)) + goto fail; + } #endif /* Initialize the thread related data */ @@ -478,6 +712,25 @@ aot_instantiate(AOTModule *module, bool is_sub_inst, goto fail; } +#if WASM_ENABLE_BULK_MEMORY != 0 +#if WASM_ENABLE_LIBC_WASI != 0 + if (!module->is_wasi_module) { +#endif + /* Only execute the memory init function for main instance because + the data segments will be dropped once initialized. + */ + if (!is_sub_inst) { + if (!execute_memory_init_function(module_inst)) { + set_error_buf(error_buf, error_buf_size, + module_inst->cur_exception); + goto fail; + } + } +#if WASM_ENABLE_LIBC_WASI != 0 + } +#endif +#endif + return module_inst; fail: @@ -498,17 +751,11 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) wasm_runtime_destroy_wasi((WASMModuleInstanceCommon*)module_inst); #endif - if (module_inst->heap_handle.ptr) - mem_allocator_destroy(module_inst->heap_handle.ptr); + if (module_inst->memories.ptr) + memories_deinstantiate(module_inst); - if (module_inst->heap_data.ptr) { -#ifndef OS_ENABLE_HW_BOUND_CHECK - wasm_runtime_free(module_inst->heap_data.ptr); -#else - os_munmap((uint8*)module_inst->memory_data.ptr - 2 * (uint64)BH_GB, - 8 * (uint64)BH_GB); -#endif - } + if (module_inst->export_funcs.ptr) + wasm_runtime_free(module_inst->export_funcs.ptr); if (module_inst->func_ptrs.ptr) wasm_runtime_free(module_inst->func_ptrs.ptr); @@ -524,11 +771,12 @@ aot_lookup_function(const AOTModuleInstance *module_inst, const char *name, const char *signature) { uint32 i; - AOTModule *module = (AOTModule*)module_inst->aot_module.ptr; + AOTFunctionInstance *export_funcs = (AOTFunctionInstance *) + module_inst->export_funcs.ptr; - for (i = 0; i < module->export_func_count; i++) - if (!strcmp(module->export_funcs[i].func_name, name)) - return &module->export_funcs[i]; + for (i = 0; i < module_inst->export_func_count; i++) + if (!strcmp(export_funcs[i].func_name, name)) + return &export_funcs[i]; (void)signature; return NULL; } @@ -565,6 +813,7 @@ static void aot_signal_handler(void *sig_addr) { AOTModuleInstance *module_inst; + AOTMemoryInstance *memory_inst; WASMJmpBuf *jmpbuf_node; uint8 *mapped_mem_start_addr, *mapped_mem_end_addr; uint8 *stack_min_addr; @@ -577,17 +826,22 @@ aot_signal_handler(void *sig_addr) && (jmpbuf_node = aot_exec_env->jmpbuf_stack_top)) { /* Get mapped mem info of current instance */ module_inst = (AOTModuleInstance *)aot_exec_env->module_inst; - mapped_mem_start_addr = (uint8*)module_inst->memory_data.ptr - - 2 * (uint64)BH_GB; - mapped_mem_end_addr = (uint8*)module_inst->memory_data.ptr - + 6 * (uint64)BH_GB; + /* Get the default memory instance */ + memory_inst = aot_get_default_memory(module_inst); + if (memory_inst) { + mapped_mem_start_addr = (uint8*)memory_inst->memory_data.ptr + - 2 * (uint64)BH_GB; + mapped_mem_end_addr = (uint8*)memory_inst->memory_data.ptr + + 6 * (uint64)BH_GB; + } /* Get stack info of current thread */ page_size = os_getpagesize(); stack_min_addr = get_stack_min_addr(aot_exec_env, page_size); - if (mapped_mem_start_addr <= (uint8*)sig_addr - && (uint8*)sig_addr < mapped_mem_end_addr) { + if (memory_inst + && (mapped_mem_start_addr <= (uint8*)sig_addr + && (uint8*)sig_addr < mapped_mem_end_addr)) { /* The address which causes segmentation fault is inside aot instance's guard regions */ aot_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS); @@ -721,7 +975,7 @@ aot_call_function(WASMExecEnv *exec_env, unsigned argc, uint32 argv[]) { AOTModuleInstance *module_inst = (AOTModuleInstance*)exec_env->module_inst; - AOTFuncType *func_type = function->func_type; + AOTFuncType *func_type = function->u.func.func_type; uint32 result_count = func_type->result_count; uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; bool ret; @@ -758,7 +1012,7 @@ aot_call_function(WASMExecEnv *exec_env, cell_num += wasm_value_type_cell_num(ext_ret_types[i]); } - ret = invoke_native_internal(exec_env, function->func_ptr, + ret = invoke_native_internal(exec_env, function->u.func.func_ptr, func_type, NULL, NULL, argv1, argc, argv); if (!ret || aot_get_exception(module_inst)) { if (argv1 != argv1_buf) @@ -789,7 +1043,7 @@ aot_call_function(WASMExecEnv *exec_env, return true; } else { - ret = invoke_native_internal(exec_env, function->func_ptr, + ret = invoke_native_internal(exec_env, function->u.func.func_ptr, func_type, NULL, NULL, argv, argc, argv); return ret && !aot_get_exception(module_inst) ? true : false; } @@ -894,24 +1148,26 @@ int32 aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, void **p_native_addr) { - uint8 *addr = mem_allocator_malloc(module_inst->heap_handle.ptr, size); + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + uint8 *addr = mem_allocator_malloc(memory_inst->heap_handle.ptr, size); if (!addr) { aot_set_exception(module_inst, "out of memory"); return 0; } if (p_native_addr) *p_native_addr = addr; - return (int32)(addr - (uint8*)module_inst->memory_data.ptr); + return (int32)(addr - (uint8*)memory_inst->memory_data.ptr); } void aot_module_free(AOTModuleInstance *module_inst, int32 ptr) { + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); if (ptr) { - uint8 *addr = (uint8*)module_inst->memory_data.ptr + ptr; - if ((uint8*)module_inst->heap_data.ptr < addr - && addr < (uint8*)module_inst->memory_data.ptr) - mem_allocator_free(module_inst->heap_handle.ptr, addr); + uint8 *addr = (uint8*)memory_inst->memory_data.ptr + ptr; + if ((uint8*)memory_inst->heap_data.ptr < addr + && addr < (uint8*)memory_inst->memory_data.ptr) + mem_allocator_free(memory_inst->heap_handle.ptr, addr); } } @@ -934,13 +1190,14 @@ bool aot_validate_app_addr(AOTModuleInstance *module_inst, int32 app_offset, uint32 size) { + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); /* integer overflow check */ if(app_offset + (int32)size < app_offset) { goto fail; } - if (module_inst->heap_base_offset <= app_offset - && app_offset + (int32)size <= (int32)module_inst->memory_data_size) { + if (memory_inst->heap_base_offset <= app_offset + && app_offset + (int32)size <= (int32)memory_inst->memory_data_size) { return true; } fail: @@ -953,15 +1210,16 @@ aot_validate_native_addr(AOTModuleInstance *module_inst, void *native_ptr, uint32 size) { uint8 *addr = (uint8*)native_ptr; - int32 memory_data_size = (int32)module_inst->memory_data_size; + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + int32 memory_data_size = (int32)memory_inst->memory_data_size; /* integer overflow check */ if (addr + size < addr) { goto fail; } - if ((uint8*)module_inst->heap_data.ptr <= addr - && addr + size <= (uint8*)module_inst->memory_data.ptr + if ((uint8*)memory_inst->heap_data.ptr <= addr + && addr + size <= (uint8*)memory_inst->memory_data.ptr + memory_data_size) { return true; } @@ -973,11 +1231,12 @@ fail: void * aot_addr_app_to_native(AOTModuleInstance *module_inst, int32 app_offset) { - int32 memory_data_size = (int32)module_inst->memory_data_size; - uint8 *addr = (uint8 *)module_inst->memory_data.ptr + app_offset; + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + int32 memory_data_size = (int32)memory_inst->memory_data_size; + uint8 *addr = (uint8 *)memory_inst->memory_data.ptr + app_offset; - if ((uint8*)module_inst->heap_data.ptr <= addr - && addr < (uint8*)module_inst->memory_data.ptr + if ((uint8*)memory_inst->heap_data.ptr <= addr + && addr < (uint8*)memory_inst->memory_data.ptr + memory_data_size) return addr; return NULL; @@ -987,12 +1246,13 @@ int32 aot_addr_native_to_app(AOTModuleInstance *module_inst, void *native_ptr) { uint8 *addr = (uint8*)native_ptr; - int32 memory_data_size = (int32)module_inst->memory_data_size; + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + int32 memory_data_size = (int32)memory_inst->memory_data_size; - if ((uint8*)module_inst->heap_data.ptr <= addr - && addr < (uint8*)module_inst->memory_data.ptr + if ((uint8*)memory_inst->heap_data.ptr <= addr + && addr < (uint8*)memory_inst->memory_data.ptr + memory_data_size) - return (int32)(addr - (uint8*)module_inst->memory_data.ptr); + return (int32)(addr - (uint8*)memory_inst->memory_data.ptr); return 0; } @@ -1002,12 +1262,13 @@ aot_get_app_addr_range(AOTModuleInstance *module_inst, int32 *p_app_start_offset, int32 *p_app_end_offset) { - int32 memory_data_size = (int32)module_inst->memory_data_size; + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + int32 memory_data_size = (int32)memory_inst->memory_data_size; - if (module_inst->heap_base_offset <= app_offset + if (memory_inst->heap_base_offset <= app_offset && app_offset < memory_data_size) { if (p_app_start_offset) - *p_app_start_offset = module_inst->heap_base_offset; + *p_app_start_offset = memory_inst->heap_base_offset; if (p_app_end_offset) *p_app_end_offset = memory_data_size; return true; @@ -1022,15 +1283,16 @@ aot_get_native_addr_range(AOTModuleInstance *module_inst, uint8 **p_native_end_addr) { uint8 *addr = (uint8*)native_ptr; - int32 memory_data_size = (int32)module_inst->memory_data_size; + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + int32 memory_data_size = (int32)memory_inst->memory_data_size; - if ((uint8*)module_inst->heap_data.ptr <= addr - && addr < (uint8*)module_inst->memory_data.ptr + if ((uint8*)memory_inst->heap_data.ptr <= addr + && addr < (uint8*)memory_inst->memory_data.ptr + memory_data_size) { if (p_native_start_addr) - *p_native_start_addr = (uint8*)module_inst->heap_data.ptr; + *p_native_start_addr = (uint8*)memory_inst->heap_data.ptr; if (p_native_end_addr) - *p_native_end_addr = (uint8*)module_inst->memory_data.ptr + *p_native_end_addr = (uint8*)memory_inst->memory_data.ptr + memory_data_size; return true; } @@ -1041,18 +1303,19 @@ aot_get_native_addr_range(AOTModuleInstance *module_inst, bool aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) { - uint8 *heap_data_old = module_inst->heap_data.ptr, *heap_data; + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + uint8 *heap_data_old = memory_inst->heap_data.ptr, *heap_data; uint32 num_bytes_per_page = - ((AOTModule*)module_inst->aot_module.ptr)->num_bytes_per_page; - uint32 cur_page_count = module_inst->mem_cur_page_count; - uint32 max_page_count = module_inst->mem_max_page_count; + ((AOTModule*)module_inst->aot_module.ptr)->memories[0].num_bytes_per_page; + uint32 cur_page_count = memory_inst->mem_cur_page_count; + uint32 max_page_count = memory_inst->mem_max_page_count; uint32 total_page_count = cur_page_count + inc_page_count; uint64 memory_data_size = (uint64)num_bytes_per_page * total_page_count; - uint32 heap_size = (uint32)((uint8*)module_inst->memory_data.ptr - - (uint8*)module_inst->heap_data.ptr); - uint32 total_size_old = heap_size + module_inst->memory_data_size; + uint32 heap_size = (uint32)((uint8*)memory_inst->memory_data.ptr + - (uint8*)memory_inst->heap_data.ptr); + uint32 total_size_old = heap_size + memory_inst->memory_data_size; uint64 total_size = heap_size + memory_data_size; - void *heap_handle_old = module_inst->heap_handle.ptr; + void *heap_handle_old = memory_inst->heap_handle.ptr; if (inc_page_count <= 0) /* No need to enlarge memory */ @@ -1069,16 +1332,25 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) return false; } +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (memory_inst->is_shared) { + /* For shared memory, we have reserved the maximum spaces during + instantiate, only change the cur_page_count here */ + memory_inst->mem_cur_page_count = total_page_count; + return true; + } +#endif + if (heap_size > 0) { /* Destroy heap's lock firstly, if its memory is re-allocated, we cannot access its lock again. */ - mem_allocator_destroy_lock(module_inst->heap_handle.ptr); + mem_allocator_destroy_lock(memory_inst->heap_handle.ptr); } if (!(heap_data = wasm_runtime_realloc(heap_data_old, (uint32)total_size))) { if (!(heap_data = wasm_runtime_malloc((uint32)total_size))) { if (heap_size > 0) { /* Restore heap's lock if memory re-alloc failed */ - mem_allocator_reinit_lock(module_inst->heap_handle.ptr); + mem_allocator_reinit_lock(memory_inst->heap_handle.ptr); } aot_set_exception(module_inst, "fail to enlarge memory."); return false; @@ -1091,39 +1363,40 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) memset(heap_data + total_size_old, 0, (uint32)total_size - total_size_old); - module_inst->heap_data.ptr = heap_data; - module_inst->heap_data_end.ptr = heap_data + heap_size; + memory_inst->heap_data.ptr = heap_data; + memory_inst->heap_data_end.ptr = heap_data + heap_size; if (heap_size > 0) { - module_inst->heap_handle.ptr = (uint8*)heap_handle_old + memory_inst->heap_handle.ptr = (uint8*)heap_handle_old + (heap_data - heap_data_old); - if (mem_allocator_migrate(module_inst->heap_handle.ptr, + if (mem_allocator_migrate(memory_inst->heap_handle.ptr, heap_handle_old) != 0) { aot_set_exception(module_inst, "fail to enlarge memory."); return false; } } - module_inst->mem_cur_page_count = total_page_count; - module_inst->memory_data_size = (uint32)memory_data_size; - module_inst->memory_data.ptr = (uint8*)heap_data + heap_size; - module_inst->memory_data_end.ptr = (uint8*)module_inst->memory_data.ptr + memory_inst->mem_cur_page_count = total_page_count; + memory_inst->memory_data_size = (uint32)memory_data_size; + memory_inst->memory_data.ptr = (uint8*)heap_data + heap_size; + memory_inst->memory_data_end.ptr = (uint8*)memory_inst->memory_data.ptr + (uint32)memory_data_size; - module_inst->mem_bound_check_1byte = module_inst->memory_data_size - 1; - module_inst->mem_bound_check_2bytes = module_inst->memory_data_size - 2; - module_inst->mem_bound_check_4bytes = module_inst->memory_data_size - 4; - module_inst->mem_bound_check_8bytes = module_inst->memory_data_size - 8; + memory_inst->mem_bound_check_1byte = memory_inst->memory_data_size - 1; + memory_inst->mem_bound_check_2bytes = memory_inst->memory_data_size - 2; + memory_inst->mem_bound_check_4bytes = memory_inst->memory_data_size - 4; + memory_inst->mem_bound_check_8bytes = memory_inst->memory_data_size - 8; return true; } #else bool aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) { + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); uint32 num_bytes_per_page = - ((AOTModule*)module_inst->aot_module.ptr)->num_bytes_per_page; - uint32 cur_page_count = module_inst->mem_cur_page_count; - uint32 max_page_count = module_inst->mem_max_page_count; + ((AOTModule*)module_inst->aot_module.ptr)->memories[0].num_bytes_per_page; + uint32 cur_page_count = memory_inst->mem_cur_page_count; + uint32 max_page_count = memory_inst->mem_max_page_count; uint32 total_page_count = cur_page_count + inc_page_count; uint64 memory_data_size = (uint64)num_bytes_per_page * total_page_count; @@ -1137,24 +1410,24 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) return false; } - if (os_mprotect(module_inst->memory_data.ptr, memory_data_size, + if (os_mprotect(memory_inst->memory_data.ptr, memory_data_size, MMAP_PROT_READ | MMAP_PROT_WRITE) != 0) { aot_set_exception(module_inst, "fail to enlarge memory."); return false; } - memset(module_inst->memory_data_end.ptr, 0, + memset(memory_inst->memory_data_end.ptr, 0, num_bytes_per_page * inc_page_count); - module_inst->mem_cur_page_count = total_page_count; - module_inst->memory_data_size = (uint32)memory_data_size; - module_inst->memory_data_end.ptr = (uint8*)module_inst->memory_data.ptr + memory_inst->mem_cur_page_count = total_page_count; + memory_inst->memory_data_size = (uint32)memory_data_size; + memory_inst->memory_data_end.ptr = (uint8*)memory_inst->memory_data.ptr + (uint32)memory_data_size; - module_inst->mem_bound_check_1byte = module_inst->memory_data_size - 1; - module_inst->mem_bound_check_2bytes = module_inst->memory_data_size - 2; - module_inst->mem_bound_check_4bytes = module_inst->memory_data_size - 4; - module_inst->mem_bound_check_8bytes = module_inst->memory_data_size - 8; + memory_inst->mem_bound_check_1byte = memory_inst->memory_data_size - 1; + memory_inst->mem_bound_check_2bytes = memory_inst->memory_data_size - 2; + memory_inst->mem_bound_check_4bytes = memory_inst->memory_data_size - 4; + memory_inst->mem_bound_check_8bytes = memory_inst->memory_data_size - 8; return true; } #endif @@ -1389,6 +1662,7 @@ bool aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, uint32 offset, uint32 len, uint32 dst) { + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); AOTModule *aot_module; uint8 *data = NULL; uint8 *maddr; @@ -1416,7 +1690,7 @@ aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, maddr = aot_addr_app_to_native(module_inst, dst); - bh_memcpy_s(maddr, module_inst->memory_data_size - dst, + bh_memcpy_s(maddr, memory_inst->memory_data_size - dst, data + offset, len); return true; } @@ -1441,3 +1715,68 @@ aot_data_drop(AOTModuleInstance *module_inst, uint32 seg_index) return true; } #endif /* WASM_ENABLE_BULK_MEMORY */ + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +aot_set_aux_stack(WASMExecEnv *exec_env, + uint32 start_offset, uint32 size) +{ + AOTModuleInstance *module_inst = + (AOTModuleInstance*)exec_env->module_inst; + AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + + uint32 stack_top_idx = + module->llvm_aux_stack_global_index; + uint32 data_end = + module->llvm_aux_data_end; + uint32 stack_bottom = + module->llvm_aux_stack_bottom; + bool is_stack_before_data = + stack_bottom < data_end ? true : false; + + /* Check the aux stack space, currently we don't allocate space in heap */ + if ((is_stack_before_data && (size > start_offset)) + || ((!is_stack_before_data) && (start_offset - data_end < size))) + return false; + + if ((stack_bottom != (uint32)-1) && (stack_top_idx != (uint32)-1)) { + /* The aux stack top is a wasm global, + set the initial value for the global */ + uint32 global_offset = + module->globals[stack_top_idx].data_offset; + uint8 *global_addr = module_inst->global_data.ptr + global_offset; + *(int32*)global_addr = start_offset; + + /* The aux stack boundary is a constant value, + set the value to exec_env */ + exec_env->aux_stack_boundary = start_offset - size; + return true; + } + + return false; +} + +bool +aot_get_aux_stack(WASMExecEnv *exec_env, + uint32 *start_offset, uint32 *size) +{ + AOTModuleInstance *module_inst = + (AOTModuleInstance*)exec_env->module_inst; + AOTModule *module = (AOTModule *)module_inst->aot_module.ptr; + + /* The aux stack information is resolved in loader + and store in module */ + uint32 stack_bottom = module->llvm_aux_stack_bottom; + uint32 total_aux_stack_size = module->llvm_aux_stack_size; + + if (stack_bottom != 0 && total_aux_stack_size != 0) { + if (start_offset) + *start_offset = stack_bottom; + if (size) + *size = total_aux_stack_size; + return true; + } + return false; +} + +#endif diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index a2f4908a7..0678a701c 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -69,18 +69,45 @@ typedef struct AOTRelocationGroup { AOTRelocation *relocations; } AOTRelocationGroup; +/* AOT function instance */ +typedef struct AOTFunctionInstance { + char *func_name; + uint32 func_index; + bool is_import_func; + union { + struct { + AOTFuncType *func_type; + /* function pointer linked */ + void *func_ptr; + } func; + AOTImportFunc *func_import; + } u; +} AOTFunctionInstance; + typedef struct AOTModule { uint32 module_type; + /* import memories */ + uint32 import_memory_count; + AOTImportMemory *import_memories; + /* memory info */ - uint32 num_bytes_per_page; - uint32 mem_init_page_count; - uint32 mem_max_page_count; + uint32 memory_count; + AOTMemory *memories; + + /* init data */ uint32 mem_init_data_count; AOTMemInitData **mem_init_data_list; - /* table info */ - uint32 table_size; + /* import tables */ + uint32 import_table_count; + AOTImportTable *import_tables; + + /* tables */ + uint32 table_count; + AOTTable *tables; + + /* table init data info */ uint32 table_init_data_count; AOTTableInitData **table_init_data_list; @@ -110,9 +137,9 @@ typedef struct AOTModule { /* function type indexes */ uint32 *func_type_indexes; - /* export function info */ - uint32 export_func_count; - AOTExportFunc *export_funcs; + /* export info */ + uint32 export_count; + AOTExport *exports; /* start function index, -1 denotes no start function */ uint32 start_func_index; @@ -160,13 +187,15 @@ typedef union { void *ptr; } AOTPointer; -typedef struct AOTModuleInstance { +typedef struct AOTMemoryInstance { uint32 module_type; - + /* shared memory flag */ + bool is_shared; /* memory space info */ uint32 mem_cur_page_count; uint32 mem_max_page_count; uint32 memory_data_size; + uint32 __padding__; AOTPointer memory_data; AOTPointer memory_data_end; @@ -177,6 +206,21 @@ typedef struct AOTModuleInstance { AOTPointer heap_data_end; AOTPointer heap_handle; + /* boundary check constants for aot code */ + int64 mem_bound_check_heap_base; + int64 mem_bound_check_1byte; + int64 mem_bound_check_2bytes; + int64 mem_bound_check_4bytes; + int64 mem_bound_check_8bytes; +} AOTMemoryInstance; + +typedef struct AOTModuleInstance { + uint32 module_type; + + /* memories */ + uint32 memory_count; + AOTPointer memories; + /* global and table info */ uint32 global_data_size; uint32 table_size; @@ -188,6 +232,16 @@ typedef struct AOTModuleInstance { /* function type indexes */ AOTPointer func_type_indexes; + /* export info */ + uint32 export_func_count; + uint32 export_global_count; + uint32 export_mem_count; + uint32 export_tab_count; + AOTPointer export_funcs; + AOTPointer export_globals; + AOTPointer export_memories; + AOTPointer export_tables; + /* The exception buffer for current thread. */ char cur_exception[128]; /* The custom data that can be set/get by @@ -198,13 +252,6 @@ typedef struct AOTModuleInstance { /* WASI context */ AOTPointer wasi_ctx; - /* boundary check constants for aot code */ - int64 mem_bound_check_heap_base; - int64 mem_bound_check_1byte; - int64 mem_bound_check_2bytes; - int64 mem_bound_check_4bytes; - int64 mem_bound_check_8bytes; - /* others */ int32 temp_ret; uint32 llvm_stack; @@ -215,12 +262,11 @@ typedef struct AOTModuleInstance { union { uint64 _make_it_8_byte_aligned_; + AOTMemoryInstance memory_instances[1]; uint8 bytes[1]; } global_table_data; } AOTModuleInstance; -typedef AOTExportFunc AOTFunctionInstance; - /* Target info, read from ELF header of object file */ typedef struct AOTTargetInfo { /* Binary type, elf32l/elf32b/elf64l/elf64b */ @@ -467,6 +513,16 @@ bool aot_data_drop(AOTModuleInstance *module_inst, uint32 seg_index); #endif +#if WASM_ENABLE_THREAD_MGR != 0 +bool +aot_set_aux_stack(WASMExecEnv *exec_env, + uint32 start_offset, uint32 size); + +bool +aot_get_aux_stack(WASMExecEnv *exec_env, + uint32 *start_offset, uint32 *size); +#endif + #ifdef OS_ENABLE_HW_BOUND_CHECK bool aot_signal_init(); diff --git a/core/iwasm/common/wasm_exec_env.h b/core/iwasm/common/wasm_exec_env.h index fa5310e5d..625676353 100644 --- a/core/iwasm/common/wasm_exec_env.h +++ b/core/iwasm/common/wasm_exec_env.h @@ -56,8 +56,15 @@ typedef struct WASMExecEnv { /* Used to terminate or suspend the interpreter bit 0: need terminate bit 1: need suspend - bit 2: need to go into breakpoint */ - uintptr_t suspend_flags; + bit 2: need to go into breakpoint + bit 3: return from pthread_exit */ + union { + uint32 flags; + uintptr_t __padding__; + } suspend_flags; + + /* thread return value */ + void *thread_ret_value; /* Must be provided by thread library */ void* (*thread_start_routine)(void *); diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index e8c1d4586..97bb0c024 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -115,7 +115,7 @@ wasm_runtime_env_init() } #endif -#if WASM_ENABLE_THREAD_MGR != 0 +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_THREAD_MGR != 0) if (!thread_manager_init()) { goto fail5; } @@ -136,7 +136,7 @@ wasm_runtime_env_init() fail6: #endif #endif -#if WASM_ENABLE_THREAD_MGR != 0 +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_THREAD_MGR != 0) thread_manager_destroy(); fail5: #endif @@ -204,7 +204,7 @@ wasm_runtime_destroy() wasm_shared_memory_destroy(); #endif -#if WASM_ENABLE_THREAD_MGR != 0 +#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_THREAD_MGR != 0) thread_manager_destroy(); #endif @@ -565,8 +565,9 @@ wasm_exec_env_set_aux_stack(WASMExecEnv *exec_env, } #endif #if WASM_ENABLE_AOT != 0 - /* TODO: implement set aux stack in AoT mode */ - (void)module_inst; + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_set_aux_stack(exec_env, start_offset, size); + } #endif return false; } @@ -583,8 +584,9 @@ wasm_exec_env_get_aux_stack(WASMExecEnv *exec_env, } #endif #if WASM_ENABLE_AOT != 0 - /* TODO: implement get aux stack in AoT mode */ - (void)module_inst; + if (module_inst->module_type == Wasm_Module_AoT) { + return aot_get_aux_stack(exec_env, start_offset, size); + } #endif return false; } @@ -1298,7 +1300,8 @@ wasm_runtime_init_wasi(WASMModuleInstanceCommon *module_inst, #endif #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT - && !((AOTModuleInstance*)module_inst)->memory_data.ptr) + && !((AOTModuleInstance*)module_inst)-> + global_table_data.memory_instances[0].memory_data.ptr) return true; #endif @@ -1479,17 +1482,18 @@ wasm_runtime_lookup_wasi_start_function(WASMModuleInstanceCommon *module_inst) #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { AOTModuleInstance *aot_inst = (AOTModuleInstance*)module_inst; - AOTModule *module = (AOTModule*)aot_inst->aot_module.ptr; - for (i = 0; i < module->export_func_count; i++) { - if (!strcmp(module->export_funcs[i].func_name, "_start")) { - AOTFuncType *func_type = module->export_funcs[i].func_type; + AOTFunctionInstance *export_funcs = (AOTFunctionInstance *) + aot_inst->export_funcs.ptr; + for (i = 0; i < aot_inst->export_func_count; i++) { + if (!strcmp(export_funcs[i].func_name, "_start")) { + AOTFuncType *func_type = export_funcs[i].u.func.func_type; if (func_type->param_count != 0 || func_type->result_count != 0) { LOG_ERROR("Lookup wasi _start function failed: " "invalid function type.\n"); return NULL; } - return (WASMFunctionInstanceCommon*)&module->export_funcs[i]; + return (WASMFunctionInstanceCommon*)&export_funcs[i]; } } return NULL; @@ -1663,7 +1667,7 @@ wasm_application_execute_main(WASMModuleInstanceCommon *module_inst, #endif #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) - func_type = ((AOTFunctionInstance*)func)->func_type; + func_type = ((AOTFunctionInstance*)func)->u.func.func_type; #endif if (!check_main_func_type(func_type)) { @@ -1817,10 +1821,11 @@ resolve_function(const WASMModuleInstanceCommon *module_inst, #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { AOTModuleInstance *aot_inst = (AOTModuleInstance*)module_inst; - AOTModule *module = (AOTModule*)aot_inst->aot_module.ptr; - for (i = 0; i < module->export_func_count; i++) { - if (!strcmp(module->export_funcs[i].func_name, function_name)) { - ret = (WASMFunctionInstance*)&module->export_funcs[i]; + AOTFunctionInstance *export_funcs = (AOTFunctionInstance *) + aot_inst->export_funcs.ptr; + for (i = 0; i < aot_inst->export_func_count; i++) { + if (!strcmp(export_funcs[i].func_name, function_name)) { + ret = &export_funcs[i]; break; } } @@ -1924,7 +1929,7 @@ wasm_application_execute_func(WASMModuleInstanceCommon *module_inst, #endif #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { - type = ((AOTFunctionInstance*)func)->func_type; + type = ((AOTFunctionInstance*)func)->u.func.func_type; argc1 = type->param_cell_num; cell_num = argc1 > type->ret_cell_num ? argc1 : type->ret_cell_num; @@ -2845,4 +2850,3 @@ wasm_runtime_call_indirect(WASMExecEnv *exec_env, #endif return false; } - diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 91532bfd6..57bb591fc 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -67,6 +67,14 @@ typedef struct WASMRegisteredModule { } WASMRegisteredModule; #endif +#if WASM_ENABLE_SHARED_MEMORY != 0 +typedef struct WASMMemoryInstanceCommon { + uint32 module_type; + uint8 memory_inst_data[1]; +} WASMMemoryInstanceCommon; + +#endif + typedef package_type_t PackageType; typedef wasm_section_t WASMSection, AOTSection; diff --git a/core/iwasm/common/wasm_shared_memory.c b/core/iwasm/common/wasm_shared_memory.c index 38baee9c9..be99e6fe3 100644 --- a/core/iwasm/common/wasm_shared_memory.c +++ b/core/iwasm/common/wasm_shared_memory.c @@ -88,16 +88,15 @@ shared_memory_dec_reference(WASMModuleCommon *module) return -1; } -#if WASM_ENABLE_INTERP != 0 -WASMMemoryInstance* +WASMMemoryInstanceCommon* shared_memory_get_memory_inst(WASMSharedMemNode *node) { - return node->u.wasm_memory; + return node->memory_inst; } WASMSharedMemNode* shared_memory_set_memory_inst(WASMModuleCommon *module, - WASMMemoryInstance *memory) + WASMMemoryInstanceCommon *memory) { WASMSharedMemNode *node; bh_list_status ret; @@ -106,7 +105,7 @@ shared_memory_set_memory_inst(WASMModuleCommon *module, return NULL; node->module = module; - node->u.wasm_memory = memory; + node->memory_inst = memory; node->ref_count = 1; if (os_mutex_init(&node->lock) != 0) { wasm_runtime_free(node); @@ -122,6 +121,4 @@ shared_memory_set_memory_inst(WASMModuleCommon *module, return node; } -#endif /* end of WASM_ENABLE_INTERP */ - #endif /* end of WASM_ENABLE_SHARED_MEMORY */ diff --git a/core/iwasm/common/wasm_shared_memory.h b/core/iwasm/common/wasm_shared_memory.h index aed6d10e1..5a78b5fee 100644 --- a/core/iwasm/common/wasm_shared_memory.h +++ b/core/iwasm/common/wasm_shared_memory.h @@ -25,28 +25,7 @@ typedef struct WASMSharedMemNode { /* The module reference */ WASMModuleCommon *module; /* The memory information */ - union { -#if WASM_ENABLE_INTERP - WASMMemoryInstance *wasm_memory; -#endif -#if WASM_ENABLE_AOT - struct { - /* memory space info */ - uint32 mem_cur_page_count; - uint32 mem_max_page_count; - uint32 memory_data_size; - AOTPointer memory_data; - AOTPointer memory_data_end; - - /* heap space info */ - int32 heap_base_offset; - uint32 heap_data_size; - AOTPointer heap_data; - AOTPointer heap_data_end; - AOTPointer heap_handle; - } aot_memory; -#endif - } u; + WASMMemoryInstanceCommon *memory_inst; /* reference count */ uint32 ref_count; @@ -67,12 +46,12 @@ shared_memory_inc_reference(WASMModuleCommon *module); int32 shared_memory_dec_reference(WASMModuleCommon *module); -WASMMemoryInstance* +WASMMemoryInstanceCommon* shared_memory_get_memory_inst(WASMSharedMemNode *node); WASMSharedMemNode* shared_memory_set_memory_inst(WASMModuleCommon *module, - WASMMemoryInstance *memory); + WASMMemoryInstanceCommon *memory); #ifdef __cplusplus diff --git a/core/iwasm/compilation/aot.c b/core/iwasm/compilation/aot.c index 025294b6f..35cf56c55 100644 --- a/core/iwasm/compilation/aot.c +++ b/core/iwasm/compilation/aot.c @@ -344,43 +344,12 @@ fail: return NULL; } -static AOTExportFunc * -aot_create_export_funcs(const WASMModule *module, - uint32 export_func_count) -{ - AOTExportFunc *export_funcs; - uint64 size; - uint32 i, j = 0; - - /* Allocate memory */ - size = sizeof(AOTExportFunc) * (uint64)export_func_count; - if (size >= UINT32_MAX - || !(export_funcs = wasm_runtime_malloc((uint32)size))) { - aot_set_last_error("allocate memory failed."); - return NULL; - } - - /* Create each export function */ - for (i = 0; i < module->export_count; i++) { - if (module->exports[i].kind == EXPORT_KIND_FUNC) { - export_funcs[j].func_name = module->exports[i].name; - export_funcs[j].func_index = module->exports[i].index; - export_funcs[j].func_type = - module->functions[module->exports[i].index - - module->import_function_count]->func_type; - /* Function pointer to be linked in JIT mode */ - export_funcs[j].func_ptr = NULL; - j++; - } - } - return export_funcs; -} - AOTCompData* aot_create_comp_data(WASMModule *module) { AOTCompData *comp_data; - uint32 import_global_data_size = 0, global_data_size = 0, i; + uint32 import_global_data_size = 0, global_data_size = 0, i, j; + uint64 size; /* Allocate memory */ if (!(comp_data = wasm_runtime_malloc(sizeof(AOTCompData)))) { @@ -390,22 +359,49 @@ aot_create_comp_data(WASMModule *module) memset(comp_data, 0, sizeof(AOTCompData)); - /* Set memory page count */ - if (module->import_memory_count) { - comp_data->num_bytes_per_page = - module->import_memories[0].u.memory.num_bytes_per_page; - comp_data->mem_init_page_count = - module->import_memories[0].u.memory.init_page_count; - comp_data->mem_max_page_count = - module->import_memories[0].u.memory.max_page_count; + comp_data->memory_count = module->import_memory_count + module->memory_count; + + /* TODO: create import memories */ + + /* Allocate memory for memory array, reserve one AOTMemory space at least */ + if (!comp_data->memory_count) + comp_data->memory_count = 1; + + size = (uint64)comp_data->memory_count * sizeof(AOTMemory); + if (size >= UINT32_MAX + || !(comp_data->memories = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("create memories array failed.\n"); + goto fail; } - else if (module->memory_count) { - comp_data->num_bytes_per_page = - module->memories[0].num_bytes_per_page; - comp_data->mem_init_page_count = - module->memories[0].init_page_count; - comp_data->mem_max_page_count = - module->memories[0].max_page_count; + memset(comp_data->memories, 0, size); + + /* Set memory page count */ + for (i = 0; i < module->import_memory_count + module->memory_count; i++) { + if (i < module->import_memory_count) { + comp_data->memories[i].memory_flags = + module->import_memories[i].u.memory.flags; + comp_data->memories[i].num_bytes_per_page = + module->import_memories[i].u.memory.num_bytes_per_page; + comp_data->memories[i].mem_init_page_count = + module->import_memories[i].u.memory.init_page_count; + comp_data->memories[i].mem_max_page_count = + module->import_memories[i].u.memory.max_page_count; + comp_data->memories[i].num_bytes_per_page = + module->import_memories[i].u.memory.num_bytes_per_page; + } + else { + j = i - module->import_memory_count; + comp_data->memories[i].memory_flags = + module->memories[j].flags; + comp_data->memories[i].num_bytes_per_page = + module->memories[j].num_bytes_per_page; + comp_data->memories[i].mem_init_page_count = + module->memories[j].init_page_count; + comp_data->memories[i].mem_max_page_count = + module->memories[j].max_page_count; + comp_data->memories[i].num_bytes_per_page = + module->memories[j].num_bytes_per_page; + } } /* Create memory data segments */ @@ -415,11 +411,39 @@ aot_create_comp_data(WASMModule *module) aot_create_mem_init_data_list(module))) goto fail; - /* Set table size */ - if (module->import_table_count) - comp_data->table_size = module->import_tables[0].u.table.init_size; - else if (module->table_count) - comp_data->table_size = module->tables[0].init_size; + /* TODO: create import tables */ + + /* Create tables */ + comp_data->table_count = module->import_table_count + module->table_count; + + if (comp_data->table_count > 0) { + size = sizeof(AOTTable) * (uint64)comp_data->table_count; + if (size >= UINT32_MAX + || !(comp_data->tables = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("create memories array failed.\n"); + goto fail; + } + memset(comp_data->tables, 0, size); + for (i = 0; i < comp_data->table_count; i++) { + if (i < module->import_table_count) { + comp_data->tables[i].elem_type = + module->import_tables[i].u.table.elem_type; + comp_data->tables[i].table_flags = + module->import_tables[i].u.table.flags; + comp_data->tables[i].table_init_size = + module->import_tables[i].u.table.init_size; + comp_data->tables[i].table_max_size = + module->import_tables[i].u.table.max_size; + } + else { + j = i - module->import_table_count; + comp_data->tables[i].elem_type = module->tables[i].elem_type; + comp_data->tables[i].table_flags = module->tables[i].flags; + comp_data->tables[i].table_init_size = module->tables[i].init_size; + comp_data->tables[i].table_max_size = module->tables[i].max_size; + } + } + } /* Create table data segments */ comp_data->table_init_data_count = module->table_seg_count; @@ -463,15 +487,11 @@ aot_create_comp_data(WASMModule *module) && !(comp_data->funcs = aot_create_funcs(module))) goto fail; - /* Create export functions */ - for (i = 0; i < module->export_count; i++) - if (module->exports[i].kind == EXPORT_KIND_FUNC) - comp_data->export_func_count++; - - if (comp_data->export_func_count - && !(comp_data->export_funcs = aot_create_export_funcs - (module, comp_data->export_func_count))) - goto fail; + /* Create llvm aux stack informations */ + comp_data->llvm_aux_stack_global_index = module->llvm_aux_stack_global_index; + comp_data->llvm_aux_data_end = module->llvm_aux_data_end; + comp_data->llvm_aux_stack_bottom = module->llvm_aux_stack_bottom; + comp_data->llvm_aux_stack_size = module->llvm_aux_stack_size; comp_data->start_func_index = module->start_function; comp_data->wasm_module = module; @@ -490,10 +510,22 @@ aot_destroy_comp_data(AOTCompData *comp_data) if (!comp_data) return; + if (comp_data->import_memories) + wasm_runtime_free(comp_data->import_memories); + + if (comp_data->memories) + wasm_runtime_free(comp_data->memories); + if (comp_data->mem_init_data_list) aot_destroy_mem_init_data_list(comp_data->mem_init_data_list, comp_data->mem_init_data_count); + if (comp_data->import_tables) + wasm_runtime_free(comp_data->import_tables); + + if (comp_data->tables) + wasm_runtime_free(comp_data->tables); + if (comp_data->table_init_data_list) aot_destroy_table_init_data_list(comp_data->table_init_data_list, comp_data->table_init_data_count); @@ -514,9 +546,6 @@ aot_destroy_comp_data(AOTCompData *comp_data) if (comp_data->funcs) aot_destroy_funcs(comp_data->funcs, comp_data->func_count); - if (comp_data->export_funcs) - wasm_runtime_free(comp_data->export_funcs); - wasm_runtime_free(comp_data); } diff --git a/core/iwasm/compilation/aot.h b/core/iwasm/compilation/aot.h index 7a533fa75..abf86d854 100644 --- a/core/iwasm/compilation/aot.h +++ b/core/iwasm/compilation/aot.h @@ -19,6 +19,30 @@ extern "C" { typedef InitializerExpression AOTInitExpr; typedef WASMType AOTFuncType; +typedef WASMExport AOTExport; + +/** + * Import memory + */ +typedef struct AOTImportMemory { + char *module_name; + char *memory_name; + uint32 memory_flags; + uint32 num_bytes_per_page; + uint32 mem_init_page_count; + uint32 mem_max_page_count; +} AOTImportMemory; + +/** + * Memory information + */ +typedef struct AOTMemory { + /* memory info */ + uint32 memory_flags; + uint32 num_bytes_per_page; + uint32 mem_init_page_count; + uint32 mem_max_page_count; +} AOTMemory; /** * A segment of memory init data @@ -38,10 +62,32 @@ typedef struct AOTMemInitData { uint8 bytes[1]; } AOTMemInitData; +/** + * Import table + */ +typedef struct AOTImportTable { + char *module_name; + char *table_name; + uint32 table_flags; + uint32 table_init_size; + uint32 table_max_size; +} AOTImportTable; + +/** + * Table + */ +typedef struct AOTTable { + uint32 elem_type; + uint32 table_flags; + uint32 table_init_size; + uint32 table_max_size; +} AOTTable; + /** * A segment of table init data */ typedef struct AOTTableInitData { + uint32 table_index; /* Start address of init data */ AOTInitExpr offset; /* Function index count */ @@ -110,47 +156,50 @@ typedef struct AOTFunc { uint8 *code; } AOTFunc; -/** - * Export function - */ -typedef struct AOTExportFunc { - char *func_name; - AOTFuncType *func_type; - /* function pointer linked */ - void *func_ptr; - uint32 func_index; -} AOTExportFunc; - typedef struct AOTCompData { - /* Memory and memory init data info */ - uint32 num_bytes_per_page; - uint32 mem_init_page_count; - uint32 mem_max_page_count; + /* Import memories */ + uint32 import_memory_count; + AOTImportMemory *import_memories; + + /* Memories */ + uint32 memory_count; + AOTMemory *memories; + + /* Memory init data info */ uint32 mem_init_data_count; AOTMemInitData **mem_init_data_list; - /* Table and table init data info */ - uint32 table_size; - AOTTableInitData **table_init_data_list; + /* Import tables */ + uint32 import_table_count; + AOTImportTable *import_tables; + + /* Tables */ + uint32 table_count; + AOTTable *tables; + + /* Table init data info */ uint32 table_init_data_count; + AOTTableInitData **table_init_data_list; - AOTImportGlobal *import_globals; + /* Import globals */ uint32 import_global_count; + AOTImportGlobal *import_globals; - AOTGlobal *globals; + /* Globals */ uint32 global_count; + AOTGlobal *globals; - AOTFuncType **func_types; + /* Function types */ uint32 func_type_count; + AOTFuncType **func_types; - AOTImportFunc *import_funcs; + /* Import functions */ uint32 import_func_count; + AOTImportFunc *import_funcs; - AOTFunc **funcs; + /* Functions */ uint32 func_count; - - AOTExportFunc *export_funcs; - uint32 export_func_count; + AOTFunc **funcs; uint32 start_func_index; uint32 addr_data_size; diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index 2095388e0..8b9c39bf1 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -152,12 +152,30 @@ get_mem_init_data_list_size(AOTMemInitData **mem_init_data_list, return size; } +static uint32 +get_import_memory_size(AOTCompData *comp_data) +{ + /* currently we only emit import_memory_count = 0 */ + return sizeof(uint32); +} + +static uint32 +get_memory_size(AOTCompData *comp_data) +{ + /* memory_count + count * (memory_flags + num_bytes_per_page + + init_page_count + max_page_count) */ + return (uint32)(sizeof(uint32) + + comp_data->memory_count * sizeof(uint32) * 4); +} + static uint32 get_mem_info_size(AOTCompData *comp_data) { - /* num bytes per page + init page count + max page count - + init data count + init data list */ - return (uint32)sizeof(uint32) * 4 + /* import_memory_size + memory_size + + init_data_count + init_data_list */ + return get_import_memory_size(comp_data) + + get_memory_size(comp_data) + + (uint32)sizeof(uint32) + get_mem_init_data_list_size(comp_data->mem_init_data_list, comp_data->mem_init_data_count); } @@ -165,9 +183,10 @@ get_mem_info_size(AOTCompData *comp_data) static uint32 get_table_init_data_size(AOTTableInitData *table_init_data) { - /* init expr type (4 bytes) + init expr value (8 bytes) + /* table_index + init expr type (4 bytes) + init expr value (8 bytes) + func index count (4 bytes) + func indexes */ - return (uint32)(sizeof(uint32) + sizeof(uint64) + sizeof(uint32) + return (uint32)(sizeof(uint32) + sizeof(uint32) + + sizeof(uint64) + sizeof(uint32) + sizeof(uint32) * table_init_data->func_index_count); } @@ -185,11 +204,30 @@ get_table_init_data_list_size(AOTTableInitData **table_init_data_list, return size; } +static uint32 +get_import_table_size(AOTCompData *comp_data) +{ + /* currently we only emit import_table_count = 0 */ + return sizeof(uint32); +} + +static uint32 +get_table_size(AOTCompData *comp_data) +{ + /* table_count + table_count * (elem_type + table_flags + * + init_size + max_size) */ + return (uint32)(sizeof(uint32) + + comp_data->table_count * sizeof(uint32) * 4); +} + static uint32 get_table_info_size(AOTCompData *comp_data) { - /* table size + init data count + init data list */ - return (uint32)sizeof(uint32) * 2 + /* import_table size + table_size + + init data count + init data list */ + return get_import_table_size(comp_data) + + get_table_size(comp_data) + + (uint32)sizeof(uint32) + get_table_init_data_list_size(comp_data->table_init_data_list, comp_data->table_init_data_count); } @@ -412,23 +450,22 @@ get_func_section_size(AOTCompData *comp_data, AOTObjectData *obj_data) } static uint32 -get_export_func_size(AOTExportFunc *export_func) +get_export_size(AOTExport *export) { - /* export func index + export func name */ - return (uint32)sizeof(uint32) - + get_string_size(export_func->func_name); + /* export index + export kind + 1 byte padding + export name */ + return (uint32)sizeof(uint32) + sizeof(uint8) + 1 + + get_string_size(export->name); } static uint32 -get_export_funcs_size(AOTExportFunc *export_funcs, - uint32 export_func_count) +get_exports_size(AOTExport *exports, uint32 export_count) { - AOTExportFunc *export_func = export_funcs; + AOTExport *export = exports; uint32 size = 0, i; - for (i = 0; i < export_func_count; i++, export_func++) { + for (i = 0; i < export_count; i++, export++) { size = align_uint(size, 4); - size += get_export_func_size(export_func); + size += get_export_size(export); } return size; } @@ -436,10 +473,10 @@ get_export_funcs_size(AOTExportFunc *export_funcs, static uint32 get_export_section_size(AOTCompData *comp_data) { - /* export func count + export funcs */ + /* export count + exports */ return (uint32)sizeof(uint32) - + get_export_funcs_size(comp_data->export_funcs, - comp_data->export_func_count); + + get_exports_size(comp_data->wasm_module->exports, + comp_data->wasm_module->export_count); } static uint32 @@ -887,11 +924,24 @@ aot_emit_mem_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, *p_offset = offset = align_uint(offset, 4); - EMIT_U32(comp_data->num_bytes_per_page); - EMIT_U32(comp_data->mem_init_page_count); - EMIT_U32(comp_data->mem_max_page_count); - EMIT_U32(comp_data->mem_init_data_count); + /* Emit import memory count, only emit 0 currently. + TODO: emit the actual import memory count and + the full import memory info. */ + EMIT_U32(0); + /* Emit memory count */ + EMIT_U32(comp_data->memory_count); + /* Emit memory items */ + for (i = 0; i < comp_data->memory_count; i++) { + EMIT_U32(comp_data->memories[i].memory_flags); + EMIT_U32(comp_data->memories[i].num_bytes_per_page); + EMIT_U32(comp_data->memories[i].mem_init_page_count); + EMIT_U32(comp_data->memories[i].mem_max_page_count); + } + + /* Emit mem init data count */ + EMIT_U32(comp_data->mem_init_data_count); + /* Emit mem init data items */ for (i = 0; i < comp_data->mem_init_data_count; i++) { offset = align_uint(offset, 4); #if WASM_ENABLE_BULK_MEMORY != 0 @@ -931,11 +981,27 @@ aot_emit_table_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, *p_offset = offset = align_uint(offset, 4); - EMIT_U32(comp_data->table_size); - EMIT_U32(comp_data->table_init_data_count); + /* Emit import table count, only emit 0 currently. + TODO: emit the actual import table count and + the full import table info. */ + EMIT_U32(0); + /* Emit table count */ + EMIT_U32(comp_data->table_count); + /* Emit table items */ + for (i = 0; i < comp_data->table_count; i++) { + EMIT_U32(comp_data->tables[i].elem_type); + EMIT_U32(comp_data->tables[i].table_flags); + EMIT_U32(comp_data->tables[i].table_init_size); + EMIT_U32(comp_data->tables[i].table_max_size); + } + + /* Emit table init data count */ + EMIT_U32(comp_data->table_init_data_count); + /* Emit table init data items */ for (i = 0; i < comp_data->table_init_data_count; i++) { offset = align_uint(offset, 4); + EMIT_U32(init_datas[i]->table_index); EMIT_U32(init_datas[i]->offset.init_expr_type); EMIT_U64(init_datas[i]->offset.u.i64); EMIT_U32(init_datas[i]->func_index_count); @@ -1211,19 +1277,22 @@ aot_emit_export_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, AOTCompData *comp_data, AOTObjectData *obj_data) { uint32 section_size = get_export_section_size(comp_data); - AOTExportFunc *func = comp_data->export_funcs;; - uint32 i, offset = *p_offset, export_func_count = comp_data->export_func_count; + AOTExport *export = comp_data->wasm_module->exports; + uint32 export_count = comp_data->wasm_module->export_count; + uint32 i, offset = *p_offset; *p_offset = offset = align_uint(offset, 4); EMIT_U32(AOT_SECTION_TYPE_EXPORT); EMIT_U32(section_size); - EMIT_U32(export_func_count); + EMIT_U32(export_count); - for (i = 0; i < export_func_count; i++, func++) { + for (i = 0; i < export_count; i++, export++) { offset = align_uint(offset, 4); - EMIT_U32(func->func_index); - EMIT_STR(func->func_name); + EMIT_U32(export->index); + EMIT_U8(export->kind); + EMIT_U8(0); + EMIT_STR(export->name); } if (offset - *p_offset != section_size + sizeof(uint32) * 2) { diff --git a/core/iwasm/compilation/aot_emit_control.c b/core/iwasm/compilation/aot_emit_control.c index b6d38bf20..a2799a7e5 100644 --- a/core/iwasm/compilation/aot_emit_control.c +++ b/core/iwasm/compilation/aot_emit_control.c @@ -102,6 +102,14 @@ format_block_name(char *name, uint32 name_size, &value, &block_curr, 1); \ } while (0) +#define BUILD_ICMP(op, left, right, res, name) do { \ + if (!(res = LLVMBuildICmp(comp_ctx->builder, op, \ + left, right, name))) { \ + aot_set_last_error("llvm build icmp failed."); \ + goto fail; \ + } \ + } while (0) + #define ADD_TO_PARAM_PHIS(block, value, idx) do { \ LLVMBasicBlockRef block_curr = CURR_BLOCK(); \ LLVMAddIncoming(block->param_phis[idx], \ @@ -614,6 +622,99 @@ fail: return false; } +#if WASM_ENABLE_THREAD_MGR != 0 +bool +check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef terminate_addr, terminate_flags, flag, offset, res; + LLVMBasicBlockRef terminate_check_block, non_terminate_block; + AOTFuncType *aot_func_type = func_ctx->aot_func->func_type; + LLVMBasicBlockRef terminate_block; + + /* Offset of suspend_flags */ + offset = I32_CONST(5); + CHECK_LLVM_CONST(offset); + + if (!(terminate_addr = + LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->exec_env, + &offset, 1, "terminate_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!(terminate_addr = + LLVMBuildBitCast(comp_ctx->builder, + terminate_addr, + INT32_PTR_TYPE, "terminate_addr_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!(terminate_flags = + LLVMBuildLoad(comp_ctx->builder, + terminate_addr, "terminate_flags"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + /* Set terminate_flags memory accecc to volatile, so that the value + will always be loaded from memory rather than register */ + LLVMSetVolatile(terminate_flags, true); + + CREATE_BLOCK(terminate_check_block, "terminate_check"); + MOVE_BLOCK_AFTER_CURR(terminate_check_block); + + CREATE_BLOCK(non_terminate_block, "non_terminate"); + MOVE_BLOCK_AFTER_CURR(non_terminate_block); + + BUILD_ICMP(LLVMIntSGT, terminate_flags, I32_ZERO, res, "need_terminate"); + BUILD_COND_BR(res, terminate_check_block, non_terminate_block); + + /* Move builder to terminate check block */ + SET_BUILDER_POS(terminate_check_block); + + CREATE_BLOCK(terminate_block, "terminate"); + MOVE_BLOCK_AFTER_CURR(terminate_block); + + if (!(flag = + LLVMBuildAnd(comp_ctx->builder, terminate_flags, + I32_ONE, "termination_flag"))) { + aot_set_last_error("llvm build AND failed"); + return false; + } + + BUILD_ICMP(LLVMIntSGT, flag, I32_ZERO, res, "need_terminate"); + BUILD_COND_BR(res, terminate_block, non_terminate_block); + + /* Move builder to terminate block */ + SET_BUILDER_POS(terminate_block); + if (aot_func_type->result_count) { + switch (aot_func_type->types[aot_func_type->param_count]) { + case VALUE_TYPE_I32: + LLVMBuildRet(comp_ctx->builder, I32_ZERO); + break; + case VALUE_TYPE_I64: + LLVMBuildRet(comp_ctx->builder, I64_ZERO); + break; + case VALUE_TYPE_F32: + LLVMBuildRet(comp_ctx->builder, F32_ZERO); + break; + case VALUE_TYPE_F64: + LLVMBuildRet(comp_ctx->builder, F64_ZERO); + break; + } + } + else { + LLVMBuildRetVoid(comp_ctx->builder); + } + + /* Move builder to terminate block */ + SET_BUILDER_POS(non_terminate_block); + return true; + +fail: + return false; +} +#endif /* End of WASM_ENABLE_THREAD_MGR */ + bool aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 br_depth, uint8 **p_frame_ip) @@ -624,6 +725,14 @@ aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, char name[32]; uint32 i, param_index, result_index; +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx)) + return false; + } +#endif + if (!(block_dst = get_target_block(func_ctx, br_depth))) { return false; } @@ -680,6 +789,14 @@ aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 i, param_index, result_index; uint64 size; +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx)) + return false; + } +#endif + POP_COND(value_cmp); if (!LLVMIsConstant(value_cmp)) { /* Compare value is not constant, create condition br IR */ @@ -798,6 +915,14 @@ aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint64 size; char name[32]; +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx)) + return false; + } +#endif + POP_I32(value_cmp); if (!LLVMIsConstant(value_cmp)) { /* Compare value is not constant, create switch IR */ diff --git a/core/iwasm/compilation/aot_emit_control.h b/core/iwasm/compilation/aot_emit_control.h index fa73d29fb..c0c029056 100644 --- a/core/iwasm/compilation/aot_emit_control.h +++ b/core/iwasm/compilation/aot_emit_control.h @@ -53,6 +53,11 @@ aot_handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint8 **p_frame_ip); +#if WASM_ENABLE_THREAD_MGR != 0 +bool +check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); +#endif + #ifdef __cplusplus } /* end of extern "C" */ #endif diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index 1897cfd80..af7362f50 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -326,6 +326,14 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, bool ret = false; char buf[32]; +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx)) + return false; + } +#endif + /* Check function index */ if (func_idx >= import_func_count + func_count) { aot_set_last_error("Function index out of range."); @@ -514,6 +522,14 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, char buf[32], *func_name = "aot_call_indirect"; uint32 i, cell_num = 0, ret_cell_num, argv_cell_num; +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx)) + return false; + } +#endif + /* prepare function type of aot_call_indirect */ func_param_types[0] = comp_ctx->exec_env_type; /* exec_env */ func_param_types[1] = INT8_TYPE; /* check_func_type */ diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index b6d497582..ff3922c8b 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -42,16 +42,16 @@ get_memory_check_bound(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef mem_check_bound = NULL; switch (bytes) { case 1: - mem_check_bound = func_ctx->mem_bound_check_1byte; + mem_check_bound = func_ctx->mem_info[0].mem_bound_check_1byte; break; case 2: - mem_check_bound = func_ctx->mem_bound_check_2bytes; + mem_check_bound = func_ctx->mem_info[0].mem_bound_check_2bytes; break; case 4: - mem_check_bound = func_ctx->mem_bound_check_4bytes; + mem_check_bound = func_ctx->mem_info[0].mem_bound_check_4bytes; break; case 8: - mem_check_bound = func_ctx->mem_bound_check_8bytes; + mem_check_bound = func_ctx->mem_info[0].mem_bound_check_8bytes; break; default: bh_assert(0); @@ -80,17 +80,26 @@ check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMBasicBlockRef check_succ; AOTValue *aot_value; +#if WASM_ENABLE_SHARED_MEMORY != 0 + bool is_shared_memory = + comp_ctx->comp_data->memories[0].memory_flags & 0x02; +#endif CHECK_LLVM_CONST(offset_const); /* Get memory base address and memory data size */ +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (func_ctx->mem_space_unchanged || is_shared_memory) { +#else if (func_ctx->mem_space_unchanged) { - mem_base_addr = func_ctx->mem_base_addr; +#endif + mem_base_addr = func_ctx->mem_info[0].mem_base_addr; } else { - if (!(mem_base_addr = LLVMBuildLoad(comp_ctx->builder, - func_ctx->mem_base_addr, - "mem_base"))) { + if (!(mem_base_addr = + LLVMBuildLoad(comp_ctx->builder, + func_ctx->mem_info[0].mem_base_addr, + "mem_base"))) { aot_set_last_error("llvm build load failed."); goto fail; } @@ -103,8 +112,10 @@ check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* return addres directly if constant offset and inside memory space */ if (LLVMIsConstant(addr)) { int64 mem_offset = (int64)LLVMConstIntGetSExtValue(addr) + (int64)offset; - uint32 num_bytes_per_page = comp_ctx->comp_data->num_bytes_per_page; - uint32 init_page_count = comp_ctx->comp_data->mem_init_page_count; + uint32 num_bytes_per_page = + comp_ctx->comp_data->memories[0].num_bytes_per_page; + uint32 init_page_count = + comp_ctx->comp_data->memories[0].mem_init_page_count; int64 mem_data_size = num_bytes_per_page * init_page_count; if (mem_data_size > 0 && mem_offset >= 0 @@ -141,8 +152,9 @@ check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - BUILD_ICMP(LLVMIntSGT, func_ctx->mem_bound_check_heap_base, offset1, - cmp1, "cmp1"); + BUILD_ICMP(LLVMIntSGT, + func_ctx->mem_info[0].mem_bound_check_heap_base, + offset1, cmp1, "cmp1"); BUILD_ICMP(LLVMIntSGT, offset1, mem_check_bound, cmp2, "cmp2"); BUILD_OP(Or, cmp1, cmp2, cmp, "cmp"); @@ -448,32 +460,19 @@ fail: static LLVMValueRef get_memory_size(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { - uint32 offset = offsetof(AOTModuleInstance, mem_cur_page_count); - LLVMValueRef mem_size_offset, mem_size_ptr, mem_size; + LLVMValueRef mem_size; - /* mem_size_offset = aot_inst + offset */ - mem_size_offset = I32_CONST(offset); - CHECK_LLVM_CONST(mem_size_offset); - if (!(mem_size_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder, - func_ctx->aot_inst, - &mem_size_offset, 1, - "mem_size_ptr_tmp"))) { - aot_set_last_error("llvm build inbounds gep failed."); - return NULL; + if (func_ctx->mem_space_unchanged) { + mem_size = func_ctx->mem_info[0].mem_cur_page_count_addr; } - - /* cast to int32* */ - if (!(mem_size_ptr = LLVMBuildBitCast(comp_ctx->builder, mem_size_ptr, - INT32_PTR_TYPE, "mem_size_ptr"))) { - aot_set_last_error("llvm build bitcast failed."); - return NULL; - } - - /* load memory size, or current page count */ - if (!(mem_size = LLVMBuildLoad(comp_ctx->builder, - mem_size_ptr, "mem_size"))) { - aot_set_last_error("llvm build load failed."); - return NULL; + else { + if (!(mem_size = + LLVMBuildLoad(comp_ctx->builder, + func_ctx->mem_info[0].mem_cur_page_count_addr, + "mem_size"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } } return mem_size; @@ -614,17 +613,24 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef mem_base_addr; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMBasicBlockRef check_succ; - uint32 off = offsetof(AOTModuleInstance, memory_data_size); - LLVMValueRef mem_size_offset, mem_size_ptr, mem_size; + LLVMValueRef mem_size; /* Get memory base address and memory data size */ +#if WASM_ENABLE_SHARED_MEMORY != 0 + bool is_shared_memory = + comp_ctx->comp_data->memories[0].memory_flags & 0x02; + + if (func_ctx->mem_space_unchanged || is_shared_memory) { +#else if (func_ctx->mem_space_unchanged) { - mem_base_addr = func_ctx->mem_base_addr; +#endif + mem_base_addr = func_ctx->mem_info[0].mem_base_addr; } else { - if (!(mem_base_addr = LLVMBuildLoad(comp_ctx->builder, - func_ctx->mem_base_addr, - "mem_base"))) { + if (!(mem_base_addr = + LLVMBuildLoad(comp_ctx->builder, + func_ctx->mem_info[0].mem_base_addr, + "mem_base"))) { aot_set_last_error("llvm build load failed."); goto fail; } @@ -634,8 +640,10 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (LLVMIsConstant(offset) && LLVMIsConstant(bytes)) { uint64 mem_offset = (uint64)LLVMConstIntGetZExtValue(offset); uint64 mem_len = (uint64)LLVMConstIntGetZExtValue(bytes); - uint32 num_bytes_per_page = comp_ctx->comp_data->num_bytes_per_page; - uint32 init_page_count = comp_ctx->comp_data->mem_init_page_count; + uint32 num_bytes_per_page = + comp_ctx->comp_data->memories[0].num_bytes_per_page; + uint32 init_page_count = + comp_ctx->comp_data->memories[0].mem_init_page_count; uint32 mem_data_size = num_bytes_per_page * init_page_count; if (mem_data_size > 0 && mem_offset + mem_len <= mem_data_size) { @@ -652,27 +660,8 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* mem_size_offset = aot_inst + off */ - mem_size_offset = I32_CONST(off); - if (!(mem_size_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder, - func_ctx->aot_inst, - &mem_size_offset, 1, - "mem_size_ptr_tmp"))) { - aot_set_last_error("llvm build inbounds gep failed."); - return NULL; - } - - /* cast to int32* */ - if (!(mem_size_ptr = LLVMBuildBitCast(comp_ctx->builder, mem_size_ptr, - INT32_PTR_TYPE, "mem_size_ptr"))) { - aot_set_last_error("llvm build bitcast failed."); - return NULL; - } - - /* load memory size */ - if (!(mem_size = LLVMBuildLoad(comp_ctx->builder, - mem_size_ptr, "mem_size"))) { - aot_set_last_error("llvm build load failed."); - return NULL; + if (!(mem_size = get_memory_size(comp_ctx, func_ctx))) { + goto fail; } ADD_BASIC_BLOCK(check_succ, "check_succ"); diff --git a/core/iwasm/compilation/aot_emit_variable.c b/core/iwasm/compilation/aot_emit_variable.c index 4f5e63f18..f9e18a5fd 100644 --- a/core/iwasm/compilation/aot_emit_variable.c +++ b/core/iwasm/compilation/aot_emit_variable.c @@ -111,8 +111,9 @@ compile_global(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { AOTCompData *comp_data = comp_ctx->comp_data; uint32 import_global_count = comp_data->import_global_count; - uint32 global_base_offset = offsetof(AOTModuleInstance, - global_table_data.bytes); + uint32 global_base_offset = + offsetof(AOTModuleInstance, global_table_data.bytes) + + sizeof(AOTMemoryInstance) * comp_ctx->comp_data->memory_count; uint32 global_offset; uint8 global_type; LLVMValueRef offset, global_ptr, global; diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 813513eb7..753f67e27 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -186,142 +186,279 @@ static bool create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMTypeRef int8_ptr_type, uint32 func_index) { - LLVMValueRef offset; + LLVMValueRef offset, mem_info_base; + uint32 memory_count; WASMModule *module = comp_ctx->comp_data->wasm_module; WASMFunction *func = module->functions[func_index]; bool mem_space_unchanged = (!func->has_op_memory_grow && !func->has_op_func_call) || (!module->possible_memory_grow); +#if WASM_ENABLE_SHARED_MEMORY != 0 + bool is_shared_memory; +#endif func_ctx->mem_space_unchanged = mem_space_unchanged; - /* Load memory base address */ - offset = I32_CONST(offsetof(AOTModuleInstance, memory_data.ptr)); - if (!(func_ctx->mem_base_addr = - LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->aot_inst, - &offset, 1, "mem_base_addr_offset"))) { - aot_set_last_error("llvm build in bounds gep failed"); + memory_count = module->memory_count + module->import_memory_count; + /* If the module dosen't have memory, reserve + one mem_info space with empty content */ + if (memory_count == 0) + memory_count = 1; + + if (!(func_ctx->mem_info = + wasm_runtime_malloc(sizeof(AOTMemInfo) * memory_count))) { return false; } - if (!(func_ctx->mem_base_addr = - LLVMBuildBitCast(comp_ctx->builder, func_ctx->mem_base_addr, + memset(func_ctx->mem_info, 0, sizeof(AOTMemInfo)); + + /* Currently we only create memory info for memory 0 */ + /* Load memory base address */ +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared_memory = comp_ctx->comp_data->memories[0].memory_flags & 0x02 + ? true : false; + if (is_shared_memory) { + LLVMValueRef shared_mem_addr; + offset = I32_CONST(offsetof(AOTModuleInstance, memories)); + if (!offset) { + aot_set_last_error("create llvm const failed."); + return false; + } + + /* aot_inst->memories */ + if (!(shared_mem_addr = + LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->aot_inst, + &offset, 1, "shared_mem_addr_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!(shared_mem_addr = + LLVMBuildBitCast(comp_ctx->builder, + shared_mem_addr, int8_ptr_type, + "shared_mem_addr_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + /* aot_inst->memories[0] */ + if (!(shared_mem_addr = + LLVMBuildLoad(comp_ctx->builder, + shared_mem_addr, "shared_mem_addr"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + if (!(shared_mem_addr = + LLVMBuildBitCast(comp_ctx->builder, + shared_mem_addr, int8_ptr_type, + "shared_mem_addr_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + if (!(shared_mem_addr = + LLVMBuildLoad(comp_ctx->builder, + shared_mem_addr, "shared_mem_addr"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + offset = I32_CONST(offsetof(AOTMemoryInstance, memory_data.ptr)); + if (!(func_ctx->mem_info[0].mem_base_addr = + LLVMBuildInBoundsGEP(comp_ctx->builder, shared_mem_addr, + &offset, 1, "mem_base_addr_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + offset = I32_CONST(offsetof(AOTMemoryInstance, mem_cur_page_count)); + if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = + LLVMBuildInBoundsGEP(comp_ctx->builder, shared_mem_addr, + &offset, 1, "mem_cur_page_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + } + else +#endif + { + offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data) + + offsetof(AOTMemoryInstance, memory_data.ptr)); + if (!(func_ctx->mem_info[0].mem_base_addr = + LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->aot_inst, + &offset, 1, "mem_base_addr_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data) + + offsetof(AOTMemoryInstance, mem_cur_page_count)); + if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = + LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->aot_inst, + &offset, 1, "mem_cur_page_offset"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + } + /* Store mem info base address before cast */ + mem_info_base = func_ctx->mem_info[0].mem_base_addr; + + if (!(func_ctx->mem_info[0].mem_base_addr = + LLVMBuildBitCast(comp_ctx->builder, + func_ctx->mem_info[0].mem_base_addr, int8_ptr_type, "mem_base_addr_ptr"))) { aot_set_last_error("llvm build bit cast failed"); return false; } + if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = + LLVMBuildBitCast(comp_ctx->builder, + func_ctx->mem_info[0].mem_cur_page_count_addr, + INT32_PTR_TYPE, "mem_cur_page_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } if (mem_space_unchanged) { - if (!(func_ctx->mem_base_addr = - LLVMBuildLoad(comp_ctx->builder, func_ctx->mem_base_addr, + if (!(func_ctx->mem_info[0].mem_base_addr = + LLVMBuildLoad(comp_ctx->builder, + func_ctx->mem_info[0].mem_base_addr, + "mem_base_addr"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = + LLVMBuildLoad(comp_ctx->builder, + func_ctx->mem_info[0].mem_cur_page_count_addr, + "mem_cur_page_count_addr"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + } +#if WASM_ENABLE_SHARED_MEMORY != 0 + else if (is_shared_memory) { + /* The base address for shared memory will never changed, + we can load the value here */ + if (!(func_ctx->mem_info[0].mem_base_addr = + LLVMBuildLoad(comp_ctx->builder, + func_ctx->mem_info[0].mem_base_addr, "mem_base_addr"))) { aot_set_last_error("llvm build load failed"); return false; } } +#endif /* Load memory bound check constants */ - offset = I32_CONST(offsetof(AOTModuleInstance, mem_bound_check_1byte)); - if (!(func_ctx->mem_bound_check_1byte = - LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->aot_inst, + offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_1byte) + - offsetof(AOTMemoryInstance, memory_data.ptr)); + if (!(func_ctx->mem_info[0].mem_bound_check_1byte = + LLVMBuildInBoundsGEP(comp_ctx->builder, mem_info_base, &offset, 1, "bound_check_1byte_offset"))) { aot_set_last_error("llvm build in bounds gep failed"); return false; } - if (!(func_ctx->mem_bound_check_1byte = - LLVMBuildBitCast(comp_ctx->builder, func_ctx->mem_bound_check_1byte, + if (!(func_ctx->mem_info[0].mem_bound_check_1byte = + LLVMBuildBitCast(comp_ctx->builder, + func_ctx->mem_info[0].mem_bound_check_1byte, INT64_PTR_TYPE, "bound_check_1byte_ptr"))) { aot_set_last_error("llvm build bit cast failed"); return false; } if (mem_space_unchanged) { - if (!(func_ctx->mem_bound_check_1byte = - LLVMBuildLoad(comp_ctx->builder, func_ctx->mem_bound_check_1byte, - "bound_check_1byte"))) { + if (!(func_ctx->mem_info[0].mem_bound_check_1byte = + LLVMBuildLoad(comp_ctx->builder, + func_ctx->mem_info[0].mem_bound_check_1byte, + "bound_check_1byte"))) { aot_set_last_error("llvm build load failed"); return false; } } - offset = I32_CONST(offsetof(AOTModuleInstance, mem_bound_check_2bytes)); - if (!(func_ctx->mem_bound_check_2bytes = - LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->aot_inst, + offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_2bytes) + - offsetof(AOTMemoryInstance, memory_data.ptr)); + if (!(func_ctx->mem_info[0].mem_bound_check_2bytes = + LLVMBuildInBoundsGEP(comp_ctx->builder, mem_info_base, &offset, 1, "bound_check_2bytes_offset"))) { aot_set_last_error("llvm build in bounds gep failed"); return false; } - if (!(func_ctx->mem_bound_check_2bytes = - LLVMBuildBitCast(comp_ctx->builder, func_ctx->mem_bound_check_2bytes, + if (!(func_ctx->mem_info[0].mem_bound_check_2bytes = + LLVMBuildBitCast(comp_ctx->builder, + func_ctx->mem_info[0].mem_bound_check_2bytes, INT64_PTR_TYPE, "bound_check_2bytes_ptr"))) { aot_set_last_error("llvm build bit cast failed"); return false; } if (mem_space_unchanged) { - if (!(func_ctx->mem_bound_check_2bytes = - LLVMBuildLoad(comp_ctx->builder, func_ctx->mem_bound_check_2bytes, - "bound_check_2bytes"))) { + if (!(func_ctx->mem_info[0].mem_bound_check_2bytes = + LLVMBuildLoad(comp_ctx->builder, + func_ctx->mem_info[0].mem_bound_check_2bytes, + "bound_check_2bytes"))) { aot_set_last_error("llvm build load failed"); return false; } } - offset = I32_CONST(offsetof(AOTModuleInstance, mem_bound_check_4bytes)); - if (!(func_ctx->mem_bound_check_4bytes = - LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->aot_inst, + offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_4bytes) + - offsetof(AOTMemoryInstance, memory_data.ptr)); + if (!(func_ctx->mem_info[0].mem_bound_check_4bytes = + LLVMBuildInBoundsGEP(comp_ctx->builder, mem_info_base, &offset, 1, "bound_check_4bytes_offset"))) { aot_set_last_error("llvm build in bounds gep failed"); return false; } - if (!(func_ctx->mem_bound_check_4bytes = - LLVMBuildBitCast(comp_ctx->builder, func_ctx->mem_bound_check_4bytes, + if (!(func_ctx->mem_info[0].mem_bound_check_4bytes = + LLVMBuildBitCast(comp_ctx->builder, + func_ctx->mem_info[0].mem_bound_check_4bytes, INT64_PTR_TYPE, "bound_check_4bytes_ptr"))) { aot_set_last_error("llvm build bit cast failed"); return false; } if (mem_space_unchanged) { - if (!(func_ctx->mem_bound_check_4bytes = - LLVMBuildLoad(comp_ctx->builder, func_ctx->mem_bound_check_4bytes, - "bound_check_4bytes"))) { + if (!(func_ctx->mem_info[0].mem_bound_check_4bytes = + LLVMBuildLoad(comp_ctx->builder, + func_ctx->mem_info[0].mem_bound_check_4bytes, + "bound_check_4bytes"))) { aot_set_last_error("llvm build load failed"); return false; } } - offset = I32_CONST(offsetof(AOTModuleInstance, mem_bound_check_8bytes)); - if (!(func_ctx->mem_bound_check_8bytes = - LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->aot_inst, + offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_8bytes) + - offsetof(AOTMemoryInstance, memory_data.ptr)); + if (!(func_ctx->mem_info[0].mem_bound_check_8bytes = + LLVMBuildInBoundsGEP(comp_ctx->builder, mem_info_base, &offset, 1, "bound_check_8bytes_offset"))) { aot_set_last_error("llvm build in bounds gep failed"); return false; } - if (!(func_ctx->mem_bound_check_8bytes = - LLVMBuildBitCast(comp_ctx->builder, func_ctx->mem_bound_check_8bytes, + if (!(func_ctx->mem_info[0].mem_bound_check_8bytes = + LLVMBuildBitCast(comp_ctx->builder, + func_ctx->mem_info[0].mem_bound_check_8bytes, INT64_PTR_TYPE, "bound_check_8bytes_ptr"))) { aot_set_last_error("llvm build bit cast failed"); return false; } if (mem_space_unchanged) { - if (!(func_ctx->mem_bound_check_8bytes = - LLVMBuildLoad(comp_ctx->builder, func_ctx->mem_bound_check_8bytes, - "bound_check_8bytes"))) { + if (!(func_ctx->mem_info[0].mem_bound_check_8bytes = + LLVMBuildLoad(comp_ctx->builder, + func_ctx->mem_info[0].mem_bound_check_8bytes, + "bound_check_8bytes"))) { aot_set_last_error("llvm build load failed"); return false; } } /* Load bound_check_heap_base */ - offset = I32_CONST(offsetof(AOTModuleInstance, mem_bound_check_heap_base)); - if (!(func_ctx->mem_bound_check_heap_base = - LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->aot_inst, + offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_heap_base) + - offsetof(AOTMemoryInstance, memory_data.ptr)); + if (!(func_ctx->mem_info[0].mem_bound_check_heap_base = + LLVMBuildInBoundsGEP(comp_ctx->builder, mem_info_base, &offset, 1, "bound_check_heap_base_offset"))) { aot_set_last_error("llvm build in bounds gep failed"); return false; } - if (!(func_ctx->mem_bound_check_heap_base = - LLVMBuildBitCast(comp_ctx->builder, func_ctx->mem_bound_check_heap_base, + if (!(func_ctx->mem_info[0].mem_bound_check_heap_base = + LLVMBuildBitCast(comp_ctx->builder, + func_ctx->mem_info[0].mem_bound_check_heap_base, INT64_PTR_TYPE, "bound_check_heap_base_tmp"))) { aot_set_last_error("llvm build bit cast failed"); return false; } - if (!(func_ctx->mem_bound_check_heap_base = - LLVMBuildLoad(comp_ctx->builder, func_ctx->mem_bound_check_heap_base, + if (!(func_ctx->mem_info[0].mem_bound_check_heap_base = + LLVMBuildLoad(comp_ctx->builder, + func_ctx->mem_info[0].mem_bound_check_heap_base, "bound_check_heap_base"))) { aot_set_last_error("llvm build load failed"); return false; @@ -616,6 +753,8 @@ aot_create_func_context(AOTCompData *comp_data, AOTCompContext *comp_ctx, return func_ctx; fail: + if (func_ctx->mem_info) + wasm_runtime_free(func_ctx->mem_info); if (func_ctx->exception_blocks) wasm_runtime_free(func_ctx->exception_blocks); aot_block_stack_destroy(&func_ctx->block_stack); @@ -630,6 +769,8 @@ aot_destroy_func_contexts(AOTFuncContext **func_ctxes, uint32 count) for (i = 0; i < count; i++) if (func_ctxes[i]) { + if (func_ctxes[i]->mem_info) + wasm_runtime_free(func_ctxes[i]->mem_info); if (func_ctxes[i]->exception_blocks) wasm_runtime_free(func_ctxes[i]->exception_blocks); aot_block_stack_destroy(&func_ctxes[i]->block_stack); @@ -929,6 +1070,9 @@ aot_create_comp_context(AOTCompData *comp_data, if (option->enable_bulk_memory) comp_ctx->enable_bulk_memory = true; + if (option->enable_thread_mgr) + comp_ctx->enable_thread_mgr = true; + if (option->is_jit_mode) { /* Create LLVM execution engine */ LLVMInitializeMCJITCompilerOptions(&jit_options, sizeof(jit_options)); diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 9fd28a622..82df590e3 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -99,6 +99,16 @@ typedef struct AOTCheckedAddr { uint32 bytes; } AOTCheckedAddr, *AOTCheckedAddrList; +typedef struct AOTMemInfo { + LLVMValueRef mem_base_addr; + LLVMValueRef mem_cur_page_count_addr; + LLVMValueRef mem_bound_check_heap_base; + LLVMValueRef mem_bound_check_1byte; + LLVMValueRef mem_bound_check_2bytes; + LLVMValueRef mem_bound_check_4bytes; + LLVMValueRef mem_bound_check_8bytes; +} AOTMemInfo; + typedef struct AOTFuncContext { AOTFunc *aot_func; LLVMValueRef func; @@ -111,12 +121,7 @@ typedef struct AOTFuncContext { LLVMValueRef native_stack_bound; LLVMValueRef last_alloca; - LLVMValueRef mem_base_addr; - LLVMValueRef mem_bound_check_heap_base; - LLVMValueRef mem_bound_check_1byte; - LLVMValueRef mem_bound_check_2bytes; - LLVMValueRef mem_bound_check_4bytes; - LLVMValueRef mem_bound_check_8bytes; + AOTMemInfo *mem_info; LLVMValueRef cur_exception; @@ -196,6 +201,9 @@ typedef struct AOTCompContext { /* Bounday Check */ bool enable_bound_check; + /* Thread Manager */ + bool enable_thread_mgr; + /* Whether optimize the JITed code */ bool optimize; @@ -235,6 +243,7 @@ typedef struct AOTCompOption{ char *target_cpu; char *cpu_features; bool enable_bulk_memory; + bool enable_thread_mgr; bool is_sgx_platform; uint32 opt_level; uint32 size_level; diff --git a/core/iwasm/include/aot_export.h b/core/iwasm/include/aot_export.h index 981c9edca..778400d2f 100644 --- a/core/iwasm/include/aot_export.h +++ b/core/iwasm/include/aot_export.h @@ -40,6 +40,7 @@ typedef struct AOTCompOption{ char *target_cpu; char *cpu_features; bool enable_bulk_memory; + bool enable_thread_mgr; bool is_sgx_platform; uint32_t opt_level; uint32_t size_level; diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 211ba5bf7..66c257c8d 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -933,8 +933,8 @@ wasm_interp_call_func_import(WASMModuleInstance *module_inst, #if WASM_ENABLE_THREAD_MGR != 0 #define CHECK_SUSPEND_FLAGS() do { \ - if (exec_env->suspend_flags != 0) { \ - if (exec_env->suspend_flags & 0x01) { \ + if (exec_env->suspend_flags.flags != 0) { \ + if (exec_env->suspend_flags.flags & 0x01) { \ /* terminate current thread */ \ return; \ } \ diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index fe34f91e8..2307ead86 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -979,8 +979,8 @@ wasm_interp_call_func_import(WASMModuleInstance *module_inst, #if WASM_ENABLE_THREAD_MGR != 0 #define CHECK_SUSPEND_FLAGS() do { \ - if (exec_env->suspend_flags != 0) { \ - if (exec_env->suspend_flags & 0x01) { \ + if (exec_env->suspend_flags.flags != 0) { \ + if (exec_env->suspend_flags.flags & 0x01) { \ /* terminate current thread */ \ return; \ } \ diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index f846f8d2c..7d75a31b4 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -2538,10 +2538,10 @@ load_from_sections(WASMModule *module, WASMSection *sections, && global->is_mutable && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST - && (global->init_expr.u.i32 == + && (global->init_expr.u.i32 <= llvm_heap_base_global->init_expr.u.i32 - || global->init_expr.u.i32 == - llvm_data_end_global->init_expr.u.i32)) { + && llvm_data_end_global->init_expr.u.i32 <= + llvm_heap_base_global->init_expr.u.i32)) { llvm_stack_top_global = global; llvm_stack_top = global->init_expr.u.i32; stack_top_global_index = global_index; @@ -2592,7 +2592,7 @@ load_from_sections(WASMModule *module, WASMSection *sections, if (module->memory_count) { memory = &module->memories[0]; init_memory_size = (uint64)memory->num_bytes_per_page * - memory->init_page_count; + memory->init_page_count; if (llvm_heap_base <= init_memory_size && llvm_data_end <= init_memory_size) { /* Reset memory info to decrease memory usage */ diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index b56aa720d..2e1152428 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -1440,8 +1440,6 @@ load_from_sections(WASMModule *module, WASMSection *sections, WASMGlobal *llvm_stack_top_global = NULL, *global; uint32 llvm_data_end = UINT32_MAX, llvm_heap_base = UINT32_MAX; uint32 llvm_stack_top = UINT32_MAX, global_index, i; - uint32 data_end_global_index = UINT32_MAX; - uint32 heap_base_global_index = UINT32_MAX; uint32 stack_top_global_index = UINT32_MAX; BlockAddr *block_addr_cache; uint64 total_size; @@ -1563,7 +1561,6 @@ load_from_sections(WASMModule *module, WASMSection *sections, && !global->is_mutable && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST) { - heap_base_global_index = global_index; llvm_heap_base_global = global; llvm_heap_base = global->init_expr.u.i32; LOG_VERBOSE("found llvm __heap_base global, value: %d\n", @@ -1577,7 +1574,6 @@ load_from_sections(WASMModule *module, WASMSection *sections, && !global->is_mutable && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST) { - data_end_global_index = global_index; llvm_data_end_global = global; llvm_data_end = global->init_expr.u.i32; LOG_VERBOSE("found llvm __data_end global, value: %d\n", @@ -1588,26 +1584,29 @@ load_from_sections(WASMModule *module, WASMSection *sections, } if (llvm_data_end_global && llvm_heap_base_global) { - if ((data_end_global_index == heap_base_global_index + 1 - && (int32)data_end_global_index > 1) - || (heap_base_global_index == data_end_global_index + 1 - && (int32)heap_base_global_index > 1)) { - global_index = - data_end_global_index < heap_base_global_index - ? data_end_global_index - 1 : heap_base_global_index - 1; + /* Resolve aux stack top global */ + for (global_index = 0; global_index < module->global_count; global_index++) { global = module->globals + global_index; - if (global->type == VALUE_TYPE_I32 + if (global != llvm_data_end_global + && global != llvm_heap_base_global + && global->type == VALUE_TYPE_I32 && global->is_mutable && global->init_expr.init_expr_type == - INIT_EXPR_TYPE_I32_CONST) { + INIT_EXPR_TYPE_I32_CONST + && (global->init_expr.u.i32 <= + llvm_heap_base_global->init_expr.u.i32 + && llvm_data_end_global->init_expr.u.i32 <= + llvm_heap_base_global->init_expr.u.i32)) { llvm_stack_top_global = global; llvm_stack_top = global->init_expr.u.i32; stack_top_global_index = global_index; LOG_VERBOSE("found llvm stack top global, " "value: %d, global index: %d\n", llvm_stack_top, global_index); + break; } } + module->llvm_aux_data_end = llvm_data_end; module->llvm_aux_stack_bottom = llvm_stack_top; module->llvm_aux_stack_size = llvm_stack_top > llvm_data_end diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 5212ca1cf..4f8007a1b 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -142,7 +142,7 @@ memory_instantiate(WASMModuleInstance *module_inst, ref_count = shared_memory_inc_reference( (WASMModuleCommon *)module_inst->module); bh_assert(ref_count > 0); - memory = shared_memory_get_memory_inst(node); + memory = (WASMMemoryInstance *)shared_memory_get_memory_inst(node); bh_assert(memory); (void)ref_count; @@ -160,6 +160,7 @@ memory_instantiate(WASMModuleInstance *module_inst, return NULL; } + memory->module_type = Wasm_Module_Bytecode; memory->num_bytes_per_page = num_bytes_per_page; memory->cur_page_count = init_page_count; memory->max_page_count = max_page_count; @@ -194,7 +195,8 @@ memory_instantiate(WASMModuleInstance *module_inst, if (is_shared_memory) { memory->is_shared = true; if (!shared_memory_set_memory_inst( - (WASMModuleCommon *)module_inst->module, memory)) { + (WASMModuleCommon *)module_inst->module, + (WASMMemoryInstanceCommon *)memory)) { set_error_buf(error_buf, error_buf_size, "Instantiate memory failed:" "allocate memory failed."); @@ -1765,7 +1767,7 @@ wasm_set_aux_stack(WASMExecEnv *exec_env, || ((!is_stack_before_data) && (start_offset - data_end < size))) return false; - if (stack_bottom) { + if ((stack_bottom != (uint32)-1) && (stack_top_idx != (uint32)-1)) { /* The aux stack top is a wasm global, set the initial value for the global */ uint8 *global_addr = diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index f8b12c629..361a76111 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -22,10 +22,10 @@ typedef struct WASMTableInstance WASMTableInstance; typedef struct WASMGlobalInstance WASMGlobalInstance; typedef struct WASMMemoryInstance { -#if WASM_ENABLE_SHARED_MEMORY != 0 - /* shared memory flag */ + /* Module type */ + uint32 module_type; + /* Shared memory flag */ bool is_shared; -#endif /* Number bytes per page */ uint32 num_bytes_per_page; /* Current page count */ diff --git a/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c b/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c index a19ac6f5b..c13e8c57a 100644 --- a/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c +++ b/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c @@ -489,7 +489,8 @@ pthread_start_routine(void *arg) if(!wasm_runtime_call_indirect(exec_env, routine_args->elem_index, 1, argv)) { - wasm_cluster_spread_exception(exec_env); + if (wasm_runtime_get_exception(module_inst)) + wasm_cluster_spread_exception(exec_env); } /* destroy pthread key values */ @@ -673,6 +674,15 @@ pthread_exit_wrapper(wasm_exec_env_t exec_env, int32 retval_offset) if (!args) return; +#ifdef OS_ENABLE_HW_BOUND_CHECK + /* If hardware bound check enabled, don't deinstantiate module inst + and thread info node here for AoT module, as they will be freed + in pthread_start_routine */ + if (exec_env->jmpbuf_stack_top) { + wasm_cluster_exit_thread(exec_env, (void *)(uintptr_t)retval_offset); + } +#endif + /* destroy pthread key values */ call_key_destructor(exec_env); diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index d1d8a531b..8a55265da 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -290,6 +290,11 @@ thread_manager_start_routine(void *arg) exec_env->handle = os_self_thread(); ret = exec_env->thread_start_routine(exec_env); +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (exec_env->suspend_flags.flags & 0x08) + ret = exec_env->thread_ret_value; +#endif + /* Routine exit */ /* Free aux stack space */ free_aux_stack(cluster, @@ -376,6 +381,25 @@ wasm_cluster_exit_thread(WASMExecEnv *exec_env, void *retval) { WASMCluster *cluster; +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (exec_env->jmpbuf_stack_top) { + WASMJmpBuf *jmpbuf_node; + + /* Store the return value in exec_env */ + exec_env->thread_ret_value = retval; + exec_env->suspend_flags.flags |= 0x08; + + /* Free all jmpbuf_node except the last one */ + while (exec_env->jmpbuf_stack_top->prev) { + jmpbuf_node = wasm_exec_env_pop_jmpbuf(exec_env); + wasm_runtime_free(jmpbuf_node); + } + jmpbuf_node = exec_env->jmpbuf_stack_top; + os_longjmp(jmpbuf_node->jmpbuf, 1); + return; + } +#endif + cluster = wasm_exec_env_get_cluster(exec_env); bh_assert(cluster); @@ -396,7 +420,7 @@ int32 wasm_cluster_cancel_thread(WASMExecEnv *exec_env) { /* Set the termination flag */ - exec_env->suspend_flags |= 0x01; + exec_env->suspend_flags.flags |= 0x01; return 0; } @@ -446,7 +470,7 @@ void wasm_cluster_suspend_thread(WASMExecEnv *exec_env) { /* Set the suspend flag */ - exec_env->suspend_flags |= 0x02; + exec_env->suspend_flags.flags |= 0x02; } static void @@ -479,7 +503,7 @@ wasm_cluster_suspend_all_except_self(WASMCluster *cluster, void wasm_cluster_resume_thread(WASMExecEnv *exec_env) { - exec_env->suspend_flags &= ~0x02; + exec_env->suspend_flags.flags &= ~0x02; } static void diff --git a/doc/pthread_library.md b/doc/pthread_library.md index 622bdfa27..953f76a80 100644 --- a/doc/pthread_library.md +++ b/doc/pthread_library.md @@ -79,6 +79,13 @@ Then build the program with this command: # -Wl,--no-check-features: the errno.o in wasi-sysroot is not compatible with pthread feature, pass this option to avoid errors ``` +**Build AoT module** + +You can build the wasm module into AoT module with pthread support, please pass option `--enable-multi-thread` to wamrc: +``` bash +wamrc --enable-multi-thread -o test.aot test.wasm +``` + Currently WAMR disables pthread library by default. To run the module with pthread support, please build the runtime with `-DWAMR_BUILD_LIB_PTHREAD=1` ``` bash cd ${WAMR_ROOT}/product-mini/platforms/linux @@ -87,10 +94,10 @@ cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 make # Then you can run the wasm module above: ./iwasm test.wasm +# Or the AoT module: +# ./iwasm test.aot ``` -> Note: Currently pthread library is not supported in AoT mode. - [Here](../samples/multi-thread) is also a sample to show how wasm-apps use pthread APIs to create threads, and how to build it with cmake. You can build this sample and have a try: ``` bash cd ${WAMR_ROOT}/samples/multi-thread diff --git a/product-mini/platforms/linux/CMakeLists.txt b/product-mini/platforms/linux/CMakeLists.txt index e7a5a0786..6b1b6e753 100644 --- a/product-mini/platforms/linux/CMakeLists.txt +++ b/product-mini/platforms/linux/CMakeLists.txt @@ -65,6 +65,11 @@ if (NOT DEFINED WAMR_BUILD_MULTI_MODULE) set (WAMR_BUILD_MULTI_MODULE 0) endif () +if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) + # Disable pthread library by default + set (WAMR_BUILD_LIB_PTHREAD 0) +endif () + if (NOT DEFINED WAMR_BUILD_MINI_LOADER) # Disable wasm mini loader by default set (WAMR_BUILD_MINI_LOADER 0) diff --git a/wamr-compiler/CMakeLists.txt b/wamr-compiler/CMakeLists.txt index 02fadd359..7cdf2fb2a 100644 --- a/wamr-compiler/CMakeLists.txt +++ b/wamr-compiler/CMakeLists.txt @@ -16,6 +16,8 @@ add_definitions(-DWASM_ENABLE_INTERP=1) add_definitions(-DWASM_ENABLE_WAMR_COMPILER=1) add_definitions(-DWASM_ENABLE_BULK_MEMORY=1) add_definitions(-DWASM_DISABLE_HW_BOUND_CHECK=1) +add_definitions(-DWASM_ENABLE_SHARED_MEMORY=1) +add_definitions(-DWASM_ENABLE_THREAD_MGR=1) # Set WAMR_BUILD_TARGET, currently values supported: # "X86_64", "AMD_64", "X86_32", "ARM_32", "MIPS_32", "XTENSA_32" diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index 7b54cd879..3c72dc381 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -40,6 +40,8 @@ print_help() printf(" llvmir-unopt Unoptimized LLVM IR\n"); printf(" llvmir-opt Optimized LLVM IR\n"); printf(" --enable-bulk-memory Enable the post-MVP bulk memory feature\n"); + printf(" --enable-multi-thread Enable multi-thread feature, the dependent features bulk-memory and\n"); + printf(" thread-mgr will be enabled automatically\n"); printf(" -v=n Set log verbose level (0 to 5, default is 2), larger with more log\n"); printf("Examples: wamrc -o test.aot test.wasm\n"); printf(" wamrc --target=i386 -o test.aot test.wasm\n"); @@ -140,6 +142,10 @@ main(int argc, char *argv[]) else if (!strcmp(argv[0], "--enable-bulk-memory")) { option.enable_bulk_memory = true; } + else if (!strcmp(argv[0], "--enable-multi-thread")) { + option.enable_bulk_memory = true; + option.enable_thread_mgr = true; + } else return print_help(); }