From 92852f3719572b80a06de2df3de5bc662183022c Mon Sep 17 00:00:00 2001 From: WenLY1 <130950131+WenLY1@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:51:42 +0800 Subject: [PATCH 1/6] Implement a first version of shared heap feature (#3789) ps. https://github.com/bytecodealliance/wasm-micro-runtime/issues/3546 --- build-scripts/config_common.cmake | 5 + build-scripts/runtime_lib.cmake | 5 + core/config.h | 4 + core/iwasm/aot/aot_runtime.h | 4 + core/iwasm/common/wasm_memory.c | 470 +++++++++++++++++- core/iwasm/common/wasm_memory.h | 26 + core/iwasm/common/wasm_native.c | 13 + core/iwasm/include/wasm_export.h | 55 ++ core/iwasm/interpreter/wasm_interp_classic.c | 69 ++- core/iwasm/interpreter/wasm_interp_fast.c | 30 +- core/iwasm/interpreter/wasm_runtime.h | 15 + .../libraries/shared-heap/shared_heap.cmake | 8 + .../shared-heap/shared_heap_wrapper.c | 55 ++ .../libraries/thread-mgr/thread_manager.c | 77 +++ .../libraries/thread-mgr/thread_manager.h | 12 + 15 files changed, 826 insertions(+), 22 deletions(-) create mode 100644 core/iwasm/libraries/shared-heap/shared_heap.cmake create mode 100644 core/iwasm/libraries/shared-heap/shared_heap_wrapper.c diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 12fc06bd7..0a5f82106 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -256,6 +256,11 @@ if (WAMR_BUILD_SHARED_MEMORY EQUAL 1) else () add_definitions (-DWASM_ENABLE_SHARED_MEMORY=0) endif () +if (WAMR_BUILD_SHARED_HEAP EQUAL 1) + add_definitions (-DWASM_ENABLE_SHARED_HEAP=1) + message (" Shared heap enabled") +endif() + if (WAMR_BUILD_MEMORY64 EQUAL 1) # if native is 32-bit or cross-compiled to 32-bit if (NOT WAMR_BUILD_TARGET MATCHES ".*64.*") diff --git a/build-scripts/runtime_lib.cmake b/build-scripts/runtime_lib.cmake index 3ab0cff4f..a2aaf425b 100644 --- a/build-scripts/runtime_lib.cmake +++ b/build-scripts/runtime_lib.cmake @@ -119,6 +119,10 @@ if (WAMR_BUILD_LIB_WASI_THREADS EQUAL 1) set (WAMR_BUILD_SHARED_MEMORY 1) endif () +if (WAMR_BUILD_SHARED_HEAP EQUAL 1) + include (${IWASM_DIR}/libraries/shared-heap/shared_heap.cmake) +endif () + if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) set (WAMR_BUILD_THREAD_MGR 1) include (${IWASM_DIR}/libraries/debug-engine/debug_engine.cmake) @@ -193,6 +197,7 @@ set (source_all ${LIBC_EMCC_SOURCE} ${LIB_RATS_SOURCE} ${DEBUG_ENGINE_SOURCE} + ${LIB_SHARED_HEAP_SOURCE} ) set (WAMR_RUNTIME_LIB_SOURCE ${source_all}) diff --git a/core/config.h b/core/config.h index a25eb543e..a7925b008 100644 --- a/core/config.h +++ b/core/config.h @@ -688,4 +688,8 @@ #endif #endif /* WASM_ENABLE_FUZZ_TEST != 0 */ +#ifndef WASM_ENABLE_SHARED_HEAP +#define WASM_ENABLE_SHARED_HEAP 0 +#endif + #endif /* end of _CONFIG_H_ */ diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 56d11a22d..4c3676f88 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -119,6 +119,10 @@ typedef struct AOTModuleInstanceExtra { bh_list *sub_module_inst_list; WASMModuleInstanceCommon **import_func_module_insts; #endif + +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; +#endif } AOTModuleInstanceExtra; #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 82eebbf30..dc9c4aa1b 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -13,6 +13,10 @@ #include "../common/wasm_shared_memory.h" #endif +#if WASM_ENABLE_THREAD_MGR != 0 +#include "../libraries/thread-mgr/thread_manager.h" +#endif + typedef enum Memory_Mode { MEMORY_MODE_UNKNOWN = 0, MEMORY_MODE_POOL, @@ -24,6 +28,11 @@ static Memory_Mode memory_mode = MEMORY_MODE_UNKNOWN; static mem_allocator_t pool_allocator = NULL; +#if WASM_ENABLE_SHARED_HEAP != 0 +static WASMSharedHeap *shared_heap_list = NULL; +static korp_mutex shared_heap_list_lock; +#endif + static enlarge_memory_error_callback_t enlarge_memory_error_cb; static void *enlarge_memory_error_user_data; @@ -132,16 +141,370 @@ is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) #endif } +#if WASM_ENABLE_SHARED_HEAP != 0 +static void * +wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size); +static void +wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, + uint64 map_size); + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, + "Operation of shared heap failed: %s", string); + } +} + +static void * +runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +WASMSharedHeap * +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, + uint32 error_buf_size) +{ + uint64 heap_struct_size = sizeof(WASMSharedHeap); + uint32 size = init_args->size; + WASMSharedHeap *heap; + + if (!(heap = runtime_malloc(heap_struct_size, error_buf, error_buf_size))) { + goto fail1; + } + if (!(heap->heap_handle = + runtime_malloc(mem_allocator_get_heap_struct_size(), error_buf, + error_buf_size))) { + goto fail2; + } + heap->start_off_mem64 = UINT64_MAX - heap->size + 1; + heap->start_off_mem32 = UINT32_MAX - heap->size + 1; + + size = align_uint(size, os_getpagesize()); + if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { + set_error_buf(error_buf, error_buf_size, "invalid size of shared heap"); + goto fail3; + } + + if (!(heap->base_addr = wasm_mmap_linear_memory(size, size))) { + goto fail3; + } + if (!mem_allocator_create_with_struct_and_pool( + heap->heap_handle, heap_struct_size, heap->base_addr, size)) { + set_error_buf(error_buf, error_buf_size, "init share heap failed"); + goto fail4; + } + + os_mutex_lock(&shared_heap_list_lock); + if (shared_heap_list == NULL) { + shared_heap_list = heap; + } + else { + heap->next = shared_heap_list; + shared_heap_list = heap; + } + os_mutex_unlock(&shared_heap_list_lock); + return heap; + +fail4: + wasm_munmap_linear_memory(heap->base_addr, size, size); + +fail3: + wasm_runtime_free(heap->heap_handle); +fail2: + wasm_runtime_free(heap); +fail1: + return NULL; +} + +bool +wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + return wasm_cluster_attach_shared_heap(module_inst, shared_heap); +#else + return wasm_runtime_attach_shared_heap_internal(module_inst, shared_heap); +#endif +} + +bool +wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap) +{ + uint64 linear_mem_size = 0; + WASMMemoryInstance *memory = NULL; + WASMSharedHeap *heap = (WASMSharedHeap *)shared_heap; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + // check if linear memory and shared heap are overlapped + linear_mem_size = memory->memory_data_size; + + if ((memory->is_memory64 && linear_mem_size > heap->start_off_mem64) + || (!memory->is_memory64 && linear_mem_size > heap->start_off_mem32)) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + return false; + } + + if (module_inst->module_type == Wasm_Module_Bytecode) { + if (((WASMModuleInstance *)module_inst)->e->shared_heap) { + LOG_WARNING("A shared heap is already attached"); + return false; + } + ((WASMModuleInstance *)module_inst)->e->shared_heap = heap; + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + return true; +} + +void +wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_detach_shared_heap(module_inst); +#else + wasm_runtime_detach_shared_heap_internal(module_inst); +#endif +} + +void +wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) +{ + if (module_inst->module_type == Wasm_Module_Bytecode) { + ((WASMModuleInstance *)module_inst)->e->shared_heap = NULL; + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } +} + +static bool +is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, + bool is_memory64, uint64 app_offset, uint32 bytes) +{ + WASMSharedHeap *heap = NULL; + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; + } + else if (module_inst_comm->module_type == Wasm_Module_AoT) { + // TODO + } + + if (!heap) { + return false; + } + + if (!is_memory64) { + if (app_offset >= heap->start_off_mem32 + && app_offset <= UINT32_MAX - bytes + 1) { + return true; + } + } + else { + if (app_offset >= heap->start_off_mem64 + && app_offset <= UINT64_MAX - bytes + 1) { + return true; + } + } + return false; +} + +static bool +is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, + uint8 *addr, uint32 bytes) +{ + WASMSharedHeap *heap = NULL; + + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; + } + else if (module_inst_comm->module_type == Wasm_Module_AoT) { + // TODO + } + + if (heap && addr >= heap->base_addr + && addr + bytes <= heap->base_addr + heap->size + && addr + bytes > addr) { + return true; + } + return false; +} + +static uint64 +shared_heap_addr_native_to_app(WASMModuleInstanceCommon *module_inst, + WASMMemoryInstance *memory, void *addr) +{ + WASMSharedHeap *heap = NULL; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (!heap) { + LOG_WARNING("Wasm module doesn't attach to a shared heap"); + return 0; + } + if (!addr) { + LOG_WARNING("Invalid address"); + return 0; + } + + if (memory && memory->is_memory64) { + return heap->start_off_mem64 + ((uint8 *)addr - heap->base_addr); + } + else if (memory && !memory->is_memory64) { + return heap->start_off_mem32 + ((uint8 *)addr - heap->base_addr); + } + return 0; +} + +static void * +shared_heap_addr_app_to_native(WASMModuleInstanceCommon *module_inst, + WASMMemoryInstance *memory, uint64 ptr) +{ + void *addr = NULL; + WASMSharedHeap *heap = NULL; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (!heap) { + LOG_WARNING("Wasm module doesn't attach to a shared heap"); + return NULL; + } + + if (!memory) { + LOG_WARNING("Wasm memory is not initialized"); + return NULL; + } + + if (memory->is_memory64) { + addr = heap->base_addr + (ptr - heap->start_off_mem64); + } + else { + addr = heap->base_addr + (ptr - heap->start_off_mem32); + } + + return addr; +} + +static uint64 +shared_heap_get_addr_start(WASMSharedHeap *heap, WASMMemoryInstance *memory) +{ + uint64 shared_heap_start = 0; + + if (!heap || !memory) { + LOG_ERROR("Invalid heap or memory"); + return 0; + } + + if (memory && !memory->is_memory64) { + shared_heap_start = heap->start_off_mem32; + } + else if (memory && memory->is_memory64) { + shared_heap_start = heap->start_off_mem64; + } + + return shared_heap_start; +} + +uint64 +wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, + uint64_t size, void **p_native_addr) +{ + WASMSharedHeap *heap = NULL; + WASMMemoryInstance *memory = NULL; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (heap) { + *p_native_addr = mem_allocator_malloc(heap->heap_handle, size); + + return shared_heap_addr_native_to_app(module_inst, memory, + *p_native_addr); + } + else { + LOG_WARNING("Wasm module doesn't attach to a shared heap"); + } + return 0; +} + +void +wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr) +{ + WASMSharedHeap *heap = NULL; + WASMMemoryInstance *memory = NULL; + void *addr = NULL; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (!heap) { + LOG_WARNING("Wasm module doesn't attach to a shared heap"); + return; + } + + addr = shared_heap_addr_app_to_native(module_inst, memory, ptr); + + if (heap) { + mem_allocator_free(heap->base_addr, addr); + } +} +#endif + bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, const MemAllocOption *alloc_option) { + bool ret = false; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (os_mutex_init(&shared_heap_list_lock)) { + return false; + } +#endif if (mem_alloc_type == Alloc_With_Pool) { - return wasm_memory_init_with_pool(alloc_option->pool.heap_buf, - alloc_option->pool.heap_size); + ret = wasm_memory_init_with_pool(alloc_option->pool.heap_buf, + alloc_option->pool.heap_size); } else if (mem_alloc_type == Alloc_With_Allocator) { - return wasm_memory_init_with_allocator( + ret = wasm_memory_init_with_allocator( #if WASM_MEM_ALLOC_WITH_USER_DATA != 0 alloc_option->allocator.user_data, #endif @@ -151,16 +514,48 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, } else if (mem_alloc_type == Alloc_With_System_Allocator) { memory_mode = MEMORY_MODE_SYSTEM_ALLOCATOR; - return true; + ret = true; } - else { - return false; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (!ret) { + os_mutex_destroy(&shared_heap_list_lock); + } +#endif + + return ret; +} + +#if WASM_ENABLE_SHARED_HEAP != 0 +static void +wasm_runtime_shared_heap_destroy() +{ + WASMSharedHeap *heap = shared_heap_list; + WASMSharedHeap *cur; + int ret = 0; + + while (heap) { + cur = heap; + heap = heap->next; + ret = ret + mem_allocator_destroy(cur->heap_handle); + wasm_runtime_free(cur->heap_handle); + wasm_munmap_linear_memory(cur->base_addr, cur->size, cur->size); + wasm_runtime_free(cur); + } + + if (ret != 0) { + LOG_ERROR("Memory leak detected in shared heap"); } } +#endif void wasm_runtime_memory_destroy(void) { + +#if WASM_ENABLE_SHARED_HEAP != 0 + wasm_runtime_shared_heap_destroy(); +#endif + if (memory_mode == MEMORY_MODE_POOL) { #if BH_ENABLE_GC_VERIFY == 0 (void)mem_allocator_destroy(pool_allocator); @@ -342,7 +737,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, /* boundary overflow check */ if (size > max_linear_memory_size || app_offset > max_linear_memory_size - size) { - goto fail; + goto shared_heap_bound_check; } SHARED_MEMORY_LOCK(memory_inst); @@ -354,6 +749,13 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); +shared_heap_bound_check: +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_offset, size)) { + return true; + } +#endif fail: wasm_set_exception(module_inst, "out of bounds memory access"); return false; @@ -439,6 +841,13 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, return true; } +#if WASM_ENABLE_SHARED_HEAP != 0 + else if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, + size)) { + SHARED_MEMORY_UNLOCK(memory_inst); + return true; + } +#endif SHARED_MEMORY_UNLOCK(memory_inst); fail: @@ -475,6 +884,19 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); return addr; } +#if WASM_ENABLE_SHARED_HEAP != 0 + else if (is_app_addr_in_shared_heap(module_inst_comm, + memory_inst->is_memory64, + app_offset, 1)) { + uint64 heap_start = shared_heap_get_addr_start( + module_inst->e->shared_heap, memory_inst); + uint64 heap_offset = (uint64)app_offset - heap_start; + + addr = module_inst->e->shared_heap->base_addr + heap_offset; + SHARED_MEMORY_UNLOCK(memory_inst); + return addr; + } +#endif SHARED_MEMORY_UNLOCK(memory_inst); return NULL; } @@ -499,6 +921,11 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, bounds_checks = is_bounds_checks_enabled(module_inst_comm); +#if WASM_ENABLE_SHARED_HEAP != 0 + /* If shared heap is enabled, bounds check is always needed */ + bounds_checks = true; +#endif + memory_inst = wasm_get_default_memory(module_inst); if (!memory_inst) { return 0; @@ -513,6 +940,17 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); return ret; } + else { +#if WASM_ENABLE_SHARED_HEAP != 0 + uint64 shared_heap_start = shared_heap_get_addr_start( + module_inst->e->shared_heap, memory_inst); + ret = + (uint64)(addr - (uint8 *)module_inst->e->shared_heap->base_addr) + + shared_heap_start; + SHARED_MEMORY_UNLOCK(memory_inst); + return ret; +#endif + } } /* If bounds checks is disabled, return the offset directly */ else if (addr != NULL) { @@ -765,6 +1203,11 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, #else WASMMemoryInstance *memory = wasm_get_default_memory(module); #endif + +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *heap; +#endif + uint8 *memory_data_old, *memory_data_new, *heap_data_old; uint32 num_bytes_per_page, heap_size; uint32 cur_page_count, max_page_count, total_page_count; @@ -797,6 +1240,19 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, total_page_count = inc_page_count + cur_page_count; total_size_new = num_bytes_per_page * (uint64)total_page_count; +#if WASM_ENABLE_SHARED_HEAP != 0 + heap = module->e->shared_heap; + if (memory->is_memory64 && total_size_new > heap->start_off_mem64) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } + else if (!memory->is_memory64 && total_size_new > heap->start_off_mem32) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } +#endif if (inc_page_count <= 0) /* No need to enlarge memory */ return true; diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index 2f20d3f68..be89772e3 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -41,6 +41,32 @@ SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size) #define SET_LINEAR_MEMORY_SIZE(memory, size) memory->memory_data_size = size #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +WASMSharedHeap * +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, + uint32 error_buf_size); + +bool +wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap); +bool +wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap); + +void +wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst); +void +wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst); + +uint64 +wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, + uint64 size, void **p_native_addr); + +void +wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, + uint64 ptr); +#endif + bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, const MemAllocOption *alloc_option); diff --git a/core/iwasm/common/wasm_native.c b/core/iwasm/common/wasm_native.c index 0ff3053fa..060bb2c3d 100644 --- a/core/iwasm/common/wasm_native.c +++ b/core/iwasm/common/wasm_native.c @@ -33,6 +33,11 @@ uint32 get_spectest_export_apis(NativeSymbol **p_libc_builtin_apis); #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +uint32 +get_lib_shared_heap_export_apis(NativeSymbol **p_shared_heap_apis); +#endif + uint32 get_libc_wasi_export_apis(NativeSymbol **p_libc_wasi_apis); @@ -512,6 +517,14 @@ wasm_native_init() goto fail; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + n_native_symbols = get_lib_shared_heap_export_apis(&native_symbols); + if (n_native_symbols > 0 + && !wasm_native_register_natives("env", native_symbols, + n_native_symbols)) + goto fail; +#endif + #if WASM_ENABLE_BASE_LIB != 0 n_native_symbols = get_base_lib_export_apis(&native_symbols); if (n_native_symbols > 0 diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 569c4deaa..5ce617ef1 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -135,6 +135,9 @@ typedef struct wasm_section_t { struct WASMExecEnv; typedef struct WASMExecEnv *wasm_exec_env_t; +struct WASMSharedHeap; +typedef struct WASMSharedHeap *wasm_shared_heap_t; + /* Package Type */ typedef enum { Wasm_Module_Bytecode = 0, @@ -320,6 +323,12 @@ typedef enum { WASM_LOG_LEVEL_VERBOSE = 4 } log_level_t; +#if WASM_ENABLE_SHARED_HEAP != 0 +typedef struct SharedHeapInitArgs { + uint32 size; +} SharedHeapInitArgs; +#endif + /** * Initialize the WASM runtime environment, and also initialize * the memory allocator with system allocator, which calls os_malloc @@ -2110,6 +2119,52 @@ wasm_runtime_detect_native_stack_overflow_size(wasm_exec_env_t exec_env, WASM_RUNTIME_API_EXTERN bool wasm_runtime_is_underlying_binary_freeable(const wasm_module_t module); +#if WASM_ENABLE_SHARED_HEAP != 0 +/** + * Create a shared heap + * @param init_args the initialization arguments + * @param error_buf buffer to output the error info if failed + * @param error_buf_size the size of the error buffer + */ +WASM_RUNTIME_API_EXTERN wasm_shared_heap_t +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, + uint32 error_buf_size); + +/** + * Attach a shared heap to a module instance + * @param module_inst the module instance + * @param shared_heap the shared heap + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst, + wasm_shared_heap_t shared_heap); + +/** + * Detach a shared heap from a module instance + * @param module_inst the module instance + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst); + +/** + * Allocate memory from a shared heap + * @param module_inst the module instance + * @param size required memory size + * @param p_native_addr native address of allocated memory + */ +WASM_RUNTIME_API_EXTERN uint64 +wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64 size, + void **p_native_addr); + +/** + * Free the memory allocated from shared heap + * @param module_inst the module instance + * @param ptr the offset in wasm app + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr); +#endif + #ifdef __cplusplus } #endif diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 67f8c2d45..f39ae3468 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -48,6 +48,31 @@ typedef float64 CellType_F64; #if WASM_ENABLE_MEMORY64 == 0 +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 +#define GOTO_OUT_OF_BOUNDS goto out_of_bounds; +#else +#define GOTO_OUT_OF_BOUNDS +#endif + +#if WASM_ENABLE_SHARED_HEAP != 0 +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ + do { \ + if (offset1 + bytes >= UINT32_MAX - module->shared_heap->size \ + && offset1 + bytes <= UINT32_MAX) { \ + uint64 heap_start = UINT32_MAX - module->shared_heap->size; \ + uint64 heap_offset = (uint64)offset1 - heap_start; \ + maddr = module->shared_heap->data + heap_offset; \ + } \ + else { \ + GOTO_OUT_OF_BOUNDS; \ + } \ + } while (0) +#else +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS +#endif + #if (!defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0) #define CHECK_MEMORY_OVERFLOW(bytes) \ @@ -58,7 +83,7 @@ typedef float64 CellType_F64; be in valid range, no need to check it again. */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ @@ -69,7 +94,7 @@ typedef float64 CellType_F64; bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ } while (0) #else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ @@ -82,22 +107,40 @@ typedef float64 CellType_F64; #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ maddr = memory->memory_data + (uint32)(start); \ + uint64 offset1 = start; \ } while (0) #endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ #else /* else of WASM_ENABLE_MEMORY64 == 0 */ -#define CHECK_MEMORY_OVERFLOW(bytes) \ - do { \ - uint64 offset1 = (uint64)offset + (uint64)addr; \ - /* If memory64 is enabled, offset1, offset1 + bytes can overflow */ \ - if (disable_bounds_checks \ - || (offset1 >= offset && offset1 + bytes >= offset1 \ - && offset1 + bytes <= get_linear_mem_size())) \ - maddr = memory->memory_data + offset1; \ - else \ - goto out_of_bounds; \ +#if WASM_ENABLE_SHARED_HEAP != 0 +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ + do { \ + if (offset1 + bytes >= UINT64_MAX - module->shared_heap->size \ + && offset1 + bytes <= UINT64_MAX) { \ + uint64 heap_start = UINT64_MAX - module->shared_heap->size; \ + uint64 heap_offset = (uint64)offset1 - heap_start; \ + maddr = module->shared_heap->data + heap_offset; \ + } \ + else \ + goto out_of_bounds; \ + } while (0) +#else +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) goto out_of_bounds; +#endif + +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + /* If memory64 is enabled, offset1, offset1 + bytes can \ + * overflow */ \ + if (disable_bounds_checks \ + || (offset1 >= offset && offset1 + bytes >= offset1 \ + && offset1 + bytes <= get_linear_mem_size())) \ + maddr = memory->memory_data + offset1; \ + else \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ @@ -110,7 +153,7 @@ typedef float64 CellType_F64; bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ } while (0) #endif /* end of WASM_ENABLE_MEMORY64 == 0 */ diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 1d7ca8f90..84170f213 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -37,6 +37,31 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 +#define GOTO_OUT_OF_BOUNDS goto out_of_bounds; +#else +#define GOTO_OUT_OF_BOUNDS +#endif + +#if WASM_ENABLE_SHARED_HEAP != 0 +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ + do { \ + if (offset1 + bytes >= UINT32_MAX - module->e->shared_heap->size \ + && offset1 + bytes <= UINT32_MAX) { \ + uint64 heap_start = UINT32_MAX - module->e->shared_heap->size; \ + uint64 heap_offset = (uint64)offset1 - heap_start; \ + maddr = module->e->shared_heap->base_addr + heap_offset; \ + } \ + else { \ + GOTO_OUT_OF_BOUNDS; \ + } \ + } while (0) +#else +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS +#endif + #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 #define CHECK_MEMORY_OVERFLOW(bytes) \ @@ -47,7 +72,7 @@ typedef float64 CellType_F64; be in valid range, no need to check it again. */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ @@ -58,7 +83,7 @@ typedef float64 CellType_F64; bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ } while (0) #else #define CHECK_MEMORY_OVERFLOW(bytes) \ @@ -70,6 +95,7 @@ typedef float64 CellType_F64; #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ maddr = memory->memory_data + (uint32)(start); \ + uint64 offset1 = start; \ } while (0) #endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 8666541f2..3fa3682c0 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -92,6 +92,17 @@ typedef union { uint32 u32[2]; } MemBound; +#if WASM_ENABLE_SHARED_HEAP != 0 +typedef struct WASMSharedHeap { + struct WASMSharedHeap *next; + void *heap_handle; + uint8_t *base_addr; + uint32_t size; + uint64 start_off_mem64; + uint64 start_off_mem32; +} WASMSharedHeap; +#endif + struct WASMMemoryInstance { /* Module type */ uint32 module_type; @@ -353,6 +364,10 @@ typedef struct WASMModuleInstanceExtra { uint32 max_aux_stack_used; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; +#endif + #if WASM_ENABLE_DEBUG_INTERP != 0 \ || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ && WASM_ENABLE_LAZY_JIT != 0) diff --git a/core/iwasm/libraries/shared-heap/shared_heap.cmake b/core/iwasm/libraries/shared-heap/shared_heap.cmake new file mode 100644 index 000000000..ec91dabcd --- /dev/null +++ b/core/iwasm/libraries/shared-heap/shared_heap.cmake @@ -0,0 +1,8 @@ +# Copyright (C) 2024 Xiaomi Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (LIB_SHARED_HEAP ${CMAKE_CURRENT_LIST_DIR}) +add_definitions (-DWASM_ENABLE_SHARED_HEAP=1) +include_directories(${LIB_SHARED_HEAP_DIR}) +file (GLOB source_all ${LIB_SHARED_HEAP}/*.c) +set (LIB_SHARED_HEAP_SOURCE ${source_all}) \ No newline at end of file diff --git a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c new file mode 100644 index 000000000..164568ac2 --- /dev/null +++ b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2024 Xiaomi Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_common.h" +#include "bh_log.h" +#include "wasm_export.h" +#include "../interpreter/wasm.h" +#include "../common/wasm_runtime_common.h" +/* clang-format off */ +#define validate_native_addr(addr, size) \ + wasm_runtime_validate_native_addr(module_inst, addr, size) + +#define module_shared_malloc(size, p_native_addr) \ + wasm_runtime_shared_heap_malloc(module_inst, size, p_native_addr) + +#define module_shared_free(offset) \ + wasm_runtime_shared_heap_free(module_inst, offset) +/* clang-format on */ + +static uint32 +shared_malloc_wrapper(wasm_exec_env_t exec_env, uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + return (uint32)module_shared_malloc((uint64)size, NULL); +} + +static void +shared_free_wrapper(wasm_exec_env_t exec_env, void *ptr) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + if (!validate_native_addr(ptr, (uint64)sizeof(uint32))) + return; + + module_shared_free(addr_native_to_app(ptr)); +} + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } +/* clang-format on */ + +static NativeSymbol native_symbols_shared_heap[] = { + REG_NATIVE_FUNC(shared_malloc, "(i)i"), + REG_NATIVE_FUNC(shared_free, "(*)"), +}; + +uint32 +get_lib_shared_heap_export_apis(NativeSymbol **p_shared_heap_apis) +{ + *p_shared_heap_apis = native_symbols_shared_heap; + return sizeof(native_symbols_shared_heap) / sizeof(NativeSymbol); +} \ No newline at end of file diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index ebb56ba7c..27ff1441f 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -1402,6 +1402,83 @@ wasm_cluster_spread_custom_data(WASMModuleInstanceCommon *module_inst, } } +#if WASM_ENABLE_SHARED_HEAP != 0 +static void +attach_shared_heap_visitor(void *node, void *heap) +{ + WASMExecEnv *curr_exec_env = (WASMExecEnv *)node; + WASMModuleInstanceCommon *module_inst = get_module_inst(curr_exec_env); + + wasm_runtime_attach_shared_heap_internal(module_inst, heap); +} + +static void +detach_shared_heap_visitor(void *node, void *heap) +{ + WASMExecEnv *curr_exec_env = (WASMExecEnv *)node; + WASMModuleInstanceCommon *module_inst = get_module_inst(curr_exec_env); + + (void)heap; + wasm_runtime_detach_shared_heap_internal(module_inst); +} + +bool +wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *heap) +{ + WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst); + + if (module_inst->module_type == Wasm_Module_Bytecode) { + if (((WASMModuleInstance *)module_inst)->e->shared_heap) { + LOG_WARNING("A shared heap is already attached"); + return false; + } + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (exec_env == NULL) { + /* Maybe threads have not been started yet. */ + return wasm_runtime_attach_shared_heap_internal(module_inst, heap); + } + else { + WASMCluster *cluster; + + cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + + os_mutex_lock(&cluster->lock); + traverse_list(&cluster->exec_env_list, attach_shared_heap_visitor, + heap); + os_mutex_unlock(&cluster->lock); + } + return true; +} + +void +wasm_cluster_detach_shared_heap(WASMModuleInstanceCommon *module_inst) +{ + WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst); + + if (exec_env == NULL) { + /* Maybe threads have not been started yet. */ + wasm_runtime_detach_shared_heap_internal(module_inst); + } + else { + WASMCluster *cluster; + + cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + + os_mutex_lock(&cluster->lock); + traverse_list(&cluster->exec_env_list, detach_shared_heap_visitor, + NULL); + os_mutex_unlock(&cluster->lock); + } +} +#endif + #if WASM_ENABLE_MODULE_INST_CONTEXT != 0 struct inst_set_context_data { void *key; diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.h b/core/iwasm/libraries/thread-mgr/thread_manager.h index 7ad6c772a..90d97b0be 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.h +++ b/core/iwasm/libraries/thread-mgr/thread_manager.h @@ -11,6 +11,9 @@ #include "wasm_export.h" #include "../interpreter/wasm.h" #include "../common/wasm_runtime_common.h" +#if WASM_ENABLE_SHARED_HEAP != 0 +#include "../common/wasm_memory.h" +#endif #ifdef __cplusplus extern "C" { @@ -167,6 +170,15 @@ wasm_cluster_set_context(WASMModuleInstanceCommon *module_inst, void *key, bool wasm_cluster_is_thread_terminated(WASMExecEnv *exec_env); +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *heap); + +void +wasm_cluster_detach_shared_heap(WASMModuleInstanceCommon *module_inst); +#endif + #if WASM_ENABLE_DEBUG_INTERP != 0 #define WAMR_SIG_TRAP (5) #define WAMR_SIG_STOP (19) From 5e20cf383e90cb63750fc90408791c7ffba6fffa Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 18 Sep 2024 14:53:41 +0800 Subject: [PATCH 2/6] Refactor shared heap feature for interpreter mode (#3794) To add test cases and samples. --- core/iwasm/common/wasm_memory.c | 490 +++++++++--------- core/iwasm/common/wasm_shared_memory.c | 42 +- core/iwasm/interpreter/wasm_interp_classic.c | 215 +++++--- core/iwasm/interpreter/wasm_interp_fast.c | 168 ++++-- core/iwasm/interpreter/wasm_runtime.h | 4 +- .../libraries/thread-mgr/thread_manager.c | 9 + 6 files changed, 543 insertions(+), 385 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index dc9c4aa1b..48c87e766 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -218,7 +218,6 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, fail4: wasm_munmap_linear_memory(heap->base_addr, size, size); - fail3: wasm_runtime_free(heap->heap_handle); fail2: @@ -227,6 +226,45 @@ fail1: return NULL; } +bool +wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap) +{ + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module_inst); + uint64 linear_mem_size; + + if (!memory) + return false; + + linear_mem_size = memory->memory_data_size; + + /* check if linear memory and shared heap are overlapped */ + if ((memory->is_memory64 && linear_mem_size > shared_heap->start_off_mem64) + || (!memory->is_memory64 + && linear_mem_size > shared_heap->start_off_mem32)) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + return false; + } + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + if (((WASMModuleInstance *)module_inst)->e->shared_heap) { + LOG_WARNING("A shared heap is already attached"); + return false; + } + ((WASMModuleInstance *)module_inst)->e->shared_heap = shared_heap; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } +#endif + + return true; +} + bool wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, WASMSharedHeap *shared_heap) @@ -238,42 +276,19 @@ wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, #endif } -bool -wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, - WASMSharedHeap *shared_heap) +void +wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) { - uint64 linear_mem_size = 0; - WASMMemoryInstance *memory = NULL; - WASMSharedHeap *heap = (WASMSharedHeap *)shared_heap; - +#if WASM_ENABLE_INTERP != 0 if (module_inst->module_type == Wasm_Module_Bytecode) { - memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); + ((WASMModuleInstance *)module_inst)->e->shared_heap = NULL; } - else if (module_inst->module_type == Wasm_Module_AoT) { +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { // TODO } - - // check if linear memory and shared heap are overlapped - linear_mem_size = memory->memory_data_size; - - if ((memory->is_memory64 && linear_mem_size > heap->start_off_mem64) - || (!memory->is_memory64 && linear_mem_size > heap->start_off_mem32)) { - LOG_WARNING("Linear memory address is overlapped with shared heap"); - return false; - } - - if (module_inst->module_type == Wasm_Module_Bytecode) { - if (((WASMModuleInstance *)module_inst)->e->shared_heap) { - LOG_WARNING("A shared heap is already attached"); - return false; - } - ((WASMModuleInstance *)module_inst)->e->shared_heap = heap; - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } - - return true; +#endif } void @@ -286,28 +301,28 @@ wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst) #endif } -void -wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) +static WASMSharedHeap * +get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) { - if (module_inst->module_type == Wasm_Module_Bytecode) { - ((WASMModuleInstance *)module_inst)->e->shared_heap = NULL; +#if WASM_ENABLE_INTERP != 0 + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + return ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; } - else if (module_inst->module_type == Wasm_Module_AoT) { +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst_comm->module_type == Wasm_Module_AoT) { // TODO + return NULL; } +#endif + return NULL; } static bool -is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, +is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, bool is_memory64, uint64 app_offset, uint32 bytes) { - WASMSharedHeap *heap = NULL; - if (module_inst_comm->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; - } - else if (module_inst_comm->module_type == Wasm_Module_AoT) { - // TODO - } + WASMSharedHeap *heap = get_shared_heap(module_inst); if (!heap) { return false; @@ -325,21 +340,15 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, return true; } } + return false; } static bool -is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, +is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, uint8 *addr, uint32 bytes) { - WASMSharedHeap *heap = NULL; - - if (module_inst_comm->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; - } - else if (module_inst_comm->module_type == Wasm_Module_AoT) { - // TODO - } + WASMSharedHeap *heap = get_shared_heap(module_inst); if (heap && addr >= heap->base_addr && addr + bytes <= heap->base_addr + heap->size @@ -349,156 +358,72 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, return false; } -static uint64 -shared_heap_addr_native_to_app(WASMModuleInstanceCommon *module_inst, - WASMMemoryInstance *memory, void *addr) -{ - WASMSharedHeap *heap = NULL; - - if (module_inst->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } - - if (!heap) { - LOG_WARNING("Wasm module doesn't attach to a shared heap"); - return 0; - } - if (!addr) { - LOG_WARNING("Invalid address"); - return 0; - } - - if (memory && memory->is_memory64) { - return heap->start_off_mem64 + ((uint8 *)addr - heap->base_addr); - } - else if (memory && !memory->is_memory64) { - return heap->start_off_mem32 + ((uint8 *)addr - heap->base_addr); - } - return 0; -} - -static void * -shared_heap_addr_app_to_native(WASMModuleInstanceCommon *module_inst, - WASMMemoryInstance *memory, uint64 ptr) -{ - void *addr = NULL; - WASMSharedHeap *heap = NULL; - - if (module_inst->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } - - if (!heap) { - LOG_WARNING("Wasm module doesn't attach to a shared heap"); - return NULL; - } - - if (!memory) { - LOG_WARNING("Wasm memory is not initialized"); - return NULL; - } - - if (memory->is_memory64) { - addr = heap->base_addr + (ptr - heap->start_off_mem64); - } - else { - addr = heap->base_addr + (ptr - heap->start_off_mem32); - } - - return addr; -} - -static uint64 -shared_heap_get_addr_start(WASMSharedHeap *heap, WASMMemoryInstance *memory) -{ - uint64 shared_heap_start = 0; - - if (!heap || !memory) { - LOG_ERROR("Invalid heap or memory"); - return 0; - } - - if (memory && !memory->is_memory64) { - shared_heap_start = heap->start_off_mem32; - } - else if (memory && memory->is_memory64) { - shared_heap_start = heap->start_off_mem64; - } - - return shared_heap_start; -} - uint64 wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, uint64_t size, void **p_native_addr) { - WASMSharedHeap *heap = NULL; - WASMMemoryInstance *memory = NULL; + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module_inst); + WASMSharedHeap *shared_heap = get_shared_heap(module_inst); - if (module_inst->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; - memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } + if (!memory || !shared_heap) + return 0; - if (heap) { - *p_native_addr = mem_allocator_malloc(heap->heap_handle, size); + *p_native_addr = mem_allocator_malloc(shared_heap->heap_handle, size); + if (!*p_native_addr) + return 0; - return shared_heap_addr_native_to_app(module_inst, memory, - *p_native_addr); - } - else { - LOG_WARNING("Wasm module doesn't attach to a shared heap"); - } - return 0; + if (memory->is_memory64) + return shared_heap->start_off_mem64 + + ((uint8 *)*p_native_addr - shared_heap->base_addr); + else + return shared_heap->start_off_mem32 + + ((uint8 *)*p_native_addr - shared_heap->base_addr); } void -wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr) +wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr) { - WASMSharedHeap *heap = NULL; - WASMMemoryInstance *memory = NULL; - void *addr = NULL; + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module_inst); + WASMSharedHeap *shared_heap = get_shared_heap(module_inst); + uint8 *addr = NULL; - if (module_inst->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; - memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } - - if (!heap) { - LOG_WARNING("Wasm module doesn't attach to a shared heap"); + if (!memory || !shared_heap) { return; } - addr = shared_heap_addr_app_to_native(module_inst, memory, ptr); - - if (heap) { - mem_allocator_free(heap->base_addr, addr); + if (memory->is_memory64) { + if (ptr < shared_heap->start_off_mem64) { /* ptr can not > UINT64_MAX */ + LOG_WARNING("The address to free isn't in shared heap"); + return; + } + addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem64); } + else { + if (ptr < shared_heap->start_off_mem32 || ptr > UINT32_MAX) { + LOG_WARNING("The address to free isn't in shared heap"); + return; + } + addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem32); + } + + mem_allocator_free(shared_heap->heap_handle, addr); } -#endif +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, const MemAllocOption *alloc_option) { bool ret = false; + #if WASM_ENABLE_SHARED_HEAP != 0 if (os_mutex_init(&shared_heap_list_lock)) { return false; } #endif + if (mem_alloc_type == Alloc_With_Pool) { ret = wasm_memory_init_with_pool(alloc_option->pool.heap_buf, alloc_option->pool.heap_size); @@ -516,6 +441,10 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, memory_mode = MEMORY_MODE_SYSTEM_ALLOCATOR; ret = true; } + else { + ret = false; + } + #if WASM_ENABLE_SHARED_HEAP != 0 if (!ret) { os_mutex_destroy(&shared_heap_list_lock); @@ -527,33 +456,27 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, #if WASM_ENABLE_SHARED_HEAP != 0 static void -wasm_runtime_shared_heap_destroy() +wasm_runtime_destroy_shared_heaps() { WASMSharedHeap *heap = shared_heap_list; WASMSharedHeap *cur; - int ret = 0; while (heap) { cur = heap; heap = heap->next; - ret = ret + mem_allocator_destroy(cur->heap_handle); + mem_allocator_destroy(cur->heap_handle); wasm_runtime_free(cur->heap_handle); wasm_munmap_linear_memory(cur->base_addr, cur->size, cur->size); wasm_runtime_free(cur); } - - if (ret != 0) { - LOG_ERROR("Memory leak detected in shared heap"); - } } #endif void wasm_runtime_memory_destroy(void) { - #if WASM_ENABLE_SHARED_HEAP != 0 - wasm_runtime_shared_heap_destroy(); + wasm_runtime_destroy_shared_heaps(); #endif if (memory_mode == MEMORY_MODE_POOL) { @@ -730,6 +653,13 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, goto fail; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_offset, size)) { + return true; + } +#endif + #if WASM_ENABLE_MEMORY64 != 0 if (memory_inst->is_memory64) max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; @@ -737,7 +667,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, /* boundary overflow check */ if (size > max_linear_memory_size || app_offset > max_linear_memory_size - size) { - goto shared_heap_bound_check; + goto fail; } SHARED_MEMORY_LOCK(memory_inst); @@ -749,13 +679,6 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); -shared_heap_bound_check: -#if WASM_ENABLE_SHARED_HEAP != 0 - if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_offset, size)) { - return true; - } -#endif fail: wasm_set_exception(module_inst, "out of bounds memory access"); return false; @@ -766,6 +689,7 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, uint64 app_str_offset) { WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; char *str, *str_end; @@ -776,22 +700,42 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, return true; } - if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, NULL, - &app_end_offset)) + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { goto fail; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_str_offset, 1)) { + WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + str = (char *)shared_heap->base_addr + + (memory_inst->is_memory64 + ? (app_str_offset - shared_heap->start_off_mem64) + : (app_str_offset - shared_heap->start_off_mem32)); + str_end = (char *)shared_heap->base_addr + shared_heap->size; + } + else +#endif + { + if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, + NULL, &app_end_offset)) + goto fail; #if WASM_ENABLE_MEMORY64 != 0 - if (module_inst->memories[0]->is_memory64) - max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; + if (memory_inst->is_memory64) + max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; #endif - /* boundary overflow check, max start offset can only be size - 1, while end - * offset can be size */ - if (app_str_offset >= max_linear_memory_size - || app_end_offset > max_linear_memory_size) - goto fail; + /* boundary overflow check, max start offset can be size - 1, while end + offset can be size */ + if (app_str_offset >= max_linear_memory_size + || app_end_offset > max_linear_memory_size) + goto fail; + + str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset); + str_end = str + (app_end_offset - app_str_offset); + } - str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset); - str_end = str + (app_end_offset - app_str_offset); while (str < str_end && *str != '\0') str++; if (str == str_end) @@ -833,6 +777,12 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, goto fail; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size)) { + return true; + } +#endif + SHARED_MEMORY_LOCK(memory_inst); if (memory_inst->memory_data <= addr @@ -841,13 +791,6 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, return true; } -#if WASM_ENABLE_SHARED_HEAP != 0 - else if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, - size)) { - SHARED_MEMORY_UNLOCK(memory_inst); - return true; - } -#endif SHARED_MEMORY_UNLOCK(memory_inst); fail: @@ -874,6 +817,23 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, return NULL; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_offset, 1)) { + WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + uint64 shared_heap_start = 0; + + if (memory_inst && !memory_inst->is_memory64) { + shared_heap_start = shared_heap->start_off_mem32; + } + else if (memory_inst && memory_inst->is_memory64) { + shared_heap_start = shared_heap->start_off_mem64; + } + + return shared_heap->base_addr + app_offset - shared_heap_start; + } +#endif + SHARED_MEMORY_LOCK(memory_inst); addr = memory_inst->memory_data + (uintptr_t)app_offset; @@ -884,19 +844,6 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); return addr; } -#if WASM_ENABLE_SHARED_HEAP != 0 - else if (is_app_addr_in_shared_heap(module_inst_comm, - memory_inst->is_memory64, - app_offset, 1)) { - uint64 heap_start = shared_heap_get_addr_start( - module_inst->e->shared_heap, memory_inst); - uint64 heap_offset = (uint64)app_offset - heap_start; - - addr = module_inst->e->shared_heap->base_addr + heap_offset; - SHARED_MEMORY_UNLOCK(memory_inst); - return addr; - } -#endif SHARED_MEMORY_UNLOCK(memory_inst); return NULL; } @@ -931,6 +878,22 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, return 0; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1)) { + WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + uint64 shared_heap_start = 0; + + if (memory_inst && !memory_inst->is_memory64) { + shared_heap_start = shared_heap->start_off_mem32; + } + else if (memory_inst && memory_inst->is_memory64) { + shared_heap_start = shared_heap->start_off_mem64; + } + + return shared_heap_start + (addr - shared_heap->base_addr); + } +#endif + SHARED_MEMORY_LOCK(memory_inst); if (bounds_checks) { @@ -940,17 +903,6 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); return ret; } - else { -#if WASM_ENABLE_SHARED_HEAP != 0 - uint64 shared_heap_start = shared_heap_get_addr_start( - module_inst->e->shared_heap, memory_inst); - ret = - (uint64)(addr - (uint8 *)module_inst->e->shared_heap->base_addr) - + shared_heap_start; - SHARED_MEMORY_UNLOCK(memory_inst); - return ret; -#endif - } } /* If bounds checks is disabled, return the offset directly */ else if (addr != NULL) { @@ -1039,6 +991,10 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst); uint8 *native_addr; bool bounds_checks; +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; + bool is_in_shared_heap = false; +#endif bh_assert(app_buf_addr <= UINTPTR_MAX && app_buf_size <= UINTPTR_MAX); @@ -1047,9 +1003,25 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, return false; } - native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, + memory_inst->is_memory64, app_buf_addr, + app_buf_size)) { + shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module_inst); + native_addr = shared_heap->base_addr + + (memory_inst->is_memory64 + ? (app_buf_addr - shared_heap->start_off_mem64) + : (app_buf_addr - shared_heap->start_off_mem32)); + is_in_shared_heap = true; + } + else +#endif + { + native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; + } - bounds_checks = is_bounds_checks_enabled((wasm_module_inst_t)module_inst); + bounds_checks = + is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst); if (!bounds_checks) { if (app_buf_addr == 0) { @@ -1058,6 +1030,24 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, goto success; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_in_shared_heap) { + const char *str, *str_end; + + /* The whole string must be in the linear memory */ + str = (const char *)native_addr; + str_end = (const char *)shared_heap->base_addr + shared_heap->size; + while (str < str_end && *str != '\0') + str++; + if (str == str_end) { + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; + } + else + goto success; + } +#endif + /* No need to check the app_offset and buf_size if memory access boundary check with hardware trap is enabled */ #ifndef OS_ENABLE_HW_BOUND_CHECK @@ -1205,7 +1195,7 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, #endif #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *heap; + WASMSharedHeap *shared_heap; #endif uint8 *memory_data_old, *memory_data_new, *heap_data_old; @@ -1241,16 +1231,20 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, total_size_new = num_bytes_per_page * (uint64)total_page_count; #if WASM_ENABLE_SHARED_HEAP != 0 - heap = module->e->shared_heap; - if (memory->is_memory64 && total_size_new > heap->start_off_mem64) { - LOG_WARNING("Linear memory address is overlapped with shared heap"); - ret = false; - goto return_func; - } - else if (!memory->is_memory64 && total_size_new > heap->start_off_mem32) { - LOG_WARNING("Linear memory address is overlapped with shared heap"); - ret = false; - goto return_func; + shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module); + if (shared_heap) { + if (memory->is_memory64 + && total_size_new > shared_heap->start_off_mem64) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } + else if (!memory->is_memory64 + && total_size_new > shared_heap->start_off_mem32) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } } #endif if (inc_page_count <= 0) diff --git a/core/iwasm/common/wasm_shared_memory.c b/core/iwasm/common/wasm_shared_memory.c index 9cfdd0926..e0d86d6f4 100644 --- a/core/iwasm/common/wasm_shared_memory.c +++ b/core/iwasm/common/wasm_shared_memory.c @@ -243,6 +243,29 @@ map_try_release_wait_info(HashMap *wait_hash_map, AtomicWaitInfo *wait_info, destroy_wait_info(wait_info); } +#if WASM_ENABLE_SHARED_HEAP != 0 +static bool +is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, + uint8 *addr, uint32 bytes) +{ + WASMSharedHeap *shared_heap = NULL; + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + shared_heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } +#endif + + return shared_heap && addr >= shared_heap->base_addr + && addr + bytes <= shared_heap->base_addr + shared_heap->size; +} +#endif + uint32 wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, uint64 expect, int64 timeout, bool wait64) @@ -271,9 +294,17 @@ wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, } shared_memory_lock(module_inst->memories[0]); - if ((uint8 *)address < module_inst->memories[0]->memory_data - || (uint8 *)address + (wait64 ? 8 : 4) - > module_inst->memories[0]->memory_data_end) { + if ( +#if WASM_ENABLE_SHARED_HEAP != 0 + /* not in shared heap */ + !is_native_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, + address, wait64 ? 8 : 4) + && +#endif + /* and not in linear memory */ + ((uint8 *)address < module_inst->memories[0]->memory_data + || (uint8 *)address + (wait64 ? 8 : 4) + > module_inst->memories[0]->memory_data_end)) { shared_memory_unlock(module_inst->memories[0]); wasm_runtime_set_exception(module, "out of bounds memory access"); return -1; @@ -397,6 +428,11 @@ wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, shared_memory_lock(module_inst->memories[0]); out_of_bounds = +#if WASM_ENABLE_SHARED_HEAP != 0 + /* not in shared heap */ + !is_native_addr_in_shared_heap(module, address, 4) && +#endif + /* and not in linear memory */ ((uint8 *)address < module_inst->memories[0]->memory_data || (uint8 *)address + 4 > module_inst->memories[0]->memory_data_end); shared_memory_unlock(module_inst->memories[0]); diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index f39ae3468..ecbe408ff 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -46,105 +46,89 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if WASM_ENABLE_MEMORY64 == 0 - -#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ - || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ - || WASM_ENABLE_BULK_MEMORY != 0 -#define GOTO_OUT_OF_BOUNDS goto out_of_bounds; -#else -#define GOTO_OUT_OF_BOUNDS -#endif - #if WASM_ENABLE_SHARED_HEAP != 0 -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ - do { \ - if (offset1 + bytes >= UINT32_MAX - module->shared_heap->size \ - && offset1 + bytes <= UINT32_MAX) { \ - uint64 heap_start = UINT32_MAX - module->shared_heap->size; \ - uint64 heap_offset = (uint64)offset1 - heap_start; \ - maddr = module->shared_heap->data + heap_offset; \ - } \ - else { \ - GOTO_OUT_OF_BOUNDS; \ - } \ - } while (0) +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 1) + +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) + +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else #else -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) #endif +#if WASM_ENABLE_MEMORY64 == 0 + #if (!defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0) #define CHECK_MEMORY_OVERFLOW(bytes) \ do { \ uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ /* If offset1 is in valid range, maddr must also \ be in valid range, no need to check it again. */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ + goto out_of_bounds; \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ /* App heap space is not valid space for \ bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ - } while (0) -#else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ - WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ -#define CHECK_MEMORY_OVERFLOW(bytes) \ - do { \ - uint64 offset1 = (uint64)offset + (uint64)addr; \ - maddr = memory->memory_data + offset1; \ + goto out_of_bounds; \ } while (0) -#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ - do { \ - maddr = memory->memory_data + (uint32)(start); \ - uint64 offset1 = start; \ +#else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ + WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ + +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ } while (0) + +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ + } while (0) + #endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ #else /* else of WASM_ENABLE_MEMORY64 == 0 */ -#if WASM_ENABLE_SHARED_HEAP != 0 -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ - do { \ - if (offset1 + bytes >= UINT64_MAX - module->shared_heap->size \ - && offset1 + bytes <= UINT64_MAX) { \ - uint64 heap_start = UINT64_MAX - module->shared_heap->size; \ - uint64 heap_offset = (uint64)offset1 - heap_start; \ - maddr = module->shared_heap->data + heap_offset; \ - } \ - else \ - goto out_of_bounds; \ +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + /* If memory64 is enabled, offset1, offset1 + bytes can overflow */ \ + if (disable_bounds_checks \ + || (offset1 >= offset && offset1 + bytes >= offset1 \ + && offset1 + bytes <= get_linear_mem_size())) \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ } while (0) -#else -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) goto out_of_bounds; -#endif -#define CHECK_MEMORY_OVERFLOW(bytes) \ - do { \ - uint64 offset1 = (uint64)offset + (uint64)addr; \ - /* If memory64 is enabled, offset1, offset1 + bytes can \ - * overflow */ \ - if (disable_bounds_checks \ - || (offset1 >= offset && offset1 + bytes >= offset1 \ - && offset1 + bytes <= get_linear_mem_size())) \ - maddr = memory->memory_data + offset1; \ - else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ - } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ uint64 offset1 = (uint64)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ /* If memory64 is enabled, offset1 + bytes can overflow */ \ if (disable_bounds_checks \ || (offset1 + bytes >= offset1 \ @@ -153,7 +137,7 @@ typedef float64 CellType_F64; bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ + goto out_of_bounds; \ } while (0) #endif /* end of WASM_ENABLE_MEMORY64 == 0 */ @@ -1648,6 +1632,22 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (memory) is_memory64 = memory->is_memory64; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap = module->e->shared_heap; + uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; +#if WASM_ENABLE_MEMORY64 != 0 + uint64 shared_heap_start_off = + shared_heap ? (is_memory64 ? shared_heap->start_off_mem64 + : shared_heap->start_off_mem32) + : 0; + uint64 shared_heap_end_off = + shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0; +#else + uint64 shared_heap_start_off = + shared_heap ? shared_heap->start_off_mem32 : 0; + uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0; +#endif +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_MULTI_MEMORY != 0 uint32 memidx = 0; uint32 memidx_cached = (uint32)-1; @@ -3498,8 +3498,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, str_obj = (WASMString)wasm_stringref_obj_get_value( stringref_obj); - memory_inst = module->memories[mem_idx]; - maddr = memory_inst->memory_data + addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } if (opcode == WASM_OP_STRING_ENCODE_WTF16) { flag = WTF16; @@ -3666,8 +3673,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, addr = POP_I32(); stringview_wtf8_obj = POP_REF(); - memory_inst = module->memories[mem_idx]; - maddr = memory_inst->memory_data + addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } bytes_written = wasm_string_encode( (WASMString)wasm_stringview_wtf8_obj_get_value( @@ -5694,9 +5708,18 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr); #else - if ((uint64)(uint32)addr + bytes > linear_mem_size) - goto out_of_bounds; - maddr = memory->memory_data + (uint32)addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)addr, + bytes)) + shared_heap_addr_app_to_native((uint64)(uint32)addr, + maddr); + else +#endif + { + if ((uint64)(uint32)addr + bytes > linear_mem_size) + goto out_of_bounds; + maddr = memory->memory_data + (uint32)addr; + } #endif if (bh_bitmap_get_bit(module->e->common.data_dropped, @@ -5746,15 +5769,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_THREAD_MGR != 0 linear_mem_size = get_linear_mem_size(); #endif + + dlen = linear_mem_size - dst; + /* dst boundary check */ #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); -#else - if ((uint64)dst + len > linear_mem_size) - goto out_of_bounds; - mdst = memory->memory_data + dst; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) + dlen = shared_heap_end_off - dst + 1; #endif - dlen = linear_mem_size - dst; +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) { + shared_heap_addr_app_to_native((uint64)dst, mdst); + dlen = shared_heap_end_off - dst + 1; + } + else +#endif + { + if ((uint64)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + dst; + } +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_MULTI_MEMORY != 0 /* src memidx */ @@ -5770,9 +5808,16 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc); #else - if ((uint64)src + len > linear_mem_size) - goto out_of_bounds; - msrc = memory->memory_data + src; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)src, len)) + shared_heap_addr_app_to_native((uint64)src, msrc); + else +#endif + { + if ((uint64)src + len > linear_mem_size) + goto out_of_bounds; + msrc = memory->memory_data + src; + } #endif #if WASM_ENABLE_MEMORY64 == 0 @@ -5809,9 +5854,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #else - if ((uint64)(uint32)dst + len > linear_mem_size) - goto out_of_bounds; - mdst = memory->memory_data + (uint32)dst; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)dst, len)) + shared_heap_addr_app_to_native((uint64)(uint32)dst, + mdst); + else +#endif + { + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; + } #endif memset(mdst, fill_val, len); diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 84170f213..82b6ff71b 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -37,29 +37,20 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ - || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ - || WASM_ENABLE_BULK_MEMORY != 0 -#define GOTO_OUT_OF_BOUNDS goto out_of_bounds; -#else -#define GOTO_OUT_OF_BOUNDS -#endif - #if WASM_ENABLE_SHARED_HEAP != 0 -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ - do { \ - if (offset1 + bytes >= UINT32_MAX - module->e->shared_heap->size \ - && offset1 + bytes <= UINT32_MAX) { \ - uint64 heap_start = UINT32_MAX - module->e->shared_heap->size; \ - uint64 heap_offset = (uint64)offset1 - heap_start; \ - maddr = module->e->shared_heap->base_addr + heap_offset; \ - } \ - else { \ - GOTO_OUT_OF_BOUNDS; \ - } \ - } while (0) +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 1) + +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) + +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else #else -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) #endif #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ @@ -67,35 +58,39 @@ typedef float64 CellType_F64; #define CHECK_MEMORY_OVERFLOW(bytes) \ do { \ uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ /* If offset1 is in valid range, maddr must also \ be in valid range, no need to check it again. */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ + goto out_of_bounds; \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ /* App heap space is not valid space for \ bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ + goto out_of_bounds; \ } while (0) #else -#define CHECK_MEMORY_OVERFLOW(bytes) \ - do { \ - uint64 offset1 = (uint64)offset + (uint64)addr; \ - maddr = memory->memory_data + offset1; \ +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ } while (0) -#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ - do { \ - maddr = memory->memory_data + (uint32)(start); \ - uint64 offset1 = start; \ +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ } while (0) #endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ @@ -1542,6 +1537,24 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 bool is_return_call = false; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap = module->e ? module->e->shared_heap : NULL; + uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; + /* +#if WASM_ENABLE_MEMORY64 != 0 + uint64 shared_heap_start_off = + shared_heap ? (is_memory64 ? shared_heap->start_off_mem64 + : shared_heap->start_off_mem32) + : 0; + uint64 shared_heap_end_off = + shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0; +#else + */ /* TODO: uncomment the code when memory64 is enabled for fast-interp */ + uint64 shared_heap_start_off = + shared_heap ? shared_heap->start_off_mem32 : 0; + uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0; +/* #endif */ +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_LABELS_AS_VALUES != 0 #define HANDLE_OPCODE(op) &&HANDLE_##op @@ -2857,8 +2870,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, str_obj = (WASMString)wasm_stringref_obj_get_value( stringref_obj); - memory_inst = module->memories[mem_idx]; - maddr = memory_inst->memory_data + addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } if (opcode == WASM_OP_STRING_ENCODE_WTF16) { flag = WTF16; @@ -3025,8 +3045,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, addr = POP_I32(); stringview_wtf8_obj = POP_REF(); - memory_inst = module->memories[mem_idx]; - maddr = memory_inst->memory_data + addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } bytes_written = wasm_string_encode( (WASMString)wasm_stringview_wtf8_obj_get_value( @@ -5011,9 +5038,18 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr); #else - if ((uint64)(uint32)addr + bytes > linear_mem_size) - goto out_of_bounds; - maddr = memory->memory_data + (uint32)addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)addr, + bytes)) + shared_heap_addr_app_to_native((uint64)(uint32)addr, + maddr); + else +#endif + { + if ((uint64)(uint32)addr + bytes > linear_mem_size) + goto out_of_bounds; + maddr = memory->memory_data + (uint32)addr; + } #endif if (bh_bitmap_get_bit(module->e->common.data_dropped, segment)) { @@ -5046,6 +5082,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, { uint32 dst, src, len; uint8 *mdst, *msrc; + uint64 dlen; len = POP_I32(); src = POP_I32(); @@ -5055,22 +5092,43 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, linear_mem_size = get_linear_mem_size(); #endif + dlen = linear_mem_size - dst; + #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc); CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); -#else - if ((uint64)(uint32)src + len > linear_mem_size) - goto out_of_bounds; - msrc = memory->memory_data + (uint32)src; - - if ((uint64)(uint32)dst + len > linear_mem_size) - goto out_of_bounds; - mdst = memory->memory_data + (uint32)dst; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) + dlen = shared_heap_end_off - dst + 1; #endif +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)src, len)) + shared_heap_addr_app_to_native((uint64)src, msrc); + else +#endif + { + if ((uint64)(uint32)src + len > linear_mem_size) + goto out_of_bounds; + msrc = memory->memory_data + (uint32)src; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) { + shared_heap_addr_app_to_native((uint64)dst, mdst); + dlen = shared_heap_end_off - dst + 1; + } + else +#endif + { + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; + } +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ /* allowing the destination and source to overlap */ - bh_memmove_s(mdst, (uint32)(linear_mem_size - dst), - msrc, len); + bh_memmove_s(mdst, (uint32)dlen, msrc, len); break; } case WASM_OP_MEMORY_FILL: @@ -5089,9 +5147,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #else - if ((uint64)(uint32)dst + len > linear_mem_size) - goto out_of_bounds; - mdst = memory->memory_data + (uint32)dst; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)dst, len)) + shared_heap_addr_app_to_native((uint64)(uint32)dst, + mdst); + else +#endif + { + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; + } #endif memset(mdst, fill_val, len); diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 3fa3682c0..3c4e5faa7 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -96,8 +96,8 @@ typedef union { typedef struct WASMSharedHeap { struct WASMSharedHeap *next; void *heap_handle; - uint8_t *base_addr; - uint32_t size; + uint8 *base_addr; + uint32 size; uint64 start_off_mem64; uint64 start_off_mem32; } WASMSharedHeap; diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index 27ff1441f..46a1bb329 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -1449,10 +1449,19 @@ wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst, bh_assert(cluster); os_mutex_lock(&cluster->lock); + /* Try attaching shared heap to this module instance first + to ensure that we can attach it to all other instances. */ + if (!wasm_runtime_attach_shared_heap_internal(module_inst, heap)) { + os_mutex_unlock(&cluster->lock); + return false; + } + /* Detach the shared heap so it can be attached again. */ + wasm_runtime_detach_shared_heap_internal(module_inst); traverse_list(&cluster->exec_env_list, attach_shared_heap_visitor, heap); os_mutex_unlock(&cluster->lock); } + return true; } From 4dacef2d600d3476cf8ec8dbe962cc540fbb258d Mon Sep 17 00:00:00 2001 From: WenLY1 <130950131+WenLY1@users.noreply.github.com> Date: Fri, 20 Sep 2024 14:24:38 +0800 Subject: [PATCH 3/6] shared heap: Fix some issues and add basic unit test case (#3801) Fix some issues and add basic unit test case for shared heap feature. Signed-off-by: wenlingyun1 --- core/iwasm/common/wasm_memory.c | 47 +++--- core/iwasm/common/wasm_memory.h | 3 +- core/iwasm/common/wasm_runtime_common.c | 39 ++--- core/iwasm/include/wasm_export.h | 30 ++-- core/iwasm/interpreter/wasm_runtime.h | 2 +- .../shared-heap/shared_heap_wrapper.c | 4 +- tests/unit/CMakeLists.txt | 1 + tests/unit/shared-heap/CMakeLists.txt | 59 ++++++++ tests/unit/shared-heap/shared_heap_test.cc | 142 ++++++++++++++++++ .../unit/shared-heap/wasm-apps/CMakeLists.txt | 39 +++++ tests/unit/shared-heap/wasm-apps/test.c | 22 +++ 11 files changed, 332 insertions(+), 56 deletions(-) create mode 100644 tests/unit/shared-heap/CMakeLists.txt create mode 100644 tests/unit/shared-heap/shared_heap_test.cc create mode 100644 tests/unit/shared-heap/wasm-apps/CMakeLists.txt create mode 100644 tests/unit/shared-heap/wasm-apps/test.c diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 48c87e766..fe6b7a074 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -148,22 +148,13 @@ static void wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size); -static void -set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) -{ - if (error_buf != NULL) { - snprintf(error_buf, error_buf_size, - "Operation of shared heap failed: %s", string); - } -} - static void * -runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size) +runtime_malloc(uint64 size) { void *mem; if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { - set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + LOG_WARNING("Allocate memory failed"); return NULL; } @@ -172,27 +163,32 @@ runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size) } WASMSharedHeap * -wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, - uint32 error_buf_size) +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) { uint64 heap_struct_size = sizeof(WASMSharedHeap); uint32 size = init_args->size; WASMSharedHeap *heap; - if (!(heap = runtime_malloc(heap_struct_size, error_buf, error_buf_size))) { + if (size == 0) { goto fail1; } + + if (!(heap = runtime_malloc(heap_struct_size))) { + goto fail1; + } + if (!(heap->heap_handle = - runtime_malloc(mem_allocator_get_heap_struct_size(), error_buf, - error_buf_size))) { + runtime_malloc(mem_allocator_get_heap_struct_size()))) { goto fail2; } + + size = align_uint(size, os_getpagesize()); + heap->size = size; heap->start_off_mem64 = UINT64_MAX - heap->size + 1; heap->start_off_mem32 = UINT32_MAX - heap->size + 1; - size = align_uint(size, os_getpagesize()); if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { - set_error_buf(error_buf, error_buf_size, "invalid size of shared heap"); + LOG_WARNING("Invalid size of shared heap"); goto fail3; } @@ -201,7 +197,7 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, } if (!mem_allocator_create_with_struct_and_pool( heap->heap_handle, heap_struct_size, heap->base_addr, size)) { - set_error_buf(error_buf, error_buf_size, "init share heap failed"); + LOG_WARNING("init share heap failed"); goto fail4; } @@ -365,20 +361,25 @@ wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, WASMMemoryInstance *memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); WASMSharedHeap *shared_heap = get_shared_heap(module_inst); + void *native_addr = NULL; if (!memory || !shared_heap) return 0; - *p_native_addr = mem_allocator_malloc(shared_heap->heap_handle, size); - if (!*p_native_addr) + native_addr = mem_allocator_malloc(shared_heap->heap_handle, size); + if (!native_addr) return 0; + if (p_native_addr) { + *p_native_addr = native_addr; + } + if (memory->is_memory64) return shared_heap->start_off_mem64 - + ((uint8 *)*p_native_addr - shared_heap->base_addr); + + ((uint8 *)native_addr - shared_heap->base_addr); else return shared_heap->start_off_mem32 - + ((uint8 *)*p_native_addr - shared_heap->base_addr); + + ((uint8 *)native_addr - shared_heap->base_addr); } void diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index be89772e3..cc6418e84 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -43,8 +43,7 @@ SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size) #if WASM_ENABLE_SHARED_HEAP != 0 WASMSharedHeap * -wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, - uint32 error_buf_size); +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); bool wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 314dc7ddb..c827b28fe 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -4498,6 +4498,11 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, uint32 *argv, uint32 argc, uint32 *argv_ret) { WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); +#if WASM_ENABLE_MEMORY64 != 0 + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module); + bool is_memory64 = memory ? memory->is_memory64 : false; +#endif typedef void (*NativeRawFuncPtr)(WASMExecEnv *, uint64 *); NativeRawFuncPtr invoke_native_raw = (NativeRawFuncPtr)func_ptr; uint64 argv_buf[16] = { 0 }, *argv1 = argv_buf, *argv_dst, size, arg_i64; @@ -4525,11 +4530,11 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, #endif { *(uint32 *)argv_dst = arg_i32 = *argv_src++; - /* TODO: memory64 if future there is a way for supporting - * wasm64 and wasm32 in libc at the same time, remove the - * macro control */ -#if WASM_ENABLE_MEMORY64 == 0 - if (signature) { + if (signature +#if WASM_ENABLE_MEMORY64 != 0 + && !is_memory64 +#endif + ) { if (signature[i + 1] == '*') { /* param is a pointer */ if (signature[i + 2] == '~') @@ -4558,7 +4563,6 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, module, (uint64)arg_i32); } } -#endif break; } case VALUE_TYPE_I64: @@ -4568,7 +4572,7 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, GET_I64_FROM_ADDR(argv_src)); argv_src += 2; arg_i64 = *argv_dst; - if (signature) { + if (signature && is_memory64) { /* TODO: memory64 pointer with length need a new symbol * to represent type i64, with '~' still represent i32 * length */ @@ -4729,9 +4733,6 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, fail: if (argv1 != argv_buf) wasm_runtime_free(argv1); -#if WASM_ENABLE_MEMORY64 == 0 - (void)arg_i64; -#endif return ret; } @@ -5655,6 +5656,11 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, uint32 *argv_ret) { WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); +#if WASM_ENABLE_MEMORY64 != 0 + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module); + bool is_memory64 = memory ? memory->is_memory64 : false; +#endif uint64 argv_buf[32] = { 0 }, *argv1 = argv_buf, *ints, *stacks, size, arg_i64; uint32 *argv_src = argv, i, argc1, n_ints = 0, n_stacks = 0; @@ -5720,11 +5726,11 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, { arg_i32 = *argv_src++; arg_i64 = arg_i32; - /* TODO: memory64 if future there is a way for supporting - * wasm64 and wasm32 in libc at the same time, remove the - * macro control */ -#if WASM_ENABLE_MEMORY64 == 0 - if (signature) { + if (signature +#if WASM_ENABLE_MEMORY64 != 0 + && !is_memory64 +#endif + ) { if (signature[i + 1] == '*') { /* param is a pointer */ if (signature[i + 2] == '~') @@ -5751,7 +5757,6 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, module, (uint64)arg_i32); } } -#endif if (n_ints < MAX_REG_INTS) ints[n_ints++] = arg_i64; else @@ -5763,7 +5768,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, { arg_i64 = GET_I64_FROM_ADDR(argv_src); argv_src += 2; - if (signature) { + if (signature && is_memory64) { /* TODO: memory64 pointer with length need a new symbol * to represent type i64, with '~' still represent i32 * length */ diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 5ce617ef1..ffdafcef0 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -323,11 +323,9 @@ typedef enum { WASM_LOG_LEVEL_VERBOSE = 4 } log_level_t; -#if WASM_ENABLE_SHARED_HEAP != 0 typedef struct SharedHeapInitArgs { - uint32 size; + uint32_t size; } SharedHeapInitArgs; -#endif /** * Initialize the WASM runtime environment, and also initialize @@ -2119,21 +2117,21 @@ wasm_runtime_detect_native_stack_overflow_size(wasm_exec_env_t exec_env, WASM_RUNTIME_API_EXTERN bool wasm_runtime_is_underlying_binary_freeable(const wasm_module_t module); -#if WASM_ENABLE_SHARED_HEAP != 0 /** * Create a shared heap + * * @param init_args the initialization arguments - * @param error_buf buffer to output the error info if failed - * @param error_buf_size the size of the error buffer + * @return the shared heap created */ WASM_RUNTIME_API_EXTERN wasm_shared_heap_t -wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, - uint32 error_buf_size); +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); /** * Attach a shared heap to a module instance + * * @param module_inst the module instance * @param shared_heap the shared heap + * @return true if success, false if failed */ WASM_RUNTIME_API_EXTERN bool wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst, @@ -2141,6 +2139,7 @@ wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst, /** * Detach a shared heap from a module instance + * * @param module_inst the module instance */ WASM_RUNTIME_API_EXTERN void @@ -2148,22 +2147,29 @@ wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst); /** * Allocate memory from a shared heap + * * @param module_inst the module instance * @param size required memory size * @param p_native_addr native address of allocated memory + * + * @return return the allocated memory address, which re-uses part of the wasm + * address space and is in the range of [UINT32 - shared_heap_size + 1, UINT32] + * (when the wasm memory is 32-bit) or [UINT64 - shared_heap_size + 1, UINT64] + * (when the wasm memory is 64-bit). Note that it is not an absolute address. + * Return non-zero if success, zero if failed. */ -WASM_RUNTIME_API_EXTERN uint64 -wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64 size, +WASM_RUNTIME_API_EXTERN uint64_t +wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64_t size, void **p_native_addr); /** * Free the memory allocated from shared heap + * * @param module_inst the module instance * @param ptr the offset in wasm app */ WASM_RUNTIME_API_EXTERN void -wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr); -#endif +wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64_t ptr); #ifdef __cplusplus } diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 3c4e5faa7..d4905a10d 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -97,7 +97,7 @@ typedef struct WASMSharedHeap { struct WASMSharedHeap *next; void *heap_handle; uint8 *base_addr; - uint32 size; + uint64 size; uint64 start_off_mem64; uint64 start_off_mem32; } WASMSharedHeap; diff --git a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c index 164568ac2..978687f3c 100644 --- a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c +++ b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c @@ -31,8 +31,10 @@ shared_free_wrapper(wasm_exec_env_t exec_env, void *ptr) { wasm_module_inst_t module_inst = get_module_inst(exec_env); - if (!validate_native_addr(ptr, (uint64)sizeof(uint32))) + if (!validate_native_addr(ptr, (uint64)sizeof(uintptr_t))) { + LOG_WARNING("Invalid app address"); return; + } module_shared_free(addr_native_to_app(ptr)); } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 40c9bb6ae..9f7a69229 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -50,3 +50,4 @@ add_subdirectory(linux-perf) add_subdirectory(gc) add_subdirectory(memory64) add_subdirectory(tid-allocator) +add_subdirectory(shared-heap) \ No newline at end of file diff --git a/tests/unit/shared-heap/CMakeLists.txt b/tests/unit/shared-heap/CMakeLists.txt new file mode 100644 index 000000000..6baf420f8 --- /dev/null +++ b/tests/unit/shared-heap/CMakeLists.txt @@ -0,0 +1,59 @@ +# Copyright (C) 2024 Xiaomi Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) + +project(test-shared-heap) + +add_definitions(-DRUN_ON_LINUX) + +set(WAMR_BUILD_APP_FRAMEWORK 0) +set(WAMR_BUILD_AOT 0) +set(WAMR_BUILD_INTERP 1) +set(WAMR_BUILD_FAST_INTERP 1) +set(WAMR_BUILD_JIT 0) +set(WAMR_BUILD_MEMORY64 1) +set(WAMR_BUILD_SHARED_HEAP 1) + +# Compile wasm modules +add_subdirectory(wasm-apps) + +# if only load this CMake other than load it as subdirectory +include(../unit_common.cmake) + +set(LLVM_SRC_ROOT "${WAMR_ROOT_DIR}/core/deps/llvm") + +if (NOT EXISTS "${LLVM_SRC_ROOT}/build") + message(FATAL_ERROR "Cannot find LLVM dir: ${LLVM_SRC_ROOT}/build") +endif () + +set(CMAKE_PREFIX_PATH "${LLVM_SRC_ROOT}/build;${CMAKE_PREFIX_PATH}") +find_package(LLVM REQUIRED CONFIG) +include_directories(${LLVM_INCLUDE_DIRS}) +add_definitions(${LLVM_DEFINITIONS}) +message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") +message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") + +include(${IWASM_DIR}/compilation/iwasm_compl.cmake) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +file(GLOB_RECURSE source_all ${CMAKE_CURRENT_SOURCE_DIR}/*.cc) + +set(UNIT_SOURCE ${source_all}) + +aux_source_directory(. SRC_LIST) + +set(unit_test_sources + ${UNIT_SOURCE} + ${WAMR_RUNTIME_LIB_SOURCE} + ${UNCOMMON_SHARED_SOURCE} + ${SRC_LIST} + ) + +# Now simply link against gtest or gtest_main as needed. Eg +add_executable(shared_heap_test ${unit_test_sources}) + +target_link_libraries(shared_heap_test ${LLVM_AVAILABLE_LIBS} gtest_main) + +gtest_discover_tests(shared_heap_test) \ No newline at end of file diff --git a/tests/unit/shared-heap/shared_heap_test.cc b/tests/unit/shared-heap/shared_heap_test.cc new file mode 100644 index 000000000..5e45d3111 --- /dev/null +++ b/tests/unit/shared-heap/shared_heap_test.cc @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2024 Xiaomi Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "test_helper.h" +#include "gtest/gtest.h" + +#include "bh_read_file.h" +#include "wasm_runtime_common.h" + +class shared_heap_test : public testing::Test +{ + protected: + virtual void SetUp() {} + static void SetUpTestCase() {} + virtual void TearDown() {} + WAMRRuntimeRAII<512 * 1024> runtime; +}; + +struct ret_env { + wasm_exec_env_t exec_env; + wasm_module_t wasm_module; + wasm_module_inst_t wasm_module_inst; + unsigned char *wasm_file_buf; + char error_buf[128]; +}; + +struct ret_env +load_wasm(char *wasm_file_tested, unsigned int app_heap_size) +{ + std::string wasm_mem_page = wasm_file_tested; + const char *wasm_file = strdup(wasm_mem_page.c_str()); + wasm_module_inst_t wasm_module_inst = nullptr; + wasm_module_t wasm_module = nullptr; + wasm_exec_env_t exec_env = nullptr; + unsigned char *wasm_file_buf = nullptr; + unsigned int wasm_file_size = 0; + unsigned int stack_size = 16 * 1024, heap_size = app_heap_size; + char error_buf[128] = { 0 }; + struct ret_env ret_module_env; + + memset(ret_module_env.error_buf, 0, 128); + wasm_file_buf = + (unsigned char *)bh_read_file_to_buffer(wasm_file, &wasm_file_size); + if (!wasm_file_buf) { + goto fail; + } + + wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, error_buf, + sizeof(error_buf)); + if (!wasm_module) { + memcpy(ret_module_env.error_buf, error_buf, 128); + goto fail; + } + + wasm_module_inst = wasm_runtime_instantiate( + wasm_module, stack_size, heap_size, error_buf, sizeof(error_buf)); + if (!wasm_module_inst) { + memcpy(ret_module_env.error_buf, error_buf, 128); + goto fail; + } + + exec_env = wasm_runtime_create_exec_env(wasm_module_inst, stack_size); + +fail: + ret_module_env.exec_env = exec_env; + ret_module_env.wasm_module = wasm_module; + ret_module_env.wasm_module_inst = wasm_module_inst; + ret_module_env.wasm_file_buf = wasm_file_buf; + + return ret_module_env; +} + +void +destroy_module_env(struct ret_env module_env) +{ + if (module_env.exec_env) { + wasm_runtime_destroy_exec_env(module_env.exec_env); + } + + if (module_env.wasm_module_inst) { + wasm_runtime_deinstantiate(module_env.wasm_module_inst); + } + + if (module_env.wasm_module) { + wasm_runtime_unload(module_env.wasm_module); + } + + if (module_env.wasm_file_buf) { + wasm_runtime_free(module_env.wasm_file_buf); + } +} + +TEST_F(shared_heap_test, test_shared_heap) +{ + struct ret_env tmp_module_env; + WASMFunctionInstanceCommon *func_test = nullptr; + bool ret = false; + uint32 argv[1] = { 65535 }; + const char *exception = nullptr; + SharedHeapInitArgs args; + WASMSharedHeap *shared_heap = nullptr; + + args.size = 1024; + shared_heap = wasm_runtime_create_shared_heap(&args); + tmp_module_env = load_wasm((char *)"test.wasm", 0); + + if (!shared_heap) { + printf("Failed to create shared heap\n"); + goto test_failed; + } + if (!wasm_runtime_attach_shared_heap(tmp_module_env.wasm_module_inst, shared_heap)) { + printf("Failed to attach shared heap\n"); + goto test_failed; + } + func_test = wasm_runtime_lookup_function( + tmp_module_env.wasm_module_inst, "test"); + if (!func_test) { + printf("\nFailed to wasm_runtime_lookup_function!\n"); + goto test_failed; + } + + ret = + wasm_runtime_call_wasm(tmp_module_env.exec_env, func_test, 1, argv); + if (!ret) { + printf("\nFailed to wasm_runtime_call_wasm!\n"); + const char *s = wasm_runtime_get_exception(tmp_module_env.wasm_module_inst); + printf("exception: %s\n", s); + goto test_failed; + } + + wasm_runtime_detach_shared_heap(tmp_module_env.wasm_module_inst); + + EXPECT_EQ(10, argv[0]); + + destroy_module_env(tmp_module_env); + return; +test_failed: + destroy_module_env(tmp_module_env); + EXPECT_EQ(1, 0); +} diff --git a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt new file mode 100644 index 000000000..3627a2c14 --- /dev/null +++ b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt @@ -0,0 +1,39 @@ +# Copyright (C) 2024 Xiaomi Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(wasm-apps) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../..) + +set(CMAKE_SYSTEM_PROCESSOR wasm32) +set(CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set(WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set(CMAKE_C_FLAGS "-nostdlib -pthread -Qunused-arguments") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -z stack-size=8192 -nostdlib -O0") +set(CMAKE_C_COMPILER_TARGET "wasm32") +set(CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +set(DEFINED_SYMBOLS + "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") + +set(CMAKE_EXE_LINKER_FLAGS + "-Wl,--no-entry \ + -Wl,--initial-memory=65536 \ + -Wl,--export-all \ + -Wl,--allow-undefined" + ) + +add_executable(test.wasm test.c) +target_link_libraries(test.wasm) + +add_custom_command(TARGET test.wasm POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/test.wasm + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMENT "Copy test.wasm to the same directory of google test" + ) \ No newline at end of file diff --git a/tests/unit/shared-heap/wasm-apps/test.c b/tests/unit/shared-heap/wasm-apps/test.c new file mode 100644 index 000000000..ce59903c6 --- /dev/null +++ b/tests/unit/shared-heap/wasm-apps/test.c @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2024 Xiaomi Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +extern void * +shared_malloc(int size); +extern void +shared_free(void *offset); + +int +test() +{ + int *ptr = (int *)shared_malloc(10); + + *ptr = 10; + int a = *ptr; + shared_free(ptr); + return a; +} \ No newline at end of file From c4aa1deda50d9428863fe73104bdfa6dd018129f Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Fri, 20 Sep 2024 16:13:20 +0800 Subject: [PATCH 4/6] Add shared heap sample (#3806) --- .github/scripts/codeql_buildscript.sh | 30 ++ .../compilation_on_android_ubuntu.yml | 8 + .github/workflows/compilation_on_macos.yml | 8 + .github/workflows/nightly_run.yml | 8 + build-scripts/config_common.cmake | 2 +- .../shared-heap/shared_heap_wrapper.c | 2 +- samples/shared-heap/CMakeLists.txt | 92 ++++++ samples/shared-heap/src/main.c | 309 ++++++++++++++++++ samples/shared-heap/wasm-apps/CMakeLists.txt | 43 +++ samples/shared-heap/wasm-apps/test1.c | 30 ++ samples/shared-heap/wasm-apps/test2.c | 18 + 11 files changed, 548 insertions(+), 2 deletions(-) create mode 100644 samples/shared-heap/CMakeLists.txt create mode 100644 samples/shared-heap/src/main.c create mode 100644 samples/shared-heap/wasm-apps/CMakeLists.txt create mode 100644 samples/shared-heap/wasm-apps/test1.c create mode 100644 samples/shared-heap/wasm-apps/test2.c diff --git a/.github/scripts/codeql_buildscript.sh b/.github/scripts/codeql_buildscript.sh index 3e523775f..f1c412a82 100755 --- a/.github/scripts/codeql_buildscript.sh +++ b/.github/scripts/codeql_buildscript.sh @@ -126,6 +126,16 @@ if [[ $? != 0 ]]; then exit 1; fi +# build iwasm with multi-memory enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MULTI_MEMORY=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with multi-memory enabled!" + exit 1; +fi + # build iwasm with hardware boundary check disabled cd ${WAMR_DIR}/product-mini/platforms/linux rm -rf build && mkdir build && cd build @@ -280,3 +290,23 @@ if [[ $? != 0 ]]; then echo "Failed to build iwasm with linux perf support enabled!" exit 1; fi + +# build iwasm with shared heap enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_SHARED_HEAP=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with shared heap enabled!" + exit 1; +fi + +# build iwasm with dynamic aot debug enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_DYNAMIC_AOT_DEBUG=1 +make -j +if [[ $? != 0 ]]; + echo "Failed to build iwasm dynamic aot debug enabled!" + exit 1; +fi diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 249a6f276..27592fde6 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -578,6 +578,14 @@ jobs: ./run.sh test1 ./run.sh test2 + - name: Build Sample [shared-heap] + run: | + cd samples/shared-heap + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./shared_heap_test + test: needs: [ diff --git a/.github/workflows/compilation_on_macos.yml b/.github/workflows/compilation_on_macos.yml index 3b92f4525..f92bba62f 100644 --- a/.github/workflows/compilation_on_macos.yml +++ b/.github/workflows/compilation_on_macos.yml @@ -386,3 +386,11 @@ jobs: ./build.sh ./run.sh test1 ./run.sh test2 + + - name: Build Sample [shared-heap] + run: | + cd samples/shared-heap + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./shared_heap_test diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index f39085d09..832c34d70 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -593,6 +593,14 @@ jobs: exit $? working-directory: ./wamr-app-framework/samples/simple + - name: Build Sample [shared-heap] + run: | + cd samples/shared-heap + mkdir build && cd build + cmake .. + cmake --build . --config Debug --parallel 4 + ./shared_heap_test + test: needs: [ diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 5af944e46..f91fbcdb0 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -498,7 +498,7 @@ if (WAMR_BUILD_MODULE_INST_CONTEXT EQUAL 1) message (" Module instance context enabled") endif () if (WAMR_BUILD_GC_HEAP_VERIFY EQUAL 1) - add_definitions (-DWASM_ENABLE_GC_VERIFY=1) + add_definitions (-DBH_ENABLE_GC_VERIFY=1) message (" GC heap verification enabled") endif () if ("$ENV{COLLECT_CODE_COVERAGE}" STREQUAL "1" OR COLLECT_CODE_COVERAGE EQUAL 1) diff --git a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c index 978687f3c..76e662dac 100644 --- a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c +++ b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c @@ -54,4 +54,4 @@ get_lib_shared_heap_export_apis(NativeSymbol **p_shared_heap_apis) { *p_shared_heap_apis = native_symbols_shared_heap; return sizeof(native_symbols_shared_heap) / sizeof(NativeSymbol); -} \ No newline at end of file +} diff --git a/samples/shared-heap/CMakeLists.txt b/samples/shared-heap/CMakeLists.txt new file mode 100644 index 000000000..382ddd546 --- /dev/null +++ b/samples/shared-heap/CMakeLists.txt @@ -0,0 +1,92 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + project (shared_heap_test) +else() + project (shared_heap_test C ASM) +endif() + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" + +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Debug) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) +set (WAMR_BUILD_LIBC_BUILTIN 1) +set (WAMR_BUILD_LIBC_WASI 0) +set (WAMR_BUILD_SHARED_HEAP 1) +set (WAMR_BUILD_GC_HEAP_VERIFY 1) + +if (NOT MSVC) + # linker flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif () + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +################ application related ################ +include_directories(${CMAKE_CURRENT_LIST_DIR}/src) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (shared_heap_test src/main.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (shared_heap_test PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (APPLE) + target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread) +else () + target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread -lrt) +endif () + +add_subdirectory(wasm-apps) diff --git a/samples/shared-heap/src/main.c b/samples/shared-heap/src/main.c new file mode 100644 index 000000000..416652a3e --- /dev/null +++ b/samples/shared-heap/src/main.c @@ -0,0 +1,309 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_platform.h" +#include "bh_read_file.h" + +typedef struct thread_arg { + bh_queue *queue; + wasm_module_inst_t module_inst; +} thread_arg; + +static void * +thread1_callback(void *arg) +{ + thread_arg *targ = arg; + wasm_module_inst_t module_inst = targ->module_inst; + bh_queue *queue = targ->queue; + wasm_exec_env_t exec_env; + wasm_function_inst_t my_shared_malloc_func; + wasm_function_inst_t my_shared_free_func; + uint32 i, argv[2]; + + /* lookup wasm functions */ + if (!(my_shared_malloc_func = + wasm_runtime_lookup_function(module_inst, "my_shared_malloc")) + || !(my_shared_free_func = + wasm_runtime_lookup_function(module_inst, "my_shared_free"))) { + printf("Failed to lookup function.\n"); + } + + /* create exec env */ + if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { + printf("Failed to create exec env.\n"); + return NULL; + } + + /* allocate memory with wasm_runtime_shared_heap_malloc and send it + to wasm app2 */ + for (i = 0; i < 5; i++) { + uint8 *buf; + uint64 offset; + + offset = wasm_runtime_shared_heap_malloc(module_inst, 1024 * (i + 1), + (void **)&buf); + + if (offset == 0) { + printf("Failed to allocate memory from shared heap\n"); + break; + } + + snprintf(buf, 1024, "Hello, this is buf %u allocated from shared heap", + i + 1); + + printf("wasm app1 send buf: %s\n\n", buf); + if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + printf("Failed to post message to queue\n"); + wasm_runtime_shared_heap_free(module_inst, offset); + break; + } + } + + /* allocate memory by calling my_shared_malloc function and send it + to wasm app2 */ + for (i = 5; i < 10; i++) { + uint8 *buf; + + argv[0] = 1024 * (i + 1); + argv[1] = i + 1; + wasm_runtime_call_wasm(exec_env, my_shared_malloc_func, 2, argv); + + if (wasm_runtime_get_exception(module_inst)) { + printf("Failed to call 'my_shared_malloc` function: %s\n", + wasm_runtime_get_exception(module_inst)); + break; + } + if (argv[0] == 0) { + printf("Failed to allocate memory from shared heap\n"); + break; + } + + buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); + + printf("wasm app1 send buf: %s\n\n", buf); + if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + printf("Failed to post message to queue\n"); + wasm_runtime_shared_heap_free(module_inst, argv[0]); + break; + } + } + + wasm_runtime_destroy_exec_env(exec_env); + + return NULL; +} + +static void +queue_callback(void *message, void *arg) +{ + bh_message_t msg = (bh_message_t)message; + wasm_exec_env_t exec_env = arg; + wasm_module_inst_t module_inst = wasm_runtime_get_module_inst(exec_env); + wasm_function_inst_t print_buf_func; + uint32 argv[2]; + + /* lookup wasm function */ + if (!(print_buf_func = + wasm_runtime_lookup_function(module_inst, "print_buf"))) { + printf("Failed to lookup function.\n"); + return; + } + + char *buf = bh_message_payload(msg); + printf("wasm app's native queue received buf: %s\n\n", buf); + + /* call wasm function */ + argv[0] = wasm_runtime_addr_native_to_app(module_inst, buf); + wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); +} + +static void * +thread2_callback(void *arg) +{ + thread_arg *targ = arg; + bh_queue *queue = targ->queue; + wasm_module_inst_t module_inst = targ->module_inst; + wasm_exec_env_t exec_env; + + /* create exec env */ + if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { + printf("Failed to create exec env.\n"); + return NULL; + } + + /* enter queue's message loop until bh_queue_exit_loop_run + is called */ + bh_queue_enter_loop_run(queue, queue_callback, exec_env); + + wasm_runtime_destroy_exec_env(exec_env); + + return NULL; +} + +static char global_heap_buf[512 * 1024]; + +int +main(int argc, char **argv) +{ + char *wasm_file1 = NULL, *wasm_file2 = NULL; + uint8 *wasm_file1_buf = NULL, *wasm_file2_buf = NULL; + uint32 wasm_file1_size, wasm_file2_size; + wasm_module_t wasm_module1 = NULL, wasm_module2 = NULL; + wasm_module_inst_t module_inst1 = NULL; + wasm_module_inst_t module_inst2 = NULL; + wasm_shared_heap_t shared_heap = NULL; + bh_queue *queue = NULL; + RuntimeInitArgs init_args; + SharedHeapInitArgs heap_init_args; + char error_buf[128] = { 0 }; + int ret = -1; + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + /* init wasm runtime */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + /* create queue */ + if (!(queue = bh_queue_create())) { + printf("Create queue failed.\n"); + goto fail; + } + + /* read wasm file */ + wasm_file1 = "./wasm-apps/test1.wasm"; + if (!(wasm_file1_buf = + bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { + printf("Open wasm file %s failed.\n", wasm_file1); + goto fail; + } + + /* load wasm file */ + wasm_module1 = wasm_runtime_load((uint8 *)wasm_file1_buf, wasm_file1_size, + error_buf, sizeof(error_buf)); + if (!wasm_module1) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* instantiate module */ + module_inst1 = wasm_runtime_instantiate(wasm_module1, 65536, 0, error_buf, + sizeof(error_buf)); + if (!module_inst1) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* read wasm file */ + wasm_file2 = "./wasm-apps/test2.wasm"; + if (!(wasm_file2_buf = + bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { + printf("Open wasm file %s failed.\n", wasm_file1); + goto fail; + } + + /* load wasm file */ + wasm_module2 = wasm_runtime_load((uint8 *)wasm_file2_buf, wasm_file2_size, + error_buf, sizeof(error_buf)); + if (!wasm_module2) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* instantiate module */ + module_inst2 = wasm_runtime_instantiate(wasm_module2, 65536, 0, error_buf, + sizeof(error_buf)); + if (!module_inst2) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* create shared heap */ + memset(&heap_init_args, 0, sizeof(heap_init_args)); + heap_init_args.size = 65536; + shared_heap = wasm_runtime_create_shared_heap(&heap_init_args); + if (!shared_heap) { + printf("Create shared heap failed. error: %s\n", error_buf); + goto fail; + } + + /* attach module instance 1 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(module_inst1, shared_heap)) { + printf("Attach shared heap failed.\n"); + goto fail; + } + + /* attach module instance 2 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(module_inst2, shared_heap)) { + printf("Attach shared heap failed.\n"); + goto fail; + } + + /* create thread 1 */ + struct thread_arg targ1 = { 0 }; + korp_tid tid1; + targ1.queue = queue; + targ1.module_inst = module_inst1; + if (os_thread_create(&tid1, thread1_callback, &targ1, + APP_THREAD_STACK_SIZE_DEFAULT)) { + printf("Failed to create thread 1\n"); + goto fail; + } + + /* create thread 2 */ + struct thread_arg targ2 = { 0 }; + korp_tid tid2; + targ2.queue = queue; + targ2.module_inst = module_inst2; + if (os_thread_create(&tid2, thread2_callback, &targ2, + APP_THREAD_STACK_SIZE_DEFAULT)) { + printf("Failed to create thread 2\n"); + os_thread_join(tid1, NULL); + goto fail; + } + + /* wait until all messages are post to wasm app2 and wasm app2 + handles all of them, then exit the queue message loop */ + usleep(2000); + bh_queue_exit_loop_run(queue); + + os_thread_join(tid1, NULL); + os_thread_join(tid2, NULL); + + ret = 0; + +fail: + if (module_inst2) + wasm_runtime_deinstantiate(module_inst2); + + if (module_inst1) + wasm_runtime_deinstantiate(module_inst1); + + if (wasm_module2) + wasm_runtime_unload(wasm_module2); + + if (wasm_module1) + wasm_runtime_unload(wasm_module1); + + if (wasm_file2_buf) + wasm_runtime_free(wasm_file2_buf); + + if (wasm_file1_buf) + wasm_runtime_free(wasm_file1_buf); + + if (queue) + bh_queue_destroy(queue); + + wasm_runtime_destroy(); + + return ret; +} diff --git a/samples/shared-heap/wasm-apps/CMakeLists.txt b/samples/shared-heap/wasm-apps/CMakeLists.txt new file mode 100644 index 000000000..819c4aca3 --- /dev/null +++ b/samples/shared-heap/wasm-apps/CMakeLists.txt @@ -0,0 +1,43 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(wasm-apps) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +if (APPLE) + set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) + set (CMAKE_C_LINK_FLAGS "") + set (CMAKE_CXX_LINK_FLAGS "") +endif () + +set (CMAKE_SYSTEM_PROCESSOR wasm32) +set (CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set (WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set (CMAKE_C_FLAGS "-nostdlib -Qunused-arguments -z stack-size=32768") +set (CMAKE_C_COMPILER_TARGET "wasm32") +set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +set (DEFINED_SYMBOLS "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") + +set (CMAKE_EXE_LINKER_FLAGS + "-Wl,--initial-memory=65536, \ + -Wl,--no-entry,--strip-all, \ + -Wl,--export=__heap_base,--export=__data_end \ + -Wl,--export=__wasm_call_ctors \ + -Wl,--export=my_shared_malloc \ + -Wl,--export=my_shared_free \ + -Wl,--export=print_buf \ + -Wl,--allow-undefined" +) + +add_executable(test1.wasm test1.c) +target_link_libraries(test1.wasm) + +add_executable(test2.wasm test2.c) +target_link_libraries(test2.wasm) diff --git a/samples/shared-heap/wasm-apps/test1.c b/samples/shared-heap/wasm-apps/test1.c new file mode 100644 index 000000000..9186df2ce --- /dev/null +++ b/samples/shared-heap/wasm-apps/test1.c @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include + +extern void * +shared_malloc(uint32_t size); +extern void +shared_free(void *ptr); + +void * +my_shared_malloc(uint32_t size, uint32_t index) +{ + char *buf = shared_malloc(size); + + if (buf) + snprintf(buf, 1024, "Hello, this is buf %u allocated from shared heap", + index); + + return buf; +} + +void +my_shared_free(void *ptr) +{ + shared_free(ptr); +} diff --git a/samples/shared-heap/wasm-apps/test2.c b/samples/shared-heap/wasm-apps/test2.c new file mode 100644 index 000000000..12af9e52b --- /dev/null +++ b/samples/shared-heap/wasm-apps/test2.c @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +#include + +extern void +shared_free(void *ptr); + +void +print_buf(char *buf) +{ + printf("wasm app2's wasm func received buf: %s\n\n", buf); + shared_free(buf); +} From 9ba36e284c201b52716b314d088a68163a419f7c Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Sun, 29 Sep 2024 12:50:59 +0800 Subject: [PATCH 5/6] Implement shared heap for AOT (#3815) --- .../compilation_on_android_ubuntu.yml | 1 + .github/workflows/compilation_on_macos.yml | 1 + .github/workflows/nightly_run.yml | 1 + core/config.h | 4 +- core/iwasm/aot/aot_runtime.c | 21 ++ core/iwasm/aot/aot_runtime.h | 8 + core/iwasm/common/wasm_memory.c | 117 ++++++-- core/iwasm/common/wasm_memory.h | 4 + core/iwasm/common/wasm_runtime_common.c | 18 ++ core/iwasm/common/wasm_shared_memory.c | 7 +- core/iwasm/compilation/aot_emit_memory.c | 278 +++++++++++++++++- core/iwasm/compilation/aot_llvm.c | 78 +++++ core/iwasm/compilation/aot_llvm.h | 5 + core/iwasm/include/aot_comp_option.h | 1 + core/iwasm/interpreter/wasm_interp_classic.c | 10 +- core/iwasm/interpreter/wasm_loader.c | 3 + core/iwasm/interpreter/wasm_mini_loader.c | 3 + core/iwasm/interpreter/wasm_runtime.c | 8 + core/iwasm/interpreter/wasm_runtime.h | 11 +- .../shared-heap/shared_heap_wrapper.c | 8 +- .../libraries/thread-mgr/thread_manager.c | 10 - doc/build_wamr.md | 16 + samples/shared-heap/CMakeLists.txt | 37 ++- samples/shared-heap/src/main.c | 43 ++- samples/shared-heap/wasm-apps/CMakeLists.txt | 6 +- samples/shared-heap/wasm-apps/test1.c | 48 ++- samples/shared-heap/wasm-apps/test2.c | 4 +- tests/unit/shared-heap/wasm-apps/test.c | 10 +- wamr-compiler/main.c | 4 + 29 files changed, 684 insertions(+), 81 deletions(-) diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 27592fde6..6be445944 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -585,6 +585,7 @@ jobs: cmake .. cmake --build . --config Debug --parallel 4 ./shared_heap_test + ./shared_heap_test --aot test: needs: diff --git a/.github/workflows/compilation_on_macos.yml b/.github/workflows/compilation_on_macos.yml index f92bba62f..5b1edac85 100644 --- a/.github/workflows/compilation_on_macos.yml +++ b/.github/workflows/compilation_on_macos.yml @@ -394,3 +394,4 @@ jobs: cmake .. cmake --build . --config Debug --parallel 4 ./shared_heap_test + ./shared_heap_test --aot diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index 832c34d70..5149c830f 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -600,6 +600,7 @@ jobs: cmake .. cmake --build . --config Debug --parallel 4 ./shared_heap_test + ./shared_heap_test --aot test: needs: diff --git a/core/config.h b/core/config.h index c49b63ca2..6bab4da90 100644 --- a/core/config.h +++ b/core/config.h @@ -396,7 +396,9 @@ #define APP_HEAP_SIZE_DEFAULT (8 * 1024) #endif #define APP_HEAP_SIZE_MIN (256) -#define APP_HEAP_SIZE_MAX (512 * 1024 * 1024) +/* The ems memory allocator supports maximal heap size 1GB, + see ems_gc_internal.h */ +#define APP_HEAP_SIZE_MAX (1024 * 1024 * 1024) /* Default min/max gc heap size of each app */ #ifndef GC_HEAP_SIZE_DEFAULT diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 63a3c83c9..fae534b7e 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -57,6 +57,9 @@ bh_static_assert(sizeof(AOTMemoryInstance) == 120); bh_static_assert(offsetof(AOTTableInstance, elems) == 24); bh_static_assert(offsetof(AOTModuleInstanceExtra, stack_sizes) == 0); +bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj) + == 8); +bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_start_off) == 16); bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3); @@ -1885,6 +1888,24 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, extra->stack_sizes = aot_get_data_section_addr(module, AOT_STACK_SIZES_SECTION_NAME, NULL); + /* + * The AOT code checks whether the n bytes to access are in shared heap + * by checking whether the beginning address meets: + * addr >= start_off && addr <= end_off - n-bytes + 1 + * where n is 1/2/4/8/16 and `end_off - n-bytes + 1` is constant, e.g., + * UINT32_MAX, UINT32_MAX-1, UINT32_MAX-3 for n = 1, 2 or 4 in 32-bit + * target. To simplify the check, when shared heap is disabled, we set + * the start off to UINT64_MAX in 64-bit target and UINT32_MAX in 32-bit + * target, so in the checking, the above formula will be false, we don't + * need to check whether the shared heap is enabled or not in the AOT + * code. + */ +#if UINTPTR_MAX == UINT64_MAX + extra->shared_heap_start_off.u64 = UINT64_MAX; +#else + extra->shared_heap_start_off.u32[0] = UINT32_MAX; +#endif + #if WASM_ENABLE_PERF_PROFILING != 0 total_size = sizeof(AOTFuncPerfProfInfo) * ((uint64)module->import_func_count + module->func_count); diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 2728e57ef..bf5e4366c 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -111,6 +111,14 @@ typedef struct AOTFunctionInstance { typedef struct AOTModuleInstanceExtra { DefPointer(const uint32 *, stack_sizes); + /* + * Adjusted shared heap based addr to simple the calculation + * in the aot code. The value is: + * shared_heap->base_addr - shared_heap->start_off + */ + DefPointer(uint8 *, shared_heap_base_addr_adj); + MemBound shared_heap_start_off; + WASMModuleInstanceExtraCommon common; AOTFunctionInstance **functions; uint32 function_count; diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index e70cc2a5b..d2d89e595 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -165,7 +165,7 @@ runtime_malloc(uint64 size) WASMSharedHeap * wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) { - uint64 heap_struct_size = sizeof(WASMSharedHeap); + uint64 heap_struct_size = sizeof(WASMSharedHeap), map_size; uint32 size = init_args->size; WASMSharedHeap *heap; @@ -192,7 +192,18 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) goto fail3; } - if (!(heap->base_addr = wasm_mmap_linear_memory(size, size))) { +#ifndef OS_ENABLE_HW_BOUND_CHECK + map_size = size; +#else + /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: + * ea = i + memarg.offset + * both i and memarg.offset are u32 in range 0 to 4G + * so the range of ea is 0 to 8G + */ + map_size = 8 * (uint64)BH_GB; +#endif + + if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) { goto fail3; } if (!mem_allocator_create_with_struct_and_pool( @@ -213,7 +224,7 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) return heap; fail4: - wasm_munmap_linear_memory(heap->base_addr, size, size); + wasm_munmap_linear_memory(heap->base_addr, size, map_size); fail3: wasm_runtime_free(heap->heap_handle); fail2: @@ -245,18 +256,52 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, #if WASM_ENABLE_INTERP != 0 if (module_inst->module_type == Wasm_Module_Bytecode) { - if (((WASMModuleInstance *)module_inst)->e->shared_heap) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + if (e->shared_heap) { LOG_WARNING("A shared heap is already attached"); return false; } - ((WASMModuleInstance *)module_inst)->e->shared_heap = shared_heap; - } + e->shared_heap = shared_heap; +#if WASM_ENABLE_JIT != 0 +#if UINTPTR_MAX == UINT64_MAX + if (memory->is_memory64) + e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; + else + e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u64; +#else + e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; #endif +#endif /* end of WASM_ENABLE_JIT != 0 */ + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + if (e->shared_heap) { + LOG_WARNING("A shared heap is already attached"); + return false; + } + e->shared_heap = shared_heap; +#if UINTPTR_MAX == UINT64_MAX + if (memory->is_memory64) + e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; + else + e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u64; +#else + e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; #endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ return true; } @@ -277,14 +322,32 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) { #if WASM_ENABLE_INTERP != 0 if (module_inst->module_type == Wasm_Module_Bytecode) { - ((WASMModuleInstance *)module_inst)->e->shared_heap = NULL; - } + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + e->shared_heap = NULL; +#if WASM_ENABLE_JIT != 0 +#if UINTPTR_MAX == UINT64_MAX + e->shared_heap_start_off.u64 = UINT64_MAX; +#else + e->shared_heap_start_off.u32[0] = UINT32_MAX; #endif + e->shared_heap_base_addr_adj = NULL; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + e->shared_heap = NULL; +#if UINTPTR_MAX == UINT64_MAX + e->shared_heap_start_off.u64 = UINT64_MAX; +#else + e->shared_heap_start_off.u32[0] = UINT32_MAX; #endif + e->shared_heap_base_addr_adj = NULL; + } +#endif /* end of WASM_ENABLE_AOT != 0 */ } void @@ -307,13 +370,21 @@ get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) #endif #if WASM_ENABLE_AOT != 0 if (module_inst_comm->module_type == Wasm_Module_AoT) { - // TODO - return NULL; + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst_comm) + ->e; + return e->shared_heap; } #endif return NULL; } +WASMSharedHeap * +wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) +{ + return get_shared_heap(module_inst_comm); +} + static bool is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, bool is_memory64, uint64 app_offset, uint32 bytes) @@ -324,6 +395,10 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, return false; } + if (bytes == 0) { + bytes = 1; + } + if (!is_memory64) { if (app_offset >= heap->start_off_mem32 && app_offset <= UINT32_MAX - bytes + 1) { @@ -457,17 +532,23 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, #if WASM_ENABLE_SHARED_HEAP != 0 static void -wasm_runtime_destroy_shared_heaps() +destroy_shared_heaps() { WASMSharedHeap *heap = shared_heap_list; WASMSharedHeap *cur; + uint64 map_size; while (heap) { cur = heap; heap = heap->next; mem_allocator_destroy(cur->heap_handle); wasm_runtime_free(cur->heap_handle); - wasm_munmap_linear_memory(cur->base_addr, cur->size, cur->size); +#ifndef OS_ENABLE_HW_BOUND_CHECK + map_size = cur->size; +#else + map_size = 8 * (uint64)BH_GB; +#endif + wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size); wasm_runtime_free(cur); } } @@ -477,7 +558,7 @@ void wasm_runtime_memory_destroy(void) { #if WASM_ENABLE_SHARED_HEAP != 0 - wasm_runtime_destroy_shared_heaps(); + destroy_shared_heaps(); #endif if (memory_mode == MEMORY_MODE_POOL) { @@ -1178,7 +1259,7 @@ wasm_mremap_linear_memory(void *mapped_mem, uint64 old_size, uint64 new_size, } static void * -wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size) +wasm_mmap_linear_memory(uint64 map_size, uint64 commit_size) { return wasm_mremap_linear_memory(NULL, 0, map_size, commit_size); } diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index cc6418e84..bceea0ee4 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -54,9 +54,13 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, void wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst); + void wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst); +WASMSharedHeap * +wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm); + uint64 wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, uint64 size, void **p_native_addr); diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index dc20edba2..aa929e971 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -185,6 +185,9 @@ static bool is_sig_addr_in_guard_pages(void *sig_addr, WASMModuleInstance *module_inst) { WASMMemoryInstance *memory_inst; +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; +#endif uint8 *mapped_mem_start_addr = NULL; uint8 *mapped_mem_end_addr = NULL; uint32 i; @@ -202,6 +205,21 @@ is_sig_addr_in_guard_pages(void *sig_addr, WASMModuleInstance *module_inst) } } +#if WASM_ENABLE_SHARED_HEAP != 0 + shared_heap = + wasm_runtime_get_shared_heap((WASMModuleInstanceCommon *)module_inst); + if (shared_heap) { + mapped_mem_start_addr = shared_heap->base_addr; + mapped_mem_end_addr = shared_heap->base_addr + 8 * (uint64)BH_GB; + if (mapped_mem_start_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr < mapped_mem_end_addr) { + /* The address which causes segmentation fault is inside + the shared heap's guard regions */ + return true; + } + } +#endif + return false; } diff --git a/core/iwasm/common/wasm_shared_memory.c b/core/iwasm/common/wasm_shared_memory.c index e0d86d6f4..2027a5720 100644 --- a/core/iwasm/common/wasm_shared_memory.c +++ b/core/iwasm/common/wasm_shared_memory.c @@ -8,6 +8,9 @@ #if WASM_ENABLE_THREAD_MGR != 0 #include "../libraries/thread-mgr/thread_manager.h" #endif +#if WASM_ENABLE_AOT != 0 +#include "../aot/aot_runtime.h" +#endif /* * Note: this lock can be per memory. @@ -257,7 +260,9 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, #endif #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { - // TODO + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + shared_heap = e->shared_heap; } #endif diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 806150ff1..869a1dbb2 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -118,10 +118,10 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { LLVMValueRef offset_const = MEMORY64_COND_VALUE(I64_CONST(offset), I32_CONST(offset)); - LLVMValueRef addr, maddr, offset1, cmp1, cmp2, cmp; + LLVMValueRef addr, maddr, maddr_phi = NULL, offset1, cmp1, cmp2, cmp; LLVMValueRef mem_base_addr, mem_check_bound; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef check_succ; + LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; AOTValue *aot_value_top; uint32 local_idx_of_aot_value = 0; uint64 const_value; @@ -131,6 +131,11 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, bool is_shared_memory = comp_ctx->comp_data->memories[0].flags & SHARED_MEMORY_FLAG; #endif +#if WASM_ENABLE_MEMORY64 == 0 + bool is_memory64 = false; +#else + bool is_memory64 = IS_MEMORY64; +#endif is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; @@ -268,8 +273,137 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* offset1 = offset + addr; */ + /* TODO: check whether integer overflow occurs when memory is 64-bit + and boundary check is enabled */ BUILD_OP(Add, offset_const, addr, offset1, "offset1"); + if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; + LLVMValueRef is_in_shared_heap, shared_heap_check_bound = NULL; + + /* Add basic blocks */ + ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap"); + ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); + ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); + + LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr); + LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, + app_addr_in_shared_heap); + LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + if (!(maddr_phi = + LLVMBuildPhi(comp_ctx->builder, + enable_segue ? INT8_PTR_TYPE_GS : INT8_PTR_TYPE, + "maddr_phi"))) { + aot_set_last_error("llvm build phi failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); + + if (!is_target_64bit) { + /* Check whether interger overflow occurs in addr + offset */ + LLVMBasicBlockRef check_integer_overflow_end; + ADD_BASIC_BLOCK(check_integer_overflow_end, + "check_integer_overflow_end"); + LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); + + BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, + cmp1, check_integer_overflow_end)) { + goto fail; + } + SET_BUILD_POS(check_integer_overflow_end); + } + + shared_heap_check_bound = + is_memory64 ? I64_CONST(UINT64_MAX - bytes + 1) + : (comp_ctx->pointer_size == sizeof(uint64) + ? I64_CONST(UINT32_MAX - bytes + 1) + : I32_CONST(UINT32_MAX - bytes + 1)); + CHECK_LLVM_CONST(shared_heap_check_bound); + + /* Check whether the bytes to access are in shared heap */ + if (!comp_ctx->enable_bound_check) { + /* Use IntUGT but not IntUGE to compare, since (1) in the ems + memory allocator, the hmu node includes hmu header and hmu + memory, only the latter is returned to the caller as the + allocated memory, the hmu header isn't returned so the + first byte of the shared heap won't be accesed, (2) using + IntUGT gets better performance than IntUGE in some cases */ + BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off, + is_in_shared_heap, "is_in_shared_heap"); + /* We don't check the shared heap's upper boundary if boundary + check isn't enabled, the runtime may also use the guard pages + of shared heap to check the boundary if hardware boundary + check feature is enabled. */ + } + else { + /* Use IntUGT but not IntUGE to compare, same as above */ + BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off, + cmp1, "cmp1"); + /* Check the shared heap's upper boundary if boundary check is + enabled */ + BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp2, + "cmp2"); + BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap"); + } + + if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap, + app_addr_in_shared_heap, app_addr_in_linear_mem)) { + aot_set_last_error("llvm build cond br failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap); + + /* Get native address inside shared heap */ + if (!(maddr = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->shared_heap_base_addr_adj, + &offset1, 1, "maddr_shared_heap"))) { + aot_set_last_error("llvm build inbounds gep failed"); + goto fail; + } + + if (enable_segue) { + LLVMValueRef mem_base_addr_u64, maddr_u64, offset_to_mem_base; + + if (!(maddr_u64 = LLVMBuildPtrToInt(comp_ctx->builder, maddr, + I64_TYPE, "maddr_u64")) + || !(mem_base_addr_u64 = + LLVMBuildPtrToInt(comp_ctx->builder, mem_base_addr, + I64_TYPE, "mem_base_addr_u64"))) { + aot_set_last_error("llvm build ptr to int failed"); + goto fail; + } + if (!(offset_to_mem_base = + LLVMBuildSub(comp_ctx->builder, maddr_u64, + mem_base_addr_u64, "offset_to_mem_base"))) { + aot_set_last_error("llvm build sub failed"); + goto fail; + } + if (!(maddr = LLVMBuildIntToPtr( + comp_ctx->builder, offset_to_mem_base, INT8_PTR_TYPE_GS, + "maddr_shared_heap_segue"))) { + aot_set_last_error("llvm build int to ptr failed."); + goto fail; + } + } + + LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1); + + if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { + aot_set_last_error("llvm build br failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + } + if (comp_ctx->enable_bound_check && !(is_local_of_aot_value && aot_checked_addr_list_find(func_ctx, local_idx_of_aot_value, @@ -305,10 +439,16 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); } else { - /* Check integer overflow */ - BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2"); - BUILD_OP(Or, cmp1, cmp2, cmp, "cmp"); + if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + /* Check integer overflow has been checked above */ + BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); + } + else { + /* Check integer overflow */ + BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); + BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2"); + BUILD_OP(Or, cmp1, cmp2, cmp, "cmp"); + } } /* Add basic blocks */ @@ -354,7 +494,19 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } } - return maddr; + + if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); + if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { + aot_set_last_error("llvm build br failed"); + goto fail; + } + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + return maddr_phi; + } + else + return maddr; fail: return NULL; } @@ -985,10 +1137,15 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef offset, LLVMValueRef bytes) { LLVMValueRef maddr, max_addr, cmp; - LLVMValueRef mem_base_addr; + LLVMValueRef mem_base_addr, maddr_phi = NULL; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef check_succ; + LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; LLVMValueRef mem_size; +#if WASM_ENABLE_MEMORY64 == 0 + bool is_memory64 = false; +#else + bool is_memory64 = IS_MEMORY64; +#endif /* Get memory base address and memory data size */ #if WASM_ENABLE_SHARED_MEMORY != 0 @@ -1053,9 +1210,96 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, offset = LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset"); bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len"); + if (!offset || !bytes) { + aot_set_last_error("llvm build zext failed."); + goto fail; + } + /* TODO: check whether integer overflow occurs when memory is 64-bit + and boundary check is enabled */ BUILD_OP(Add, offset, bytes, max_addr, "max_addr"); + + if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; + LLVMValueRef shared_heap_start_off, shared_heap_check_bound; + LLVMValueRef max_offset, cmp1, cmp2, is_in_shared_heap; + + /* Add basic blocks */ + ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap"); + ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); + ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); + + LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr); + LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, + app_addr_in_shared_heap); + LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE, + "maddr_phi"))) { + aot_set_last_error("llvm build phi failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); + + shared_heap_start_off = func_ctx->shared_heap_start_off; + if (comp_ctx->pointer_size == sizeof(uint32)) { + if (!(shared_heap_start_off = + LLVMBuildZExt(comp_ctx->builder, shared_heap_start_off, + I64_TYPE, "shared_heap_start_off_u64"))) { + aot_set_last_error("llvm build zext failed"); + goto fail; + } + } + shared_heap_check_bound = + is_memory64 ? I64_CONST(UINT64_MAX) : I64_CONST(UINT32_MAX); + CHECK_LLVM_CONST(shared_heap_check_bound); + + /* Check whether the bytes to access are in shared heap */ + if (!comp_ctx->enable_bound_check) { + /* Use IntUGT but not IntUGE to compare, same as the check + in aot_check_memory_overflow */ + BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off, + is_in_shared_heap, "is_in_shared_heap"); + } + else { + BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off, + cmp1, "cmp1"); + BUILD_OP(Add, max_addr, I64_NEG_ONE, max_offset, "max_offset"); + BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_check_bound, cmp2, + "cmp2"); + BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap"); + } + + if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap, + app_addr_in_shared_heap, app_addr_in_linear_mem)) { + aot_set_last_error("llvm build cond br failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap); + + /* Get native address inside shared heap */ + if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->shared_heap_base_addr_adj, + &offset, 1, "maddr_shared_heap"))) { + aot_set_last_error("llvm build inbounds gep failed"); + goto fail; + } + LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1); + + if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { + aot_set_last_error("llvm build br failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + } + BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr"); + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, check_succ)) { @@ -1068,11 +1312,23 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("llvm build add failed."); goto fail; } - return maddr; + + if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); + if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { + aot_set_last_error("llvm build br failed"); + goto fail; + } + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + return maddr_phi; + } + else + return maddr; fail: return NULL; } -#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 or WASM_ENABLE_STRINGREF != 0 */ +#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 || WASM_ENABLE_STRINGREF != 0 */ #if WASM_ENABLE_BULK_MEMORY != 0 bool diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 820a55e96..fb1c4308b 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1518,6 +1518,75 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } +static bool +create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef offset, base_addr_p, start_off_p, cmp; + uint32 offset_u32; + + /* Load aot_inst->e->shared_heap_base_addr_adj */ + offset_u32 = get_module_inst_extra_offset(comp_ctx); +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->is_jit_mode) + offset_u32 += + offsetof(WASMModuleInstanceExtra, shared_heap_base_addr_adj); + else +#endif + offset_u32 += + offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj); + offset = I32_CONST(offset_u32); + CHECK_LLVM_CONST(offset); + + if (!(base_addr_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, 1, + "shared_heap_base_addr_adj_p"))) { + aot_set_last_error("llvm build inbounds gep failed"); + return false; + } + if (!(func_ctx->shared_heap_base_addr_adj = + LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, base_addr_p, + "shared_heap_base_addr_adj"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + + /* Load aot_inst->e->shared_heap_start_off */ + offset_u32 = get_module_inst_extra_offset(comp_ctx); +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->is_jit_mode) + offset_u32 += offsetof(WASMModuleInstanceExtra, shared_heap_start_off); + else +#endif + offset_u32 += offsetof(AOTModuleInstanceExtra, shared_heap_start_off); + offset = I32_CONST(offset_u32); + CHECK_LLVM_CONST(offset); + + if (!(start_off_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->aot_inst, &offset, 1, + "shared_heap_start_off_p"))) { + aot_set_last_error("llvm build inbounds gep failed"); + return false; + } + if (!(func_ctx->shared_heap_start_off = LLVMBuildLoad2( + comp_ctx->builder, + comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE, + start_off_p, "shared_heap_start_off"))) { + aot_set_last_error("llvm build load failed"); + return false; + } + + if (!(cmp = LLVMBuildIsNotNull(comp_ctx->builder, + func_ctx->shared_heap_base_addr_adj, + "has_shared_heap"))) { + aot_set_last_error("llvm build is not null failed"); + return false; + } + + return true; +fail: + return false; +} + static bool create_cur_exception(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { @@ -1808,6 +1877,12 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx, goto fail; } + /* Load shared heap, shared heap start off mem32 or mem64 */ + if (comp_ctx->enable_shared_heap + && !create_shared_heap_info(comp_ctx, func_ctx)) { + goto fail; + } + return func_ctx; fail: @@ -2619,6 +2694,9 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) if (option->enable_gc) comp_ctx->enable_gc = true; + if (option->enable_shared_heap) + comp_ctx->enable_shared_heap = true; + comp_ctx->opt_level = option->opt_level; comp_ctx->size_level = option->size_level; diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 43212e502..0dce988bc 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -242,6 +242,9 @@ typedef struct AOTFuncContext { bool mem_space_unchanged; AOTCheckedAddrList checked_addr_list; + LLVMValueRef shared_heap_base_addr_adj; + LLVMValueRef shared_heap_start_off; + LLVMBasicBlockRef got_exception_block; LLVMBasicBlockRef func_return_block; LLVMValueRef exception_id_phi; @@ -467,6 +470,8 @@ typedef struct AOTCompContext { /* Enable GC */ bool enable_gc; + bool enable_shared_heap; + uint32 opt_level; uint32 size_level; diff --git a/core/iwasm/include/aot_comp_option.h b/core/iwasm/include/aot_comp_option.h index 98f33a160..4df80ec58 100644 --- a/core/iwasm/include/aot_comp_option.h +++ b/core/iwasm/include/aot_comp_option.h @@ -71,6 +71,7 @@ typedef struct AOTCompOption { bool enable_llvm_pgo; bool enable_stack_estimation; bool quick_invoke_c_api_import; + bool enable_shared_heap; char *use_prof_file; uint32_t opt_level; uint32_t size_level; diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index ecbe408ff..cb385ce9c 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -47,8 +47,14 @@ typedef float64 CellType_F64; #endif #if WASM_ENABLE_SHARED_HEAP != 0 -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && (app_addr) >= shared_heap_start_off \ +#if WASM_ENABLE_MULTI_MEMORY != 0 +/* Only enable shared heap for the default memory */ +#define is_default_memory (memidx == 0) +#else +#define is_default_memory true +#endif +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ && (app_addr) <= shared_heap_end_off - bytes + 1) #define shared_heap_addr_app_to_native(app_addr, native_addr) \ diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index ff3501e3d..e61e7677d 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -5330,6 +5330,9 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, option.enable_memory_profiling = true; option.enable_stack_estimation = true; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + option.enable_shared_heap = true; +#endif module->comp_ctx = aot_create_comp_context(module->comp_data, &option); if (!module->comp_ctx) { diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 968eaf009..ff0a4f403 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -2158,6 +2158,9 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, option.enable_memory_profiling = true; option.enable_stack_estimation = true; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + option.enable_shared_heap = true; +#endif module->comp_ctx = aot_create_comp_context(module->comp_data, &option); if (!module->comp_ctx) { diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index e4142ab88..d474f6eae 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -2791,6 +2791,14 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, } } +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 +#if UINTPTR_MAX == UINT64_MAX + module_inst->e->shared_heap_start_off.u64 = UINT64_MAX; +#else + module_inst->e->shared_heap_start_off.u32[0] = UINT32_MAX; +#endif +#endif + #if WASM_ENABLE_GC != 0 /* Initialize the table data with init expr */ for (i = 0; i < module->table_count; i++) { diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 48c333f4c..2644b0016 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -92,7 +92,6 @@ typedef union { uint32 u32[2]; } MemBound; -#if WASM_ENABLE_SHARED_HEAP != 0 typedef struct WASMSharedHeap { struct WASMSharedHeap *next; void *heap_handle; @@ -101,7 +100,6 @@ typedef struct WASMSharedHeap { uint64 start_off_mem64; uint64 start_off_mem32; } WASMSharedHeap; -#endif struct WASMMemoryInstance { /* Module type */ @@ -366,6 +364,15 @@ typedef struct WASMModuleInstanceExtra { #if WASM_ENABLE_SHARED_HEAP != 0 WASMSharedHeap *shared_heap; +#if WASM_ENABLE_JIT != 0 + /* + * Adjusted shared heap based addr to simple the calculation + * in the aot code. The value is: + * shared_heap->base_addr - shared_heap->start_off + */ + uint8 *shared_heap_base_addr_adj; + MemBound shared_heap_start_off; +#endif #endif #if WASM_ENABLE_DEBUG_INTERP != 0 \ diff --git a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c index 76e662dac..b7b78307a 100644 --- a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c +++ b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c @@ -20,14 +20,14 @@ /* clang-format on */ static uint32 -shared_malloc_wrapper(wasm_exec_env_t exec_env, uint32 size) +shared_heap_malloc_wrapper(wasm_exec_env_t exec_env, uint32 size) { wasm_module_inst_t module_inst = get_module_inst(exec_env); return (uint32)module_shared_malloc((uint64)size, NULL); } static void -shared_free_wrapper(wasm_exec_env_t exec_env, void *ptr) +shared_heap_free_wrapper(wasm_exec_env_t exec_env, void *ptr) { wasm_module_inst_t module_inst = get_module_inst(exec_env); @@ -45,8 +45,8 @@ shared_free_wrapper(wasm_exec_env_t exec_env, void *ptr) /* clang-format on */ static NativeSymbol native_symbols_shared_heap[] = { - REG_NATIVE_FUNC(shared_malloc, "(i)i"), - REG_NATIVE_FUNC(shared_free, "(*)"), + REG_NATIVE_FUNC(shared_heap_malloc, "(i)i"), + REG_NATIVE_FUNC(shared_heap_free, "(*)"), }; uint32 diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index 46a1bb329..55e0526c3 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -1428,16 +1428,6 @@ wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst, { WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst); - if (module_inst->module_type == Wasm_Module_Bytecode) { - if (((WASMModuleInstance *)module_inst)->e->shared_heap) { - LOG_WARNING("A shared heap is already attached"); - return false; - } - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } - if (exec_env == NULL) { /* Maybe threads have not been started yet. */ return wasm_runtime_attach_shared_heap_internal(module_inst, heap); diff --git a/doc/build_wamr.md b/doc/build_wamr.md index 2adb17f6a..4537ee084 100644 --- a/doc/build_wamr.md +++ b/doc/build_wamr.md @@ -300,6 +300,22 @@ Currently we only profile the memory consumption of module, module_instance and wasm_runtime_get_context ``` +#### **Shared heap among wasm apps and host native** +- **WAMR_BUILD_SHARED_HEAP**=1/0, default to disable if not set +> Note: If it is enabled, allow to create one or more shared heaps, and attach one to a module instance, the belows APIs ared provided: +```C + wasm_runtime_create_shared_heap + wasm_runtime_attach_shared_heap + wasm_runtime_detach_shared_heap + wasm_runtime_shared_heap_malloc + wasm_runtime_shared_heap_free +``` +And the wasm app can calls below APIs to allocate/free memory from/to the shared heap if it is attached to the app's module instance: +```C + void *shared_heap_malloc(); + void shared_heap_free(void *ptr); +``` + **Combination of configurations:** We can combine the configurations. For example, if we want to disable interpreter, enable AOT and WASI, we can run command: diff --git a/samples/shared-heap/CMakeLists.txt b/samples/shared-heap/CMakeLists.txt index 382ddd546..6346d077e 100644 --- a/samples/shared-heap/CMakeLists.txt +++ b/samples/shared-heap/CMakeLists.txt @@ -48,6 +48,7 @@ if (NOT CMAKE_BUILD_TYPE) endif () set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_FAST_INTERP 1) set (WAMR_BUILD_AOT 1) set (WAMR_BUILD_JIT 0) set (WAMR_BUILD_LIBC_BUILTIN 1) @@ -72,7 +73,11 @@ endif () set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) -add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) +add_library(vmlib STATIC ${WAMR_RUNTIME_LIB_SOURCE}) +if (MSVC) + target_compile_definitions(vmlib PRIVATE WASM_API_EXTERN=) +endif() +target_link_libraries(vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) ################ application related ################ include_directories(${CMAKE_CURRENT_LIST_DIR}/src) @@ -90,3 +95,33 @@ else () endif () add_subdirectory(wasm-apps) + +if (WAMR_BUILD_AOT EQUAL 1) + set (WAMR_COMPILER_DIR ${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build) + message (CHECK_START "Detecting WAMR_COMPILER at ${WAMR_COMPILER_DIR}") + find_file (WAMR_COMPILER + wamrc + PATHS "${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH + ) + if (WAMR_COMPILER) + message (CHECK_PASS "found") + else() + message (CHECK_FAIL "not found") + endif() + if (NOT EXISTS ${WAMR_COMPILER}) + message (FATAL_ERROR "Please build wamrc under ${WAMR_ROOT_DIR}/wamr-compiler") + else() + message (STATUS "WAMR_COMPILER is ${WAMR_COMPILER}") + endif() + + add_custom_target( + wasm_to_aot + ALL + DEPENDS wasm-apps/test1.wasm wasm-apps/test2.wasm ${WAMR_COMPILER} + COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test1.aot wasm-apps/test1.wasm + COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test2.aot wasm-apps/test2.wasm + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) +endif() diff --git a/samples/shared-heap/src/main.c b/samples/shared-heap/src/main.c index 416652a3e..f4024f08c 100644 --- a/samples/shared-heap/src/main.c +++ b/samples/shared-heap/src/main.c @@ -19,15 +19,15 @@ thread1_callback(void *arg) wasm_module_inst_t module_inst = targ->module_inst; bh_queue *queue = targ->queue; wasm_exec_env_t exec_env; - wasm_function_inst_t my_shared_malloc_func; - wasm_function_inst_t my_shared_free_func; + wasm_function_inst_t my_shared_heap_malloc_func; + wasm_function_inst_t my_shared_heap_free_func; uint32 i, argv[2]; /* lookup wasm functions */ - if (!(my_shared_malloc_func = - wasm_runtime_lookup_function(module_inst, "my_shared_malloc")) - || !(my_shared_free_func = - wasm_runtime_lookup_function(module_inst, "my_shared_free"))) { + if (!(my_shared_heap_malloc_func = wasm_runtime_lookup_function( + module_inst, "my_shared_heap_malloc")) + || !(my_shared_heap_free_func = wasm_runtime_lookup_function( + module_inst, "my_shared_heap_free"))) { printf("Failed to lookup function.\n"); } @@ -62,17 +62,17 @@ thread1_callback(void *arg) } } - /* allocate memory by calling my_shared_malloc function and send it + /* allocate memory by calling my_shared_heap_malloc function and send it to wasm app2 */ for (i = 5; i < 10; i++) { uint8 *buf; argv[0] = 1024 * (i + 1); argv[1] = i + 1; - wasm_runtime_call_wasm(exec_env, my_shared_malloc_func, 2, argv); + wasm_runtime_call_wasm(exec_env, my_shared_heap_malloc_func, 2, argv); if (wasm_runtime_get_exception(module_inst)) { - printf("Failed to call 'my_shared_malloc` function: %s\n", + printf("Failed to call 'my_shared_heap_malloc' function: %s\n", wasm_runtime_get_exception(module_inst)); break; } @@ -118,6 +118,10 @@ queue_callback(void *message, void *arg) /* call wasm function */ argv[0] = wasm_runtime_addr_native_to_app(module_inst, buf); wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); + if (wasm_runtime_get_exception(module_inst)) { + printf("Failed to call 'print_buf' function: %s\n", + wasm_runtime_get_exception(module_inst)); + } } static void * @@ -159,8 +163,17 @@ main(int argc, char **argv) RuntimeInitArgs init_args; SharedHeapInitArgs heap_init_args; char error_buf[128] = { 0 }; + bool aot_mode = false; int ret = -1; + if (argc > 1 && !strcmp(argv[1], "--aot")) + aot_mode = true; + + if (!aot_mode) + printf("Test shared heap in interpreter mode\n\n"); + else + printf("Test shared heap in AOT mode\n\n"); + memset(&init_args, 0, sizeof(RuntimeInitArgs)); init_args.mem_alloc_type = Alloc_With_Pool; @@ -180,7 +193,10 @@ main(int argc, char **argv) } /* read wasm file */ - wasm_file1 = "./wasm-apps/test1.wasm"; + if (!aot_mode) + wasm_file1 = "./wasm-apps/test1.wasm"; + else + wasm_file1 = "./wasm-apps/test1.aot"; if (!(wasm_file1_buf = bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { printf("Open wasm file %s failed.\n", wasm_file1); @@ -204,7 +220,10 @@ main(int argc, char **argv) } /* read wasm file */ - wasm_file2 = "./wasm-apps/test2.wasm"; + if (!aot_mode) + wasm_file2 = "./wasm-apps/test2.wasm"; + else + wasm_file2 = "./wasm-apps/test2.aot"; if (!(wasm_file2_buf = bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { printf("Open wasm file %s failed.\n", wasm_file1); @@ -273,7 +292,7 @@ main(int argc, char **argv) /* wait until all messages are post to wasm app2 and wasm app2 handles all of them, then exit the queue message loop */ - usleep(2000); + usleep(10000); bh_queue_exit_loop_run(queue); os_thread_join(tid1, NULL); diff --git a/samples/shared-heap/wasm-apps/CMakeLists.txt b/samples/shared-heap/wasm-apps/CMakeLists.txt index 819c4aca3..c0010af6a 100644 --- a/samples/shared-heap/wasm-apps/CMakeLists.txt +++ b/samples/shared-heap/wasm-apps/CMakeLists.txt @@ -26,12 +26,12 @@ set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") set (DEFINED_SYMBOLS "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") set (CMAKE_EXE_LINKER_FLAGS - "-Wl,--initial-memory=65536, \ + "-O0 -Wl,--initial-memory=65536, \ -Wl,--no-entry,--strip-all, \ -Wl,--export=__heap_base,--export=__data_end \ -Wl,--export=__wasm_call_ctors \ - -Wl,--export=my_shared_malloc \ - -Wl,--export=my_shared_free \ + -Wl,--export=my_shared_heap_malloc \ + -Wl,--export=my_shared_heap_free \ -Wl,--export=print_buf \ -Wl,--allow-undefined" ) diff --git a/samples/shared-heap/wasm-apps/test1.c b/samples/shared-heap/wasm-apps/test1.c index 9186df2ce..c8fe0c755 100644 --- a/samples/shared-heap/wasm-apps/test1.c +++ b/samples/shared-heap/wasm-apps/test1.c @@ -5,26 +5,56 @@ #include #include +#include extern void * -shared_malloc(uint32_t size); +shared_heap_malloc(uint32_t size); extern void -shared_free(void *ptr); +shared_heap_free(void *ptr); void * -my_shared_malloc(uint32_t size, uint32_t index) +my_shared_heap_malloc(uint32_t size, uint32_t index) { - char *buf = shared_malloc(size); + char *buf1 = NULL, *buf2 = NULL, *buf; - if (buf) - snprintf(buf, 1024, "Hello, this is buf %u allocated from shared heap", - index); + buf1 = shared_heap_malloc(128); + if (!buf1) + return NULL; + buf1[0] = 'H'; + buf1[1] = 'e'; + buf1[2] = 'l'; + buf1[3] = 'l'; + buf1[4] = 'o'; + buf1[5] = ','; + buf1[6] = ' '; + + buf2 = shared_heap_malloc(128); + if (!buf2) { + shared_heap_free(buf1); + return NULL; + } + + snprintf(buf2, 128, "this is buf %u allocated from shared heap", index); + + buf = shared_heap_malloc(size); + if (!buf) { + shared_heap_free(buf1); + shared_heap_free(buf2); + return NULL; + } + + memset(buf, 0, size); + memcpy(buf, buf1, strlen(buf1)); + memcpy(buf + strlen(buf1), buf2, strlen(buf2)); + + shared_heap_free(buf1); + shared_heap_free(buf2); return buf; } void -my_shared_free(void *ptr) +my_shared_heap_free(void *ptr) { - shared_free(ptr); + shared_heap_free(ptr); } diff --git a/samples/shared-heap/wasm-apps/test2.c b/samples/shared-heap/wasm-apps/test2.c index 12af9e52b..b63efcd1a 100644 --- a/samples/shared-heap/wasm-apps/test2.c +++ b/samples/shared-heap/wasm-apps/test2.c @@ -8,11 +8,11 @@ #include extern void -shared_free(void *ptr); +shared_heap_free(void *ptr); void print_buf(char *buf) { printf("wasm app2's wasm func received buf: %s\n\n", buf); - shared_free(buf); + shared_heap_free(buf); } diff --git a/tests/unit/shared-heap/wasm-apps/test.c b/tests/unit/shared-heap/wasm-apps/test.c index ce59903c6..b83ee64ff 100644 --- a/tests/unit/shared-heap/wasm-apps/test.c +++ b/tests/unit/shared-heap/wasm-apps/test.c @@ -6,17 +6,17 @@ #include extern void * -shared_malloc(int size); +shared_heap_malloc(int size); extern void -shared_free(void *offset); +shared_heap_free(void *offset); int test() { - int *ptr = (int *)shared_malloc(10); + int *ptr = (int *)shared_heap_malloc(10); *ptr = 10; int a = *ptr; - shared_free(ptr); + shared_heap_free(ptr); return a; -} \ No newline at end of file +} diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index 8dca712cd..3efe344e6 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -206,6 +206,7 @@ print_help() printf(" --enable-linux-perf Enable linux perf support\n"); #endif printf(" --mllvm=