From 92852f3719572b80a06de2df3de5bc662183022c Mon Sep 17 00:00:00 2001 From: WenLY1 <130950131+WenLY1@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:51:42 +0800 Subject: [PATCH] Implement a first version of shared heap feature (#3789) ps. https://github.com/bytecodealliance/wasm-micro-runtime/issues/3546 --- build-scripts/config_common.cmake | 5 + build-scripts/runtime_lib.cmake | 5 + core/config.h | 4 + core/iwasm/aot/aot_runtime.h | 4 + core/iwasm/common/wasm_memory.c | 470 +++++++++++++++++- core/iwasm/common/wasm_memory.h | 26 + core/iwasm/common/wasm_native.c | 13 + core/iwasm/include/wasm_export.h | 55 ++ core/iwasm/interpreter/wasm_interp_classic.c | 69 ++- core/iwasm/interpreter/wasm_interp_fast.c | 30 +- core/iwasm/interpreter/wasm_runtime.h | 15 + .../libraries/shared-heap/shared_heap.cmake | 8 + .../shared-heap/shared_heap_wrapper.c | 55 ++ .../libraries/thread-mgr/thread_manager.c | 77 +++ .../libraries/thread-mgr/thread_manager.h | 12 + 15 files changed, 826 insertions(+), 22 deletions(-) create mode 100644 core/iwasm/libraries/shared-heap/shared_heap.cmake create mode 100644 core/iwasm/libraries/shared-heap/shared_heap_wrapper.c diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 12fc06bd7..0a5f82106 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -256,6 +256,11 @@ if (WAMR_BUILD_SHARED_MEMORY EQUAL 1) else () add_definitions (-DWASM_ENABLE_SHARED_MEMORY=0) endif () +if (WAMR_BUILD_SHARED_HEAP EQUAL 1) + add_definitions (-DWASM_ENABLE_SHARED_HEAP=1) + message (" Shared heap enabled") +endif() + if (WAMR_BUILD_MEMORY64 EQUAL 1) # if native is 32-bit or cross-compiled to 32-bit if (NOT WAMR_BUILD_TARGET MATCHES ".*64.*") diff --git a/build-scripts/runtime_lib.cmake b/build-scripts/runtime_lib.cmake index 3ab0cff4f..a2aaf425b 100644 --- a/build-scripts/runtime_lib.cmake +++ b/build-scripts/runtime_lib.cmake @@ -119,6 +119,10 @@ if (WAMR_BUILD_LIB_WASI_THREADS EQUAL 1) set (WAMR_BUILD_SHARED_MEMORY 1) endif () +if (WAMR_BUILD_SHARED_HEAP EQUAL 1) + include (${IWASM_DIR}/libraries/shared-heap/shared_heap.cmake) +endif () + if (WAMR_BUILD_DEBUG_INTERP EQUAL 1) set (WAMR_BUILD_THREAD_MGR 1) include (${IWASM_DIR}/libraries/debug-engine/debug_engine.cmake) @@ -193,6 +197,7 @@ set (source_all ${LIBC_EMCC_SOURCE} ${LIB_RATS_SOURCE} ${DEBUG_ENGINE_SOURCE} + ${LIB_SHARED_HEAP_SOURCE} ) set (WAMR_RUNTIME_LIB_SOURCE ${source_all}) diff --git a/core/config.h b/core/config.h index a25eb543e..a7925b008 100644 --- a/core/config.h +++ b/core/config.h @@ -688,4 +688,8 @@ #endif #endif /* WASM_ENABLE_FUZZ_TEST != 0 */ +#ifndef WASM_ENABLE_SHARED_HEAP +#define WASM_ENABLE_SHARED_HEAP 0 +#endif + #endif /* end of _CONFIG_H_ */ diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 56d11a22d..4c3676f88 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -119,6 +119,10 @@ typedef struct AOTModuleInstanceExtra { bh_list *sub_module_inst_list; WASMModuleInstanceCommon **import_func_module_insts; #endif + +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; +#endif } AOTModuleInstanceExtra; #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 82eebbf30..dc9c4aa1b 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -13,6 +13,10 @@ #include "../common/wasm_shared_memory.h" #endif +#if WASM_ENABLE_THREAD_MGR != 0 +#include "../libraries/thread-mgr/thread_manager.h" +#endif + typedef enum Memory_Mode { MEMORY_MODE_UNKNOWN = 0, MEMORY_MODE_POOL, @@ -24,6 +28,11 @@ static Memory_Mode memory_mode = MEMORY_MODE_UNKNOWN; static mem_allocator_t pool_allocator = NULL; +#if WASM_ENABLE_SHARED_HEAP != 0 +static WASMSharedHeap *shared_heap_list = NULL; +static korp_mutex shared_heap_list_lock; +#endif + static enlarge_memory_error_callback_t enlarge_memory_error_cb; static void *enlarge_memory_error_user_data; @@ -132,16 +141,370 @@ is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) #endif } +#if WASM_ENABLE_SHARED_HEAP != 0 +static void * +wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size); +static void +wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, + uint64 map_size); + +static void +set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) +{ + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, + "Operation of shared heap failed: %s", string); + } +} + +static void * +runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size) +{ + void *mem; + + if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) { + set_error_buf(error_buf, error_buf_size, "allocate memory failed"); + return NULL; + } + + memset(mem, 0, (uint32)size); + return mem; +} + +WASMSharedHeap * +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, + uint32 error_buf_size) +{ + uint64 heap_struct_size = sizeof(WASMSharedHeap); + uint32 size = init_args->size; + WASMSharedHeap *heap; + + if (!(heap = runtime_malloc(heap_struct_size, error_buf, error_buf_size))) { + goto fail1; + } + if (!(heap->heap_handle = + runtime_malloc(mem_allocator_get_heap_struct_size(), error_buf, + error_buf_size))) { + goto fail2; + } + heap->start_off_mem64 = UINT64_MAX - heap->size + 1; + heap->start_off_mem32 = UINT32_MAX - heap->size + 1; + + size = align_uint(size, os_getpagesize()); + if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { + set_error_buf(error_buf, error_buf_size, "invalid size of shared heap"); + goto fail3; + } + + if (!(heap->base_addr = wasm_mmap_linear_memory(size, size))) { + goto fail3; + } + if (!mem_allocator_create_with_struct_and_pool( + heap->heap_handle, heap_struct_size, heap->base_addr, size)) { + set_error_buf(error_buf, error_buf_size, "init share heap failed"); + goto fail4; + } + + os_mutex_lock(&shared_heap_list_lock); + if (shared_heap_list == NULL) { + shared_heap_list = heap; + } + else { + heap->next = shared_heap_list; + shared_heap_list = heap; + } + os_mutex_unlock(&shared_heap_list_lock); + return heap; + +fail4: + wasm_munmap_linear_memory(heap->base_addr, size, size); + +fail3: + wasm_runtime_free(heap->heap_handle); +fail2: + wasm_runtime_free(heap); +fail1: + return NULL; +} + +bool +wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + return wasm_cluster_attach_shared_heap(module_inst, shared_heap); +#else + return wasm_runtime_attach_shared_heap_internal(module_inst, shared_heap); +#endif +} + +bool +wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap) +{ + uint64 linear_mem_size = 0; + WASMMemoryInstance *memory = NULL; + WASMSharedHeap *heap = (WASMSharedHeap *)shared_heap; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + // check if linear memory and shared heap are overlapped + linear_mem_size = memory->memory_data_size; + + if ((memory->is_memory64 && linear_mem_size > heap->start_off_mem64) + || (!memory->is_memory64 && linear_mem_size > heap->start_off_mem32)) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + return false; + } + + if (module_inst->module_type == Wasm_Module_Bytecode) { + if (((WASMModuleInstance *)module_inst)->e->shared_heap) { + LOG_WARNING("A shared heap is already attached"); + return false; + } + ((WASMModuleInstance *)module_inst)->e->shared_heap = heap; + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + return true; +} + +void +wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_THREAD_MGR != 0 + wasm_cluster_detach_shared_heap(module_inst); +#else + wasm_runtime_detach_shared_heap_internal(module_inst); +#endif +} + +void +wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) +{ + if (module_inst->module_type == Wasm_Module_Bytecode) { + ((WASMModuleInstance *)module_inst)->e->shared_heap = NULL; + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } +} + +static bool +is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, + bool is_memory64, uint64 app_offset, uint32 bytes) +{ + WASMSharedHeap *heap = NULL; + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; + } + else if (module_inst_comm->module_type == Wasm_Module_AoT) { + // TODO + } + + if (!heap) { + return false; + } + + if (!is_memory64) { + if (app_offset >= heap->start_off_mem32 + && app_offset <= UINT32_MAX - bytes + 1) { + return true; + } + } + else { + if (app_offset >= heap->start_off_mem64 + && app_offset <= UINT64_MAX - bytes + 1) { + return true; + } + } + return false; +} + +static bool +is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, + uint8 *addr, uint32 bytes) +{ + WASMSharedHeap *heap = NULL; + + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; + } + else if (module_inst_comm->module_type == Wasm_Module_AoT) { + // TODO + } + + if (heap && addr >= heap->base_addr + && addr + bytes <= heap->base_addr + heap->size + && addr + bytes > addr) { + return true; + } + return false; +} + +static uint64 +shared_heap_addr_native_to_app(WASMModuleInstanceCommon *module_inst, + WASMMemoryInstance *memory, void *addr) +{ + WASMSharedHeap *heap = NULL; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (!heap) { + LOG_WARNING("Wasm module doesn't attach to a shared heap"); + return 0; + } + if (!addr) { + LOG_WARNING("Invalid address"); + return 0; + } + + if (memory && memory->is_memory64) { + return heap->start_off_mem64 + ((uint8 *)addr - heap->base_addr); + } + else if (memory && !memory->is_memory64) { + return heap->start_off_mem32 + ((uint8 *)addr - heap->base_addr); + } + return 0; +} + +static void * +shared_heap_addr_app_to_native(WASMModuleInstanceCommon *module_inst, + WASMMemoryInstance *memory, uint64 ptr) +{ + void *addr = NULL; + WASMSharedHeap *heap = NULL; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (!heap) { + LOG_WARNING("Wasm module doesn't attach to a shared heap"); + return NULL; + } + + if (!memory) { + LOG_WARNING("Wasm memory is not initialized"); + return NULL; + } + + if (memory->is_memory64) { + addr = heap->base_addr + (ptr - heap->start_off_mem64); + } + else { + addr = heap->base_addr + (ptr - heap->start_off_mem32); + } + + return addr; +} + +static uint64 +shared_heap_get_addr_start(WASMSharedHeap *heap, WASMMemoryInstance *memory) +{ + uint64 shared_heap_start = 0; + + if (!heap || !memory) { + LOG_ERROR("Invalid heap or memory"); + return 0; + } + + if (memory && !memory->is_memory64) { + shared_heap_start = heap->start_off_mem32; + } + else if (memory && memory->is_memory64) { + shared_heap_start = heap->start_off_mem64; + } + + return shared_heap_start; +} + +uint64 +wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, + uint64_t size, void **p_native_addr) +{ + WASMSharedHeap *heap = NULL; + WASMMemoryInstance *memory = NULL; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (heap) { + *p_native_addr = mem_allocator_malloc(heap->heap_handle, size); + + return shared_heap_addr_native_to_app(module_inst, memory, + *p_native_addr); + } + else { + LOG_WARNING("Wasm module doesn't attach to a shared heap"); + } + return 0; +} + +void +wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr) +{ + WASMSharedHeap *heap = NULL; + WASMMemoryInstance *memory = NULL; + void *addr = NULL; + + if (module_inst->module_type == Wasm_Module_Bytecode) { + heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (!heap) { + LOG_WARNING("Wasm module doesn't attach to a shared heap"); + return; + } + + addr = shared_heap_addr_app_to_native(module_inst, memory, ptr); + + if (heap) { + mem_allocator_free(heap->base_addr, addr); + } +} +#endif + bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, const MemAllocOption *alloc_option) { + bool ret = false; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (os_mutex_init(&shared_heap_list_lock)) { + return false; + } +#endif if (mem_alloc_type == Alloc_With_Pool) { - return wasm_memory_init_with_pool(alloc_option->pool.heap_buf, - alloc_option->pool.heap_size); + ret = wasm_memory_init_with_pool(alloc_option->pool.heap_buf, + alloc_option->pool.heap_size); } else if (mem_alloc_type == Alloc_With_Allocator) { - return wasm_memory_init_with_allocator( + ret = wasm_memory_init_with_allocator( #if WASM_MEM_ALLOC_WITH_USER_DATA != 0 alloc_option->allocator.user_data, #endif @@ -151,16 +514,48 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, } else if (mem_alloc_type == Alloc_With_System_Allocator) { memory_mode = MEMORY_MODE_SYSTEM_ALLOCATOR; - return true; + ret = true; } - else { - return false; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (!ret) { + os_mutex_destroy(&shared_heap_list_lock); + } +#endif + + return ret; +} + +#if WASM_ENABLE_SHARED_HEAP != 0 +static void +wasm_runtime_shared_heap_destroy() +{ + WASMSharedHeap *heap = shared_heap_list; + WASMSharedHeap *cur; + int ret = 0; + + while (heap) { + cur = heap; + heap = heap->next; + ret = ret + mem_allocator_destroy(cur->heap_handle); + wasm_runtime_free(cur->heap_handle); + wasm_munmap_linear_memory(cur->base_addr, cur->size, cur->size); + wasm_runtime_free(cur); + } + + if (ret != 0) { + LOG_ERROR("Memory leak detected in shared heap"); } } +#endif void wasm_runtime_memory_destroy(void) { + +#if WASM_ENABLE_SHARED_HEAP != 0 + wasm_runtime_shared_heap_destroy(); +#endif + if (memory_mode == MEMORY_MODE_POOL) { #if BH_ENABLE_GC_VERIFY == 0 (void)mem_allocator_destroy(pool_allocator); @@ -342,7 +737,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, /* boundary overflow check */ if (size > max_linear_memory_size || app_offset > max_linear_memory_size - size) { - goto fail; + goto shared_heap_bound_check; } SHARED_MEMORY_LOCK(memory_inst); @@ -354,6 +749,13 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); +shared_heap_bound_check: +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_offset, size)) { + return true; + } +#endif fail: wasm_set_exception(module_inst, "out of bounds memory access"); return false; @@ -439,6 +841,13 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, return true; } +#if WASM_ENABLE_SHARED_HEAP != 0 + else if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, + size)) { + SHARED_MEMORY_UNLOCK(memory_inst); + return true; + } +#endif SHARED_MEMORY_UNLOCK(memory_inst); fail: @@ -475,6 +884,19 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); return addr; } +#if WASM_ENABLE_SHARED_HEAP != 0 + else if (is_app_addr_in_shared_heap(module_inst_comm, + memory_inst->is_memory64, + app_offset, 1)) { + uint64 heap_start = shared_heap_get_addr_start( + module_inst->e->shared_heap, memory_inst); + uint64 heap_offset = (uint64)app_offset - heap_start; + + addr = module_inst->e->shared_heap->base_addr + heap_offset; + SHARED_MEMORY_UNLOCK(memory_inst); + return addr; + } +#endif SHARED_MEMORY_UNLOCK(memory_inst); return NULL; } @@ -499,6 +921,11 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, bounds_checks = is_bounds_checks_enabled(module_inst_comm); +#if WASM_ENABLE_SHARED_HEAP != 0 + /* If shared heap is enabled, bounds check is always needed */ + bounds_checks = true; +#endif + memory_inst = wasm_get_default_memory(module_inst); if (!memory_inst) { return 0; @@ -513,6 +940,17 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); return ret; } + else { +#if WASM_ENABLE_SHARED_HEAP != 0 + uint64 shared_heap_start = shared_heap_get_addr_start( + module_inst->e->shared_heap, memory_inst); + ret = + (uint64)(addr - (uint8 *)module_inst->e->shared_heap->base_addr) + + shared_heap_start; + SHARED_MEMORY_UNLOCK(memory_inst); + return ret; +#endif + } } /* If bounds checks is disabled, return the offset directly */ else if (addr != NULL) { @@ -765,6 +1203,11 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, #else WASMMemoryInstance *memory = wasm_get_default_memory(module); #endif + +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *heap; +#endif + uint8 *memory_data_old, *memory_data_new, *heap_data_old; uint32 num_bytes_per_page, heap_size; uint32 cur_page_count, max_page_count, total_page_count; @@ -797,6 +1240,19 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, total_page_count = inc_page_count + cur_page_count; total_size_new = num_bytes_per_page * (uint64)total_page_count; +#if WASM_ENABLE_SHARED_HEAP != 0 + heap = module->e->shared_heap; + if (memory->is_memory64 && total_size_new > heap->start_off_mem64) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } + else if (!memory->is_memory64 && total_size_new > heap->start_off_mem32) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } +#endif if (inc_page_count <= 0) /* No need to enlarge memory */ return true; diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index 2f20d3f68..be89772e3 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -41,6 +41,32 @@ SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size) #define SET_LINEAR_MEMORY_SIZE(memory, size) memory->memory_data_size = size #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +WASMSharedHeap * +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, + uint32 error_buf_size); + +bool +wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap); +bool +wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap); + +void +wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst); +void +wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst); + +uint64 +wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, + uint64 size, void **p_native_addr); + +void +wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, + uint64 ptr); +#endif + bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, const MemAllocOption *alloc_option); diff --git a/core/iwasm/common/wasm_native.c b/core/iwasm/common/wasm_native.c index 0ff3053fa..060bb2c3d 100644 --- a/core/iwasm/common/wasm_native.c +++ b/core/iwasm/common/wasm_native.c @@ -33,6 +33,11 @@ uint32 get_spectest_export_apis(NativeSymbol **p_libc_builtin_apis); #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +uint32 +get_lib_shared_heap_export_apis(NativeSymbol **p_shared_heap_apis); +#endif + uint32 get_libc_wasi_export_apis(NativeSymbol **p_libc_wasi_apis); @@ -512,6 +517,14 @@ wasm_native_init() goto fail; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + n_native_symbols = get_lib_shared_heap_export_apis(&native_symbols); + if (n_native_symbols > 0 + && !wasm_native_register_natives("env", native_symbols, + n_native_symbols)) + goto fail; +#endif + #if WASM_ENABLE_BASE_LIB != 0 n_native_symbols = get_base_lib_export_apis(&native_symbols); if (n_native_symbols > 0 diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 569c4deaa..5ce617ef1 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -135,6 +135,9 @@ typedef struct wasm_section_t { struct WASMExecEnv; typedef struct WASMExecEnv *wasm_exec_env_t; +struct WASMSharedHeap; +typedef struct WASMSharedHeap *wasm_shared_heap_t; + /* Package Type */ typedef enum { Wasm_Module_Bytecode = 0, @@ -320,6 +323,12 @@ typedef enum { WASM_LOG_LEVEL_VERBOSE = 4 } log_level_t; +#if WASM_ENABLE_SHARED_HEAP != 0 +typedef struct SharedHeapInitArgs { + uint32 size; +} SharedHeapInitArgs; +#endif + /** * Initialize the WASM runtime environment, and also initialize * the memory allocator with system allocator, which calls os_malloc @@ -2110,6 +2119,52 @@ wasm_runtime_detect_native_stack_overflow_size(wasm_exec_env_t exec_env, WASM_RUNTIME_API_EXTERN bool wasm_runtime_is_underlying_binary_freeable(const wasm_module_t module); +#if WASM_ENABLE_SHARED_HEAP != 0 +/** + * Create a shared heap + * @param init_args the initialization arguments + * @param error_buf buffer to output the error info if failed + * @param error_buf_size the size of the error buffer + */ +WASM_RUNTIME_API_EXTERN wasm_shared_heap_t +wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, + uint32 error_buf_size); + +/** + * Attach a shared heap to a module instance + * @param module_inst the module instance + * @param shared_heap the shared heap + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst, + wasm_shared_heap_t shared_heap); + +/** + * Detach a shared heap from a module instance + * @param module_inst the module instance + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst); + +/** + * Allocate memory from a shared heap + * @param module_inst the module instance + * @param size required memory size + * @param p_native_addr native address of allocated memory + */ +WASM_RUNTIME_API_EXTERN uint64 +wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64 size, + void **p_native_addr); + +/** + * Free the memory allocated from shared heap + * @param module_inst the module instance + * @param ptr the offset in wasm app + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr); +#endif + #ifdef __cplusplus } #endif diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 67f8c2d45..f39ae3468 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -48,6 +48,31 @@ typedef float64 CellType_F64; #if WASM_ENABLE_MEMORY64 == 0 +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 +#define GOTO_OUT_OF_BOUNDS goto out_of_bounds; +#else +#define GOTO_OUT_OF_BOUNDS +#endif + +#if WASM_ENABLE_SHARED_HEAP != 0 +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ + do { \ + if (offset1 + bytes >= UINT32_MAX - module->shared_heap->size \ + && offset1 + bytes <= UINT32_MAX) { \ + uint64 heap_start = UINT32_MAX - module->shared_heap->size; \ + uint64 heap_offset = (uint64)offset1 - heap_start; \ + maddr = module->shared_heap->data + heap_offset; \ + } \ + else { \ + GOTO_OUT_OF_BOUNDS; \ + } \ + } while (0) +#else +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS +#endif + #if (!defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0) #define CHECK_MEMORY_OVERFLOW(bytes) \ @@ -58,7 +83,7 @@ typedef float64 CellType_F64; be in valid range, no need to check it again. */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ @@ -69,7 +94,7 @@ typedef float64 CellType_F64; bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ } while (0) #else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ @@ -82,22 +107,40 @@ typedef float64 CellType_F64; #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ maddr = memory->memory_data + (uint32)(start); \ + uint64 offset1 = start; \ } while (0) #endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ #else /* else of WASM_ENABLE_MEMORY64 == 0 */ -#define CHECK_MEMORY_OVERFLOW(bytes) \ - do { \ - uint64 offset1 = (uint64)offset + (uint64)addr; \ - /* If memory64 is enabled, offset1, offset1 + bytes can overflow */ \ - if (disable_bounds_checks \ - || (offset1 >= offset && offset1 + bytes >= offset1 \ - && offset1 + bytes <= get_linear_mem_size())) \ - maddr = memory->memory_data + offset1; \ - else \ - goto out_of_bounds; \ +#if WASM_ENABLE_SHARED_HEAP != 0 +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ + do { \ + if (offset1 + bytes >= UINT64_MAX - module->shared_heap->size \ + && offset1 + bytes <= UINT64_MAX) { \ + uint64 heap_start = UINT64_MAX - module->shared_heap->size; \ + uint64 heap_offset = (uint64)offset1 - heap_start; \ + maddr = module->shared_heap->data + heap_offset; \ + } \ + else \ + goto out_of_bounds; \ + } while (0) +#else +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) goto out_of_bounds; +#endif + +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + /* If memory64 is enabled, offset1, offset1 + bytes can \ + * overflow */ \ + if (disable_bounds_checks \ + || (offset1 >= offset && offset1 + bytes >= offset1 \ + && offset1 + bytes <= get_linear_mem_size())) \ + maddr = memory->memory_data + offset1; \ + else \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ @@ -110,7 +153,7 @@ typedef float64 CellType_F64; bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ } while (0) #endif /* end of WASM_ENABLE_MEMORY64 == 0 */ diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 1d7ca8f90..84170f213 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -37,6 +37,31 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif +#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ + || WASM_ENABLE_BULK_MEMORY != 0 +#define GOTO_OUT_OF_BOUNDS goto out_of_bounds; +#else +#define GOTO_OUT_OF_BOUNDS +#endif + +#if WASM_ENABLE_SHARED_HEAP != 0 +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ + do { \ + if (offset1 + bytes >= UINT32_MAX - module->e->shared_heap->size \ + && offset1 + bytes <= UINT32_MAX) { \ + uint64 heap_start = UINT32_MAX - module->e->shared_heap->size; \ + uint64 heap_offset = (uint64)offset1 - heap_start; \ + maddr = module->e->shared_heap->base_addr + heap_offset; \ + } \ + else { \ + GOTO_OUT_OF_BOUNDS; \ + } \ + } while (0) +#else +#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS +#endif + #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 #define CHECK_MEMORY_OVERFLOW(bytes) \ @@ -47,7 +72,7 @@ typedef float64 CellType_F64; be in valid range, no need to check it again. */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ @@ -58,7 +83,7 @@ typedef float64 CellType_F64; bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - goto out_of_bounds; \ + CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ } while (0) #else #define CHECK_MEMORY_OVERFLOW(bytes) \ @@ -70,6 +95,7 @@ typedef float64 CellType_F64; #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ maddr = memory->memory_data + (uint32)(start); \ + uint64 offset1 = start; \ } while (0) #endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 8666541f2..3fa3682c0 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -92,6 +92,17 @@ typedef union { uint32 u32[2]; } MemBound; +#if WASM_ENABLE_SHARED_HEAP != 0 +typedef struct WASMSharedHeap { + struct WASMSharedHeap *next; + void *heap_handle; + uint8_t *base_addr; + uint32_t size; + uint64 start_off_mem64; + uint64 start_off_mem32; +} WASMSharedHeap; +#endif + struct WASMMemoryInstance { /* Module type */ uint32 module_type; @@ -353,6 +364,10 @@ typedef struct WASMModuleInstanceExtra { uint32 max_aux_stack_used; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; +#endif + #if WASM_ENABLE_DEBUG_INTERP != 0 \ || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ && WASM_ENABLE_LAZY_JIT != 0) diff --git a/core/iwasm/libraries/shared-heap/shared_heap.cmake b/core/iwasm/libraries/shared-heap/shared_heap.cmake new file mode 100644 index 000000000..ec91dabcd --- /dev/null +++ b/core/iwasm/libraries/shared-heap/shared_heap.cmake @@ -0,0 +1,8 @@ +# Copyright (C) 2024 Xiaomi Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set (LIB_SHARED_HEAP ${CMAKE_CURRENT_LIST_DIR}) +add_definitions (-DWASM_ENABLE_SHARED_HEAP=1) +include_directories(${LIB_SHARED_HEAP_DIR}) +file (GLOB source_all ${LIB_SHARED_HEAP}/*.c) +set (LIB_SHARED_HEAP_SOURCE ${source_all}) \ No newline at end of file diff --git a/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c new file mode 100644 index 000000000..164568ac2 --- /dev/null +++ b/core/iwasm/libraries/shared-heap/shared_heap_wrapper.c @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2024 Xiaomi Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_common.h" +#include "bh_log.h" +#include "wasm_export.h" +#include "../interpreter/wasm.h" +#include "../common/wasm_runtime_common.h" +/* clang-format off */ +#define validate_native_addr(addr, size) \ + wasm_runtime_validate_native_addr(module_inst, addr, size) + +#define module_shared_malloc(size, p_native_addr) \ + wasm_runtime_shared_heap_malloc(module_inst, size, p_native_addr) + +#define module_shared_free(offset) \ + wasm_runtime_shared_heap_free(module_inst, offset) +/* clang-format on */ + +static uint32 +shared_malloc_wrapper(wasm_exec_env_t exec_env, uint32 size) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + return (uint32)module_shared_malloc((uint64)size, NULL); +} + +static void +shared_free_wrapper(wasm_exec_env_t exec_env, void *ptr) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + if (!validate_native_addr(ptr, (uint64)sizeof(uint32))) + return; + + module_shared_free(addr_native_to_app(ptr)); +} + +/* clang-format off */ +#define REG_NATIVE_FUNC(func_name, signature) \ + { #func_name, func_name##_wrapper, signature, NULL } +/* clang-format on */ + +static NativeSymbol native_symbols_shared_heap[] = { + REG_NATIVE_FUNC(shared_malloc, "(i)i"), + REG_NATIVE_FUNC(shared_free, "(*)"), +}; + +uint32 +get_lib_shared_heap_export_apis(NativeSymbol **p_shared_heap_apis) +{ + *p_shared_heap_apis = native_symbols_shared_heap; + return sizeof(native_symbols_shared_heap) / sizeof(NativeSymbol); +} \ No newline at end of file diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index ebb56ba7c..27ff1441f 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -1402,6 +1402,83 @@ wasm_cluster_spread_custom_data(WASMModuleInstanceCommon *module_inst, } } +#if WASM_ENABLE_SHARED_HEAP != 0 +static void +attach_shared_heap_visitor(void *node, void *heap) +{ + WASMExecEnv *curr_exec_env = (WASMExecEnv *)node; + WASMModuleInstanceCommon *module_inst = get_module_inst(curr_exec_env); + + wasm_runtime_attach_shared_heap_internal(module_inst, heap); +} + +static void +detach_shared_heap_visitor(void *node, void *heap) +{ + WASMExecEnv *curr_exec_env = (WASMExecEnv *)node; + WASMModuleInstanceCommon *module_inst = get_module_inst(curr_exec_env); + + (void)heap; + wasm_runtime_detach_shared_heap_internal(module_inst); +} + +bool +wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *heap) +{ + WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst); + + if (module_inst->module_type == Wasm_Module_Bytecode) { + if (((WASMModuleInstance *)module_inst)->e->shared_heap) { + LOG_WARNING("A shared heap is already attached"); + return false; + } + } + else if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } + + if (exec_env == NULL) { + /* Maybe threads have not been started yet. */ + return wasm_runtime_attach_shared_heap_internal(module_inst, heap); + } + else { + WASMCluster *cluster; + + cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + + os_mutex_lock(&cluster->lock); + traverse_list(&cluster->exec_env_list, attach_shared_heap_visitor, + heap); + os_mutex_unlock(&cluster->lock); + } + return true; +} + +void +wasm_cluster_detach_shared_heap(WASMModuleInstanceCommon *module_inst) +{ + WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst); + + if (exec_env == NULL) { + /* Maybe threads have not been started yet. */ + wasm_runtime_detach_shared_heap_internal(module_inst); + } + else { + WASMCluster *cluster; + + cluster = wasm_exec_env_get_cluster(exec_env); + bh_assert(cluster); + + os_mutex_lock(&cluster->lock); + traverse_list(&cluster->exec_env_list, detach_shared_heap_visitor, + NULL); + os_mutex_unlock(&cluster->lock); + } +} +#endif + #if WASM_ENABLE_MODULE_INST_CONTEXT != 0 struct inst_set_context_data { void *key; diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.h b/core/iwasm/libraries/thread-mgr/thread_manager.h index 7ad6c772a..90d97b0be 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.h +++ b/core/iwasm/libraries/thread-mgr/thread_manager.h @@ -11,6 +11,9 @@ #include "wasm_export.h" #include "../interpreter/wasm.h" #include "../common/wasm_runtime_common.h" +#if WASM_ENABLE_SHARED_HEAP != 0 +#include "../common/wasm_memory.h" +#endif #ifdef __cplusplus extern "C" { @@ -167,6 +170,15 @@ wasm_cluster_set_context(WASMModuleInstanceCommon *module_inst, void *key, bool wasm_cluster_is_thread_terminated(WASMExecEnv *exec_env); +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *heap); + +void +wasm_cluster_detach_shared_heap(WASMModuleInstanceCommon *module_inst); +#endif + #if WASM_ENABLE_DEBUG_INTERP != 0 #define WAMR_SIG_TRAP (5) #define WAMR_SIG_STOP (19)