gnh1201/wasm-micro-runtime
1
0
Fork 0
mirror of https://github.com/bytecodealliance/wasm-micro-runtime.git synced 2025-05-29 13:07:10 +00:00
Code Issues Actions 10 Packages Projects Releases Wiki Activity

first draft of shared heap enhancement in interpreter and runtime

Browse Source
This commit is contained in:
TL 2025-01-23 18:20:01 +08:00
parent 3179b10d8a
commit 013306d67c
2 changed files with 232 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

267
core/iwasm/common/wasm_memory.c
View File

@ -143,7 +143,7 @@ is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst)
#if WASM_ENABLE_SHARED_HEAP != 0
static void *
wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size);
wasm_mmap_linear_memory(uint64 map_size, uint64 commit_size);
static void
wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size,
uint64 map_size);
@ -177,39 +177,53 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args)
goto fail1;
}
if (!(heap->heap_handle =
runtime_malloc(mem_allocator_get_heap_struct_size()))) {
goto fail2;
}
size = align_uint(size, os_getpagesize());
heap->size = size;
heap->start_off_mem64 = UINT64_MAX - heap->size + 1;
heap->start_off_mem32 = UINT32_MAX - heap->size + 1;
heap->attached_count = 0;
if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) {
LOG_WARNING("Invalid size of shared heap");
goto fail3;
if (init_args->pre_allocated_addr != NULL) {
/* Create shared heap from a pre allocated buffer, its size need to
* align with system page */
if (size != init_args->size) {
LOG_WARNING("Pre allocated size need to be aligned with system "
"page size to create shared heap");
goto fail1;
}
heap->heap_handle = NULL;
}
else {
if (!(heap->heap_handle =
runtime_malloc(mem_allocator_get_heap_struct_size()))) {
goto fail2;
}
if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) {
LOG_WARNING("Invalid size of shared heap");
goto fail3;
}
#ifndef OS_ENABLE_HW_BOUND_CHECK
map_size = size;
map_size = size;
#else
/* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
* ea = i + memarg.offset
* both i and memarg.offset are u32 in range 0 to 4G
* so the range of ea is 0 to 8G
*/
map_size = 8 * (uint64)BH_GB;
/* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
* ea = i + memarg.offset
* both i and memarg.offset are u32 in range 0 to 4G
* so the range of ea is 0 to 8G
*/
map_size = 8 * (uint64)BH_GB;
#endif
if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) {
goto fail3;
}
if (!mem_allocator_create_with_struct_and_pool(
heap->heap_handle, heap_struct_size, heap->base_addr, size)) {
LOG_WARNING("init share heap failed");
goto fail4;
if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) {
goto fail3;
}
if (!mem_allocator_create_with_struct_and_pool(
heap->heap_handle, heap_struct_size, heap->base_addr, size)) {
LOG_WARNING("init share heap failed");
goto fail4;
}
}
os_mutex_lock(&shared_heap_list_lock);
@ -233,6 +247,74 @@ fail1:
return NULL;
}
WASMSharedHeap *
wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body)
{
WASMSharedHeap *cur;
bool heap_handle_exist = false;
if (!head || !body) {
LOG_WARNING("Invalid shared heap to chain.");
return NULL;
}
os_mutex_lock(&shared_heap_list_lock);
if (head->attached_count != 0 || body->attached_count != 0) {
LOG_WARNING("To create shared heap chain, all shared heap need to be "
"detached first.");
os_mutex_unlock(&shared_heap_list_lock);
return NULL;
}
for (cur = head; cur; cur = cur->chain_next) {
if (cur->heap_handle && heap_handle_exist) {
LOG_WARNING(
"To create shared heap chain, only one of shared heap can "
"dynamically shared_heap_malloc and shared_heap_free, the rest "
"can only be pre-allocated shared heap");
os_mutex_unlock(&shared_heap_list_lock);
return NULL;
}
if (cur->heap_handle)
heap_handle_exist = true;
}
head->start_off_mem64 = body->start_off_mem64 - head->size + 1;
head->start_off_mem32 = body->start_off_mem32 - head->size + 1;
head->chain_next = body;
os_mutex_unlock(&shared_heap_list_lock);
return head;
}
WASMSharedHeap *
wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain)
{
WASMSharedHeap *cur;
if (!head || !head->chain_next) {
LOG_WARNING("Invalid shared heap chain to disconnect the head from.");
return NULL;
}
os_mutex_lock(&shared_heap_list_lock);
if (head->attached_count != 0) {
LOG_WARNING("To disconnect the shared heap head from the shared heap "
"chain, the shared heap chain needs to be detached first.");
os_mutex_unlock(&shared_heap_list_lock);
return NULL;
}
for (cur = head; cur && cur->chain_next; cur = cur->chain_next) {
cur->start_off_mem64 = UINT64_MAX - cur->size + 1;
cur->start_off_mem32 = UINT32_MAX - cur->size + 1;
if (!entire_chain)
break;
}
os_mutex_unlock(&shared_heap_list_lock);
return cur;
}
bool
wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *shared_heap)
@ -303,6 +385,9 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst,
}
#endif /* end of WASM_ENABLE_AOT != 0 */
os_mutex_lock(&shared_heap_list_lock);
shared_heap->attached_count++;
os_mutex_unlock(&shared_heap_list_lock);
return true;
}
@ -324,6 +409,11 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst)
if (module_inst->module_type == Wasm_Module_Bytecode) {
WASMModuleInstanceExtra *e =
(WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e;
if (e->shared_heap != NULL) {
os_mutex_lock(&shared_heap_list_lock);
e->shared_heap->attached_count--;
os_mutex_unlock(&shared_heap_list_lock);
}
e->shared_heap = NULL;
#if WASM_ENABLE_JIT != 0
#if UINTPTR_MAX == UINT64_MAX
@ -339,6 +429,11 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst)
if (module_inst->module_type == Wasm_Module_AoT) {
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e;
if (e->shared_heap != NULL) {
os_mutex_lock(&shared_heap_list_lock);
e->shared_heap->attached_count--;
os_mutex_unlock(&shared_heap_list_lock);
}
e->shared_heap = NULL;
#if UINTPTR_MAX == UINT64_MAX
e->shared_heap_start_off.u64 = UINT64_MAX;
@ -387,9 +482,11 @@ wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm)
static bool
is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
bool is_memory64, uint64 app_offset, uint32 bytes)
bool is_memory64, uint64 app_offset, uint32 bytes,
WASMSharedHeap **target_heap)
{
WASMSharedHeap *heap = get_shared_heap(module_inst);
WASMSharedHeap *heap = get_shared_heap(module_inst), *cur;
uint64 shared_heap_start, shared_heap_end;
if (!heap) {
return false;
@ -399,57 +496,66 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
bytes = 1;
}
if (!is_memory64) {
if (app_offset >= heap->start_off_mem32
&& app_offset <= UINT32_MAX - bytes + 1) {
return true;
}
}
else {
if (app_offset >= heap->start_off_mem64
&& app_offset <= UINT64_MAX - bytes + 1) {
for (cur = heap; cur; cur = cur->chain_next) {
shared_heap_start =
cur->start_off_mem64 ? is_memory64 : cur->start_off_mem32;
shared_heap_end = shared_heap_start + cur->size;
if (app_offset >= shared_heap_start && app_offset <= shared_heap_end) {
if (target_heap)
*target_heap = cur;
return true;
}
}
if (target_heap)
*target_heap = NULL;
return false;
}
static bool
is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
uint8 *addr, uint32 bytes)
uint8 *addr, uint32 bytes,
WASMSharedHeap **target_heap)
{
WASMSharedHeap *heap = get_shared_heap(module_inst);
uintptr_t base_addr;
uintptr_t addr_int;
uintptr_t end_addr;
WASMSharedHeap *cur, *heap_head = get_shared_heap(module_inst);
uintptr_t base_addr, addr_int, end_addr;
if (!heap) {
if (!heap_head) {
return false;
}
base_addr = (uintptr_t)heap->base_addr;
addr_int = (uintptr_t)addr;
if (addr_int < base_addr) {
return false;
/* Iterate through shared heap chain to find whether native addr in one of
* shared heap */
for (cur = heap_head; cur != NULL; cur = cur->chain_next) {
base_addr = (uintptr_t)cur->base_addr;
addr_int = (uintptr_t)addr;
if (addr_int < base_addr) {
continue;
}
end_addr = addr_int + bytes;
/* Check for overflow */
if (end_addr <= addr_int) {
continue;
}
if (end_addr > base_addr + cur->size) {
continue;
}
if (target_heap)
*target_heap = cur;
return true;
}
end_addr = addr_int + bytes;
/* Check for overflow */
if (end_addr <= addr_int) {
return false;
}
if (end_addr > base_addr + heap->size) {
return false;
}
return true;
if (target_heap)
*target_heap = NULL;
return false;
}
uint64
wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst,
uint64_t size, void **p_native_addr)
uint64 size, void **p_native_addr)
{
WASMMemoryInstance *memory =
wasm_get_default_memory((WASMModuleInstance *)module_inst);
@ -459,6 +565,14 @@ wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst,
if (!memory || !shared_heap)
return 0;
while (shared_heap && !shared_heap->heap_handle) {
shared_heap = shared_heap->chain_next;
}
if (!shared_heap) {
LOG_WARNING("Can't allocate from pre allocated shared heap");
return 0;
}
native_addr = mem_allocator_malloc(shared_heap->heap_handle, size);
if (!native_addr)
return 0;
@ -487,6 +601,14 @@ wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr)
return;
}
while (shared_heap && !shared_heap->heap_handle) {
shared_heap = shared_heap->chain_next;
}
if (!shared_heap) {
LOG_WARNING("The address to free is from pre allocated shared heap");
return;
}
if (memory->is_memory64) {
if (ptr < shared_heap->start_off_mem64) { /* ptr can not > UINT64_MAX */
LOG_WARNING("The address to free isn't in shared heap");
@ -564,14 +686,16 @@ destroy_shared_heaps()
while (heap) {
cur = heap;
heap = heap->next;
mem_allocator_destroy(cur->heap_handle);
wasm_runtime_free(cur->heap_handle);
if (cur->heap_handle) {
mem_allocator_destroy(cur->heap_handle);
wasm_runtime_free(cur->heap_handle);
#ifndef OS_ENABLE_HW_BOUND_CHECK
map_size = cur->size;
map_size = cur->size;
#else
map_size = 8 * (uint64)BH_GB;
map_size = 8 * (uint64)BH_GB;
#endif
wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size);
wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size);
}
wasm_runtime_free(cur);
}
os_mutex_destroy(&shared_heap_list_lock);
@ -761,7 +885,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm,
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_offset, size)) {
app_offset, size, NULL)) {
return true;
}
#endif
@ -812,9 +936,9 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm,
}
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap;
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_str_offset, 1)) {
WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm);
app_str_offset, 1, &shared_heap)) {
str = (char *)shared_heap->base_addr
+ (memory_inst->is_memory64
? (app_str_offset - shared_heap->start_off_mem64)
@ -884,7 +1008,8 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm,
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size)) {
if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size,
NULL)) {
return true;
}
#endif
@ -910,6 +1035,7 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm,
{
WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm;
WASMMemoryInstance *memory_inst;
WASMSharedHeap *shared_heap;
uint8 *addr;
bool bounds_checks;
@ -925,8 +1051,7 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm,
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_offset, 1)) {
WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm);
app_offset, 1, &shared_heap)) {
uint64 shared_heap_start = 0;
if (memory_inst && !memory_inst->is_memory64) {
@ -985,8 +1110,9 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm,
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1)) {
WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm);
WASMSharedHeap *shared_heap;
if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1,
&shared_heap)) {
uint64 shared_heap_start = 0;
if (memory_inst && !memory_inst->is_memory64) {
@ -1112,8 +1238,7 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst,
memory_inst->is_memory64, app_buf_addr,
app_buf_size)) {
shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module_inst);
app_buf_size, &shared_heap)) {
native_addr = shared_heap->base_addr
+ (memory_inst->is_memory64
? (app_buf_addr - shared_heap->start_off_mem64)

50
core/iwasm/interpreter/wasm_interp_classic.c
View File

@ -53,9 +53,34 @@ typedef float64 CellType_F64;
#else
#define is_default_memory true
#endif
#define app_addr_in_shared_heap(app_addr, bytes) \
(shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \
&& (app_addr) <= shared_heap_end_off - bytes + 1)
#if WASM_ENABLE_MEMORY64
#define get_shared_heap_start_off(shared_heap) \
(is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32)
#else
#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32)
#endif
#define app_addr_in_shared_heap(app_addr, bytes) \
(shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \
&& (app_addr) <= shared_heap_end_off - bytes + 0) \
|| ({ \
bool in_chain = false; \
WASMSharedHeap *cur; \
uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \
for (cur = shared_heap; cur; cur = cur->chain_next) { \
cur_shared_heap_start_off = get_shared_heap_start_off(cur); \
cur_shared_heap_end_off = \
cur_shared_heap_start_off + cur->size - 1; \
if ((app_addr) >= cur_shared_heap_start_off \
&& (app_addr) <= cur_shared_heap_end_off - bytes + 0) { \
shared_heap_start_off = cur_shared_heap_start_off; \
shared_heap_end_off = cur_shared_heap_end_off; \
shared_heap_base_addr = cur->base_addr; \
in_chain = true; \
break; \
} \
} \
in_chain; \
})
#define shared_heap_addr_app_to_native(app_addr, native_addr) \
native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off)
@ -1647,18 +1672,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap = module->e->shared_heap;
uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL;
#if WASM_ENABLE_MEMORY64 != 0
uint64 shared_heap_start_off =
shared_heap ? (is_memory64 ? shared_heap->start_off_mem64
: shared_heap->start_off_mem32)
: 0;
shared_heap ? get_shared_heap_start_off(shared_heap) : 0;
uint64 shared_heap_end_off =
shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0;
#else
uint64 shared_heap_start_off =
shared_heap ? shared_heap->start_off_mem32 : 0;
uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0;
#endif
shared_heap
? (get_shared_heap_start_off(shared_heap) + shared_heap->size - 1)
: 0;
#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */
#if WASM_ENABLE_MULTI_MEMORY != 0
uint32 memidx = 0;
@ -1697,7 +1716,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
goto got_exception;
}
HANDLE_OP(WASM_OP_NOP) { HANDLE_OP_END(); }
HANDLE_OP(WASM_OP_NOP)
{
HANDLE_OP_END();
}
#if WASM_ENABLE_EXCE_HANDLING != 0
HANDLE_OP(WASM_OP_RETHROW)