diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 99260bfc8..704d52547 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -145,7 +145,6 @@ jobs: "-DWAMR_BUILD_SIMD=1", "-DWAMR_BUILD_TAIL_CALL=1", "-DWAMR_DISABLE_HW_BOUND_CHECK=1", - "-DWAMR_ENABLE_SHARED_MEMORY_MMAP=1", ] os: [ubuntu-22.04] platform: [android, linux] diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 22783a509..f370927e8 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -248,12 +248,6 @@ if (WAMR_BUILD_SHARED_MEMORY EQUAL 1) else () add_definitions (-DWASM_ENABLE_SHARED_MEMORY=0) endif () -if (WAMR_ENABLE_SHARED_MEMORY_MMAP EQUAL 1) - add_definitions (-DWASM_ENABLE_SHARED_MEMORY_MMAP=1) - message (" Shared memory allocated using mmap enabled") -else () - add_definitions (-DWASM_ENABLE_SHARED_MEMORY_MMAP=0) -endif () if (WAMR_BUILD_THREAD_MGR EQUAL 1) message (" Thread manager enabled") endif () diff --git a/core/config.h b/core/config.h index 25c45107e..fea449f81 100644 --- a/core/config.h +++ b/core/config.h @@ -188,6 +188,10 @@ #define WASM_ENABLE_APP_FRAMEWORK 0 #endif +#ifndef WASM_HAVE_MREMAP +#define WASM_HAVE_MREMAP 0 +#endif + /* Bulk memory operation */ #ifndef WASM_ENABLE_BULK_MEMORY #define WASM_ENABLE_BULK_MEMORY 0 @@ -521,9 +525,4 @@ #define WASM_ENABLE_QUICK_AOT_ENTRY 1 #endif -/* Disable mmap based shared memory by default */ -#ifndef WASM_ENABLE_SHARED_MEMORY_MMAP -#define WASM_ENABLE_SHARED_MEMORY_MMAP 0 -#endif - #endif /* end of _CONFIG_H_ */ diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 731cacbea..6a5681a6e 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -340,9 +340,6 @@ tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module, static void memories_deinstantiate(AOTModuleInstance *module_inst) { -#ifdef WASM_LINEAR_MEMORY_MMAP - uint64 map_size; -#endif uint32 i; AOTMemoryInstance *memory_inst; @@ -364,23 +361,7 @@ memories_deinstantiate(AOTModuleInstance *module_inst) } if (memory_inst->memory_data) { -#ifndef OS_ENABLE_HW_BOUND_CHECK -#ifdef WASM_LINEAR_MEMORY_MMAP - if (shared_memory_is_shared(memory_inst)) { - map_size = (uint64)memory_inst->num_bytes_per_page - * memory_inst->max_page_count; - wasm_munmap_linear_memory(memory_inst->memory_data, - map_size, map_size); - } - else -#endif - wasm_runtime_free(memory_inst->memory_data); -#else - map_size = (uint64)memory_inst->num_bytes_per_page - * memory_inst->cur_page_count; - wasm_munmap_linear_memory(memory_inst->memory_data, map_size, - 8 * (uint64)BH_GB); -#endif + wasm_deallocate_linear_memory(memory_inst); } } } @@ -402,14 +383,10 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, uint32 heap_offset = num_bytes_per_page * init_page_count; uint64 memory_data_size, max_memory_data_size; uint8 *p = NULL, *global_addr; -#ifdef WASM_LINEAR_MEMORY_MMAP - uint8 *mapped_mem = NULL; - uint64 map_size; -#endif + bool is_shared_memory = false; #if WASM_ENABLE_SHARED_MEMORY != 0 - bool is_shared_memory = memory->memory_flags & 0x02 ? true : false; - + is_shared_memory = memory->memory_flags & 0x02 ? true : false; /* Shared memory */ if (is_shared_memory && parent != NULL) { AOTMemoryInstance *shared_memory_instance; @@ -519,55 +496,18 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, module->aux_stack_size); LOG_VERBOSE(" heap offset: %u, heap size: %d\n", heap_offset, heap_size); - memory_data_size = (uint64)num_bytes_per_page * init_page_count; max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; - bh_assert(memory_data_size <= UINT32_MAX); bh_assert(max_memory_data_size <= 4 * (uint64)BH_GB); (void)max_memory_data_size; -#ifndef OS_ENABLE_HW_BOUND_CHECK -#if WASM_ENABLE_SHARED_MEMORY != 0 - if (is_shared_memory) { -#if WASM_ENABLE_SHARED_MEMORY_MMAP != 0 - map_size = max_memory_data_size; - if (max_memory_data_size > 0 - && !(p = mapped_mem = - wasm_mmap_linear_memory(map_size, &max_memory_data_size, - error_buf, error_buf_size))) { - return NULL; - } -#else - /* Allocate maximum memory size when memory is shared */ - if (max_memory_data_size > 0 - && !(p = runtime_malloc(max_memory_data_size, error_buf, - error_buf_size))) { - return NULL; - } -#endif - } - else -#endif /* end of WASM_ENABLE_SHARED_MEMORY != 0 */ - { - /* Allocate initial memory size when memory is not shared */ - if (memory_data_size > 0 - && !(p = runtime_malloc(memory_data_size, error_buf, - error_buf_size))) { - return NULL; - } - } -#else /* else of OS_ENABLE_HW_BOUND_CHECK */ - /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: - * ea = i + memarg.offset - * both i and memarg.offset are u32 in range 0 to 4G - * so the range of ea is 0 to 8G - */ - map_size = 8 * (uint64)BH_GB; - if (!(p = mapped_mem = wasm_mmap_linear_memory( - map_size, &memory_data_size, error_buf, error_buf_size))) { - set_error_buf(error_buf, error_buf_size, "mmap memory failed"); + if (wasm_allocate_linear_memory(&p, is_shared_memory, num_bytes_per_page, + init_page_count, max_page_count, + &memory_data_size) + != BHT_OK) { + set_error_buf(error_buf, error_buf_size, + "allocate linear memory failed"); return NULL; } -#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ memory_inst->module_type = Wasm_Module_AoT; memory_inst->num_bytes_per_page = num_bytes_per_page; @@ -617,16 +557,8 @@ fail2: if (heap_size > 0) wasm_runtime_free(memory_inst->heap_handle); fail1: -#ifdef WASM_LINEAR_MEMORY_MMAP - if (mapped_mem) - wasm_munmap_linear_memory(mapped_mem, memory_data_size, map_size); - else -#endif - { - if (memory_inst->memory_data) - wasm_runtime_free(memory_inst->memory_data); - } - memory_inst->memory_data = NULL; + wasm_deallocate_linear_memory(memory_inst); + return NULL; } diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 3941c224a..019e0c129 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -42,6 +42,14 @@ static void (*free_func)(void *ptr) = NULL; static unsigned int global_pool_size; +static uint32 +align_as_and_cast(uint64 size, uint64 alignment) +{ + uint64 aligned_size = (size + alignment - 1) & ~(alignment - 1); + + return aligned_size > UINT32_MAX ? UINT32_MAX : (uint32)aligned_size; +} + static bool wasm_memory_init_with_pool(void *mem, unsigned int bytes) { @@ -629,7 +637,61 @@ wasm_runtime_set_mem_bound_check_bytes(WASMMemoryInstance *memory, #endif } -#ifndef OS_ENABLE_HW_BOUND_CHECK +static void +wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size) +{ +#ifdef BH_PLATFORM_WINDOWS + os_mem_decommit(mapped_mem, commit_size); +#else + (void)commit_size; +#endif + os_munmap(mapped_mem, map_size); +} + +static void * +wasm_mremap_linear_memory(void *mapped_mem, uint64 old_size, uint64 new_size, + uint64 commit_size) +{ + void *new_mem; + + bh_assert(new_size > 0); + bh_assert(new_size > old_size); + + if (mapped_mem) { + new_mem = os_mremap(mapped_mem, old_size, new_size); + } + else { + new_mem = os_mmap(NULL, new_size, MMAP_PROT_NONE, MMAP_MAP_NONE, + os_get_invalid_handle()); + } + if (!new_mem) { + return NULL; + } + +#ifdef BH_PLATFORM_WINDOWS + if (commit_size > 0 + && !os_mem_commit(new_mem, commit_size, + MMAP_PROT_READ | MMAP_PROT_WRITE)) { + os_munmap(new_mem, new_size); + return NULL; + } +#endif + + if (os_mprotect(new_mem, commit_size, MMAP_PROT_READ | MMAP_PROT_WRITE) + != 0) { + wasm_munmap_linear_memory(new_mem, new_size, new_size); + return NULL; + } + + return new_mem; +} + +static void * +wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size) +{ + return wasm_mremap_linear_memory(NULL, 0, map_size, commit_size); +} + bool wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count) { @@ -638,7 +700,7 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count) uint32 num_bytes_per_page, heap_size, total_size_old = 0; uint32 cur_page_count, max_page_count, total_page_count; uint64 total_size_new; - bool ret = true; + bool ret = true, full_size_mmaped; enlarge_memory_error_reason_t failure_reason = INTERNAL_ERROR; if (!memory) { @@ -646,12 +708,20 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count) goto return_func; } - heap_data_old = memory->heap_data; - heap_size = (uint32)(memory->heap_data_end - memory->heap_data); +#ifdef OS_ENABLE_HW_BOUND_CHECK + full_size_mmaped = true; +#elif WASM_ENABLE_SHARED_MEMORY != 0 + full_size_mmaped = shared_memory_is_shared(memory); +#else + full_size_mmaped = false; +#endif memory_data_old = memory->memory_data; total_size_old = memory->memory_data_size; + heap_data_old = memory->heap_data; + heap_size = (uint32)(memory->heap_data_end - memory->heap_data); + num_bytes_per_page = memory->num_bytes_per_page; cur_page_count = memory->cur_page_count; max_page_count = memory->max_page_count; @@ -681,166 +751,65 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count) total_size_new = UINT32_MAX; } -#if WASM_ENABLE_SHARED_MEMORY != 0 - if (shared_memory_is_shared(memory)) { - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - SET_LINEAR_MEMORY_SIZE(memory, (uint32)total_size_new); - memory->memory_data_end = memory->memory_data + (uint32)total_size_new; - - wasm_runtime_set_mem_bound_check_bytes(memory, total_size_new); - return true; - } + if (full_size_mmaped) { +#ifdef BH_PLATFORM_WINDOWS + if (!os_mem_commit(memory->memory_data_end, + (uint32)total_size_new - total_size_old, + MMAP_PROT_READ | MMAP_PROT_WRITE)) { + ret = false; + goto return_func; + } #endif - if (heap_size > 0) { - if (mem_allocator_is_heap_corrupted(memory->heap_handle)) { - wasm_runtime_show_app_heap_corrupted_prompt(); - ret = false; - goto return_func; - } - } - - if (!(memory_data_new = - wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) { - if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) { - ret = false; - goto return_func; - } - if (memory_data_old) { - bh_memcpy_s(memory_data_new, (uint32)total_size_new, - memory_data_old, total_size_old); - wasm_runtime_free(memory_data_old); - } - } - - memset(memory_data_new + total_size_old, 0, - (uint32)total_size_new - total_size_old); - - if (heap_size > 0) { - if (mem_allocator_migrate(memory->heap_handle, - (char *)heap_data_old - + (memory_data_new - memory_data_old), - heap_size) + if (os_mprotect(memory->memory_data_end, + (uint32)total_size_new - total_size_old, + MMAP_PROT_READ | MMAP_PROT_WRITE) != 0) { - /* Don't return here as memory->memory_data is obsolete and - must be updated to be correctly used later. */ +#ifdef BH_PLATFORM_WINDOWS + os_mem_decommit(memory->memory_data_end, + (uint32)total_size_new - total_size_old); +#endif ret = false; + goto return_func; } } + else { + if (heap_size > 0) { + if (mem_allocator_is_heap_corrupted(memory->heap_handle)) { + wasm_runtime_show_app_heap_corrupted_prompt(); + ret = false; + goto return_func; + } + } - memory->heap_data = memory_data_new + (heap_data_old - memory_data_old); - memory->heap_data_end = memory->heap_data + heap_size; + if (!(memory_data_new = wasm_mremap_linear_memory( + memory_data_old, total_size_old, (uint32)total_size_new, + (uint32)total_size_new))) { + ret = false; + goto return_func; + } - memory->num_bytes_per_page = num_bytes_per_page; - memory->cur_page_count = total_page_count; - memory->max_page_count = max_page_count; - memory->memory_data_size = (uint32)total_size_new; - - memory->memory_data = memory_data_new; - memory->memory_data_end = memory_data_new + (uint32)total_size_new; - - wasm_runtime_set_mem_bound_check_bytes(memory, total_size_new); + if (heap_size > 0) { + if (mem_allocator_migrate(memory->heap_handle, + (char *)heap_data_old + + (memory_data_new - memory_data_old), + heap_size) + != 0) { + /* Don't return here as memory->memory_data is obsolete and + must be updated to be correctly used later. */ + ret = false; + } + } + memory->heap_data = memory_data_new + (heap_data_old - memory_data_old); + memory->heap_data_end = memory->heap_data + heap_size; + memory->memory_data = memory_data_new; #if defined(os_writegsbase) - /* write base addr of linear memory to GS segment register */ - os_writegsbase(memory_data_new); + /* write base addr of linear memory to GS segment register */ + os_writegsbase(memory_data_new); #endif - -return_func: - if (!ret && enlarge_memory_error_cb) { - WASMExecEnv *exec_env = NULL; - -#if WASM_ENABLE_INTERP != 0 - if (module->module_type == Wasm_Module_Bytecode) - exec_env = - ((WASMModuleInstanceExtra *)module->e)->common.cur_exec_env; -#endif -#if WASM_ENABLE_AOT != 0 - if (module->module_type == Wasm_Module_AoT) - exec_env = - ((AOTModuleInstanceExtra *)module->e)->common.cur_exec_env; -#endif - - enlarge_memory_error_cb(inc_page_count, total_size_old, 0, - failure_reason, - (WASMModuleInstanceCommon *)module, exec_env, - enlarge_memory_error_user_data); } - return ret; -} -#else -bool -wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count) -{ - WASMMemoryInstance *memory = wasm_get_default_memory(module); - uint32 num_bytes_per_page, total_size_old = 0; - uint32 cur_page_count, max_page_count, total_page_count; - uint64 total_size_new; - bool ret = true; - enlarge_memory_error_reason_t failure_reason = INTERNAL_ERROR; - - if (!memory) { - ret = false; - goto return_func; - } - - num_bytes_per_page = memory->num_bytes_per_page; - cur_page_count = memory->cur_page_count; - max_page_count = memory->max_page_count; - total_size_old = num_bytes_per_page * cur_page_count; - total_page_count = inc_page_count + cur_page_count; - total_size_new = num_bytes_per_page * (uint64)total_page_count; - - if (inc_page_count <= 0) - /* No need to enlarge memory */ - return true; - - if (total_page_count < cur_page_count) { /* integer overflow */ - ret = false; - goto return_func; - } - - if (total_page_count > max_page_count) { - failure_reason = MAX_SIZE_REACHED; - ret = false; - goto return_func; - } - - bh_assert(total_size_new <= 4 * (uint64)BH_GB); - if (total_size_new > UINT32_MAX) { - /* Resize to 1 page with size 4G-1 */ - num_bytes_per_page = UINT32_MAX; - total_page_count = max_page_count = 1; - total_size_new = UINT32_MAX; - } - -#ifdef BH_PLATFORM_WINDOWS - if (!os_mem_commit(memory->memory_data_end, - (uint32)total_size_new - total_size_old, - MMAP_PROT_READ | MMAP_PROT_WRITE)) { - ret = false; - goto return_func; - } -#endif - - if (os_mprotect(memory->memory_data_end, - (uint32)total_size_new - total_size_old, - MMAP_PROT_READ | MMAP_PROT_WRITE) - != 0) { -#ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(memory->memory_data_end, - (uint32)total_size_new - total_size_old); -#endif - ret = false; - goto return_func; - } - - /* The increased pages are filled with zero by the OS when os_mmap, - no need to memset it again here */ - memory->num_bytes_per_page = num_bytes_per_page; memory->cur_page_count = total_page_count; memory->max_page_count = max_page_count; @@ -872,7 +841,6 @@ return_func: return ret; } -#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ void wasm_runtime_set_enlarge_mem_error_callback( @@ -899,3 +867,75 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) return ret; } + +void +wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst) +{ + uint64 map_size; + + bh_assert(memory_inst); + bh_assert(memory_inst->memory_data); + +#ifndef OS_ENABLE_HW_BOUND_CHECK +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (shared_memory_is_shared(memory_inst)) { + map_size = (uint64)memory_inst->num_bytes_per_page + * memory_inst->max_page_count; + } + else +#endif + { + map_size = (uint64)memory_inst->num_bytes_per_page + * memory_inst->cur_page_count; + } +#else + map_size = 8 * (uint64)BH_GB; +#endif + wasm_munmap_linear_memory(memory_inst->memory_data, + memory_inst->memory_data_size, map_size); + memory_inst->memory_data = NULL; +} + +int +wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory, + uint64 num_bytes_per_page, uint64 init_page_count, + uint64 max_page_count, uint64 *memory_data_size) +{ + uint64 map_size, page_size; + + bh_assert(data); + bh_assert(memory_data_size); + +#ifndef OS_ENABLE_HW_BOUND_CHECK +#if WASM_ENABLE_SHARED_MEMORY != 0 + if (is_shared_memory) { + /* Allocate maximum memory size when memory is shared */ + map_size = max_page_count * num_bytes_per_page; + } + else +#endif + { + map_size = init_page_count * num_bytes_per_page; + } +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ + /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: + * ea = i + memarg.offset + * both i and memarg.offset are u32 in range 0 to 4G + * so the range of ea is 0 to 8G + */ + map_size = 8 * (uint64)BH_GB; +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ + + page_size = os_getpagesize(); + *memory_data_size = init_page_count * num_bytes_per_page; + bh_assert(*memory_data_size <= UINT32_MAX); + align_as_and_cast(*memory_data_size, page_size); + + if (map_size > 0) { + if (!(*data = wasm_mmap_linear_memory(map_size, *memory_data_size))) { + return BHT_ERROR; + } + } + + return BHT_OK; +} \ No newline at end of file diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index 9b74db526..381266b61 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -42,6 +42,14 @@ void wasm_runtime_set_enlarge_mem_error_callback( const enlarge_memory_error_callback_t callback, void *user_data); +void +wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst); + +int +wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory, + uint64 num_bytes_per_page, uint64 init_page_count, + uint64 max_page_count, uint64 *memory_data_size); + #ifdef __cplusplus } #endif diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index eaee4f225..767e417b3 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -6269,67 +6269,3 @@ wasm_runtime_set_linux_perf(bool flag) enable_linux_perf = flag; } #endif - -#ifdef WASM_LINEAR_MEMORY_MMAP -void -wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size) -{ -#ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(mapped_mem, commit_size); -#else - (void)commit_size; -#endif - os_munmap(mapped_mem, map_size); -} - -void * -wasm_mmap_linear_memory(uint64_t map_size, uint64 *io_memory_data_size, - char *error_buf, uint32 error_buf_size) -{ - uint64 page_size = os_getpagesize(); - void *mapped_mem = NULL; - uint64 memory_data_size; - - bh_assert(io_memory_data_size); - - memory_data_size = - (*io_memory_data_size + page_size - 1) & ~(page_size - 1); - - if (memory_data_size > UINT32_MAX) - memory_data_size = UINT32_MAX; - - if (!(mapped_mem = os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE, - os_get_invalid_handle()))) { - set_error_buf(error_buf, error_buf_size, "mmap memory failed"); - goto fail1; - } - -#ifdef BH_PLATFORM_WINDOWS - if (memory_data_size > 0 - && !os_mem_commit(mapped_mem, memory_data_size, - MMAP_PROT_READ | MMAP_PROT_WRITE)) { - set_error_buf(error_buf, error_buf_size, "commit memory failed"); - os_munmap(mapped_mem, map_size); - goto fail1; - } -#endif - - if (os_mprotect(mapped_mem, memory_data_size, - MMAP_PROT_READ | MMAP_PROT_WRITE) - != 0) { - set_error_buf(error_buf, error_buf_size, "mprotect memory failed"); - goto fail2; - } - - /* Newly allocated pages are filled with zero by the OS, we don't fill it - * again here */ - - *io_memory_data_size = memory_data_size; - - return mapped_mem; -fail2: - wasm_munmap_linear_memory(mapped_mem, memory_data_size, map_size); -fail1: - return NULL; -} -#endif diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 039244ff6..cb9731521 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -320,11 +320,6 @@ LOAD_I16(void *addr) #define SHARED_MEMORY_UNLOCK(memory) (void)0 #endif -#if defined(OS_ENABLE_HW_BOUND_CHECK) \ - || (WASM_ENABLE_SHARED_MEMORY != 0 && WASM_ENABLE_SHARED_MEMORY_MMAP != 0) -#define WASM_LINEAR_MEMORY_MMAP -#endif - typedef struct WASMModuleCommon { /* Module type, for module loaded from WASM bytecode binary, this field is Wasm_Module_Bytecode, and this structure should @@ -421,16 +416,6 @@ typedef struct WASMRegisteredModule { } WASMRegisteredModule; #endif -typedef struct WASMMemoryInstanceCommon { - uint32 module_type; - - /* The following uint8[1] member is a dummy just to indicate - some module_type dependent members follow. - Typically it should be accessed by casting to the corresponding - actual module_type dependent structure, not via this member. */ - uint8 memory_inst_data[1]; -} WASMMemoryInstanceCommon; - typedef package_type_t PackageType; typedef wasm_section_t WASMSection, AOTSection; @@ -1098,14 +1083,6 @@ wasm_runtime_quick_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, void wasm_runtime_show_app_heap_corrupted_prompt(); -void -wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, - uint64 map_size); - -void * -wasm_mmap_linear_memory(uint64_t map_size, uint64 *io_memory_data_size, - char *error_buf, uint32 error_buf_size); - #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 void wasm_runtime_destroy_custom_sections(WASMCustomSection *section_list); diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 003ceb1f5..a3a544cac 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -115,9 +115,6 @@ static void memories_deinstantiate(WASMModuleInstance *module_inst, WASMMemoryInstance **memories, uint32 count) { -#ifdef WASM_LINEAR_MEMORY_MMAP - uint64 map_size; -#endif uint32 i; if (memories) { for (i = 0; i < count; i++) { @@ -144,23 +141,7 @@ memories_deinstantiate(WASMModuleInstance *module_inst, memories[i]->heap_handle = NULL; } if (memories[i]->memory_data) { -#ifndef OS_ENABLE_HW_BOUND_CHECK -#ifdef WASM_LINEAR_MEMORY_MMAP - if (shared_memory_is_shared(memories[i])) { - map_size = (uint64)memories[i]->num_bytes_per_page - * memories[i]->max_page_count; - wasm_munmap_linear_memory(memories[i]->memory_data, - map_size, map_size); - } - else -#endif - wasm_runtime_free(memories[i]->memory_data); -#else - map_size = (uint64)memories[i]->num_bytes_per_page - * memories[i]->cur_page_count; - wasm_munmap_linear_memory(memories[i]->memory_data, - map_size, 8 * (uint64)BH_GB); -#endif + wasm_deallocate_linear_memory(memories[i]); } } } @@ -182,13 +163,10 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, uint32 inc_page_count, aux_heap_base, global_idx; uint32 bytes_of_last_page, bytes_to_page_end; uint8 *global_addr; -#ifdef WASM_LINEAR_MEMORY_MMAP - uint8 *mapped_mem = NULL; - uint64 map_size; -#endif + bool is_shared_memory = false; #if WASM_ENABLE_SHARED_MEMORY != 0 - bool is_shared_memory = flags & 0x02 ? true : false; + is_shared_memory = flags & 0x02 ? true : false; /* shared memory */ if (is_shared_memory && parent != NULL) { @@ -197,6 +175,10 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, shared_memory_inc_reference(memory); return memory; } +#else + (void)parent; + (void)memory_idx; + (void)flags; #endif /* end of WASM_ENABLE_SHARED_MEMORY */ if (heap_size > 0 && module_inst->module->malloc_function != (uint32)-1 @@ -296,57 +278,20 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, num_bytes_per_page, init_page_count, max_page_count); LOG_VERBOSE(" heap offset: %u, heap size: %d\n", heap_offset, heap_size); - memory_data_size = (uint64)num_bytes_per_page * init_page_count; max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; - bh_assert(memory_data_size <= UINT32_MAX); bh_assert(max_memory_data_size <= 4 * (uint64)BH_GB); (void)max_memory_data_size; bh_assert(memory != NULL); -#ifndef OS_ENABLE_HW_BOUND_CHECK -#if WASM_ENABLE_SHARED_MEMORY != 0 - if (is_shared_memory) { - /* Allocate maximum memory size when memory is shared */ -#if WASM_ENABLE_SHARED_MEMORY_MMAP != 0 - map_size = max_memory_data_size; - if (max_memory_data_size > 0 - && !(memory->memory_data = mapped_mem = - wasm_mmap_linear_memory(map_size, &max_memory_data_size, - error_buf, error_buf_size))) { - goto fail1; - } -#else - if (max_memory_data_size > 0 - && !(memory->memory_data = runtime_malloc( - max_memory_data_size, error_buf, error_buf_size))) { - goto fail1; - } -#endif + if (wasm_allocate_linear_memory(&memory->memory_data, is_shared_memory, + num_bytes_per_page, init_page_count, + max_page_count, &memory_data_size) + != BHT_OK) { + set_error_buf(error_buf, error_buf_size, + "allocate linear memory failed"); + return NULL; } - else -#endif /* end of WASM_ENABLE_SHARED_MEMORY != 0 */ - { - /* Allocate initial memory size when memory is not shared */ - if (memory_data_size > 0 - && !(memory->memory_data = runtime_malloc( - memory_data_size, error_buf, error_buf_size))) { - goto fail1; - } - } -#else /* else of OS_ENABLE_HW_BOUND_CHECK */ - /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: - * ea = i + memarg.offset - * both i and memarg.offset are u32 in range 0 to 4G - * so the range of ea is 0 to 8G - */ - map_size = 8 * (uint64)BH_GB; - if (!(memory->memory_data = mapped_mem = wasm_mmap_linear_memory( - map_size, &memory_data_size, error_buf, error_buf_size))) { - set_error_buf(error_buf, error_buf_size, "mmap memory failed"); - goto fail1; - } -#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ memory->module_type = Wasm_Module_Bytecode; memory->num_bytes_per_page = num_bytes_per_page; @@ -364,13 +309,13 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, if (!(memory->heap_handle = runtime_malloc( (uint64)heap_struct_size, error_buf, error_buf_size))) { - goto fail2; + goto fail1; } if (!mem_allocator_create_with_struct_and_pool( memory->heap_handle, heap_struct_size, memory->heap_data, heap_size)) { set_error_buf(error_buf, error_buf_size, "init app heap failed"); - goto fail3; + goto fail2; } } @@ -388,20 +333,13 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, LOG_VERBOSE("Memory instantiate success."); return memory; -fail3: +fail2: if (heap_size > 0) wasm_runtime_free(memory->heap_handle); -fail2: -#ifdef WASM_LINEAR_MEMORY_MMAP - if (mapped_mem) - wasm_munmap_linear_memory(mapped_mem, memory_data_size, map_size); - else -#endif - { - if (memory->memory_data) - wasm_runtime_free(memory->memory_data); - } fail1: + if (memory->memory_data) + wasm_deallocate_linear_memory(memory); + return NULL; } @@ -633,7 +571,7 @@ fail: * Destroy function instances. */ static void -functions_deinstantiate(WASMFunctionInstance *functions, uint32 count) +functions_deinstantiate(WASMFunctionInstance *functions) { if (functions) { wasm_runtime_free(functions); @@ -1561,6 +1499,7 @@ set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode, #if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \ && WASM_ENABLE_LAZY_JIT != 0) /* No possible multi-tier JIT */ + (void)first_time_set; module_inst->e->running_mode = running_mode; if (running_mode == Mode_Interp) { @@ -2342,12 +2281,10 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst) } tables_deinstantiate(module_inst); - functions_deinstantiate(module_inst->e->functions, - module_inst->e->function_count); + functions_deinstantiate(module_inst->e->functions); #if WASM_ENABLE_TAGS != 0 tags_deinstantiate(module_inst->e->tags, module_inst->e->import_tag_ptrs); #endif - globals_deinstantiate(module_inst->e->globals); export_functions_deinstantiate(module_inst->export_functions); #if WASM_ENABLE_TAGS != 0 diff --git a/core/shared/platform/common/memory/mremap.c b/core/shared/platform/common/memory/mremap.c new file mode 100644 index 000000000..bbd287e77 --- /dev/null +++ b/core/shared/platform/common/memory/mremap.c @@ -0,0 +1,12 @@ +/* + * Copyright (C) 2024 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_memutils.h" + +void * +os_mremap(void *old_addr, size_t old_size, size_t new_size) +{ + return bh_memory_remap_slow(old_addr, old_size, new_size); +} diff --git a/core/shared/platform/common/memory/platform_api_memory.cmake b/core/shared/platform/common/memory/platform_api_memory.cmake new file mode 100644 index 000000000..9f06c1391 --- /dev/null +++ b/core/shared/platform/common/memory/platform_api_memory.cmake @@ -0,0 +1,4 @@ +# Copyright (C) 2024 Amazon Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +file (GLOB_RECURSE PLATFORM_COMMON_MEMORY_SOURCE ${CMAKE_CURRENT_LIST_DIR}/*.c) diff --git a/core/shared/platform/common/posix/platform_api_posix.cmake b/core/shared/platform/common/posix/platform_api_posix.cmake index 2bf9fab4a..17ee04f82 100644 --- a/core/shared/platform/common/posix/platform_api_posix.cmake +++ b/core/shared/platform/common/posix/platform_api_posix.cmake @@ -16,4 +16,19 @@ else() set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE}) endif() +# This is to support old CMake version. Newer version of CMake could use +# list APPEND/POP_BACK methods. +include(CheckSymbolExists) +set (CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE ${CMAKE_REQUIRED_DEFINITIONS}) +check_symbol_exists (mremap "sys/mman.h" MREMAP_EXISTS) +list (REMOVE_AT CMAKE_REQUIRED_DEFINITIONS 0) + +if(MREMAP_EXISTS) + add_definitions (-DWASM_HAVE_MREMAP=1) +else() + add_definitions (-DWASM_HAVE_MREMAP=0) + include (${CMAKE_CURRENT_LIST_DIR}/../memory/platform_api_memory.cmake) + set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE}) +endif() + set (PLATFORM_COMMON_POSIX_SOURCE ${source_all} ) diff --git a/core/shared/platform/common/posix/posix_memmap.c b/core/shared/platform/common/posix/posix_memmap.c index 2baf5aeb7..afc549cdd 100644 --- a/core/shared/platform/common/posix/posix_memmap.c +++ b/core/shared/platform/common/posix/posix_memmap.c @@ -3,6 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ +#if !defined(_GNU_SOURCE) && WASM_HAVE_MREMAP != 0 +/* Enable mremap */ +#define _GNU_SOURCE +#include "bh_memutils.h" +#endif + #include "platform_api_vmcore.h" #if defined(__APPLE__) || defined(__MACH__) @@ -236,6 +242,23 @@ os_munmap(void *addr, size_t size) } } +#if WASM_HAVE_MREMAP != 0 +void * +os_mremap(void *old_addr, size_t old_size, size_t new_size) +{ + void *ptr = mremap(old_addr, old_size, new_size, MREMAP_MAYMOVE); + + if (ptr == MAP_FAILED) { +#if BH_ENABLE_TRACE_MMAP != 0 + os_printf("mremap failed: %d\n", errno); +#endif + return bh_memory_remap_slow(old_addr, old_size, new_size); + } + + return ptr; +} +#endif + int os_mprotect(void *addr, size_t size, int prot) { diff --git a/core/shared/platform/darwin/shared_platform.cmake b/core/shared/platform/darwin/shared_platform.cmake index 5eecd65c7..3680f3d3a 100644 --- a/core/shared/platform/darwin/shared_platform.cmake +++ b/core/shared/platform/darwin/shared_platform.cmake @@ -12,6 +12,9 @@ include (${CMAKE_CURRENT_LIST_DIR}/../common/posix/platform_api_posix.cmake) file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) +include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake) +set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE}) + set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_POSIX_SOURCE}) file (GLOB header ${PLATFORM_SHARED_DIR}/../include/*.h) diff --git a/core/shared/platform/include/platform_api_vmcore.h b/core/shared/platform/include/platform_api_vmcore.h index 4a9d29cc0..1c948f58c 100644 --- a/core/shared/platform/include/platform_api_vmcore.h +++ b/core/shared/platform/include/platform_api_vmcore.h @@ -132,7 +132,7 @@ enum { MMAP_MAP_32BIT = 1, /* Don't interpret addr as a hint: place the mapping at exactly that address. */ - MMAP_MAP_FIXED = 2 + MMAP_MAP_FIXED = 2, }; void * @@ -142,6 +142,11 @@ os_munmap(void *addr, size_t size); int os_mprotect(void *addr, size_t size, int prot); +/* Doesn't guarantee that protection flags will be preserved. + os_mprotect() must be called after remapping. */ +void * +os_mremap(void *old_addr, size_t old_size, size_t new_size); + #if (WASM_MEM_DUAL_BUS_MIRROR != 0) void * os_get_dbus_mirror(void *ibus); diff --git a/core/shared/platform/linux-sgx/platform_internal.h b/core/shared/platform/linux-sgx/platform_internal.h index c96768e32..2cc34dfca 100644 --- a/core/shared/platform/linux-sgx/platform_internal.h +++ b/core/shared/platform/linux-sgx/platform_internal.h @@ -79,6 +79,8 @@ os_get_invalid_handle() return -1; } +#define os_getpagesize getpagesize + #ifdef __cplusplus } #endif diff --git a/core/shared/platform/linux-sgx/shared_platform.cmake b/core/shared/platform/linux-sgx/shared_platform.cmake index e8e167005..9cd765be4 100644 --- a/core/shared/platform/linux-sgx/shared_platform.cmake +++ b/core/shared/platform/linux-sgx/shared_platform.cmake @@ -37,6 +37,9 @@ else() set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE}) endif() +include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake) +set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE}) + file (GLOB source_all_untrusted ${PLATFORM_SHARED_DIR}/untrusted/*.c) set (PLATFORM_SHARED_SOURCE ${source_all}) diff --git a/core/shared/platform/nuttx/platform_internal.h b/core/shared/platform/nuttx/platform_internal.h index 2fb80a6e3..0b54d85a9 100644 --- a/core/shared/platform/nuttx/platform_internal.h +++ b/core/shared/platform/nuttx/platform_internal.h @@ -44,6 +44,8 @@ typedef pthread_t korp_thread; typedef pthread_rwlock_t korp_rwlock; typedef sem_t korp_sem; +#define os_getpagesize getpagesize + #define OS_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER #define BH_APPLET_PRESERVED_STACK_SIZE (2 * BH_KB) diff --git a/core/shared/platform/nuttx/shared_platform.cmake b/core/shared/platform/nuttx/shared_platform.cmake index 1cbe2218b..ff70cc031 100644 --- a/core/shared/platform/nuttx/shared_platform.cmake +++ b/core/shared/platform/nuttx/shared_platform.cmake @@ -16,5 +16,6 @@ if (WAMR_BUILD_LIBC_WASI EQUAL 1) set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE}) endif () -set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE}) +include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake) +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE} ${PLATFORM_COMMON_MEMORY_SOURCE}) diff --git a/core/shared/platform/windows/shared_platform.cmake b/core/shared/platform/windows/shared_platform.cmake index 3c531c17c..502a8a2ed 100644 --- a/core/shared/platform/windows/shared_platform.cmake +++ b/core/shared/platform/windows/shared_platform.cmake @@ -20,6 +20,9 @@ else() set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE}) endif() +include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake) +set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE}) + set (PLATFORM_SHARED_SOURCE ${source_all}) file (GLOB header ${PLATFORM_SHARED_DIR}/../include/*.h) diff --git a/core/shared/utils/bh_memutils.c b/core/shared/utils/bh_memutils.c new file mode 100644 index 000000000..a655d8ac2 --- /dev/null +++ b/core/shared/utils/bh_memutils.c @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2024 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_memutils.h" + +void * +bh_memory_remap_slow(void *old_addr, size_t old_size, size_t new_size) +{ + void *new_memory = + os_mmap(NULL, new_size, MMAP_PROT_WRITE | MMAP_PROT_READ, 0, -1); + if (!new_memory) { + return NULL; + } + /* + * bh_memcpy_s can't be used as it doesn't support values bigger than + * UINT32_MAX + */ + memcpy(new_memory, old_addr, new_size < old_size ? new_size : old_size); + os_munmap(old_addr, old_size); + + return new_memory; +} diff --git a/core/shared/utils/bh_memutils.h b/core/shared/utils/bh_memutils.h new file mode 100644 index 000000000..7581860bd --- /dev/null +++ b/core/shared/utils/bh_memutils.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2024 Amazon Inc. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_MEMUTILS_H +#define _BH_MEMUTILS_H + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Remaps memory by mapping a new region, copying data from the old + * region and umapping the old region. + * + * Unless the behavior is desired, in most cases os_mremap should be used + * as it's at worst equally slow as this function, and on some platforms + * (e.g. posix with mremap) os_mremap will perform better. + * + * @param old_addr an old address. + * @param old_size a size of the old address. + * @param new_size a size of the new memory region. + * @return a pointer to the new memory region. + */ +void * +bh_memory_remap_slow(void *old_addr, size_t old_size, size_t new_size); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _BH_MEMUTILS_H */ diff --git a/doc/memory_usage.md b/doc/memory_usage.md index ec0624c6f..88b792f0c 100644 --- a/doc/memory_usage.md +++ b/doc/memory_usage.md @@ -51,14 +51,9 @@ They are in the number of WASM pages, each of which is of 65536 bytes. The `max` is optional for non-shared memory. When omitted, it effectivily means unlimited. -If `OS_ENABLE_HW_BOUND_CHECK` is enabled, the memory is allocated via -`os_mmap` and `os_mem_commit`/`os_mprotect`. -Otherwise, it's allocated from the global heap. +The linear memory is allocated via `os_mmap` and `os_mem_commit`/`os_mprotect`. -If the memory is shared and `OS_ENABLE_HW_BOUND_CHECK` is not enabled, -the `max` size of memory is allocated on instantiation. - -Otherwise, the `min` size of memory is allocated on instantiation. +The `min` size of memory is allocated on instantiation. It can later grow up to the `max` size via the `memory.grow` instruction. Libc heap diff --git a/product-mini/platforms/linux-sgx/CMakeLists.txt b/product-mini/platforms/linux-sgx/CMakeLists.txt index 0b2b4afec..81d6244b1 100644 --- a/product-mini/platforms/linux-sgx/CMakeLists.txt +++ b/product-mini/platforms/linux-sgx/CMakeLists.txt @@ -176,3 +176,10 @@ else() OUTPUT_VARIABLE cmdOutput ) endif() + +if (WAMR_BUILD_SPEC_TEST EQUAL 1) + execute_process( + COMMAND bash -c "sed -i -E 's/0x1000000<\\/ReservedMemMaxSize>/0x8000000<\\/ReservedMemMaxSize>/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Enclave/Enclave.config.xml" + OUTPUT_VARIABLE cmdOutput + ) +endif() diff --git a/product-mini/platforms/nuttx/wamr.mk b/product-mini/platforms/nuttx/wamr.mk index 748fb6a87..9c1aeae4e 100644 --- a/product-mini/platforms/nuttx/wamr.mk +++ b/product-mini/platforms/nuttx/wamr.mk @@ -391,6 +391,7 @@ CSRCS += nuttx_platform.c \ posix_thread.c \ posix_time.c \ posix_sleep.c \ + mremap.c \ mem_alloc.c \ ems_kfc.c \ ems_alloc.c \ @@ -401,6 +402,7 @@ CSRCS += nuttx_platform.c \ bh_hashmap.c \ bh_list.c \ bh_log.c \ + bh_memutils.c \ bh_queue.c \ bh_vector.c \ bh_read_file.c \ @@ -416,6 +418,7 @@ CSRCS += nuttx_platform.c \ ASRCS += $(INVOKE_NATIVE) VPATH += $(SHARED_ROOT)/platform/nuttx +VPATH += $(SHARED_ROOT)/platform/common/memory VPATH += $(SHARED_ROOT)/platform/common/posix VPATH += $(SHARED_ROOT)/platform/common/libc-util VPATH += $(SHARED_ROOT)/mem-alloc