diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 8c27465d3..d842159f5 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -145,6 +145,7 @@ jobs: "-DWAMR_BUILD_SIMD=1", "-DWAMR_BUILD_TAIL_CALL=1", "-DWAMR_DISABLE_HW_BOUND_CHECK=1", + "-DWAMR_ENABLE_SHARED_MEMORY_MMAP=1", ] os: [ubuntu-22.04] platform: [android, linux] diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 62c5163ac..b9d5a0413 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -248,6 +248,12 @@ if (WAMR_BUILD_SHARED_MEMORY EQUAL 1) else () add_definitions (-DWASM_ENABLE_SHARED_MEMORY=0) endif () +if (WAMR_ENABLE_SHARED_MEMORY_MMAP EQUAL 1) + add_definitions (-DWASM_ENABLE_SHARED_MEMORY_MMAP=1) + message (" Shared memory allocated using mmap enabled") +else () + add_definitions (-DWASM_ENABLE_SHARED_MEMORY_MMAP=0) +endif () if (WAMR_BUILD_THREAD_MGR EQUAL 1) message (" Thread manager enabled") endif () diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 0fa4f7939..4f9635a7c 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -340,6 +340,9 @@ tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module, static void memories_deinstantiate(AOTModuleInstance *module_inst) { +#ifdef WASM_LINEAR_MEMORY_MMAP + uint64 map_size; +#endif uint32 i; AOTMemoryInstance *memory_inst; @@ -362,14 +365,21 @@ memories_deinstantiate(AOTModuleInstance *module_inst) if (memory_inst->memory_data) { #ifndef OS_ENABLE_HW_BOUND_CHECK - wasm_runtime_free(memory_inst->memory_data); -#else -#ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(memory_inst->memory_data, - memory_inst->num_bytes_per_page - * memory_inst->cur_page_count); +#ifdef WASM_LINEAR_MEMORY_MMAP + if (shared_memory_is_shared(memory_inst)) { + map_size = (uint64)memory_inst->num_bytes_per_page + * memory_inst->max_page_count; + wasm_munmap_linear_memory(memory_inst->memory_data, + map_size, map_size); + } + else #endif - os_munmap(memory_inst->memory_data, 8 * (uint64)BH_GB); + wasm_runtime_free(memory_inst->memory_data); +#else + map_size = (uint64)memory_inst->num_bytes_per_page + * memory_inst->cur_page_count; + wasm_munmap_linear_memory(memory_inst->memory_data, map_size, + 8 * (uint64)BH_GB); #endif } } @@ -392,10 +402,9 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, uint32 heap_offset = num_bytes_per_page * init_page_count; uint64 memory_data_size, max_memory_data_size; uint8 *p = NULL, *global_addr; -#ifdef OS_ENABLE_HW_BOUND_CHECK - uint8 *mapped_mem; - uint64 map_size = 8 * (uint64)BH_GB; - uint64 page_size = os_getpagesize(); +#ifdef WASM_LINEAR_MEMORY_MMAP + uint8 *mapped_mem = NULL; + uint64 map_size; #endif #if WASM_ENABLE_SHARED_MEMORY != 0 @@ -519,15 +528,25 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, #ifndef OS_ENABLE_HW_BOUND_CHECK #if WASM_ENABLE_SHARED_MEMORY != 0 if (is_shared_memory) { +#if WASM_ENABLE_SHARED_MEMORY_MMAP != 0 + map_size = max_memory_data_size; + if (max_memory_data_size > 0 + && !(p = mapped_mem = + wasm_mmap_linear_memory(map_size, &max_memory_data_size, + error_buf, error_buf_size))) { + return NULL; + } +#else /* Allocate maximum memory size when memory is shared */ if (max_memory_data_size > 0 && !(p = runtime_malloc(max_memory_data_size, error_buf, error_buf_size))) { return NULL; } +#endif } else -#endif +#endif /* end of WASM_ENABLE_SHARED_MEMORY != 0 */ { /* Allocate initial memory size when memory is not shared */ if (memory_data_size > 0 @@ -536,43 +555,18 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, return NULL; } } -#else /* else of OS_ENABLE_HW_BOUND_CHECK */ - memory_data_size = (memory_data_size + page_size - 1) & ~(page_size - 1); - +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: * ea = i + memarg.offset * both i and memarg.offset are u32 in range 0 to 4G * so the range of ea is 0 to 8G */ - if (!(p = mapped_mem = os_mmap(NULL, map_size, MMAP_PROT_NONE, - MMAP_MAP_NONE, os_get_invalid_handle()))) { + map_size = 8 * (uint64)BH_GB; + if (!(p = mapped_mem = wasm_mmap_linear_memory( + map_size, &memory_data_size, error_buf, error_buf_size))) { set_error_buf(error_buf, error_buf_size, "mmap memory failed"); return NULL; } - -#ifdef BH_PLATFORM_WINDOWS - if (!os_mem_commit(p, memory_data_size, MMAP_PROT_READ | MMAP_PROT_WRITE)) { - set_error_buf(error_buf, error_buf_size, "commit memory failed"); - os_munmap(mapped_mem, map_size); - return NULL; - } -#endif - - if (os_mprotect(p, memory_data_size, MMAP_PROT_READ | MMAP_PROT_WRITE) - != 0) { - set_error_buf(error_buf, error_buf_size, "mprotect memory failed"); -#ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(p, memory_data_size); -#endif - os_munmap(mapped_mem, map_size); - return NULL; - } - - /* Newly allocated pages are filled with zero by the OS, we don't fill it - * again here */ - - if (memory_data_size > UINT32_MAX) - memory_data_size = UINT32_MAX; #endif /* end of OS_ENABLE_HW_BOUND_CHECK */ memory_inst->module_type = Wasm_Module_AoT; @@ -623,16 +617,15 @@ fail2: if (heap_size > 0) wasm_runtime_free(memory_inst->heap_handle); fail1: -#ifndef OS_ENABLE_HW_BOUND_CHECK - if (memory_inst->memory_data) - wasm_runtime_free(memory_inst->memory_data); -#else -#ifdef BH_PLATFORM_WINDOWS - if (memory_inst->memory_data) - os_mem_decommit(p, memory_data_size); -#endif - os_munmap(mapped_mem, map_size); +#ifdef WASM_LINEAR_MEMORY_MMAP + if (mapped_mem) + wasm_munmap_linear_memory(mapped_mem, memory_data_size, map_size); + else #endif + { + if (memory_inst->memory_data) + wasm_runtime_free(memory_inst->memory_data); + } memory_inst->memory_data = NULL; return NULL; } diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index e245cf50b..5c75d82af 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -6229,3 +6229,67 @@ wasm_runtime_set_linux_perf(bool flag) enable_linux_perf = flag; } #endif + +#ifdef WASM_LINEAR_MEMORY_MMAP +void +wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size) +{ +#ifdef BH_PLATFORM_WINDOWS + os_mem_decommit(mapped_mem, commit_size); +#else + (void)commit_size; +#endif + os_munmap(mapped_mem, map_size); +} + +void * +wasm_mmap_linear_memory(uint64_t map_size, uint64 *io_memory_data_size, + char *error_buf, uint32 error_buf_size) +{ + uint64 page_size = os_getpagesize(); + void *mapped_mem = NULL; + uint64 memory_data_size; + + bh_assert(io_memory_data_size); + + memory_data_size = + (*io_memory_data_size + page_size - 1) & ~(page_size - 1); + + if (memory_data_size > UINT32_MAX) + memory_data_size = UINT32_MAX; + + if (!(mapped_mem = os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE, + os_get_invalid_handle()))) { + set_error_buf(error_buf, error_buf_size, "mmap memory failed"); + goto fail1; + } + +#ifdef BH_PLATFORM_WINDOWS + if (memory_data_size > 0 + && !os_mem_commit(mapped_mem, memory_data_size, + MMAP_PROT_READ | MMAP_PROT_WRITE)) { + set_error_buf(error_buf, error_buf_size, "commit memory failed"); + os_munmap(mapped_mem, map_size); + goto fail1; + } +#endif + + if (os_mprotect(mapped_mem, memory_data_size, + MMAP_PROT_READ | MMAP_PROT_WRITE) + != 0) { + set_error_buf(error_buf, error_buf_size, "mprotect memory failed"); + goto fail2; + } + + /* Newly allocated pages are filled with zero by the OS, we don't fill it + * again here */ + + *io_memory_data_size = memory_data_size; + + return mapped_mem; +fail2: + wasm_munmap_linear_memory(mapped_mem, memory_data_size, map_size); +fail1: + return NULL; +} +#endif diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 606eea019..039244ff6 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -320,6 +320,11 @@ LOAD_I16(void *addr) #define SHARED_MEMORY_UNLOCK(memory) (void)0 #endif +#if defined(OS_ENABLE_HW_BOUND_CHECK) \ + || (WASM_ENABLE_SHARED_MEMORY != 0 && WASM_ENABLE_SHARED_MEMORY_MMAP != 0) +#define WASM_LINEAR_MEMORY_MMAP +#endif + typedef struct WASMModuleCommon { /* Module type, for module loaded from WASM bytecode binary, this field is Wasm_Module_Bytecode, and this structure should @@ -1093,6 +1098,14 @@ wasm_runtime_quick_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, void wasm_runtime_show_app_heap_corrupted_prompt(); +void +wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, + uint64 map_size); + +void * +wasm_mmap_linear_memory(uint64_t map_size, uint64 *io_memory_data_size, + char *error_buf, uint32 error_buf_size); + #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 void wasm_runtime_destroy_custom_sections(WASMCustomSection *section_list); diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 7cda841e5..267e3d59d 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -115,6 +115,9 @@ static void memories_deinstantiate(WASMModuleInstance *module_inst, WASMMemoryInstance **memories, uint32 count) { +#ifdef WASM_LINEAR_MEMORY_MMAP + uint64 map_size; +#endif uint32 i; if (memories) { for (i = 0; i < count; i++) { @@ -142,15 +145,21 @@ memories_deinstantiate(WASMModuleInstance *module_inst, } if (memories[i]->memory_data) { #ifndef OS_ENABLE_HW_BOUND_CHECK - wasm_runtime_free(memories[i]->memory_data); -#else -#ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(memories[i]->memory_data, - memories[i]->num_bytes_per_page - * memories[i]->cur_page_count); +#ifdef WASM_LINEAR_MEMORY_MMAP + if (shared_memory_is_shared(memories[i])) { + map_size = (uint64)memories[i]->num_bytes_per_page + * memories[i]->max_page_count; + wasm_munmap_linear_memory(memories[i]->memory_data, + map_size, map_size); + } + else #endif - os_munmap((uint8 *)memories[i]->memory_data, - 8 * (uint64)BH_GB); + wasm_runtime_free(memories[i]->memory_data); +#else + map_size = (uint64)memories[i]->num_bytes_per_page + * memories[i]->cur_page_count; + wasm_munmap_linear_memory(memories[i]->memory_data, + map_size, 8 * (uint64)BH_GB); #endif } } @@ -173,10 +182,9 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, uint32 inc_page_count, aux_heap_base, global_idx; uint32 bytes_of_last_page, bytes_to_page_end; uint8 *global_addr; -#ifdef OS_ENABLE_HW_BOUND_CHECK - uint8 *mapped_mem; - uint64 map_size = 8 * (uint64)BH_GB; - uint64 page_size = os_getpagesize(); +#ifdef WASM_LINEAR_MEMORY_MMAP + uint8 *mapped_mem = NULL; + uint64 map_size; #endif #if WASM_ENABLE_SHARED_MEMORY != 0 @@ -295,18 +303,29 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, (void)max_memory_data_size; bh_assert(memory != NULL); + #ifndef OS_ENABLE_HW_BOUND_CHECK #if WASM_ENABLE_SHARED_MEMORY != 0 if (is_shared_memory) { /* Allocate maximum memory size when memory is shared */ +#if WASM_ENABLE_SHARED_MEMORY_MMAP != 0 + map_size = max_memory_data_size; + if (max_memory_data_size > 0 + && !(memory->memory_data = mapped_mem = + wasm_mmap_linear_memory(map_size, &max_memory_data_size, + error_buf, error_buf_size))) { + goto fail1; + } +#else if (max_memory_data_size > 0 && !(memory->memory_data = runtime_malloc( max_memory_data_size, error_buf, error_buf_size))) { goto fail1; } +#endif } else -#endif +#endif /* end of WASM_ENABLE_SHARED_MEMORY != 0 */ { /* Allocate initial memory size when memory is not shared */ if (memory_data_size > 0 @@ -315,43 +334,18 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, goto fail1; } } -#else /* else of OS_ENABLE_HW_BOUND_CHECK */ - memory_data_size = (memory_data_size + page_size - 1) & ~(page_size - 1); - +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: * ea = i + memarg.offset * both i and memarg.offset are u32 in range 0 to 4G * so the range of ea is 0 to 8G */ - if (!(memory->memory_data = mapped_mem = - os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE, - os_get_invalid_handle()))) { + map_size = 8 * (uint64)BH_GB; + if (!(memory->memory_data = mapped_mem = wasm_mmap_linear_memory( + map_size, &memory_data_size, error_buf, error_buf_size))) { set_error_buf(error_buf, error_buf_size, "mmap memory failed"); goto fail1; } - -#ifdef BH_PLATFORM_WINDOWS - if (memory_data_size > 0 - && !os_mem_commit(mapped_mem, memory_data_size, - MMAP_PROT_READ | MMAP_PROT_WRITE)) { - set_error_buf(error_buf, error_buf_size, "commit memory failed"); - os_munmap(mapped_mem, map_size); - goto fail1; - } -#endif - - if (os_mprotect(mapped_mem, memory_data_size, - MMAP_PROT_READ | MMAP_PROT_WRITE) - != 0) { - set_error_buf(error_buf, error_buf_size, "mprotect memory failed"); - goto fail2; - } - - /* Newly allocated pages are filled with zero by the OS, we don't fill it - * again here */ - - if (memory_data_size > UINT32_MAX) - memory_data_size = UINT32_MAX; #endif /* end of OS_ENABLE_HW_BOUND_CHECK */ memory->module_type = Wasm_Module_Bytecode; @@ -398,15 +392,15 @@ fail3: if (heap_size > 0) wasm_runtime_free(memory->heap_handle); fail2: -#ifndef OS_ENABLE_HW_BOUND_CHECK - if (memory->memory_data) - wasm_runtime_free(memory->memory_data); -#else -#ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(mapped_mem, memory_data_size); -#endif - os_munmap(mapped_mem, map_size); +#ifdef WASM_LINEAR_MEMORY_MMAP + if (mapped_mem) + wasm_munmap_linear_memory(mapped_mem, memory_data_size, map_size); + else #endif + { + if (memory->memory_data) + wasm_runtime_free(memory->memory_data); + } fail1: return NULL; } diff --git a/core/shared/platform/android/platform_internal.h b/core/shared/platform/android/platform_internal.h index 42e4e726c..4449f21e8 100644 --- a/core/shared/platform/android/platform_internal.h +++ b/core/shared/platform/android/platform_internal.h @@ -80,8 +80,6 @@ typedef jmp_buf korp_jmpbuf; #define os_longjmp longjmp #define os_alloca alloca -#define os_getpagesize getpagesize - typedef void (*os_signal_handler)(void *sig_addr); int @@ -101,6 +99,8 @@ os_sigreturn(); #endif /* end of BUILD_TARGET_X86_64/AMD_64/AARCH64/RISCV64 */ #endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ +#define os_getpagesize getpagesize + typedef long int __syscall_slong_t; #if __ANDROID_API__ < 19 diff --git a/core/shared/platform/cosmopolitan/platform_internal.h b/core/shared/platform/cosmopolitan/platform_internal.h index 02cd78be4..66515e305 100644 --- a/core/shared/platform/cosmopolitan/platform_internal.h +++ b/core/shared/platform/cosmopolitan/platform_internal.h @@ -98,8 +98,6 @@ typedef jmp_buf korp_jmpbuf; #define os_longjmp longjmp #define os_alloca alloca -#define os_getpagesize getpagesize - typedef void (*os_signal_handler)(void *sig_addr); int @@ -119,6 +117,8 @@ os_sigreturn(); #endif /* end of BUILD_TARGET_X86_64/AMD_64/AARCH64/RISCV64 */ #endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ +#define os_getpagesize getpagesize + #ifdef __cplusplus } #endif diff --git a/core/shared/platform/darwin/platform_internal.h b/core/shared/platform/darwin/platform_internal.h index 30b89624e..1cbecdc20 100644 --- a/core/shared/platform/darwin/platform_internal.h +++ b/core/shared/platform/darwin/platform_internal.h @@ -82,8 +82,6 @@ typedef jmp_buf korp_jmpbuf; #define os_longjmp longjmp #define os_alloca alloca -#define os_getpagesize getpagesize - typedef void (*os_signal_handler)(void *sig_addr); int @@ -103,6 +101,8 @@ os_sigreturn(); #endif /* end of BUILD_TARGET_X86_64/AMD_64/AARCH64/RISCV64 */ #endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ +#define os_getpagesize getpagesize + #if WASM_DISABLE_WAKEUP_BLOCKING_OP == 0 #define OS_ENABLE_WAKEUP_BLOCKING_OP #endif diff --git a/core/shared/platform/freebsd/platform_internal.h b/core/shared/platform/freebsd/platform_internal.h index 5241c6456..bfdfe1493 100644 --- a/core/shared/platform/freebsd/platform_internal.h +++ b/core/shared/platform/freebsd/platform_internal.h @@ -85,8 +85,6 @@ typedef jmp_buf korp_jmpbuf; #define os_longjmp longjmp #define os_alloca alloca -#define os_getpagesize getpagesize - typedef void (*os_signal_handler)(void *sig_addr); int @@ -106,6 +104,8 @@ os_sigreturn(); #endif /* end of BUILD_TARGET_X86_64/AMD_64/AARCH64/RISCV64 */ #endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ +#define os_getpagesize getpagesize + #if WASM_DISABLE_WAKEUP_BLOCKING_OP == 0 #define OS_ENABLE_WAKEUP_BLOCKING_OP #endif diff --git a/core/shared/platform/linux/platform_internal.h b/core/shared/platform/linux/platform_internal.h index 335070bf8..aeddc4ccf 100644 --- a/core/shared/platform/linux/platform_internal.h +++ b/core/shared/platform/linux/platform_internal.h @@ -95,8 +95,6 @@ typedef jmp_buf korp_jmpbuf; #define os_longjmp longjmp #define os_alloca alloca -#define os_getpagesize getpagesize - typedef void (*os_signal_handler)(void *sig_addr); int @@ -116,6 +114,8 @@ os_sigreturn(); #endif /* end of BUILD_TARGET_X86_64/AMD_64/AARCH64/RISCV64 */ #endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ +#define os_getpagesize getpagesize + #if WASM_DISABLE_WAKEUP_BLOCKING_OP == 0 #define OS_ENABLE_WAKEUP_BLOCKING_OP #endif diff --git a/core/shared/platform/vxworks/platform_internal.h b/core/shared/platform/vxworks/platform_internal.h index 930ff7777..1b870c70e 100644 --- a/core/shared/platform/vxworks/platform_internal.h +++ b/core/shared/platform/vxworks/platform_internal.h @@ -79,8 +79,6 @@ typedef jmp_buf korp_jmpbuf; #define os_longjmp longjmp #define os_alloca alloca -#define os_getpagesize getpagesize - typedef void (*os_signal_handler)(void *sig_addr); int @@ -100,6 +98,8 @@ os_sigreturn(); #endif /* end of BUILD_TARGET_X86_64/AMD_64/AARCH64 */ #endif /* end of WASM_DISABLE_HW_BOUND_CHECK */ +#define os_getpagesize getpagesize + static inline os_file_handle os_get_invalid_handle() {