From fd5030e02e5b9a3d490380d0893046b40075ad06 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Fri, 22 Jul 2022 11:05:40 +0800 Subject: [PATCH] Implement interpreter hw bound check (#1309) Implement boundary check with hardware trap for interpreter on 64-bit platforms: - To improve the performance of interpreter and Fast JIT - To prepare for multi-tier compilation for the feature Linux/MacOS/Windows 64-bit are enabled. --- core/iwasm/aot/aot_runtime.c | 92 ++--- core/iwasm/aot/aot_runtime.h | 10 +- core/iwasm/common/wasm_runtime_common.c | 115 +++++- core/iwasm/common/wasm_runtime_common.h | 20 + core/iwasm/interpreter/wasm_interp_classic.c | 35 +- core/iwasm/interpreter/wasm_interp_fast.c | 35 +- core/iwasm/interpreter/wasm_runtime.c | 375 +++++++++++++++++-- core/iwasm/interpreter/wasm_runtime.h | 10 + 8 files changed, 591 insertions(+), 101 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 3a608117e..00a22ffff 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -518,7 +518,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, #endif if (os_mprotect(p, total_size, MMAP_PROT_READ | MMAP_PROT_WRITE) != 0) { - set_error_buf(error_buf, error_buf_size, "mprotec memory failed"); + set_error_buf(error_buf, error_buf_size, "mprotect memory failed"); #ifdef BH_PLATFORM_WINDOWS os_mem_decommit(p, total_size); #endif @@ -1186,12 +1186,12 @@ aot_lookup_function(const AOTModuleInstance *module_inst, const char *name, #ifdef OS_ENABLE_HW_BOUND_CHECK -static os_thread_local_attribute WASMExecEnv *aot_exec_env = NULL; - #ifndef BH_PLATFORM_WINDOWS -static void -aot_signal_handler(void *sig_addr) +void +aot_signal_handler(WASMSignalInfo *sig_info) { + WASMExecEnv *exec_env_tls = sig_info->exec_env_tls; + void *sig_addr = sig_info->sig_addr; AOTModuleInstance *module_inst; AOTMemoryInstance *memory_inst; WASMJmpBuf *jmpbuf_node; @@ -1202,10 +1202,10 @@ aot_signal_handler(void *sig_addr) uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; /* Check whether current thread is running aot function */ - if (aot_exec_env && aot_exec_env->handle == os_self_thread() - && (jmpbuf_node = aot_exec_env->jmpbuf_stack_top)) { + if (exec_env_tls && exec_env_tls->handle == os_self_thread() + && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { /* Get mapped mem info of current instance */ - module_inst = (AOTModuleInstance *)aot_exec_env->module_inst; + module_inst = (AOTModuleInstance *)exec_env_tls->module_inst; /* Get the default memory instance */ memory_inst = aot_get_default_memory(module_inst); if (memory_inst) { @@ -1222,7 +1222,7 @@ aot_signal_handler(void *sig_addr) && (mapped_mem_start_addr <= (uint8 *)sig_addr && (uint8 *)sig_addr < mapped_mem_end_addr)) { /* The address which causes segmentation fault is inside - aot instance's guard regions */ + the memory instance's guard regions */ aot_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS); os_longjmp(jmpbuf_node->jmpbuf, 1); @@ -1238,9 +1238,11 @@ aot_signal_handler(void *sig_addr) } } #else /* else of BH_PLATFORM_WINDOWS */ -static LONG -aot_exception_handler(EXCEPTION_POINTERS *exce_info) +LONG +aot_exception_handler(WASMSignalInfo *sig_info) { + WASMExecEnv *exec_env_tls = sig_info->exec_env_tls; + EXCEPTION_POINTERS *exce_info = sig_info->exce_info; PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord; uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1]; AOTModuleInstance *module_inst; @@ -1250,9 +1252,9 @@ aot_exception_handler(EXCEPTION_POINTERS *exce_info) uint8 *mapped_mem_end_addr = NULL; uint32 page_size = os_getpagesize(); - if (aot_exec_env && aot_exec_env->handle == os_self_thread() - && (jmpbuf_node = aot_exec_env->jmpbuf_stack_top)) { - module_inst = (AOTModuleInstance *)aot_exec_env->module_inst; + if (exec_env_tls && exec_env_tls->handle == os_self_thread() + && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { + module_inst = (AOTModuleInstance *)exec_env_tls->module_inst; if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { /* Get the default memory instance */ memory_inst = aot_get_default_memory(module_inst); @@ -1293,32 +1295,6 @@ aot_exception_handler(EXCEPTION_POINTERS *exce_info) } #endif /* end of BH_PLATFORM_WINDOWS */ -bool -aot_signal_init() -{ -#ifndef BH_PLATFORM_WINDOWS - return os_thread_signal_init(aot_signal_handler) == 0 ? true : false; -#else - if (os_thread_signal_init() != 0) - return false; - - if (!AddVectoredExceptionHandler(1, aot_exception_handler)) { - os_thread_signal_destroy(); - return false; - } -#endif - return true; -} - -void -aot_signal_destroy() -{ -#ifdef BH_PLATFORM_WINDOWS - RemoveVectoredExceptionHandler(aot_exception_handler); -#endif - os_thread_signal_destroy(); -} - static bool invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, const WASMType *func_type, @@ -1326,7 +1302,7 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, uint32 *argv, uint32 argc, uint32 *argv_ret) { AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst; - WASMExecEnv **p_aot_exec_env = &aot_exec_env; + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop; uint32 page_size = os_getpagesize(); uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; @@ -1348,7 +1324,7 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, return false; } - if (aot_exec_env && (aot_exec_env != exec_env)) { + if (exec_env_tls && (exec_env_tls != exec_env)) { aot_set_exception(module_inst, "invalid exec env"); return false; } @@ -1360,7 +1336,7 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node); - aot_exec_env = exec_env; + wasm_runtime_set_exec_env_tls(exec_env); if (os_setjmp(jmpbuf_node.jmpbuf) == 0) { /* Quick call with func_ptr if the function signature is simple */ if (!signature && param_count == 1 && types[0] == VALUE_TYPE_I32) { @@ -1406,7 +1382,7 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, jmpbuf_node_pop = wasm_exec_env_pop_jmpbuf(exec_env); bh_assert(&jmpbuf_node == jmpbuf_node_pop); if (!exec_env->jmpbuf_stack_top) { - *p_aot_exec_env = NULL; + wasm_runtime_set_exec_env_tls(NULL); } if (!ret) { os_sigreturn(); @@ -1594,7 +1570,7 @@ aot_create_exec_env_and_call_function(AOTModuleInstance *module_inst, bool ret; #if defined(OS_ENABLE_HW_BOUND_CHECK) - existing_exec_env = exec_env = aot_exec_env; + existing_exec_env = exec_env = wasm_runtime_get_exec_env_tls(); #elif WASM_ENABLE_THREAD_MGR != 0 existing_exec_env = exec_env = wasm_clusters_search_exec_env((WASMModuleInstanceCommon *)module_inst); @@ -1611,7 +1587,7 @@ aot_create_exec_env_and_call_function(AOTModuleInstance *module_inst, ret = aot_call_function(exec_env, func, argc, argv); - /* don't destroy the exec_env if it's searched from the cluster */ + /* don't destroy the exec_env if it isn't created in this function */ if (!existing_exec_env) wasm_exec_env_destroy(exec_env); @@ -1707,6 +1683,9 @@ execute_malloc_function(AOTModuleInstance *module_inst, AOTFunctionInstance *retain_func, uint32 size, uint32 *p_result) { +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif uint32 argv[2], argc; bool ret; @@ -1718,13 +1697,13 @@ execute_malloc_function(AOTModuleInstance *module_inst, } #ifdef OS_ENABLE_HW_BOUND_CHECK - if (aot_exec_env != NULL) { - bh_assert(aot_exec_env->module_inst + if (exec_env_tls != NULL) { + bh_assert(exec_env_tls->module_inst == (WASMModuleInstanceCommon *)module_inst); - ret = aot_call_function(aot_exec_env, malloc_func, argc, argv); + ret = aot_call_function(exec_env_tls, malloc_func, argc, argv); if (retain_func && ret) { - ret = aot_call_function(aot_exec_env, retain_func, 1, argv); + ret = aot_call_function(exec_env_tls, retain_func, 1, argv); } } else @@ -1748,14 +1727,17 @@ static bool execute_free_function(AOTModuleInstance *module_inst, AOTFunctionInstance *free_func, uint32 offset) { +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif uint32 argv[2]; argv[0] = offset; #ifdef OS_ENABLE_HW_BOUND_CHECK - if (aot_exec_env != NULL) { - bh_assert(aot_exec_env->module_inst + if (exec_env_tls != NULL) { + bh_assert(exec_env_tls->module_inst == (WASMModuleInstanceCommon *)module_inst); - return aot_call_function(aot_exec_env, free_func, 1, argv); + return aot_call_function(exec_env_tls, free_func, 1, argv); } else #endif @@ -2197,8 +2179,8 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) return false; } - memset(memory_inst->memory_data_end.ptr, 0, - num_bytes_per_page * inc_page_count); + /* The increased pages are filled with zero by the OS when os_mmap, + no need to memset it again here */ memory_inst->cur_page_count = total_page_count; memory_inst->memory_data_size = (uint32)total_size; diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index c3838b585..b335b98ba 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -684,11 +684,13 @@ aot_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size); #endif #ifdef OS_ENABLE_HW_BOUND_CHECK -bool -aot_signal_init(); - +#ifndef BH_PLATFORM_WINDOWS void -aot_signal_destroy(); +aot_signal_handler(WASMSignalInfo *sig_info); +#else +LONG +aot_exception_handler(WASMSignalInfo *sig_info); +#endif #endif void diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index d357391ef..6ef54bf65 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -117,6 +117,96 @@ runtime_malloc(uint64 size, WASMModuleInstanceCommon *module_inst, return mem; } +#ifdef OS_ENABLE_HW_BOUND_CHECK +/* The exec_env of thread local storage, set before calling function + and used in signal handler, as we cannot get it from the argument + of signal handler */ +static os_thread_local_attribute WASMExecEnv *exec_env_tls = NULL; + +#ifndef BH_PLATFORM_WINDOWS +static void +runtime_signal_handler(void *sig_addr) +{ + WASMModuleInstanceCommon *module_inst; + WASMSignalInfo sig_info; + + sig_info.exec_env_tls = exec_env_tls; + sig_info.sig_addr = sig_addr; + if (exec_env_tls) { + module_inst = exec_env_tls->module_inst; +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + wasm_signal_handler(&sig_info); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + aot_signal_handler(&sig_info); +#endif + } +} +#else +static LONG +runtime_exception_handler(EXCEPTION_POINTERS *exce_info) +{ + WASMModuleInstanceCommon *module_inst; + WASMSignalInfo sig_info; + + sig_info.exec_env_tls = exec_env_tls; + sig_info.exce_info = exce_info; + if (exec_env_tls) { + module_inst = exec_env_tls->module_inst; +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) + return wasm_exception_handler(&sig_info); +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) + return aot_exception_handler(&sig_info); +#endif + } + return EXCEPTION_CONTINUE_SEARCH; +} +#endif /* end of BH_PLATFORM_WINDOWS */ + +static bool +runtime_signal_init() +{ +#ifndef BH_PLATFORM_WINDOWS + return os_thread_signal_init(runtime_signal_handler) == 0 ? true : false; +#else + if (os_thread_signal_init() != 0) + return false; + + if (!AddVectoredExceptionHandler(1, runtime_exception_handler)) { + os_thread_signal_destroy(); + return false; + } +#endif + return true; +} + +static void +runtime_signal_destroy() +{ +#ifdef BH_PLATFORM_WINDOWS + RemoveVectoredExceptionHandler(aot_exception_handler); +#endif + os_thread_signal_destroy(); +} + +void +wasm_runtime_set_exec_env_tls(WASMExecEnv *exec_env) +{ + exec_env_tls = exec_env; +} + +WASMExecEnv * +wasm_runtime_get_exec_env_tls() +{ + return exec_env_tls; +} +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ + static bool wasm_runtime_env_init() { @@ -149,12 +239,13 @@ wasm_runtime_env_init() } #endif -#if WASM_ENABLE_AOT != 0 #ifdef OS_ENABLE_HW_BOUND_CHECK - if (!aot_signal_init()) { + if (!runtime_signal_init()) { goto fail6; } #endif + +#if WASM_ENABLE_AOT != 0 #if WASM_ENABLE_DEBUG_AOT != 0 if (!jit_debug_engine_init()) { goto fail7; @@ -178,10 +269,10 @@ fail8: jit_debug_engine_destroy(); fail7: #endif -#ifdef OS_ENABLE_HW_BOUND_CHECK - aot_signal_destroy(); -fail6: #endif +#ifdef OS_ENABLE_HW_BOUND_CHECK + runtime_signal_destroy(); +fail6: #endif #if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_THREAD_MGR != 0) thread_manager_destroy(); @@ -238,9 +329,10 @@ wasm_runtime_destroy() #if WASM_ENABLE_DEBUG_AOT != 0 jit_debug_engine_destroy(); #endif -#ifdef OS_ENABLE_HW_BOUND_CHECK - aot_signal_destroy(); #endif + +#ifdef OS_ENABLE_HW_BOUND_CHECK + runtime_signal_destroy(); #endif /* runtime env destroy */ @@ -953,26 +1045,23 @@ wasm_runtime_init_thread_env(void) return false; #endif -#if WASM_ENABLE_AOT != 0 #ifdef OS_ENABLE_HW_BOUND_CHECK - if (!aot_signal_init()) { + if (!runtime_signal_init()) { #ifdef BH_PLATFORM_WINDOWS os_thread_env_destroy(); #endif return false; } #endif -#endif + return true; } void wasm_runtime_destroy_thread_env(void) { -#if WASM_ENABLE_AOT != 0 #ifdef OS_ENABLE_HW_BOUND_CHECK - aot_signal_destroy(); -#endif + runtime_signal_destroy(); #endif #ifdef BH_PLATFORM_WINDOWS diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index a4bec59c5..5cc9c52da 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -407,6 +407,26 @@ typedef struct wasm_frame_t { const char *func_name_wp; } WASMCApiFrame; +#ifdef OS_ENABLE_HW_BOUND_CHECK +/* Signal info passing to interp/aot signal handler */ +typedef struct WASMSignalInfo { + WASMExecEnv *exec_env_tls; +#ifndef BH_PLATFORM_WINDOWS + void *sig_addr; +#else + EXCEPTION_POINTERS *exce_info; +#endif +} WASMSignalInfo; + +/* Set exec_env of thread local storage */ +void +wasm_runtime_set_exec_env_tls(WASMExecEnv *exec_env); + +/* Get exec_env of thread local storage */ +WASMExecEnv * +wasm_runtime_get_exec_env_tls(void); +#endif + /* See wasm_export.h for description */ WASM_RUNTIME_API_EXTERN bool wasm_runtime_init(void); diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 8504a6860..72689eb1f 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -24,6 +24,7 @@ typedef float64 CellType_F64; #define BR_TABLE_TMP_BUF_LEN 32 +#ifndef OS_ENABLE_HW_BOUND_CHECK #define CHECK_MEMORY_OVERFLOW(bytes) \ do { \ uint64 offset1 = (uint64)offset + (uint64)addr; \ @@ -45,6 +46,18 @@ typedef float64 CellType_F64; else \ goto out_of_bounds; \ } while (0) +#else +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + maddr = memory->memory_data + offset1; \ + } while (0) + +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + maddr = memory->memory_data + (uint32)(start); \ + } while (0) +#endif #define CHECK_ATOMIC_MEMORY_ACCESS() \ do { \ @@ -1006,10 +1019,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMInterpFrame *prev_frame) { WASMMemoryInstance *memory = module->default_memory; - uint32 num_bytes_per_page = memory ? memory->num_bytes_per_page : 0; uint8 *global_data = module->global_data; +#ifndef OS_ENABLE_HW_BOUND_CHECK + uint32 num_bytes_per_page = memory ? memory->num_bytes_per_page : 0; uint32 linear_mem_size = memory ? num_bytes_per_page * memory->cur_page_count : 0; +#endif WASMType **wasm_types = module->module->types; WASMGlobalInstance *globals = module->globals, *global; uint8 opcode_IMPDEP = WASM_OP_IMPDEP; @@ -1959,8 +1974,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, PUSH_I32(prev_page_count); /* update memory instance ptr and memory size */ memory = module->default_memory; +#ifndef OS_ENABLE_HW_BOUND_CHECK linear_mem_size = num_bytes_per_page * memory->cur_page_count; +#endif } (void)reserved; @@ -2988,11 +3005,16 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, seg_len = (uint64)module->module->data_segments[segment] ->data_length; data = module->module->data_segments[segment]->data; +#ifndef OS_ENABLE_HW_BOUND_CHECK if (offset + bytes > seg_len) goto out_of_bounds; bh_memcpy_s(maddr, linear_mem_size - addr, data + offset, (uint32)bytes); +#else + bh_memcpy_s(maddr, (uint32)bytes, data + offset, + (uint32)bytes); +#endif break; } case WASM_OP_DATA_DROP: @@ -3019,8 +3041,11 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); /* allowing the destination and source to overlap */ +#ifndef OS_ENABLE_HW_BOUND_CHECK bh_memmove_s(mdst, linear_mem_size - dst, msrc, len); - +#else + bh_memmove_s(mdst, len, msrc, len); +#endif break; } case WASM_OP_MEMORY_FILL: @@ -3681,8 +3706,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* update memory instance ptr and memory size */ memory = module->default_memory; +#ifndef OS_ENABLE_HW_BOUND_CHECK if (memory) linear_mem_size = num_bytes_per_page * memory->cur_page_count; +#endif if (wasm_get_exception(module)) goto got_exception; } @@ -3759,8 +3786,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; #endif +#ifndef OS_ENABLE_HW_BOUND_CHECK out_of_bounds: wasm_set_exception(module, "out of bounds memory access"); +#endif got_exception: #if WASM_ENABLE_DEBUG_INTERP != 0 @@ -3808,11 +3837,13 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, } argc = function->param_cell_num; +#ifndef OS_ENABLE_HW_BOUND_CHECK if ((uint8 *)&prev_frame < exec_env->native_stack_boundary) { wasm_set_exception((WASMModuleInstance *)exec_env->module_inst, "native stack overflow"); return; } +#endif if (!(frame = ALLOC_FRAME(exec_env, frame_size, (WASMInterpFrame *)prev_frame))) diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index cb8252e76..178d4f1cf 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -18,6 +18,7 @@ typedef int64 CellType_I64; typedef float32 CellType_F32; typedef float64 CellType_F64; +#ifndef OS_ENABLE_HW_BOUND_CHECK #define CHECK_MEMORY_OVERFLOW(bytes) \ do { \ uint64 offset1 = (uint64)offset + (uint64)addr; \ @@ -39,6 +40,18 @@ typedef float64 CellType_F64; else \ goto out_of_bounds; \ } while (0) +#else +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + maddr = memory->memory_data + offset1; \ + } while (0) + +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + maddr = memory->memory_data + (uint32)(start); \ + } while (0) +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ #define CHECK_ATOMIC_MEMORY_ACCESS(align) \ do { \ @@ -1080,10 +1093,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMInterpFrame *prev_frame) { WASMMemoryInstance *memory = module->default_memory; +#ifndef OS_ENABLE_HW_BOUND_CHECK uint32 num_bytes_per_page = memory ? memory->num_bytes_per_page : 0; - uint8 *global_data = module->global_data; uint32 linear_mem_size = memory ? num_bytes_per_page * memory->cur_page_count : 0; +#endif + uint8 *global_data = module->global_data; WASMGlobalInstance *globals = module->globals, *global; uint8 opcode_IMPDEP = WASM_OP_IMPDEP; WASMInterpFrame *frame = NULL; @@ -1797,8 +1812,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, frame_lp[addr_ret] = prev_page_count; /* update memory instance ptr and memory size */ memory = module->default_memory; +#ifndef OS_ENABLE_HW_BOUND_CHECK linear_mem_size = num_bytes_per_page * memory->cur_page_count; +#endif } (void)reserved; @@ -2907,11 +2924,16 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, seg_len = (uint64)module->module->data_segments[segment] ->data_length; data = module->module->data_segments[segment]->data; +#ifndef OS_ENABLE_HW_BOUND_CHECK if (offset + bytes > seg_len) goto out_of_bounds; bh_memcpy_s(maddr, linear_mem_size - addr, data + offset, (uint32)bytes); +#else + bh_memcpy_s(maddr, (uint32)bytes, data + offset, + (uint32)bytes); +#endif break; } case WASM_OP_DATA_DROP: @@ -2937,8 +2959,11 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); /* allowing the destination and source to overlap */ +#ifndef OS_ENABLE_HW_BOUND_CHECK bh_memmove_s(mdst, linear_mem_size - dst, msrc, len); - +#else + bh_memmove_s(mdst, len, msrc, len); +#endif break; } case WASM_OP_MEMORY_FILL: @@ -3694,8 +3719,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* update memory instance ptr and memory size */ memory = module->default_memory; +#ifndef OS_ENABLE_HW_BOUND_CHECK if (memory) linear_mem_size = num_bytes_per_page * memory->cur_page_count; +#endif if (wasm_get_exception(module)) goto got_exception; } @@ -3761,8 +3788,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; #endif +#ifndef OS_ENABLE_HW_BOUND_CHECK out_of_bounds: wasm_set_exception(module, "out of bounds memory access"); +#endif got_exception: SYNC_ALL_TO_FRAME(); @@ -3813,11 +3842,13 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, } argc = function->param_cell_num; +#ifndef OS_ENABLE_HW_BOUND_CHECK if ((uint8 *)&prev_frame < exec_env->native_stack_boundary) { wasm_set_exception((WASMModuleInstance *)exec_env->module_inst, "native stack overflow"); return; } +#endif if (!(frame = ALLOC_FRAME(exec_env, frame_size, (WASMInterpFrame *)prev_frame))) diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 568b9db1d..8371c06ea 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -130,8 +130,19 @@ memories_deinstantiate(WASMModuleInstance *module_inst, wasm_runtime_free(memories[i]->heap_handle); memories[i]->heap_handle = NULL; } - if (memories[i]->memory_data) + if (memories[i]->memory_data) { +#ifndef OS_ENABLE_HW_BOUND_CHECK wasm_runtime_free(memories[i]->memory_data); +#else +#ifdef BH_PLATFORM_WINDOWS + os_mem_decommit(memories[i]->memory_data, + memories[i]->num_bytes_per_page + * memories[i]->cur_page_count); +#endif + os_munmap((uint8 *)memories[i]->memory_data, + 8 * (uint64)BH_GB); +#endif + } wasm_runtime_free(memories[i]); } } @@ -153,6 +164,11 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, uint32 inc_page_count, aux_heap_base, global_idx; uint32 bytes_of_last_page, bytes_to_page_end; uint8 *global_addr; +#ifdef OS_ENABLE_HW_BOUND_CHECK + uint8 *mapped_mem; + uint64 map_size = 8 * (uint64)BH_GB; + uint64 page_size = os_getpagesize(); +#endif #if WASM_ENABLE_SHARED_MEMORY != 0 bool is_shared_memory = flags & 0x02 ? true : false; @@ -268,11 +284,45 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, return NULL; } +#ifndef OS_ENABLE_HW_BOUND_CHECK if (memory_data_size > 0 && !(memory->memory_data = runtime_malloc(memory_data_size, error_buf, error_buf_size))) { goto fail1; } +#else + memory_data_size = (memory_data_size + page_size - 1) & ~(page_size - 1); + + /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: + * ea = i + memarg.offset + * both i and memarg.offset are u32 in range 0 to 4G + * so the range of ea is 0 to 8G + */ + if (memory_data_size >= UINT32_MAX + || !(memory->memory_data = mapped_mem = + os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE))) { + set_error_buf(error_buf, error_buf_size, "mmap memory failed"); + goto fail1; + } + +#ifdef BH_PLATFORM_WINDOWS + if (!os_mem_commit(mapped_mem, memory_data_size, + MMAP_PROT_READ | MMAP_PROT_WRITE)) { + set_error_buf(error_buf, error_buf_size, "commit memory failed"); + os_munmap(mapped_mem, map_size); + goto fail1; + } +#endif + + if (os_mprotect(mapped_mem, memory_data_size, + MMAP_PROT_READ | MMAP_PROT_WRITE) + != 0) { + set_error_buf(error_buf, error_buf_size, "mprotect memory failed"); + goto fail2; + } + /* Newly allocated pages are filled with zero by the OS, we don't fill it + * again here */ +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ memory->module_type = Wasm_Module_Bytecode; memory->num_bytes_per_page = num_bytes_per_page; @@ -327,8 +377,15 @@ fail3: if (heap_size > 0) wasm_runtime_free(memory->heap_handle); fail2: +#ifndef OS_ENABLE_HW_BOUND_CHECK if (memory->memory_data) wasm_runtime_free(memory->memory_data); +#else +#ifdef BH_PLATFORM_WINDOWS + os_mem_decommit(mapped_mem, memory_data_size); +#endif + os_munmap(mapped_mem, map_size); +#endif fail1: wasm_runtime_free(memory); return NULL; @@ -949,6 +1006,9 @@ execute_malloc_function(WASMModuleInstance *module_inst, WASMFunctionInstance *retain_func, uint32 size, uint32 *p_result) { +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif uint32 argv[2], argc; bool ret; @@ -967,12 +1027,26 @@ execute_malloc_function(WASMModuleInstance *module_inst, argc = 2; } - ret = wasm_create_exec_env_and_call_function(module_inst, malloc_func, argc, - argv, false); +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (exec_env_tls != NULL) { + bh_assert(exec_env_tls->module_inst + == (WASMModuleInstanceCommon *)module_inst); + ret = wasm_call_function(exec_env_tls, malloc_func, argc, argv); - if (retain_func && ret) { - ret = wasm_create_exec_env_and_call_function(module_inst, retain_func, - 1, argv, false); + if (retain_func && ret) { + ret = wasm_call_function(exec_env_tls, retain_func, 1, argv); + } + } + else +#endif + { + ret = wasm_create_exec_env_and_call_function(module_inst, malloc_func, + argc, argv, false); + + if (retain_func && ret) { + ret = wasm_create_exec_env_and_call_function( + module_inst, retain_func, 1, argv, false); + } } if (ret) @@ -984,11 +1058,24 @@ static bool execute_free_function(WASMModuleInstance *module_inst, WASMFunctionInstance *free_func, uint32 offset) { +#ifdef OS_ENABLE_HW_BOUND_CHECK + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); +#endif uint32 argv[2]; argv[0] = offset; - return wasm_create_exec_env_and_call_function(module_inst, free_func, 1, - argv, false); +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (exec_env_tls != NULL) { + bh_assert(exec_env_tls->module_inst + == (WASMModuleInstanceCommon *)module_inst); + return wasm_call_function(exec_env_tls, free_func, 1, argv); + } + else +#endif + { + return wasm_create_exec_env_and_call_function(module_inst, free_func, 1, + argv, false); + } } #if WASM_ENABLE_MULTI_MODULE != 0 @@ -1704,6 +1791,195 @@ clear_wasi_proc_exit_exception(WASMModuleInstance *module_inst) #endif } +#ifdef OS_ENABLE_HW_BOUND_CHECK + +#ifndef BH_PLATFORM_WINDOWS +void +wasm_signal_handler(WASMSignalInfo *sig_info) +{ + WASMExecEnv *exec_env_tls = sig_info->exec_env_tls; + void *sig_addr = sig_info->sig_addr; + WASMModuleInstance *module_inst; + WASMMemoryInstance *memory_inst; + WASMJmpBuf *jmpbuf_node; + uint8 *mapped_mem_start_addr = NULL; + uint8 *mapped_mem_end_addr = NULL; + uint8 *stack_min_addr; + uint32 page_size; + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; + + /* Check whether current thread is running wasm function */ + if (exec_env_tls && exec_env_tls->handle == os_self_thread() + && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { + /* Get mapped mem info of current instance */ + module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; + /* Get the default memory instance */ + memory_inst = module_inst->default_memory; + if (memory_inst) { + mapped_mem_start_addr = (uint8 *)memory_inst->memory_data; + mapped_mem_end_addr = + (uint8 *)memory_inst->memory_data + 8 * (uint64)BH_GB; + } + + /* Get stack info of current thread */ + page_size = os_getpagesize(); + stack_min_addr = os_thread_get_stack_boundary(); + + if (memory_inst + && (mapped_mem_start_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr < mapped_mem_end_addr)) { + /* The address which causes segmentation fault is inside + the memory instance's guard regions */ + wasm_set_exception(module_inst, "out of bounds memory access"); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } + else if (stack_min_addr - page_size <= (uint8 *)sig_addr + && (uint8 *)sig_addr + < stack_min_addr + page_size * guard_page_count) { + /* The address which causes segmentation fault is inside + native thread's guard page */ + wasm_set_exception(module_inst, "native stack overflow"); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } + } +} +#else /* else of BH_PLATFORM_WINDOWS */ +LONG +wasm_exception_handler(WASMSignalInfo *sig_info) +{ + WASMExecEnv *exec_env_tls = sig_info->exec_env_tls; + EXCEPTION_POINTERS *exce_info = sig_info->exce_info; + PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord; + uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1]; + WASMModuleInstance *module_inst; + WASMMemoryInstance *memory_inst; + WASMJmpBuf *jmpbuf_node; + uint8 *mapped_mem_start_addr = NULL; + uint8 *mapped_mem_end_addr = NULL; + uint32 page_size = os_getpagesize(); + + if (exec_env_tls && exec_env_tls->handle == os_self_thread() + && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { + module_inst = (WASMModuleInstance *)exec_env_tls->module_inst; + if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { + /* Get the default memory instance */ + memory_inst = module_inst->default_memory; + if (memory_inst) { + mapped_mem_start_addr = (uint8 *)memory_inst->memory_data; + mapped_mem_end_addr = + (uint8 *)memory_inst->memory_data + 8 * (uint64)BH_GB; + if (mapped_mem_start_addr <= (uint8 *)sig_addr + && (uint8 *)sig_addr < mapped_mem_end_addr) { + /* The address which causes segmentation fault is inside + the memory instance's guard regions. + Set exception and let the wasm func continue to run, when + the wasm func returns, the caller will check whether the + exception is thrown and return to runtime. */ + wasm_set_exception(module_inst, + "out of bounds memory access"); + /* Skip current instruction */ + return EXCEPTION_CONTINUE_SEARCH; + } + } + } + else if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) { + /* Set stack overflow exception and let the wasm func continue + to run, when the wasm func returns, the caller will check + whether the exception is thrown and return to runtime, and + the damaged stack will be recovered by _resetstkoflw(). */ + wasm_set_exception(module_inst, "native stack overflow"); + return EXCEPTION_CONTINUE_SEARCH; + } + } + + os_printf("Unhandled exception thrown: exception code: 0x%lx, " + "exception address: %p, exception information: %p\n", + ExceptionRecord->ExceptionCode, ExceptionRecord->ExceptionAddress, + sig_addr); + return EXCEPTION_CONTINUE_SEARCH; +} +#endif /* end of BH_PLATFORM_WINDOWS */ + +static void +call_wasm_with_hw_bound_check(WASMModuleInstance *module_inst, + WASMExecEnv *exec_env, + WASMFunctionInstance *function, unsigned argc, + uint32 argv[]) +{ + WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls(); + WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop; + uint32 page_size = os_getpagesize(); + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; +#ifdef BH_PLATFORM_WINDOWS + const char *exce; + int result; +#endif + bool ret = true; + + /* Check native stack overflow firstly to ensure we have enough + native stack to run the following codes before actually calling + the aot function in invokeNative function. */ + if ((uint8 *)&exec_env_tls < exec_env->native_stack_boundary + + page_size * (guard_page_count + 1)) { + wasm_set_exception(module_inst, "native stack overflow"); + return; + } + + if (exec_env_tls && (exec_env_tls != exec_env)) { + wasm_set_exception(module_inst, "invalid exec env"); + return; + } + + if (!os_thread_signal_inited()) { + wasm_set_exception(module_inst, "thread signal env not inited"); + return; + } + + wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node); + + wasm_runtime_set_exec_env_tls(exec_env); + if (os_setjmp(jmpbuf_node.jmpbuf) == 0) { +#ifndef BH_PLATFORM_WINDOWS + wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv); +#else + __try { + wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv); + } __except (wasm_get_exception(module_inst) + ? EXCEPTION_EXECUTE_HANDLER + : EXCEPTION_CONTINUE_SEARCH) { + /* exception was thrown in wasm_exception_handler */ + ret = false; + } + if ((exce = wasm_get_exception(module_inst)) + && strstr(exce, "native stack overflow")) { + /* After a stack overflow, the stack was left + in a damaged state, let the CRT repair it */ + result = _resetstkoflw(); + bh_assert(result != 0); + } +#endif + } + else { + /* Exception has been set in signal handler before calling longjmp */ + ret = false; + } + + jmpbuf_node_pop = wasm_exec_env_pop_jmpbuf(exec_env); + bh_assert(&jmpbuf_node == jmpbuf_node_pop); + if (!exec_env->jmpbuf_stack_top) { + wasm_runtime_set_exec_env_tls(NULL); + } + if (!ret) { + os_sigreturn(); + os_signal_unmask(); + } + (void)jmpbuf_node_pop; +} +#define interp_call_wasm call_wasm_with_hw_bound_check +#else +#define interp_call_wasm wasm_interp_call_wasm +#endif + bool wasm_call_function(WASMExecEnv *exec_env, WASMFunctionInstance *function, unsigned argc, uint32 argv[]) @@ -1714,7 +1990,7 @@ wasm_call_function(WASMExecEnv *exec_env, WASMFunctionInstance *function, /* set thread handle and stack boundary */ wasm_exec_env_set_thread_info(exec_env); - wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv); + interp_call_wasm(module_inst, exec_env, function, argc, argv); (void)clear_wasi_proc_exit_exception(module_inst); return !wasm_get_exception(module_inst) ? true : false; } @@ -1725,15 +2001,17 @@ wasm_create_exec_env_and_call_function(WASMModuleInstance *module_inst, unsigned argc, uint32 argv[], bool enable_debug) { - WASMExecEnv *exec_env; + WASMExecEnv *exec_env, *existing_exec_env = NULL; bool ret; -#if WASM_ENABLE_THREAD_MGR != 0 - WASMExecEnv *existing_exec_env = NULL; - - if (!(existing_exec_env = exec_env = wasm_clusters_search_exec_env( - (WASMModuleInstanceCommon *)module_inst))) { +#if defined(OS_ENABLE_HW_BOUND_CHECK) + existing_exec_env = exec_env = wasm_runtime_get_exec_env_tls(); +#elif WASM_ENABLE_THREAD_MGR != 0 + existing_exec_env = exec_env = + wasm_clusters_search_exec_env((WASMModuleInstanceCommon *)module_inst); #endif + + if (!existing_exec_env) { if (!(exec_env = wasm_exec_env_create((WASMModuleInstanceCommon *)module_inst, module_inst->default_wasm_stack_size))) { @@ -1742,20 +2020,18 @@ wasm_create_exec_env_and_call_function(WASMModuleInstance *module_inst, } #if WASM_ENABLE_THREAD_MGR != 0 - if (enable_debug) { #if WASM_ENABLE_DEBUG_INTERP != 0 + if (enable_debug) { wasm_runtime_start_debug_instance(exec_env); -#endif } - } #endif +#endif + } ret = wasm_call_function(exec_env, func, argc, argv); -#if WASM_ENABLE_THREAD_MGR != 0 - /* don't destroy the exec_env if it's searched from the cluster */ + /* don't destroy the exec_env if it isn't created in this function */ if (!existing_exec_env) -#endif wasm_exec_env_destroy(exec_env); return ret; @@ -2111,6 +2387,7 @@ wasm_get_native_addr_range(WASMModuleInstance *module_inst, uint8 *native_ptr, return false; } +#ifndef OS_ENABLE_HW_BOUND_CHECK bool wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) { @@ -2195,6 +2472,57 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) return ret; } +#else +bool +wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) +{ + WASMMemoryInstance *memory = module->default_memory; + uint32 num_bytes_per_page, total_page_count; + + if (!memory) + return false; + + total_page_count = inc_page_count + memory->cur_page_count; + + if (inc_page_count <= 0) + /* No need to enlarge memory */ + return true; + + if (total_page_count < memory->cur_page_count /* integer overflow */ + || total_page_count > memory->max_page_count) { + return false; + } + + num_bytes_per_page = memory->num_bytes_per_page; + +#ifdef BH_PLATFORM_WINDOWS + if (!os_mem_commit(memory->memory_data_end, + num_bytes_per_page * inc_page_count, + MMAP_PROT_READ | MMAP_PROT_WRITE)) { + return false; + } +#endif + + if (os_mprotect(memory->memory_data_end, + num_bytes_per_page * inc_page_count, + MMAP_PROT_READ | MMAP_PROT_WRITE) + != 0) { +#ifdef BH_PLATFORM_WINDOWS + os_mem_decommit(memory->memory_data_end, + num_bytes_per_page * inc_page_count); +#endif + return false; + } + + /* The increased pages are filled with zero by the OS when os_mmap, + no need to memset it again here */ + + memory->cur_page_count = total_page_count; + memory->memory_data_end = + memory->memory_data + num_bytes_per_page * total_page_count; + return true; +} +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_REF_TYPES != 0 bool @@ -2279,7 +2607,7 @@ wasm_call_indirect(WASMExecEnv *exec_env, uint32_t tbl_idx, function_inst = module_inst->functions + function_indices; - wasm_interp_call_wasm(module_inst, exec_env, function_inst, argc, argv); + interp_call_wasm(module_inst, exec_env, function_inst, argc, argv); (void)clear_wasi_proc_exit_exception(module_inst); return !wasm_get_exception(module_inst) ? true : false; @@ -2415,9 +2743,6 @@ wasm_get_module_mem_consumption(const WASMModule *module, mem_conspn->total_size += mem_conspn->table_segs_size; mem_conspn->total_size += mem_conspn->data_segs_size; mem_conspn->total_size += mem_conspn->const_strs_size; -#if WASM_ENABLE_AOT != 0 - mem_conspn->total_size += mem_conspn->aot_code_size; -#endif } void diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index c780e6a11..9c26d97ee 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -385,6 +385,16 @@ bool wasm_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size); #endif +#ifdef OS_ENABLE_HW_BOUND_CHECK +#ifndef BH_PLATFORM_WINDOWS +void +wasm_signal_handler(WASMSignalInfo *sig_info); +#else +LONG +wasm_exception_handler(WASMSignalInfo *sig_info); +#endif +#endif + void wasm_get_module_mem_consumption(const WASMModule *module, WASMModuleMemConsumption *mem_conspn);