Implement interpreter hw bound check (#1309)

Implement boundary check with hardware trap for interpreter on
64-bit platforms:
- To improve the performance of interpreter and Fast JIT
- To prepare for multi-tier compilation for the feature

Linux/MacOS/Windows 64-bit are enabled.
This commit is contained in:
Wenyong Huang 2022-07-22 11:05:40 +08:00 committed by GitHub
parent 32c94161d1
commit fd5030e02e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 591 additions and 101 deletions

View File

@ -518,7 +518,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
#endif
if (os_mprotect(p, total_size, MMAP_PROT_READ | MMAP_PROT_WRITE) != 0) {
set_error_buf(error_buf, error_buf_size, "mprotec memory failed");
set_error_buf(error_buf, error_buf_size, "mprotect memory failed");
#ifdef BH_PLATFORM_WINDOWS
os_mem_decommit(p, total_size);
#endif
@ -1186,12 +1186,12 @@ aot_lookup_function(const AOTModuleInstance *module_inst, const char *name,
#ifdef OS_ENABLE_HW_BOUND_CHECK
static os_thread_local_attribute WASMExecEnv *aot_exec_env = NULL;
#ifndef BH_PLATFORM_WINDOWS
static void
aot_signal_handler(void *sig_addr)
void
aot_signal_handler(WASMSignalInfo *sig_info)
{
WASMExecEnv *exec_env_tls = sig_info->exec_env_tls;
void *sig_addr = sig_info->sig_addr;
AOTModuleInstance *module_inst;
AOTMemoryInstance *memory_inst;
WASMJmpBuf *jmpbuf_node;
@ -1202,10 +1202,10 @@ aot_signal_handler(void *sig_addr)
uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT;
/* Check whether current thread is running aot function */
if (aot_exec_env && aot_exec_env->handle == os_self_thread()
&& (jmpbuf_node = aot_exec_env->jmpbuf_stack_top)) {
if (exec_env_tls && exec_env_tls->handle == os_self_thread()
&& (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) {
/* Get mapped mem info of current instance */
module_inst = (AOTModuleInstance *)aot_exec_env->module_inst;
module_inst = (AOTModuleInstance *)exec_env_tls->module_inst;
/* Get the default memory instance */
memory_inst = aot_get_default_memory(module_inst);
if (memory_inst) {
@ -1222,7 +1222,7 @@ aot_signal_handler(void *sig_addr)
&& (mapped_mem_start_addr <= (uint8 *)sig_addr
&& (uint8 *)sig_addr < mapped_mem_end_addr)) {
/* The address which causes segmentation fault is inside
aot instance's guard regions */
the memory instance's guard regions */
aot_set_exception_with_id(module_inst,
EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS);
os_longjmp(jmpbuf_node->jmpbuf, 1);
@ -1238,9 +1238,11 @@ aot_signal_handler(void *sig_addr)
}
}
#else /* else of BH_PLATFORM_WINDOWS */
static LONG
aot_exception_handler(EXCEPTION_POINTERS *exce_info)
LONG
aot_exception_handler(WASMSignalInfo *sig_info)
{
WASMExecEnv *exec_env_tls = sig_info->exec_env_tls;
EXCEPTION_POINTERS *exce_info = sig_info->exce_info;
PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord;
uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1];
AOTModuleInstance *module_inst;
@ -1250,9 +1252,9 @@ aot_exception_handler(EXCEPTION_POINTERS *exce_info)
uint8 *mapped_mem_end_addr = NULL;
uint32 page_size = os_getpagesize();
if (aot_exec_env && aot_exec_env->handle == os_self_thread()
&& (jmpbuf_node = aot_exec_env->jmpbuf_stack_top)) {
module_inst = (AOTModuleInstance *)aot_exec_env->module_inst;
if (exec_env_tls && exec_env_tls->handle == os_self_thread()
&& (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) {
module_inst = (AOTModuleInstance *)exec_env_tls->module_inst;
if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
/* Get the default memory instance */
memory_inst = aot_get_default_memory(module_inst);
@ -1293,32 +1295,6 @@ aot_exception_handler(EXCEPTION_POINTERS *exce_info)
}
#endif /* end of BH_PLATFORM_WINDOWS */
bool
aot_signal_init()
{
#ifndef BH_PLATFORM_WINDOWS
return os_thread_signal_init(aot_signal_handler) == 0 ? true : false;
#else
if (os_thread_signal_init() != 0)
return false;
if (!AddVectoredExceptionHandler(1, aot_exception_handler)) {
os_thread_signal_destroy();
return false;
}
#endif
return true;
}
void
aot_signal_destroy()
{
#ifdef BH_PLATFORM_WINDOWS
RemoveVectoredExceptionHandler(aot_exception_handler);
#endif
os_thread_signal_destroy();
}
static bool
invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr,
const WASMType *func_type,
@ -1326,7 +1302,7 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr,
uint32 *argv, uint32 argc, uint32 *argv_ret)
{
AOTModuleInstance *module_inst = (AOTModuleInstance *)exec_env->module_inst;
WASMExecEnv **p_aot_exec_env = &aot_exec_env;
WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop;
uint32 page_size = os_getpagesize();
uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT;
@ -1348,7 +1324,7 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr,
return false;
}
if (aot_exec_env && (aot_exec_env != exec_env)) {
if (exec_env_tls && (exec_env_tls != exec_env)) {
aot_set_exception(module_inst, "invalid exec env");
return false;
}
@ -1360,7 +1336,7 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr,
wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node);
aot_exec_env = exec_env;
wasm_runtime_set_exec_env_tls(exec_env);
if (os_setjmp(jmpbuf_node.jmpbuf) == 0) {
/* Quick call with func_ptr if the function signature is simple */
if (!signature && param_count == 1 && types[0] == VALUE_TYPE_I32) {
@ -1406,7 +1382,7 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr,
jmpbuf_node_pop = wasm_exec_env_pop_jmpbuf(exec_env);
bh_assert(&jmpbuf_node == jmpbuf_node_pop);
if (!exec_env->jmpbuf_stack_top) {
*p_aot_exec_env = NULL;
wasm_runtime_set_exec_env_tls(NULL);
}
if (!ret) {
os_sigreturn();
@ -1594,7 +1570,7 @@ aot_create_exec_env_and_call_function(AOTModuleInstance *module_inst,
bool ret;
#if defined(OS_ENABLE_HW_BOUND_CHECK)
existing_exec_env = exec_env = aot_exec_env;
existing_exec_env = exec_env = wasm_runtime_get_exec_env_tls();
#elif WASM_ENABLE_THREAD_MGR != 0
existing_exec_env = exec_env =
wasm_clusters_search_exec_env((WASMModuleInstanceCommon *)module_inst);
@ -1611,7 +1587,7 @@ aot_create_exec_env_and_call_function(AOTModuleInstance *module_inst,
ret = aot_call_function(exec_env, func, argc, argv);
/* don't destroy the exec_env if it's searched from the cluster */
/* don't destroy the exec_env if it isn't created in this function */
if (!existing_exec_env)
wasm_exec_env_destroy(exec_env);
@ -1707,6 +1683,9 @@ execute_malloc_function(AOTModuleInstance *module_inst,
AOTFunctionInstance *retain_func, uint32 size,
uint32 *p_result)
{
#ifdef OS_ENABLE_HW_BOUND_CHECK
WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
#endif
uint32 argv[2], argc;
bool ret;
@ -1718,13 +1697,13 @@ execute_malloc_function(AOTModuleInstance *module_inst,
}
#ifdef OS_ENABLE_HW_BOUND_CHECK
if (aot_exec_env != NULL) {
bh_assert(aot_exec_env->module_inst
if (exec_env_tls != NULL) {
bh_assert(exec_env_tls->module_inst
== (WASMModuleInstanceCommon *)module_inst);
ret = aot_call_function(aot_exec_env, malloc_func, argc, argv);
ret = aot_call_function(exec_env_tls, malloc_func, argc, argv);
if (retain_func && ret) {
ret = aot_call_function(aot_exec_env, retain_func, 1, argv);
ret = aot_call_function(exec_env_tls, retain_func, 1, argv);
}
}
else
@ -1748,14 +1727,17 @@ static bool
execute_free_function(AOTModuleInstance *module_inst,
AOTFunctionInstance *free_func, uint32 offset)
{
#ifdef OS_ENABLE_HW_BOUND_CHECK
WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
#endif
uint32 argv[2];
argv[0] = offset;
#ifdef OS_ENABLE_HW_BOUND_CHECK
if (aot_exec_env != NULL) {
bh_assert(aot_exec_env->module_inst
if (exec_env_tls != NULL) {
bh_assert(exec_env_tls->module_inst
== (WASMModuleInstanceCommon *)module_inst);
return aot_call_function(aot_exec_env, free_func, 1, argv);
return aot_call_function(exec_env_tls, free_func, 1, argv);
}
else
#endif
@ -2197,8 +2179,8 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count)
return false;
}
memset(memory_inst->memory_data_end.ptr, 0,
num_bytes_per_page * inc_page_count);
/* The increased pages are filled with zero by the OS when os_mmap,
no need to memset it again here */
memory_inst->cur_page_count = total_page_count;
memory_inst->memory_data_size = (uint32)total_size;

View File

@ -684,11 +684,13 @@ aot_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size);
#endif
#ifdef OS_ENABLE_HW_BOUND_CHECK
bool
aot_signal_init();
#ifndef BH_PLATFORM_WINDOWS
void
aot_signal_destroy();
aot_signal_handler(WASMSignalInfo *sig_info);
#else
LONG
aot_exception_handler(WASMSignalInfo *sig_info);
#endif
#endif
void

View File

@ -117,6 +117,96 @@ runtime_malloc(uint64 size, WASMModuleInstanceCommon *module_inst,
return mem;
}
#ifdef OS_ENABLE_HW_BOUND_CHECK
/* The exec_env of thread local storage, set before calling function
and used in signal handler, as we cannot get it from the argument
of signal handler */
static os_thread_local_attribute WASMExecEnv *exec_env_tls = NULL;
#ifndef BH_PLATFORM_WINDOWS
static void
runtime_signal_handler(void *sig_addr)
{
WASMModuleInstanceCommon *module_inst;
WASMSignalInfo sig_info;
sig_info.exec_env_tls = exec_env_tls;
sig_info.sig_addr = sig_addr;
if (exec_env_tls) {
module_inst = exec_env_tls->module_inst;
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode)
wasm_signal_handler(&sig_info);
#endif
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT)
aot_signal_handler(&sig_info);
#endif
}
}
#else
static LONG
runtime_exception_handler(EXCEPTION_POINTERS *exce_info)
{
WASMModuleInstanceCommon *module_inst;
WASMSignalInfo sig_info;
sig_info.exec_env_tls = exec_env_tls;
sig_info.exce_info = exce_info;
if (exec_env_tls) {
module_inst = exec_env_tls->module_inst;
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode)
return wasm_exception_handler(&sig_info);
#endif
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT)
return aot_exception_handler(&sig_info);
#endif
}
return EXCEPTION_CONTINUE_SEARCH;
}
#endif /* end of BH_PLATFORM_WINDOWS */
static bool
runtime_signal_init()
{
#ifndef BH_PLATFORM_WINDOWS
return os_thread_signal_init(runtime_signal_handler) == 0 ? true : false;
#else
if (os_thread_signal_init() != 0)
return false;
if (!AddVectoredExceptionHandler(1, runtime_exception_handler)) {
os_thread_signal_destroy();
return false;
}
#endif
return true;
}
static void
runtime_signal_destroy()
{
#ifdef BH_PLATFORM_WINDOWS
RemoveVectoredExceptionHandler(aot_exception_handler);
#endif
os_thread_signal_destroy();
}
void
wasm_runtime_set_exec_env_tls(WASMExecEnv *exec_env)
{
exec_env_tls = exec_env;
}
WASMExecEnv *
wasm_runtime_get_exec_env_tls()
{
return exec_env_tls;
}
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
static bool
wasm_runtime_env_init()
{
@ -149,12 +239,13 @@ wasm_runtime_env_init()
}
#endif
#if WASM_ENABLE_AOT != 0
#ifdef OS_ENABLE_HW_BOUND_CHECK
if (!aot_signal_init()) {
if (!runtime_signal_init()) {
goto fail6;
}
#endif
#if WASM_ENABLE_AOT != 0
#if WASM_ENABLE_DEBUG_AOT != 0
if (!jit_debug_engine_init()) {
goto fail7;
@ -178,10 +269,10 @@ fail8:
jit_debug_engine_destroy();
fail7:
#endif
#ifdef OS_ENABLE_HW_BOUND_CHECK
aot_signal_destroy();
fail6:
#endif
#ifdef OS_ENABLE_HW_BOUND_CHECK
runtime_signal_destroy();
fail6:
#endif
#if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_THREAD_MGR != 0)
thread_manager_destroy();
@ -238,9 +329,10 @@ wasm_runtime_destroy()
#if WASM_ENABLE_DEBUG_AOT != 0
jit_debug_engine_destroy();
#endif
#ifdef OS_ENABLE_HW_BOUND_CHECK
aot_signal_destroy();
#endif
#ifdef OS_ENABLE_HW_BOUND_CHECK
runtime_signal_destroy();
#endif
/* runtime env destroy */
@ -953,26 +1045,23 @@ wasm_runtime_init_thread_env(void)
return false;
#endif
#if WASM_ENABLE_AOT != 0
#ifdef OS_ENABLE_HW_BOUND_CHECK
if (!aot_signal_init()) {
if (!runtime_signal_init()) {
#ifdef BH_PLATFORM_WINDOWS
os_thread_env_destroy();
#endif
return false;
}
#endif
#endif
return true;
}
void
wasm_runtime_destroy_thread_env(void)
{
#if WASM_ENABLE_AOT != 0
#ifdef OS_ENABLE_HW_BOUND_CHECK
aot_signal_destroy();
#endif
runtime_signal_destroy();
#endif
#ifdef BH_PLATFORM_WINDOWS

View File

@ -407,6 +407,26 @@ typedef struct wasm_frame_t {
const char *func_name_wp;
} WASMCApiFrame;
#ifdef OS_ENABLE_HW_BOUND_CHECK
/* Signal info passing to interp/aot signal handler */
typedef struct WASMSignalInfo {
WASMExecEnv *exec_env_tls;
#ifndef BH_PLATFORM_WINDOWS
void *sig_addr;
#else
EXCEPTION_POINTERS *exce_info;
#endif
} WASMSignalInfo;
/* Set exec_env of thread local storage */
void
wasm_runtime_set_exec_env_tls(WASMExecEnv *exec_env);
/* Get exec_env of thread local storage */
WASMExecEnv *
wasm_runtime_get_exec_env_tls(void);
#endif
/* See wasm_export.h for description */
WASM_RUNTIME_API_EXTERN bool
wasm_runtime_init(void);

View File

@ -24,6 +24,7 @@ typedef float64 CellType_F64;
#define BR_TABLE_TMP_BUF_LEN 32
#ifndef OS_ENABLE_HW_BOUND_CHECK
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
@ -45,6 +46,18 @@ typedef float64 CellType_F64;
else \
goto out_of_bounds; \
} while (0)
#else
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
maddr = memory->memory_data + offset1; \
} while (0)
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \
maddr = memory->memory_data + (uint32)(start); \
} while (0)
#endif
#define CHECK_ATOMIC_MEMORY_ACCESS() \
do { \
@ -1006,10 +1019,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
WASMInterpFrame *prev_frame)
{
WASMMemoryInstance *memory = module->default_memory;
uint32 num_bytes_per_page = memory ? memory->num_bytes_per_page : 0;
uint8 *global_data = module->global_data;
#ifndef OS_ENABLE_HW_BOUND_CHECK
uint32 num_bytes_per_page = memory ? memory->num_bytes_per_page : 0;
uint32 linear_mem_size =
memory ? num_bytes_per_page * memory->cur_page_count : 0;
#endif
WASMType **wasm_types = module->module->types;
WASMGlobalInstance *globals = module->globals, *global;
uint8 opcode_IMPDEP = WASM_OP_IMPDEP;
@ -1959,8 +1974,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
PUSH_I32(prev_page_count);
/* update memory instance ptr and memory size */
memory = module->default_memory;
#ifndef OS_ENABLE_HW_BOUND_CHECK
linear_mem_size =
num_bytes_per_page * memory->cur_page_count;
#endif
}
(void)reserved;
@ -2988,11 +3005,16 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
seg_len = (uint64)module->module->data_segments[segment]
->data_length;
data = module->module->data_segments[segment]->data;
#ifndef OS_ENABLE_HW_BOUND_CHECK
if (offset + bytes > seg_len)
goto out_of_bounds;
bh_memcpy_s(maddr, linear_mem_size - addr,
data + offset, (uint32)bytes);
#else
bh_memcpy_s(maddr, (uint32)bytes, data + offset,
(uint32)bytes);
#endif
break;
}
case WASM_OP_DATA_DROP:
@ -3019,8 +3041,11 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst);
/* allowing the destination and source to overlap */
#ifndef OS_ENABLE_HW_BOUND_CHECK
bh_memmove_s(mdst, linear_mem_size - dst, msrc, len);
#else
bh_memmove_s(mdst, len, msrc, len);
#endif
break;
}
case WASM_OP_MEMORY_FILL:
@ -3681,8 +3706,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
/* update memory instance ptr and memory size */
memory = module->default_memory;
#ifndef OS_ENABLE_HW_BOUND_CHECK
if (memory)
linear_mem_size = num_bytes_per_page * memory->cur_page_count;
#endif
if (wasm_get_exception(module))
goto got_exception;
}
@ -3759,8 +3786,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
goto got_exception;
#endif
#ifndef OS_ENABLE_HW_BOUND_CHECK
out_of_bounds:
wasm_set_exception(module, "out of bounds memory access");
#endif
got_exception:
#if WASM_ENABLE_DEBUG_INTERP != 0
@ -3808,11 +3837,13 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
}
argc = function->param_cell_num;
#ifndef OS_ENABLE_HW_BOUND_CHECK
if ((uint8 *)&prev_frame < exec_env->native_stack_boundary) {
wasm_set_exception((WASMModuleInstance *)exec_env->module_inst,
"native stack overflow");
return;
}
#endif
if (!(frame =
ALLOC_FRAME(exec_env, frame_size, (WASMInterpFrame *)prev_frame)))

View File

@ -18,6 +18,7 @@ typedef int64 CellType_I64;
typedef float32 CellType_F32;
typedef float64 CellType_F64;
#ifndef OS_ENABLE_HW_BOUND_CHECK
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
@ -39,6 +40,18 @@ typedef float64 CellType_F64;
else \
goto out_of_bounds; \
} while (0)
#else
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
maddr = memory->memory_data + offset1; \
} while (0)
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \
maddr = memory->memory_data + (uint32)(start); \
} while (0)
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
#define CHECK_ATOMIC_MEMORY_ACCESS(align) \
do { \
@ -1080,10 +1093,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
WASMInterpFrame *prev_frame)
{
WASMMemoryInstance *memory = module->default_memory;
#ifndef OS_ENABLE_HW_BOUND_CHECK
uint32 num_bytes_per_page = memory ? memory->num_bytes_per_page : 0;
uint8 *global_data = module->global_data;
uint32 linear_mem_size =
memory ? num_bytes_per_page * memory->cur_page_count : 0;
#endif
uint8 *global_data = module->global_data;
WASMGlobalInstance *globals = module->globals, *global;
uint8 opcode_IMPDEP = WASM_OP_IMPDEP;
WASMInterpFrame *frame = NULL;
@ -1797,8 +1812,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
frame_lp[addr_ret] = prev_page_count;
/* update memory instance ptr and memory size */
memory = module->default_memory;
#ifndef OS_ENABLE_HW_BOUND_CHECK
linear_mem_size =
num_bytes_per_page * memory->cur_page_count;
#endif
}
(void)reserved;
@ -2907,11 +2924,16 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
seg_len = (uint64)module->module->data_segments[segment]
->data_length;
data = module->module->data_segments[segment]->data;
#ifndef OS_ENABLE_HW_BOUND_CHECK
if (offset + bytes > seg_len)
goto out_of_bounds;
bh_memcpy_s(maddr, linear_mem_size - addr,
data + offset, (uint32)bytes);
#else
bh_memcpy_s(maddr, (uint32)bytes, data + offset,
(uint32)bytes);
#endif
break;
}
case WASM_OP_DATA_DROP:
@ -2937,8 +2959,11 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst);
/* allowing the destination and source to overlap */
#ifndef OS_ENABLE_HW_BOUND_CHECK
bh_memmove_s(mdst, linear_mem_size - dst, msrc, len);
#else
bh_memmove_s(mdst, len, msrc, len);
#endif
break;
}
case WASM_OP_MEMORY_FILL:
@ -3694,8 +3719,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
/* update memory instance ptr and memory size */
memory = module->default_memory;
#ifndef OS_ENABLE_HW_BOUND_CHECK
if (memory)
linear_mem_size = num_bytes_per_page * memory->cur_page_count;
#endif
if (wasm_get_exception(module))
goto got_exception;
}
@ -3761,8 +3788,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
goto got_exception;
#endif
#ifndef OS_ENABLE_HW_BOUND_CHECK
out_of_bounds:
wasm_set_exception(module, "out of bounds memory access");
#endif
got_exception:
SYNC_ALL_TO_FRAME();
@ -3813,11 +3842,13 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
}
argc = function->param_cell_num;
#ifndef OS_ENABLE_HW_BOUND_CHECK
if ((uint8 *)&prev_frame < exec_env->native_stack_boundary) {
wasm_set_exception((WASMModuleInstance *)exec_env->module_inst,
"native stack overflow");
return;
}
#endif
if (!(frame =
ALLOC_FRAME(exec_env, frame_size, (WASMInterpFrame *)prev_frame)))

View File

@ -130,8 +130,19 @@ memories_deinstantiate(WASMModuleInstance *module_inst,
wasm_runtime_free(memories[i]->heap_handle);
memories[i]->heap_handle = NULL;
}
if (memories[i]->memory_data)
if (memories[i]->memory_data) {
#ifndef OS_ENABLE_HW_BOUND_CHECK
wasm_runtime_free(memories[i]->memory_data);
#else
#ifdef BH_PLATFORM_WINDOWS
os_mem_decommit(memories[i]->memory_data,
memories[i]->num_bytes_per_page
* memories[i]->cur_page_count);
#endif
os_munmap((uint8 *)memories[i]->memory_data,
8 * (uint64)BH_GB);
#endif
}
wasm_runtime_free(memories[i]);
}
}
@ -153,6 +164,11 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page,
uint32 inc_page_count, aux_heap_base, global_idx;
uint32 bytes_of_last_page, bytes_to_page_end;
uint8 *global_addr;
#ifdef OS_ENABLE_HW_BOUND_CHECK
uint8 *mapped_mem;
uint64 map_size = 8 * (uint64)BH_GB;
uint64 page_size = os_getpagesize();
#endif
#if WASM_ENABLE_SHARED_MEMORY != 0
bool is_shared_memory = flags & 0x02 ? true : false;
@ -268,11 +284,45 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page,
return NULL;
}
#ifndef OS_ENABLE_HW_BOUND_CHECK
if (memory_data_size > 0
&& !(memory->memory_data =
runtime_malloc(memory_data_size, error_buf, error_buf_size))) {
goto fail1;
}
#else
memory_data_size = (memory_data_size + page_size - 1) & ~(page_size - 1);
/* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
* ea = i + memarg.offset
* both i and memarg.offset are u32 in range 0 to 4G
* so the range of ea is 0 to 8G
*/
if (memory_data_size >= UINT32_MAX
|| !(memory->memory_data = mapped_mem =
os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE))) {
set_error_buf(error_buf, error_buf_size, "mmap memory failed");
goto fail1;
}
#ifdef BH_PLATFORM_WINDOWS
if (!os_mem_commit(mapped_mem, memory_data_size,
MMAP_PROT_READ | MMAP_PROT_WRITE)) {
set_error_buf(error_buf, error_buf_size, "commit memory failed");
os_munmap(mapped_mem, map_size);
goto fail1;
}
#endif
if (os_mprotect(mapped_mem, memory_data_size,
MMAP_PROT_READ | MMAP_PROT_WRITE)
!= 0) {
set_error_buf(error_buf, error_buf_size, "mprotect memory failed");
goto fail2;
}
/* Newly allocated pages are filled with zero by the OS, we don't fill it
* again here */
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
memory->module_type = Wasm_Module_Bytecode;
memory->num_bytes_per_page = num_bytes_per_page;
@ -327,8 +377,15 @@ fail3:
if (heap_size > 0)
wasm_runtime_free(memory->heap_handle);
fail2:
#ifndef OS_ENABLE_HW_BOUND_CHECK
if (memory->memory_data)
wasm_runtime_free(memory->memory_data);
#else
#ifdef BH_PLATFORM_WINDOWS
os_mem_decommit(mapped_mem, memory_data_size);
#endif
os_munmap(mapped_mem, map_size);
#endif
fail1:
wasm_runtime_free(memory);
return NULL;
@ -949,6 +1006,9 @@ execute_malloc_function(WASMModuleInstance *module_inst,
WASMFunctionInstance *retain_func, uint32 size,
uint32 *p_result)
{
#ifdef OS_ENABLE_HW_BOUND_CHECK
WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
#endif
uint32 argv[2], argc;
bool ret;
@ -967,12 +1027,26 @@ execute_malloc_function(WASMModuleInstance *module_inst,
argc = 2;
}
ret = wasm_create_exec_env_and_call_function(module_inst, malloc_func, argc,
argv, false);
#ifdef OS_ENABLE_HW_BOUND_CHECK
if (exec_env_tls != NULL) {
bh_assert(exec_env_tls->module_inst
== (WASMModuleInstanceCommon *)module_inst);
ret = wasm_call_function(exec_env_tls, malloc_func, argc, argv);
if (retain_func && ret) {
ret = wasm_create_exec_env_and_call_function(module_inst, retain_func,
1, argv, false);
if (retain_func && ret) {
ret = wasm_call_function(exec_env_tls, retain_func, 1, argv);
}
}
else
#endif
{
ret = wasm_create_exec_env_and_call_function(module_inst, malloc_func,
argc, argv, false);
if (retain_func && ret) {
ret = wasm_create_exec_env_and_call_function(
module_inst, retain_func, 1, argv, false);
}
}
if (ret)
@ -984,11 +1058,24 @@ static bool
execute_free_function(WASMModuleInstance *module_inst,
WASMFunctionInstance *free_func, uint32 offset)
{
#ifdef OS_ENABLE_HW_BOUND_CHECK
WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
#endif
uint32 argv[2];
argv[0] = offset;
return wasm_create_exec_env_and_call_function(module_inst, free_func, 1,
argv, false);
#ifdef OS_ENABLE_HW_BOUND_CHECK
if (exec_env_tls != NULL) {
bh_assert(exec_env_tls->module_inst
== (WASMModuleInstanceCommon *)module_inst);
return wasm_call_function(exec_env_tls, free_func, 1, argv);
}
else
#endif
{
return wasm_create_exec_env_and_call_function(module_inst, free_func, 1,
argv, false);
}
}
#if WASM_ENABLE_MULTI_MODULE != 0
@ -1704,6 +1791,195 @@ clear_wasi_proc_exit_exception(WASMModuleInstance *module_inst)
#endif
}
#ifdef OS_ENABLE_HW_BOUND_CHECK
#ifndef BH_PLATFORM_WINDOWS
void
wasm_signal_handler(WASMSignalInfo *sig_info)
{
WASMExecEnv *exec_env_tls = sig_info->exec_env_tls;
void *sig_addr = sig_info->sig_addr;
WASMModuleInstance *module_inst;
WASMMemoryInstance *memory_inst;
WASMJmpBuf *jmpbuf_node;
uint8 *mapped_mem_start_addr = NULL;
uint8 *mapped_mem_end_addr = NULL;
uint8 *stack_min_addr;
uint32 page_size;
uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT;
/* Check whether current thread is running wasm function */
if (exec_env_tls && exec_env_tls->handle == os_self_thread()
&& (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) {
/* Get mapped mem info of current instance */
module_inst = (WASMModuleInstance *)exec_env_tls->module_inst;
/* Get the default memory instance */
memory_inst = module_inst->default_memory;
if (memory_inst) {
mapped_mem_start_addr = (uint8 *)memory_inst->memory_data;
mapped_mem_end_addr =
(uint8 *)memory_inst->memory_data + 8 * (uint64)BH_GB;
}
/* Get stack info of current thread */
page_size = os_getpagesize();
stack_min_addr = os_thread_get_stack_boundary();
if (memory_inst
&& (mapped_mem_start_addr <= (uint8 *)sig_addr
&& (uint8 *)sig_addr < mapped_mem_end_addr)) {
/* The address which causes segmentation fault is inside
the memory instance's guard regions */
wasm_set_exception(module_inst, "out of bounds memory access");
os_longjmp(jmpbuf_node->jmpbuf, 1);
}
else if (stack_min_addr - page_size <= (uint8 *)sig_addr
&& (uint8 *)sig_addr
< stack_min_addr + page_size * guard_page_count) {
/* The address which causes segmentation fault is inside
native thread's guard page */
wasm_set_exception(module_inst, "native stack overflow");
os_longjmp(jmpbuf_node->jmpbuf, 1);
}
}
}
#else /* else of BH_PLATFORM_WINDOWS */
LONG
wasm_exception_handler(WASMSignalInfo *sig_info)
{
WASMExecEnv *exec_env_tls = sig_info->exec_env_tls;
EXCEPTION_POINTERS *exce_info = sig_info->exce_info;
PEXCEPTION_RECORD ExceptionRecord = exce_info->ExceptionRecord;
uint8 *sig_addr = (uint8 *)ExceptionRecord->ExceptionInformation[1];
WASMModuleInstance *module_inst;
WASMMemoryInstance *memory_inst;
WASMJmpBuf *jmpbuf_node;
uint8 *mapped_mem_start_addr = NULL;
uint8 *mapped_mem_end_addr = NULL;
uint32 page_size = os_getpagesize();
if (exec_env_tls && exec_env_tls->handle == os_self_thread()
&& (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) {
module_inst = (WASMModuleInstance *)exec_env_tls->module_inst;
if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
/* Get the default memory instance */
memory_inst = module_inst->default_memory;
if (memory_inst) {
mapped_mem_start_addr = (uint8 *)memory_inst->memory_data;
mapped_mem_end_addr =
(uint8 *)memory_inst->memory_data + 8 * (uint64)BH_GB;
if (mapped_mem_start_addr <= (uint8 *)sig_addr
&& (uint8 *)sig_addr < mapped_mem_end_addr) {
/* The address which causes segmentation fault is inside
the memory instance's guard regions.
Set exception and let the wasm func continue to run, when
the wasm func returns, the caller will check whether the
exception is thrown and return to runtime. */
wasm_set_exception(module_inst,
"out of bounds memory access");
/* Skip current instruction */
return EXCEPTION_CONTINUE_SEARCH;
}
}
}
else if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) {
/* Set stack overflow exception and let the wasm func continue
to run, when the wasm func returns, the caller will check
whether the exception is thrown and return to runtime, and
the damaged stack will be recovered by _resetstkoflw(). */
wasm_set_exception(module_inst, "native stack overflow");
return EXCEPTION_CONTINUE_SEARCH;
}
}
os_printf("Unhandled exception thrown: exception code: 0x%lx, "
"exception address: %p, exception information: %p\n",
ExceptionRecord->ExceptionCode, ExceptionRecord->ExceptionAddress,
sig_addr);
return EXCEPTION_CONTINUE_SEARCH;
}
#endif /* end of BH_PLATFORM_WINDOWS */
static void
call_wasm_with_hw_bound_check(WASMModuleInstance *module_inst,
WASMExecEnv *exec_env,
WASMFunctionInstance *function, unsigned argc,
uint32 argv[])
{
WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop;
uint32 page_size = os_getpagesize();
uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT;
#ifdef BH_PLATFORM_WINDOWS
const char *exce;
int result;
#endif
bool ret = true;
/* Check native stack overflow firstly to ensure we have enough
native stack to run the following codes before actually calling
the aot function in invokeNative function. */
if ((uint8 *)&exec_env_tls < exec_env->native_stack_boundary
+ page_size * (guard_page_count + 1)) {
wasm_set_exception(module_inst, "native stack overflow");
return;
}
if (exec_env_tls && (exec_env_tls != exec_env)) {
wasm_set_exception(module_inst, "invalid exec env");
return;
}
if (!os_thread_signal_inited()) {
wasm_set_exception(module_inst, "thread signal env not inited");
return;
}
wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node);
wasm_runtime_set_exec_env_tls(exec_env);
if (os_setjmp(jmpbuf_node.jmpbuf) == 0) {
#ifndef BH_PLATFORM_WINDOWS
wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv);
#else
__try {
wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv);
} __except (wasm_get_exception(module_inst)
? EXCEPTION_EXECUTE_HANDLER
: EXCEPTION_CONTINUE_SEARCH) {
/* exception was thrown in wasm_exception_handler */
ret = false;
}
if ((exce = wasm_get_exception(module_inst))
&& strstr(exce, "native stack overflow")) {
/* After a stack overflow, the stack was left
in a damaged state, let the CRT repair it */
result = _resetstkoflw();
bh_assert(result != 0);
}
#endif
}
else {
/* Exception has been set in signal handler before calling longjmp */
ret = false;
}
jmpbuf_node_pop = wasm_exec_env_pop_jmpbuf(exec_env);
bh_assert(&jmpbuf_node == jmpbuf_node_pop);
if (!exec_env->jmpbuf_stack_top) {
wasm_runtime_set_exec_env_tls(NULL);
}
if (!ret) {
os_sigreturn();
os_signal_unmask();
}
(void)jmpbuf_node_pop;
}
#define interp_call_wasm call_wasm_with_hw_bound_check
#else
#define interp_call_wasm wasm_interp_call_wasm
#endif
bool
wasm_call_function(WASMExecEnv *exec_env, WASMFunctionInstance *function,
unsigned argc, uint32 argv[])
@ -1714,7 +1990,7 @@ wasm_call_function(WASMExecEnv *exec_env, WASMFunctionInstance *function,
/* set thread handle and stack boundary */
wasm_exec_env_set_thread_info(exec_env);
wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv);
interp_call_wasm(module_inst, exec_env, function, argc, argv);
(void)clear_wasi_proc_exit_exception(module_inst);
return !wasm_get_exception(module_inst) ? true : false;
}
@ -1725,15 +2001,17 @@ wasm_create_exec_env_and_call_function(WASMModuleInstance *module_inst,
unsigned argc, uint32 argv[],
bool enable_debug)
{
WASMExecEnv *exec_env;
WASMExecEnv *exec_env, *existing_exec_env = NULL;
bool ret;
#if WASM_ENABLE_THREAD_MGR != 0
WASMExecEnv *existing_exec_env = NULL;
if (!(existing_exec_env = exec_env = wasm_clusters_search_exec_env(
(WASMModuleInstanceCommon *)module_inst))) {
#if defined(OS_ENABLE_HW_BOUND_CHECK)
existing_exec_env = exec_env = wasm_runtime_get_exec_env_tls();
#elif WASM_ENABLE_THREAD_MGR != 0
existing_exec_env = exec_env =
wasm_clusters_search_exec_env((WASMModuleInstanceCommon *)module_inst);
#endif
if (!existing_exec_env) {
if (!(exec_env =
wasm_exec_env_create((WASMModuleInstanceCommon *)module_inst,
module_inst->default_wasm_stack_size))) {
@ -1742,20 +2020,18 @@ wasm_create_exec_env_and_call_function(WASMModuleInstance *module_inst,
}
#if WASM_ENABLE_THREAD_MGR != 0
if (enable_debug) {
#if WASM_ENABLE_DEBUG_INTERP != 0
if (enable_debug) {
wasm_runtime_start_debug_instance(exec_env);
#endif
}
}
#endif
#endif
}
ret = wasm_call_function(exec_env, func, argc, argv);
#if WASM_ENABLE_THREAD_MGR != 0
/* don't destroy the exec_env if it's searched from the cluster */
/* don't destroy the exec_env if it isn't created in this function */
if (!existing_exec_env)
#endif
wasm_exec_env_destroy(exec_env);
return ret;
@ -2111,6 +2387,7 @@ wasm_get_native_addr_range(WASMModuleInstance *module_inst, uint8 *native_ptr,
return false;
}
#ifndef OS_ENABLE_HW_BOUND_CHECK
bool
wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count)
{
@ -2195,6 +2472,57 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count)
return ret;
}
#else
bool
wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count)
{
WASMMemoryInstance *memory = module->default_memory;
uint32 num_bytes_per_page, total_page_count;
if (!memory)
return false;
total_page_count = inc_page_count + memory->cur_page_count;
if (inc_page_count <= 0)
/* No need to enlarge memory */
return true;
if (total_page_count < memory->cur_page_count /* integer overflow */
|| total_page_count > memory->max_page_count) {
return false;
}
num_bytes_per_page = memory->num_bytes_per_page;
#ifdef BH_PLATFORM_WINDOWS
if (!os_mem_commit(memory->memory_data_end,
num_bytes_per_page * inc_page_count,
MMAP_PROT_READ | MMAP_PROT_WRITE)) {
return false;
}
#endif
if (os_mprotect(memory->memory_data_end,
num_bytes_per_page * inc_page_count,
MMAP_PROT_READ | MMAP_PROT_WRITE)
!= 0) {
#ifdef BH_PLATFORM_WINDOWS
os_mem_decommit(memory->memory_data_end,
num_bytes_per_page * inc_page_count);
#endif
return false;
}
/* The increased pages are filled with zero by the OS when os_mmap,
no need to memset it again here */
memory->cur_page_count = total_page_count;
memory->memory_data_end =
memory->memory_data + num_bytes_per_page * total_page_count;
return true;
}
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
#if WASM_ENABLE_REF_TYPES != 0
bool
@ -2279,7 +2607,7 @@ wasm_call_indirect(WASMExecEnv *exec_env, uint32_t tbl_idx,
function_inst = module_inst->functions + function_indices;
wasm_interp_call_wasm(module_inst, exec_env, function_inst, argc, argv);
interp_call_wasm(module_inst, exec_env, function_inst, argc, argv);
(void)clear_wasi_proc_exit_exception(module_inst);
return !wasm_get_exception(module_inst) ? true : false;
@ -2415,9 +2743,6 @@ wasm_get_module_mem_consumption(const WASMModule *module,
mem_conspn->total_size += mem_conspn->table_segs_size;
mem_conspn->total_size += mem_conspn->data_segs_size;
mem_conspn->total_size += mem_conspn->const_strs_size;
#if WASM_ENABLE_AOT != 0
mem_conspn->total_size += mem_conspn->aot_code_size;
#endif
}
void

View File

@ -385,6 +385,16 @@ bool
wasm_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size);
#endif
#ifdef OS_ENABLE_HW_BOUND_CHECK
#ifndef BH_PLATFORM_WINDOWS
void
wasm_signal_handler(WASMSignalInfo *sig_info);
#else
LONG
wasm_exception_handler(WASMSignalInfo *sig_info);
#endif
#endif
void
wasm_get_module_mem_consumption(const WASMModule *module,
WASMModuleMemConsumption *mem_conspn);