diff --git a/core/config.h b/core/config.h index 878a42280..7f0c01789 100644 --- a/core/config.h +++ b/core/config.h @@ -267,6 +267,9 @@ stack overflow exception if the guard boudary is reached */ #define RESERVED_BYTES_TO_NATIVE_STACK_BOUNDARY (512) +/* Guard page count for stack overflow check with hardware trap */ +#define STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT 3 + /* Default wasm block address cache size and conflict list size */ #ifndef BLOCK_ADDR_CACHE_SIZE #define BLOCK_ADDR_CACHE_SIZE 64 diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 139c0d034..0657804e6 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1110,17 +1110,8 @@ aot_lookup_function(const AOTModuleInstance *module_inst, #ifdef OS_ENABLE_HW_BOUND_CHECK -#define STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT 3 - static os_thread_local_attribute WASMExecEnv *aot_exec_env = NULL; -static inline uint8 * -get_stack_min_addr(WASMExecEnv *exec_env, uint32 page_size) -{ - uintptr_t stack_bound = (uintptr_t)exec_env->native_stack_boundary; - return (uint8*)(stack_bound & ~(uintptr_t)(page_size -1 )); -} - static void aot_signal_handler(void *sig_addr) { @@ -1149,7 +1140,7 @@ aot_signal_handler(void *sig_addr) /* Get stack info of current thread */ page_size = os_getpagesize(); - stack_min_addr = get_stack_min_addr(aot_exec_env, page_size); + stack_min_addr = os_thread_get_stack_boundary(); if (memory_inst && (mapped_mem_start_addr <= (uint8*)sig_addr @@ -1182,27 +1173,6 @@ aot_signal_destroy() os_signal_destroy(); } -#if defined(__GNUC__) -__attribute__((no_sanitize_address)) static uint32 -#else -static uint32 -#endif -touch_pages(uint8 *stack_min_addr, uint32 page_size) -{ - uint8 sum = 0; - while (1) { - volatile uint8 *touch_addr = - (volatile uint8*)os_alloca(page_size / 2); - if (touch_addr < stack_min_addr + page_size) { - sum += *(stack_min_addr + page_size - 1); - break; - } - *touch_addr = 0; - sum += *touch_addr; - } - return sum; -} - static bool invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, const WASMType *func_type, const char *signature, @@ -1211,10 +1181,9 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, { AOTModuleInstance *module_inst = (AOTModuleInstance*)exec_env->module_inst; WASMExecEnv **p_aot_exec_env = &aot_exec_env; - WASMJmpBuf *jmpbuf_node, *jmpbuf_node_pop; + WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop; uint32 page_size = os_getpagesize(); uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; - uint8 *stack_min_addr = get_stack_min_addr(exec_env, page_size); bool ret; /* Check native stack overflow firstly to ensure we have enough @@ -1226,33 +1195,17 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, return false; } - if (aot_exec_env - && (aot_exec_env != exec_env)) { + if (aot_exec_env && (aot_exec_env != exec_env)) { aot_set_exception(module_inst, "invalid exec env"); return false; } - if (!exec_env->jmpbuf_stack_top) { - /* Touch each stack page to ensure that it has been mapped: the OS may - lazily grow the stack mapping as a guard page is hit. */ - (void)touch_pages(stack_min_addr, page_size); - /* First time to call aot function, protect one page */ - if (os_mprotect(stack_min_addr, page_size * guard_page_count, - MMAP_PROT_NONE) != 0) { - aot_set_exception(module_inst, "set protected page failed"); - return false; - } - } + os_thread_init_stack_guard_pages(); - if (!(jmpbuf_node = wasm_runtime_malloc(sizeof(WASMJmpBuf)))) { - aot_set_exception_with_id(module_inst, EXCE_OUT_OF_MEMORY); - return false; - } - - wasm_exec_env_push_jmpbuf(exec_env, jmpbuf_node); + wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node); aot_exec_env = exec_env; - if (os_setjmp(jmpbuf_node->jmpbuf) == 0) { + if (os_setjmp(jmpbuf_node.jmpbuf) == 0) { ret = wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature, attachment, argv, argc, argv_ret); @@ -1263,12 +1216,8 @@ invoke_native_with_hw_bound_check(WASMExecEnv *exec_env, void *func_ptr, } jmpbuf_node_pop = wasm_exec_env_pop_jmpbuf(exec_env); - bh_assert(jmpbuf_node == jmpbuf_node_pop); - wasm_runtime_free(jmpbuf_node); + bh_assert(&jmpbuf_node == jmpbuf_node_pop); if (!exec_env->jmpbuf_stack_top) { - /* Unprotect the guard page when the nested call depth is zero */ - os_mprotect(stack_min_addr, page_size * guard_page_count, - MMAP_PROT_READ | MMAP_PROT_WRITE); *p_aot_exec_env = NULL; } os_sigreturn(); @@ -1293,6 +1242,9 @@ aot_call_function(WASMExecEnv *exec_env, uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; bool ret; + /* set thread handle and stack boundary */ + wasm_exec_env_set_thread_info(exec_env); + if (ext_ret_count > 0) { uint32 cell_num = 0, i; uint8 *ext_ret_types = func_type->types + func_type->param_count + 1; @@ -1375,7 +1327,7 @@ aot_call_function(WASMExecEnv *exec_env, case VALUE_TYPE_V128: argv_ret += 4; break; -#endif - +#endif default: bh_assert(0); break; @@ -1436,8 +1388,6 @@ aot_create_exec_env_and_call_function(AOTModuleInstance *module_inst, return false; } - /* set thread handle and stack boundary */ - wasm_exec_env_set_thread_info(exec_env); #if WASM_ENABLE_THREAD_MGR != 0 } #endif @@ -2265,7 +2215,7 @@ aot_call_indirect(WASMExecEnv *exec_env, case VALUE_TYPE_V128: argv_ret += 4; break; -#endif - +#endif default: bh_assert(0); break; diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 82f8a0f8c..8e122906c 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -1122,9 +1122,6 @@ wasm_runtime_call_wasm(WASMExecEnv *exec_env, return false; } - /* set thread handle and stack boundary */ - wasm_exec_env_set_thread_info(exec_env); - #if WASM_ENABLE_REF_TYPES != 0 wasm_runtime_prepare_call_function(exec_env, function); #endif diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index aa4176fc9..08c2bd241 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -1647,6 +1647,10 @@ wasm_call_function(WASMExecEnv *exec_env, unsigned argc, uint32 argv[]) { WASMModuleInstance *module_inst = (WASMModuleInstance*)exec_env->module_inst; + + /* set thread handle and stack boundary */ + wasm_exec_env_set_thread_info(exec_env); + wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv); (void)clear_wasi_proc_exit_exception(module_inst); return !wasm_get_exception(module_inst) ? true : false; @@ -1674,8 +1678,6 @@ wasm_create_exec_env_and_call_function(WASMModuleInstance *module_inst, return false; } - /* set thread handle and stack boundary */ - wasm_exec_env_set_thread_info(exec_env); #if WASM_ENABLE_THREAD_MGR != 0 } #endif diff --git a/core/shared/platform/android/platform_init.c b/core/shared/platform/android/platform_init.c index 5353308c1..04b1e1545 100644 --- a/core/shared/platform/android/platform_init.c +++ b/core/shared/platform/android/platform_init.c @@ -20,6 +20,9 @@ bh_platform_init() void bh_platform_destroy() { +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_thread_destroy_stack_guard_pages(); +#endif } int os_printf(const char *fmt, ...) diff --git a/core/shared/platform/android/platform_internal.h b/core/shared/platform/android/platform_internal.h index b8b99f16a..73cbffac3 100644 --- a/core/shared/platform/android/platform_internal.h +++ b/core/shared/platform/android/platform_internal.h @@ -56,6 +56,8 @@ typedef pthread_mutex_t korp_mutex; typedef pthread_cond_t korp_cond; typedef pthread_t korp_thread; +#define os_thread_local_attribute __thread + #if WASM_DISABLE_HW_BOUND_CHECK == 0 #if defined(BUILD_TARGET_X86_64) \ || defined(BUILD_TARGET_AMD_64) \ @@ -65,8 +67,6 @@ typedef pthread_t korp_thread; #define OS_ENABLE_HW_BOUND_CHECK -#define os_thread_local_attribute __thread - typedef jmp_buf korp_jmpbuf; #define os_setjmp setjmp @@ -75,6 +75,10 @@ typedef jmp_buf korp_jmpbuf; #define os_getpagesize getpagesize +bool os_thread_init_stack_guard_pages(); + +void os_thread_destroy_stack_guard_pages(); + typedef void (*os_signal_handler)(void *sig_addr); int os_signal_init(os_signal_handler handler); diff --git a/core/shared/platform/common/posix/posix_thread.c b/core/shared/platform/common/posix/posix_thread.c index 3580eca74..54b52c839 100644 --- a/core/shared/platform/common/posix/posix_thread.c +++ b/core/shared/platform/common/posix/posix_thread.c @@ -25,6 +25,9 @@ static void *os_thread_wrapper(void *arg) targ->stack = (void *)((uintptr_t)(&arg) & (uintptr_t)~0xfff); BH_FREE(targ); start_func(thread_arg); +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_thread_destroy_stack_guard_pages(); +#endif return NULL; } @@ -239,22 +242,32 @@ int os_thread_detach(korp_tid thread) void os_thread_exit(void *retval) { +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_thread_destroy_stack_guard_pages(); +#endif return pthread_exit(retval); } +static os_thread_local_attribute uint8 *thread_stack_boundary = NULL; + uint8 *os_thread_get_stack_boundary() { - pthread_t self = pthread_self(); + pthread_t self; #ifdef __linux__ pthread_attr_t attr; size_t guard_size; #endif uint8 *addr = NULL; - size_t stack_size; - int page_size = getpagesize(); - size_t max_stack_size = (size_t) - (APP_THREAD_STACK_SIZE_MAX + page_size - 1) - & ~(page_size - 1); + size_t stack_size, max_stack_size; + int page_size; + + if (thread_stack_boundary) + return thread_stack_boundary; + + page_size = getpagesize(); + self = pthread_self(); + max_stack_size = (size_t)(APP_THREAD_STACK_SIZE_MAX + page_size - 1) + & ~(page_size - 1); if (max_stack_size < APP_THREAD_STACK_SIZE_DEFAULT) max_stack_size = APP_THREAD_STACK_SIZE_DEFAULT; @@ -284,6 +297,7 @@ uint8 *os_thread_get_stack_boundary() } #endif + thread_stack_boundary = addr; return addr; } @@ -291,12 +305,71 @@ uint8 *os_thread_get_stack_boundary() #define SIG_ALT_STACK_SIZE (32 * 1024) +/* Whether the stack pages are touched and guard pages are set */ +static os_thread_local_attribute bool stack_guard_pages_inited = false; + /* The signal alternate stack base addr */ static uint8 *sigalt_stack_base_addr; /* The signal handler passed to os_signal_init() */ static os_signal_handler signal_handler; +#if defined(__GNUC__) +__attribute__((no_sanitize_address)) static uint32 +#else +static uint32 +#endif +touch_pages(uint8 *stack_min_addr, uint32 page_size) +{ + uint8 sum = 0; + while (1) { + volatile uint8 *touch_addr = + (volatile uint8*)os_alloca(page_size / 2); + if (touch_addr < stack_min_addr + page_size) { + sum += *(stack_min_addr + page_size - 1); + break; + } + *touch_addr = 0; + sum += *touch_addr; + } + return sum; +} + +bool +os_thread_init_stack_guard_pages() +{ + uint32 page_size = os_getpagesize(); + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; + uint8 *stack_min_addr = os_thread_get_stack_boundary(); + + if (!stack_guard_pages_inited) { + /* Touch each stack page to ensure that it has been mapped: the OS + may lazily grow the stack mapping as a guard page is hit. */ + (void)touch_pages(stack_min_addr, page_size); + /* First time to call aot function, protect guard pages */ + if (os_mprotect(stack_min_addr, page_size * guard_page_count, + MMAP_PROT_NONE) != 0) { + return false; + } + stack_guard_pages_inited = true; + } + return true; +} + +void +os_thread_destroy_stack_guard_pages() +{ + if (stack_guard_pages_inited) { + uint32 page_size = os_getpagesize(); + uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT; + uint8 *stack_min_addr = os_thread_get_stack_boundary(); + + os_mprotect(stack_min_addr, page_size * guard_page_count, + MMAP_PROT_READ | MMAP_PROT_WRITE); + stack_guard_pages_inited = false; + } +} + static void mask_signals(int how) { diff --git a/core/shared/platform/darwin/platform_init.c b/core/shared/platform/darwin/platform_init.c index 17aeb8baa..23cd00604 100644 --- a/core/shared/platform/darwin/platform_init.c +++ b/core/shared/platform/darwin/platform_init.c @@ -14,6 +14,9 @@ bh_platform_init() void bh_platform_destroy() { +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_thread_destroy_stack_guard_pages(); +#endif } int diff --git a/core/shared/platform/darwin/platform_internal.h b/core/shared/platform/darwin/platform_internal.h index 75348df15..66596b07f 100644 --- a/core/shared/platform/darwin/platform_internal.h +++ b/core/shared/platform/darwin/platform_internal.h @@ -57,6 +57,8 @@ typedef pthread_mutex_t korp_mutex; typedef pthread_cond_t korp_cond; typedef pthread_t korp_thread; +#define os_thread_local_attribute __thread + #if WASM_DISABLE_HW_BOUND_CHECK == 0 #if defined(BUILD_TARGET_X86_64) \ || defined(BUILD_TARGET_AMD_64) \ @@ -66,8 +68,6 @@ typedef pthread_t korp_thread; #define OS_ENABLE_HW_BOUND_CHECK -#define os_thread_local_attribute __thread - typedef jmp_buf korp_jmpbuf; #define os_setjmp setjmp @@ -76,6 +76,10 @@ typedef jmp_buf korp_jmpbuf; #define os_getpagesize getpagesize +bool os_thread_init_stack_guard_pages(); + +void os_thread_destroy_stack_guard_pages(); + typedef void (*os_signal_handler)(void *sig_addr); int os_signal_init(os_signal_handler handler); diff --git a/core/shared/platform/linux/platform_init.c b/core/shared/platform/linux/platform_init.c index 17aeb8baa..23cd00604 100644 --- a/core/shared/platform/linux/platform_init.c +++ b/core/shared/platform/linux/platform_init.c @@ -14,6 +14,9 @@ bh_platform_init() void bh_platform_destroy() { +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_thread_destroy_stack_guard_pages(); +#endif } int diff --git a/core/shared/platform/linux/platform_internal.h b/core/shared/platform/linux/platform_internal.h index 7a15f85e6..0154bdf5d 100644 --- a/core/shared/platform/linux/platform_internal.h +++ b/core/shared/platform/linux/platform_internal.h @@ -56,6 +56,8 @@ typedef pthread_mutex_t korp_mutex; typedef pthread_cond_t korp_cond; typedef pthread_t korp_thread; +#define os_thread_local_attribute __thread + #if WASM_DISABLE_HW_BOUND_CHECK == 0 #if defined(BUILD_TARGET_X86_64) \ || defined(BUILD_TARGET_AMD_64) \ @@ -65,8 +67,6 @@ typedef pthread_t korp_thread; #define OS_ENABLE_HW_BOUND_CHECK -#define os_thread_local_attribute __thread - typedef jmp_buf korp_jmpbuf; #define os_setjmp setjmp @@ -75,6 +75,10 @@ typedef jmp_buf korp_jmpbuf; #define os_getpagesize getpagesize +bool os_thread_init_stack_guard_pages(); + +void os_thread_destroy_stack_guard_pages(); + typedef void (*os_signal_handler)(void *sig_addr); int os_signal_init(os_signal_handler handler); diff --git a/core/shared/platform/vxworks/platform_init.c b/core/shared/platform/vxworks/platform_init.c index 17aeb8baa..23cd00604 100644 --- a/core/shared/platform/vxworks/platform_init.c +++ b/core/shared/platform/vxworks/platform_init.c @@ -14,6 +14,9 @@ bh_platform_init() void bh_platform_destroy() { +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_thread_destroy_stack_guard_pages(); +#endif } int diff --git a/core/shared/platform/vxworks/platform_internal.h b/core/shared/platform/vxworks/platform_internal.h index a8479361e..254ad68a6 100644 --- a/core/shared/platform/vxworks/platform_internal.h +++ b/core/shared/platform/vxworks/platform_internal.h @@ -55,6 +55,8 @@ typedef pthread_mutex_t korp_mutex; typedef pthread_cond_t korp_cond; typedef pthread_t korp_thread; +#define os_thread_local_attribute __thread + #if WASM_DISABLE_HW_BOUND_CHECK == 0 #if defined(BUILD_TARGET_X86_64) \ || defined(BUILD_TARGET_AMD_64) \ @@ -64,8 +66,6 @@ typedef pthread_t korp_thread; #define OS_ENABLE_HW_BOUND_CHECK -#define os_thread_local_attribute __thread - typedef jmp_buf korp_jmpbuf; #define os_setjmp setjmp @@ -74,6 +74,10 @@ typedef jmp_buf korp_jmpbuf; #define os_getpagesize getpagesize +bool os_thread_init_stack_guard_pages(); + +void os_thread_destroy_stack_guard_pages(); + typedef void (*os_signal_handler)(void *sig_addr); int os_signal_init(os_signal_handler handler); diff --git a/core/shared/platform/zephyr/zephyr_time.c b/core/shared/platform/zephyr/zephyr_time.c index b6c03f586..6a9db0806 100644 --- a/core/shared/platform/zephyr/zephyr_time.c +++ b/core/shared/platform/zephyr/zephyr_time.c @@ -8,6 +8,6 @@ uint64 os_time_get_boot_microsecond() { - return k_uptime_get_32() * 1000; + return k_uptime_get() * 1000; }