diff --git a/README.md b/README.md index df10d7179..1163f49ad 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ iwasm VM core - [Sign-extension operators](https://github.com/WebAssembly/sign-extension-ops) - [Bulk memory operations](https://github.com/WebAssembly/bulk-memory-operations) - [Shared memmory](https://github.com/WebAssembly/threads/blob/main/proposals/threads/Overview.md#shared-linear-memory) +- [Multi-value](https://github.com/WebAssembly/multi-value) ### Performance and memory usage The WAMR performance, footprint and memory usage data are available at the [performance](../../wiki/Performance) wiki page. diff --git a/core/app-mgr/app-manager/module_wasm_app.c b/core/app-mgr/app-manager/module_wasm_app.c index 95ac95a78..4d30c4a08 100644 --- a/core/app-mgr/app-manager/module_wasm_app.c +++ b/core/app-mgr/app-manager/module_wasm_app.c @@ -556,7 +556,7 @@ wasm_app_module_init(void) static bool wasm_app_module_install(request_t * msg) { - unsigned int m_data_size, heap_size; + unsigned int m_data_size, heap_size, stack_size; unsigned int timeout, timers, err_size; char *properties; int properties_offset; @@ -842,9 +842,13 @@ wasm_app_module_install(request_t * msg) goto fail; } + stack_size = APP_THREAD_STACK_SIZE_DEFAULT; +#ifdef OS_ENABLE_HW_BOUND_CHECK + stack_size += 4 * BH_KB; +#endif /* Create WASM app thread. */ if (os_thread_create(&wasm_app_data->thread_id, wasm_app_routine, - (void*) m_data, APP_THREAD_STACK_SIZE_DEFAULT) != 0) { + (void*) m_data, stack_size) != 0) { module_data_list_remove(m_data); SEND_ERR_RESPONSE(msg->mid, "Install WASM app failed: create app thread failed."); diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 8803213e7..af0d8fdb5 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -551,11 +551,19 @@ load_func_types(const uint8 **p_buf, const uint8 *buf_end, /* Create each function type */ for (i = 0; i < module->func_type_count; i++) { uint32 param_count, result_count; + uint32 param_cell_num, ret_cell_num; uint64 size1; read_uint32(buf, buf_end, param_count); read_uint32(buf, buf_end, result_count); + if (param_count > UINT16_MAX || result_count > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: " + "param count or result count too large"); + return false; + } + size1 = (uint64)param_count + (uint64)result_count; size = offsetof(AOTFuncType, types) + size1; if (!(func_types[i] = loader_malloc @@ -563,9 +571,22 @@ load_func_types(const uint8 **p_buf, const uint8 *buf_end, return false; } - func_types[i]->param_count = param_count; - func_types[i]->result_count = result_count; + func_types[i]->param_count = (uint16)param_count; + func_types[i]->result_count = (uint16)result_count; read_byte_array(buf, buf_end, func_types[i]->types, (uint32)size1); + + param_cell_num = wasm_get_cell_num(func_types[i]->types, param_count); + ret_cell_num = wasm_get_cell_num(func_types[i]->types + param_count, + result_count); + if (param_cell_num > UINT16_MAX || ret_cell_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "AOT module load failed: " + "param count or result count too large"); + return false; + } + + func_types[i]->param_cell_num = (uint16)param_cell_num; + func_types[i]->ret_cell_num = (uint16)ret_cell_num; } *p_buf = buf; diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 0e0b574d0..7805b1f9e 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -722,9 +722,77 @@ aot_call_function(WASMExecEnv *exec_env, { AOTModuleInstance *module_inst = (AOTModuleInstance*)exec_env->module_inst; AOTFuncType *func_type = function->func_type; - bool ret = invoke_native_internal(exec_env, function->func_ptr, - func_type, NULL, NULL, argv, argc, argv); - return ret && !aot_get_exception(module_inst) ? true : false; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; + bool ret; + + if (ext_ret_count > 0) { + uint32 cell_num = 0, i; + uint8 *ext_ret_types = func_type->types + func_type->param_count + 1; + uint32 argv1_buf[32], *argv1 = argv1_buf, *ext_rets = NULL; + uint32 *argv_ret = argv; + uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count); + uint64 size; + + /* Allocate memory all arguments */ + size = sizeof(uint32) * (uint64)argc /* original arguments */ + + sizeof(void*) * (uint64)ext_ret_count /* extra result values' addr */ + + sizeof(uint32) * (uint64)ext_ret_cell; /* extra result values */ + if (size > sizeof(argv1_buf) + && !(argv1 = runtime_malloc(size, module_inst->cur_exception, + sizeof(module_inst->cur_exception)))) { + aot_set_exception_with_id(module_inst, EXCE_OUT_OF_MEMORY); + return false; + } + + /* Copy original arguments */ + bh_memcpy_s(argv1, (uint32)size, argv, sizeof(uint32) * argc); + + /* Get the extra result value's address */ + ext_rets = argv1 + argc + sizeof(void*)/sizeof(uint32) * ext_ret_count; + + /* Append each extra result value's address to original arguments */ + for (i = 0; i < ext_ret_count; i++) { + *(uintptr_t*)(argv1 + argc + sizeof(void*) / sizeof(uint32) * i) = + (uintptr_t)(ext_rets + cell_num); + cell_num += wasm_value_type_cell_num(ext_ret_types[i]); + } + + ret = invoke_native_internal(exec_env, function->func_ptr, + func_type, NULL, NULL, argv1, argc, argv); + if (!ret || aot_get_exception(module_inst)) { + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + return false; + } + + /* Get extra result values */ + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + argv_ret++; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + argv_ret += 2; + break; + default: + bh_assert(0); + break; + } + ext_rets = argv1 + argc + sizeof(void*)/sizeof(uint32) * ext_ret_count; + bh_memcpy_s(argv_ret, sizeof(uint32) * cell_num, + ext_rets, sizeof(uint32) * cell_num); + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + + return true; + } + else { + ret = invoke_native_internal(exec_env, function->func_ptr, + func_type, NULL, NULL, argv, argc, argv); + return ret && !aot_get_exception(module_inst) ? true : false; + } } bool @@ -1183,10 +1251,12 @@ aot_call_indirect(WASMExecEnv *exec_env, void **func_ptrs = (void**)module_inst->func_ptrs.ptr, *func_ptr; uint32 table_size = module_inst->table_size; uint32 func_idx, func_type_idx1; + uint32 ext_ret_count; AOTImportFunc *import_func; const char *signature = NULL; void *attachment = NULL; char buf[128]; + bool ret; /* this function is called from native code, so exec_env->handle and exec_env->native_stack_boundary must have been set, we don't set @@ -1241,9 +1311,77 @@ aot_call_indirect(WASMExecEnv *exec_env, } } - return invoke_native_internal(exec_env, func_ptr, - func_type, signature, attachment, - argv, argc, argv); + ext_ret_count = func_type->result_count > 1 + ? func_type->result_count - 1 : 0; + if (ext_ret_count > 0) { + uint32 argv1_buf[32], *argv1 = argv1_buf; + uint32 *ext_rets = NULL, *argv_ret = argv; + uint32 cell_num = 0, i; + uint8 *ext_ret_types = func_type->types + func_type->param_count + 1; + uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count); + uint64 size; + + /* Allocate memory all arguments */ + size = sizeof(uint32) * (uint64)argc /* original arguments */ + + sizeof(void*) * (uint64)ext_ret_count /* extra result values' addr */ + + sizeof(uint32) * (uint64)ext_ret_cell; /* extra result values */ + if (size > sizeof(argv1_buf) + && !(argv1 = runtime_malloc(size, module_inst->cur_exception, + sizeof(module_inst->cur_exception)))) { + aot_set_exception_with_id(module_inst, EXCE_OUT_OF_MEMORY); + return false; + } + + /* Copy original arguments */ + bh_memcpy_s(argv1, (uint32)size, argv, sizeof(uint32) * argc); + + /* Get the extra result value's address */ + ext_rets = argv1 + argc + sizeof(void*)/sizeof(uint32) * ext_ret_count; + + /* Append each extra result value's address to original arguments */ + for (i = 0; i < ext_ret_count; i++) { + *(uintptr_t*)(argv1 + argc + sizeof(void*) / sizeof(uint32) * i) = + (uintptr_t)(ext_rets + cell_num); + cell_num += wasm_value_type_cell_num(ext_ret_types[i]); + } + + ret = invoke_native_internal(exec_env, func_ptr, + func_type, signature, attachment, + argv1, argc, argv); + if (!ret || aot_get_exception(module_inst)) { + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + return false; + } + + /* Get extra result values */ + switch (func_type->types[func_type->param_count]) { + case VALUE_TYPE_I32: + case VALUE_TYPE_F32: + argv_ret++; + break; + case VALUE_TYPE_I64: + case VALUE_TYPE_F64: + argv_ret += 2; + break; + default: + bh_assert(0); + break; + } + ext_rets = argv1 + argc + sizeof(void*)/sizeof(uint32) * ext_ret_count; + bh_memcpy_s(argv_ret, sizeof(uint32) * cell_num, + ext_rets, sizeof(uint32) * cell_num); + + if (argv1 != argv1_buf) + wasm_runtime_free(argv1); + + return true; + } + else { + return invoke_native_internal(exec_env, func_ptr, + func_type, signature, attachment, + argv, argc, argv); + } } #if WASM_ENABLE_BULK_MEMORY != 0 diff --git a/core/iwasm/common/arch/invokeNative_aarch64.s b/core/iwasm/common/arch/invokeNative_aarch64.s index 7204a958e..68390cb54 100644 --- a/core/iwasm/common/arch/invokeNative_aarch64.s +++ b/core/iwasm/common/arch/invokeNative_aarch64.s @@ -44,7 +44,7 @@ invokeNative: cmp x21, #0 beq call_func - /* Fill all stack args: reserve stack space and fill ony by one */ + /* Fill all stack args: reserve stack space and fill one by one */ mov x23, sp bic sp, x23, #15 /* Ensure stack is 16 bytes aligned */ lsl x23, x21, #3 /* x23 = nstacks * 8 */ diff --git a/core/iwasm/common/arch/invokeNative_arm_vfp.s b/core/iwasm/common/arch/invokeNative_arm_vfp.s index 0020f28a5..679fdedf1 100644 --- a/core/iwasm/common/arch/invokeNative_arm_vfp.s +++ b/core/iwasm/common/arch/invokeNative_arm_vfp.s @@ -52,7 +52,7 @@ invokeNative: beq call_func - /* Fill all stack args: reserve stack space and fill ony by one */ + /* Fill all stack args: reserve stack space and fill one by one */ add r4, r4, #64 /* r4 points to stack args */ bic sp, sp, #7 /* Ensure stack is 8 byte aligned */ mov r7, r5, lsl#2 /* r7 = nstacks * 4 */ diff --git a/core/iwasm/common/wasm_native.c b/core/iwasm/common/wasm_native.c index 3dd38c389..fcf52de08 100644 --- a/core/iwasm/common/wasm_native.c +++ b/core/iwasm/common/wasm_native.c @@ -54,7 +54,7 @@ check_symbol_signature(const WASMType *type, const char *signature) if (*p++ != '(') return false; - if ((uint32)(p_end - p) < type->param_count + 1) + if ((uint32)(p_end - p) < (uint32)(type->param_count + 1)) /* signatures of parameters, and ')' */ return false; diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index fa1700df2..66d79b888 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -1916,7 +1916,7 @@ wasm_application_execute_func(WASMModuleInstanceCommon *module_inst, { WASMFunctionInstanceCommon *func; WASMType *type = NULL; - uint32 argc1, *argv1 = NULL; + uint32 argc1, *argv1 = NULL, cell_num, j, k = 0; int32 i, p; uint64 total_size; const char *exception; @@ -1946,12 +1946,16 @@ wasm_application_execute_func(WASMModuleInstanceCommon *module_inst, } type = wasm_func->u.func->func_type; argc1 = wasm_func->param_cell_num; + cell_num = argc1 > wasm_func->ret_cell_num ? + argc1 : wasm_func->ret_cell_num; } #endif #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { type = ((AOTFunctionInstance*)func)->func_type; - argc1 = wasm_type_param_cell_num(type); + argc1 = type->param_cell_num; + cell_num = argc1 > type->ret_cell_num ? + argc1 : type->ret_cell_num; } #endif @@ -1961,7 +1965,7 @@ wasm_application_execute_func(WASMModuleInstanceCommon *module_inst, goto fail; } - total_size = sizeof(uint32) * (uint64)(argc1 > 2 ? argc1 : 2); + total_size = sizeof(uint32) * (uint64)(cell_num > 2 ? cell_num : 2); if ((!(argv1 = runtime_malloc((uint32)total_size, module_inst, NULL, 0)))) { goto fail; @@ -2076,16 +2080,18 @@ wasm_application_execute_func(WASMModuleInstanceCommon *module_inst, } /* print return value */ - if (type->result_count > 0) { - switch (type->types[type->param_count]) { + for (j = 0; j < type->result_count; j++) { + switch (type->types[type->param_count + j]) { case VALUE_TYPE_I32: - os_printf("0x%x:i32", argv1[0]); + os_printf("0x%x:i32", argv1[k]); + k++; break; case VALUE_TYPE_I64: { union { uint64 val; uint32 parts[2]; } u; - u.parts[0] = argv1[0]; - u.parts[1] = argv1[1]; + u.parts[0] = argv1[k]; + u.parts[1] = argv1[k + 1]; + k += 2; #ifdef PRIx64 os_printf("0x%"PRIx64":i64", u.val); #else @@ -2099,17 +2105,21 @@ wasm_application_execute_func(WASMModuleInstanceCommon *module_inst, break; } case VALUE_TYPE_F32: - os_printf("%.7g:f32", *(float32*)argv1); + os_printf("%.7g:f32", *(float32*)(argv1 + k)); + k++; break; case VALUE_TYPE_F64: { union { float64 val; uint32 parts[2]; } u; - u.parts[0] = argv1[0]; - u.parts[1] = argv1[1]; + u.parts[0] = argv1[k]; + u.parts[1] = argv1[k + 1]; + k += 2; os_printf("%.7g:f64", u.val); break; } } + if (j < (uint32)(type->result_count - 1)) + os_printf(","); } os_printf("\n"); @@ -2302,6 +2312,8 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, uint32 argv_buf[32], *argv1 = argv_buf, *fps, *ints, *stacks, size; uint32 *argv_src = argv, i, argc1, n_ints = 0, n_fps = 0, n_stacks = 0; uint32 arg_i32, ptr_len; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; bool ret = false; n_ints++; /* exec env */ @@ -2355,6 +2367,13 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, } } + for (i = 0; i < ext_ret_count; i++) { + if (n_ints < MAX_REG_INTS) + n_ints++; + else + n_stacks++; + } + argc1 = MAX_REG_INTS + MAX_REG_FLOATS + n_stacks; if (argc1 > sizeof(argv_buf) / sizeof(uint32)) { size = sizeof(uint32) * (uint32)argc1; @@ -2458,6 +2477,14 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, } } + /* Save extra result values' address to argv1 */ + for (i = 0; i < ext_ret_count; i++) { + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = *(uint32*)argv_src++; + else + stacks[n_stacks++] = *(uint32*)argv_src++; + } + exec_env->attachment = attachment; if (func_type->result_count == 0) { invokeNative_Void(func_ptr, argv1, n_stacks); @@ -2521,15 +2548,17 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); uint32 argv_buf[32], *argv1 = argv_buf, argc1, i, j = 0; uint32 arg_i32, ptr_len; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; uint64 size; bool ret = false; #if defined(BUILD_TARGET_X86_32) - argc1 = argc + 2; + argc1 = argc + ext_ret_count + 2; #else /* arm/thumb/mips/xtensa, 64-bit data must be 8 bytes aligned, so we need to allocate more memory. */ - argc1 = func_type->param_count * 2 + 2; + argc1 = func_type->param_count * 2 + ext_ret_count + 2; #endif if (argc1 > sizeof(argv_buf) / sizeof(uint32)) { @@ -2598,7 +2627,10 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, } } - argc1 = j; + /* Save extra result values' address to argv1 */ + word_copy(argv1 + j, argv, ext_ret_count); + + argc1 = j + ext_ret_count; exec_env->attachment = attachment; if (func_type->result_count == 0) { invokeNative_Void(func_ptr, argv1, argc1); @@ -2678,7 +2710,10 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, uint64 argv_buf[32], *argv1 = argv_buf, *fps, *ints, *stacks, size, arg_i64; uint32 *argv_src = argv, i, argc1, n_ints = 0, n_stacks = 0; uint32 arg_i32, ptr_len; + uint32 result_count = func_type->result_count; + uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0; bool ret = false; + #if defined(_WIN32) || defined(_WIN32_) /* important difference in calling conventions */ #define n_fps n_ints @@ -2686,7 +2721,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, int n_fps = 0; #endif - argc1 = 1 + MAX_REG_FLOATS + func_type->param_count + 2; + argc1 = 1 + MAX_REG_FLOATS + func_type->param_count + ext_ret_count; if (argc1 > sizeof(argv_buf) / sizeof(uint64)) { size = sizeof(uint64) * (uint64)argc1; if (!(argv1 = runtime_malloc((uint32)size, exec_env->module_inst, @@ -2764,11 +2799,21 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, } } + /* Save extra result values' address to argv1 */ + for (i = 0; i < ext_ret_count; i++) { + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = *(uint64*)argv_src; + else + stacks[n_stacks++] = *(uint64*)argv_src; + argv_src += 2; + } + exec_env->attachment = attachment; - if (func_type->result_count == 0) { + if (result_count == 0) { invokeNative_Void(func_ptr, argv1, n_stacks); } else { + /* Invoke the native function and get the first result value */ switch (func_type->types[func_type->param_count]) { case VALUE_TYPE_I32: argv_ret[0] = (uint32)invokeNative_Int32(func_ptr, argv1, n_stacks); diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index 617bf6e65..f0f2539c6 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -92,14 +92,21 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) AOTFuncContext *func_ctx = comp_ctx->func_ctxes[func_index]; uint8 *frame_ip = func_ctx->aot_func->code, opcode, *p_f32, *p_f64; uint8 *frame_ip_end = frame_ip + func_ctx->aot_func->code_size; - uint32 block_ret_type, br_depth, *br_depths, br_count; + uint8 *param_types = NULL; + uint8 *result_types = NULL; + uint8 value_type; + uint16 param_count; + uint16 result_count; + uint32 br_depth, *br_depths, br_count; uint32 func_idx, type_idx, mem_idx, local_idx, global_idx, i; uint32 bytes = 4, align, offset; + uint32 type_index; bool sign = true; int32 i32_const; int64 i64_const; float32 f32_const; float64 f64_const; + AOTFuncType *func_type = NULL; /* Start to translate the opcodes */ LLVMPositionBuilderAtEnd(comp_ctx->builder, @@ -119,11 +126,37 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) case WASM_OP_BLOCK: case WASM_OP_LOOP: case WASM_OP_IF: - read_leb_uint32(frame_ip, frame_ip_end, block_ret_type); + value_type = *frame_ip++; + if (value_type == VALUE_TYPE_I32 + || value_type == VALUE_TYPE_I64 + || value_type == VALUE_TYPE_F32 + || value_type == VALUE_TYPE_F64 + || value_type == VALUE_TYPE_VOID) { + param_count = 0; + param_types = NULL; + if (value_type == VALUE_TYPE_VOID) { + result_count = 0; + result_types = NULL; + } + else { + result_count = 1; + result_types = &value_type; + } + } + else { + frame_ip--; + read_leb_uint32(frame_ip, frame_ip_end, type_index); + func_type = comp_ctx->comp_data->func_types[type_index]; + param_count = func_type->param_count; + param_types = func_type->types; + result_count = func_type->result_count; + result_types = func_type->types + param_count; + } if (!aot_compile_op_block(comp_ctx, func_ctx, &frame_ip, frame_ip_end, - (uint32)(BLOCK_TYPE_BLOCK + opcode - WASM_OP_BLOCK), - block_ret_type)) + (uint32)(LABEL_TYPE_BLOCK + opcode - WASM_OP_BLOCK), + param_count, param_types, + result_count, result_types)) return false; break; diff --git a/core/iwasm/compilation/aot_emit_control.c b/core/iwasm/compilation/aot_emit_control.c index bbb470a2c..b6d38bf20 100644 --- a/core/iwasm/compilation/aot_emit_control.c +++ b/core/iwasm/compilation/aot_emit_control.c @@ -19,13 +19,13 @@ enum { static void format_block_name(char *name, uint32 name_size, - uint32 block_index, uint32 block_type, - uint32 label_type) + uint32 block_index, uint32 label_type, + uint32 label_id) { - if (block_type != BLOCK_TYPE_FUNCTION) + if (label_type != LABEL_TYPE_FUNCTION) snprintf(name, name_size, "%s%d%s%s", - block_name_prefix[block_type], block_index, - "_", block_name_suffix[label_type]); + block_name_prefix[label_type], block_index, + "_", block_name_suffix[label_id]); else snprintf(name, name_size, "%s", "func_end"); } @@ -69,28 +69,44 @@ format_block_name(char *name, uint32 name_size, #define SET_BUILDER_POS(llvm_block) \ LLVMPositionBuilderAtEnd(comp_ctx->builder, llvm_block) -#define CREATE_RETURN_VALUE_PHI(block) do { \ - if (block->return_type != VALUE_TYPE_VOID \ - && !block->return_value_phi) { \ - LLVMBasicBlockRef block_curr = CURR_BLOCK(); \ - SET_BUILDER_POS(block->llvm_end_block); \ - if (!(block->return_value_phi = \ - LLVMBuildPhi(comp_ctx->builder, \ - TO_LLVM_TYPE(block->return_type),\ - "phi"))) { \ - aot_set_last_error("llvm build phi failed."); \ - goto fail; \ - } \ - SET_BUILDER_POS(block_curr); \ - } \ +#define CREATE_RESULT_VALUE_PHIS(block) do { \ + if (block->result_count && !block->result_phis) { \ + uint32 i; \ + uint64 size; \ + LLVMBasicBlockRef block_curr = CURR_BLOCK(); \ + /* Allocate memory */ \ + size = sizeof(LLVMValueRef) * (uint64)block->result_count; \ + if (size >= UINT32_MAX \ + || !(block->result_phis = \ + wasm_runtime_malloc((uint32)size))) { \ + aot_set_last_error("allocate memory failed."); \ + goto fail; \ + } \ + SET_BUILDER_POS(block->llvm_end_block); \ + for (i = 0; i < block->result_count; i++) { \ + if (!(block->result_phis[i] = \ + LLVMBuildPhi(comp_ctx->builder, \ + TO_LLVM_TYPE(block->result_types[i]), \ + "phi"))) { \ + aot_set_last_error("llvm build phi failed."); \ + goto fail; \ + } \ + } \ + SET_BUILDER_POS(block_curr); \ + } \ } while (0) -#define ADD_TO_RETURN_PHI(block, value) do { \ - LLVMBasicBlockRef block_curr = CURR_BLOCK(); \ - LLVMAddIncoming(block->return_value_phi, \ - &value, &block_curr, 1); \ +#define ADD_TO_RESULT_PHIS(block, value, idx) do { \ + LLVMBasicBlockRef block_curr = CURR_BLOCK(); \ + LLVMAddIncoming(block->result_phis[idx], \ + &value, &block_curr, 1); \ } while (0) +#define ADD_TO_PARAM_PHIS(block, value, idx) do { \ + LLVMBasicBlockRef block_curr = CURR_BLOCK(); \ + LLVMAddIncoming(block->param_phis[idx], \ + &value, &block_curr, 1); \ + } while (0) static LLVMBasicBlockRef find_next_llvm_end_block(AOTBlock *block) @@ -126,15 +142,20 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTBlock *block = func_ctx->block_stack.block_list_end; AOTBlock *block_prev; uint8 *frame_ip; + uint32 i; + AOTFuncType *func_type; aot_checked_addr_list_destroy(func_ctx); - if (block->block_type == BLOCK_TYPE_IF + if (block->label_type == LABEL_TYPE_IF && block->llvm_else_block && !block->skip_wasm_code_else && *p_frame_ip <= block->wasm_code_else) { /* Clear value stack and start to translate else branch */ aot_value_stack_destroy(&block->value_stack); + /* Recover parameters of else branch */ + for (i = 0; i < block->param_count; i++) + PUSH(block->else_param_phis[i], block->param_types[i]); SET_BUILDER_POS(block->llvm_else_block); *p_frame_ip = block->wasm_code_else + 1; return true; @@ -144,7 +165,7 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, block_prev = block->prev; block = aot_block_stack_pop(&func_ctx->block_stack); - if (block->block_type == BLOCK_TYPE_IF) { + if (block->label_type == LABEL_TYPE_IF) { if (block->llvm_else_block && !block->skip_wasm_code_else && *p_frame_ip <= block->wasm_code_else) { @@ -178,15 +199,39 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, /* Pop block, push its return value, and destroy the block */ block = aot_block_stack_pop(&func_ctx->block_stack); - if (block->return_type != VALUE_TYPE_VOID) { - bh_assert(block->return_value_phi); - if (block->block_type != BLOCK_TYPE_FUNCTION) - PUSH(block->return_value_phi, block->return_type); - else - LLVMBuildRet(comp_ctx->builder, block->return_value_phi); + func_type = func_ctx->aot_func->func_type; + for (i = 0; i < block->result_count; i++) { + bh_assert(block->result_phis[i]); + if (block->label_type != LABEL_TYPE_FUNCTION) { + PUSH(block->result_phis[i], block->result_types[i]); + } + else { + /* Store extra return values to function parameters */ + if (i != 0) { + uint32 param_index = func_type->param_count + i; + if (!LLVMBuildStore(comp_ctx->builder, + block->result_phis[i], + LLVMGetParam(func_ctx->func, param_index))) { + aot_set_last_error("llvm build store failed."); + goto fail; + } + } + } } - else if (block->block_type == BLOCK_TYPE_FUNCTION) { - LLVMBuildRetVoid(comp_ctx->builder); + if (block->label_type == LABEL_TYPE_FUNCTION) { + if (block->result_count) { + /* Return the first return value */ + if (!LLVMBuildRet(comp_ctx->builder, block->result_phis[0])) { + aot_set_last_error("llvm build return failed."); + goto fail; + } + } + else { + if (!LLVMBuildRetVoid(comp_ctx->builder)) { + aot_set_last_error("llvm build return void failed."); + goto fail; + } + } } aot_block_destroy(block); return true; @@ -194,10 +239,116 @@ fail: return false; } +static bool +push_aot_block_to_stack_and_pass_params(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + AOTBlock *block) +{ + uint32 i, param_index; + LLVMValueRef value; + uint64 size; + char name[32]; + LLVMBasicBlockRef block_curr = CURR_BLOCK(); + + if (block->param_count) { + size = sizeof(LLVMValueRef) * (uint64)block->param_count; + if (size >= UINT32_MAX + || !(block->param_phis = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return false; + } + + if (block->label_type == LABEL_TYPE_IF + && !block->skip_wasm_code_else + && !(block->else_param_phis = wasm_runtime_malloc((uint32)size))) { + wasm_runtime_free(block->param_phis); + block->param_phis = NULL; + aot_set_last_error("allocate memory failed."); + return false; + } + + /* Create param phis */ + for (i = 0; i < block->param_count; i++) { + SET_BUILDER_POS(block->llvm_entry_block); + snprintf(name, sizeof(name), "%s%d_phi%d", + block_name_prefix[block->label_type], + block->block_index, i); + if (!(block->param_phis[i] = + LLVMBuildPhi(comp_ctx->builder, + TO_LLVM_TYPE(block->param_types[i]), + name))) { + aot_set_last_error("llvm build phi failed."); + goto fail; + } + + if (block->label_type == LABEL_TYPE_IF + && !block->skip_wasm_code_else + && block->llvm_else_block) { + /* Build else param phis */ + SET_BUILDER_POS(block->llvm_else_block); + snprintf(name, sizeof(name), "else%d_phi%d", + block->block_index, i); + if (!(block->else_param_phis[i] = + LLVMBuildPhi(comp_ctx->builder, + TO_LLVM_TYPE(block->param_types[i]), + name))) { + aot_set_last_error("llvm build phi failed."); + goto fail; + } + } + } + SET_BUILDER_POS(block_curr); + + /* Pop param values from current block's + * value stack and add to param phis. + */ + for (i = 0; i < block->param_count; i++) { + param_index = block->param_count - 1 - i; + POP(value, block->param_types[param_index]); + ADD_TO_PARAM_PHIS(block, value, param_index); + if (block->label_type == LABEL_TYPE_IF + && !block->skip_wasm_code_else) { + if (block->llvm_else_block) { + /* has else branch, add to else param phis */ + LLVMAddIncoming(block->else_param_phis[param_index], + &value, &block_curr, 1); + } + else { + /* no else branch, add to result phis */ + CREATE_RESULT_VALUE_PHIS(block); + ADD_TO_RESULT_PHIS(block, value, param_index); + } + } + } + } + + /* Push the new block to block stack */ + aot_block_stack_push(&func_ctx->block_stack, block); + + /* Push param phis to the new block */ + for (i = 0; i < block->param_count; i++) { + PUSH(block->param_phis[i], block->param_types[i]); + } + + return true; + +fail: + if (block->param_phis) { + wasm_runtime_free(block->param_phis); + block->param_phis = NULL; + } + if (block->else_param_phis) { + wasm_runtime_free(block->else_param_phis); + block->else_param_phis = NULL; + } + return false; +} + bool aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint8 **p_frame_ip, uint8 *frame_ip_end, - uint32 block_type, uint32 block_ret_type) + uint32 label_type, uint32 param_count, uint8 *param_types, + uint32 result_count, uint8 *result_types) { BlockAddr block_addr_cache[BLOCK_ADDR_CACHE_SIZE][BLOCK_ADDR_CONFLICT_SIZE]; AOTBlock *block; @@ -215,7 +366,7 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Get block info */ if (!(wasm_loader_find_block_addr((BlockAddr*)block_addr_cache, - *p_frame_ip, frame_ip_end, (uint8)block_type, + *p_frame_ip, frame_ip_end, (uint8)label_type, &else_addr, &end_addr, NULL, 0))) { aot_set_last_error("find block end addr failed."); return false; @@ -226,51 +377,66 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("allocate memory failed."); return false; } + memset(block, 0, sizeof(AOTBlock)); + if (param_count + && !(block->param_types = wasm_runtime_malloc(param_count))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + if (result_count) { + if (!(block->result_types = wasm_runtime_malloc(result_count))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + } /* Init aot block data */ - memset(block, 0, sizeof(AOTBlock)); - block->block_type = block_type; - block->return_type = (uint8)block_ret_type; + block->label_type = label_type; + block->param_count = param_count; + memcpy(block->param_types, param_types, param_count); + block->result_count = result_count; + memcpy(block->result_types, result_types, result_count); block->wasm_code_else = else_addr; block->wasm_code_end = end_addr; - block->block_index = func_ctx->block_stack.block_index[block_type]; - func_ctx->block_stack.block_index[block_type]++; + block->block_index = func_ctx->block_stack.block_index[label_type]; + func_ctx->block_stack.block_index[label_type]++; - if (block_type == BLOCK_TYPE_BLOCK - || block_type == BLOCK_TYPE_LOOP) { + if (label_type == LABEL_TYPE_BLOCK + || label_type == LABEL_TYPE_LOOP) { /* Create block */ format_block_name(name, sizeof(name), - block->block_index, block_type, LABEL_BEGIN); + block->block_index, label_type, LABEL_BEGIN); CREATE_BLOCK(block->llvm_entry_block, name); MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block); /* Jump to the entry block */ BUILD_BR(block->llvm_entry_block); + if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, block)) + goto fail; /* Start to translate the block */ SET_BUILDER_POS(block->llvm_entry_block); - aot_block_stack_push(&func_ctx->block_stack, block); - if (block_type == BLOCK_TYPE_LOOP) + if (label_type == LABEL_TYPE_LOOP) aot_checked_addr_list_destroy(func_ctx); } - else if (block_type == BLOCK_TYPE_IF) { + else if (label_type == LABEL_TYPE_IF) { POP_COND(value); if (!LLVMIsConstant(value)) { /* Compare value is not constant, create condition br IR */ /* Create entry block */ format_block_name(name, sizeof(name), - block->block_index, block_type, LABEL_BEGIN); + block->block_index, label_type, LABEL_BEGIN); CREATE_BLOCK(block->llvm_entry_block, name); MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block); /* Create end block */ format_block_name(name, sizeof(name), - block->block_index, block_type, LABEL_END); + block->block_index, label_type, LABEL_END); CREATE_BLOCK(block->llvm_end_block, name); MOVE_BLOCK_AFTER(block->llvm_end_block, block->llvm_entry_block); if (else_addr) { /* Create else block */ format_block_name(name, sizeof(name), - block->block_index, block_type, LABEL_ELSE); + block->block_index, label_type, LABEL_ELSE); CREATE_BLOCK(block->llvm_else_block, name); MOVE_BLOCK_AFTER(block->llvm_else_block, block->llvm_entry_block); /* Create condition br IR */ @@ -283,49 +449,48 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, block->llvm_end_block); block->is_reachable = true; } + if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, block)) + goto fail; /* Start to translate if branch of BLOCK if */ SET_BUILDER_POS(block->llvm_entry_block); - aot_block_stack_push(&func_ctx->block_stack, block); } else { if ((int32)LLVMConstIntGetZExtValue(value) != 0) { - /* Compare value is not 0, condtion is true, else branch of + /* Compare value is not 0, condition is true, else branch of BLOCK if cannot be reached */ block->skip_wasm_code_else = true; /* Create entry block */ format_block_name(name, sizeof(name), - block->block_index, block_type, LABEL_BEGIN); + block->block_index, label_type, LABEL_BEGIN); CREATE_BLOCK(block->llvm_entry_block, name); MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block); /* Jump to the entry block */ BUILD_BR(block->llvm_entry_block); + if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, block)) + goto fail; /* Start to translate the if branch */ SET_BUILDER_POS(block->llvm_entry_block); - aot_block_stack_push(&func_ctx->block_stack, block); } else { - /* Compare value is not 0, condtion is false, if branch of + /* Compare value is not 0, condition is false, if branch of BLOCK if cannot be reached */ if (else_addr) { /* Create else block */ format_block_name(name, sizeof(name), - block->block_index, block_type, LABEL_ELSE); + block->block_index, label_type, LABEL_ELSE); CREATE_BLOCK(block->llvm_else_block, name); MOVE_BLOCK_AFTER_CURR(block->llvm_else_block); /* Jump to the else block */ BUILD_BR(block->llvm_else_block); + if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, block)) + goto fail; /* Start to translate the else branch */ SET_BUILDER_POS(block->llvm_else_block); *p_frame_ip = else_addr + 1; - aot_block_stack_push(&func_ctx->block_stack, block); } else { - if (block->return_type != VALUE_TYPE_VOID) { - aot_set_last_error("WASM value stack underflow."); - goto fail; - } /* skip the block */ - wasm_runtime_free(block); + aot_block_destroy(block); *p_frame_ip = end_addr + 1; } } @@ -338,7 +503,7 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; fail: - wasm_runtime_free(block); + aot_block_destroy(block); return false; } @@ -349,13 +514,14 @@ aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, AOTBlock *block = func_ctx->block_stack.block_list_end; LLVMValueRef value; char name[32]; + uint32 i, result_index; /* Check block */ if (!block) { aot_set_last_error("WASM block stack underflow."); return false; } - if (block->block_type != BLOCK_TYPE_IF + if (block->label_type != LABEL_TYPE_IF || (!block->skip_wasm_code_else && !block->llvm_else_block)) { aot_set_last_error("Invalid WASM block type."); @@ -365,7 +531,7 @@ aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Create end block if needed */ if (!block->llvm_end_block) { format_block_name(name, sizeof(name), - block->block_index, block->block_type, LABEL_END); + block->block_index, block->label_type, LABEL_END); CREATE_BLOCK(block->llvm_end_block, name); if (block->llvm_else_block) MOVE_BLOCK_AFTER(block->llvm_end_block, block->llvm_else_block); @@ -376,10 +542,11 @@ aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, block->is_reachable = true; /* Comes from the if branch of BLOCK if */ - if (block->return_type != VALUE_TYPE_VOID) { - POP(value, block->return_type); - CREATE_RETURN_VALUE_PHI(block); - ADD_TO_RETURN_PHI(block, value); + CREATE_RESULT_VALUE_PHIS(block); + for (i = 0; i < block->result_count; i++) { + result_index = block->result_count - 1 - i; + POP(value, block->result_types[result_index]); + ADD_TO_RESULT_PHIS(block, value, result_index); } /* Jump to end block */ @@ -387,8 +554,12 @@ aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!block->skip_wasm_code_else && block->llvm_else_block) { - /* Clear value stack and start to translate else branch */ + /* Clear value stack, recover param values + * and start to translate else branch. + */ aot_value_stack_destroy(&block->value_stack); + for (i = 0; i < block->param_count; i++) + PUSH(block->else_param_phis[i], block->param_types[i]); SET_BUILDER_POS(block->llvm_else_block); aot_checked_addr_list_destroy(func_ctx); return true; @@ -409,6 +580,7 @@ aot_compile_op_end(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef value; LLVMBasicBlockRef next_llvm_end_block; char name[32]; + uint32 i, result_index; /* Check block stack */ if (!(block = func_ctx->block_stack.block_list_end)) { @@ -419,17 +591,18 @@ aot_compile_op_end(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Create the end block */ if (!block->llvm_end_block) { format_block_name(name, sizeof(name), - block->block_index, block->block_type, LABEL_END); + block->block_index, block->label_type, LABEL_END); CREATE_BLOCK(block->llvm_end_block, name); if ((next_llvm_end_block = find_next_llvm_end_block(block))) MOVE_BLOCK_BEFORE(block->llvm_end_block, next_llvm_end_block); } - /* Handle block return value */ - if (block->return_type != VALUE_TYPE_VOID) { - POP(value, block->return_type); - CREATE_RETURN_VALUE_PHI(block); - ADD_TO_RETURN_PHI(block, value); + /* Handle block result values */ + CREATE_RESULT_VALUE_PHIS(block); + for (i = 0; i < block->result_count; i++) { + result_index = block->result_count - 1 - i; + POP(value, block->result_types[result_index]); + ADD_TO_RESULT_PHIS(block, value, result_index); } /* Jump to the end block */ @@ -446,16 +619,23 @@ aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 br_depth, uint8 **p_frame_ip) { AOTBlock *block_dst; - LLVMValueRef value_ret; + LLVMValueRef value_ret, value_param; LLVMBasicBlockRef next_llvm_end_block; char name[32]; + uint32 i, param_index, result_index; if (!(block_dst = get_target_block(func_ctx, br_depth))) { return false; } - if (block_dst->block_type == BLOCK_TYPE_LOOP) { + if (block_dst->label_type == LABEL_TYPE_LOOP) { /* Dest block is Loop block */ + /* Handle Loop parameters */ + for (i = 0; i < block_dst->param_count; i++) { + param_index = block_dst->param_count - 1 - i; + POP(value_param, block_dst->param_types[param_index]); + ADD_TO_PARAM_PHIS(block_dst, value_param, param_index); + } BUILD_BR(block_dst->llvm_entry_block); } else { @@ -463,7 +643,7 @@ aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Create the end block */ if (!block_dst->llvm_end_block) { format_block_name(name, sizeof(name), - block_dst->block_index, block_dst->block_type, + block_dst->block_index, block_dst->label_type, LABEL_END); CREATE_BLOCK(block_dst->llvm_end_block, name); if ((next_llvm_end_block = find_next_llvm_end_block(block_dst))) @@ -473,13 +653,13 @@ aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, block_dst->is_reachable = true; - /* Handle return value */ - if (block_dst->return_type != VALUE_TYPE_VOID) { - POP(value_ret, block_dst->return_type); - CREATE_RETURN_VALUE_PHI(block_dst); - ADD_TO_RETURN_PHI(block_dst, value_ret); + /* Handle result values */ + CREATE_RESULT_VALUE_PHIS(block_dst); + for (i = 0; i < block_dst->result_count; i++) { + result_index = block_dst->result_count - 1 - i; + POP(value_ret, block_dst->result_types[result_index]); + ADD_TO_RESULT_PHIS(block_dst, value_ret, result_index); } - /* Jump to the end block */ BUILD_BR(block_dst->llvm_end_block); } @@ -494,9 +674,11 @@ aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 br_depth, uint8 **p_frame_ip) { AOTBlock *block_dst; - LLVMValueRef value_cmp, value_ret; + LLVMValueRef value_cmp, value, *values = NULL; LLVMBasicBlockRef llvm_else_block, next_llvm_end_block; char name[32]; + uint32 i, param_index, result_index; + uint64 size; POP_COND(value_cmp); if (!LLVMIsConstant(value_cmp)) { @@ -509,8 +691,29 @@ aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, CREATE_BLOCK(llvm_else_block, "br_if_else"); MOVE_BLOCK_AFTER_CURR(llvm_else_block); - if (block_dst->block_type == BLOCK_TYPE_LOOP) { + if (block_dst->label_type == LABEL_TYPE_LOOP) { /* Dest block is Loop block */ + /* Handle Loop parameters */ + if (block_dst->param_count) { + size = sizeof(LLVMValueRef) * (uint64)block_dst->param_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + for (i = 0; i < block_dst->param_count; i++) { + param_index = block_dst->param_count - 1 - i; + POP(value, block_dst->param_types[param_index]); + ADD_TO_PARAM_PHIS(block_dst, value, param_index); + values[param_index] = value; + } + for (i = 0; i < block_dst->param_count; i++) { + PUSH(values[i], block_dst->param_types[i]); + } + wasm_runtime_free(values); + values = NULL; + } + BUILD_COND_BR(value_cmp, block_dst->llvm_entry_block, llvm_else_block); @@ -522,7 +725,7 @@ aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Create the end block */ if (!block_dst->llvm_end_block) { format_block_name(name, sizeof(name), - block_dst->block_index, block_dst->block_type, + block_dst->block_index, block_dst->label_type, LABEL_END); CREATE_BLOCK(block_dst->llvm_end_block, name); if ((next_llvm_end_block = find_next_llvm_end_block(block_dst))) @@ -530,15 +733,29 @@ aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, next_llvm_end_block); } - /* Set reachable flag and create condtion br IR */ + /* Set reachable flag and create condition br IR */ block_dst->is_reachable = true; - /* Handle return value */ - if (block_dst->return_type != VALUE_TYPE_VOID) { - POP(value_ret, block_dst->return_type); - CREATE_RETURN_VALUE_PHI(block_dst); - ADD_TO_RETURN_PHI(block_dst, value_ret); - PUSH(value_ret, block_dst->return_type); + /* Handle result values */ + if (block_dst->result_count) { + size = sizeof(LLVMValueRef) * (uint64)block_dst->result_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + CREATE_RESULT_VALUE_PHIS(block_dst); + for (i = 0; i < block_dst->result_count; i++) { + result_index = block_dst->result_count - 1 - i; + POP(value, block_dst->result_types[result_index]); + values[result_index] = value; + ADD_TO_RESULT_PHIS(block_dst, value, result_index); + } + for (i = 0; i < block_dst->result_count; i++) { + PUSH(values[i], block_dst->result_types[i]); + } + wasm_runtime_free(values); + values = NULL; } /* Condition jump to end block */ @@ -551,16 +768,18 @@ aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } else { if ((int32)LLVMConstIntGetZExtValue(value_cmp) != 0) { - /* Compare value is not 0, condtion is true, same as op_br */ + /* Compare value is not 0, condition is true, same as op_br */ return aot_compile_op_br(comp_ctx, func_ctx, br_depth, p_frame_ip); } else { - /* Compare value is not 0, condtion is false, skip br_if */ + /* Compare value is not 0, condition is false, skip br_if */ return true; } } return true; fail: + if (values) + wasm_runtime_free(values); return false; } @@ -569,12 +788,14 @@ aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 *br_depths, uint32 br_count, uint8 **p_frame_ip) { - uint32 i; - LLVMValueRef value_switch, value_cmp, value_case, value_ret = NULL; + uint32 i, j; + LLVMValueRef value_switch, value_cmp, value_case, value, *values = NULL; LLVMBasicBlockRef default_llvm_block = NULL, target_llvm_block; LLVMBasicBlockRef next_llvm_end_block; AOTBlock *target_block; uint32 br_depth, depth_idx; + uint32 param_index, result_index; + uint64 size; char name[32]; POP_I32(value_cmp); @@ -585,13 +806,13 @@ aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!target_block) return false; - if (target_block->block_type != BLOCK_TYPE_LOOP) { + if (target_block->label_type != LABEL_TYPE_LOOP) { /* Dest block is Block/If/Function block */ /* Create the end block */ if (!target_block->llvm_end_block) { format_block_name(name, sizeof(name), target_block->block_index, - target_block->block_type, + target_block->label_type, LABEL_END); CREATE_BLOCK(target_block->llvm_end_block, name); if ((next_llvm_end_block = @@ -599,18 +820,50 @@ aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, MOVE_BLOCK_BEFORE(target_block->llvm_end_block, next_llvm_end_block); } - /* Handle return value */ - if (target_block->return_type != VALUE_TYPE_VOID) { - POP(value_ret, target_block->return_type); - CREATE_RETURN_VALUE_PHI(target_block); - ADD_TO_RETURN_PHI(target_block, value_ret); - PUSH(value_ret, target_block->return_type); + /* Handle result values */ + if (target_block->result_count) { + size = sizeof(LLVMValueRef) * (uint64)target_block->result_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + CREATE_RESULT_VALUE_PHIS(target_block); + for (j = 0; j < target_block->result_count; j++) { + result_index = target_block->result_count - 1 - j; + POP(value, target_block->result_types[result_index]); + values[result_index] = value; + ADD_TO_RESULT_PHIS(target_block, value, result_index); + } + for (j = 0; j < target_block->result_count; j++) { + PUSH(values[j], target_block->result_types[j]); + } + wasm_runtime_free(values); } target_block->is_reachable = true; if (i == br_count) default_llvm_block = target_block->llvm_end_block; } else { + /* Handle Loop parameters */ + if (target_block->param_count) { + size = sizeof(LLVMValueRef) * (uint64)target_block->param_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + for (j = 0; j < target_block->param_count; j++) { + param_index = target_block->param_count - 1 - j; + POP(value, target_block->param_types[param_index]); + values[param_index] = value; + ADD_TO_PARAM_PHIS(target_block, value, param_index); + } + for (j = 0; j < target_block->param_count; j++) { + PUSH(values[j], target_block->param_types[j]); + } + wasm_runtime_free(values); + } if (i == br_count) default_llvm_block = target_block->llvm_entry_block; } @@ -630,7 +883,7 @@ aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, target_block = get_target_block(func_ctx, br_depths[i]); if (!target_block) return false; - target_llvm_block = target_block->block_type != BLOCK_TYPE_LOOP + target_llvm_block = target_block->label_type != LABEL_TYPE_LOOP ? target_block->llvm_end_block : target_block->llvm_entry_block; LLVMAddCase(value_switch, value_case, target_llvm_block); @@ -648,6 +901,8 @@ aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return aot_compile_op_br(comp_ctx, func_ctx, br_depth, p_frame_ip); } fail: + if (values) + wasm_runtime_free(values); return false; } @@ -657,14 +912,38 @@ aot_compile_op_return(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { AOTBlock *block_func = func_ctx->block_stack.block_list_head; LLVMValueRef value; + AOTFuncType *func_type; + uint32 i, param_index, result_index; bh_assert(block_func); - if (block_func->return_type != VALUE_TYPE_VOID) { - POP(value, block_func->return_type); - LLVMBuildRet(comp_ctx->builder, value); + func_type = func_ctx->aot_func->func_type; + + if (block_func->result_count) { + /* Store extra result values to function parameters */ + for (i = 0; i < block_func->result_count - 1; i++) { + result_index = block_func->result_count - 1 - i; + POP(value, block_func->result_types[result_index]); + param_index = func_type->param_count + result_index; + if (!LLVMBuildStore(comp_ctx->builder, + value, + LLVMGetParam(func_ctx->func, param_index))) { + aot_set_last_error("llvm build store failed."); + goto fail; + } + } + /* Return the first result value */ + POP(value, block_func->result_types[0]); + if (!LLVMBuildRet(comp_ctx->builder, value)) { + aot_set_last_error("llvm build return failed."); + goto fail; + } + } + else { + if (!LLVMBuildRetVoid(comp_ctx->builder)) { + aot_set_last_error("llvm build return void failed."); + goto fail; + } } - else - LLVMBuildRetVoid(comp_ctx->builder); return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); fail: diff --git a/core/iwasm/compilation/aot_emit_control.h b/core/iwasm/compilation/aot_emit_control.h index d222ea607..fa73d29fb 100644 --- a/core/iwasm/compilation/aot_emit_control.h +++ b/core/iwasm/compilation/aot_emit_control.h @@ -15,7 +15,8 @@ extern "C" { bool aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint8 **p_frame_ip, uint8 *frame_ip_end, - uint32 block_type, uint32 block_ret_type); + uint32 label_type, uint32 param_count, uint8 *param_types, + uint32 result_count, uint8 *result_types); bool aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index ed45edda9..1897cfd80 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -64,9 +64,9 @@ check_exception_thrown(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) /* Load the first byte of aot_module_inst->cur_exception, and check whether it is '\0'. If yes, no exception was thrown. */ if (!(value = LLVMBuildLoad(comp_ctx->builder, func_ctx->cur_exception, - "exce_value")) + "exce_value")) || !(cmp = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, - value, I8_ZERO, "cmp"))) { + value, I8_ZERO, "cmp"))) { aot_set_last_error("llvm build icmp failed."); return false; } @@ -183,7 +183,7 @@ call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } - if (param_count > 64) { + if (param_cell_num > 64) { aot_set_last_error("prepare native arguments failed: " "maximum 64 parameter cell number supported."); return false; @@ -309,17 +309,22 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 import_func_count = comp_ctx->comp_data->import_func_count; AOTImportFunc *import_funcs = comp_ctx->comp_data->import_funcs; uint32 func_count = comp_ctx->func_ctx_count, param_cell_num = 0; + uint32 ext_ret_cell_num = 0, cell_num = 0; AOTFuncContext **func_ctxes = comp_ctx->func_ctxes; AOTFuncType *func_type; AOTFunc *aot_func; LLVMTypeRef *param_types = NULL, ret_type; + LLVMTypeRef ext_ret_ptr_type; LLVMValueRef *param_values = NULL, value_ret = NULL, func; LLVMValueRef import_func_idx, res; - int32 i, j = 0, param_count; + LLVMValueRef ext_ret, ext_ret_ptr, ext_ret_idx; + int32 i, j = 0, param_count, result_count, ext_ret_count; uint64 total_size; uint32 callee_cell_num; uint8 wasm_ret_type; + uint8 *ext_ret_types = NULL; bool ret = false; + char buf[32]; /* Check function index */ if (func_idx >= import_func_count + func_count) { @@ -335,11 +340,19 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_func->func_type; /* Get param cell number */ - param_cell_num = wasm_type_param_cell_num(func_type); + param_cell_num = func_type->param_cell_num; - /* Allocate memory for parameters */ + /* Allocate memory for parameters. + * Parameters layout: + * - exec env + * - wasm function's parameters + * - extra results'(except the first one) addresses + */ param_count = (int32)func_type->param_count; - total_size = sizeof(LLVMValueRef) * (uint64)(param_count + 1); + result_count = (int32)func_type->result_count; + ext_ret_count = result_count > 1 ? result_count - 1 : 0; + total_size = sizeof(LLVMValueRef) * (uint64)(param_count + 1 + + ext_ret_count); if (total_size >= UINT32_MAX || !(param_values = wasm_runtime_malloc((uint32)total_size))) { aot_set_last_error("Allocate memory failed."); @@ -402,9 +415,50 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, && !check_stack_boundary(comp_ctx, func_ctx, callee_cell_num)) goto fail; + /* Prepare parameters for extra results */ + if (ext_ret_count > 0) { + ext_ret_types = func_type->types + param_count + 1; + ext_ret_cell_num = + wasm_get_cell_num(ext_ret_types, ext_ret_count); + if (ext_ret_cell_num > 64) { + aot_set_last_error("prepare extra results's return " + "address arguments failed: " + "maximum 64 parameter cell number supported."); + goto fail; + } + + for (i = 0; i < ext_ret_count; i++) { + if (!(ext_ret_idx = I32_CONST(cell_num)) + || !(ext_ret_ptr_type = + LLVMPointerType(TO_LLVM_TYPE(ext_ret_types[i]), 0))) { + aot_set_last_error("llvm add const or pointer type failed."); + goto fail; + } + + snprintf(buf, sizeof(buf), "func%d_ext_ret%d_ptr", func_idx, i); + if (!(ext_ret_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder, + func_ctx->argv_buf, + &ext_ret_idx, 1, buf))) { + aot_set_last_error("llvm build GEP failed."); + goto fail; + } + snprintf(buf, sizeof(buf), "func%d_ext_ret%d_ptr_cast", func_idx, i); + if (!(ext_ret_ptr = LLVMBuildBitCast(comp_ctx->builder, + ext_ret_ptr, + ext_ret_ptr_type, + buf))) { + aot_set_last_error("llvm build bit cast failed."); + goto fail; + } + param_values[1 + param_count + i] = ext_ret_ptr; + cell_num += wasm_value_type_cell_num(ext_ret_types[i]); + } + } + /* Call the function */ if (!(value_ret = LLVMBuildCall(comp_ctx->builder, func, - param_values, (uint32)param_count + 1, + param_values, + (uint32)param_count + 1 + ext_ret_count, (func_type->result_count > 0 ? "call" : "")))) { aot_set_last_error("LLVM build call failed."); @@ -419,8 +473,21 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - if (func_type->result_count > 0) + if (func_type->result_count > 0) { + /* Push the first result to stack */ PUSH(value_ret, func_type->types[func_type->param_count]); + /* Load extra result from its address and push to stack */ + for (i = 0; i < ext_ret_count; i++) { + snprintf(buf, sizeof(buf), "func%d_ext_ret%d", func_idx, i); + if (!(ext_ret = LLVMBuildLoad(comp_ctx->builder, + param_values[1 + param_count + i], + buf))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + PUSH(ext_ret, ext_ret_types[i]); + } + } ret = true; fail: @@ -437,15 +504,15 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef func_type_idx, LLVMValueRef table_elem_idx, LLVMTypeRef *param_types, LLVMValueRef *param_values, uint32 param_count, uint32 param_cell_num, - LLVMTypeRef ret_type, uint8 wasm_ret_type, - LLVMValueRef *p_value_ret, LLVMValueRef *p_res) + uint32 result_count, uint8 *wasm_ret_types, + LLVMValueRef *value_rets, LLVMValueRef *p_res) { LLVMTypeRef func_type, func_ptr_type, func_param_types[6]; - LLVMTypeRef ret_ptr_type, elem_ptr_type; - LLVMValueRef func, elem_idx, elem_ptr; - LLVMValueRef func_param_values[6], value_ret = NULL, res = NULL; + LLVMTypeRef ret_type, ret_ptr_type, elem_ptr_type; + LLVMValueRef func, ret_idx, ret_ptr, elem_idx, elem_ptr; + LLVMValueRef func_param_values[6], res = NULL; char buf[32], *func_name = "aot_call_indirect"; - uint32 i, cell_num = 0; + uint32 i, cell_num = 0, ret_cell_num, argv_cell_num; /* prepare function type of aot_call_indirect */ func_param_types[0] = comp_ctx->exec_env_type; /* exec_env */ @@ -482,7 +549,9 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } - if (param_count > 64) { + ret_cell_num = wasm_get_cell_num(wasm_ret_types, result_count); + argv_cell_num = param_cell_num > ret_cell_num ? param_cell_num : ret_cell_num; + if (argv_cell_num > 64) { aot_set_last_error("prepare native arguments failed: " "maximum 64 parameter cell number supported."); return false; @@ -533,24 +602,31 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return false; } - /* get function return value */ - if (wasm_ret_type != VALUE_TYPE_VOID) { - if (!(ret_ptr_type = LLVMPointerType(ret_type, 0))) { - aot_set_last_error("llvm add pointer type failed."); + /* get function result values */ + cell_num = 0; + for (i = 0; i < result_count; i++) { + ret_type = TO_LLVM_TYPE(wasm_ret_types[i]); + if (!(ret_idx = I32_CONST(cell_num)) + || !(ret_ptr_type = LLVMPointerType(ret_type, 0))) { + aot_set_last_error("llvm add const or pointer type failed."); return false; } - if (!(value_ret = LLVMBuildBitCast(comp_ctx->builder, func_ctx->argv_buf, - ret_ptr_type, "argv_ret"))) { - aot_set_last_error("llvm build bit cast failed."); + snprintf(buf, sizeof(buf), "argv_ret%d", i); + if (!(ret_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder, + func_ctx->argv_buf, &ret_idx, 1, buf)) + || !(ret_ptr = LLVMBuildBitCast(comp_ctx->builder, ret_ptr, + ret_ptr_type, buf))) { + aot_set_last_error("llvm build GEP or bit cast failed."); return false; } - if (!(*p_value_ret = LLVMBuildLoad(comp_ctx->builder, value_ret, - "value_ret"))) { + snprintf(buf, sizeof(buf), "ret%d", i); + if (!(value_rets[i] = LLVMBuildLoad(comp_ctx->builder, ret_ptr, buf))) { aot_set_last_error("llvm build load failed."); return false; } + cell_num += wasm_value_type_cell_num(wasm_ret_types[i]); } *p_res = res; @@ -563,12 +639,12 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { AOTFuncType *func_type; LLVMValueRef elem_idx, ftype_idx; - LLVMValueRef *param_values = NULL, value_ret = NULL, res = NULL; - LLVMTypeRef *param_types = NULL, ret_type; - int32 i, param_count; + LLVMValueRef *param_values = NULL, *value_rets = NULL, res = NULL; + LLVMTypeRef *param_types = NULL; + int32 i, param_count, result_count; uint32 param_cell_num; uint64 total_size; - uint8 wasm_ret_type; + uint8 *wasm_ret_types = NULL; bool ret; /* Check function type index */ @@ -582,7 +658,9 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, func_type = comp_ctx->comp_data->func_types[type_idx]; - param_cell_num = wasm_type_param_cell_num(func_type); + param_cell_num = func_type->param_cell_num; + result_count = func_type->result_count; + wasm_ret_types = func_type->types + func_type->param_count; POP_I32(elem_idx); @@ -598,16 +676,6 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, for (i = 0; i < param_count; i++) param_types[i] = TO_LLVM_TYPE(func_type->types[i]); - /* Resolve return type of the LLVM function */ - if (func_type->result_count) { - wasm_ret_type = func_type->types[func_type->param_count]; - ret_type = TO_LLVM_TYPE(wasm_ret_type); - } - else { - wasm_ret_type = VALUE_TYPE_VOID; - ret_type = VOID_TYPE; - } - /* Allocate memory for parameters */ total_size = sizeof(LLVMValueRef) * (uint64)param_count; if (total_size >= UINT32_MAX @@ -620,16 +688,25 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, for (i = param_count - 1; i >= 0; i--) POP(param_values[i], func_type->types[i]); + /* Allocate memory for result values */ + total_size = sizeof(LLVMValueRef) * (uint64)result_count; + if (total_size >= UINT32_MAX + || !(value_rets = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("Allocate memory failed."); + goto fail; + } + memset(value_rets, 0, total_size); + if (!call_aot_call_indirect_func(comp_ctx, func_ctx, func_type, ftype_idx, elem_idx, param_types, param_values, param_count, param_cell_num, - ret_type, wasm_ret_type, - &value_ret, &res)) + result_count, wasm_ret_types, + value_rets, &res)) goto fail; - if (func_type->result_count > 0) - PUSH(value_ret, func_type->types[func_type->param_count]); + for (i = 0; i < func_type->result_count; i++) + PUSH(value_rets[i], func_type->types[func_type->param_count + i]); /* Check whether there was exception thrown when executing the function */ if (!check_call_return(comp_ctx, func_ctx, res)) @@ -638,6 +715,8 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, ret = true; fail: + if (value_rets) + wasm_runtime_free(value_rets); if (param_values) wasm_runtime_free(param_values); if (param_types) diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 7ae2e025d..813513eb7 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -41,9 +41,14 @@ aot_add_llvm_func(AOTCompContext *comp_ctx, AOTFuncType *aot_func_type, uint64 size; uint32 i, j = 0, param_count = (uint64)aot_func_type->param_count; - /* aot context as first parameter */ + /* exec env as first parameter */ param_count++; + /* Extra wasm function results(except the first one)'s address are + * appended to aot function parameters. */ + if (aot_func_type->result_count > 1) + param_count += aot_func_type->result_count - 1; + /* Initialize parameter types of the LLVM function */ size = sizeof(LLVMTypeRef) * ((uint64)param_count); if (size >= UINT32_MAX @@ -56,6 +61,15 @@ aot_add_llvm_func(AOTCompContext *comp_ctx, AOTFuncType *aot_func_type, param_types[j++] = comp_ctx->exec_env_type; for (i = 0; i < aot_func_type->param_count; i++) param_types[j++] = TO_LLVM_TYPE(aot_func_type->types[i]); + /* Extra results' address */ + for (i = 1; i < aot_func_type->result_count; i++, j++) { + param_types[j] = + TO_LLVM_TYPE(aot_func_type->types[aot_func_type->param_count + i]); + if (!(param_types[j] = LLVMPointerType(param_types[j], 0))) { + aot_set_last_error("llvm get pointer type failed."); + goto fail; + } + } /* Resolve return type of the LLVM function */ if (aot_func_type->result_count) @@ -92,6 +106,16 @@ fail: return func; } +static void +free_block_memory(AOTBlock *block) +{ + if (block->param_types) + wasm_runtime_free(block->param_types); + if (block->result_types) + wasm_runtime_free(block->result_types); + wasm_runtime_free(block); +} + /** * Create first AOTBlock, or function block for the function */ @@ -100,22 +124,33 @@ aot_create_func_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, AOTFunc *func, AOTFuncType *aot_func_type) { AOTBlock *aot_block; + uint32 param_count = aot_func_type->param_count, + result_count = aot_func_type->result_count; /* Allocate memory */ if (!(aot_block = wasm_runtime_malloc(sizeof(AOTBlock)))) { aot_set_last_error("allocate memory failed."); return NULL; } - memset(aot_block, 0, sizeof(AOTBlock)); + if (param_count + && !(aot_block->param_types = wasm_runtime_malloc(param_count))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + if (result_count) { + if (!(aot_block->result_types = wasm_runtime_malloc(result_count))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + } - /* Set block type and return type */ - aot_block->block_type = BLOCK_TYPE_FUNCTION; - if (aot_func_type->result_count) - aot_block->return_type = aot_func_type->types[aot_func_type->param_count]; - else - aot_block->return_type = VALUE_TYPE_VOID; - + /* Set block data */ + aot_block->label_type = LABEL_TYPE_FUNCTION; + aot_block->param_count = param_count; + memcpy(aot_block->param_types, aot_func_type->types, param_count); + aot_block->result_count = result_count; + memcpy(aot_block->result_types, aot_func_type->types + param_count, result_count); aot_block->wasm_code_end = func->code + func->code_size; /* Add function entry block */ @@ -129,7 +164,7 @@ aot_create_func_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return aot_block; fail: - wasm_runtime_free(aot_block); + free_block_memory(aot_block); return NULL; } @@ -1288,6 +1323,16 @@ void aot_block_destroy(AOTBlock *block) { aot_value_stack_destroy(&block->value_stack); + if (block->param_types) + wasm_runtime_free(block->param_types); + if (block->param_phis) + wasm_runtime_free(block->param_phis); + if (block->else_param_phis) + wasm_runtime_free(block->else_param_phis); + if (block->result_types) + wasm_runtime_free(block->result_types); + if (block->result_phis) + wasm_runtime_free(block->result_phis); wasm_runtime_free(block); } diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index f94f7c754..9fd28a622 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -48,10 +48,8 @@ typedef struct AOTBlock { /* Block index */ uint32 block_index; - /* BLOCK_TYPE_BLOCK/LOOP/IF/FUNCTION */ - uint32 block_type; - /* VALUE_TYPE_I32/I64/F32/F64/VOID */ - uint8 return_type; + /* LABEL_TYPE_BLOCK/LOOP/IF/FUNCTION */ + uint32 label_type; /* Whether it is reachable */ bool is_reachable; /* Whether skip translation of wasm else branch */ @@ -72,8 +70,16 @@ typedef struct AOTBlock { /* WASM operation stack */ AOTValueStack value_stack; - /* Return value of this block, a PHI node */ - LLVMValueRef return_value_phi; + /* Param count/types/PHIs of this block */ + uint32 param_count; + uint8 *param_types; + LLVMValueRef *param_phis; + LLVMValueRef *else_param_phis; + + /* Result count/types/PHIs of this block */ + uint32 result_count; + uint8 *result_types; + LLVMValueRef *result_phis; } AOTBlock; /** diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index b54d09aa3..fd7f03a82 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -66,10 +66,10 @@ extern "C" { #define EXPORT_KIND_MEMORY 2 #define EXPORT_KIND_GLOBAL 3 -#define BLOCK_TYPE_BLOCK 0 -#define BLOCK_TYPE_LOOP 1 -#define BLOCK_TYPE_IF 2 -#define BLOCK_TYPE_FUNCTION 3 +#define LABEL_TYPE_BLOCK 0 +#define LABEL_TYPE_LOOP 1 +#define LABEL_TYPE_IF 2 +#define LABEL_TYPE_FUNCTION 3 typedef struct WASMModule WASMModule; typedef struct WASMFunction WASMFunction; @@ -98,9 +98,10 @@ typedef struct InitializerExpression { } InitializerExpression; typedef struct WASMType { - uint32 param_count; - /* only one result is supported currently */ - uint32 result_count; + uint16 param_count; + uint16 result_count; + uint16 param_cell_num; + uint16 ret_cell_num; /* types of params and results */ uint8 types[1]; } WASMType; @@ -206,7 +207,7 @@ typedef struct WASMFunction { uint16 ret_cell_num; /* cell num of local variables */ uint16 local_cell_num; - /* offset of each local, including function paramameters + /* offset of each local, including function parameters and local variables */ uint16 *local_offsets; @@ -351,9 +352,21 @@ typedef struct WASMModule { #endif } WASMModule; +typedef struct BlockType { + /* Block type may be expressed in one of two forms: + * either by the type of the single return value or + * by a type index of module. + */ + union { + uint8 value_type; + WASMType *type; + } u; + bool is_value_type; +} BlockType; + typedef struct WASMBranchBlock { - uint8 block_type; - uint8 return_type; + uint8 label_type; + uint32 cell_num; uint8 *target_addr; uint32 *frame_sp; } WASMBranchBlock; @@ -421,40 +434,28 @@ wasm_value_type_size(uint8 value_type) inline static uint16 wasm_value_type_cell_num(uint8 value_type) { - switch (value_type) { - case VALUE_TYPE_I32: - case VALUE_TYPE_F32: - return 1; - case VALUE_TYPE_I64: - case VALUE_TYPE_F64: - return 2; - default: - bh_assert(0); + if (value_type == VALUE_TYPE_VOID) + return 0; + else if (value_type == VALUE_TYPE_I32 + || value_type == VALUE_TYPE_F32) + return 1; + else if (value_type == VALUE_TYPE_I64 + || value_type == VALUE_TYPE_F64) + return 2; + else { + bh_assert(0); } return 0; } -inline static uint16 +inline static uint32 wasm_get_cell_num(const uint8 *types, uint32 type_count) { uint32 cell_num = 0; uint32 i; for (i = 0; i < type_count; i++) cell_num += wasm_value_type_cell_num(types[i]); - return (uint16)cell_num; -} - -inline static uint16 -wasm_type_param_cell_num(const WASMType *type) -{ - return wasm_get_cell_num(type->types, type->param_count); -} - -inline static uint16 -wasm_type_return_cell_num(const WASMType *type) -{ - return wasm_get_cell_num(type->types + type->param_count, - type->result_count); + return cell_num; } inline static bool @@ -467,8 +468,45 @@ wasm_type_equal(const WASMType *type1, const WASMType *type2) ? true : false; } +static inline uint32 +block_type_get_param_types(BlockType *block_type, + uint8 **p_param_types) +{ + uint32 param_count = 0; + if (!block_type->is_value_type) { + WASMType *wasm_type = block_type->u.type; + *p_param_types = wasm_type->types; + param_count = wasm_type->param_count; + } + else { + *p_param_types = NULL; + param_count = 0; + } + + return param_count; +} + +static inline uint32 +block_type_get_result_types(BlockType *block_type, + uint8 **p_result_types) +{ + uint32 result_count = 0; + if (block_type->is_value_type) { + if (block_type->u.value_type != VALUE_TYPE_VOID) { + *p_result_types = &block_type->u.value_type; + result_count = 1; + } + } + else { + WASMType *wasm_type = block_type->u.type; + *p_result_types = wasm_type->types + wasm_type->param_count; + result_count = wasm_type->result_count; + } + return result_count; +} + #ifdef __cplusplus } /* end of extern "C" */ #endif -#endif /* end of _WASM_H_ */ \ No newline at end of file +#endif /* end of _WASM_H_ */ diff --git a/core/iwasm/interpreter/wasm_interp.h b/core/iwasm/interpreter/wasm_interp.h index 9883d2654..c3c2c1b34 100644 --- a/core/iwasm/interpreter/wasm_interp.h +++ b/core/iwasm/interpreter/wasm_interp.h @@ -27,8 +27,8 @@ typedef struct WASMInterpFrame { uint8 *ip; #if WASM_ENABLE_FAST_INTERP != 0 - /* return offset of current frame. - the callee will put return value here */ + /* return offset of the first return value of current frame. + the callee will put return values here continuously */ uint32 ret_offset; uint32 *lp; uint32 operand[1]; diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 5316bd92d..211ba5bf7 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -414,13 +414,13 @@ read_leb(const uint8 *buf, uint32 *p_offset, uint32 maxbits, bool sign) frame_sp += 2; \ } while (0) -#define PUSH_CSP(type, ret_type, _target_addr) do { \ - bh_assert(frame_csp < frame->csp_boundary); \ - frame_csp->block_type = type; \ - frame_csp->return_type = ret_type; \ - frame_csp->target_addr = _target_addr; \ - frame_csp->frame_sp = frame_sp; \ - frame_csp++; \ +#define PUSH_CSP(_label_type, cell_num, _target_addr) do { \ + bh_assert(frame_csp < frame->csp_boundary); \ + frame_csp->label_type = _label_type; \ + frame_csp->cell_num = cell_num; \ + frame_csp->target_addr = _target_addr; \ + frame_csp->frame_sp = frame_sp; \ + frame_csp++; \ } while (0) #define POP_I32() (--frame_sp, *(int32*)frame_sp) @@ -442,21 +442,15 @@ read_leb(const uint8 *buf, uint32 *p_offset, uint32 maxbits, bool sign) #define POP_CSP_N(n) do { \ uint32 *frame_sp_old = frame_sp; \ + uint32 cell_num = 0; \ POP_CSP_CHECK_OVERFLOW(n + 1); \ frame_csp -= n; \ frame_ip = (frame_csp - 1)->target_addr; \ - /* copy return value of block */ \ + /* copy arity values of block */ \ frame_sp = (frame_csp - 1)->frame_sp; \ - switch ((frame_csp - 1)->return_type) { \ - case VALUE_TYPE_I32: \ - case VALUE_TYPE_F32: \ - PUSH_I32(*(frame_sp_old - 1)); \ - break; \ - case VALUE_TYPE_I64: \ - case VALUE_TYPE_F64: \ - PUSH_I64(GET_I64_FROM_ADDR(frame_sp_old - 2)); \ - break; \ - } \ + cell_num = (frame_csp - 1)->cell_num; \ + word_copy(frame_sp, frame_sp_old - cell_num, cell_num); \ + frame_sp += cell_num; \ } while (0) /* Pop the given number of elements from the given frame's stack. */ @@ -974,6 +968,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint8 *global_data = module->global_data; uint32 linear_mem_size = memory ? num_bytes_per_page * memory->cur_page_count : 0; WASMTableInstance *table = module->default_table; + WASMType **wasm_types = module->module->types; WASMGlobalInstance *globals = module->globals, *global; uint8 opcode_IMPDEP = WASM_OP_IMPDEP; WASMInterpFrame *frame = NULL; @@ -984,7 +979,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMBranchBlock *frame_csp = NULL; BlockAddr *cache_items; uint8 *frame_ip_end = frame_ip + 1; - uint8 opcode, block_ret_type; + uint8 opcode; uint32 *depths = NULL; uint32 depth_buf[BR_TABLE_TMP_BUF_LEN]; uint32 i, depth, cond, count, fidx, tidx, frame_size = 0; @@ -993,7 +988,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint8 *else_addr, *end_addr, *maddr = NULL; uint32 local_idx, local_offset, global_idx; uint8 local_type, *global_addr; - uint32 cache_index; + uint32 cache_index, type_index, cell_num; + uint8 value_type; #if WASM_ENABLE_LABELS_AS_VALUES != 0 #define HANDLE_OPCODE(op) &&HANDLE_##op @@ -1016,9 +1012,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP (WASM_OP_NOP): HANDLE_OP_END (); - HANDLE_OP (WASM_OP_BLOCK): - block_ret_type = *frame_ip++; + HANDLE_OP (EXT_OP_BLOCK): + read_leb_uint32(frame_ip, frame_ip_end, type_index); + cell_num = wasm_types[type_index]->ret_cell_num; + goto handle_op_block; + HANDLE_OP (WASM_OP_BLOCK): + value_type = *frame_ip++; + cell_num = wasm_value_type_cell_num(value_type); +handle_op_block: cache_index = ((uintptr_t)frame_ip) & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); cache_items = exec_env->block_addr_cache[cache_index]; if (cache_items[0].start_addr == frame_ip) { @@ -1029,24 +1031,37 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } else if (!wasm_loader_find_block_addr((BlockAddr*)exec_env->block_addr_cache, frame_ip, (uint8*)-1, - BLOCK_TYPE_BLOCK, + LABEL_TYPE_BLOCK, &else_addr, &end_addr, NULL, 0)) { wasm_set_exception(module, "find block address failed"); goto got_exception; } - PUSH_CSP(BLOCK_TYPE_BLOCK, block_ret_type, end_addr); + PUSH_CSP(LABEL_TYPE_BLOCK, cell_num, end_addr); HANDLE_OP_END (); + HANDLE_OP (EXT_OP_LOOP): + read_leb_uint32(frame_ip, frame_ip_end, type_index); + cell_num = wasm_types[type_index]->param_cell_num; + goto handle_op_loop; + HANDLE_OP (WASM_OP_LOOP): - block_ret_type = *frame_ip++; - PUSH_CSP(BLOCK_TYPE_LOOP, block_ret_type, frame_ip); + value_type = *frame_ip++; + cell_num = wasm_value_type_cell_num(value_type); +handle_op_loop: + PUSH_CSP(LABEL_TYPE_LOOP, cell_num, frame_ip); HANDLE_OP_END (); - HANDLE_OP (WASM_OP_IF): - block_ret_type = *frame_ip++; + HANDLE_OP (EXT_OP_IF): + read_leb_uint32(frame_ip, frame_ip_end, type_index); + cell_num = wasm_types[type_index]->ret_cell_num; + goto handle_op_if; + HANDLE_OP (WASM_OP_IF): + value_type = *frame_ip++; + cell_num = wasm_value_type_cell_num(value_type); +handle_op_if: cache_index = ((uintptr_t)frame_ip) & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1); cache_items = exec_env->block_addr_cache[cache_index]; if (cache_items[0].start_addr == frame_ip) { @@ -1059,7 +1074,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } else if (!wasm_loader_find_block_addr((BlockAddr*)exec_env->block_addr_cache, frame_ip, (uint8*)-1, - BLOCK_TYPE_IF, + LABEL_TYPE_IF, &else_addr, &end_addr, NULL, 0)) { wasm_set_exception(module, "find block address failed"); @@ -1068,7 +1083,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, cond = (uint32)POP_I32(); - PUSH_CSP(BLOCK_TYPE_IF, block_ret_type, end_addr); + PUSH_CSP(LABEL_TYPE_IF, cell_num, end_addr); /* condition of the if branch is false, else condition is met */ if (cond == 0) { @@ -1106,6 +1121,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_SUSPEND_FLAGS(); #endif read_leb_uint32(frame_ip, frame_ip_end, depth); +label_pop_csp_n: POP_CSP_N(depth); HANDLE_OP_END (); @@ -1116,7 +1132,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, read_leb_uint32(frame_ip, frame_ip_end, depth); cond = (uint32)POP_I32(); if (cond) - POP_CSP_N(depth); + goto label_pop_csp_n; HANDLE_OP_END (); HANDLE_OP (WASM_OP_BR_TABLE): @@ -1147,7 +1163,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, wasm_runtime_free(depths); depths = NULL; } - POP_CSP_N(depth); + goto label_pop_csp_n; HANDLE_OP_END (); HANDLE_OP (WASM_OP_RETURN): @@ -1192,7 +1208,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, wasm_set_exception(module, "type index is overflow"); goto got_exception; } - cur_type = module->module->types[tidx]; + cur_type = wasm_types[tidx]; /* to skip 0x00 here */ frame_ip++; @@ -2705,6 +2721,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP (EXT_OP_TEE_LOCAL_FAST_I64): HANDLE_OP (EXT_OP_COPY_STACK_TOP): HANDLE_OP (EXT_OP_COPY_STACK_TOP_I64): + HANDLE_OP (EXT_OP_COPY_STACK_VALUES): { wasm_set_exception(module, "WASM interp failed: unsupported opcode."); goto got_exception; @@ -2753,7 +2770,6 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, else { WASMFunction *cur_wasm_func = cur_func->u.func; WASMType *func_type; - uint8 ret_type; func_type = cur_wasm_func->func_type; @@ -2790,10 +2806,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, (uint32)(cur_func->local_cell_num * 4)); /* Push function block as first block */ - ret_type = func_type->result_count - ? cur_func->param_types[func_type->param_count] - : VALUE_TYPE_VOID; - PUSH_CSP(BLOCK_TYPE_FUNCTION, ret_type, frame_ip_end - 1); + cell_num = func_type->ret_cell_num; + PUSH_CSP(LABEL_TYPE_FUNCTION, cell_num, frame_ip_end - 1); wasm_exec_env_set_cur_frame(exec_env, (WASMRuntimeFrame*)frame); } @@ -2837,7 +2851,8 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMInterpFrame *frame, *outs_area; /* Allocate sufficient cells for all kinds of return values. */ - unsigned all_cell_num = 2, i; + unsigned all_cell_num = function->ret_cell_num > 2 ? + function->ret_cell_num : 2, i; /* This frame won't be used by JITed code, so only allocate interp frame here. */ unsigned frame_size = wasm_interp_interp_frame_size(all_cell_num); diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index b6509a7cf..fe34f91e8 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -679,21 +679,121 @@ trunc_f64_to_int(WASMModuleInstance *module, PUSH_##dst_op_type(value); \ } while (0) -#define RECOVER_BR_INFO() do { \ - uint16 stack_index, ret_cell_num; \ - stack_index = *(uint16*)frame_ip; \ - frame_ip += sizeof(uint16); \ - ret_cell_num = *(uint8*)frame_ip; \ - frame_ip += sizeof(uint8); \ - if (ret_cell_num == 1) \ - frame_lp[stack_index] = \ - frame_lp[*(int16*)frame_ip]; \ - else if (ret_cell_num == 2) { \ - *(int64*)(frame_lp + stack_index) = \ - *(int64*)(frame_lp + *(int16*)frame_ip);\ - } \ - frame_ip += sizeof(int16); \ - frame_ip = *(uint8**)frame_ip; \ +static bool +copy_stack_values(WASMModuleInstance *module, + uint32 *frame_lp, + uint32 arity, + uint32 total_cell_num, + const uint8 *cells, + const int16 *src_offsets, + const uint16 *dst_offsets) +{ + /* To avoid the overlap issue between src offsets and dst offset, + * we use 2 steps to do the copy. First step, copy the src values + * to a tmp buf. Second step, copy the values from tmp buf to dst. + */ + uint32 buf[16] = {0}, i; + uint32 *tmp_buf = buf; + uint8 cell; + int16 src, buf_index = 0; + uint16 dst; + + /* Allocate memory if the buf is not large enough */ + if (total_cell_num > sizeof(buf)/sizeof(uint32)) { + uint64 total_size = sizeof(uint32) * (uint64)total_cell_num; + if (total_size >= UINT32_MAX + || !(tmp_buf = wasm_runtime_malloc((uint32)total_size))) { + wasm_set_exception(module, + "WASM interp failed: allocate memory failed."); + return false; + } + } + + /* 1) Copy values from src to tmp buf */ + for (i = 0; i < arity; i++) { + cell = cells[i]; + src = src_offsets[i]; + if (cell == 1) + tmp_buf[buf_index] = frame_lp[src]; + else + *(uint64*)(tmp_buf + buf_index) = *(uint64*)(frame_lp + src); + buf_index += cell; + } + + /* 2) Copy values from tmp buf to dest */ + buf_index = 0; + for (i = 0; i < arity; i++) { + cell = cells[i]; + dst = dst_offsets[i]; + if (cell == 1) + frame_lp[dst] = tmp_buf[buf_index]; + else + *(uint64*)(frame_lp + dst) = *(uint64*)(tmp_buf + buf_index); + buf_index += cell; + } + + if (tmp_buf != buf) { + wasm_runtime_free(tmp_buf); + } + + return true; +} + +#define RECOVER_BR_INFO() do { \ + uint32 arity; \ + /* read arity */ \ + arity = *(uint32*)frame_ip; \ + frame_ip += sizeof(arity); \ + if (arity) { \ + uint32 total_cell; \ + uint16 *dst_offsets = NULL; \ + uint8 *cells; \ + int16 *src_offsets = NULL; \ + /* read total cell num */ \ + total_cell = *(uint32*)frame_ip; \ + frame_ip += sizeof(total_cell); \ + /* cells */ \ + cells = (uint8 *)frame_ip; \ + frame_ip += arity * sizeof(uint8); \ + /* src offsets */ \ + src_offsets = (int16 *)frame_ip; \ + frame_ip += arity * sizeof(int16); \ + /* dst offsets */ \ + dst_offsets = (uint16*)frame_ip; \ + frame_ip += arity * sizeof(uint16); \ + if (arity == 1) { \ + if (cells[0] == 1) \ + frame_lp[dst_offsets[0]] = \ + frame_lp[src_offsets[0]]; \ + else if (cells[0] == 2) { \ + *(int64*)(frame_lp + dst_offsets[0]) = \ + *(int64*)(frame_lp + src_offsets[0]); \ + } \ + } \ + else { \ + if (!copy_stack_values(module, frame_lp, \ + arity, total_cell, \ + cells, src_offsets, \ + dst_offsets)) \ + goto got_exception; \ + } \ + } \ + frame_ip = *(uint8**)frame_ip; \ + } while (0) + +#define SKIP_BR_INFO() do { \ + uint32 arity; \ + /* read and skip arity */ \ + arity = *(uint32*)frame_ip; \ + frame_ip += sizeof(arity); \ + if (arity) { \ + /* skip total cell num */ \ + frame_ip += sizeof(uint32); \ + /* skip cells, src offsets and dst offsets */ \ + frame_ip += (sizeof(uint8) + sizeof(int16) + sizeof(uint16)) * arity; \ + } \ + /* skip target address */ \ + frame_ip += sizeof(uint8*); \ } while (0) static inline int32 @@ -1034,6 +1134,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_THREAD_MGR != 0 CHECK_SUSPEND_FLAGS(); #endif +recover_br_info: RECOVER_BR_INFO(); HANDLE_OP_END (); @@ -1044,10 +1145,9 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, cond = frame_lp[GET_OFFSET()]; if (cond) - RECOVER_BR_INFO(); - else { - frame_ip += (2 + 1 + 2 + sizeof(uint8*)); - } + goto recover_br_info; + else + SKIP_BR_INFO(); HANDLE_OP_END (); @@ -1062,16 +1162,44 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (!(didx >= 0 && (uint32)didx < count)) didx = count; - frame_ip += (didx * ((2 + 1 + 2 + sizeof(uint8*)))); - RECOVER_BR_INFO(); - HANDLE_OP_END (); + while (didx--) + SKIP_BR_INFO(); + + goto recover_br_info; HANDLE_OP (WASM_OP_RETURN): - if (cur_func->ret_cell_num == 2) { - *((uint64 *)(prev_frame->lp + prev_frame->ret_offset)) = - GET_OPERAND(uint64, 0); - } else if (cur_func->ret_cell_num == 1) { - prev_frame->lp[prev_frame->ret_offset] = GET_OPERAND(int32, 0);; + { + uint32 ret_idx; + WASMType *func_type; + uint32 off, ret_offset; + uint8 *ret_types; + if (cur_func->is_import_func +#if WASM_ENABLE_MULTI_MODULE != 0 + && !cur_func->import_func_inst +#endif + ) + func_type = cur_func->u.func_import->func_type; + else + func_type = cur_func->u.func->func_type; + + /* types of each return value */ + ret_types = func_type->types + func_type->param_count; + ret_offset = prev_frame->ret_offset; + + for (ret_idx = 0, off = sizeof(int16) * (func_type->result_count - 1); + ret_idx < func_type->result_count; + ret_idx++, off -= sizeof(int16)) { + if (ret_types[ret_idx] == VALUE_TYPE_I64 + || ret_types[ret_idx] == VALUE_TYPE_F64) { + *((uint64 *)(prev_frame->lp + ret_offset)) = + GET_OPERAND(uint64, off); + ret_offset += 2; + } + else { + prev_frame->lp[ret_offset] = GET_OPERAND(int32, off); + ret_offset++; + } + } } goto return_func; @@ -1128,7 +1256,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, && !cur_func->import_func_inst #endif ) - cur_func_type = cur_func->u.func_import->func_type; + cur_func_type = cur_func->u.func_import->func_type; else cur_func_type = cur_func->u.func->func_type; if (!wasm_type_equal(cur_type, cur_func_type)) { @@ -2350,6 +2478,37 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, *(uint64*)(frame_lp + addr2) = *(uint64*)(frame_lp + addr1); HANDLE_OP_END (); + HANDLE_OP (EXT_OP_COPY_STACK_VALUES): + { + uint32 values_count, total_cell; + uint8 *cells; + int16 *src_offsets = NULL; + uint16 *dst_offsets = NULL; + + /* read values_count */ + values_count = *(uint32*)frame_ip; + frame_ip += sizeof(values_count); + /* read total cell num */ + total_cell = *(uint32*)frame_ip; + frame_ip += sizeof(total_cell); + /* cells */ + cells = (uint8 *)frame_ip; + frame_ip += values_count * sizeof(uint8); + /* src offsets */ + src_offsets = (int16 *)frame_ip; + frame_ip += values_count * sizeof(int16); + /* dst offsets */ + dst_offsets = (uint16*)frame_ip; + frame_ip += values_count * sizeof(uint16); + + if (!copy_stack_values(module, frame_lp, + values_count, total_cell, + cells, src_offsets, + dst_offsets)) + goto got_exception; + + HANDLE_OP_END (); + } HANDLE_OP (WASM_OP_SET_LOCAL): HANDLE_OP (WASM_OP_TEE_LOCAL): { @@ -2567,6 +2726,9 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP (WASM_OP_LOOP): HANDLE_OP (WASM_OP_END): HANDLE_OP (WASM_OP_NOP): + HANDLE_OP (EXT_OP_BLOCK): + HANDLE_OP (EXT_OP_LOOP): + HANDLE_OP (EXT_OP_IF): { wasm_set_exception(module, "WASM interp failed: unsupported opcode."); goto got_exception; @@ -2596,8 +2758,22 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } } frame_ip += cur_func->param_count * sizeof(int16); - if (cur_func->ret_cell_num != 0) + if (cur_func->ret_cell_num != 0) { + /* Get the first return value's offset. Since loader emit all return + * values' offset so we must skip remain return values' offsets. + */ + WASMType *func_type; + if (cur_func->is_import_func +#if WASM_ENABLE_MULTI_MODULE != 0 + && !cur_func->import_func_inst +#endif + ) + func_type = cur_func->u.func_import->func_type; + else + func_type = cur_func->u.func->func_type; frame->ret_offset = GET_OFFSET(); + frame_ip += 2 * (func_type->result_count - 1); + } SYNC_ALL_TO_FRAME(); prev_frame = frame; } @@ -2712,7 +2888,8 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMInterpFrame *frame, *outs_area; /* Allocate sufficient cells for all kinds of return values. */ - unsigned all_cell_num = 2, i; + unsigned all_cell_num = function->ret_cell_num > 2 ? + function->ret_cell_num : 2, i; /* This frame won't be used by JITed code, so only allocate interp frame here. */ unsigned frame_size = wasm_interp_interp_frame_size(all_cell_num); diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index b22cc8488..f846f8d2c 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -389,6 +389,7 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, { const uint8 *p = buf, *p_end = buf_end, *p_org; uint32 type_count, param_count, result_count, i, j; + uint32 param_cell_num, ret_cell_num; uint64 total_size; uint8 flag; WASMType *type; @@ -419,14 +420,16 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, CHECK_BUF(p, p_end, param_count); p += param_count; read_leb_uint32(p, p_end, result_count); - if (result_count > 1) { - set_error_buf(error_buf, error_buf_size, - "Load type section failed: invalid result arity."); - return false; - } CHECK_BUF(p, p_end, result_count); p = p_org; + if (param_count > UINT16_MAX || result_count > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "Load type section failed: " + "param count or result count too large"); + return false; + } + total_size = offsetof(WASMType, types) + sizeof(uint8) * (uint64)(param_count + result_count); if (!(type = module->types[i] = @@ -435,8 +438,8 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, } /* Resolve param types and result types */ - type->param_count = param_count; - type->result_count = result_count; + type->param_count = (uint16)param_count; + type->result_count = (uint16)result_count; for (j = 0; j < param_count; j++) { CHECK_BUF(p, p_end, 1); type->types[j] = read_uint8(p); @@ -446,6 +449,18 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, CHECK_BUF(p, p_end, 1); type->types[param_count + j] = read_uint8(p); } + + param_cell_num = wasm_get_cell_num(type->types, param_count); + ret_cell_num = wasm_get_cell_num(type->types + param_count, + result_count); + if (param_cell_num > UINT16_MAX || ret_cell_num > UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "Load type section failed: " + "param count or result count too large"); + return false; + } + type->param_cell_num = (uint16)param_cell_num; + type->ret_cell_num = (uint16)ret_cell_num; } } @@ -1742,8 +1757,8 @@ load_function_section(const uint8 *buf, const uint8 *buf_end, } } - func->param_cell_num = wasm_type_param_cell_num(func->func_type); - func->ret_cell_num = wasm_type_return_cell_num(func->func_type); + func->param_cell_num = func->func_type->param_cell_num; + func->ret_cell_num = func->func_type->ret_cell_num; func->local_cell_num = wasm_get_cell_num(func->local_types, func->local_count); @@ -2929,7 +2944,7 @@ bool wasm_loader_find_block_addr(BlockAddr *block_addr_cache, const uint8 *start_addr, const uint8 *code_end_addr, - uint8 block_type, + uint8 label_type, uint8 **p_else_addr, uint8 **p_end_addr, char *error_buf, @@ -2977,8 +2992,20 @@ wasm_loader_find_block_addr(BlockAddr *block_addr_cache, block_nested_depth++; break; + case EXT_OP_BLOCK: + case EXT_OP_LOOP: + case EXT_OP_IF: + /* block type */ + skip_leb_uint32(p, p_end); + if (block_nested_depth < sizeof(block_stack)/sizeof(BlockAddr)) { + block_stack[block_nested_depth].start_addr = p; + block_stack[block_nested_depth].else_addr = NULL; + } + block_nested_depth++; + break; + case WASM_OP_ELSE: - if (block_type == BLOCK_TYPE_IF && block_nested_depth == 1) + if (label_type == LABEL_TYPE_IF && block_nested_depth == 1) else_addr = (uint8*)(p - 1); if (block_nested_depth - 1 < sizeof(block_stack)/sizeof(BlockAddr)) block_stack[block_nested_depth - 1].else_addr = (uint8*)(p - 1); @@ -2986,7 +3013,7 @@ wasm_loader_find_block_addr(BlockAddr *block_addr_cache, case WASM_OP_END: if (block_nested_depth == 1) { - if (block_type == BLOCK_TYPE_IF) + if (label_type == LABEL_TYPE_IF) *p_else_addr = else_addr; *p_end_addr = (uint8*)(p - 1); @@ -3324,8 +3351,8 @@ typedef struct BranchBlockPatch { #endif typedef struct BranchBlock { - uint8 block_type; - uint8 return_type; + uint8 label_type; + BlockType block_type; uint8 *start_addr; uint8 *else_addr; uint8 *end_addr; @@ -3334,6 +3361,8 @@ typedef struct BranchBlock { uint16 dynamic_offset; uint8 *code_compiled; BranchBlockPatch *patch_list; + /* This is used to save params frame_offset of of if block */ + int16 *param_frame_offsets; #endif /* Indicate the operand stack is in polymorphic state. @@ -3710,14 +3739,14 @@ wasm_loader_push_pop_frame_ref(WASMLoaderContext *ctx, uint8 pop_cnt, } static bool -wasm_loader_push_frame_csp(WASMLoaderContext *ctx, uint8 type, - uint8 ret_type, uint8* start_addr, +wasm_loader_push_frame_csp(WASMLoaderContext *ctx, uint8 label_type, + BlockType block_type, uint8* start_addr, char *error_buf, uint32 error_buf_size) { CHECK_CSP_PUSH(); memset(ctx->frame_csp, 0, sizeof(BranchBlock)); - ctx->frame_csp->block_type = type; - ctx->frame_csp->return_type = ret_type; + ctx->frame_csp->label_type = label_type; + ctx->frame_csp->block_type = block_type; ctx->frame_csp->start_addr = start_addr; ctx->frame_csp->stack_cell_num = ctx->stack_cell_num; #if WASM_ENABLE_FAST_INTERP != 0 @@ -3738,45 +3767,18 @@ wasm_loader_pop_frame_csp(WASMLoaderContext *ctx, char *error_buf, uint32 error_buf_size) { CHECK_CSP_POP(); +#if WASM_ENABLE_FAST_INTERP != 0 + if ((ctx->frame_csp - 1)->param_frame_offsets) + wasm_runtime_free((ctx->frame_csp - 1)->param_frame_offsets); +#endif ctx->frame_csp--; ctx->csp_num--; + return true; fail: return false; } -static bool -wasm_loader_check_br(WASMLoaderContext *ctx, uint32 depth, - char *error_buf, uint32 error_buf_size) -{ - BranchBlock *target_block, *cur_block; - int32 available_stack_cell; - - if (ctx->csp_num < depth + 1) { - set_error_buf(error_buf, error_buf_size, - "WASM module load failed: unknown label, " - "unexpected end of section or function"); - return false; - } - - target_block = ctx->frame_csp - (depth + 1); - cur_block = ctx->frame_csp - 1; - - available_stack_cell = (int32) - (ctx->stack_cell_num - cur_block->stack_cell_num); - - if (available_stack_cell <= 0 && target_block->is_stack_polymorphic) - return true; - - if (target_block->block_type != BLOCK_TYPE_LOOP) { - uint8 type = target_block->return_type; - if (!check_stack_top_values(ctx->frame_ref, available_stack_cell, - type, error_buf, error_buf_size)) - return false; - } - return true; -} - #if WASM_ENABLE_FAST_INTERP != 0 #if WASM_ENABLE_ABS_LABEL_ADDR != 0 @@ -4101,24 +4103,61 @@ static bool wasm_loader_emit_br_info(WASMLoaderContext *ctx, BranchBlock *frame_csp, char *error_buf, uint32 error_buf_size) { - emit_operand(ctx, frame_csp->dynamic_offset); - if (frame_csp->block_type == BLOCK_TYPE_LOOP || - frame_csp->return_type == VALUE_TYPE_VOID) { - emit_byte(ctx, 0); - emit_operand(ctx, 0); - } - else if (frame_csp->return_type == VALUE_TYPE_I32 - || frame_csp->return_type == VALUE_TYPE_F32) { - emit_byte(ctx, 1); - emit_operand(ctx, *(int16*)(ctx->frame_offset - 1)); - } - else if (frame_csp->return_type == VALUE_TYPE_I64 - || frame_csp->return_type == VALUE_TYPE_F64) { - emit_byte(ctx, 2); - emit_operand(ctx, *(int16*)(ctx->frame_offset - 2)); + /* br info layout: + * a) arity of target block + * b) total cell num of arity values + * c) each arity value's cell num + * d) each arity value's src frame offset + * e) each arity values's dst dynamic offset + * f) branch target address + * + * Note: b-e are omitted when arity is 0 so that + * interpreter can recover the br info quickly. + */ + BlockType *block_type = &frame_csp->block_type; + uint8 *types = NULL, cell; + uint32 arity = 0; + int32 i; + int16 *frame_offset = ctx->frame_offset; + uint16 dynamic_offset; + + /* Note: loop's arity is different from if and block. loop's arity is + * its parameter count while if and block arity is result count. + */ + if (frame_csp->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(block_type, &types); + else + arity = block_type_get_result_types(block_type, &types); + + /* Part a */ + emit_uint32(ctx, arity); + + if (arity) { + /* Part b */ + emit_uint32(ctx, wasm_get_cell_num(types, arity)); + /* Part c */ + for (i = (int32)arity - 1; i >= 0; i--) { + cell = wasm_value_type_cell_num(types[i]); + emit_byte(ctx, cell); + } + /* Part d */ + for (i = (int32)arity - 1; i >= 0; i--) { + cell = wasm_value_type_cell_num(types[i]); + frame_offset -= cell; + emit_operand(ctx, *(int16*)(frame_offset)); + } + /* Part e */ + dynamic_offset = frame_csp->dynamic_offset + + wasm_get_cell_num(types, arity); + for (i = (int32)arity - 1; i >= 0; i--) { + cell = wasm_value_type_cell_num(types[i]); + dynamic_offset -= cell; + emit_operand(ctx, dynamic_offset); + } } - if (frame_csp->block_type == BLOCK_TYPE_LOOP) { + /* Part f */ + if (frame_csp->label_type == LABEL_TYPE_LOOP) { wasm_loader_emit_ptr(ctx, frame_csp->code_compiled); } else { @@ -4129,6 +4168,7 @@ wasm_loader_emit_br_info(WASMLoaderContext *ctx, BranchBlock *frame_csp, /* label address, to be patched */ wasm_loader_emit_ptr(ctx, NULL); } + return true; } @@ -4521,50 +4561,143 @@ fail: #if WASM_ENABLE_FAST_INTERP != 0 static bool -reserve_block_ret(WASMLoaderContext *loader_ctx, uint8 opcode, bool disable_emit, +reserve_block_ret(WASMLoaderContext *loader_ctx, + uint8 opcode, bool disable_emit, char *error_buf, uint32 error_buf_size) { int16 operand_offset = 0; - uint8 block_depth = 0; - if (opcode == WASM_OP_ELSE) - block_depth = 1; - else - block_depth = 0; + BranchBlock *block = (opcode == WASM_OP_ELSE) ? + loader_ctx->frame_csp - 1 : loader_ctx->frame_csp; + BlockType *block_type = &block->block_type; + uint8 *return_types = NULL; + uint32 return_count = 0, value_count = 0, total_cel_num = 0; + int32 i = 0; + int16 dynamic_offset, dynamic_offset_org, + *frame_offset = NULL, *frame_offset_org = NULL; - if ((loader_ctx->frame_csp - block_depth)->return_type != VALUE_TYPE_VOID) { - uint8 return_cells; - if ((loader_ctx->frame_csp - block_depth)->return_type == VALUE_TYPE_I32 - || (loader_ctx->frame_csp - block_depth)->return_type == VALUE_TYPE_F32) - return_cells = 1; - else - return_cells = 2; - if ((loader_ctx->frame_csp - block_depth)->dynamic_offset != - *(loader_ctx->frame_offset - return_cells)) { + return_count = block_type_get_result_types(block_type, &return_types); + /* If there is only one return value, use EXT_OP_COPY_STACK_TOP/_I64 instead + * of EXT_OP_COPY_STACK_VALUES for interpreter performance. */ + if (return_count == 1) { + uint8 cell = wasm_value_type_cell_num(return_types[0]); + if (block->dynamic_offset != *(loader_ctx->frame_offset - cell)) { /* insert op_copy before else opcode */ if (opcode == WASM_OP_ELSE) skip_label(); - - if (return_cells == 1) - emit_label(EXT_OP_COPY_STACK_TOP); - else - emit_label(EXT_OP_COPY_STACK_TOP_I64); - emit_operand(loader_ctx, *(loader_ctx->frame_offset - return_cells)); - emit_operand(loader_ctx, (loader_ctx->frame_csp - block_depth)->dynamic_offset); + emit_label(cell == 1 ? EXT_OP_COPY_STACK_TOP : EXT_OP_COPY_STACK_TOP_I64); + emit_operand(loader_ctx, *(loader_ctx->frame_offset - cell)); + emit_operand(loader_ctx, block->dynamic_offset); if (opcode == WASM_OP_ELSE) { - *(loader_ctx->frame_offset - return_cells) = - (loader_ctx->frame_csp - block_depth)->dynamic_offset; + *(loader_ctx->frame_offset - cell) = block->dynamic_offset; } else { - loader_ctx->frame_offset -= return_cells; - loader_ctx->dynamic_offset = loader_ctx->frame_csp->dynamic_offset; - PUSH_OFFSET_TYPE((loader_ctx->frame_csp - block_depth)->return_type); + loader_ctx->frame_offset -= cell; + loader_ctx->dynamic_offset = block->dynamic_offset; + PUSH_OFFSET_TYPE(return_types[0]); wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); } if (opcode == WASM_OP_ELSE) emit_label(opcode); } + return true; + } + + /* Copy stack top values to block's results which are in dynamic space. + * The instruction format: + * Part a: values count + * Part b: all values total cell num + * Part c: each value's cell_num, src offset and dst offset + * Part d: each value's src offset and dst offset + * Part e: each value's dst offset + */ + frame_offset = frame_offset_org = loader_ctx->frame_offset; + dynamic_offset = dynamic_offset_org = + block->dynamic_offset + + wasm_get_cell_num(return_types, return_count); + + /* First traversal to get the count of values needed to be copied. */ + for (i = (int32)return_count - 1; i >= 0; i--) { + uint8 cells = wasm_value_type_cell_num(return_types[i]); + + frame_offset -= cells; + dynamic_offset -= cells; + if (dynamic_offset != *frame_offset) { + value_count++; + total_cel_num += cells; + } + } + + if (value_count) { + uint32 j = 0; + uint8 *emit_data = NULL, *cells = NULL; + int16 *src_offsets = NULL; + uint16 *dst_offsets = NULL; + uint64 size = (uint64)value_count * (sizeof(*cells) + + sizeof(*src_offsets) + + sizeof(*dst_offsets)); + + /* Allocate memory for the emit data */ + if (!(emit_data = loader_malloc(size, error_buf, error_buf_size))) + return false; + + cells = emit_data; + src_offsets = (int16 *)(cells + value_count); + dst_offsets = (uint16 *)(src_offsets + value_count); + + /* insert op_copy before else opcode */ + if (opcode == WASM_OP_ELSE) + skip_label(); + emit_label(EXT_OP_COPY_STACK_VALUES); + /* Part a) */ + emit_uint32(loader_ctx, value_count); + /* Part b) */ + emit_uint32(loader_ctx, total_cel_num); + + /* Second traversal to get each value's cell num, src offset and dst offset. */ + frame_offset = frame_offset_org; + dynamic_offset = dynamic_offset_org; + for (i = (int32)return_count - 1, j = 0; i >= 0; i--) { + uint8 cell = wasm_value_type_cell_num(return_types[i]); + frame_offset -= cell; + dynamic_offset -= cell; + if (dynamic_offset != *frame_offset) { + /* cell num */ + cells[j] = cell; + /* src offset */ + src_offsets[j] = *frame_offset; + /* dst offset */ + dst_offsets[j] = dynamic_offset; + j++; + } + if (opcode == WASM_OP_ELSE) { + *frame_offset = dynamic_offset; + } + else { + loader_ctx->frame_offset = frame_offset; + loader_ctx->dynamic_offset = dynamic_offset; + PUSH_OFFSET_TYPE(return_types[i]); + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + loader_ctx->frame_offset = frame_offset_org; + loader_ctx->dynamic_offset = dynamic_offset_org; + } + } + + bh_assert(j == value_count); + + /* Emit the cells, src_offsets and dst_offsets */ + for (j = 0; j < value_count; j++) + emit_byte(loader_ctx, cells[j]); + for (j = 0; j < value_count; j++) + emit_operand(loader_ctx, src_offsets[j]); + for (j = 0; j < value_count; j++) + emit_operand(loader_ctx, dst_offsets[j]); + + if (opcode == WASM_OP_ELSE) + emit_label(opcode); + + wasm_runtime_free(emit_data); } return true; @@ -4572,7 +4705,6 @@ reserve_block_ret(WASMLoaderContext *loader_ctx, uint8 opcode, bool disable_emit fail: return false; } - #endif /* WASM_ENABLE_FAST_INTERP */ #define RESERVE_BLOCK_RET() do { \ @@ -4593,11 +4725,11 @@ fail: goto fail; \ } while (0) -#define PUSH_CSP(type, ret_type, _start_addr) do { \ - if (!wasm_loader_push_frame_csp(loader_ctx, type, ret_type, \ - _start_addr, error_buf, \ - error_buf_size)) \ - goto fail; \ +#define PUSH_CSP(label_type, block_type, _start_addr) do { \ + if (!wasm_loader_push_frame_csp(loader_ctx, label_type, block_type, \ + _start_addr, error_buf, \ + error_buf_size)) \ + goto fail; \ } while (0) #define POP_CSP() do { \ @@ -4663,7 +4795,7 @@ check_memory_access_align(uint8 opcode, uint32 align, } static bool -is_block_type_valid(uint8 type) +is_value_type(uint8 type) { return type == VALUE_TYPE_I32 || type == VALUE_TYPE_I64 || @@ -4672,13 +4804,77 @@ is_block_type_valid(uint8 type) type == VALUE_TYPE_VOID; } -#define CHECK_BLOCK_TYPE(type) do { \ - if (!is_block_type_valid(type)) { \ - set_error_buf(error_buf, error_buf_size, \ - "WASM module load failed: invalid block type"); \ - goto fail; \ - } \ - } while (0) +static bool +wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, + char *error_buf, uint32 error_buf_size) +{ + BranchBlock *target_block, *cur_block; + BlockType *target_block_type; + uint8 *types = NULL, *frame_ref; + uint32 arity = 0; + int32 i, available_stack_cell; + uint16 cell_num; + + if (loader_ctx->csp_num < depth + 1) { + set_error_buf(error_buf, error_buf_size, + "WASM module load failed: unknown label, " + "unexpected end of section or function"); + return false; + } + + cur_block = loader_ctx->frame_csp - 1; + target_block = loader_ctx->frame_csp - (depth + 1); + target_block_type = &target_block->block_type; + frame_ref = loader_ctx->frame_ref; + + /* Note: loop's arity is different from if and block. loop's arity is + * its parameter count while if and block arity is result count. + */ + if (target_block->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(target_block_type, &types); + else + arity = block_type_get_result_types(target_block_type, &types); + + /* If the stack is in polymorphic state, just clear the stack + * and then re-push the values to make the stack top values + * match block type. */ + if (cur_block->is_stack_polymorphic) { + for (i = (int32)arity -1; i >= 0; i--) { +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(types[i]); +#endif + POP_TYPE(types[i]); + } + for (i = 0; i < (int32)arity; i++) { +#if WASM_ENABLE_FAST_INTERP != 0 + bool disable_emit = true; + int16 operand_offset = 0; + PUSH_OFFSET_TYPE(types[i]); +#endif + PUSH_TYPE(types[i]); + } + return true; + } + + available_stack_cell = (int32) + (loader_ctx->stack_cell_num - cur_block->stack_cell_num); + + /* Check stack top values match target block type */ + for (i = (int32)arity -1; i >= 0; i--) { + if (!check_stack_top_values(frame_ref, available_stack_cell, + types[i], + error_buf, error_buf_size)) + return false; + cell_num = wasm_value_type_cell_num(types[i]); + frame_ref -= cell_num; + available_stack_cell -= cell_num; + } + + return true; + +fail: + return false; +} static BranchBlock * check_branch_block(WASMLoaderContext *loader_ctx, @@ -4703,94 +4899,174 @@ fail: } static bool -check_branch_block_ret(WASMLoaderContext *loader_ctx, - BranchBlock *frame_csp_tmp, - char *error_buf, uint32 error_buf_size) +check_block_stack(WASMLoaderContext *loader_ctx, BranchBlock *block, + char *error_buf, uint32 error_buf_size) { + BlockType *block_type = &block->block_type; + uint8 *return_types = NULL; + uint32 return_count = 0; + int32 available_stack_cell, return_cell_num, i; + uint8 *frame_ref = NULL; + + available_stack_cell = (int32) + (loader_ctx->stack_cell_num + - block->stack_cell_num); + + return_count = block_type_get_result_types(block_type, &return_types); + return_cell_num = return_count > 0 ? + wasm_get_cell_num(return_types, return_count) : 0; + + /* If the stack is in polymorphic state, just clear the stack + * and then re-push the values to make the stack top values + * match block type. */ + if (block->is_stack_polymorphic) { + for (i = (int32)return_count -1; i >= 0; i--) { #if WASM_ENABLE_FAST_INTERP != 0 - BranchBlock *cur_block = loader_ctx->frame_csp - 1; - bool disable_emit = true; - int16 operand_offset = 0; + POP_OFFSET_TYPE(return_types[i]); #endif - if (frame_csp_tmp->block_type != BLOCK_TYPE_LOOP) { - uint8 block_return_type = frame_csp_tmp->return_type; -#if WASM_ENABLE_FAST_INTERP != 0 - /* If the stack is in polymorphic state, do fake pop and push on - offset stack to keep the depth of offset stack to be the same - with ref stack */ - if (cur_block->is_stack_polymorphic) { - POP_OFFSET_TYPE(block_return_type); - PUSH_OFFSET_TYPE(block_return_type); + POP_TYPE(return_types[i]); } + + /* Check stack is empty */ + if (loader_ctx->stack_cell_num != block->stack_cell_num) { + set_error_buf(error_buf, error_buf_size, + "WASM module load failed: " + "type mismatch: stack size does not match block type"); + goto fail; + } + + for (i = 0; i < (int32)return_count; i++) { +#if WASM_ENABLE_FAST_INTERP != 0 + bool disable_emit = true; + int16 operand_offset = 0; + PUSH_OFFSET_TYPE(return_types[i]); #endif - POP_TYPE(block_return_type); - PUSH_TYPE(block_return_type); + PUSH_TYPE(return_types[i]); + } + return true; } + + /* Check stack cell num equals return cell num */ + if (available_stack_cell != return_cell_num) { + set_error_buf(error_buf, error_buf_size, + "WASM module load failed: " + "type mismatch: stack size does not match block type"); + goto fail; + } + + /* Check stack values match return types */ + frame_ref = loader_ctx->frame_ref; + for (i = (int32)return_count -1; i >= 0; i--) { + if (!check_stack_top_values(frame_ref, available_stack_cell, + return_types[i], + error_buf, error_buf_size)) + return false; + frame_ref -= wasm_value_type_cell_num(return_types[i]); + available_stack_cell -= wasm_value_type_cell_num(return_types[i]); + } + return true; + fail: return false; } +#if WASM_ENABLE_FAST_INTERP != 0 +/* Copy parameters to dynamic space. + * 1) POP original parameter out; + * 2) Push and copy original values to dynamic space. + * The copy instruction format: + * Part a: param count + * Part b: all param total cell num + * Part c: each param's cell_num, src offset and dst offset + * Part d: each param's src offset + * Part e: each param's dst offset + */ static bool -check_block_stack(WASMLoaderContext *ctx, BranchBlock *block, - char *error_buf, uint32 error_buf_size) +copy_params_to_dynamic_space(WASMLoaderContext *loader_ctx, bool is_if_block, + char* error_buf, uint32 error_buf_size) { - uint8 type = block->return_type; - int32 available_stack_cell = (int32) - (ctx->stack_cell_num - block->stack_cell_num); + int16 *frame_offset = NULL; + uint8 *cells = NULL, cell; + int16 *src_offsets = NULL; + uint8 *emit_data = NULL; + uint32 i; + BranchBlock *block = loader_ctx->frame_csp - 1; + BlockType *block_type = &block->block_type; + WASMType *wasm_type = block_type->u.type; + uint32 param_count = block_type->u.type->param_count; + int16 condition_offset = 0; + bool disable_emit = false; + int16 operand_offset = 0; - if (type != VALUE_TYPE_VOID - && available_stack_cell <= 0 - && block->is_stack_polymorphic) { - if (!(wasm_loader_push_frame_ref(ctx, type, error_buf, error_buf_size)) -#if WASM_ENABLE_FAST_INTERP != 0 - || !(wasm_loader_push_frame_offset(ctx, type, true, 0, error_buf, error_buf_size)) -#endif - ) - return false; - return true; - } + uint64 size = (uint64)param_count * (sizeof(*cells) + + sizeof(*src_offsets)); - if (type != VALUE_TYPE_VOID - && available_stack_cell == 1 - && *(ctx->frame_ref - 1) == VALUE_TYPE_ANY) { - if (type == VALUE_TYPE_I32 || type == VALUE_TYPE_F32) { - /* If the stack top is a value of any type, change its type to the - * same as block return type and return success */ - *(ctx->frame_ref - 1) = type; - } - else { - if (!(wasm_loader_push_frame_ref(ctx, VALUE_TYPE_I32, - error_buf, error_buf_size)) -#if WASM_ENABLE_FAST_INTERP != 0 - || !(wasm_loader_push_frame_offset(ctx, VALUE_TYPE_I32, - true, 0, - error_buf, error_buf_size)) -#endif - ) - return false; - *(ctx->frame_ref - 1) = *(ctx->frame_ref - 2) = type; - } - return true; - } + /* For if block, we also need copy the condition operand offset. */ + if (is_if_block) + size += sizeof(*cells) + sizeof(*src_offsets); - if (((type == VALUE_TYPE_I32 || type == VALUE_TYPE_F32) - && available_stack_cell != 1) - || ((type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64) - && available_stack_cell != 2) - || (type == VALUE_TYPE_VOID && available_stack_cell > 0)) { - set_error_buf(error_buf, error_buf_size, - "WASM module load failed: " - "type mismatch: stack size does not match block type"); + /* Allocate memory for the emit data */ + if (!(emit_data = loader_malloc(size, error_buf, error_buf_size))) return false; + + cells = emit_data; + src_offsets = (int16 *)(cells + param_count); + + if (is_if_block) + condition_offset = *loader_ctx->frame_offset; + + /* POP original parameter out */ + for (i = 0; i < param_count; i++) { + POP_OFFSET_TYPE(wasm_type->types[param_count - i - 1]); + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + } + frame_offset = loader_ctx->frame_offset; + + /* Get each param's cell num and src offset */ + for (i = 0; i < param_count; i++) { + cell = wasm_value_type_cell_num(wasm_type->types[i]); + cells[i] = cell; + src_offsets[i] = *frame_offset; + frame_offset += cell; } - if (!check_stack_top_values(ctx->frame_ref, available_stack_cell, - type, error_buf, error_buf_size)) - return false; + /* emit copy instruction */ + emit_label(EXT_OP_COPY_STACK_VALUES); + /* Part a) */ + emit_uint32(loader_ctx, is_if_block ? param_count + 1 : param_count); + /* Part b) */ + emit_uint32(loader_ctx, is_if_block ? + wasm_type->param_cell_num + 1 : + wasm_type->param_cell_num); + /* Part c) */ + for (i = 0; i < param_count; i++) + emit_byte(loader_ctx, cells[i]); + if (is_if_block) + emit_byte(loader_ctx, 1); + + /* Part d) */ + for (i = 0; i < param_count; i++) + emit_operand(loader_ctx, src_offsets[i]); + if (is_if_block) + emit_operand(loader_ctx, condition_offset); + + /* Part e) */ + /* Push to dynamic space. The push will emit the dst offset. */ + for (i = 0; i < param_count; i++) + PUSH_OFFSET_TYPE(wasm_type->types[i]); + if (is_if_block) + PUSH_OFFSET_TYPE(VALUE_TYPE_I32); + + /* Free the emit data */ + wasm_runtime_free(emit_data); return true; + +fail: + return false; } +#endif /* reset the stack to the state of before entering the last block */ #if WASM_ENABLE_FAST_INTERP != 0 @@ -4817,6 +5093,9 @@ check_block_stack(WASMLoaderContext *ctx, BranchBlock *block, cur_block->is_stack_polymorphic = flag; \ } while (0) +#define BLOCK_HAS_PARAM(block_type) \ + (!block_type.is_value_type && block_type.u.type->param_count > 0) + static bool wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, BlockAddr *block_addr_cache, @@ -4824,12 +5103,13 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, { uint8 *p = func->code, *p_end = func->code + func->code_size, *p_org; uint32 param_count, local_count, global_count; - uint8 *param_types, ret_type, *local_types, local_type, global_type; + uint8 *param_types, *local_types, local_type, global_type; + BlockType func_type; uint16 *local_offsets, local_offset; uint32 count, i, local_idx, global_idx, u32, align, mem_offset; int32 i32, i32_const = 0; int64 i64; - uint8 opcode, u8, block_return_type; + uint8 opcode, u8; bool return_value = false; WASMLoaderContext *loader_ctx; BranchBlock *frame_csp_tmp; @@ -4854,8 +5134,9 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, param_count = func->func_type->param_count; param_types = func->func_type->types; - ret_type = func->func_type->result_count - ? param_types[param_count] : VALUE_TYPE_VOID; + + func_type.is_value_type = false; + func_type.u.type = func->func_type; local_count = func->local_count; local_types = func->local_types; @@ -4882,7 +5163,7 @@ re_scan: } #endif - PUSH_CSP(BLOCK_TYPE_FUNCTION, ret_type, p); + PUSH_CSP(LABEL_TYPE_FUNCTION, func_type, p); while (p < p_end) { opcode = *p++; @@ -4904,43 +5185,142 @@ re_scan: #endif break; - case WASM_OP_BLOCK: - /* 0x40/0x7F/0x7E/0x7D/0x7C */ - block_return_type = read_uint8(p); - CHECK_BLOCK_TYPE(block_return_type); - PUSH_CSP(BLOCK_TYPE_BLOCK, block_return_type, p); -#if WASM_ENABLE_FAST_INTERP != 0 - skip_label(); -#endif - break; - - case WASM_OP_LOOP: - /* 0x40/0x7F/0x7E/0x7D/0x7C */ - block_return_type = read_uint8(p); - CHECK_BLOCK_TYPE(block_return_type); - PUSH_CSP(BLOCK_TYPE_LOOP, block_return_type, p); -#if WASM_ENABLE_FAST_INTERP != 0 - skip_label(); - (loader_ctx->frame_csp - 1)->code_compiled = - loader_ctx->p_code_compiled; -#endif - break; - case WASM_OP_IF: POP_I32(); - /* 0x40/0x7F/0x7E/0x7D/0x7C */ - block_return_type = read_uint8(p); - CHECK_BLOCK_TYPE(block_return_type); - PUSH_CSP(BLOCK_TYPE_IF, block_return_type, p); + goto handle_op_block_and_loop; + case WASM_OP_BLOCK: + case WASM_OP_LOOP: +handle_op_block_and_loop: + { + uint8 value_type; + BlockType block_type; + + value_type = read_uint8(p); + if (is_value_type(value_type)) { + /* If the first byte is one of these special values: + * 0x40/0x7F/0x7E/0x7D/0x7C, take it as the type of + * the single return value. */ + block_type.is_value_type = true; + block_type.u.value_type = value_type; + } + else { + uint32 type_index; + /* Resolve the leb128 encoded type index as block type */ + p--; + read_leb_uint32(p, p_end, type_index); + if (type_index >= module->type_count) { + set_error_buf(error_buf, error_buf_size, + "WASM loader prepare bytecode failed: " + "unknown type"); + goto fail; + } + block_type.is_value_type = false; + block_type.u.type = module->types[type_index]; +#if WASM_ENABLE_FAST_INTERP == 0 \ + && WASM_ENABLE_WAMR_COMPILER == 0 \ + && WASM_ENABLE_JIT == 0 + /* If block use type index as block type, change the opcode + * to new extended opcode so that interpreter can resolve the + * block quickly. + */ + *(p - 2) = EXT_OP_BLOCK + (opcode - WASM_OP_BLOCK); +#endif + } + + /* Pop block parameters from stack */ + if (BLOCK_HAS_PARAM(block_type)) { + WASMType *wasm_type = block_type.u.type; + for (i = 0; i < block_type.u.type->param_count; i++) + POP_TYPE(wasm_type->types[wasm_type->param_count - i - 1]); + } + + PUSH_CSP(LABEL_TYPE_BLOCK + (opcode - WASM_OP_BLOCK), block_type, p); + + /* Pass parameters to block */ + if (BLOCK_HAS_PARAM(block_type)) { + for (i = 0; i < block_type.u.type->param_count; i++) + PUSH_TYPE(block_type.u.type->types[i]); + } + #if WASM_ENABLE_FAST_INTERP != 0 - emit_empty_label_addr_and_frame_ip(PATCH_ELSE); - emit_empty_label_addr_and_frame_ip(PATCH_END); + if (opcode == WASM_OP_BLOCK) { + skip_label(); + } else if (opcode == WASM_OP_LOOP) { + skip_label(); + if (BLOCK_HAS_PARAM(block_type)) { + /* Make sure params are in dynamic space */ + if (!copy_params_to_dynamic_space(loader_ctx, + false, + error_buf, + error_buf_size)) + goto fail; + } + (loader_ctx->frame_csp - 1)->code_compiled = + loader_ctx->p_code_compiled; + } else if (opcode == WASM_OP_IF) { + /* If block has parameters, we should make sure they are in + * dynamic space. Otherwise, when else branch is missing, + * the later opcode may consume incorrect operand offset. + * Spec case: + * (func (export "params-id") (param i32) (result i32) + * (i32.const 1) + * (i32.const 2) + * (if (param i32 i32) (result i32 i32) (local.get 0) (then)) + * (i32.add) + * ) + * + * So we should emit a copy instruction before the if. + * + * And we also need to save the parameter offsets and + * recover them before entering else branch. + * + */ + if (BLOCK_HAS_PARAM(block_type)) { + BranchBlock *block = loader_ctx->frame_csp - 1; + uint64 size; + + /* skip the if condition operand offset */ + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + /* skip the if label */ + skip_label(); + /* Emit a copy instruction */ + if (!copy_params_to_dynamic_space(loader_ctx, + true, + error_buf, + error_buf_size)) + goto fail; + + /* Emit the if instruction */ + emit_label(opcode); + /* Emit the new condition operand offset */ + POP_OFFSET_TYPE(VALUE_TYPE_I32); + + /* Save top param_count values of frame_offset stack, so that + * we can recover it before executing else branch */ + size = sizeof(int16) * + (uint64)block_type.u.type->param_cell_num; + if (!(block->param_frame_offsets = + loader_malloc(size, error_buf, error_buf_size))) + goto fail; + bh_memcpy_s(block->param_frame_offsets, + size, + loader_ctx->frame_offset - size/sizeof(int16), + size); + } + + emit_empty_label_addr_and_frame_ip(PATCH_ELSE); + emit_empty_label_addr_and_frame_ip(PATCH_END); + } #endif break; + } case WASM_OP_ELSE: + { + BlockType block_type = (loader_ctx->frame_csp - 1)->block_type; + if (loader_ctx->csp_num < 2 - || (loader_ctx->frame_csp - 1)->block_type != BLOCK_TYPE_IF) { + || (loader_ctx->frame_csp - 1)->label_type != LABEL_TYPE_IF) { set_error_buf(error_buf, error_buf_size, "WASM loader prepare bytecode failed: " "opcode else found without matched opcode if"); @@ -4963,25 +5343,63 @@ re_scan: #endif RESET_STACK(); SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(false); + + /* Pass parameters to if-false branch */ + if (BLOCK_HAS_PARAM(block_type)) { + for (i = 0; i < block_type.u.type->param_count; i++) + PUSH_TYPE(block_type.u.type->types[i]); + } + +#if WASM_ENABLE_FAST_INTERP != 0 + /* Recover top param_count values of frame_offset stack */ + if (BLOCK_HAS_PARAM((block_type))) { + uint32 size; + BranchBlock *block = loader_ctx->frame_csp - 1; + size = sizeof(int16) * + block_type.u.type->param_cell_num; + bh_memcpy_s(loader_ctx->frame_offset, size, + block->param_frame_offsets, size); + loader_ctx->frame_offset += (size/sizeof(int16)); + } +#endif + break; + } case WASM_OP_END: { + BranchBlock *cur_block = loader_ctx->frame_csp - 1; /* check whether block stack matches its result type */ - if (!check_block_stack(loader_ctx, loader_ctx->frame_csp - 1, + if (!check_block_stack(loader_ctx, cur_block, error_buf, error_buf_size)) goto fail; - /* if has return value, but no else branch, fail */ - if ((loader_ctx->frame_csp - 1)->block_type == BLOCK_TYPE_IF - && (loader_ctx->frame_csp - 1)->return_type != VALUE_TYPE_VOID - && !(loader_ctx->frame_csp - 1)->else_addr) { - set_error_buf(error_buf, error_buf_size, - "WASM module load failed: " - "type mismatch: if has return value and else is missing"); - - goto fail; + /* if no else branch, and return types do not match param types, fail */ + if (cur_block->label_type == LABEL_TYPE_IF + && !cur_block->else_addr) { + uint32 param_count = 0, ret_count = 0; + uint8 *param_types = NULL, *ret_types = NULL; + BlockType *block_type = &cur_block->block_type; + if (block_type->is_value_type) { + if (block_type->u.value_type != VALUE_TYPE_VOID) { + ret_count = 1; + ret_types = &block_type->u.value_type; + } + } + else { + param_count = block_type->u.type->param_count; + ret_count = block_type->u.type->result_count; + param_types = block_type->u.type->types; + ret_types = block_type->u.type->types + param_count; + } + if (param_count != ret_count + || (param_count && memcmp(param_types, ret_types, param_count))) { + set_error_buf(error_buf, error_buf_size, + "WASM module load failed: " + "type mismatch: else branch missing"); + goto fail; + } } POP_CSP(); @@ -4993,9 +5411,17 @@ re_scan: apply_label_patch(loader_ctx, 0, PATCH_END); free_label_patch_list(loader_ctx->frame_csp); - if (loader_ctx->frame_csp->block_type == BLOCK_TYPE_FUNCTION) { + if (loader_ctx->frame_csp->label_type == LABEL_TYPE_FUNCTION) { + int32 idx; + uint8 ret_type; + emit_label(WASM_OP_RETURN); - POP_OFFSET_TYPE(loader_ctx->frame_csp->return_type); + for (idx = (int32)func->func_type->result_count - 1; + idx >= 0; idx--) { + ret_type = *(func->func_type->types + + func->func_type->param_count + idx); + POP_OFFSET_TYPE(ret_type); + } } #endif if (loader_ctx->csp_num > 0) { @@ -5019,10 +5445,6 @@ re_scan: error_buf, error_buf_size))) goto fail; - if (!check_branch_block_ret(loader_ctx, frame_csp_tmp, - error_buf, error_buf_size)) - goto fail; - RESET_STACK(); SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); break; @@ -5036,16 +5458,13 @@ re_scan: error_buf, error_buf_size))) goto fail; - if (!check_branch_block_ret(loader_ctx, frame_csp_tmp, - error_buf, error_buf_size)) - goto fail; - break; } case WASM_OP_BR_TABLE: { - uint8 ret_type; + uint8 *ret_types = NULL; + uint32 ret_count = 0; read_leb_uint32(p, p_end, count); #if WASM_ENABLE_FAST_INTERP != 0 @@ -5055,23 +5474,30 @@ re_scan: /* TODO: check the const */ for (i = 0; i <= count; i++) { - if (!(frame_csp_tmp = check_branch_block(loader_ctx, &p, p_end, - error_buf, error_buf_size))) - goto fail; - - if (!check_branch_block_ret(loader_ctx, frame_csp_tmp, - error_buf, error_buf_size)) + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, + error_buf, error_buf_size))) goto fail; if (i == 0) { - ret_type = frame_csp_tmp->block_type == BLOCK_TYPE_LOOP ? - VALUE_TYPE_VOID : frame_csp_tmp->return_type; + if (frame_csp_tmp->label_type != LABEL_TYPE_LOOP) + ret_count = + block_type_get_result_types(&frame_csp_tmp->block_type, + &ret_types); } else { + uint8 *tmp_ret_types = NULL; + uint32 tmp_ret_count = 0; + /* Check whether all table items have the same return type */ - uint8 tmp_ret_type = frame_csp_tmp->block_type == BLOCK_TYPE_LOOP ? - VALUE_TYPE_VOID : frame_csp_tmp->return_type; - if (ret_type != tmp_ret_type) { + if (frame_csp_tmp->label_type != LABEL_TYPE_LOOP) + tmp_ret_count = + block_type_get_result_types(&frame_csp_tmp->block_type, + &tmp_ret_types); + + if (ret_count != tmp_ret_count + || (ret_count + && 0 != memcmp(ret_types, tmp_ret_types, ret_count))) { set_error_buf(error_buf, error_buf_size, "WASM loader prepare bytecode failed: " "type mismatch: br_table targets must " @@ -5088,16 +5514,21 @@ re_scan: case WASM_OP_RETURN: { - POP_TYPE(ret_type); - PUSH_TYPE(ret_type); - + int32 idx; + uint8 ret_type; + for (idx = (int32)func->func_type->result_count - 1; idx >= 0; idx--) { + ret_type = *(func->func_type->types + + func->func_type->param_count + idx); + POP_TYPE(ret_type); #if WASM_ENABLE_FAST_INTERP != 0 - // emit the offset after return opcode - POP_OFFSET_TYPE(ret_type); + // emit the offset after return opcode + POP_OFFSET_TYPE(ret_type); #endif + } RESET_STACK(); SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + break; } @@ -5135,10 +5566,14 @@ re_scan: } } - if (func_type->result_count) { - PUSH_TYPE(func_type->types[func_type->param_count]); + for (i = 0; i < func_type->result_count; i++) { + PUSH_TYPE(func_type->types[func_type->param_count + i]); #if WASM_ENABLE_FAST_INTERP != 0 - PUSH_OFFSET_TYPE(func_type->types[func_type->param_count]); + /* Here we emit each return value's dynamic_offset. But in fact + * these offsets are continuous, so interpreter only need to get + * the first return value's offset. + */ + PUSH_OFFSET_TYPE(func_type->types[func_type->param_count + i]); #endif } @@ -5194,10 +5629,10 @@ re_scan: } } - if (func_type->result_count > 0) { - PUSH_TYPE(func_type->types[func_type->param_count]); + for (i = 0; i < func_type->result_count; i++) { + PUSH_TYPE(func_type->types[func_type->param_count + i]); #if WASM_ENABLE_FAST_INTERP != 0 - PUSH_OFFSET_TYPE(func_type->types[func_type->param_count]); + PUSH_OFFSET_TYPE(func_type->types[func_type->param_count + i]); #endif } diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index d969c2203..b56aa720d 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -284,6 +284,7 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, { const uint8 *p = buf, *p_end = buf_end, *p_org; uint32 type_count, param_count, result_count, i, j; + uint32 param_cell_num, ret_cell_num; uint64 total_size; uint8 flag; WASMType *type; @@ -310,10 +311,11 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, CHECK_BUF(p, p_end, param_count); p += param_count; read_leb_uint32(p, p_end, result_count); - bh_assert(result_count <= 1); CHECK_BUF(p, p_end, result_count); p = p_org; + bh_assert(param_count <= UINT16_MAX && result_count <= UINT16_MAX); + total_size = offsetof(WASMType, types) + sizeof(uint8) * (uint64)(param_count + result_count); if (!(type = module->types[i] = @@ -322,8 +324,8 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, } /* Resolve param types and result types */ - type->param_count = param_count; - type->result_count = result_count; + type->param_count = (uint16)param_count; + type->result_count = (uint16)result_count; for (j = 0; j < param_count; j++) { CHECK_BUF(p, p_end, 1); type->types[j] = read_uint8(p); @@ -333,6 +335,13 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, CHECK_BUF(p, p_end, 1); type->types[param_count + j] = read_uint8(p); } + + param_cell_num = wasm_get_cell_num(type->types, param_count); + ret_cell_num = wasm_get_cell_num(type->types + param_count, + result_count); + bh_assert(param_cell_num <= UINT16_MAX && ret_cell_num <= UINT16_MAX); + type->param_cell_num = (uint16)param_cell_num; + type->ret_cell_num = (uint16)ret_cell_num; } } @@ -953,8 +962,8 @@ load_function_section(const uint8 *buf, const uint8 *buf_end, } } - func->param_cell_num = wasm_type_param_cell_num(func->func_type); - func->ret_cell_num = wasm_type_return_cell_num(func->func_type); + func->param_cell_num = func->func_type->param_cell_num; + func->ret_cell_num = func->func_type->ret_cell_num; func->local_cell_num = wasm_get_cell_num(func->local_types, func->local_count); @@ -1958,7 +1967,7 @@ bool wasm_loader_find_block_addr(BlockAddr *block_addr_cache, const uint8 *start_addr, const uint8 *code_end_addr, - uint8 block_type, + uint8 label_type, uint8 **p_else_addr, uint8 **p_end_addr, char *error_buf, @@ -2006,8 +2015,20 @@ wasm_loader_find_block_addr(BlockAddr *block_addr_cache, block_nested_depth++; break; + case EXT_OP_BLOCK: + case EXT_OP_LOOP: + case EXT_OP_IF: + /* block type */ + skip_leb_uint32(p, p_end); + if (block_nested_depth < sizeof(block_stack)/sizeof(BlockAddr)) { + block_stack[block_nested_depth].start_addr = p; + block_stack[block_nested_depth].else_addr = NULL; + } + block_nested_depth++; + break; + case WASM_OP_ELSE: - if (block_type == BLOCK_TYPE_IF && block_nested_depth == 1) + if (label_type == LABEL_TYPE_IF && block_nested_depth == 1) else_addr = (uint8*)(p - 1); if (block_nested_depth - 1 < sizeof(block_stack)/sizeof(BlockAddr)) block_stack[block_nested_depth - 1].else_addr = (uint8*)(p - 1); @@ -2015,7 +2036,7 @@ wasm_loader_find_block_addr(BlockAddr *block_addr_cache, case WASM_OP_END: if (block_nested_depth == 1) { - if (block_type == BLOCK_TYPE_IF) + if (label_type == LABEL_TYPE_IF) *p_else_addr = else_addr; *p_end_addr = (uint8*)(p - 1); @@ -2347,8 +2368,8 @@ typedef struct BranchBlockPatch { #endif typedef struct BranchBlock { - uint8 block_type; - uint8 return_type; + uint8 label_type; + BlockType block_type; uint8 *start_addr; uint8 *else_addr; uint8 *end_addr; @@ -2357,6 +2378,8 @@ typedef struct BranchBlock { uint16 dynamic_offset; uint8 *code_compiled; BranchBlockPatch *patch_list; + /* This is used to save params frame_offset of of if block */ + int16 *param_frame_offsets; #endif /* Indicate the operand stack is in polymorphic state. @@ -2714,14 +2737,14 @@ wasm_loader_push_pop_frame_ref(WASMLoaderContext *ctx, uint8 pop_cnt, } static bool -wasm_loader_push_frame_csp(WASMLoaderContext *ctx, uint8 type, - uint8 ret_type, uint8* start_addr, +wasm_loader_push_frame_csp(WASMLoaderContext *ctx, uint8 label_type, + BlockType block_type, uint8* start_addr, char *error_buf, uint32 error_buf_size) { CHECK_CSP_PUSH(); memset(ctx->frame_csp, 0, sizeof(BranchBlock)); - ctx->frame_csp->block_type = type; - ctx->frame_csp->return_type = ret_type; + ctx->frame_csp->label_type = label_type; + ctx->frame_csp->block_type = block_type; ctx->frame_csp->start_addr = start_addr; ctx->frame_csp->stack_cell_num = ctx->stack_cell_num; #if WASM_ENABLE_FAST_INTERP != 0 @@ -2742,38 +2765,15 @@ wasm_loader_pop_frame_csp(WASMLoaderContext *ctx, char *error_buf, uint32 error_buf_size) { CHECK_CSP_POP(); +#if WASM_ENABLE_FAST_INTERP != 0 + if ((ctx->frame_csp - 1)->param_frame_offsets) + wasm_runtime_free((ctx->frame_csp - 1)->param_frame_offsets); +#endif ctx->frame_csp--; ctx->csp_num--; return true; } -static bool -wasm_loader_check_br(WASMLoaderContext *ctx, uint32 depth, - char *error_buf, uint32 error_buf_size) -{ - BranchBlock *target_block, *cur_block; - int32 available_stack_cell; - - bh_assert(ctx->csp_num >= depth + 1); - - target_block = ctx->frame_csp - (depth + 1); - cur_block = ctx->frame_csp - 1; - - available_stack_cell = (int32) - (ctx->stack_cell_num - cur_block->stack_cell_num); - - if (available_stack_cell <= 0 && target_block->is_stack_polymorphic) - return true; - - if (target_block->block_type != BLOCK_TYPE_LOOP) { - uint8 type = target_block->return_type; - if (!check_stack_top_values(ctx->frame_ref, available_stack_cell, - type, error_buf, error_buf_size)) - return false; - } - return true; -} - #if WASM_ENABLE_FAST_INTERP != 0 #if WASM_ENABLE_ABS_LABEL_ADDR != 0 @@ -3093,24 +3093,56 @@ static bool wasm_loader_emit_br_info(WASMLoaderContext *ctx, BranchBlock *frame_csp, char *error_buf, uint32 error_buf_size) { - emit_operand(ctx, frame_csp->dynamic_offset); - if (frame_csp->block_type == BLOCK_TYPE_LOOP || - frame_csp->return_type == VALUE_TYPE_VOID) { - emit_byte(ctx, 0); - emit_operand(ctx, 0); + /* br info layout: + * a) arity of target block + * b) total cell num of arity values + * c) each arity value's cell num + * d) each arity value's src frame offset + * e) each arity values's dst dynamic offset + * f) branch target address + */ + BlockType *block_type = &frame_csp->block_type; + uint8 *types = NULL, cell; + uint32 arity = 0; + int32 i; + int16 *frame_offset = ctx->frame_offset; + uint16 dynamic_offset; + + /* Note: loop's arity is different from if and block. loop's arity is + * its parameter count while if and block arity is result count. + */ + if (frame_csp->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(block_type, &types); + else + arity = block_type_get_result_types(block_type, &types); + + /* Part a */ + emit_uint32(ctx, arity); + /* Part b */ + emit_uint32(ctx, wasm_get_cell_num(types, arity)); + + /* Part c */ + for (i = (int32)arity - 1; i >= 0; i--) { + cell = wasm_value_type_cell_num(types[i]); + emit_byte(ctx, cell); } - else if (frame_csp->return_type == VALUE_TYPE_I32 - || frame_csp->return_type == VALUE_TYPE_F32) { - emit_byte(ctx, 1); - emit_operand(ctx, *(int16*)(ctx->frame_offset - 1)); + /* Part d */ + for (i = (int32)arity - 1; i >= 0; i--) { + cell = wasm_value_type_cell_num(types[i]); + frame_offset -= cell; + emit_operand(ctx, *(int16*)(frame_offset)); } - else if (frame_csp->return_type == VALUE_TYPE_I64 - || frame_csp->return_type == VALUE_TYPE_F64) { - emit_byte(ctx, 2); - emit_operand(ctx, *(int16*)(ctx->frame_offset - 2)); + /* Part e */ + dynamic_offset = frame_csp->dynamic_offset + + wasm_get_cell_num(types, arity); + for (i = (int32)arity - 1; i >= 0; i--) { + cell = wasm_value_type_cell_num(types[i]); + dynamic_offset -= cell; + emit_operand(ctx, dynamic_offset); } - if (frame_csp->block_type == BLOCK_TYPE_LOOP) { + /* Part f */ + if (frame_csp->label_type == LABEL_TYPE_LOOP) { wasm_loader_emit_ptr(ctx, frame_csp->code_compiled); } else { @@ -3121,6 +3153,7 @@ wasm_loader_emit_br_info(WASMLoaderContext *ctx, BranchBlock *frame_csp, /* label address, to be patched */ wasm_loader_emit_ptr(ctx, NULL); } + return true; } @@ -3513,50 +3546,116 @@ fail: #if WASM_ENABLE_FAST_INTERP != 0 static bool -reserve_block_ret(WASMLoaderContext *loader_ctx, uint8 opcode, bool disable_emit, +reserve_block_ret(WASMLoaderContext *loader_ctx, + uint8 opcode, bool disable_emit, char *error_buf, uint32 error_buf_size) { int16 operand_offset = 0; - uint8 block_depth = 0; - if (opcode == WASM_OP_ELSE) - block_depth = 1; - else - block_depth = 0; + BranchBlock *block = (opcode == WASM_OP_ELSE) ? + loader_ctx->frame_csp - 1 : loader_ctx->frame_csp; + BlockType *block_type = &block->block_type; + uint8 *return_types = NULL; + uint32 return_count = 0, value_count = 0, total_cel_num = 0; + int32 i = 0; + int16 dynamic_offset, dynamic_offset_org, + *frame_offset = NULL, *frame_offset_org = NULL; - if ((loader_ctx->frame_csp - block_depth)->return_type != VALUE_TYPE_VOID) { - uint8 return_cells; - if ((loader_ctx->frame_csp - block_depth)->return_type == VALUE_TYPE_I32 - || (loader_ctx->frame_csp - block_depth)->return_type == VALUE_TYPE_F32) - return_cells = 1; - else - return_cells = 2; - if ((loader_ctx->frame_csp - block_depth)->dynamic_offset != - *(loader_ctx->frame_offset - return_cells)) { + return_count = block_type_get_result_types(block_type, &return_types); - /* insert op_copy before else opcode */ - if (opcode == WASM_OP_ELSE) - skip_label(); + /* Copy stack top values to block's results which are in dynamic space. + * The instruction format: + * Part a: values count + * Part b: all values total cell num + * Part c: each value's cell_num, src offset and dst offset + * Part d: each value's src offset and dst offset + * Part e: each value's dst offset + */ + frame_offset = frame_offset_org = loader_ctx->frame_offset; + dynamic_offset = dynamic_offset_org = + block->dynamic_offset + + wasm_get_cell_num(return_types, return_count); - if (return_cells == 1) - emit_label(EXT_OP_COPY_STACK_TOP); - else - emit_label(EXT_OP_COPY_STACK_TOP_I64); - emit_operand(loader_ctx, *(loader_ctx->frame_offset - return_cells)); - emit_operand(loader_ctx, (loader_ctx->frame_csp - block_depth)->dynamic_offset); + /* First traversal to get the count of values needed to be copied. */ + for (i = (int32)return_count - 1; i >= 0; i--) { + uint8 cells = wasm_value_type_cell_num(return_types[i]); + frame_offset -= cells; + dynamic_offset -= cells; + if (dynamic_offset != *frame_offset) { + value_count++; + total_cel_num += cells; + } + } + + if (value_count) { + uint32 j = 0; + uint8 *emit_data = NULL, *cells = NULL; + int16 *src_offsets = NULL; + uint16 *dst_offsets = NULL; + uint64 size = (uint64)value_count * (sizeof(*cells) + + sizeof(*src_offsets) + + sizeof(*dst_offsets)); + + /* Allocate memory for the emit data */ + if (!(emit_data = loader_malloc(size, error_buf, error_buf_size))) + return false; + + cells = emit_data; + src_offsets = (int16 *)(cells + value_count); + dst_offsets = (uint16 *)(src_offsets + value_count); + + /* insert op_copy before else opcode */ + if (opcode == WASM_OP_ELSE) + skip_label(); + emit_label(EXT_OP_COPY_STACK_VALUES); + /* Part a) */ + emit_uint32(loader_ctx, value_count); + /* Part b) */ + emit_uint32(loader_ctx, total_cel_num); + + /* Second traversal to get each value's cell num, src offset and dst offset. */ + frame_offset = frame_offset_org; + dynamic_offset = dynamic_offset_org; + for (i = (int32)return_count - 1, j = 0; i >= 0; i--) { + uint8 cell = wasm_value_type_cell_num(return_types[i]); + frame_offset -= cell; + dynamic_offset -= cell; + if (dynamic_offset != *frame_offset) { + /* cell num */ + cells[j] = cell; + /* src offset */ + src_offsets[j] = *frame_offset; + /* dst offset */ + dst_offsets[j] = dynamic_offset; + j++; + } if (opcode == WASM_OP_ELSE) { - *(loader_ctx->frame_offset - return_cells) = - (loader_ctx->frame_csp - block_depth)->dynamic_offset; + *frame_offset = dynamic_offset; } else { - loader_ctx->frame_offset -= return_cells; - loader_ctx->dynamic_offset = loader_ctx->frame_csp->dynamic_offset; - PUSH_OFFSET_TYPE((loader_ctx->frame_csp - block_depth)->return_type); + loader_ctx->frame_offset = frame_offset; + loader_ctx->dynamic_offset = dynamic_offset; + PUSH_OFFSET_TYPE(return_types[i]); wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + loader_ctx->frame_offset = frame_offset_org; + loader_ctx->dynamic_offset = dynamic_offset_org; } - if (opcode == WASM_OP_ELSE) - emit_label(opcode); } + + bh_assert(j == value_count); + + /* Emit the cells, src_offsets and dst_offsets */ + for (j = 0; j < value_count; j++) + emit_byte(loader_ctx, cells[j]); + for (j = 0; j < value_count; j++) + emit_operand(loader_ctx, src_offsets[j]); + for (j = 0; j < value_count; j++) + emit_operand(loader_ctx, dst_offsets[j]); + + if (opcode == WASM_OP_ELSE) + emit_label(opcode); + + wasm_runtime_free(emit_data); } return true; @@ -3585,11 +3684,11 @@ fail: goto fail; \ } while (0) -#define PUSH_CSP(type, ret_type, _start_addr) do { \ - if (!wasm_loader_push_frame_csp(loader_ctx, type, ret_type, \ - _start_addr, error_buf, \ - error_buf_size)) \ - goto fail; \ +#define PUSH_CSP(label_type, block_type, _start_addr) do { \ + if (!wasm_loader_push_frame_csp(loader_ctx, label_type, block_type, \ + _start_addr, error_buf, \ + error_buf_size)) \ + goto fail; \ } while (0) #define POP_CSP() do { \ @@ -3618,13 +3717,88 @@ fail: + module->memory_count > 0); \ } while (0) -#define CHECK_BLOCK_TYPE(type) do { \ - bh_assert(type == VALUE_TYPE_I32 \ - || type == VALUE_TYPE_I64 \ - || type == VALUE_TYPE_F32 \ - || type == VALUE_TYPE_F64 \ - || type == VALUE_TYPE_VOID); \ - } while (0) + +static bool +is_value_type(uint8 type) +{ + return type == VALUE_TYPE_I32 || + type == VALUE_TYPE_I64 || + type == VALUE_TYPE_F32 || + type == VALUE_TYPE_F64 || + type == VALUE_TYPE_VOID; +} + +static bool +wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, + char *error_buf, uint32 error_buf_size) +{ + BranchBlock *target_block, *cur_block; + BlockType *target_block_type; + uint8 *types = NULL, *frame_ref; + uint32 arity = 0; + int32 i, available_stack_cell; + uint16 cell_num; + + if (loader_ctx->csp_num < depth + 1) { + set_error_buf(error_buf, error_buf_size, + "WASM module load failed: unknown label, " + "unexpected end of section or function"); + return false; + } + + cur_block = loader_ctx->frame_csp - 1; + target_block = loader_ctx->frame_csp - (depth + 1); + target_block_type = &target_block->block_type; + frame_ref = loader_ctx->frame_ref; + + /* Note: loop's arity is different from if and block. loop's arity is + * its parameter count while if and block arity is result count. + */ + if (target_block->label_type == LABEL_TYPE_LOOP) + arity = block_type_get_param_types(target_block_type, &types); + else + arity = block_type_get_result_types(target_block_type, &types); + + /* If the stack is in polymorphic state, just clear the stack + * and then re-push the values to make the stack top values + * match block type. */ + if (cur_block->is_stack_polymorphic) { + for (i = (int32)arity -1; i >= 0; i--) { +#if WASM_ENABLE_FAST_INTERP != 0 + POP_OFFSET_TYPE(types[i]); +#endif + POP_TYPE(types[i]); + } + for (i = 0; i < (int32)arity; i++) { +#if WASM_ENABLE_FAST_INTERP != 0 + bool disable_emit = true; + int16 operand_offset = 0; + PUSH_OFFSET_TYPE(types[i]); +#endif + PUSH_TYPE(types[i]); + } + return true; + } + + available_stack_cell = (int32) + (loader_ctx->stack_cell_num - cur_block->stack_cell_num); + + /* Check stack top values match target block type */ + for (i = (int32)arity -1; i >= 0; i--) { + if (!check_stack_top_values(frame_ref, available_stack_cell, + types[i], + error_buf, error_buf_size)) + return false; + cell_num = wasm_value_type_cell_num(types[i]); + frame_ref -= cell_num; + available_stack_cell -= cell_num; + } + + return true; + +fail: + return false; +} static BranchBlock * check_branch_block(WASMLoaderContext *loader_ctx, @@ -3649,90 +3823,165 @@ fail: } static bool -check_branch_block_ret(WASMLoaderContext *loader_ctx, - BranchBlock *frame_csp_tmp, - char *error_buf, uint32 error_buf_size) +check_block_stack(WASMLoaderContext *loader_ctx, BranchBlock *block, + char *error_buf, uint32 error_buf_size) { + BlockType *block_type = &block->block_type; + uint8 *return_types = NULL; + uint32 return_count = 0; + int32 available_stack_cell, return_cell_num, i; + uint8 *frame_ref = NULL; + + available_stack_cell = (int32) + (loader_ctx->stack_cell_num + - block->stack_cell_num); + + return_count = block_type_get_result_types(block_type, &return_types); + return_cell_num = return_count > 0 ? + wasm_get_cell_num(return_types, return_count) : 0; + + /* If the stack is in polymorphic state, just clear the stack + * and then re-push the values to make the stack top values + * match block type. */ + if (block->is_stack_polymorphic) { + for (i = (int32)return_count -1; i >= 0; i--) { #if WASM_ENABLE_FAST_INTERP != 0 - BranchBlock *cur_block = loader_ctx->frame_csp - 1; - bool disable_emit = true; - int16 operand_offset = 0; + POP_OFFSET_TYPE(return_types[i]); #endif - if (frame_csp_tmp->block_type != BLOCK_TYPE_LOOP) { - uint8 block_return_type = frame_csp_tmp->return_type; -#if WASM_ENABLE_FAST_INTERP != 0 - /* If the stack is in polymorphic state, do fake pop and push on - offset stack to keep the depth of offset stack to be the same - with ref stack */ - if (cur_block->is_stack_polymorphic) { - POP_OFFSET_TYPE(block_return_type); - PUSH_OFFSET_TYPE(block_return_type); + POP_TYPE(return_types[i]); } + + /* Check stack is empty */ + bh_assert(loader_ctx->stack_cell_num == block->stack_cell_num); + + for (i = 0; i < (int32)return_count; i++) { +#if WASM_ENABLE_FAST_INTERP != 0 + bool disable_emit = true; + int16 operand_offset = 0; + PUSH_OFFSET_TYPE(return_types[i]); #endif - POP_TYPE(block_return_type); - PUSH_TYPE(block_return_type); + PUSH_TYPE(return_types[i]); + } + return true; } + + /* Check stack cell num equals return cell num */ + bh_assert(available_stack_cell == return_cell_num); + + /* Check stack values match return types */ + frame_ref = loader_ctx->frame_ref; + for (i = (int32)return_count -1; i >= 0; i--) { + if (!check_stack_top_values(frame_ref, available_stack_cell, + return_types[i], + error_buf, error_buf_size)) + return false; + frame_ref -= wasm_value_type_cell_num(return_types[i]); + available_stack_cell -= wasm_value_type_cell_num(return_types[i]); + } + + (void)return_cell_num; return true; + fail: return false; } +#if WASM_ENABLE_FAST_INTERP != 0 +/* Copy parameters to dynamic space. + * 1) POP original parameter out; + * 2) Push and copy original values to dynamic space. + * The copy instruction format: + * Part a: param count + * Part b: all param total cell num + * Part c: each param's cell_num, src offset and dst offset + * Part d: each param's src offset + * Part e: each param's dst offset + */ static bool -check_block_stack(WASMLoaderContext *ctx, BranchBlock *block, - char *error_buf, uint32 error_buf_size) +copy_params_to_dynamic_space(WASMLoaderContext *loader_ctx, bool is_if_block, + char* error_buf, uint32 error_buf_size) { - uint8 type = block->return_type; - int32 available_stack_cell = (int32) - (ctx->stack_cell_num - block->stack_cell_num); + int16 *frame_offset = NULL; + uint8 *cells = NULL, cell; + int16 *src_offsets = NULL; + uint8 *emit_data = NULL; + uint32 i; + BranchBlock *block = loader_ctx->frame_csp - 1; + BlockType *block_type = &block->block_type; + WASMType *wasm_type = block_type->u.type; + uint32 param_count = block_type->u.type->param_count; + int16 condition_offset = 0; + bool disable_emit = false; + int16 operand_offset = 0; - if (type != VALUE_TYPE_VOID - && available_stack_cell <= 0 - && block->is_stack_polymorphic) { - if (!(wasm_loader_push_frame_ref(ctx, type, error_buf, error_buf_size)) -#if WASM_ENABLE_FAST_INTERP != 0 - || !(wasm_loader_push_frame_offset(ctx, type, true, 0, error_buf, error_buf_size)) -#endif - ) - return false; - return true; - } + uint64 size = (uint64)param_count * (sizeof(*cells) + + sizeof(*src_offsets)); - if (type != VALUE_TYPE_VOID - && available_stack_cell == 1 - && *(ctx->frame_ref - 1) == VALUE_TYPE_ANY) { - if (type == VALUE_TYPE_I32 || type == VALUE_TYPE_F32) { - /* If the stack top is a value of any type, change its type to the - * same as block return type and return success */ - *(ctx->frame_ref - 1) = type; - } - else { - if (!(wasm_loader_push_frame_ref(ctx, VALUE_TYPE_I32, - error_buf, error_buf_size)) -#if WASM_ENABLE_FAST_INTERP != 0 - || !(wasm_loader_push_frame_offset(ctx, VALUE_TYPE_I32, - true, 0, - error_buf, error_buf_size)) -#endif - ) - return false; - *(ctx->frame_ref - 1) = *(ctx->frame_ref - 2) = type; - } - return true; - } + /* For if block, we also need copy the condition operand offset. */ + if (is_if_block) + size += sizeof(*cells) + sizeof(*src_offsets); - bh_assert(!(((type == VALUE_TYPE_I32 || type == VALUE_TYPE_F32) - && available_stack_cell != 1) - || ((type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64) - && available_stack_cell != 2) - || (type == VALUE_TYPE_VOID - && available_stack_cell > 0))); - - if (!check_stack_top_values(ctx->frame_ref, available_stack_cell, - type, error_buf, error_buf_size)) + /* Allocate memory for the emit data */ + if (!(emit_data = loader_malloc(size, error_buf, error_buf_size))) return false; + cells = emit_data; + src_offsets = (int16 *)(cells + param_count); + + if (is_if_block) + condition_offset = *loader_ctx->frame_offset; + + /* POP original parameter out */ + for (i = 0; i < param_count; i++) { + POP_OFFSET_TYPE(wasm_type->types[param_count - i - 1]); + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + } + frame_offset = loader_ctx->frame_offset; + + /* Get each param's cell num and src offset */ + for (i = 0; i < param_count; i++) { + cell = wasm_value_type_cell_num(wasm_type->types[i]); + cells[i] = cell; + src_offsets[i] = *frame_offset; + frame_offset += cell; + } + + /* emit copy instruction */ + emit_label(EXT_OP_COPY_STACK_VALUES); + /* Part a) */ + emit_uint32(loader_ctx, is_if_block ? param_count + 1 : param_count); + /* Part b) */ + emit_uint32(loader_ctx, is_if_block ? + wasm_type->param_cell_num + 1 : + wasm_type->param_cell_num); + /* Part c) */ + for (i = 0; i < param_count; i++) + emit_byte(loader_ctx, cells[i]); + if (is_if_block) + emit_byte(loader_ctx, 1); + + /* Part d) */ + for (i = 0; i < param_count; i++) + emit_operand(loader_ctx, src_offsets[i]); + if (is_if_block) + emit_operand(loader_ctx, condition_offset); + + /* Part e) */ + /* Push to dynamic space. The push will emit the dst offset. */ + for (i = 0; i < param_count; i++) + PUSH_OFFSET_TYPE(wasm_type->types[i]); + if (is_if_block) + PUSH_OFFSET_TYPE(VALUE_TYPE_I32); + + /* Free the emit data */ + wasm_runtime_free(emit_data); + return true; + +fail: + return false; } +#endif /* reset the stack to the state of before entering the last block */ #if WASM_ENABLE_FAST_INTERP != 0 @@ -3759,6 +4008,9 @@ check_block_stack(WASMLoaderContext *ctx, BranchBlock *block, cur_block->is_stack_polymorphic = flag; \ } while (0) +#define BLOCK_HAS_PARAM(block_type) \ + (!block_type.is_value_type && block_type.u.type->param_count > 0) + static bool wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, BlockAddr *block_addr_cache, @@ -3766,12 +4018,13 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, { uint8 *p = func->code, *p_end = func->code + func->code_size, *p_org; uint32 param_count, local_count, global_count; - uint8 *param_types, ret_type, *local_types, local_type, global_type; + uint8 *param_types, *local_types, local_type, global_type; + BlockType func_type; uint16 *local_offsets, local_offset; uint32 count, i, local_idx, global_idx, u32, align, mem_offset; int32 i32, i32_const = 0; int64 i64; - uint8 opcode, u8, block_return_type; + uint8 opcode, u8; bool return_value = false; WASMLoaderContext *loader_ctx; BranchBlock *frame_csp_tmp; @@ -3796,8 +4049,9 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, param_count = func->func_type->param_count; param_types = func->func_type->types; - ret_type = func->func_type->result_count - ? param_types[param_count] : VALUE_TYPE_VOID; + + func_type.is_value_type = false; + func_type.u.type = func->func_type; local_count = func->local_count; local_types = func->local_types; @@ -3824,7 +4078,7 @@ re_scan: } #endif - PUSH_CSP(BLOCK_TYPE_FUNCTION, ret_type, p); + PUSH_CSP(LABEL_TYPE_FUNCTION, func_type, p); while (p < p_end) { opcode = *p++; @@ -3846,44 +4100,137 @@ re_scan: #endif break; - case WASM_OP_BLOCK: - /* 0x40/0x7F/0x7E/0x7D/0x7C */ - block_return_type = read_uint8(p); - CHECK_BLOCK_TYPE(block_return_type); - PUSH_CSP(BLOCK_TYPE_BLOCK, block_return_type, p); -#if WASM_ENABLE_FAST_INTERP != 0 - skip_label(); -#endif - break; - - case WASM_OP_LOOP: - /* 0x40/0x7F/0x7E/0x7D/0x7C */ - block_return_type = read_uint8(p); - CHECK_BLOCK_TYPE(block_return_type); - PUSH_CSP(BLOCK_TYPE_LOOP, block_return_type, p); -#if WASM_ENABLE_FAST_INTERP != 0 - skip_label(); - (loader_ctx->frame_csp - 1)->code_compiled = - loader_ctx->p_code_compiled; -#endif - break; - case WASM_OP_IF: POP_I32(); - /* 0x40/0x7F/0x7E/0x7D/0x7C */ - block_return_type = read_uint8(p); - CHECK_BLOCK_TYPE(block_return_type); - PUSH_CSP(BLOCK_TYPE_IF, block_return_type, p); + goto handle_op_block_and_loop; + case WASM_OP_BLOCK: + case WASM_OP_LOOP: +handle_op_block_and_loop: + { + uint8 value_type; + BlockType block_type; + + value_type = read_uint8(p); + if (is_value_type(value_type)) { + /* If the first byte is one of these special values: + * 0x40/0x7F/0x7E/0x7D/0x7C, take it as the type of + * the single return value. */ + block_type.is_value_type = true; + block_type.u.value_type = value_type; + } + else { + uint32 type_index; + /* Resolve the leb128 encoded type index as block type */ + p--; + read_leb_uint32(p, p_end, type_index); + bh_assert(type_index < module->type_count); + block_type.is_value_type = false; + block_type.u.type = module->types[type_index]; +#if WASM_ENABLE_FAST_INTERP == 0 \ + && WASM_ENABLE_WAMR_COMPILER == 0 \ + && WASM_ENABLE_JIT == 0 + /* If block use type index as block type, change the opcode + * to new extended opcode so that interpreter can resolve the + * block quickly. + */ + *(p - 2) = EXT_OP_BLOCK + (opcode - WASM_OP_BLOCK); +#endif + } + + /* Pop block parameters from stack */ + if (BLOCK_HAS_PARAM(block_type)) { + WASMType *wasm_type = block_type.u.type; + for (i = 0; i < block_type.u.type->param_count; i++) + POP_TYPE(wasm_type->types[wasm_type->param_count - i - 1]); + } + + PUSH_CSP(LABEL_TYPE_BLOCK + (opcode - WASM_OP_BLOCK), block_type, p); + + /* Pass parameters to block */ + if (BLOCK_HAS_PARAM(block_type)) { + for (i = 0; i < block_type.u.type->param_count; i++) + PUSH_TYPE(block_type.u.type->types[i]); + } + #if WASM_ENABLE_FAST_INTERP != 0 - emit_empty_label_addr_and_frame_ip(PATCH_ELSE); - emit_empty_label_addr_and_frame_ip(PATCH_END); + if (opcode == WASM_OP_BLOCK) { + skip_label(); + } else if (opcode == WASM_OP_LOOP) { + skip_label(); + if (BLOCK_HAS_PARAM(block_type)) { + /* Make sure params are in dynamic space */ + if (!copy_params_to_dynamic_space(loader_ctx, + false, + error_buf, + error_buf_size)) + goto fail; + } + (loader_ctx->frame_csp - 1)->code_compiled = + loader_ctx->p_code_compiled; + } else if (opcode == WASM_OP_IF) { + /* If block has parameters, we should make sure they are in + * dynamic space. Otherwise, when else branch is missing, + * the later opcode may consume incorrect operand offset. + * Spec case: + * (func (export "params-id") (param i32) (result i32) + * (i32.const 1) + * (i32.const 2) + * (if (param i32 i32) (result i32 i32) (local.get 0) (then)) + * (i32.add) + * ) + * + * So we should emit a copy instruction before the if. + * + * And we also need to save the parameter offsets and + * recover them before entering else branch. + * + */ + if (BLOCK_HAS_PARAM(block_type)) { + BranchBlock *block = loader_ctx->frame_csp - 1; + uint64 size; + + /* skip the if condition operand offset */ + wasm_loader_emit_backspace(loader_ctx, sizeof(int16)); + /* skip the if label */ + skip_label(); + /* Emit a copy instruction */ + if (!copy_params_to_dynamic_space(loader_ctx, + true, + error_buf, + error_buf_size)) + goto fail; + + /* Emit the if instruction */ + emit_label(opcode); + /* Emit the new condition operand offset */ + POP_OFFSET_TYPE(VALUE_TYPE_I32); + + /* Save top param_count values of frame_offset stack, so that + * we can recover it before executing else branch */ + size = sizeof(int16) * + (uint64)block_type.u.type->param_cell_num; + if (!(block->param_frame_offsets = + loader_malloc(size, error_buf, error_buf_size))) + goto fail; + bh_memcpy_s(block->param_frame_offsets, + size, + loader_ctx->frame_offset - size/sizeof(int16), + size); + } + + emit_empty_label_addr_and_frame_ip(PATCH_ELSE); + emit_empty_label_addr_and_frame_ip(PATCH_END); + } #endif break; + } case WASM_OP_ELSE: + { + BlockType block_type = (loader_ctx->frame_csp - 1)->block_type; bh_assert(loader_ctx->csp_num >= 2 - && (loader_ctx->frame_csp - 1)->block_type - == BLOCK_TYPE_IF); + && (loader_ctx->frame_csp - 1)->label_type + == LABEL_TYPE_IF); /* check whether if branch's stack matches its result type */ if (!check_block_stack(loader_ctx, loader_ctx->frame_csp - 1, @@ -3901,20 +4248,63 @@ re_scan: #endif RESET_STACK(); SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(false); + + /* Pass parameters to if-false branch */ + if (BLOCK_HAS_PARAM(block_type)) { + for (i = 0; i < block_type.u.type->param_count; i++) + PUSH_TYPE(block_type.u.type->types[i]); + } + +#if WASM_ENABLE_FAST_INTERP != 0 + /* Recover top param_count values of frame_offset stack */ + if (BLOCK_HAS_PARAM((block_type))) { + uint32 size; + BranchBlock *block = loader_ctx->frame_csp - 1; + size = sizeof(int16) * + block_type.u.type->param_cell_num; + bh_memcpy_s(loader_ctx->frame_offset, size, + block->param_frame_offsets, size); + loader_ctx->frame_offset += (size/sizeof(int16)); + } +#endif + break; + } case WASM_OP_END: { + BranchBlock *cur_block = loader_ctx->frame_csp - 1; /* check whether block stack matches its result type */ - if (!check_block_stack(loader_ctx, loader_ctx->frame_csp - 1, + if (!check_block_stack(loader_ctx, cur_block, error_buf, error_buf_size)) goto fail; - /* if has return value, but no else branch, fail */ - bh_assert(!((loader_ctx->frame_csp - 1)->block_type == BLOCK_TYPE_IF - && (loader_ctx->frame_csp - 1)->return_type != VALUE_TYPE_VOID - && !(loader_ctx->frame_csp - 1)->else_addr)); + /* if no else branch, and return types do not match param types, fail */ + if (cur_block->label_type == LABEL_TYPE_IF + && !cur_block->else_addr) { + uint32 param_count = 0, ret_count = 0; + uint8 *param_types = NULL, *ret_types = NULL; + BlockType *block_type = &cur_block->block_type; + if (block_type->is_value_type) { + if (block_type->u.value_type != VALUE_TYPE_VOID) { + ret_count = 1; + ret_types = &block_type->u.value_type; + } + } + else { + param_count = block_type->u.type->param_count; + ret_count = block_type->u.type->result_count; + param_types = block_type->u.type->types; + ret_types = block_type->u.type->types + param_count; + } + bh_assert(param_count == ret_count + && (!param_count + || !memcmp(param_types, ret_types, param_count))); + (void)ret_types; + (void)ret_count; + (void)param_types; + } POP_CSP(); @@ -3925,9 +4315,17 @@ re_scan: apply_label_patch(loader_ctx, 0, PATCH_END); free_label_patch_list(loader_ctx->frame_csp); - if (loader_ctx->frame_csp->block_type == BLOCK_TYPE_FUNCTION) { + if (loader_ctx->frame_csp->label_type == LABEL_TYPE_FUNCTION) { + int32 idx; + uint8 ret_type; + emit_label(WASM_OP_RETURN); - POP_OFFSET_TYPE(loader_ctx->frame_csp->return_type); + for (idx = (int32)func->func_type->result_count - 1; + idx >= 0; idx--) { + ret_type = *(func->func_type->types + + func->func_type->param_count + idx); + POP_OFFSET_TYPE(ret_type); + } } #endif if (loader_ctx->csp_num > 0) { @@ -3951,10 +4349,6 @@ re_scan: error_buf, error_buf_size))) goto fail; - if (!check_branch_block_ret(loader_ctx, frame_csp_tmp, - error_buf, error_buf_size)) - goto fail; - RESET_STACK(); SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); break; @@ -3968,16 +4362,13 @@ re_scan: error_buf, error_buf_size))) goto fail; - if (!check_branch_block_ret(loader_ctx, frame_csp_tmp, - error_buf, error_buf_size)) - goto fail; - break; } case WASM_OP_BR_TABLE: { - uint8 ret_type; + uint8 *ret_types = NULL; + uint32 ret_count = 0; read_leb_uint32(p, p_end, count); #if WASM_ENABLE_FAST_INTERP != 0 @@ -3987,45 +4378,37 @@ re_scan: /* TODO: check the const */ for (i = 0; i <= count; i++) { - if (!(frame_csp_tmp = check_branch_block(loader_ctx, &p, p_end, - error_buf, error_buf_size))) + if (!(frame_csp_tmp = + check_branch_block(loader_ctx, &p, p_end, + error_buf, error_buf_size))) goto fail; - - if (!check_branch_block_ret(loader_ctx, frame_csp_tmp, - error_buf, error_buf_size)) - goto fail; - - if (i == 0) { - ret_type = frame_csp_tmp->block_type == BLOCK_TYPE_LOOP ? - VALUE_TYPE_VOID : frame_csp_tmp->return_type; - } - else { - /* Check whether all table items have the same return type */ - uint8 tmp_ret_type = frame_csp_tmp->block_type == BLOCK_TYPE_LOOP ? - VALUE_TYPE_VOID : frame_csp_tmp->return_type; - bh_assert(ret_type == tmp_ret_type); - (void)tmp_ret_type; - } } RESET_STACK(); SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); - (void)ret_type; + + (void)ret_count; + (void)ret_types; break; } case WASM_OP_RETURN: { - POP_TYPE(ret_type); - PUSH_TYPE(ret_type); - + int32 idx; + uint8 ret_type; + for (idx = (int32)func->func_type->result_count - 1; idx >= 0; idx--) { + ret_type = *(func->func_type->types + + func->func_type->param_count + idx); + POP_TYPE(ret_type); #if WASM_ENABLE_FAST_INTERP != 0 - // emit the offset after return opcode - POP_OFFSET_TYPE(ret_type); + // emit the offset after return opcode + POP_OFFSET_TYPE(ret_type); #endif + } RESET_STACK(); SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true); + break; } @@ -4059,10 +4442,14 @@ re_scan: } } - if (func_type->result_count) { - PUSH_TYPE(func_type->types[func_type->param_count]); + for (i = 0; i < func_type->result_count; i++) { + PUSH_TYPE(func_type->types[func_type->param_count + i]); #if WASM_ENABLE_FAST_INTERP != 0 - PUSH_OFFSET_TYPE(func_type->types[func_type->param_count]); + /* Here we emit each return value's dynamic_offset. But in fact + * these offsets are continuous, so interpreter only need to get + * the first return value's offset. + */ + PUSH_OFFSET_TYPE(func_type->types[func_type->param_count + i]); #endif } @@ -4104,10 +4491,10 @@ re_scan: } } - if (func_type->result_count > 0) { - PUSH_TYPE(func_type->types[func_type->param_count]); + for (i = 0; i < func_type->result_count; i++) { + PUSH_TYPE(func_type->types[func_type->param_count + i]); #if WASM_ENABLE_FAST_INTERP != 0 - PUSH_OFFSET_TYPE(func_type->types[func_type->param_count]); + PUSH_OFFSET_TYPE(func_type->types[func_type->param_count + i]); #endif } diff --git a/core/iwasm/interpreter/wasm_opcode.h b/core/iwasm/interpreter/wasm_opcode.h index 99b4f0b0a..abb60b8ed 100644 --- a/core/iwasm/interpreter/wasm_opcode.h +++ b/core/iwasm/interpreter/wasm_opcode.h @@ -255,8 +255,11 @@ typedef enum WASMOpcode { EXT_OP_TEE_LOCAL_FAST_I64 = 0xcb, EXT_OP_COPY_STACK_TOP = 0xcc, EXT_OP_COPY_STACK_TOP_I64 = 0xcd, - - WASM_OP_IMPDEP = 0xce, + EXT_OP_COPY_STACK_VALUES = 0xce, + EXT_OP_BLOCK = 0xcf, /* block with blocktype */ + EXT_OP_LOOP = 0xd0, /* loop with blocktype */ + EXT_OP_IF = 0xd1, /* if with blocktype */ + WASM_OP_IMPDEP = 0xd2, /* Post-MVP extend op prefix */ WASM_OP_MISC_PREFIX = 0xfc, @@ -499,7 +502,11 @@ static type _name[WASM_INSTRUCTION_NUM] = { \ HANDLE_OPCODE (EXT_OP_TEE_LOCAL_FAST_I64), /* 0xcb */ \ HANDLE_OPCODE (EXT_OP_COPY_STACK_TOP), /* 0xcc */ \ HANDLE_OPCODE (EXT_OP_COPY_STACK_TOP_I64), /* 0xcd */ \ - HANDLE_OPCODE (WASM_OP_IMPDEP), /* 0xce */ \ + HANDLE_OPCODE (EXT_OP_COPY_STACK_VALUES), /* 0xce */ \ + HANDLE_OPCODE (EXT_OP_BLOCK), /* 0xcf */ \ + HANDLE_OPCODE (EXT_OP_LOOP), /* 0xd0 */ \ + HANDLE_OPCODE (EXT_OP_IF), /* 0xd1 */ \ + HANDLE_OPCODE (WASM_OP_IMPDEP), /* 0xd2 */ \ }; \ do { \ _name[WASM_OP_MISC_PREFIX] = \ diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 4ea874e4e..5212ca1cf 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -509,9 +509,9 @@ functions_instantiate(const WASMModule *module, import->u.function.field_name); function->u.func_import = &import->u.function; function->param_cell_num = - wasm_type_param_cell_num(import->u.function.func_type); + import->u.function.func_type->param_cell_num; function->ret_cell_num = - wasm_type_return_cell_num(import->u.function.func_type); + import->u.function.func_type->ret_cell_num; function->param_count = (uint16)function->u.func_import->func_type->param_count; function->param_types = function->u.func_import->func_type->types; @@ -1804,4 +1804,4 @@ wasm_get_aux_stack(WASMExecEnv *exec_env, } return false; } -#endif \ No newline at end of file +#endif diff --git a/product-mini/platforms/linux/CMakeLists.txt b/product-mini/platforms/linux/CMakeLists.txt index 053e76eac..e7a5a0786 100644 --- a/product-mini/platforms/linux/CMakeLists.txt +++ b/product-mini/platforms/linux/CMakeLists.txt @@ -112,4 +112,4 @@ install (TARGETS libiwasm DESTINATION lib) set_target_properties (libiwasm PROPERTIES OUTPUT_NAME iwasm) -target_link_libraries (libiwasm ${LLVM_AVAILABLE_LIBS} -lm -ldl -lpthread) \ No newline at end of file +target_link_libraries (libiwasm ${LLVM_AVAILABLE_LIBS} -lm -ldl -lpthread) diff --git a/test-tools/binarydump-tool/CMakeLists.txt b/test-tools/binarydump-tool/CMakeLists.txt new file mode 100644 index 000000000..d821c8062 --- /dev/null +++ b/test-tools/binarydump-tool/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 2.8) + +project(binarydump) + +add_definitions (-Wextra -pedantic -Wno-unused-parameter) + +add_executable (binarydump binarydump.c) + diff --git a/test-tools/binarydump-tool/binarydump.c b/test-tools/binarydump-tool/binarydump.c new file mode 100644 index 000000000..cd19eecad --- /dev/null +++ b/test-tools/binarydump-tool/binarydump.c @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include + +static unsigned char* +read_file_to_buffer (const char *filename, int *ret_size) +{ + unsigned char *buffer; + FILE *file; + int file_size, read_size; + + if (!(file = fopen (filename, "r"))) + return NULL; + + fseek (file, 0, SEEK_END); + file_size = ftell (file); + fseek (file, 0, SEEK_SET); + + if (!(buffer = malloc (file_size))) { + fclose (file); + return NULL; + } + + read_size = fread (buffer, 1, file_size, file); + fclose (file); + + if (read_size < file_size) { + free (buffer); + return NULL; + } + + *ret_size = file_size; + + return buffer; +} + +static int +print_help () +{ + printf ("Usage: binarydump -o -n input_file\n"); + printf ("Options:\n"); + printf (" -o Place the output into \n"); + printf (" -n The name of array \n"); + + return -1; +} + +static bool +bin_file_dump (const unsigned char *file, int size, + const char *bin_file_output, + const char *array_name) +{ + unsigned i = 0; + const unsigned char *p = file, *p_end = file + size; + FILE *file_output = fopen(bin_file_output, "wb+"); + + if (!file_output) + return false; + + fprintf(file_output, "\nunsigned char __aligned(4) %s[] = {\n ", array_name); + + while (p < p_end) { + fprintf(file_output, "0x%02X", *p++); + + if (p == p_end) + break; + + fprintf(file_output, ","); + + if ((++i % 12) != 0) + fprintf(file_output, " "); + else + fprintf(file_output, "\n "); + } + + fprintf(file_output, "\n};\n"); + + fclose(file_output); + return true; +} + +int +main (int argc, char *argv[]) +{ + unsigned char *file; + int size; + bool ret; + const char *bin_file_input, *array_file_output = NULL, *array_name = NULL; + + for (argc--, argv++; argc > 0 && argv[0][0] == '-'; argc--, argv++) { + if (!strcmp (argv[0], "-o")) { + ++argv; + if (--argc == 0) + return print_help (); + array_file_output = *argv; + } + else if (!strcmp (argv[0], "-n")) { + ++argv; + if (--argc == 0) + return print_help (); + array_name = *argv; + } + else + return print_help (); + } + + if (!array_file_output || !array_name) + return print_help (); + + bin_file_input = *argv; + + if (!(file = read_file_to_buffer (bin_file_input, &size))) + return -1; + + ret = bin_file_dump (file, size, array_file_output, array_name); + + free (file); + + return ret ? 0 : -1; +}