From 66cd90d8471904597a5d043b22cb7fbc8c09eb15 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Thu, 2 Jun 2022 16:41:48 +0800 Subject: [PATCH] Fix fast jit issues (#1201) --- .../fast-jit/cg/x86-64/jit_codegen_x86_64.cpp | 172 +++++++++++++++--- core/iwasm/fast-jit/fe/jit_emit_function.c | 64 +++---- core/iwasm/fast-jit/fe/jit_emit_memory.c | 19 +- core/iwasm/fast-jit/fe/jit_emit_numberic.c | 130 +++++++++++-- core/iwasm/fast-jit/fe/jit_emit_parametric.c | 39 ++-- core/iwasm/fast-jit/jit_frontend.h | 2 + core/iwasm/interpreter/wasm_runtime.c | 60 ++++-- core/iwasm/interpreter/wasm_runtime.h | 18 +- 8 files changed, 397 insertions(+), 107 deletions(-) diff --git a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp index 3db6a0918..58c62df0b 100644 --- a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp +++ b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp @@ -2880,6 +2880,68 @@ alu_imm_imm_to_r_f32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, return mov_imm_to_r_f32(a, reg_no_dst, data); } +static bool +alu_r_m_float(x86::Assembler &a, ALU_OP op, int32 reg_no, x86::Mem &m, + bool is_f32) +{ + switch (op) { + case ADD: + { + if (is_f32) + a.addss(regs_float[reg_no], m); + else + a.addsd(regs_float[reg_no], m); + break; + } + case SUB: + { + if (is_f32) + a.subss(regs_float[reg_no], m); + else + a.subsd(regs_float[reg_no], m); + break; + } + case MUL: + { + if (is_f32) + a.mulss(regs_float[reg_no], m); + else + a.mulsd(regs_float[reg_no], m); + break; + } + case DIV_S: + { + if (is_f32) + a.divss(regs_float[reg_no], m); + else + a.divsd(regs_float[reg_no], m); + break; + } + case MAX: + { + if (is_f32) + a.maxss(regs_float[reg_no], m); + else + a.maxsd(regs_float[reg_no], m); + break; + } + case MIN: + { + if (is_f32) + a.minss(regs_float[reg_no], m); + else + a.minsd(regs_float[reg_no], m); + break; + } + default: + { + bh_assert(0); + return false; + } + } + return true; +} + /** * Encode float alu operation of imm and reg, and save result to reg * @@ -2902,10 +2964,9 @@ alu_imm_r_to_r_f32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, a.movaps(cache, regs_float[reg_no2_src]); /* imm -> gp -> xmm */ - mov_imm_to_r_f32(a, reg_no2_src, data1_src); + mov_imm_to_r_f32(a, reg_no_dst, data1_src); - a.addss(regs_float[reg_no2_src], cache); - return true; + return alu_r_m_float(a, op, reg_no_dst, cache, true); } /** @@ -2930,8 +2991,9 @@ alu_r_imm_to_r_f32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, Imm imm(*(uint32 *)&data2_src); mov_imm_to_m(a, cache, imm, 4); - a.addss(regs_float[reg_no1_src], cache); - return true; + mov_r_to_r_f32(a, reg_no_dst, reg_no1_src); + + return alu_r_m_float(a, op, reg_no_dst, cache, true); } /** @@ -3076,10 +3138,9 @@ alu_imm_r_to_r_f64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, a.movapd(cache, regs_float[reg_no2_src]); /* imm -> gp -> xmm */ - mov_imm_to_r_f64(a, reg_no2_src, data1_src); + mov_imm_to_r_f64(a, reg_no_dst, data1_src); - a.addsd(regs_float[reg_no2_src], cache); - return true; + return alu_r_m_float(a, op, reg_no_dst, cache, false); } /** @@ -3104,8 +3165,9 @@ alu_r_imm_to_r_f64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, Imm imm(*(uint64 *)&data2_src); mov_imm_to_m(a, cache, imm, 8); - a.addsd(regs_float[reg_no1_src], cache); - return true; + mov_r_to_r_f64(a, reg_no_dst, reg_no1_src); + + return alu_r_m_float(a, op, reg_no_dst, cache, false); } /** @@ -3359,18 +3421,42 @@ bit_r_imm_i64(x86::Assembler &a, BIT_OP op, int32 reg_no, int64 data) switch (op) { case OR: - if (data != 0) - a.or_(regs_i64[reg_no], imm); + if (data != 0) { + if (data >= INT32_MIN && data <= INT32_MAX) { + imm.setValue((int32)data); + a.or_(regs_i64[reg_no], imm); + } + else { + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.or_(regs_i64[reg_no], regs_i64[REG_I64_FREE_IDX]); + } + } break; case XOR: if (data == -1LL) a.not_(regs_i64[reg_no]); - else if (data != 0) - a.xor_(regs_i64[reg_no], imm); + else if (data != 0) { + if (data >= INT32_MIN && data <= INT32_MAX) { + imm.setValue((int32)data); + a.xor_(regs_i64[reg_no], imm); + } + else { + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.xor_(regs_i64[reg_no], regs_i64[REG_I64_FREE_IDX]); + } + } break; case AND: - if (data != -1LL) - a.and_(regs_i64[reg_no], imm); + if (data != -1LL) { + if (data >= INT32_MIN && data <= INT32_MAX) { + imm.setValue((int32)data); + a.and_(regs_i64[reg_no], imm); + } + else { + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.and_(regs_i64[reg_no], regs_i64[REG_I64_FREE_IDX]); + } + } break; default: bh_assert(0); @@ -4958,6 +5044,7 @@ fail: * Encode detecting the cmp flags in reg, and jmp to the relative address * according to the condition opcode * + * @param cc the compiler context * @param a the assembler to emit the code * @param reg_no the no of register which contains cmp flags of cmp result * @param op the condition opcode to jmp @@ -4966,12 +5053,13 @@ fail: * @return return the next address of native code after encoded */ static bool -cmp_r_and_jmp_relative(x86::Assembler &a, int32 reg_no, COND_OP op, - int32 offset) +cmp_r_and_jmp_relative(JitCompContext *cc, x86::Assembler &a, int32 reg_no, + COND_OP op, int32 offset) { Imm target(INT32_MAX); char *stream = (char *)a.code()->sectionById(0)->buffer().data() + a.code()->sectionById(0)->buffer().size(); + bool fp_cmp = cc->last_cmp_on_fp; switch (op) { case EQ: @@ -4981,17 +5069,46 @@ cmp_r_and_jmp_relative(x86::Assembler &a, int32 reg_no, COND_OP op, a.jne(target); break; case GTS: - a.jg(target); + { + if (fp_cmp) { + a.jnbe(target); + } + else { + a.jg(target); + } break; + } case LES: - a.jng(target); + { + if (fp_cmp) { + a.jbe(target); + } + else { + a.jng(target); + } break; + } case GES: - a.jge(target); + { + if (fp_cmp) { + a.jnb(target); + } + else { + + a.jnl(target); + } break; + } case LTS: - a.jl(target); + { + if (fp_cmp) { + a.jb(target); + } + else { + a.jl(target); + } break; + } case GTU: a.ja(target); break; @@ -5074,7 +5191,8 @@ lower_select(JitCompContext *cc, x86::Assembler &a, COND_OP op, JitReg r0, } if (r3 && r0 != r3) { - if (!cmp_r_and_jmp_relative(a, jit_reg_no(r1), op, (int32)size_mov2)) + if (!cmp_r_and_jmp_relative(cc, a, jit_reg_no(r1), op, + (int32)size_mov2)) return false; a.embedDataArray(TypeId::kInt8, stream_mov2, size_mov2); } @@ -6398,7 +6516,7 @@ static uint8 hreg_info_F32[3][16] = { { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1 }, { 1, 1, 1, 1, 1, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0 }, /* caller_saved_native */ + 1, 1, 1, 1, 1, 1, 1, 1 }, /* caller_saved_native */ { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* caller_saved_jitted */ }; @@ -6409,7 +6527,7 @@ static uint8 hreg_info_F64[3][16] = { { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0 }, { 1, 1, 1, 1, 1, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0 }, /* caller_saved_native */ + 1, 1, 1, 1, 1, 1, 1, 1 }, /* caller_saved_native */ { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* caller_saved_jitted */ }; @@ -6466,12 +6584,16 @@ jit_codegen_get_hreg_by_name(const char *name) return jit_reg_new(JIT_REG_KIND_I32, REG_ECX_IDX); else if (strcmp(name, "edx") == 0) return jit_reg_new(JIT_REG_KIND_I32, REG_EDX_IDX); + else if (strcmp(name, "esi") == 0) + return jit_reg_new(JIT_REG_KIND_I32, REG_ESI_IDX); else if (strcmp(name, "rax") == 0) return jit_reg_new(JIT_REG_KIND_I64, REG_RAX_IDX); else if (strcmp(name, "rcx") == 0) return jit_reg_new(JIT_REG_KIND_I64, REG_RCX_IDX); else if (strcmp(name, "rdx") == 0) return jit_reg_new(JIT_REG_KIND_I64, REG_RDX_IDX); + else if (strcmp(name, "r9") == 0) + return jit_reg_new(JIT_REG_KIND_I64, REG_R9_IDX); else if (strcmp(name, "xmm0") == 0) return jit_reg_new(JIT_REG_KIND_F32, 0); else if (strcmp(name, "xmm0_f64") == 0) diff --git a/core/iwasm/fast-jit/fe/jit_emit_function.c b/core/iwasm/fast-jit/fe/jit_emit_function.c index 39a532a54..3bf616474 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_function.c +++ b/core/iwasm/fast-jit/fe/jit_emit_function.c @@ -255,13 +255,10 @@ pack_argv(JitCompContext *cc) static bool unpack_argv(JitCompContext *cc, const WASMType *func_type, JitReg argv) { - /* argv to stack*/ - uint32 i, top_by_cell, offset_by_cell; + uint32 i, offset_by_cell = 0; JitReg value; - /* stack top */ - top_by_cell = cc->jit_frame->sp - cc->jit_frame->lp; - offset_by_cell = 0; + /* push results in argv to stack*/ for (i = 0; i < func_type->result_count; i++) { switch (func_type->types[func_type->param_count + i]) { case VALUE_TYPE_I32: @@ -272,40 +269,32 @@ unpack_argv(JitCompContext *cc, const WASMType *func_type, JitReg argv) { value = jit_cc_new_reg_I32(cc); GEN_INSN(LDI32, value, argv, NEW_CONST(I32, offset_by_cell)); - GEN_INSN(STI32, value, cc->fp_reg, - NEW_CONST(I32, offset_of_local(top_by_cell - + offset_by_cell))); - offset_by_cell += 1; + PUSH_I32(value); + offset_by_cell += 4; break; } case VALUE_TYPE_I64: { value = jit_cc_new_reg_I64(cc); GEN_INSN(LDI64, value, argv, NEW_CONST(I32, offset_by_cell)); - GEN_INSN(STI64, value, cc->fp_reg, - NEW_CONST(I32, offset_of_local(top_by_cell - + offset_by_cell))); - offset_by_cell += 2; + PUSH_I64(value); + offset_by_cell += 8; break; } case VALUE_TYPE_F32: { value = jit_cc_new_reg_F32(cc); GEN_INSN(LDF32, value, argv, NEW_CONST(I32, offset_by_cell)); - GEN_INSN(STF32, value, cc->fp_reg, - NEW_CONST(I32, offset_of_local(top_by_cell - + offset_by_cell))); - offset_by_cell += 1; + PUSH_F32(value); + offset_by_cell += 4; break; } case VALUE_TYPE_F64: { value = jit_cc_new_reg_F64(cc); GEN_INSN(LDF64, value, argv, NEW_CONST(I32, offset_by_cell)); - GEN_INSN(STF64, value, cc->fp_reg, - NEW_CONST(I32, offset_of_local(top_by_cell - + offset_by_cell))); - offset_by_cell += 2; + PUSH_F64(value); + offset_by_cell += 8; break; } default: @@ -316,6 +305,9 @@ unpack_argv(JitCompContext *cc, const WASMType *func_type, JitReg argv) } } + /* Update the committed_sp as the callee has updated the frame sp */ + cc->jit_frame->committed_sp = cc->jit_frame->sp; + return true; fail: return false; @@ -325,11 +317,14 @@ bool jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx, uint32 tbl_idx) { - JitReg element_indices, native_ret, argv; + JitReg elem_idx, native_ret, argv; +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + JitReg edx_hreg, r9_hreg; +#endif WASMType *func_type; JitInsn *insn; - POP_I32(element_indices); + POP_I32(elem_idx); func_type = cc->cur_wasm_module->types[type_idx]; if (!pre_call(cc, func_type)) { @@ -341,23 +336,34 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx, #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) /* Set native_ret to x86::eax */ native_ret = jit_codegen_get_hreg_by_name("eax"); + + edx_hreg = jit_codegen_get_hreg_by_name("edx"); + GEN_INSN(MOV, edx_hreg, elem_idx); + elem_idx = edx_hreg; + + r9_hreg = jit_codegen_get_hreg_by_name("r9"); + GEN_INSN(MOV, r9_hreg, argv); + argv = r9_hreg; #else native_ret = jit_cc_new_reg_I32(cc); #endif insn = GEN_INSN(CALLNATIVE, native_ret, - NEW_CONST(PTR, (uintptr_t)wasm_call_indirect), 5); + NEW_CONST(PTR, (uintptr_t)jit_call_indirect), 6); if (!insn) { goto fail; } *(jit_insn_opndv(insn, 2)) = cc->exec_env_reg; *(jit_insn_opndv(insn, 3)) = NEW_CONST(I32, tbl_idx); - *(jit_insn_opndv(insn, 4)) = element_indices; - *(jit_insn_opndv(insn, 5)) = NEW_CONST(I32, func_type->param_count); - *(jit_insn_opndv(insn, 6)) = argv; + *(jit_insn_opndv(insn, 4)) = elem_idx; + *(jit_insn_opndv(insn, 5)) = NEW_CONST(I32, type_idx); + *(jit_insn_opndv(insn, 6)) = NEW_CONST(I32, func_type->param_cell_num); + *(jit_insn_opndv(insn, 7)) = argv; +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) jit_lock_reg_in_insn(cc, insn, native_ret); +#endif /* Check whether there is exception thrown */ GEN_INSN(CMP, cc->cmp_reg, native_ret, NEW_CONST(I32, 0)); @@ -370,10 +376,6 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx, goto fail; } - if (!post_return(cc, func_type)) { - goto fail; - } - /* Clear part of memory regs and table regs as their values may be changed in the function call */ if (cc->cur_wasm_module->possible_memory_grow) diff --git a/core/iwasm/fast-jit/fe/jit_emit_memory.c b/core/iwasm/fast-jit/fe/jit_emit_memory.c index 0a878b095..431a7e896 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_memory.c +++ b/core/iwasm/fast-jit/fe/jit_emit_memory.c @@ -476,6 +476,9 @@ bool jit_compile_op_memory_grow(JitCompContext *cc, uint32 mem_idx) { JitReg delta, module_inst, grow_result, res, memory_inst, prev_page_count; +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + JitReg esi_hreg; +#endif JitInsn *insn; /* WASMMemoryInstance->cur_page_count before enlarging */ @@ -491,15 +494,25 @@ jit_compile_op_memory_grow(JitCompContext *cc, uint32 mem_idx) #else grow_result = jit_cc_new_reg_I32(cc); #endif + POP_I32(delta); +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + esi_hreg = jit_codegen_get_hreg_by_name("esi"); + GEN_INSN(MOV, esi_hreg, delta); + delta = esi_hreg; +#endif + module_inst = get_module_inst_reg(cc->jit_frame); + insn = GEN_INSN(CALLNATIVE, grow_result, NEW_CONST(PTR, (uintptr_t)wasm_enlarge_memory), 2); - if (insn) { - *(jit_insn_opndv(insn, 2)) = module_inst; - *(jit_insn_opndv(insn, 3)) = delta; + if (!insn) { + goto fail; } + *(jit_insn_opndv(insn, 2)) = module_inst; + *(jit_insn_opndv(insn, 3)) = delta; + #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) jit_lock_reg_in_insn(cc, insn, grow_result); #endif diff --git a/core/iwasm/fast-jit/fe/jit_emit_numberic.c b/core/iwasm/fast-jit/fe/jit_emit_numberic.c index ee2cdbdc9..84c96e785 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_numberic.c +++ b/core/iwasm/fast-jit/fe/jit_emit_numberic.c @@ -1322,36 +1322,117 @@ jit_compile_op_i64_shift(JitCompContext *cc, IntShift shift_op) return compile_op_int_shift(cc, shift_op, false); } +static float32 +negf(float32 f32) +{ + return -f32; +} + +static float64 +neg(float64 f64) +{ + return -f64; +} + +static bool +compile_op_float_math(JitCompContext *cc, FloatMath math_op, bool is_f32) +{ + JitReg value, res, xmm0; + JitInsn *insn = NULL; + void *func = NULL; + +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + if (is_f32) { + res = xmm0 = jit_codegen_get_hreg_by_name("xmm0"); + } + else { + res = jit_codegen_get_hreg_by_name("xmm0_f64"); + xmm0 = jit_codegen_get_hreg_by_name("xmm0"); + } +#else + if (is_f32) + res = jit_cc_new_reg_F32(cc); + else + res = jit_cc_new_reg_F64(cc); +#endif + + if (is_f32) + POP_F32(value); + else + POP_F64(value); + + switch (math_op) { + case FLOAT_ABS: + func = is_f32 ? (void *)fabsf : (void *)fabs; + break; + case FLOAT_NEG: + func = is_f32 ? (void *)negf : (void *)neg; + break; + case FLOAT_CEIL: + func = is_f32 ? (void *)ceilf : (void *)ceil; + break; + case FLOAT_FLOOR: + func = is_f32 ? (void *)floorf : (void *)floor; + break; + case FLOAT_TRUNC: + func = is_f32 ? (void *)truncf : (void *)trunc; + break; + case FLOAT_NEAREST: + func = is_f32 ? (void *)rintf : (void *)rint; + break; + case FLOAT_SQRT: + func = is_f32 ? (void *)sqrtf : (void *)sqrt; + break; + } + + insn = GEN_INSN(CALLNATIVE, res, NEW_CONST(PTR, (uintptr_t)func), 1); + if (!insn) { + goto fail; + } + *(jit_insn_opndv(insn, 2)) = value; + + if (is_f32) + PUSH_F32(res); + else + PUSH_F64(res); + +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + jit_lock_reg_in_insn(cc, insn, xmm0); +#endif + return true; +fail: + return false; +} + bool jit_compile_op_f32_math(JitCompContext *cc, FloatMath math_op) { - bh_assert(0); - return false; + return compile_op_float_math(cc, math_op, true); } bool jit_compile_op_f64_math(JitCompContext *cc, FloatMath math_op) { - bh_assert(0); - return false; + return compile_op_float_math(cc, math_op, false); } -bool -jit_compile_op_f32_arithmetic(JitCompContext *cc, FloatArithmetic arith_op) -{ - bh_assert(0); - return false; -} - -bool -jit_compile_op_f64_arithmetic(JitCompContext *cc, FloatArithmetic arith_op) +static bool +compile_op_float_arithmetic(JitCompContext *cc, FloatArithmetic arith_op, + bool is_f32) { JitReg lhs, rhs, res; - POP_F64(rhs); - POP_F64(lhs); + if (is_f32) { + POP_F32(rhs); + POP_F32(lhs); + res = jit_cc_new_reg_F32(cc); + } + else { + POP_F64(rhs); + POP_F64(lhs); + res = jit_cc_new_reg_F64(cc); + } - res = jit_cc_new_reg_F64(cc); switch (arith_op) { case FLOAT_ADD: { @@ -1391,13 +1472,28 @@ jit_compile_op_f64_arithmetic(JitCompContext *cc, FloatArithmetic arith_op) } } - PUSH_F64(res); + if (is_f32) + PUSH_F32(res); + else + PUSH_F64(res); return true; fail: return false; } +bool +jit_compile_op_f32_arithmetic(JitCompContext *cc, FloatArithmetic arith_op) +{ + return compile_op_float_arithmetic(cc, arith_op, true); +} + +bool +jit_compile_op_f64_arithmetic(JitCompContext *cc, FloatArithmetic arith_op) +{ + return compile_op_float_arithmetic(cc, arith_op, false); +} + bool jit_compile_op_f32_copysign(JitCompContext *cc) { diff --git a/core/iwasm/fast-jit/fe/jit_emit_parametric.c b/core/iwasm/fast-jit/fe/jit_emit_parametric.c index 24dd9d264..df0b23a7a 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_parametric.c +++ b/core/iwasm/fast-jit/fe/jit_emit_parametric.c @@ -11,6 +11,7 @@ pop_value_from_wasm_stack(JitCompContext *cc, bool is_32bit, JitReg *p_value, uint8 *p_type) { JitValue *jit_value; + JitReg value; uint8 type; if (!jit_block_stack_top(&cc->block_stack)) { @@ -29,9 +30,6 @@ pop_value_from_wasm_stack(JitCompContext *cc, bool is_32bit, JitReg *p_value, if (p_type != NULL) { *p_type = jit_value->type; } - if (p_value != NULL) { - *p_value = jit_value->value->reg; - } wasm_runtime_free(jit_value); @@ -57,19 +55,25 @@ pop_value_from_wasm_stack(JitCompContext *cc, bool is_32bit, JitReg *p_value, case VALUE_TYPE_FUNCREF: case VALUE_TYPE_EXTERNREF: #endif - pop_i32(cc->jit_frame); + value = pop_i32(cc->jit_frame); break; case VALUE_TYPE_I64: - pop_i64(cc->jit_frame); + value = pop_i64(cc->jit_frame); break; case VALUE_TYPE_F32: - pop_f32(cc->jit_frame); + value = pop_f32(cc->jit_frame); break; case VALUE_TYPE_F64: - pop_f64(cc->jit_frame); + value = pop_f64(cc->jit_frame); break; + default: + bh_assert(0); + return false; } + if (p_value != NULL) { + *p_value = value; + } return true; } @@ -99,10 +103,23 @@ jit_compile_op_select(JitCompContext *cc, bool is_select_32) return false; } - if (is_select_32) - selected = jit_cc_new_reg_I32(cc); - else - selected = jit_cc_new_reg_I64(cc); + switch (val1_type) { + case VALUE_TYPE_I32: + selected = jit_cc_new_reg_I32(cc); + break; + case VALUE_TYPE_I64: + selected = jit_cc_new_reg_I64(cc); + break; + case VALUE_TYPE_F32: + selected = jit_cc_new_reg_F32(cc); + break; + case VALUE_TYPE_F64: + selected = jit_cc_new_reg_F64(cc); + break; + default: + bh_assert(0); + return false; + } GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0)); GEN_INSN(SELECTNE, selected, cc->cmp_reg, val1, val2); diff --git a/core/iwasm/fast-jit/jit_frontend.h b/core/iwasm/fast-jit/jit_frontend.h index 760dd1692..b57f7176d 100644 --- a/core/iwasm/fast-jit/jit_frontend.h +++ b/core/iwasm/fast-jit/jit_frontend.h @@ -396,6 +396,8 @@ clear_values(JitFrame *frame) size_t total_size = sizeof(JitValueSlot) * (frame->max_locals + frame->max_stacks); memset(frame->lp, 0, total_size); + frame->committed_sp = NULL; + frame->committed_ip = NULL; clear_fixed_virtual_regs(frame); } diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index fdf207fe9..54dc9fc5f 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -56,7 +56,7 @@ wasm_load(uint8 *buf, uint32 size, char *error_buf, uint32 error_buf_size) WASMModule * wasm_load_from_sections(WASMSection *section_list, char *error_buf, - uint32_t error_buf_size) + uint32 error_buf_size) { return wasm_loader_load_from_sections(section_list, error_buf, error_buf_size); @@ -1509,8 +1509,8 @@ wasm_instantiate(WASMModule *module, bool is_sub_inst, uint32 stack_size, if (stack_size == 0) stack_size = DEFAULT_WASM_STACK_SIZE; #if WASM_ENABLE_SPEC_TEST != 0 - if (stack_size < 48 * 1024) - stack_size = 48 * 1024; + if (stack_size < 100 * 1024) + stack_size = 100 * 1024; #endif module_inst->default_wasm_stack_size = stack_size; @@ -2270,14 +2270,14 @@ wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx, } #endif /* WASM_ENABLE_REF_TYPES != 0 */ -bool -wasm_call_indirect(WASMExecEnv *exec_env, uint32_t tbl_idx, - uint32_t element_indices, uint32_t argc, uint32_t argv[]) +static bool +call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 argv[], bool check_type_idx, uint32 type_idx) { WASMModuleInstance *module_inst = NULL; WASMTableInstance *table_inst = NULL; - uint32_t function_indices = 0; - WASMFunctionInstance *function_inst = NULL; + uint32 func_idx = 0; + WASMFunctionInstance *func_inst = NULL; module_inst = (WASMModuleInstance *)exec_env->module_inst; bh_assert(module_inst); @@ -2288,7 +2288,7 @@ wasm_call_indirect(WASMExecEnv *exec_env, uint32_t tbl_idx, goto got_exception; } - if (element_indices >= table_inst->cur_size) { + if (elem_idx >= table_inst->cur_size) { wasm_set_exception(module_inst, "undefined element"); goto got_exception; } @@ -2297,8 +2297,8 @@ wasm_call_indirect(WASMExecEnv *exec_env, uint32_t tbl_idx, * please be aware that table_inst->base_addr may point * to another module's table **/ - function_indices = ((uint32_t *)table_inst->base_addr)[element_indices]; - if (function_indices == NULL_REF) { + func_idx = ((uint32 *)table_inst->base_addr)[elem_idx]; + if (func_idx == NULL_REF) { wasm_set_exception(module_inst, "uninitialized element"); goto got_exception; } @@ -2306,14 +2306,29 @@ wasm_call_indirect(WASMExecEnv *exec_env, uint32_t tbl_idx, /** * we insist to call functions owned by the module itself **/ - if (function_indices >= module_inst->function_count) { + if (func_idx >= module_inst->function_count) { wasm_set_exception(module_inst, "unknown function"); goto got_exception; } - function_inst = module_inst->functions + function_indices; + func_inst = module_inst->functions + func_idx; - wasm_interp_call_wasm(module_inst, exec_env, function_inst, argc, argv); + if (check_type_idx) { + WASMType *cur_type = module_inst->module->types[type_idx]; + WASMType *cur_func_type; + + if (func_inst->is_import_func) + cur_func_type = func_inst->u.func_import->func_type; + else + cur_func_type = func_inst->u.func->func_type; + + if (!wasm_type_equal(cur_type, cur_func_type)) { + wasm_set_exception(module_inst, "indirect call type mismatch"); + goto got_exception; + } + } + + wasm_interp_call_wasm(module_inst, exec_env, func_inst, argc, argv); (void)clear_wasi_proc_exit_exception(module_inst); return !wasm_get_exception(module_inst) ? true : false; @@ -2322,6 +2337,23 @@ got_exception: return false; } +bool +wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 argv[]) +{ + return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0); +} + +#if WASM_ENABLE_FAST_JIT != 0 +bool +jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 type_idx, uint32 argc, uint32 argv[]) +{ + return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, true, + type_idx); +} +#endif + #if WASM_ENABLE_THREAD_MGR != 0 bool wasm_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size) diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 28a64e951..5284115a6 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -296,7 +296,7 @@ wasm_load(uint8 *buf, uint32 size, char *error_buf, uint32 error_buf_size); WASMModule * wasm_load_from_sections(WASMSection *section_list, char *error_buf, - uint32_t error_buf_size); + uint32 error_buf_size); void wasm_unload(WASMModule *module); @@ -382,16 +382,22 @@ wasm_get_app_addr_range(WASMModuleInstance *module_inst, uint32 app_offset, uint32 *p_app_start_offset, uint32 *p_app_end_offset); bool -wasm_get_native_addr_range(WASMModuleInstance *module_inst, uint8_t *native_ptr, - uint8_t **p_native_start_addr, - uint8_t **p_native_end_addr); +wasm_get_native_addr_range(WASMModuleInstance *module_inst, uint8 *native_ptr, + uint8 **p_native_start_addr, + uint8 **p_native_end_addr); bool wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count); bool -wasm_call_indirect(WASMExecEnv *exec_env, uint32_t tbl_idx, - uint32_t element_indices, uint32_t argc, uint32_t argv[]); +wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 argc, uint32 argv[]); + +#if WASM_ENABLE_FAST_JIT != 0 +bool +jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, + uint32 type_idx, uint32 argc, uint32 argv[]); +#endif #if WASM_ENABLE_THREAD_MGR != 0 bool