diff --git a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp index 81eb3208b..a261807ad 100644 --- a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp +++ b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp @@ -237,7 +237,7 @@ typedef enum { ADD, SUB, MUL, DIV_S, REM_S, DIV_U, REM_U } ALU_OP; /* Bit opcode */ typedef enum { OR, XOR, AND } BIT_OP; /* Shift opcode */ -typedef enum { SHL, SHRS, SHRU } SHIFT_OP; +typedef enum { SHL, SHRS, SHRU, ROTL, ROTR } SHIFT_OP; /* Condition opcode */ typedef enum { EQ, NE, GTS, GES, LTS, LES, GTU, GEU, LTU, LEU } COND_OP; @@ -2722,7 +2722,28 @@ bit_r_r_to_r_i32(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, static bool bit_r_imm_i64(x86::Assembler &a, BIT_OP op, int32 reg_no, int64 data) { - return false; + Imm imm(data); + + switch (op) { + case OR: + if (data != 0) + a.or_(regs_i64[reg_no], imm); + break; + case XOR: + if (data == -1LL) + a.not_(regs_i64[reg_no]); + else if (data != 0) + a.xor_(regs_i64[reg_no], imm); + break; + case AND: + if (data != -1LL) + a.and_(regs_i64[reg_no], imm); + break; + default: + bh_assert(0); + break; + } + return true; } /** @@ -2738,7 +2759,21 @@ bit_r_imm_i64(x86::Assembler &a, BIT_OP op, int32 reg_no, int64 data) static bool bit_r_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, int32 reg_no_src) { - return false; + switch (op) { + case OR: + a.or_(regs_i64[reg_no_dst], regs_i64[reg_no_src]); + break; + case XOR: + a.xor_(regs_i64[reg_no_dst], regs_i64[reg_no_src]); + break; + case AND: + a.and_(regs_i64[reg_no_dst], regs_i64[reg_no_src]); + break; + default: + bh_assert(0); + break; + } + return true; } /** @@ -2756,7 +2791,25 @@ static bool bit_imm_imm_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, int32 data1_src, int64 data2_src) { - return false; + Imm imm; + + switch (op) { + case OR: + imm.setValue(data1_src | data2_src); + break; + case XOR: + imm.setValue(data1_src ^ data2_src); + break; + case AND: + imm.setValue(data1_src & data2_src); + break; + default: + bh_assert(0); + break; + } + + a.mov(regs_i64[reg_no_dst], imm); + return true; } /** @@ -2774,7 +2827,17 @@ static bool bit_imm_r_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, int64 data1_src, int32 reg_no2_src) { - return false; + if (op == AND && data1_src == 0) + a.xor_(regs_i64[reg_no_dst], regs_i64[reg_no_dst]); + else if (op == OR && data1_src == -1LL) { + Imm imm(-1LL); + a.mov(regs_i64[reg_no_dst], imm); + } + else { + mov_r_to_r_i64(a, reg_no_dst, reg_no2_src); + return bit_r_imm_i64(a, op, reg_no_dst, data1_src); + } + return true; } /** @@ -2792,7 +2855,7 @@ static bool bit_r_imm_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, int32 reg_no1_src, int64 data2_src) { - return false; + return bit_imm_r_to_r_i64(a, op, reg_no_dst, data2_src, reg_no1_src); } /** @@ -2810,6 +2873,12 @@ static bool bit_r_r_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, int32 reg_no1_src, int32 reg_no2_src) { + if (reg_no_dst != reg_no2_src) { + mov_r_to_r_i64(a, reg_no_dst, reg_no1_src); + return bit_r_r_i64(a, op, reg_no_dst, reg_no2_src); + } + else + return bit_r_r_i64(a, op, reg_no_dst, reg_no1_src); return false; } @@ -2845,6 +2914,18 @@ shift_imm_imm_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, data = ((uint32)data1_src) >> data2_src; break; } + case ROTL: + { + data = (data1_src << data2_src) + | (((uint32)data1_src) >> (32 - data2_src)); + break; + } + case ROTR: + { + data = (((uint32)data1_src) >> data2_src) + | (data1_src << (32 - data2_src)); + break; + } default: { bh_assert(0); @@ -2895,20 +2976,31 @@ shift_r_imm_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, /* SHL/SHA/SHR r/m32, imm8 */ Imm imm((uint8)data2_src); + mov_r_to_r_i32(a, reg_no_dst, reg_no1_src); switch (op) { case SHL: { - a.shl(regs_i32[reg_no1_src], imm); + a.shl(regs_i32[reg_no_dst], imm); break; } case SHRS: { - a.sar(regs_i32[reg_no1_src], imm); + a.sar(regs_i32[reg_no_dst], imm); break; } case SHRU: { - a.shr(regs_i32[reg_no1_src], imm); + a.shr(regs_i32[reg_no_dst], imm); + break; + } + case ROTL: + { + a.rol(regs_i32[reg_no_dst], imm); + break; + } + case ROTR: + { + a.ror(regs_i32[reg_no_dst], imm); break; } default: @@ -2918,7 +3010,7 @@ shift_r_imm_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, } } - return mov_r_to_r_i32(a, reg_no_dst, reg_no1_src); + return true; fail: return false; } @@ -2941,20 +3033,32 @@ shift_r_r_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, /* should be CL */ bh_assert(reg_no2_src == REG_ECX_IDX); + mov_r_to_r_i32(a, reg_no_dst, reg_no1_src); + switch (op) { case SHL: { - a.shl(regs_i32[reg_no1_src], x86::cl); + a.shl(regs_i32[reg_no_dst], x86::cl); break; } case SHRS: { - a.sar(regs_i32[reg_no1_src], x86::cl); + a.sar(regs_i32[reg_no_dst], x86::cl); break; } case SHRU: { - a.shr(regs_i32[reg_no1_src], x86::cl); + a.shr(regs_i32[reg_no_dst], x86::cl); + break; + } + case ROTL: + { + a.rol(regs_i32[reg_no_dst], x86::cl); + break; + } + case ROTR: + { + a.ror(regs_i32[reg_no_dst], x86::cl); break; } default: @@ -2964,7 +3068,7 @@ shift_r_r_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, } } - return mov_r_to_r_i32(a, reg_no_dst, reg_no1_src); + return true; fail: return false; } @@ -3002,6 +3106,18 @@ shift_imm_imm_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, data = ((uint64)data1_src) >> data2_src; break; } + case ROTL: + { + data = (data1_src << data2_src) + | (((uint64)data1_src) >> (64LL - data2_src)); + break; + } + case ROTR: + { + data = (((uint64)data1_src) >> data2_src) + | (data1_src << (64LL - data2_src)); + break; + } default: { bh_assert(0); @@ -3052,20 +3168,31 @@ shift_r_imm_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, /* SHL/SHA/SHR r/m64, imm8 */ Imm imm((uint8)data2_src); + mov_r_to_r_i64(a, reg_no_dst, reg_no1_src); switch (op) { case SHL: { - a.shl(regs_i64[reg_no1_src], imm); + a.shl(regs_i64[reg_no_dst], imm); break; } case SHRS: { - a.sar(regs_i64[reg_no1_src], imm); + a.sar(regs_i64[reg_no_dst], imm); break; } case SHRU: { - a.shr(regs_i64[reg_no1_src], imm); + a.shr(regs_i64[reg_no_dst], imm); + break; + } + case ROTL: + { + a.ror(regs_i64[reg_no_dst], imm); + break; + } + case ROTR: + { + a.ror(regs_i64[reg_no_dst], imm); break; } default: @@ -3075,7 +3202,7 @@ shift_r_imm_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, } } - return mov_r_to_r_i64(a, reg_no_dst, reg_no1_src); + return true; fail: return false; } @@ -3098,20 +3225,32 @@ shift_r_r_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, /* should be CL */ bh_assert(reg_no2_src == REG_ECX_IDX); + mov_r_to_r_i64(a, reg_no_dst, reg_no1_src); + switch (op) { case SHL: { - a.shl(regs_i64[reg_no1_src], x86::cl); + a.shl(regs_i64[reg_no_dst], x86::cl); break; } case SHRS: { - a.sar(regs_i64[reg_no1_src], x86::cl); + a.sar(regs_i64[reg_no_dst], x86::cl); break; } case SHRU: { - a.shr(regs_i64[reg_no1_src], x86::cl); + a.shr(regs_i64[reg_no_dst], x86::cl); + break; + } + case ROTL: + { + a.rol(regs_i64[reg_no_dst], x86::cl); + break; + } + case ROTR: + { + a.ror(regs_i64[reg_no_dst], x86::cl); break; } default: @@ -3121,7 +3260,7 @@ shift_r_r_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, } } - return mov_r_to_r_i64(a, reg_no_dst, reg_no1_src); + return true; fail: return false; } @@ -4215,19 +4354,24 @@ fail: } /* jmp to dst label */ -#define JMP_TO_LABEL(label_dst, label_src) \ - do { \ - if (label_is_ahead(cc, label_dst, label_src)) { \ - int32 _offset = label_offsets[label_dst] \ - - a.code()->sectionById(0)->buffer().size(); \ - Imm imm(_offset); \ - a.jmp(imm); \ - } \ - else { \ - if (!jmp_from_label_to_label(a, jmp_info_list, label_dst, \ - label_src)) \ - GOTO_FAIL; \ - } \ +#define JMP_TO_LABEL(label_dst, label_src) \ + do { \ + if (label_is_ahead(cc, label_dst, label_src)) { \ + char *stream = (char *)a.code()->sectionById(0)->buffer().data() \ + + a.code()->sectionById(0)->buffer().size(); \ + int32 _offset = label_offsets[label_dst] \ + - a.code()->sectionById(0)->buffer().size(); \ + Imm imm(INT32_MAX); \ + a.jmp(imm); \ + /* The offset written by asmjit is always 0, we patch it again, \ + 6 is the size of jmp instruciton */ \ + *(int32 *)(stream + 2) = _offset - 6; \ + } \ + else { \ + if (!jmp_from_label_to_label(a, jmp_info_list, label_dst, \ + label_src)) \ + GOTO_FAIL; \ + } \ } while (0) /** @@ -4841,6 +4985,8 @@ jit_codegen_gen_native(JitCompContext *cc) case JIT_OP_SHL: case JIT_OP_SHRS: case JIT_OP_SHRU: + case JIT_OP_ROTL: + case JIT_OP_ROTR: LOAD_3ARGS(); if (!lower_shift( cc, a, @@ -5217,7 +5363,7 @@ static const uint8 hreg_info_I32[3][7] = { /* ebp, eax, ebx, ecx, edx, edi, esi */ { 1, 0, 0, 0, 0, 0, 1 }, /* fixed, esi is freely used */ { 0, 1, 0, 1, 1, 1, 0 }, /* caller_saved_native */ - { 0, 1, 0, 1, 1, 1, 0 } /* caller_saved_jitted */ + { 0, 1, 1, 1, 1, 1, 0 } /* caller_saved_jitted */ }; static const uint8 hreg_info_I64[3][16] = { diff --git a/core/iwasm/fast-jit/fe/jit_emit_control.c b/core/iwasm/fast-jit/fe/jit_emit_control.c index 671a605cd..69d34720a 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_control.c +++ b/core/iwasm/fast-jit/fe/jit_emit_control.c @@ -241,7 +241,7 @@ push_jit_block_to_stack_and_pass_params(JitCompContext *cc, JitBlock *block, block->incoming_insn_for_else_bb = insn; } else { - if (!jit_block_add_incoming_insn(block, insn)) { + if (!jit_block_add_incoming_insn(block, insn, 2)) { jit_set_last_error(cc, "add incoming insn failed"); goto fail; } @@ -449,17 +449,9 @@ handle_op_end(JitCompContext *cc, uint8 **p_frame_ip, bool is_block_polymorphic) incoming_insn = block->incoming_insns_for_end_bb; while (incoming_insn) { insn = incoming_insn->insn; - if (insn->opcode == JIT_OP_JMP) { - *(jit_insn_opnd(insn, 0)) = - jit_basic_block_label(block->basic_block_end); - } - else if (insn->opcode == JIT_OP_BNE) { - *(jit_insn_opnd(insn, 2)) = - jit_basic_block_label(block->basic_block_end); - } - else { - bh_assert(0); - } + bh_assert(insn->opcode == JIT_OP_JMP || insn->opcode == JIT_OP_BNE); + *(jit_insn_opnd(insn, incoming_insn->opnd_idx)) = + jit_basic_block_label(block->basic_block_end); incoming_insn = incoming_insn->next; } @@ -534,7 +526,7 @@ handle_op_else(JitCompContext *cc, uint8 **p_frame_ip, jit_set_last_error(cc, "generate jmp insn failed"); return false; } - if (!jit_block_add_incoming_insn(block, insn)) { + if (!jit_block_add_incoming_insn(block, insn, 0)) { jit_set_last_error(cc, "add incoming insn failed"); return false; } @@ -821,7 +813,7 @@ handle_op_br(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip) jit_set_last_error(cc, "generate jmp insn failed"); goto fail; } - if (!jit_block_add_incoming_insn(block_dst, insn)) { + if (!jit_block_add_incoming_insn(block_dst, insn, 0)) { jit_set_last_error(cc, "add incoming insn failed"); goto fail; } @@ -847,6 +839,8 @@ jit_compile_op_br_if(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip) JitBlock *block_dst; JitReg cond; JitBasicBlock *cur_basic_block, *if_basic_block = NULL; + JitValueSlot *frame_sp_src; + JitInsn *insn; if (!(block_dst = get_target_block(cc, br_depth))) { return false; @@ -861,6 +855,41 @@ jit_compile_op_br_if(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip) /* Clear frame values */ clear_values(jit_frame); + if (block_dst->label_type == LABEL_TYPE_LOOP) { + frame_sp_src = + jit_frame->sp + - wasm_get_cell_num(block_dst->param_types, block_dst->param_count); + } + else { + frame_sp_src = jit_frame->sp + - wasm_get_cell_num(block_dst->result_types, + block_dst->result_count); + } + + if (block_dst->frame_sp_begin == frame_sp_src) { + if (block_dst->label_type == LABEL_TYPE_LOOP) { + if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0)) + || !GEN_INSN( + BNE, cc->cmp_reg, + jit_basic_block_label(block_dst->basic_block_entry), 0)) { + jit_set_last_error(cc, "generate bne insn failed"); + goto fail; + } + } + else { + if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0)) + || !(insn = GEN_INSN(BNE, cc->cmp_reg, 0, 0))) { + jit_set_last_error(cc, "generate bne insn failed"); + goto fail; + } + if (!jit_block_add_incoming_insn(block_dst, insn, 1)) { + jit_set_last_error(cc, "add incoming insn failed"); + goto fail; + } + } + return true; + } + CREATE_BASIC_BLOCK(if_basic_block); if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0)) || !GEN_INSN(BNE, cc->cmp_reg, jit_basic_block_label(if_basic_block), @@ -912,6 +941,8 @@ jit_compile_op_br_table(JitCompContext *cc, uint32 *br_depths, uint32 br_count, for (i = 0, opnd = jit_insn_opndls(insn); i < br_count + 1; i++) { JitBasicBlock *basic_block = NULL; + /* TODO: refine the code */ + CREATE_BASIC_BLOCK(basic_block); SET_BB_BEGIN_BCIP(basic_block, *p_frame_ip - 1); diff --git a/core/iwasm/fast-jit/fe/jit_emit_function.c b/core/iwasm/fast-jit/fe/jit_emit_function.c index 802ad718f..c25265d1d 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_function.c +++ b/core/iwasm/fast-jit/fe/jit_emit_function.c @@ -112,6 +112,9 @@ post_return(JitCompContext *cc, const WASMType *func_type) } } + /* Update the committed_sp as the callee has updated the frame sp */ + cc->jit_frame->committed_sp = cc->jit_frame->sp; + return true; fail: return false; @@ -127,6 +130,10 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call) JitFrame *jit_frame = cc->jit_frame; JitReg result = 0, native_ret; JitReg func_ptrs, jitted_code = 0; +#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) + JitReg eax_hreg = jit_codegen_get_hreg_by_name("eax"); + JitReg rax_hreg = jit_codegen_get_hreg_by_name("rax"); +#endif JitInsn *insn; uint32 jitted_func_idx; @@ -156,7 +163,7 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call) if (func_idx < wasm_module->import_function_count) { #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) /* Set native_ret to x86::eax */ - native_ret = jit_codegen_get_hreg_by_name("eax"); + native_ret = eax_hreg; #else native_ret = jit_cc_new_reg_I32(cc); #endif @@ -184,16 +191,14 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call) case VALUE_TYPE_FUNCREF: #endif #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) - /* Set result to x86::eax, 1 is hard reg index of eax */ - result = jit_reg_new(JIT_REG_KIND_I32, 1); + result = eax_hreg; #else result = jit_cc_new_reg_I32(cc); #endif break; case VALUE_TYPE_I64: #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) - /* Set result to x86::rax, 1 is hard reg index of rax */ - result = jit_reg_new(JIT_REG_KIND_I64, 1); + result = rax_hreg; #else result = jit_cc_new_reg_I64(cc); #endif @@ -219,7 +224,8 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call) /* Clear part of memory regs and table regs as their values may be changed in the function call */ - clear_memory_regs(jit_frame); + if (cc->cur_wasm_module->possible_memory_grow) + clear_memory_regs(jit_frame); clear_table_regs(jit_frame); /* Ignore tail call currently */ @@ -364,7 +370,8 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx, /* Clear part of memory regs and table regs as their values may be changed in the function call */ - clear_memory_regs(cc->jit_frame); + if (cc->cur_wasm_module->possible_memory_grow) + clear_memory_regs(cc->jit_frame); clear_table_regs(cc->jit_frame); return true; fail: diff --git a/core/iwasm/fast-jit/fe/jit_emit_numberic.c b/core/iwasm/fast-jit/fe/jit_emit_numberic.c index 768964dee..204a6f54f 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_numberic.c +++ b/core/iwasm/fast-jit/fe/jit_emit_numberic.c @@ -182,7 +182,7 @@ compile_op_ibinopt_const(JitCompContext *cc, JitReg left, JitReg right, if (jit_reg_is_const(left) || jit_reg_is_const(right)) { res = handle_one_const(cc, left, right, is_i32); - if (!res) + if (res) goto shortcut; } @@ -787,25 +787,44 @@ jit_compile_op_i64_bitwise(JitCompContext *cc, IntBitwise bitwise_op) DEF_UNI_INT_CONST_OPS(shl) { - if (IS_CONST_ZERO(right)) { + if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)) { return left; } + + if (jit_reg_is_const(right)) { + JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(SHL, res, left, right); + return res; + } return 0; } DEF_UNI_INT_CONST_OPS(shrs) { - if (IS_CONST_ZERO(right)) { + if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left) + || IS_CONST_ALL_ONE(left, is_i32)) { return left; } + + if (jit_reg_is_const(right)) { + JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(SHRS, res, left, right); + return res; + } return 0; } DEF_UNI_INT_CONST_OPS(shru) { - if (IS_CONST_ZERO(right)) { + if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)) { return left; } + + if (jit_reg_is_const(right)) { + JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(SHRU, res, left, right); + return res; + } return 0; } @@ -958,11 +977,15 @@ shortcut: DEF_UNI_INT_CONST_OPS(rotl) { - if (IS_CONST_ZERO(right)) + if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left) + || IS_CONST_ALL_ONE(left, is_i32)) return left; - if (IS_CONST_ZERO(left)) - return right; + if (jit_reg_is_const(right)) { + JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(ROTL, res, left, right); + return res; + } return 0; } @@ -986,7 +1009,7 @@ do_i64_const_rotl(int64 lhs, int64 rhs) static JitReg compile_int_rotl(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) { - JitReg res, tmp, shl_res, shr_res; + JitReg res; #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) JitReg ecx_hreg = jit_codegen_get_hreg_by_name("ecx"); JitReg rcx_hreg = jit_codegen_get_hreg_by_name("rcx"); @@ -1000,34 +1023,13 @@ compile_int_rotl(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) left = mov_left_to_reg(cc, is_i32, left); - if (is_i32) { - tmp = jit_cc_new_reg_I32(cc); - shl_res = jit_cc_new_reg_I32(cc); - shr_res = jit_cc_new_reg_I32(cc); - res = jit_cc_new_reg_I32(cc); - } - else { - tmp = jit_cc_new_reg_I64(cc); - shl_res = jit_cc_new_reg_I64(cc); - shr_res = jit_cc_new_reg_I64(cc); - res = jit_cc_new_reg_I64(cc); - } - - /* 32/64 - rhs */ - GEN_INSN(SUB, tmp, is_i32 ? NEW_CONST(I32, 32) : NEW_CONST(I64, 64), right); + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, right); - GEN_INSN(SHL, shl_res, left, is_i32 ? ecx_hreg : rcx_hreg); - - GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, tmp); - GEN_INSN(SHRU, shr_res, left, is_i32 ? ecx_hreg : rcx_hreg); - - GEN_INSN(OR, res, shl_res, shr_res); + GEN_INSN(ROTL, res, left, is_i32 ? ecx_hreg : rcx_hreg); GEN_INSN(MOV, ecx_hreg, ecx_hreg); #else - GEN_INSN(SHL, shl_res, left, right); - GEN_INSN(SHRU, shr_res, left, tmp); - GEN_INSN(OR, res, shl_res, shr_res); + GEN_INSN(ROTL, res, left, right); #endif shortcut: @@ -1036,11 +1038,15 @@ shortcut: DEF_UNI_INT_CONST_OPS(rotr) { - if (IS_CONST_ZERO(right)) + if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left) + || IS_CONST_ALL_ONE(left, is_i32)) return left; - if (IS_CONST_ZERO(left)) - return right; + if (jit_reg_is_const(right)) { + JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); + GEN_INSN(ROTR, res, left, right); + return res; + } return 0; } @@ -1064,7 +1070,7 @@ do_i64_const_rotr(int64 lhs, int64 rhs) static JitReg compile_int_rotr(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) { - JitReg res, tmp, shr_res, shl_res; + JitReg res; #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) JitReg ecx_hreg = jit_codegen_get_hreg_by_name("ecx"); JitReg rcx_hreg = jit_codegen_get_hreg_by_name("rcx"); @@ -1078,34 +1084,13 @@ compile_int_rotr(JitCompContext *cc, JitReg left, JitReg right, bool is_i32) left = mov_left_to_reg(cc, is_i32, left); - if (is_i32) { - tmp = jit_cc_new_reg_I32(cc); - shr_res = jit_cc_new_reg_I32(cc); - shl_res = jit_cc_new_reg_I32(cc); - res = jit_cc_new_reg_I32(cc); - } - else { - tmp = jit_cc_new_reg_I64(cc); - shr_res = jit_cc_new_reg_I64(cc); - shl_res = jit_cc_new_reg_I64(cc); - res = jit_cc_new_reg_I64(cc); - } - - /* 32/64 - rhs */ - GEN_INSN(SUB, tmp, is_i32 ? NEW_CONST(I32, 32) : NEW_CONST(I64, 64), right); + res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc); #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, right); - GEN_INSN(SHRU, shl_res, left, is_i32 ? ecx_hreg : rcx_hreg); - - GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, tmp); - GEN_INSN(SHL, shr_res, left, is_i32 ? ecx_hreg : rcx_hreg); - - GEN_INSN(OR, res, shl_res, shr_res); + GEN_INSN(ROTR, res, left, is_i32 ? ecx_hreg : rcx_hreg); GEN_INSN(MOV, ecx_hreg, ecx_hreg); #else - GEN_INSN(SHRU, shr_res, left, right); - GEN_INSN(SHL, shl_res, left, tmp); - GEN_INSN(OR, res, shr_res, shl_res); + GEN_INSN(ROTR, res, left, right); #endif shortcut: diff --git a/core/iwasm/fast-jit/fe/jit_emit_parametric.c b/core/iwasm/fast-jit/fe/jit_emit_parametric.c index 0cea93ef5..9d04ada5d 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_parametric.c +++ b/core/iwasm/fast-jit/fe/jit_emit_parametric.c @@ -104,7 +104,8 @@ jit_compile_op_select(JitCompContext *cc, bool is_select_32) else selected = jit_cc_new_reg_I64(cc); - GEN_INSN(SELECTNE, selected, cond, val1, val2); + GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0)); + GEN_INSN(SELECTNE, selected, cc->cmp_reg, val1, val2); PUSH(selected, val1_type); return true; fail: diff --git a/core/iwasm/fast-jit/fe/jit_emit_variable.c b/core/iwasm/fast-jit/fe/jit_emit_variable.c index 5439beb60..32b8dfca2 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_variable.c +++ b/core/iwasm/fast-jit/fe/jit_emit_variable.c @@ -4,6 +4,7 @@ */ #include "jit_emit_variable.h" +#include "jit_emit_exception.h" #include "../jit_frontend.h" #define CHECK_LOCAL(idx) \ @@ -248,11 +249,6 @@ jit_compile_op_set_global(JitCompContext *cc, uint32 global_idx, uint8 global_type = 0; JitReg value = 0; - if (is_aux_stack) { - jit_set_last_error(cc, "doesn't support set global_aux_stack"); - goto fail; - } - bh_assert(global_idx < cc->cur_wasm_module->import_global_count + cc->cur_wasm_module->global_count); @@ -266,6 +262,19 @@ jit_compile_op_set_global(JitCompContext *cc, uint32 global_idx, #endif { POP_I32(value); + if (is_aux_stack) { + JitReg aux_stack_bound = get_aux_stack_bound_reg(cc->jit_frame); + JitReg aux_stack_bottom = + get_aux_stack_bottom_reg(cc->jit_frame); + GEN_INSN(CMP, cc->cmp_reg, value, aux_stack_bound); + if (!(jit_emit_exception(cc, EXCE_AUX_STACK_OVERFLOW, + JIT_OP_BLEU, cc->cmp_reg, NULL))) + goto fail; + GEN_INSN(CMP, cc->cmp_reg, value, aux_stack_bottom); + if (!(jit_emit_exception(cc, EXCE_AUX_STACK_UNDERFLOW, + JIT_OP_BGTU, cc->cmp_reg, NULL))) + goto fail; + } GEN_INSN(STI32, value, get_global_data_reg(cc->jit_frame), NEW_CONST(I32, data_offset)); break; diff --git a/core/iwasm/fast-jit/jit_frontend.c b/core/iwasm/fast-jit/jit_frontend.c index 0a7c38b5d..602ce52a7 100644 --- a/core/iwasm/fast-jit/jit_frontend.c +++ b/core/iwasm/fast-jit/jit_frontend.c @@ -100,15 +100,29 @@ get_global_data_reg(JitFrame *frame) JitReg get_aux_stack_bound_reg(JitFrame *frame) { - /* TODO */ - return 0; + JitCompContext *cc = frame->cc; + + if (!frame->aux_stack_bound_reg) { + frame->aux_stack_bound_reg = cc->aux_stack_bound_reg; + GEN_INSN( + LDI32, frame->aux_stack_bound_reg, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, aux_stack_boundary.boundary))); + } + return frame->aux_stack_bound_reg; } JitReg get_aux_stack_bottom_reg(JitFrame *frame) { - /* TODO */ - return 0; + JitCompContext *cc = frame->cc; + + if (!frame->aux_stack_bottom_reg) { + frame->aux_stack_bottom_reg = cc->aux_stack_bottom_reg; + GEN_INSN( + LDI32, frame->aux_stack_bottom_reg, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, aux_stack_bottom.bottom))); + } + return frame->aux_stack_bottom_reg; } JitReg @@ -494,11 +508,13 @@ gen_commit_sp_ip(JitFrame *frame) frame->committed_sp = frame->sp; } +#if 0 /* Disable committing ip currently */ if (frame->ip != frame->committed_ip) { GEN_INSN(STPTR, NEW_CONST(PTR, (uintptr_t)frame->ip), cc->fp_reg, NEW_CONST(I32, offsetof(WASMInterpFrame, ip))); frame->committed_ip = frame->ip; } +#endif } static void @@ -691,6 +707,7 @@ init_func_translation(JitCompContext *cc) + (uint64)cur_wasm_func->max_stack_cell_num + ((uint64)cur_wasm_func->max_block_num) * sizeof(WASMBranchBlock) / 4; uint32 frame_size, outs_size, local_size, count; + uint32 i, local_off; uint64 total_size; if ((uint64)max_locals + (uint64)max_stacks >= UINT32_MAX @@ -796,6 +813,18 @@ init_func_translation(JitCompContext *cc) /* fp_reg = top */ GEN_INSN(MOV, cc->fp_reg, top); + /* Initialize local variables, set them to 0 */ + local_off = (uint32)offsetof(WASMInterpFrame, lp) + + cur_wasm_func->param_cell_num * 4; + for (i = 0; i < cur_wasm_func->local_cell_num / 2; i++, local_off += 8) { + GEN_INSN(STI64, NEW_CONST(I32, 0), cc->fp_reg, + NEW_CONST(I32, local_off)); + } + if (cur_wasm_func->local_cell_num & 1) { + GEN_INSN(STI32, NEW_CONST(I32, 0), cc->fp_reg, + NEW_CONST(I32, local_off)); + } + return jit_frame; } @@ -1084,15 +1113,11 @@ jit_compile_func(JitCompContext *cc) read_leb_uint32(frame_ip, frame_ip_end, type_idx); #if WASM_ENABLE_REF_TYPES != 0 - if (cc->enable_ref_types) { - read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); - } - else + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); +#else + frame_ip++; + tbl_idx = 0; #endif - { - frame_ip++; - tbl_idx = 0; - } if (!jit_compile_op_call_indirect(cc, type_idx, tbl_idx)) return false; @@ -1123,15 +1148,11 @@ jit_compile_func(JitCompContext *cc) read_leb_uint32(frame_ip, frame_ip_end, type_idx); #if WASM_ENABLE_REF_TYPES != 0 - if (cc->enable_ref_types) { - read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); - } - else + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); +#else + frame_ip++; + tbl_idx = 0; #endif - { - frame_ip++; - tbl_idx = 0; - } if (!jit_compile_op_call_indirect(cc, type_idx, tbl_idx)) return false; @@ -1166,10 +1187,6 @@ jit_compile_func(JitCompContext *cc) { uint32 vec_len; - if (!cc->enable_ref_types) { - goto unsupport_ref_types; - } - read_leb_uint32(frame_ip, frame_ip_end, vec_len); bh_assert(vec_len == 1); (void)vec_len; @@ -1185,10 +1202,6 @@ jit_compile_func(JitCompContext *cc) { uint32 tbl_idx; - if (!cc->enable_ref_types) { - goto unsupport_ref_types; - } - read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); if (!jit_compile_op_table_get(cc, tbl_idx)) return false; @@ -1198,10 +1211,6 @@ jit_compile_func(JitCompContext *cc) { uint32 tbl_idx; - if (!cc->enable_ref_types) { - goto unsupport_ref_types; - } - read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); if (!jit_compile_op_table_set(cc, tbl_idx)) return false; @@ -1211,10 +1220,6 @@ jit_compile_func(JitCompContext *cc) { uint32 type; - if (!cc->enable_ref_types) { - goto unsupport_ref_types; - } - read_leb_uint32(frame_ip, frame_ip_end, type); if (!jit_compile_op_ref_null(cc)) @@ -1225,20 +1230,12 @@ jit_compile_func(JitCompContext *cc) } case WASM_OP_REF_IS_NULL: { - if (!cc->enable_ref_types) { - goto unsupport_ref_types; - } - if (!jit_compile_op_ref_is_null(cc)) return false; break; } case WASM_OP_REF_FUNC: { - if (!cc->enable_ref_types) { - goto unsupport_ref_types; - } - read_leb_uint32(frame_ip, frame_ip_end, func_idx); if (!jit_compile_op_ref_func(cc, func_idx)) return false; @@ -1760,21 +1757,6 @@ jit_compile_func(JitCompContext *cc) read_leb_uint32(frame_ip, frame_ip_end, opcode1); opcode = (uint32)opcode1; -#if WASM_ENABLE_BULK_MEMORY != 0 - if (WASM_OP_MEMORY_INIT <= opcode - && opcode <= WASM_OP_MEMORY_FILL - && !cc->enable_bulk_memory) { - goto unsupport_bulk_memory; - } -#endif - -#if WASM_ENABLE_REF_TYPES != 0 - if (WASM_OP_TABLE_INIT <= opcode && opcode <= WASM_OP_TABLE_FILL - && !cc->enable_ref_types) { - goto unsupport_ref_types; - } -#endif - switch (opcode) { case WASM_OP_I32_TRUNC_SAT_S_F32: case WASM_OP_I32_TRUNC_SAT_U_F32: @@ -2063,21 +2045,6 @@ jit_compile_func(JitCompContext *cc) (void)func_idx; return true; - -#if WASM_ENABLE_REF_TYPES != 0 -unsupport_ref_types: - jit_set_last_error(cc, "reference type instruction was found, " - "try removing --disable-ref-types option"); - return false; -#endif - -#if WASM_ENABLE_BULK_MEMORY != 0 -unsupport_bulk_memory: - jit_set_last_error(cc, "bulk memory instruction was found, " - "try removing --disable-bulk-memory option"); - return false; -#endif - fail: return false; } diff --git a/core/iwasm/fast-jit/jit_ir.c b/core/iwasm/fast-jit/jit_ir.c index 3c09079f3..195180b1f 100644 --- a/core/iwasm/fast-jit/jit_ir.c +++ b/core/iwasm/fast-jit/jit_ir.c @@ -1394,7 +1394,7 @@ jit_block_stack_destroy(JitBlockStack *stack) } bool -jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn) +jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn, uint32 opnd_idx) { JitIncomingInsn *incoming_insn; @@ -1402,6 +1402,7 @@ jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn) return false; incoming_insn->insn = insn; + incoming_insn->opnd_idx = opnd_idx; incoming_insn->next = block->incoming_insns_for_end_bb; block->incoming_insns_for_end_bb = incoming_insn; return true; diff --git a/core/iwasm/fast-jit/jit_ir.def b/core/iwasm/fast-jit/jit_ir.def index 6c87eabaf..3db0c01ff 100644 --- a/core/iwasm/fast-jit/jit_ir.def +++ b/core/iwasm/fast-jit/jit_ir.def @@ -109,6 +109,8 @@ INSN(REM_U, Reg, 3, 1) INSN(SHL, Reg, 3, 1) INSN(SHRS, Reg, 3, 1) INSN(SHRU, Reg, 3, 1) +INSN(ROTL, Reg, 3, 1) +INSN(ROTR, Reg, 3, 1) INSN(OR, Reg, 3, 1) INSN(XOR, Reg, 3, 1) INSN(AND, Reg, 3, 1) diff --git a/core/iwasm/fast-jit/jit_ir.h b/core/iwasm/fast-jit/jit_ir.h index 226e5bbe1..17c798b7f 100644 --- a/core/iwasm/fast-jit/jit_ir.h +++ b/core/iwasm/fast-jit/jit_ir.h @@ -974,6 +974,7 @@ typedef struct JitFrame { typedef struct JitIncomingInsn { struct JitIncomingInsn *next; JitInsn *insn; + uint32 opnd_idx; } JitIncomingInsn, *JitIncomingInsnList; typedef struct JitBlock { @@ -1865,7 +1866,7 @@ void jit_block_stack_destroy(JitBlockStack *stack); bool -jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn); +jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn, uint32 opnd_idx); void jit_block_destroy(JitBlock *block); diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index a38500513..c6760b945 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -837,15 +837,8 @@ jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, WASMModuleInstance *module_inst = (WASMModuleInstance *)exec_env->module_inst; WASMFunctionInstance *cur_func = module_inst->functions + func_idx; - uint32 *sp_org; - sp_org = prev_frame->sp; wasm_interp_call_func_native(module_inst, exec_env, cur_func, prev_frame); - /* Restore the stack pointer of previous frame as the caller in - jitted code will just read the return value and won't decrease - the stack pointer */ - prev_frame->sp = sp_org; - return wasm_get_exception(module_inst) ? false : true; } #endif