Fix fast jit issues (#1169)

Implement bitwise 64-bit operations in codegen
Fix and refine shift IRs
Zero local variables
Remove ref-type/bulk-memory macros
Implement set aux stack
Refine clear mem registers

core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp

@@ -237,7 +237,7 @@ typedef enum { ADD, SUB, MUL, DIV_S, REM_S, DIV_U, REM_U } ALU_OP;
 /* Bit opcode */
 typedef enum { OR, XOR, AND } BIT_OP;
 /* Shift opcode */
-typedef enum { SHL, SHRS, SHRU } SHIFT_OP;
+typedef enum { SHL, SHRS, SHRU, ROTL, ROTR } SHIFT_OP;
 /* Condition opcode */
 typedef enum { EQ, NE, GTS, GES, LTS, LES, GTU, GEU, LTU, LEU } COND_OP;
 
@@ -2722,7 +2722,28 @@ bit_r_r_to_r_i32(x86::Assembler &a, BIT_OP op, int32 reg_no_dst,
 static bool
 bit_r_imm_i64(x86::Assembler &a, BIT_OP op, int32 reg_no, int64 data)
 {
-    return false;
+    Imm imm(data);
+
+    switch (op) {
+        case OR:
+            if (data != 0)
+                a.or_(regs_i64[reg_no], imm);
+            break;
+        case XOR:
+            if (data == -1LL)
+                a.not_(regs_i64[reg_no]);
+            else if (data != 0)
+                a.xor_(regs_i64[reg_no], imm);
+            break;
+        case AND:
+            if (data != -1LL)
+                a.and_(regs_i64[reg_no], imm);
+            break;
+        default:
+            bh_assert(0);
+            break;
+    }
+    return true;
 }
 
 /**
@@ -2738,7 +2759,21 @@ bit_r_imm_i64(x86::Assembler &a, BIT_OP op, int32 reg_no, int64 data)
 static bool
 bit_r_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, int32 reg_no_src)
 {
-    return false;
+    switch (op) {
+        case OR:
+            a.or_(regs_i64[reg_no_dst], regs_i64[reg_no_src]);
+            break;
+        case XOR:
+            a.xor_(regs_i64[reg_no_dst], regs_i64[reg_no_src]);
+            break;
+        case AND:
+            a.and_(regs_i64[reg_no_dst], regs_i64[reg_no_src]);
+            break;
+        default:
+            bh_assert(0);
+            break;
+    }
+    return true;
 }
 
 /**
@@ -2756,7 +2791,25 @@ static bool
 bit_imm_imm_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst,
                      int32 data1_src, int64 data2_src)
 {
-    return false;
+    Imm imm;
+
+    switch (op) {
+        case OR:
+            imm.setValue(data1_src | data2_src);
+            break;
+        case XOR:
+            imm.setValue(data1_src ^ data2_src);
+            break;
+        case AND:
+            imm.setValue(data1_src & data2_src);
+            break;
+        default:
+            bh_assert(0);
+            break;
+    }
+
+    a.mov(regs_i64[reg_no_dst], imm);
+    return true;
 }
 
 /**
@@ -2774,7 +2827,17 @@ static bool
 bit_imm_r_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst,
                    int64 data1_src, int32 reg_no2_src)
 {
-    return false;
+    if (op == AND && data1_src == 0)
+        a.xor_(regs_i64[reg_no_dst], regs_i64[reg_no_dst]);
+    else if (op == OR && data1_src == -1LL) {
+        Imm imm(-1LL);
+        a.mov(regs_i64[reg_no_dst], imm);
+    }
+    else {
+        mov_r_to_r_i64(a, reg_no_dst, reg_no2_src);
+        return bit_r_imm_i64(a, op, reg_no_dst, data1_src);
+    }
+    return true;
 }
 
 /**
@@ -2792,7 +2855,7 @@ static bool
 bit_r_imm_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst,
                    int32 reg_no1_src, int64 data2_src)
 {
-    return false;
+    return bit_imm_r_to_r_i64(a, op, reg_no_dst, data2_src, reg_no1_src);
 }
 
 /**
@@ -2810,6 +2873,12 @@ static bool
 bit_r_r_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst,
                  int32 reg_no1_src, int32 reg_no2_src)
 {
+    if (reg_no_dst != reg_no2_src) {
+        mov_r_to_r_i64(a, reg_no_dst, reg_no1_src);
+        return bit_r_r_i64(a, op, reg_no_dst, reg_no2_src);
+    }
+    else
+        return bit_r_r_i64(a, op, reg_no_dst, reg_no1_src);
     return false;
 }
 
@@ -2845,6 +2914,18 @@ shift_imm_imm_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst,
             data = ((uint32)data1_src) >> data2_src;
             break;
         }
+        case ROTL:
+        {
+            data = (data1_src << data2_src)
+                   | (((uint32)data1_src) >> (32 - data2_src));
+            break;
+        }
+        case ROTR:
+        {
+            data = (((uint32)data1_src) >> data2_src)
+                   | (data1_src << (32 - data2_src));
+            break;
+        }
         default:
         {
             bh_assert(0);
@@ -2895,20 +2976,31 @@ shift_r_imm_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst,
     /* SHL/SHA/SHR r/m32, imm8 */
     Imm imm((uint8)data2_src);
 
+    mov_r_to_r_i32(a, reg_no_dst, reg_no1_src);
     switch (op) {
         case SHL:
         {
-            a.shl(regs_i32[reg_no1_src], imm);
+            a.shl(regs_i32[reg_no_dst], imm);
             break;
         }
         case SHRS:
         {
-            a.sar(regs_i32[reg_no1_src], imm);
+            a.sar(regs_i32[reg_no_dst], imm);
             break;
         }
         case SHRU:
         {
-            a.shr(regs_i32[reg_no1_src], imm);
+            a.shr(regs_i32[reg_no_dst], imm);
+            break;
+        }
+        case ROTL:
+        {
+            a.rol(regs_i32[reg_no_dst], imm);
+            break;
+        }
+        case ROTR:
+        {
+            a.ror(regs_i32[reg_no_dst], imm);
             break;
         }
         default:
@@ -2918,7 +3010,7 @@ shift_r_imm_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst,
         }
     }
 
-    return mov_r_to_r_i32(a, reg_no_dst, reg_no1_src);
+    return true;
 fail:
     return false;
 }
@@ -2941,20 +3033,32 @@ shift_r_r_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst,
     /* should be CL */
     bh_assert(reg_no2_src == REG_ECX_IDX);
 
+    mov_r_to_r_i32(a, reg_no_dst, reg_no1_src);
+
     switch (op) {
         case SHL:
         {
-            a.shl(regs_i32[reg_no1_src], x86::cl);
+            a.shl(regs_i32[reg_no_dst], x86::cl);
             break;
         }
         case SHRS:
         {
-            a.sar(regs_i32[reg_no1_src], x86::cl);
+            a.sar(regs_i32[reg_no_dst], x86::cl);
             break;
         }
         case SHRU:
         {
-            a.shr(regs_i32[reg_no1_src], x86::cl);
+            a.shr(regs_i32[reg_no_dst], x86::cl);
+            break;
+        }
+        case ROTL:
+        {
+            a.rol(regs_i32[reg_no_dst], x86::cl);
+            break;
+        }
+        case ROTR:
+        {
+            a.ror(regs_i32[reg_no_dst], x86::cl);
             break;
         }
         default:
@@ -2964,7 +3068,7 @@ shift_r_r_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst,
         }
     }
 
-    return mov_r_to_r_i32(a, reg_no_dst, reg_no1_src);
+    return true;
 fail:
     return false;
 }
@@ -3002,6 +3106,18 @@ shift_imm_imm_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst,
             data = ((uint64)data1_src) >> data2_src;
             break;
         }
+        case ROTL:
+        {
+            data = (data1_src << data2_src)
+                   | (((uint64)data1_src) >> (64LL - data2_src));
+            break;
+        }
+        case ROTR:
+        {
+            data = (((uint64)data1_src) >> data2_src)
+                   | (data1_src << (64LL - data2_src));
+            break;
+        }
         default:
         {
             bh_assert(0);
@@ -3052,20 +3168,31 @@ shift_r_imm_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst,
     /* SHL/SHA/SHR r/m64, imm8 */
     Imm imm((uint8)data2_src);
 
+    mov_r_to_r_i64(a, reg_no_dst, reg_no1_src);
     switch (op) {
         case SHL:
         {
-            a.shl(regs_i64[reg_no1_src], imm);
+            a.shl(regs_i64[reg_no_dst], imm);
             break;
         }
         case SHRS:
         {
-            a.sar(regs_i64[reg_no1_src], imm);
+            a.sar(regs_i64[reg_no_dst], imm);
             break;
         }
         case SHRU:
         {
-            a.shr(regs_i64[reg_no1_src], imm);
+            a.shr(regs_i64[reg_no_dst], imm);
+            break;
+        }
+        case ROTL:
+        {
+            a.ror(regs_i64[reg_no_dst], imm);
+            break;
+        }
+        case ROTR:
+        {
+            a.ror(regs_i64[reg_no_dst], imm);
             break;
         }
         default:
@@ -3075,7 +3202,7 @@ shift_r_imm_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst,
         }
     }
 
-    return mov_r_to_r_i64(a, reg_no_dst, reg_no1_src);
+    return true;
 fail:
     return false;
 }
@@ -3098,20 +3225,32 @@ shift_r_r_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst,
     /* should be CL */
     bh_assert(reg_no2_src == REG_ECX_IDX);
 
+    mov_r_to_r_i64(a, reg_no_dst, reg_no1_src);
+
     switch (op) {
         case SHL:
         {
-            a.shl(regs_i64[reg_no1_src], x86::cl);
+            a.shl(regs_i64[reg_no_dst], x86::cl);
             break;
         }
         case SHRS:
         {
-            a.sar(regs_i64[reg_no1_src], x86::cl);
+            a.sar(regs_i64[reg_no_dst], x86::cl);
             break;
         }
         case SHRU:
         {
-            a.shr(regs_i64[reg_no1_src], x86::cl);
+            a.shr(regs_i64[reg_no_dst], x86::cl);
+            break;
+        }
+        case ROTL:
+        {
+            a.rol(regs_i64[reg_no_dst], x86::cl);
+            break;
+        }
+        case ROTR:
+        {
+            a.ror(regs_i64[reg_no_dst], x86::cl);
             break;
         }
         default:
@@ -3121,7 +3260,7 @@ shift_r_r_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst,
         }
     }
 
-    return mov_r_to_r_i64(a, reg_no_dst, reg_no1_src);
+    return true;
 fail:
     return false;
 }
@@ -4215,19 +4354,24 @@ fail:
 }
 
 /* jmp to dst label */
-#define JMP_TO_LABEL(label_dst, label_src)                               \
+#define JMP_TO_LABEL(label_dst, label_src)                                   \
-    do {                                                                 \
+    do {                                                                     \
-        if (label_is_ahead(cc, label_dst, label_src)) {                  \
+        if (label_is_ahead(cc, label_dst, label_src)) {                      \
-            int32 _offset = label_offsets[label_dst]                     \
+            char *stream = (char *)a.code()->sectionById(0)->buffer().data() \
-                            - a.code()->sectionById(0)->buffer().size(); \
+                           + a.code()->sectionById(0)->buffer().size();      \
-            Imm imm(_offset);                                            \
+            int32 _offset = label_offsets[label_dst]                         \
-            a.jmp(imm);                                                  \
+                            - a.code()->sectionById(0)->buffer().size();     \
-        }                                                                \
+            Imm imm(INT32_MAX);                                              \
-        else {                                                           \
+            a.jmp(imm);                                                      \
-            if (!jmp_from_label_to_label(a, jmp_info_list, label_dst,    \
+            /* The offset written by asmjit is always 0, we patch it again,  \
-                                         label_src))                     \
+               6 is the size of jmp instruciton */                           \
-                GOTO_FAIL;                                               \
+            *(int32 *)(stream + 2) = _offset - 6;                            \
-        }                                                                \
+        }                                                                    \
+        else {                                                               \
+            if (!jmp_from_label_to_label(a, jmp_info_list, label_dst,        \
+                                         label_src))                         \
+                GOTO_FAIL;                                                   \
+        }                                                                    \
     } while (0)
 
 /**
@@ -4841,6 +4985,8 @@ jit_codegen_gen_native(JitCompContext *cc)
                 case JIT_OP_SHL:
                 case JIT_OP_SHRS:
                 case JIT_OP_SHRU:
+                case JIT_OP_ROTL:
+                case JIT_OP_ROTR:
                     LOAD_3ARGS();
                     if (!lower_shift(
                             cc, a,
@@ -5217,7 +5363,7 @@ static const uint8 hreg_info_I32[3][7] = {
     /* ebp, eax, ebx, ecx, edx, edi, esi */
     { 1, 0, 0, 0, 0, 0, 1 }, /* fixed, esi is freely used */
     { 0, 1, 0, 1, 1, 1, 0 }, /* caller_saved_native */
-    { 0, 1, 0, 1, 1, 1, 0 }  /* caller_saved_jitted */
+    { 0, 1, 1, 1, 1, 1, 0 }  /* caller_saved_jitted */
 };
 
 static const uint8 hreg_info_I64[3][16] = {

core/iwasm/fast-jit/fe/jit_emit_control.c

@@ -241,7 +241,7 @@ push_jit_block_to_stack_and_pass_params(JitCompContext *cc, JitBlock *block,
                 block->incoming_insn_for_else_bb = insn;
             }
             else {
-                if (!jit_block_add_incoming_insn(block, insn)) {
+                if (!jit_block_add_incoming_insn(block, insn, 2)) {
                     jit_set_last_error(cc, "add incoming insn failed");
                     goto fail;
                 }
@@ -449,17 +449,9 @@ handle_op_end(JitCompContext *cc, uint8 **p_frame_ip, bool is_block_polymorphic)
         incoming_insn = block->incoming_insns_for_end_bb;
         while (incoming_insn) {
             insn = incoming_insn->insn;
-            if (insn->opcode == JIT_OP_JMP) {
+            bh_assert(insn->opcode == JIT_OP_JMP || insn->opcode == JIT_OP_BNE);
-                *(jit_insn_opnd(insn, 0)) =
+            *(jit_insn_opnd(insn, incoming_insn->opnd_idx)) =
-                    jit_basic_block_label(block->basic_block_end);
+                jit_basic_block_label(block->basic_block_end);
-            }
-            else if (insn->opcode == JIT_OP_BNE) {
-                *(jit_insn_opnd(insn, 2)) =
-                    jit_basic_block_label(block->basic_block_end);
-            }
-            else {
-                bh_assert(0);
-            }
             incoming_insn = incoming_insn->next;
         }
 
@@ -534,7 +526,7 @@ handle_op_else(JitCompContext *cc, uint8 **p_frame_ip,
                 jit_set_last_error(cc, "generate jmp insn failed");
                 return false;
             }
-            if (!jit_block_add_incoming_insn(block, insn)) {
+            if (!jit_block_add_incoming_insn(block, insn, 0)) {
                 jit_set_last_error(cc, "add incoming insn failed");
                 return false;
             }
@@ -821,7 +813,7 @@ handle_op_br(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip)
             jit_set_last_error(cc, "generate jmp insn failed");
             goto fail;
         }
-        if (!jit_block_add_incoming_insn(block_dst, insn)) {
+        if (!jit_block_add_incoming_insn(block_dst, insn, 0)) {
             jit_set_last_error(cc, "add incoming insn failed");
             goto fail;
         }
@@ -847,6 +839,8 @@ jit_compile_op_br_if(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip)
     JitBlock *block_dst;
     JitReg cond;
     JitBasicBlock *cur_basic_block, *if_basic_block = NULL;
+    JitValueSlot *frame_sp_src;
+    JitInsn *insn;
 
     if (!(block_dst = get_target_block(cc, br_depth))) {
         return false;
@@ -861,6 +855,41 @@ jit_compile_op_br_if(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip)
     /* Clear frame values */
     clear_values(jit_frame);
 
+    if (block_dst->label_type == LABEL_TYPE_LOOP) {
+        frame_sp_src =
+            jit_frame->sp
+            - wasm_get_cell_num(block_dst->param_types, block_dst->param_count);
+    }
+    else {
+        frame_sp_src = jit_frame->sp
+                       - wasm_get_cell_num(block_dst->result_types,
+                                           block_dst->result_count);
+    }
+
+    if (block_dst->frame_sp_begin == frame_sp_src) {
+        if (block_dst->label_type == LABEL_TYPE_LOOP) {
+            if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0))
+                || !GEN_INSN(
+                    BNE, cc->cmp_reg,
+                    jit_basic_block_label(block_dst->basic_block_entry), 0)) {
+                jit_set_last_error(cc, "generate bne insn failed");
+                goto fail;
+            }
+        }
+        else {
+            if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0))
+                || !(insn = GEN_INSN(BNE, cc->cmp_reg, 0, 0))) {
+                jit_set_last_error(cc, "generate bne insn failed");
+                goto fail;
+            }
+            if (!jit_block_add_incoming_insn(block_dst, insn, 1)) {
+                jit_set_last_error(cc, "add incoming insn failed");
+                goto fail;
+            }
+        }
+        return true;
+    }
+
     CREATE_BASIC_BLOCK(if_basic_block);
     if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0))
         || !GEN_INSN(BNE, cc->cmp_reg, jit_basic_block_label(if_basic_block),
@@ -912,6 +941,8 @@ jit_compile_op_br_table(JitCompContext *cc, uint32 *br_depths, uint32 br_count,
     for (i = 0, opnd = jit_insn_opndls(insn); i < br_count + 1; i++) {
         JitBasicBlock *basic_block = NULL;
 
+        /* TODO: refine the code */
+
         CREATE_BASIC_BLOCK(basic_block);
         SET_BB_BEGIN_BCIP(basic_block, *p_frame_ip - 1);
 

core/iwasm/fast-jit/fe/jit_emit_function.c

@@ -112,6 +112,9 @@ post_return(JitCompContext *cc, const WASMType *func_type)
         }
     }
 
+    /* Update the committed_sp as the callee has updated the frame sp */
+    cc->jit_frame->committed_sp = cc->jit_frame->sp;
+
     return true;
 fail:
     return false;
@@ -127,6 +130,10 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
     JitFrame *jit_frame = cc->jit_frame;
     JitReg result = 0, native_ret;
     JitReg func_ptrs, jitted_code = 0;
+#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
+    JitReg eax_hreg = jit_codegen_get_hreg_by_name("eax");
+    JitReg rax_hreg = jit_codegen_get_hreg_by_name("rax");
+#endif
     JitInsn *insn;
     uint32 jitted_func_idx;
 
@@ -156,7 +163,7 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
     if (func_idx < wasm_module->import_function_count) {
 #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
         /* Set native_ret to x86::eax */
-        native_ret = jit_codegen_get_hreg_by_name("eax");
+        native_ret = eax_hreg;
 #else
         native_ret = jit_cc_new_reg_I32(cc);
 #endif
@@ -184,16 +191,14 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
                 case VALUE_TYPE_FUNCREF:
 #endif
 #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
-                    /* Set result to x86::eax, 1 is hard reg index of eax */
+                    result = eax_hreg;
-                    result = jit_reg_new(JIT_REG_KIND_I32, 1);
 #else
                     result = jit_cc_new_reg_I32(cc);
 #endif
                     break;
                 case VALUE_TYPE_I64:
 #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
-                    /* Set result to x86::rax, 1 is hard reg index of rax */
+                    result = rax_hreg;
-                    result = jit_reg_new(JIT_REG_KIND_I64, 1);
 #else
                     result = jit_cc_new_reg_I64(cc);
 #endif
@@ -219,7 +224,8 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
 
     /* Clear part of memory regs and table regs as their values
        may be changed in the function call */
-    clear_memory_regs(jit_frame);
+    if (cc->cur_wasm_module->possible_memory_grow)
+        clear_memory_regs(jit_frame);
     clear_table_regs(jit_frame);
 
     /* Ignore tail call currently */
@@ -364,7 +370,8 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx,
 
     /* Clear part of memory regs and table regs as their values
        may be changed in the function call */
-    clear_memory_regs(cc->jit_frame);
+    if (cc->cur_wasm_module->possible_memory_grow)
+        clear_memory_regs(cc->jit_frame);
     clear_table_regs(cc->jit_frame);
     return true;
 fail:

core/iwasm/fast-jit/fe/jit_emit_numberic.c

@@ -182,7 +182,7 @@ compile_op_ibinopt_const(JitCompContext *cc, JitReg left, JitReg right,
 
     if (jit_reg_is_const(left) || jit_reg_is_const(right)) {
         res = handle_one_const(cc, left, right, is_i32);
-        if (!res)
+        if (res)
             goto shortcut;
     }
 
@@ -787,25 +787,44 @@ jit_compile_op_i64_bitwise(JitCompContext *cc, IntBitwise bitwise_op)
 
 DEF_UNI_INT_CONST_OPS(shl)
 {
-    if (IS_CONST_ZERO(right)) {
+    if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)) {
         return left;
     }
+
+    if (jit_reg_is_const(right)) {
+        JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
+        GEN_INSN(SHL, res, left, right);
+        return res;
+    }
     return 0;
 }
 
 DEF_UNI_INT_CONST_OPS(shrs)
 {
-    if (IS_CONST_ZERO(right)) {
+    if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)
+        || IS_CONST_ALL_ONE(left, is_i32)) {
         return left;
     }
+
+    if (jit_reg_is_const(right)) {
+        JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
+        GEN_INSN(SHRS, res, left, right);
+        return res;
+    }
     return 0;
 }
 
 DEF_UNI_INT_CONST_OPS(shru)
 {
-    if (IS_CONST_ZERO(right)) {
+    if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)) {
         return left;
     }
+
+    if (jit_reg_is_const(right)) {
+        JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
+        GEN_INSN(SHRU, res, left, right);
+        return res;
+    }
     return 0;
 }
 
@@ -958,11 +977,15 @@ shortcut:
 
 DEF_UNI_INT_CONST_OPS(rotl)
 {
-    if (IS_CONST_ZERO(right))
+    if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)
+        || IS_CONST_ALL_ONE(left, is_i32))
         return left;
 
-    if (IS_CONST_ZERO(left))
+    if (jit_reg_is_const(right)) {
-        return right;
+        JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
+        GEN_INSN(ROTL, res, left, right);
+        return res;
+    }
 
     return 0;
 }
@@ -986,7 +1009,7 @@ do_i64_const_rotl(int64 lhs, int64 rhs)
 static JitReg
 compile_int_rotl(JitCompContext *cc, JitReg left, JitReg right, bool is_i32)
 {
-    JitReg res, tmp, shl_res, shr_res;
+    JitReg res;
 #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
     JitReg ecx_hreg = jit_codegen_get_hreg_by_name("ecx");
     JitReg rcx_hreg = jit_codegen_get_hreg_by_name("rcx");
@@ -1000,34 +1023,13 @@ compile_int_rotl(JitCompContext *cc, JitReg left, JitReg right, bool is_i32)
 
     left = mov_left_to_reg(cc, is_i32, left);
 
-    if (is_i32) {
+    res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
-        tmp = jit_cc_new_reg_I32(cc);
-        shl_res = jit_cc_new_reg_I32(cc);
-        shr_res = jit_cc_new_reg_I32(cc);
-        res = jit_cc_new_reg_I32(cc);
-    }
-    else {
-        tmp = jit_cc_new_reg_I64(cc);
-        shl_res = jit_cc_new_reg_I64(cc);
-        shr_res = jit_cc_new_reg_I64(cc);
-        res = jit_cc_new_reg_I64(cc);
-    }
-
-    /* 32/64 - rhs */
-    GEN_INSN(SUB, tmp, is_i32 ? NEW_CONST(I32, 32) : NEW_CONST(I64, 64), right);
 #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
     GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, right);
-    GEN_INSN(SHL, shl_res, left, is_i32 ? ecx_hreg : rcx_hreg);
+    GEN_INSN(ROTL, res, left, is_i32 ? ecx_hreg : rcx_hreg);
-
-    GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, tmp);
-    GEN_INSN(SHRU, shr_res, left, is_i32 ? ecx_hreg : rcx_hreg);
-
-    GEN_INSN(OR, res, shl_res, shr_res);
     GEN_INSN(MOV, ecx_hreg, ecx_hreg);
 #else
-    GEN_INSN(SHL, shl_res, left, right);
+    GEN_INSN(ROTL, res, left, right);
-    GEN_INSN(SHRU, shr_res, left, tmp);
-    GEN_INSN(OR, res, shl_res, shr_res);
 #endif
 
 shortcut:
@@ -1036,11 +1038,15 @@ shortcut:
 
 DEF_UNI_INT_CONST_OPS(rotr)
 {
-    if (IS_CONST_ZERO(right))
+    if (IS_CONST_ZERO(right) || IS_CONST_ZERO(left)
+        || IS_CONST_ALL_ONE(left, is_i32))
         return left;
 
-    if (IS_CONST_ZERO(left))
+    if (jit_reg_is_const(right)) {
-        return right;
+        JitReg res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
+        GEN_INSN(ROTR, res, left, right);
+        return res;
+    }
 
     return 0;
 }
@@ -1064,7 +1070,7 @@ do_i64_const_rotr(int64 lhs, int64 rhs)
 static JitReg
 compile_int_rotr(JitCompContext *cc, JitReg left, JitReg right, bool is_i32)
 {
-    JitReg res, tmp, shr_res, shl_res;
+    JitReg res;
 #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
     JitReg ecx_hreg = jit_codegen_get_hreg_by_name("ecx");
     JitReg rcx_hreg = jit_codegen_get_hreg_by_name("rcx");
@@ -1078,34 +1084,13 @@ compile_int_rotr(JitCompContext *cc, JitReg left, JitReg right, bool is_i32)
 
     left = mov_left_to_reg(cc, is_i32, left);
 
-    if (is_i32) {
+    res = is_i32 ? jit_cc_new_reg_I32(cc) : jit_cc_new_reg_I64(cc);
-        tmp = jit_cc_new_reg_I32(cc);
-        shr_res = jit_cc_new_reg_I32(cc);
-        shl_res = jit_cc_new_reg_I32(cc);
-        res = jit_cc_new_reg_I32(cc);
-    }
-    else {
-        tmp = jit_cc_new_reg_I64(cc);
-        shr_res = jit_cc_new_reg_I64(cc);
-        shl_res = jit_cc_new_reg_I64(cc);
-        res = jit_cc_new_reg_I64(cc);
-    }
-
-    /* 32/64 - rhs */
-    GEN_INSN(SUB, tmp, is_i32 ? NEW_CONST(I32, 32) : NEW_CONST(I64, 64), right);
 #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
     GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, right);
-    GEN_INSN(SHRU, shl_res, left, is_i32 ? ecx_hreg : rcx_hreg);
+    GEN_INSN(ROTR, res, left, is_i32 ? ecx_hreg : rcx_hreg);
-
-    GEN_INSN(MOV, is_i32 ? ecx_hreg : rcx_hreg, tmp);
-    GEN_INSN(SHL, shr_res, left, is_i32 ? ecx_hreg : rcx_hreg);
-
-    GEN_INSN(OR, res, shl_res, shr_res);
     GEN_INSN(MOV, ecx_hreg, ecx_hreg);
 #else
-    GEN_INSN(SHRU, shr_res, left, right);
+    GEN_INSN(ROTR, res, left, right);
-    GEN_INSN(SHL, shl_res, left, tmp);
-    GEN_INSN(OR, res, shr_res, shl_res);
 #endif
 
 shortcut:

core/iwasm/fast-jit/fe/jit_emit_parametric.c

@@ -104,7 +104,8 @@ jit_compile_op_select(JitCompContext *cc, bool is_select_32)
     else
         selected = jit_cc_new_reg_I64(cc);
 
-    GEN_INSN(SELECTNE, selected, cond, val1, val2);
+    GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0));
+    GEN_INSN(SELECTNE, selected, cc->cmp_reg, val1, val2);
     PUSH(selected, val1_type);
     return true;
 fail:

core/iwasm/fast-jit/fe/jit_emit_variable.c

@@ -4,6 +4,7 @@
  */
 
 #include "jit_emit_variable.h"
+#include "jit_emit_exception.h"
 #include "../jit_frontend.h"
 
 #define CHECK_LOCAL(idx)                                                     \
@@ -248,11 +249,6 @@ jit_compile_op_set_global(JitCompContext *cc, uint32 global_idx,
     uint8 global_type = 0;
     JitReg value = 0;
 
-    if (is_aux_stack) {
-        jit_set_last_error(cc, "doesn't support set global_aux_stack");
-        goto fail;
-    }
-
     bh_assert(global_idx < cc->cur_wasm_module->import_global_count
                                + cc->cur_wasm_module->global_count);
 
@@ -266,6 +262,19 @@ jit_compile_op_set_global(JitCompContext *cc, uint32 global_idx,
 #endif
         {
             POP_I32(value);
+            if (is_aux_stack) {
+                JitReg aux_stack_bound = get_aux_stack_bound_reg(cc->jit_frame);
+                JitReg aux_stack_bottom =
+                    get_aux_stack_bottom_reg(cc->jit_frame);
+                GEN_INSN(CMP, cc->cmp_reg, value, aux_stack_bound);
+                if (!(jit_emit_exception(cc, EXCE_AUX_STACK_OVERFLOW,
+                                         JIT_OP_BLEU, cc->cmp_reg, NULL)))
+                    goto fail;
+                GEN_INSN(CMP, cc->cmp_reg, value, aux_stack_bottom);
+                if (!(jit_emit_exception(cc, EXCE_AUX_STACK_UNDERFLOW,
+                                         JIT_OP_BGTU, cc->cmp_reg, NULL)))
+                    goto fail;
+            }
             GEN_INSN(STI32, value, get_global_data_reg(cc->jit_frame),
                      NEW_CONST(I32, data_offset));
             break;

core/iwasm/fast-jit/jit_frontend.c

@@ -100,15 +100,29 @@ get_global_data_reg(JitFrame *frame)
 JitReg
 get_aux_stack_bound_reg(JitFrame *frame)
 {
-    /* TODO */
+    JitCompContext *cc = frame->cc;
-    return 0;
+
+    if (!frame->aux_stack_bound_reg) {
+        frame->aux_stack_bound_reg = cc->aux_stack_bound_reg;
+        GEN_INSN(
+            LDI32, frame->aux_stack_bound_reg, cc->exec_env_reg,
+            NEW_CONST(I32, offsetof(WASMExecEnv, aux_stack_boundary.boundary)));
+    }
+    return frame->aux_stack_bound_reg;
 }
 
 JitReg
 get_aux_stack_bottom_reg(JitFrame *frame)
 {
-    /* TODO */
+    JitCompContext *cc = frame->cc;
-    return 0;
+
+    if (!frame->aux_stack_bottom_reg) {
+        frame->aux_stack_bottom_reg = cc->aux_stack_bottom_reg;
+        GEN_INSN(
+            LDI32, frame->aux_stack_bottom_reg, cc->exec_env_reg,
+            NEW_CONST(I32, offsetof(WASMExecEnv, aux_stack_bottom.bottom)));
+    }
+    return frame->aux_stack_bottom_reg;
 }
 
 JitReg
@@ -494,11 +508,13 @@ gen_commit_sp_ip(JitFrame *frame)
         frame->committed_sp = frame->sp;
     }
 
+#if 0 /* Disable committing ip currently */
     if (frame->ip != frame->committed_ip) {
         GEN_INSN(STPTR, NEW_CONST(PTR, (uintptr_t)frame->ip), cc->fp_reg,
                  NEW_CONST(I32, offsetof(WASMInterpFrame, ip)));
         frame->committed_ip = frame->ip;
     }
+#endif
 }
 
 static void
@@ -691,6 +707,7 @@ init_func_translation(JitCompContext *cc)
         + (uint64)cur_wasm_func->max_stack_cell_num
         + ((uint64)cur_wasm_func->max_block_num) * sizeof(WASMBranchBlock) / 4;
     uint32 frame_size, outs_size, local_size, count;
+    uint32 i, local_off;
     uint64 total_size;
 
     if ((uint64)max_locals + (uint64)max_stacks >= UINT32_MAX
@@ -796,6 +813,18 @@ init_func_translation(JitCompContext *cc)
     /* fp_reg = top */
     GEN_INSN(MOV, cc->fp_reg, top);
 
+    /* Initialize local variables, set them to 0 */
+    local_off = (uint32)offsetof(WASMInterpFrame, lp)
+                + cur_wasm_func->param_cell_num * 4;
+    for (i = 0; i < cur_wasm_func->local_cell_num / 2; i++, local_off += 8) {
+        GEN_INSN(STI64, NEW_CONST(I32, 0), cc->fp_reg,
+                 NEW_CONST(I32, local_off));
+    }
+    if (cur_wasm_func->local_cell_num & 1) {
+        GEN_INSN(STI32, NEW_CONST(I32, 0), cc->fp_reg,
+                 NEW_CONST(I32, local_off));
+    }
+
     return jit_frame;
 }
 
@@ -1084,15 +1113,11 @@ jit_compile_func(JitCompContext *cc)
                 read_leb_uint32(frame_ip, frame_ip_end, type_idx);
 
 #if WASM_ENABLE_REF_TYPES != 0
-                if (cc->enable_ref_types) {
+                read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
-                    read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
+#else
-                }
+                frame_ip++;
-                else
+                tbl_idx = 0;
 #endif
-                {
-                    frame_ip++;
-                    tbl_idx = 0;
-                }
 
                 if (!jit_compile_op_call_indirect(cc, type_idx, tbl_idx))
                     return false;
@@ -1123,15 +1148,11 @@ jit_compile_func(JitCompContext *cc)
 
                 read_leb_uint32(frame_ip, frame_ip_end, type_idx);
 #if WASM_ENABLE_REF_TYPES != 0
-                if (cc->enable_ref_types) {
+                read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
-                    read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
+#else
-                }
+                frame_ip++;
-                else
+                tbl_idx = 0;
 #endif
-                {
-                    frame_ip++;
-                    tbl_idx = 0;
-                }
 
                 if (!jit_compile_op_call_indirect(cc, type_idx, tbl_idx))
                     return false;
@@ -1166,10 +1187,6 @@ jit_compile_func(JitCompContext *cc)
             {
                 uint32 vec_len;
 
-                if (!cc->enable_ref_types) {
-                    goto unsupport_ref_types;
-                }
-
                 read_leb_uint32(frame_ip, frame_ip_end, vec_len);
                 bh_assert(vec_len == 1);
                 (void)vec_len;
@@ -1185,10 +1202,6 @@ jit_compile_func(JitCompContext *cc)
             {
                 uint32 tbl_idx;
 
-                if (!cc->enable_ref_types) {
-                    goto unsupport_ref_types;
-                }
-
                 read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
                 if (!jit_compile_op_table_get(cc, tbl_idx))
                     return false;
@@ -1198,10 +1211,6 @@ jit_compile_func(JitCompContext *cc)
             {
                 uint32 tbl_idx;
 
-                if (!cc->enable_ref_types) {
-                    goto unsupport_ref_types;
-                }
-
                 read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
                 if (!jit_compile_op_table_set(cc, tbl_idx))
                     return false;
@@ -1211,10 +1220,6 @@ jit_compile_func(JitCompContext *cc)
             {
                 uint32 type;
 
-                if (!cc->enable_ref_types) {
-                    goto unsupport_ref_types;
-                }
-
                 read_leb_uint32(frame_ip, frame_ip_end, type);
 
                 if (!jit_compile_op_ref_null(cc))
@@ -1225,20 +1230,12 @@ jit_compile_func(JitCompContext *cc)
             }
             case WASM_OP_REF_IS_NULL:
             {
-                if (!cc->enable_ref_types) {
-                    goto unsupport_ref_types;
-                }
-
                 if (!jit_compile_op_ref_is_null(cc))
                     return false;
                 break;
             }
             case WASM_OP_REF_FUNC:
             {
-                if (!cc->enable_ref_types) {
-                    goto unsupport_ref_types;
-                }
-
                 read_leb_uint32(frame_ip, frame_ip_end, func_idx);
                 if (!jit_compile_op_ref_func(cc, func_idx))
                     return false;
@@ -1760,21 +1757,6 @@ jit_compile_func(JitCompContext *cc)
                 read_leb_uint32(frame_ip, frame_ip_end, opcode1);
                 opcode = (uint32)opcode1;
 
-#if WASM_ENABLE_BULK_MEMORY != 0
-                if (WASM_OP_MEMORY_INIT <= opcode
-                    && opcode <= WASM_OP_MEMORY_FILL
-                    && !cc->enable_bulk_memory) {
-                    goto unsupport_bulk_memory;
-                }
-#endif
-
-#if WASM_ENABLE_REF_TYPES != 0
-                if (WASM_OP_TABLE_INIT <= opcode && opcode <= WASM_OP_TABLE_FILL
-                    && !cc->enable_ref_types) {
-                    goto unsupport_ref_types;
-                }
-#endif
-
                 switch (opcode) {
                     case WASM_OP_I32_TRUNC_SAT_S_F32:
                     case WASM_OP_I32_TRUNC_SAT_U_F32:
@@ -2063,21 +2045,6 @@ jit_compile_func(JitCompContext *cc)
 
     (void)func_idx;
     return true;
-
-#if WASM_ENABLE_REF_TYPES != 0
-unsupport_ref_types:
-    jit_set_last_error(cc, "reference type instruction was found, "
-                           "try removing --disable-ref-types option");
-    return false;
-#endif
-
-#if WASM_ENABLE_BULK_MEMORY != 0
-unsupport_bulk_memory:
-    jit_set_last_error(cc, "bulk memory instruction was found, "
-                           "try removing --disable-bulk-memory option");
-    return false;
-#endif
-
 fail:
     return false;
 }

core/iwasm/fast-jit/jit_ir.c

@@ -1394,7 +1394,7 @@ jit_block_stack_destroy(JitBlockStack *stack)
 }
 
 bool
-jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn)
+jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn, uint32 opnd_idx)
 {
     JitIncomingInsn *incoming_insn;
 
@@ -1402,6 +1402,7 @@ jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn)
         return false;
 
     incoming_insn->insn = insn;
+    incoming_insn->opnd_idx = opnd_idx;
     incoming_insn->next = block->incoming_insns_for_end_bb;
     block->incoming_insns_for_end_bb = incoming_insn;
     return true;

core/iwasm/fast-jit/jit_ir.def

@@ -109,6 +109,8 @@ INSN(REM_U, Reg, 3, 1)
 INSN(SHL, Reg, 3, 1)
 INSN(SHRS, Reg, 3, 1)
 INSN(SHRU, Reg, 3, 1)
+INSN(ROTL, Reg, 3, 1)
+INSN(ROTR, Reg, 3, 1)
 INSN(OR, Reg, 3, 1)
 INSN(XOR, Reg, 3, 1)
 INSN(AND, Reg, 3, 1)

core/iwasm/fast-jit/jit_ir.h

@@ -974,6 +974,7 @@ typedef struct JitFrame {
 typedef struct JitIncomingInsn {
     struct JitIncomingInsn *next;
     JitInsn *insn;
+    uint32 opnd_idx;
 } JitIncomingInsn, *JitIncomingInsnList;
 
 typedef struct JitBlock {
@@ -1865,7 +1866,7 @@ void
 jit_block_stack_destroy(JitBlockStack *stack);
 
 bool
-jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn);
+jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn, uint32 opnd_idx);
 
 void
 jit_block_destroy(JitBlock *block);

core/iwasm/interpreter/wasm_interp_classic.c

@@ -837,15 +837,8 @@ jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
     WASMModuleInstance *module_inst =
         (WASMModuleInstance *)exec_env->module_inst;
     WASMFunctionInstance *cur_func = module_inst->functions + func_idx;
-    uint32 *sp_org;
 
-    sp_org = prev_frame->sp;
     wasm_interp_call_func_native(module_inst, exec_env, cur_func, prev_frame);
-    /* Restore the stack pointer of previous frame as the caller in
-       jitted code will just read the return value and won't decrease
-       the stack pointer */
-    prev_frame->sp = sp_org;
-
     return wasm_get_exception(module_inst) ? false : true;
 }
 #endif