diff --git a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp index bf1651f3f..b1b500600 100644 --- a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp +++ b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp @@ -246,6 +246,36 @@ typedef enum { CLZ, CTZ, POPCNT } BITCOUNT_OP; /* Condition opcode */ typedef enum { EQ, NE, GTS, GES, LTS, LES, GTU, GEU, LTU, LEU } COND_OP; +typedef union _cast_float_to_integer { + float f; + uint32 i; +} cast_float_to_integer; + +typedef union _cast_double_to_integer { + double d; + uint64 i; +} cast_double_to_integer; + +static uint32 +local_log2(uint32 data) +{ + uint32 ret = 0; + while (data >>= 1) { + ret++; + } + return ret; +} + +static uint64 +local_log2l(uint64 data) +{ + uint64 ret = 0; + while (data >>= 1) { + ret++; + } + return ret; +} + /* Jmp type */ typedef enum JmpType { JMP_DST_LABEL, /* jmp to dst label */ @@ -1157,7 +1187,8 @@ static bool mov_imm_to_r_f32(x86::Assembler &a, int32 reg_no, float data) { /* imm -> gp -> xmm */ - Imm imm(*(uint32 *)&data); + cast_float_to_integer v = { .f = data }; + Imm imm(v.i); a.mov(regs_i32[REG_I32_FREE_IDX], imm); a.movd(regs_float[reg_no], regs_i32[REG_I32_FREE_IDX]); return true; @@ -1193,7 +1224,8 @@ mov_r_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) static bool mov_imm_to_r_f64(x86::Assembler &a, int32 reg_no, double data) { - Imm imm(*(uint64 *)&data); + cast_double_to_integer v = { .d = data }; + Imm imm(v.i); a.mov(regs_i64[REG_I32_FREE_IDX], imm); /* REG_I32_FREE_IDX == REG_I64_FREE_IDX */ a.movq(regs_float[reg_no], regs_i64[REG_I64_FREE_IDX]); @@ -2266,19 +2298,10 @@ alu_r_r_imm_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, else if (data == 1) { mov_r_to_r(a, JIT_REG_KIND_I32, reg_no_dst, reg_no_src); } - else if (data == 2) { + else if (data > 0 && (data & (data - 1)) == 0x0) { mov_r_to_r(a, JIT_REG_KIND_I32, reg_no_dst, reg_no_src); - imm.setValue(1); - a.shl(regs_i32[reg_no_dst], imm); - } - else if (data == 4) { - mov_r_to_r(a, JIT_REG_KIND_I32, reg_no_dst, reg_no_src); - imm.setValue(2); - a.shl(regs_i32[reg_no_dst], imm); - } - else if (data == 8) { - mov_r_to_r(a, JIT_REG_KIND_I32, reg_no_dst, reg_no_src); - imm.setValue(3); + data = (int32)local_log2(data); + imm.setValue(data); a.shl(regs_i32[reg_no_dst], imm); } else { @@ -2539,122 +2562,6 @@ alu_r_r_to_r_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, return alu_r_r_r_i32(a, op, reg_no_dst, reg_no1_src, reg_no2_src); } -/** - * Encode int64 alu operation of reg and data, and save result to reg - * - * @param a the assembler to emit the code - * @param op the opcode of ALU operation - * @param reg_no the no of register, as first operand, and save result - * @param data the immediate data, as the second operand - * - * @return true if success, false otherwise - */ -static bool -alu_r_r_imm_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, - int32 reg_no_src, int64 data) -{ - Imm imm(data); - - switch (op) { - case ADD: - mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); - if (data == 1) - a.inc(regs_i64[reg_no_dst]); - else if (data == -1) - a.dec(regs_i64[reg_no_dst]); - else if (data != 0) { - if (data >= INT32_MIN && data <= INT32_MAX) { - imm.setValue((int32)data); - a.add(regs_i64[reg_no_dst], imm); - } - else { - a.mov(regs_i64[REG_I64_FREE_IDX], imm); - a.add(regs_i64[reg_no_dst], regs_i64[REG_I64_FREE_IDX]); - } - } - break; - case SUB: - mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); - if (data == -1) - a.inc(regs_i64[reg_no_dst]); - else if (data == 1) - a.dec(regs_i64[reg_no_dst]); - else if (data != 0) { - if (data >= INT32_MIN && data <= INT32_MAX) { - imm.setValue((int32)data); - a.sub(regs_i64[reg_no_dst], imm); - } - else { - a.mov(regs_i64[REG_I64_FREE_IDX], imm); - a.sub(regs_i64[reg_no_dst], regs_i64[REG_I64_FREE_IDX]); - } - } - break; - case MUL: - if (data == 0) - a.xor_(regs_i64[reg_no_dst], regs_i64[reg_no_dst]); - else if (data == -1) { - mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); - a.neg(regs_i64[reg_no_dst]); - } - else if (data == 1) { - mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); - } - else if (data == 2) { - mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); - imm.setValue(1); - a.shl(regs_i64[reg_no_dst], imm); - } - else if (data == 4) { - mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); - imm.setValue(2); - a.shl(regs_i64[reg_no_dst], imm); - } - else if (data == 8) { - mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); - imm.setValue(3); - a.shl(regs_i64[reg_no_dst], imm); - } - else { - a.imul(regs_i64[reg_no_dst], regs_i64[reg_no_src], imm); - } - break; - case DIV_S: - case REM_S: - bh_assert(reg_no_src == REG_RAX_IDX); - if (op == DIV_S) { - bh_assert(reg_no_dst == REG_RAX_IDX); - } - else { - bh_assert(reg_no_dst == REG_RDX_IDX); - } - a.mov(regs_i64[REG_I64_FREE_IDX], imm); - /* signed extend rax to rdx:rax */ - a.cqo(); - a.idiv(regs_i64[REG_I64_FREE_IDX]); - break; - case DIV_U: - case REM_U: - bh_assert(reg_no_src == REG_RAX_IDX); - if (op == DIV_U) { - bh_assert(reg_no_dst == REG_RAX_IDX); - } - else { - bh_assert(reg_no_dst == REG_RDX_IDX); - } - a.mov(regs_i64[REG_I64_FREE_IDX], imm); - /* unsigned extend rax to rdx:rax */ - a.xor_(regs_i64[REG_RDX_IDX], regs_i64[REG_RDX_IDX]); - a.div(regs_i64[REG_I64_FREE_IDX]); - break; - default: - bh_assert(0); - break; - } - - return true; -} - /** * Encode int64 alu operation of reg and reg, and save result to reg * @@ -2730,6 +2637,122 @@ alu_r_r_r_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, int32 reg_no1_src, return true; } +/** + * Encode int64 alu operation of reg and data, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no the no of register, as first operand, and save result + * @param data the immediate data, as the second operand + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_imm_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no_src, int64 data) +{ + Imm imm(data); + + switch (op) { + case ADD: + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); + if (data == 1) + a.inc(regs_i64[reg_no_dst]); + else if (data == -1) + a.dec(regs_i64[reg_no_dst]); + else if (data != 0) { + if (data >= INT32_MIN && data <= INT32_MAX) { + imm.setValue((int32)data); + a.add(regs_i64[reg_no_dst], imm); + } + else { + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.add(regs_i64[reg_no_dst], regs_i64[REG_I64_FREE_IDX]); + } + } + break; + case SUB: + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); + if (data == -1) + a.inc(regs_i64[reg_no_dst]); + else if (data == 1) + a.dec(regs_i64[reg_no_dst]); + else if (data != 0) { + if (data >= INT32_MIN && data <= INT32_MAX) { + imm.setValue((int32)data); + a.sub(regs_i64[reg_no_dst], imm); + } + else { + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + a.sub(regs_i64[reg_no_dst], regs_i64[REG_I64_FREE_IDX]); + } + } + break; + case MUL: + if (data == 0) + a.xor_(regs_i64[reg_no_dst], regs_i64[reg_no_dst]); + else if (data == -1) { + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); + a.neg(regs_i64[reg_no_dst]); + } + else if (data == 1) { + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); + } + else if (data > 0 && (data & (data - 1)) == 0x0) { + mov_r_to_r(a, JIT_REG_KIND_I64, reg_no_dst, reg_no_src); + data = (int64)local_log2l(data); + imm.setValue(data); + a.shl(regs_i64[reg_no_dst], imm); + } + else if (INT32_MIN <= data && data <= INT32_MAX) { + a.imul(regs_i64[reg_no_dst], regs_i64[reg_no_src], imm); + } + else { + mov_imm_to_r_i64( + a, reg_no_dst == reg_no_src ? REG_I64_FREE_IDX : reg_no_dst, + data); + alu_r_r_r_i64(a, op, reg_no_dst, + reg_no_dst == reg_no_src ? REG_I64_FREE_IDX + : reg_no_dst, + reg_no_src); + } + break; + case DIV_S: + case REM_S: + bh_assert(reg_no_src == REG_RAX_IDX); + if (op == DIV_S) { + bh_assert(reg_no_dst == REG_RAX_IDX); + } + else { + bh_assert(reg_no_dst == REG_RDX_IDX); + } + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + /* signed extend rax to rdx:rax */ + a.cqo(); + a.idiv(regs_i64[REG_I64_FREE_IDX]); + break; + case DIV_U: + case REM_U: + bh_assert(reg_no_src == REG_RAX_IDX); + if (op == DIV_U) { + bh_assert(reg_no_dst == REG_RAX_IDX); + } + else { + bh_assert(reg_no_dst == REG_RDX_IDX); + } + a.mov(regs_i64[REG_I64_FREE_IDX], imm); + /* unsigned extend rax to rdx:rax */ + a.xor_(regs_i64[REG_RDX_IDX], regs_i64[REG_RDX_IDX]); + a.div(regs_i64[REG_I64_FREE_IDX]); + break; + default: + bh_assert(0); + break; + } + + return true; +} + /** * Encode int64 alu operation of imm and imm, and save result to reg * @@ -3025,7 +3048,8 @@ alu_r_imm_to_r_f32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, /* imm -> m32 */ x86::Mem cache = x86::dword_ptr(regs_i64[hreg_info->exec_env_hreg_index], offsetof(WASMExecEnv, jit_cache)); - Imm imm(*(uint32 *)&data2_src); + cast_float_to_integer v = { .f = data2_src }; + Imm imm(v.i); mov_imm_to_m(a, cache, imm, 4); mov_r_to_r_f32(a, reg_no_dst, reg_no1_src); @@ -3214,7 +3238,8 @@ alu_r_imm_to_r_f64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, /* imm -> m64 */ x86::Mem cache = x86::qword_ptr(regs_i64[hreg_info->exec_env_hreg_index], offsetof(WASMExecEnv, jit_cache)); - Imm imm(*(uint64 *)&data2_src); + cast_double_to_integer v = { .d = data2_src }; + Imm imm(v.i); mov_imm_to_m(a, cache, imm, 8); mov_r_to_r_f64(a, reg_no_dst, reg_no1_src); @@ -3822,7 +3847,8 @@ shift_r_r_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, int32 reg_no1_src, int32 reg_no2_src) { /* should be CL */ - bh_assert(reg_no2_src == REG_ECX_IDX); + if (reg_no2_src != REG_ECX_IDX) + return false; mov_r_to_r_i32(a, reg_no_dst, reg_no1_src); @@ -4019,7 +4045,8 @@ shift_r_r_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, int32 reg_no1_src, int32 reg_no2_src) { /* should be CL */ - bh_assert(reg_no2_src == REG_ECX_IDX); + if (reg_no2_src != REG_ECX_IDX) + return false; mov_r_to_r_i64(a, reg_no_dst, reg_no1_src); @@ -5845,7 +5872,8 @@ cast_r_i64_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) static bool cast_imm_f32_to_r_i32(x86::Assembler &a, int32 reg_no, float data) { - return mov_imm_to_r_i32(a, reg_no, *(uint32 *)&data); + cast_float_to_integer v = { .f = data }; + return mov_imm_to_r_i32(a, reg_no, v.i); } /** @@ -5876,7 +5904,8 @@ cast_r_f32_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) static bool cast_imm_f64_to_r_i64(x86::Assembler &a, int32 reg_no, double data) { - return mov_imm_to_r_i64(a, reg_no, *(uint64 *)&data); + cast_double_to_integer v = { .d = data }; + return mov_imm_to_r_i64(a, reg_no, v.i); } /** @@ -6475,6 +6504,9 @@ jit_codegen_dump_native(void *begin_addr, void *end_addr) os_printf("\n"); dump_native((char *)begin_addr, (char *)end_addr - (char *)begin_addr); os_printf("\n"); +#else + (void)begin_addr; + (void)end_addr; #endif } diff --git a/core/iwasm/fast-jit/fe/jit_emit_function.c b/core/iwasm/fast-jit/fe/jit_emit_function.c index 29201a1d1..539b653ca 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_function.c +++ b/core/iwasm/fast-jit/fe/jit_emit_function.c @@ -365,23 +365,43 @@ fail: return false; } +#if WASM_ENABLE_REF_TYPES != 0 bool -jit_compile_op_ref_null(JitCompContext *cc) +jit_compile_op_ref_null(JitCompContext *cc, uint32 ref_type) { + PUSH_I32(NEW_CONST(I32, NULL_REF)); + (void)ref_type; + return true; +fail: return false; } bool jit_compile_op_ref_is_null(JitCompContext *cc) { + JitReg ref, res; + + POP_I32(ref); + + GEN_INSN(CMP, cc->cmp_reg, ref, NEW_CONST(I32, NULL_REF)); + res = jit_cc_new_reg_I32(cc); + GEN_INSN(SELECTEQ, res, cc->cmp_reg, NEW_CONST(I32, 1), NEW_CONST(I32, 0)); + PUSH_I32(res); + + return true; +fail: return false; } bool jit_compile_op_ref_func(JitCompContext *cc, uint32 func_idx) { + PUSH_I32(NEW_CONST(I32, func_idx)); + return true; +fail: return false; } +#endif #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) bool diff --git a/core/iwasm/fast-jit/fe/jit_emit_function.h b/core/iwasm/fast-jit/fe/jit_emit_function.h index ec63f50e3..7405f774c 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_function.h +++ b/core/iwasm/fast-jit/fe/jit_emit_function.h @@ -20,7 +20,7 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx, uint32 tbl_idx); bool -jit_compile_op_ref_null(JitCompContext *cc); +jit_compile_op_ref_null(JitCompContext *cc, uint32 ref_type); bool jit_compile_op_ref_is_null(JitCompContext *cc); diff --git a/core/iwasm/fast-jit/fe/jit_emit_table.c b/core/iwasm/fast-jit/fe/jit_emit_table.c index 3985a526c..f7293b4f1 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_table.c +++ b/core/iwasm/fast-jit/fe/jit_emit_table.c @@ -4,54 +4,315 @@ */ #include "jit_emit_table.h" +#include "jit_emit_exception.h" +#include "jit_emit_function.h" +#include "../../interpreter/wasm_runtime.h" #include "../jit_frontend.h" +#if WASM_ENABLE_REF_TYPES != 0 bool jit_compile_op_elem_drop(JitCompContext *cc, uint32 tbl_seg_idx) { - return false; + JitReg module, tbl_segs; + + module = get_module_reg(cc->jit_frame); + + tbl_segs = jit_cc_new_reg_ptr(cc); + GEN_INSN(LDPTR, tbl_segs, module, + NEW_CONST(I32, offsetof(WASMModule, table_segments))); + + GEN_INSN(STI32, NEW_CONST(I32, true), tbl_segs, + NEW_CONST(I32, tbl_seg_idx * sizeof(WASMTableSeg) + + offsetof(WASMTableSeg, is_dropped))); + return true; } bool jit_compile_op_table_get(JitCompContext *cc, uint32 tbl_idx) { + JitReg elem_idx, tbl_sz, tbl_data, elem_idx_long, offset, res; + + POP_I32(elem_idx); + + /* if (elem_idx >= tbl_sz) goto exception; */ + tbl_sz = get_table_cur_size_reg(cc->jit_frame, tbl_idx); + GEN_INSN(CMP, cc->cmp_reg, elem_idx, tbl_sz); + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, JIT_OP_BGEU, + cc->cmp_reg, NULL)) + goto fail; + + elem_idx_long = jit_cc_new_reg_I64(cc); + GEN_INSN(I32TOI64, elem_idx_long, elem_idx); + + offset = jit_cc_new_reg_I64(cc); + GEN_INSN(MUL, offset, elem_idx_long, NEW_CONST(I64, sizeof(uint32))); + + res = jit_cc_new_reg_I32(cc); + tbl_data = get_table_data_reg(cc->jit_frame, tbl_idx); + GEN_INSN(LDI32, res, tbl_data, offset); + PUSH_I32(res); + + return true; +fail: return false; } bool jit_compile_op_table_set(JitCompContext *cc, uint32 tbl_idx) { + JitReg elem_idx, elem_val, tbl_sz, tbl_data, elem_idx_long, offset; + + POP_I32(elem_val); + POP_I32(elem_idx); + + /* if (elem_idx >= tbl_sz) goto exception; */ + tbl_sz = get_table_cur_size_reg(cc->jit_frame, tbl_idx); + GEN_INSN(CMP, cc->cmp_reg, elem_idx, tbl_sz); + if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS, JIT_OP_BGEU, + cc->cmp_reg, NULL)) + goto fail; + + elem_idx_long = jit_cc_new_reg_I64(cc); + GEN_INSN(I32TOI64, elem_idx_long, elem_idx); + + offset = jit_cc_new_reg_I64(cc); + GEN_INSN(MUL, offset, elem_idx_long, NEW_CONST(I64, sizeof(uint32))); + + tbl_data = get_table_data_reg(cc->jit_frame, tbl_idx); + GEN_INSN(STI32, elem_val, tbl_data, offset); + + return true; +fail: return false; } +static int +wasm_init_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 elem_idx, + uint32 dst, uint32 len, uint32 src) +{ + WASMTableInstance *tbl; + uint32 tbl_sz; + WASMTableSeg *elem; + uint32 elem_len; + + tbl = inst->tables[tbl_idx]; + tbl_sz = tbl->cur_size; + if (dst > tbl_sz || tbl_sz - dst < len) + goto out_of_bounds; + + elem = inst->module->table_segments + elem_idx; + elem_len = elem->function_count; + if (src > elem_len || elem_len - src < len) + goto out_of_bounds; + + bh_memcpy_s((uint8 *)(tbl) + offsetof(WASMTableInstance, base_addr) + + dst * sizeof(uint32), + (uint32)((tbl_sz - dst) * sizeof(uint32)), + elem->func_indexes + src, (uint32)(len * sizeof(uint32))); + + return 0; +out_of_bounds: + wasm_set_exception(inst, "out of bounds table access"); + return -1; +} + bool jit_compile_op_table_init(JitCompContext *cc, uint32 tbl_idx, uint32 tbl_seg_idx) { + JitReg len, src, dst, res; + JitReg args[6] = { 0 }; + + POP_I32(len); + POP_I32(src); + POP_I32(dst); + + res = jit_cc_new_reg_I32(cc); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, tbl_idx); + args[2] = NEW_CONST(I32, tbl_seg_idx); + args[3] = dst; + args[4] = len; + args[5] = src; + + if (!jit_emit_callnative(cc, wasm_init_table, res, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) + goto fail; + + return true; +fail: return false; } +static int +wasm_copy_table(WASMModuleInstance *inst, uint32 src_tbl_idx, + uint32 dst_tbl_idx, uint32 dst_offset, uint32 len, + uint32 src_offset) +{ + WASMTableInstance *src_tbl, *dst_tbl; + uint32 src_tbl_sz, dst_tbl_sz; + + src_tbl = inst->tables[src_tbl_idx]; + src_tbl_sz = src_tbl->cur_size; + if (src_offset > src_tbl_sz || src_tbl_sz - src_offset < len) + goto out_of_bounds; + + dst_tbl = inst->tables[dst_tbl_idx]; + dst_tbl_sz = dst_tbl->cur_size; + if (dst_offset > dst_tbl_sz || dst_tbl_sz - dst_offset < len) + goto out_of_bounds; + + bh_memmove_s((uint8 *)(dst_tbl) + offsetof(WASMTableInstance, base_addr) + + dst_offset * sizeof(uint32), + (uint32)((dst_tbl_sz - dst_offset) * sizeof(uint32)), + (uint8 *)(src_tbl) + offsetof(WASMTableInstance, base_addr) + + src_offset * sizeof(uint32), + (uint32)(len * sizeof(uint32))); + + return 0; +out_of_bounds: + wasm_set_exception(inst, "out of bounds table access"); + return -1; +} + bool jit_compile_op_table_copy(JitCompContext *cc, uint32 src_tbl_idx, uint32 dst_tbl_idx) { + JitReg len, src, dst, res; + JitReg args[6] = { 0 }; + + POP_I32(len); + POP_I32(src); + POP_I32(dst); + + res = jit_cc_new_reg_I32(cc); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, src_tbl_idx); + args[2] = NEW_CONST(I32, dst_tbl_idx); + args[3] = dst; + args[4] = len; + args[5] = src; + + if (!jit_emit_callnative(cc, wasm_copy_table, res, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) + goto fail; + + return true; +fail: return false; } bool jit_compile_op_table_size(JitCompContext *cc, uint32 tbl_idx) { + JitReg res; + + res = get_table_cur_size_reg(cc->jit_frame, tbl_idx); + PUSH_I32(res); + + return true; +fail: return false; } bool jit_compile_op_table_grow(JitCompContext *cc, uint32 tbl_idx) { + JitReg tbl_sz, n, val, enlarge_ret, res; + JitReg args[4] = { 0 }; + + POP_I32(n); + POP_I32(val); + + tbl_sz = get_table_cur_size_reg(cc->jit_frame, tbl_idx); + + enlarge_ret = jit_cc_new_reg_I32(cc); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, tbl_idx); + args[2] = n; + args[3] = val; + + if (!jit_emit_callnative(cc, wasm_enlarge_table, enlarge_ret, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + /* Convert bool to uint32 */ + GEN_INSN(AND, enlarge_ret, enlarge_ret, NEW_CONST(I32, 0xFF)); + + res = jit_cc_new_reg_I32(cc); + GEN_INSN(CMP, cc->cmp_reg, enlarge_ret, NEW_CONST(I32, 1)); + GEN_INSN(SELECTEQ, res, cc->cmp_reg, tbl_sz, NEW_CONST(I32, -1)); + PUSH_I32(res); + + /* Ensure a refresh in next get memory related registers */ + clear_table_regs(cc->jit_frame); + return true; +fail: return false; } +static int +wasm_fill_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 dst, + uint32 val, uint32 len) +{ + WASMTableInstance *tbl; + uint32 tbl_sz; + + tbl = inst->tables[tbl_idx]; + tbl_sz = tbl->cur_size; + + if (dst > tbl_sz || tbl_sz - dst < len) + goto out_of_bounds; + + for (; len != 0; dst++, len--) { + ((uint32 *)(tbl->base_addr))[dst] = val; + } + + return 0; +out_of_bounds: + wasm_set_exception(inst, "out of bounds table access"); + return -1; +} + bool jit_compile_op_table_fill(JitCompContext *cc, uint32 tbl_idx) { + JitReg len, val, dst, res; + JitReg args[5] = { 0 }; + + POP_I32(len); + POP_I32(val); + POP_I32(dst); + + res = jit_cc_new_reg_I32(cc); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, tbl_idx); + args[2] = dst; + args[3] = val; + args[4] = len; + + if (!jit_emit_callnative(cc, wasm_fill_table, res, args, + sizeof(args) / sizeof(args[0]))) + goto fail; + + GEN_INSN(CMP, cc->cmp_reg, res, NEW_CONST(I32, 0)); + if (!jit_emit_exception(cc, EXCE_ALREADY_THROWN, JIT_OP_BLTS, cc->cmp_reg, + NULL)) + goto fail; + + return true; +fail: return false; } +#endif diff --git a/core/iwasm/fast-jit/fe/jit_emit_table.h b/core/iwasm/fast-jit/fe/jit_emit_table.h index eaa5893a7..acfb655f2 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_table.h +++ b/core/iwasm/fast-jit/fe/jit_emit_table.h @@ -13,6 +13,7 @@ extern "C" { #endif +#if WASM_ENABLE_REF_TYPES != 0 bool jit_compile_op_elem_drop(JitCompContext *cc, uint32 tbl_seg_idx); @@ -38,6 +39,7 @@ jit_compile_op_table_grow(JitCompContext *cc, uint32 tbl_idx); bool jit_compile_op_table_fill(JitCompContext *cc, uint32 tbl_idx); +#endif #ifdef __cplusplus } /* end of extern "C" */ diff --git a/core/iwasm/fast-jit/jit_frontend.c b/core/iwasm/fast-jit/jit_frontend.c index 6362308fe..8a887aad2 100644 --- a/core/iwasm/fast-jit/jit_frontend.c +++ b/core/iwasm/fast-jit/jit_frontend.c @@ -310,25 +310,60 @@ get_mem_bound_check_16bytes_reg(JitFrame *frame, uint32 mem_idx) JitReg get_tables_reg(JitFrame *frame) { - return 0; + JitCompContext *cc = frame->cc; + JitReg inst_reg = get_module_inst_reg(frame); + + if (!frame->tables_reg) { + frame->tables_reg = cc->tables_reg; + GEN_INSN(LDPTR, frame->tables_reg, inst_reg, + NEW_CONST(I32, offsetof(WASMModuleInstance, tables))); + } + return frame->tables_reg; } JitReg -get_table_inst_reg(JitFrame *frame, uint32 table_idx) +get_table_inst_reg(JitFrame *frame, uint32 tbl_idx) { - return 0; + JitCompContext *cc = frame->cc; + JitReg tables_reg = get_tables_reg(frame); + + if (!frame->table_regs[tbl_idx].table_inst) { + frame->table_regs[tbl_idx].table_inst = + cc->table_regs[tbl_idx].table_inst; + GEN_INSN(LDPTR, frame->table_regs[tbl_idx].table_inst, tables_reg, + NEW_CONST(I32, sizeof(WASMTableInstance *) * tbl_idx)); + } + return frame->table_regs[tbl_idx].table_inst; } JitReg -get_table_data_reg(JitFrame *frame, uint32 table_idx) +get_table_data_reg(JitFrame *frame, uint32 tbl_idx) { - return 0; + JitCompContext *cc = frame->cc; + JitReg table_reg = get_table_inst_reg(frame, tbl_idx); + + if (!frame->table_regs[tbl_idx].table_data) { + frame->table_regs[tbl_idx].table_data = + cc->table_regs[tbl_idx].table_data; + GEN_INSN(ADD, frame->table_regs[tbl_idx].table_data, table_reg, + NEW_CONST(I64, offsetof(WASMTableInstance, base_addr))); + } + return frame->table_regs[tbl_idx].table_data; } JitReg -get_table_cur_size_reg(JitFrame *frame, uint32 table_idx) +get_table_cur_size_reg(JitFrame *frame, uint32 tbl_idx) { - return 0; + JitCompContext *cc = frame->cc; + JitReg table_reg = get_table_inst_reg(frame, tbl_idx); + + if (!frame->table_regs[tbl_idx].table_cur_size) { + frame->table_regs[tbl_idx].table_cur_size = + cc->table_regs[tbl_idx].table_cur_size; + GEN_INSN(LDI32, frame->table_regs[tbl_idx].table_cur_size, table_reg, + NEW_CONST(I32, offsetof(WASMTableInstance, cur_size))); + } + return frame->table_regs[tbl_idx].table_cur_size; } void @@ -1253,14 +1288,10 @@ jit_compile_func(JitCompContext *cc) } case WASM_OP_REF_NULL: { - uint32 type; - - read_leb_uint32(frame_ip, frame_ip_end, type); - - if (!jit_compile_op_ref_null(cc)) + uint32 ref_type; + read_leb_uint32(frame_ip, frame_ip_end, ref_type); + if (!jit_compile_op_ref_null(cc, ref_type)) return false; - - (void)type; break; } case WASM_OP_REF_IS_NULL: