diff --git a/core/config.h b/core/config.h index 9017ba7c9..a86de4d74 100644 --- a/core/config.h +++ b/core/config.h @@ -94,6 +94,14 @@ #define WASM_ENABLE_LAZY_JIT 0 #endif +#ifndef WASM_ENABLE_FAST_JIT +#define WASM_ENABLE_FAST_JIT 0 +#endif + +#ifndef WASM_ENABLE_FAST_JIT_DUMP +#define WASM_ENABLE_FAST_JIT_DUMP 0 +#endif + #ifndef WASM_ENABLE_WAMR_COMPILER #define WASM_ENABLE_WAMR_COMPILER 0 #endif diff --git a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.c b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.c deleted file mode 100644 index 274e329e0..000000000 --- a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.c +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (C) 2021 Intel Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - */ - -#include "jit_codegen.h" - -bool -jit_codegen_init() -{ - return true; -} - -void -jit_codegen_destroy() -{} - -/* clang-format off */ -static const uint8 hreg_info_I4[3][7] = { - /* ebp, eax, ebx, ecx, edx, edi, esi */ - { 1, 0, 0, 0, 0, 0, 1 }, /* fixed, esi is freely used */ - { 0, 1, 0, 1, 1, 0, 0 }, /* caller_saved_native */ - { 0, 1, 0, 1, 1, 1, 0 } /* caller_saved_jitted */ -}; - -static const uint8 hreg_info_I8[3][16] = { - /* rbp, rax, rbx, rcx, rdx, rdi, rsi, rsp, - r8, r9, r10, r11, r12, r13, r14, r15 */ - { 1, 1, 1, 1, 1, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 1 }, /* fixed, rsi is freely used */ - { 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 0, 0, 0, 0, 0, 0 }, /* caller_saved_native */ - { 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 0, 0, 0, 0, 0, 0 }, /* caller_saved_jitted */ -}; - -static uint8 hreg_info_F4[3][16] = { - { 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 1, 1, 1, 1 }, /* fixed, rsi is freely used */ - { 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1 }, /* caller_saved_native */ - { 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1 }, /* caller_saved_jitted */ -}; - -static uint8 hreg_info_F8[3][16] = { - { 1, 1, 1, 1, 1, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0 }, /* fixed, rsi is freely used */ - { 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1 }, /* caller_saved_native */ - { 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1 }, /* caller_saved_jitted */ -}; - -static const JitHardRegInfo hreg_info = { - { - { 0, NULL, NULL, NULL }, /* VOID */ - - { sizeof(hreg_info_I4[0]), /* I4 */ - hreg_info_I4[0], - hreg_info_I4[1], - hreg_info_I4[2] }, - - { sizeof(hreg_info_I8[0]), /* I8 */ - hreg_info_I8[0], - hreg_info_I8[1], - hreg_info_I8[2] }, - - { sizeof(hreg_info_F4[0]), /* F4 */ - hreg_info_F4[0], - hreg_info_F4[1], - hreg_info_F4[2] }, - - { sizeof(hreg_info_F8[0]), /* F8 */ - hreg_info_F8[0], - hreg_info_F8[1], - hreg_info_F8[2] }, - - { 0, NULL, NULL, NULL }, /* V8 */ - { 0, NULL, NULL, NULL }, /* V16 */ - { 0, NULL, NULL, NULL } /* V32 */ - }, - /* frame pointer hreg index: rbp */ - 0, - /* exec_env hreg index: r15 */ - 15, - /* cmp hreg index: esi */ - 6 -}; -/* clang-format on */ - -const JitHardRegInfo * -jit_codegen_get_hreg_info() -{ - return &hreg_info; -} - -bool -jit_codegen_gen_native(JitCompContext *cc) -{ - jit_set_last_error(cc, "jit_codegen_gen_native failed"); - return false; -} - -bool -jit_codegen_lower(JitCompContext *cc) -{ - return true; -} - -void -jit_codegen_dump_native(void *begin_addr, void *end_addr) -{} - -bool -jit_codegen_call_func_jitted(void *exec_env, void *frame, void *func_inst, - void *target) -{ - return false; -} diff --git a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp new file mode 100644 index 000000000..d6b01d029 --- /dev/null +++ b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp @@ -0,0 +1,3233 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_codegen.h" +#include "jit_codecache.h" +#include "jit_compiler.h" +#include "jit_dump.h" + +#include +#include +#if WASM_ENABLE_FAST_JIT_DUMP != 0 +#include +#endif + +#define CODEGEN_CHECK_ARGS 1 + +using namespace asmjit; + +static char *code_block_switch_to_jitted_from_interp = NULL; +static char *code_block_return_to_interp_from_jitted = NULL; + +typedef enum { + REG_EBP_IDX = 0, + REG_EAX_IDX, + REG_EBX_IDX, + REG_ECX_IDX, + REG_EDX_IDX, + REG_EDI_IDX, + REG_ESI_IDX, + REG_I32_FREE_IDX = REG_ESI_IDX +} RegIndexI32; + +typedef enum { + REG_RBP_IDX = 0, + REG_RAX_IDX, + REG_RBX_IDX, + REG_RCX_IDX, + REG_RDX_IDX, + REG_RDI_IDX, + REG_RSI_IDX, + REG_RSP_IDX, + REG_R8_IDX, + REG_R9_IDX, + REG_R10_IDX, + REG_R11_IDX, + REG_R12_IDX, + REG_R13_IDX, + REG_R14_IDX, + REG_R15_IDX, + REG_I64_FREE_IDX = REG_RSI_IDX +} RegIndexI64; + +x86::Gp regs_i32[] = { x86::ebp, x86::eax, x86::ebx, x86::ecx, + x86::edx, x86::edi, x86::esi }; + +x86::Gp regs_i64[] = { + x86::rbp, x86::rax, x86::rbx, x86::rcx, x86::rdx, x86::rdi, + x86::rsi, x86::rsp, x86::r8, x86::r9, x86::r10, x86::r11, + x86::r12, x86::r13, x86::r14, x86::r15, +}; + +int +jit_codegen_interp_jitted_glue(void *exec_env, JitInterpSwitchInfo *info, + void *target) +{ + typedef int32 (*F)(const void *exec_env, void *info, const void *target); + union { + F f; + void *v; + } u; + + u.v = code_block_switch_to_jitted_from_interp; + return u.f(exec_env, info, target); +} + +#define PRINT_LINE() LOG_VERBOSE("\n", __LINE__) +#define GOTO_FAIL goto fail + +#if CODEGEN_CHECK_ARGS == 0 + +#define CHECK_EQKIND(reg0, reg1) (void)0 +#define CHECK_CONST(reg0) (void)0 +#define CHECK_NCONST(reg0) (void)0 +#define CHECK_KIND(reg0, type) (void)0 + +#else + +/* Check if two register's kind is equal */ +#define CHECK_EQKIND(reg0, reg1) \ + do { \ + if (jit_reg_kind(reg0) != jit_reg_kind(reg1)) { \ + PRINT_LINE(); \ + LOG_VERBOSE("reg type not equal:\n"); \ + jit_dump_reg(cc, reg0); \ + jit_dump_reg(cc, reg1); \ + GOTO_FAIL; \ + } \ + } while (0) + +/* Check if a register is an const */ +#define CHECK_CONST(reg0) \ + do { \ + if (!jit_reg_is_const(reg0)) { \ + PRINT_LINE(); \ + LOG_VERBOSE("reg is not const:\n"); \ + jit_dump_reg(cc, reg0); \ + GOTO_FAIL; \ + } \ + } while (0) + +/* Check if a register is not an const */ +#define CHECK_NCONST(reg0) \ + do { \ + if (jit_reg_is_const(reg0)) { \ + PRINT_LINE(); \ + LOG_VERBOSE("reg is const:\n"); \ + jit_dump_reg(cc, reg0); \ + GOTO_FAIL; \ + } \ + } while (0) + +/* Check if a register is a special type */ +#define CHECK_KIND(reg0, type) \ + do { \ + if (jit_reg_kind(reg0) != type) { \ + PRINT_LINE(); \ + LOG_VERBOSE("invalid reg type %d, expected is: %d", \ + jit_reg_kind(reg0), type); \ + jit_dump_reg(cc, reg0); \ + GOTO_FAIL; \ + } \ + } while (0) + +#endif /* end of CODEGEN_CHECK_ARGS == 0 */ + +/* Load one operand from insn and check none */ +#define LOAD_1ARG r0 = *jit_insn_opnd(insn, 0); + +/* Load two operands from insn and check if r0 is non-const */ +#define LOAD_2ARGS() \ + r0 = *jit_insn_opnd(insn, 0); \ + r1 = *jit_insn_opnd(insn, 1); \ + CHECK_NCONST(r0) + +/* Load three operands from insn and check if r0 is non-const */ +#define LOAD_3ARGS() \ + r0 = *jit_insn_opnd(insn, 0); \ + r1 = *jit_insn_opnd(insn, 1); \ + r2 = *jit_insn_opnd(insn, 2); \ + CHECK_NCONST(r0) + +/* Load three operands from insn and check none */ +#define LOAD_3ARGS_NO_ASSIGN() \ + r0 = *jit_insn_opnd(insn, 0); \ + r1 = *jit_insn_opnd(insn, 1); \ + r2 = *jit_insn_opnd(insn, 2); + +/* Load four operands from insn and check if r0 is non-const */ +#define LOAD_4ARGS() \ + r0 = *jit_insn_opnd(insn, 0); \ + r1 = *jit_insn_opnd(insn, 1); \ + r2 = *jit_insn_opnd(insn, 2); \ + r3 = *jit_insn_opnd(insn, 3); \ + CHECK_NCONST(r0) + +/** + * Encode extending register of byte to register of dword + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src tho no of src register + * @param is_signed the data is signed or unsigned + * + * @return true if success, false otherwise + */ +static bool +extend_r8_to_r32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src, + bool is_signed) +{ + return false; +} +/** + * Encode extending register of word to register of dword + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src tho no of src register + * @param is_signed the data is signed or unsigned + * + * @return true if success, false otherwise + */ +static bool +extend_r16_to_r32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src, + bool is_signed) +{ + return false; +} + +/** + * Encode extending register of byte to register of qword + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src tho no of src register + * @param is_signed the data is signed or unsigned + * + * @return true if success, false otherwise + */ +static bool +extend_r8_to_r64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src, + bool is_signed) +{ + return false; +} + +/** + * Encode extending register of word to register of qword + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src tho no of src register + * @param is_signed the data is signed or unsigned + * + * @return true if success, false otherwise + */ +static bool +extend_r16_to_r64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src, + bool is_signed) +{ + return false; +} + +/** + * Encode extending register of dword to register of qword + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src tho no of src register + * @param is_signed the data is signed or unsigned + * + * @return true if success, false otherwise + */ +static bool +extend_r32_to_r64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src, + bool is_signed) +{ + return false; +} + +/** + * Encode loading register data from memory with imm base and imm offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param is_signed the data is signed or unsigned + * @param reg_no_dst the index of dest register + * @param base the base address of the memory + * @param offset the offset address of the memory + * + * @return true if success, false otherwise + */ +static bool +ld_r_from_base_imm_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, bool is_signed, int32 reg_no_dst, + int32 base, int32 offset) +{ + return false; +} + +/** + * Encode loading register data from memory with imm base and register offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param is_signed the data is signed or unsigned + * @param reg_no_dst the index of dest register + * @param base the base address of the memory + * @param reg_no_offset the no of register which stores the offset of the memory + * + * @return true if success, false otherwise + */ +static bool +ld_r_from_base_imm_offset_r(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, bool is_signed, int32 reg_no_dst, + int32 base, int32 reg_no_offset) +{ + return false; +} + +/** + * Encode loading register data from memory with register base and imm offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param is_signed the data is signed or unsigned + * @param reg_no_dst the index of dest register + * @param reg_no_base the no of register which stores the base of the memory + * @param offset the offset address of the memory + * + * @return true if success, false otherwise + */ +static bool +ld_r_from_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, bool is_signed, int32 reg_no_dst, + int32 reg_no_base, int32 offset) +{ + return false; +} + +/** + * Encode loading register data from memory with register base and register + * offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param is_signed the data is signed or unsigned + * @param reg_no_dst the index of dest register + * @param reg_no_base the no of register which stores the base of the memory + * @param reg_no_offset the no of register which stores the offset of the memory + * + * @return true if success, false otherwise + */ +static bool +ld_r_from_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, + bool is_signed, int32 reg_no_dst, int32 reg_no_base, + int32 reg_no_offset) +{ + return false; +} + +/** + * Encode storing register data to memory with imm base and imm offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param reg_no_src the index of src register + * @param base the base address of the dst memory + * @param offset the offset address of the dst memory + * + * @return true if success, false otherwise + */ +static bool +st_r_to_base_imm_offset_imm(x86::Assembler &a, uint32 bytes_dst, + uint32 kind_dst, int32 reg_no_src, int32 base, + int32 offset) +{ + return false; +} + +/** + * Encode storing register data to memory with imm base and register offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param reg_no_src the index of src register + * @param base the base address of the dst memory + * @param reg_no_offset the no of register which stores the offset of the dst + * memory + * + * @return true if success, false otherwise + */ +static bool +st_r_to_base_imm_offset_r(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, + int32 reg_no_src, int32 base, int32 reg_no_offset) +{ + return false; +} + +/** + * Encode storing register data to memory with register base and imm offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param reg_no_src the index of src register + * @param reg_no_base the no of register which stores the base of the dst memory + * @param offset the offset address of the dst memory + * + * @return true if success, false otherwise + */ +static bool +st_r_to_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, + int32 reg_no_src, int32 reg_no_base, int32 offset) +{ + return false; +} + +/** + * Encode storing register data to memory with register base and register offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64), skipped by + * float/double + * @param kind_dst the kind of data to move, could be I32, I64, F32 or F64 + * @param reg_no_src the index of src register + * @param reg_no_base the no of register which stores the base of the dst memory + * @param reg_no_offset the no of register which stores the offset of the dst + * memory + * + * @return true if success, false otherwise + */ +static bool +st_r_to_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, uint32 kind_dst, + int32 reg_no_src, int32 reg_no_base, + int32 reg_no_offset) +{ + return false; +} + +/** + * Encode storing int32 imm data to memory with imm base and imm offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64) + * @param data_src the src immediate data + * @param base the base address of dst memory + * @param offset the offset address of dst memory + * + * @return true if success, false otherwise + */ +static bool +st_imm_to_base_imm_offset_imm(x86::Assembler &a, uint32 bytes_dst, + void *data_src, int32 base, int32 offset) +{ + return false; +} + +/** + * Encode storing int32 imm data to memory with imm base and reg offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64) + * @param data_src the src immediate data + * @param base the base address of dst memory + * @param reg_no_offset the no of register that stores the offset address + * of dst memory + * + * @return true if success, false otherwise + */ +static bool +st_imm_to_base_imm_offset_r(x86::Assembler &a, uint32 bytes_dst, void *data_src, + int32 base, int32 reg_no_offset) +{ + return false; +} + +/** + * Encode storing int32 imm data to memory with reg base and imm offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64) + * @param data_src the src immediate data + * @param reg_no_base the no of register that stores the base address + * of dst memory + * @param offset the offset address of dst memory + * + * @return true if success, false otherwise + */ +static bool +st_imm_to_base_r_offset_imm(x86::Assembler &a, uint32 bytes_dst, void *data_src, + int32 reg_no_base, int32 offset) +{ + return false; +} + +/** + * Encode storing int32 imm data to memory with reg base and reg offset + * + * @param a the assembler to emit the code + * @param bytes_dst the bytes number of the data, + * could be 1(byte), 2(short), 4(int32), 8(int64) + * @param data_src the src immediate data + * @param reg_no_base the no of register that stores the base address + * of dst memory + * @param reg_no_offset the no of register that stores the offset address + * of dst memory + * + * @return true if success, false otherwise + */ +static bool +st_imm_to_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, void *data_src, + int32 reg_no_base, int32 reg_no_offset) +{ + return false; +} + +/** + * Encode moving immediate int32 data to register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst register + * @param data the immediate data to move + * + * @return true if success, false otherwise + */ +static bool +mov_imm_to_r_i32(x86::Assembler &a, int32 reg_no, int32 data) +{ + return false; +} + +/** + * Encode moving int32 data from src register to dst register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +mov_r_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode moving immediate int64 data to register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst register + * @param data the immediate data to move + * + * @return true if success, false otherwise + */ +static bool +mov_imm_to_r_i64(x86::Assembler &a, int32 reg_no, int32 data) +{ + return false; +} + +/** + * Encode moving int64 data from src register to dst register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +mov_r_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode moving immediate float data to register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst register + * @param data the immediate data to move + * + * @return true if success, false otherwise + */ +static bool +mov_imm_to_r_f32(x86::Assembler &a, int32 reg_no, float data) +{ + return false; +} + +/** + * Encode moving float data from src register to dst register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +mov_r_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode moving immediate double data to register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst register + * @param data the immediate data to move + * + * @return true if success, false otherwise + */ +static bool +mov_imm_to_r_f64(x86::Assembler &a, int32 reg_no, double data) +{ + return false; +} + +/** + * Encode moving double data from src register to dst register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +mov_r_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encoding convert int32 immediate data to int8 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to int8 + * @param data the src int32 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_int8(x86::Assembler &a, int32 reg_no, int32 data) +{ + return false; +} + +/** + * Encoding convert int32 immediate data to int8 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register, need to be converted to int8 + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_int8(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encoding convert int32 immediate data to uint8 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to uint8 + * @param data the src int32 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_uint8(x86::Assembler &a, int32 reg_no, int32 data) +{ + return false; +} + +/** + * Encoding convert int32 immediate data to uint8 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register, need to be converted to uint8 + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_uint8(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encoding convert int32 immediate data to int16 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to int16 + * @param data the src int32 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_int16(x86::Assembler &a, int32 reg_no, int32 data) +{ + return false; +} + +/** + * Encoding convert int32 immediate data to int16 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register, need to be converted to int16 + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_int16(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encoding convert int32 immediate data to uint16 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to uint16 + * @param data the src int32 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_uint16(x86::Assembler &a, int32 reg_no, int32 data) +{ + return false; +} + +/** + * Encoding convert int32 immediate data to uint16 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register, need to be converted to uint16 + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_uint16(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encoding convert int32 immediate data to uint64 register + * + * @param a the assembler to emit the code + * @param reg_no the dst register, need to be converted to uint64 + * @param data the src int32 immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_uint64(x86::Assembler &a, int32 reg_no, int32 data) +{ + return false; +} + +/** + * Encoding convert int32 immediate data to uint64 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the dst register, need to be converted to uint64 + * @param reg_no_src the src register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_uint64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode converting int32 immediate data to float register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst float register + * @param data the src immediate data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_f32(x86::Assembler &a, int32 reg_no, int32 data) +{ + return false; +} + +/** + * Encode converting int32 register data to float register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst float register + * @param reg_no_src the no of src int32 register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode converting int32 immediate data to double register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst double register + * @param data the src immediate int32 data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_i32_to_r_f64(x86::Assembler &a, int32 reg_no, int32 data) +{ + return false; +} + +/** + * Encode converting int32 register data to double register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst double register + * @param reg_no_src the no of src int32 register + * + * @return true if success, false otherwise + */ +static bool +convert_r_i32_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode converting float immediate data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int32 register + * @param data the src immediate float data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_f32_to_r_i32(x86::Assembler &a, int32 reg_no, float data) +{ + return false; +} + +/** + * Encode converting float register data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int32 register + * @param reg_no_src the no of src float register + * + * @return true if success, false otherwise + */ +static bool +convert_r_f32_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode converting float immediate data to double register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst double register + * @param data the src immediate float data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_f32_to_r_f64(x86::Assembler &a, int32 reg_no, float data) +{ + return false; +} + +/** + * Encode converting float register data to double register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst double register + * @param reg_no_src the no of src float register + * + * @return true if success, false otherwise + */ +static bool +convert_r_f32_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode converting double immediate data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst int32 register + * @param data the src immediate double data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_f64_to_r_i32(x86::Assembler &a, int32 reg_no, double data) +{ + return false; +} + +/** + * Encode converting double register data to int32 register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst int32 register + * @param reg_no_src the no of src double register + * + * @return true if success, false otherwise + */ +static bool +convert_r_f64_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode converting double immediate data to float register data + * + * @param a the assembler to emit the code + * @param reg_no the no of dst float register + * @param data the src immediate double data + * + * @return true if success, false otherwise + */ +static bool +convert_imm_f64_to_r_f32(x86::Assembler &a, int32 reg_no, double data) +{ + return false; +} + +/** + * Encode converting double register data to float register data + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst float register + * @param reg_no_src the no of src double register + * + * @return true if success, false otherwise + */ +static bool +convert_r_f64_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode making negative from int32 immediate data to int32 register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst register + * @param data the src int32 immediate data + * + * @return true if success, false otherwise + */ +static bool +neg_imm_to_r_i32(x86::Assembler &a, int32 reg_no, int32 data) +{ + return false; +} + +/** + * Encode making negative from int32 register to int32 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +neg_r_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode making negative from int64 immediate data to int64 register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst register + * @param data the src int64 immediate data + * + * @return true if success, false otherwise + */ +static bool +neg_imm_to_r_i64(x86::Assembler &a, int32 reg_no, int64 data) +{ + return false; +} + +/** + * Encode making negative from int64 register to int64 register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +neg_r_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode making negative from float immediate data to float register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst float register + * @param data the src float immediate data + * + * @return true if success, false otherwise + */ +static bool +neg_imm_to_r_f32(x86::Assembler &a, int32 reg_no, float data) +{ + return false; +} + +/** + * Encode making negative from float register to float register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register + * + * @return true if success, false otherwise + */ +static bool +neg_r_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode making negative from double immediate data to double register + * + * @param a the assembler to emit the code + * @param reg_no the no of dst double register + * @param data the src double immediate data + * + * @return true if success, false otherwise + */ +static bool +neg_imm_to_r_f64(x86::Assembler &a, int32 reg_no, double data) +{ + return false; +} + +/** + * Encode making negative from double register to double register + * + * @param a the assembler to emit the code + * @param reg_no_dst the no of dst double register + * @param reg_no_src the no of src double register + * + * @return true if success, false otherwise + */ +static bool +neg_r_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/* Alu opcode */ +typedef enum { ADD, SUB, MUL, DIV, REM } ALU_OP; +/* Bit opcode */ +typedef enum { OR, XOR, AND } BIT_OP; +/* Shift opcode */ +typedef enum { SHL, SHRS, SHRU } SHIFT_OP; +/* Condition opcode */ +typedef enum { EQ, NE, GTS, GES, LTS, LES, GTU, GEU, LTU, LEU } COND_OP; + +static COND_OP +not_cond(COND_OP op) +{ + COND_OP not_list[] = { NE, EQ, LES, LTS, GES, GTS, LEU, LTU, GEU, GTU }; + + bh_assert(op <= LEU); + return not_list[op]; +} + +/** + * Encode int32 alu operation of reg and data, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no the no of register, as first operand, and save result + * @param data the immediate data, as the second operand + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_imm_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no_src, int32 data) +{ + return false; +} + +/** + * Encode int32 alu operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register, as first operand, and save result + * @param reg_no_src the no of register, as the second operand + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_r_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int32 alu operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_imm_to_r_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 data1_src, int32 data2_src) +{ + return false; +} + +/** + * Encode int32 alu operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_r_to_r_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 data1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int32 alu operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_r_imm_to_r_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 data2_src) +{ + return false; +} + +/** + * Encode int32 alu operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_to_r_i32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int64 alu operation of reg and data, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no the no of register, as first operand, and save result + * @param data the immediate data, as the second operand + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_imm_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no_src, int64 data) +{ + return false; +} + +/** + * Encode int64 alu operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register, as first operand, and save result + * @param reg_no_src the no of register, as the second operand + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_r_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int64 alu operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_imm_to_r_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int64 data1_src, int64 data2_src) +{ + return false; +} + +/** + * Encode int64 alu operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_r_to_r_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int64 data1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int64 alu operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_r_imm_to_r_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, int64 data2_src) +{ + return false; +} + +/** + * Encode int64 alu operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_to_r_i64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode float alu operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_imm_to_r_f32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + float data1_src, float data2_src) +{ + return false; +} + +/** + * Encode float alu operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_r_to_r_f32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + float data1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode float alu operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +alu_r_imm_to_r_f32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, float data2_src) +{ + return false; +} + +/** + * Encode float alu operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_to_r_f32(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode double alu operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_imm_to_r_f64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + double data1_src, double data2_src) +{ + return false; +} + +/** + * Encode double alu operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_imm_r_to_r_f64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + double data1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode double alu operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +alu_r_imm_to_r_f64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, double data2_src) +{ + return false; +} + +/** + * Encode double alu operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of ALU operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +alu_r_r_to_r_f64(x86::Assembler &a, ALU_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int32 bit operation of reg and data, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no the no of register, as first operand, and save result + * @param data the immediate data, as the second operand + * + * @return true if success, false otherwise + */ +static bool +bit_r_imm_i32(x86::Assembler &a, BIT_OP op, int32 reg_no, int32 data) +{ + return false; +} + +/** + * Encode int32 bit operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register, as first operand, and save result + * @param reg_no_src the no of register, as second operand + * + * @return true if success, false otherwise + */ +static bool +bit_r_r_i32(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode int32 bit operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +bit_imm_imm_to_r_i32(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 data1_src, int32 data2_src) +{ + return false; +} + +/** + * Encode int32 bit operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +bit_imm_r_to_r_i32(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 data1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int32 bit operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +bit_r_imm_to_r_i32(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 data2_src) +{ + return false; +} + +/** + * Encode int32 bit operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +bit_r_r_to_r_i32(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int64 bit operation of reg and data, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no the no of register, as first operand, and save result + * @param data the immediate data, as the second operand + * + * @return true if success, false otherwise + */ +static bool +bit_r_imm_i64(x86::Assembler &a, BIT_OP op, int32 reg_no, int64 data) +{ + return false; +} + +/** + * Encode int64 bit operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register, as first operand, and save result + * @param reg_no_src the no of register, as second operand + * + * @return true if success, false otherwise + */ +static bool +bit_r_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, int32 reg_no_src) +{ + return false; +} + +/** + * Encode int64 bit operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +bit_imm_imm_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 data1_src, int64 data2_src) +{ + return false; +} + +/** + * Encode int64 bit operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +bit_imm_r_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int64 data1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int64 bit operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +bit_r_imm_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int64 data2_src) +{ + return false; +} + +/** + * Encode int64 bit operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of BIT operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +bit_r_r_to_r_i64(x86::Assembler &a, BIT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int32 shift operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of SHIFT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +shift_imm_imm_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int32 data1_src, int32 data2_src) +{ + return false; +} + +/** + * Encode int32 shift operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of SHIFT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +shift_imm_r_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int32 data1_src, int32 reg_no2_src) +{ + /* Should have been optimized by previous lower */ + bh_assert(0); + return false; +} + +/** + * Encode int32 shift operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of SHIFT operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +shift_r_imm_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 data2_src) +{ + return false; +} + +/** + * Encode int32 shift operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of shift operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +shift_r_r_to_r_i32(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int64 shift operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of SHIFT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +shift_imm_imm_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int64 data1_src, int64 data2_src) +{ + return false; +} + +/** + * Encode int64 shift operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of SHIFT operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +shift_imm_r_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int64 data1_src, int32 reg_no2_src) +{ + /* Should have been optimized by previous lower */ + bh_assert(0); + return false; +} + +/** + * Encode int64 shift operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of SHIFT operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +shift_r_imm_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int64 data2_src) +{ + return false; +} + +/** + * Encode int64 shift operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of shift operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +shift_r_r_to_r_i64(x86::Assembler &a, SHIFT_OP op, int32 reg_no_dst, + int32 reg_no1_src, int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int32 cmp operation of reg and data, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register, as first operand + * @param data the immediate data, as the second operand + * + * @return true if success, false otherwise + */ +static bool +cmp_r_imm_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src, int32 data) +{ + return false; +} + +/** + * Encode int32 cmp operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of dst register + * @param reg_no1_src the no of src register, as first operand + * @param reg_no2_src the no of src register, as second operand + * + * @return true if success, false otherwise + */ +static bool +cmp_r_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int32 cmp operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_imm_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 data1_src, + int32 data2_src) +{ + return false; +} + +/** + * Encode int32 cmp operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_r_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 data1_src, + int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int32 cmp operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_imm_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int32 data2_src) +{ + return false; +} + +/** + * Encode int32 cmp operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_r_to_r_i32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int64 cmp operation of reg and data, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of dst register + * @param reg_no_src the no of src register, as first operand + * @param data the immediate data, as the second operand + * + * @return true if success, false otherwise + */ +static bool +cmp_r_imm_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no_src, int64 data) +{ + return false; +} + +/** + * Encode int64 cmp operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of dst register + * @param reg_no1_src the no of src register, as first operand + * @param reg_no2_src the no of src register, as second operand + * + * @return true if success, false otherwise + */ +static bool +cmp_r_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int64 cmp operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_imm_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 data1_src, + int32 data2_src) +{ + return false; +} + +/** + * Encode int64 cmp operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_r_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int64 data1_src, + int32 reg_no2_src) +{ + return false; +} + +/** + * Encode int64 cmp operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_imm_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int64 data2_src) +{ + return false; +} + +/** + * Encode int64 cmp operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_r_to_r_i64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + return false; +} + +/** + * Encode float cmp operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_imm_to_r_f32(x86::Assembler &a, int32 reg_no_dst, float data1_src, + float data2_src) +{ + return false; +} + +/** + * Encode float cmp operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_r_to_r_f32(x86::Assembler &a, int32 reg_no_dst, float data1_src, + int32 reg_no2_src) +{ + return false; +} + +/** + * Encode float cmp operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_imm_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + float data2_src) +{ + return false; +} + +/** + * Encode float cmp operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_r_to_r_f32(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + return false; +} + +/** + * Encode double cmp operation of imm and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_imm_to_r_f64(x86::Assembler &a, int32 reg_no_dst, double data1_src, + double data2_src) +{ + return false; +} + +/** + * Encode double cmp operation of imm and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param data1_src the first src immediate data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_imm_r_to_r_f64(x86::Assembler &a, int32 reg_no_dst, double data1_src, + int32 reg_no2_src) +{ + return false; +} + +/** + * Encode double cmp operation of reg and imm, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param data2_src the second src immediate data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_imm_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + double data2_src) +{ + return false; +} + +/** + * Encode double cmp operation of reg and reg, and save result to reg + * + * @param a the assembler to emit the code + * @param op the opcode of cmp operation + * @param reg_no_dst the no of register + * @param reg_no1_src the reg no of first src register data + * @param reg_no2_src the reg no of second src register data + * + * @return true if success, false otherwise + */ +static bool +cmp_r_r_to_r_f64(x86::Assembler &a, int32 reg_no_dst, int32 reg_no1_src, + int32 reg_no2_src) +{ + return false; +} + +/** + * Encode insn ld: LD_type r0, r1, r2 + * @param kind the data kind, such as I32, I64, F32 and F64 + * @param bytes_dst the byte number of dst data + * @param is_signed the data is signed or unsigned + */ +#define LD_R_R_R(kind, bytes_dst, is_signed) \ + do { \ + int32 reg_no_dst; \ + int32 base, offset; \ + bool _ret = false; \ + \ + CHECK_KIND(r1, JIT_REG_KIND_I64); \ + CHECK_KIND(r2, JIT_REG_KIND_I32); \ + base = 0; \ + offset = 0; \ + real_opnd_to_reg[1] = r2; \ + \ + reg_no_dst = jit_reg_no(r0); \ + if (jit_reg_is_const(r1)) \ + base = jit_cc_get_const_I32(cc, r1); \ + if (jit_reg_is_const(r2)) \ + offset = jit_cc_get_const_I32(cc, r2); \ + \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = ld_r_from_base_imm_offset_imm( \ + a, bytes_dst, JIT_REG_KIND_##kind, is_signed, reg_no_dst, \ + base, offset); \ + else \ + _ret = ld_r_from_base_imm_offset_r( \ + a, bytes_dst, JIT_REG_KIND_##kind, is_signed, reg_no_dst, \ + base, jit_reg_no(r2)); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = ld_r_from_base_r_offset_imm( \ + a, bytes_dst, JIT_REG_KIND_##kind, is_signed, reg_no_dst, \ + jit_reg_no(r1), offset); \ + else \ + _ret = ld_r_from_base_r_offset_r( \ + a, bytes_dst, JIT_REG_KIND_##kind, is_signed, reg_no_dst, \ + jit_reg_no(r1), jit_reg_no(r2)); \ + if (_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode insn sd: ST_type r0, r1, r2 + * @param kind the data kind, such as I32, I64, F32 and F64 + * @param bytes_dst the byte number of dst data + */ +#define ST_R_R_R(kind, type, bytes_dst) \ + do { \ + type data_src = 0; \ + int32 reg_no_src = 0; \ + int32 base, offset; \ + bool _ret = false; \ + \ + CHECK_KIND(r1, JIT_REG_KIND_I64); \ + CHECK_KIND(r2, JIT_REG_KIND_I32); \ + base = 0; \ + offset = 0; \ + real_opnd_to_reg[0] = r2; \ + real_opnd_to_reg[1] = r0; \ + \ + if (jit_reg_is_const(r0)) \ + data_src = jit_cc_get_const_##kind(cc, r0); \ + else \ + reg_no_src = jit_reg_no(r0); \ + if (jit_reg_is_const(r1)) \ + base = jit_cc_get_const_I32(cc, r1); \ + if (jit_reg_is_const(r2)) \ + offset = jit_cc_get_const_I32(cc, r2); \ + \ + if (jit_reg_is_const(r0)) { \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = st_imm_to_base_imm_offset_imm( \ + a, bytes_dst, &data_src, base, offset); \ + else \ + _ret = st_imm_to_base_imm_offset_r( \ + a, bytes_dst, &data_src, base, jit_reg_no(r2)); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = st_imm_to_base_r_offset_imm(a, bytes_dst, &data_src, \ + jit_reg_no(r1), offset); \ + else \ + _ret = st_imm_to_base_r_offset_r( \ + a, bytes_dst, &data_src, jit_reg_no(r1), jit_reg_no(r2)); \ + } \ + else if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = st_r_to_base_imm_offset_imm(a, bytes_dst, \ + JIT_REG_KIND_##kind, \ + reg_no_src, base, offset); \ + else \ + _ret = st_r_to_base_imm_offset_r( \ + a, bytes_dst, JIT_REG_KIND_##kind, reg_no_src, base, \ + jit_reg_no(r2)); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = \ + st_r_to_base_r_offset_imm(a, bytes_dst, JIT_REG_KIND_##kind, \ + reg_no_src, jit_reg_no(r1), offset); \ + else \ + _ret = st_r_to_base_r_offset_r(a, bytes_dst, JIT_REG_KIND_##kind, \ + reg_no_src, jit_reg_no(r1), \ + jit_reg_no(r2)); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode insn mov: MOV r0, r1 + * @param kind the data kind, such as I32, I64, F32 and F64 + * @param Type the data type, such as int32, int64, float32, and float64 + * @param type the abbreviation of data type, such as i32, i64, f32, and f64 + * @param bytes_dst the byte number of dst data + */ +#define MOV_R_R(kind, Type, type) \ + do { \ + bool _ret = false; \ + CHECK_EQKIND(r0, r1); \ + if (jit_reg_is_const(r1)) { \ + Type data = jit_cc_get_const_##kind(cc, r1); \ + _ret = mov_imm_to_r_##type(a, jit_reg_no(r0), data); \ + } \ + else \ + _ret = mov_r_to_r_##type(a, jit_reg_no(r0), jit_reg_no(r1)); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode mov insn, MOV r0, r1 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the src operand info + * + * @return true if success, false if failed + */ +static bool +lower_mov(JitCompContext *cc, x86::Assembler &a, JitReg r0, JitReg r1) +{ + switch (jit_reg_kind(r0)) { + case JIT_REG_KIND_I32: + MOV_R_R(I32, int32, i32); + break; + case JIT_REG_KIND_I64: + MOV_R_R(I64, int64, i64); + break; + case JIT_REG_KIND_F32: + MOV_R_R(F32, float32, f32); + break; + case JIT_REG_KIND_F64: + MOV_R_R(F64, float64, f64); + break; + default: + LOG_VERBOSE("Invalid reg type of mov: %d\n", jit_reg_kind(r0)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode insn neg: NEG r0, r1 + * @param kind the data kind, such as I32, I64, F32 and F64 + * @param Type the data type, such as int32, int64, float32, and float64 + * @param type the abbreviation of data type, such as i32, i64, f32, and f64 + */ +#define NEG_R_R(kind, Type, type) \ + do { \ + bool _ret = false; \ + CHECK_EQKIND(r0, r1); \ + if (jit_reg_is_const(r1)) { \ + Type data = jit_cc_get_const_##kind(cc, r1); \ + _ret = neg_imm_to_r_##type(a, jit_reg_no(r0), data); \ + } \ + else \ + _ret = neg_r_to_r_##type(a, jit_reg_no(r0), jit_reg_no(r1)); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode neg insn, NEG r0, r1 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the src operand info + * + * @return true if success, false if failed + */ +static bool +lower_neg(JitCompContext *cc, x86::Assembler &a, JitReg r0, JitReg r1) +{ + switch (jit_reg_kind(r0)) { + case JIT_REG_KIND_I32: + NEG_R_R(I32, int32, i32); + break; + case JIT_REG_KIND_I64: + NEG_R_R(I64, int64, i64); + break; + case JIT_REG_KIND_F32: + NEG_R_R(F32, float32, f32); + break; + case JIT_REG_KIND_F64: + NEG_R_R(F64, float64, f64); + break; + default: + LOG_VERBOSE("Invalid reg type of neg: %d\n", jit_reg_kind(r0)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode insn convert: I32TOI1 r0, r1, or I32TOI2, I32TOF32, F32TOF64, etc. + * @param kind0 the dst data kind, such as I32, I64, F32 and F64 + * @param kind1 the src data kind, such as I32, I64, F32 and F64 + * @param type0 the dst data type, such as int32, float and double + * @param type1 the src data type, such as int32, float and double + */ +#define CONVERT_R_R(kind0, kind1, type0, type1) \ + do { \ + bool _ret = false; \ + CHECK_KIND(r0, JIT_REG_KIND_##kind0); \ + CHECK_KIND(r1, JIT_REG_KIND_##kind1); \ + if (jit_reg_is_const(r1)) { \ + type1 data = jit_cc_get_const_##kind1(cc, r1); \ + _ret = \ + convert_imm_##type1##_to_r_##type0(a, jit_reg_no(r0), data); \ + } \ + else \ + _ret = convert_r_##type1##_to_r_##type0(a, jit_reg_no(r0), \ + jit_reg_no(r1)); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode insn alu: ADD/SUB/MUL/DIV/REM r0, r1, r2 + * @param kind the data kind, such as I32, I64, F32 and F64 + * @param Type the data type, such as int32, int64, float32, and float64 + * @param type the abbreviation of data type, such as i32, i64, f32, and f64 + * @param op the opcode of alu + */ +#define ALU_R_R_R(kind, Type, type, op) \ + do { \ + Type data1, data2; \ + int32 reg_no_dst; \ + bool _ret = false; \ + \ + CHECK_EQKIND(r0, r1); \ + CHECK_EQKIND(r0, r2); \ + memset(&data1, 0, sizeof(Type)); \ + memset(&data2, 0, sizeof(Type)); \ + \ + reg_no_dst = jit_reg_no(r0); \ + if (jit_reg_is_const(r1)) \ + data1 = jit_cc_get_const_##kind(cc, r1); \ + if (jit_reg_is_const(r2)) \ + data2 = jit_cc_get_const_##kind(cc, r2); \ + \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = \ + alu_imm_imm_to_r_##type(a, op, reg_no_dst, data1, data2); \ + else \ + _ret = alu_imm_r_to_r_##type(a, op, reg_no_dst, data1, \ + jit_reg_no(r2)); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = alu_r_imm_to_r_##type(a, op, reg_no_dst, jit_reg_no(r1), \ + data2); \ + else \ + _ret = alu_r_r_to_r_##type(a, op, reg_no_dst, jit_reg_no(r1), \ + jit_reg_no(r2)); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode alu insn, ADD/SUB/MUL/DIV/REM r0, r1, r2 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param op the opcode of alu operations + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the first src operand info + * @param r2 src jit register that contains the second src operand info + * + * @return true if success, false if failed + */ +static bool +lower_alu(JitCompContext *cc, x86::Assembler &a, ALU_OP op, JitReg r0, + JitReg r1, JitReg r2) +{ + switch (jit_reg_kind(r0)) { + case JIT_REG_KIND_I32: + ALU_R_R_R(I32, int32, i32, op); + break; + case JIT_REG_KIND_I64: + ALU_R_R_R(I64, int64, i64, op); + break; + case JIT_REG_KIND_F32: + ALU_R_R_R(F32, float32, f32, op); + break; + case JIT_REG_KIND_F64: + ALU_R_R_R(F64, float64, f64, op); + break; + default: + LOG_VERBOSE("Invalid reg type of alu: %d\n", jit_reg_kind(r0)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode insn bit: AND/OR/XOR r0, r1, r2 + * @param kind the data kind, such as I32, I64 + * @param Type the data type, such as int32, int64 + * @param type the abbreviation of data type, such as i32, i64 + * @param op the opcode of bit operation + */ +#define BIT_R_R_R(kind, Type, type, op) \ + do { \ + Type data1, data2; \ + int32 reg_no_dst; \ + bool _ret = false; \ + \ + CHECK_EQKIND(r0, r1); \ + CHECK_EQKIND(r0, r2); \ + memset(&data1, 0, sizeof(Type)); \ + memset(&data2, 0, sizeof(Type)); \ + \ + reg_no_dst = jit_reg_no(r0); \ + if (jit_reg_is_const(r1)) \ + data1 = jit_cc_get_const_##kind(cc, r1); \ + if (jit_reg_is_const(r2)) \ + data2 = jit_cc_get_const_##kind(cc, r2); \ + \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = \ + bit_imm_imm_to_r_##type(a, op, reg_no_dst, data1, data2); \ + else \ + _ret = bit_imm_r_to_r_##type(a, op, reg_no_dst, data1, \ + jit_reg_no(r2)); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = bit_r_imm_to_r_##type(a, op, reg_no_dst, jit_reg_no(r1), \ + data2); \ + else \ + _ret = bit_r_r_to_r_##type(a, op, reg_no_dst, jit_reg_no(r1), \ + jit_reg_no(r2)); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode bit insn, AND/OR/XOR r0, r1, r2 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param op the opcode of bit operations + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the first src operand info + * @param r2 src jit register that contains the second src operand info + * + * @return true if success, false if failed + */ +static bool +lower_bit(JitCompContext *cc, x86::Assembler &a, BIT_OP op, JitReg r0, + JitReg r1, JitReg r2) +{ + switch (jit_reg_kind(r0)) { + case JIT_REG_KIND_I32: + BIT_R_R_R(I32, int32, i32, op); + break; + case JIT_REG_KIND_I64: + BIT_R_R_R(I64, int64, i64, op); + break; + default: + LOG_VERBOSE("Invalid reg type of bit: %d\n", jit_reg_kind(r0)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode insn shift: SHL/SHRS/SHRU r0, r1, r2 + * @param kind the data kind, such as I32, I64 + * @param Type the data type, such as int32, int64 + * @param type the abbreviation of data type, such as i32, i64 + * @param op the opcode of shift operation + */ +#define SHIFT_R_R_R(kind, Type, type, op) \ + do { \ + Type data1, data2; \ + int32 reg_no_dst; \ + bool _ret = false; \ + \ + CHECK_EQKIND(r0, r1); \ + CHECK_KIND(r2, JIT_REG_KIND_I32); \ + memset(&data1, 0, sizeof(Type)); \ + memset(&data2, 0, sizeof(Type)); \ + \ + reg_no_dst = jit_reg_no(r0); \ + if (jit_reg_is_const(r1)) \ + data1 = jit_cc_get_const_##kind(cc, r1); \ + if (jit_reg_is_const(r2)) \ + data2 = jit_cc_get_const_##kind(cc, r2); \ + \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = shift_imm_imm_to_r_##type(a, op, reg_no_dst, data1, \ + data2); \ + else \ + _ret = shift_imm_r_to_r_##type(a, op, reg_no_dst, data1, \ + jit_reg_no(r2)); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = shift_r_imm_to_r_##type(a, op, reg_no_dst, jit_reg_no(r1), \ + data2); \ + else \ + _ret = shift_r_r_to_r_##type(a, op, reg_no_dst, jit_reg_no(r1), \ + jit_reg_no(r2)); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode shift insn, SHL/SHRS/SHRU r0, r1, r2 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param op the opcode of shift operations + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the first src operand info + * @param r2 src jit register that contains the second src operand info + * + * @return true if success, false if failed + */ +static bool +lower_shift(JitCompContext *cc, x86::Assembler &a, SHIFT_OP op, JitReg r0, + JitReg r1, JitReg r2) +{ + switch (jit_reg_kind(r0)) { + case JIT_REG_KIND_I32: + SHIFT_R_R_R(I32, int32, i32, op); + break; + case JIT_REG_KIND_I64: + SHIFT_R_R_R(I64, int64, i64, op); + break; + default: + LOG_VERBOSE("Invalid reg type of shift: %d\n", jit_reg_kind(r0)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode insn cmp: CMP r0, r1, r2 + * @param kind the data kind, such as I32, I64, F32 and F64 + * @param Type the data type, such as int32, int64, float32, and float64 + * @param type the abbreviation of data type, such as i32, i64, f32, and f64 + */ +#define CMP_R_R_R(kind, Type, type) \ + do { \ + Type data1, data2; \ + int32 reg_no_dst; \ + bool _ret = false; \ + \ + CHECK_KIND(r0, JIT_REG_KIND_I32); \ + CHECK_KIND(r1, JIT_REG_KIND_##kind); \ + CHECK_EQKIND(r1, r2); \ + memset(&data1, 0, sizeof(Type)); \ + memset(&data2, 0, sizeof(Type)); \ + \ + reg_no_dst = jit_reg_no(r0); \ + if (jit_reg_is_const(r1)) \ + data1 = jit_cc_get_const_##kind(cc, r1); \ + if (jit_reg_is_const(r2)) \ + data2 = jit_cc_get_const_##kind(cc, r2); \ + \ + if (jit_reg_is_const(r1)) { \ + if (jit_reg_is_const(r2)) \ + _ret = cmp_imm_imm_to_r_##type(a, reg_no_dst, data1, data2); \ + else \ + _ret = cmp_imm_r_to_r_##type(a, reg_no_dst, data1, \ + jit_reg_no(r2)); \ + } \ + else if (jit_reg_is_const(r2)) \ + _ret = \ + cmp_r_imm_to_r_##type(a, reg_no_dst, jit_reg_no(r1), data2); \ + else \ + _ret = cmp_r_r_to_r_##type(a, reg_no_dst, jit_reg_no(r1), \ + jit_reg_no(r2)); \ + if (!_ret) \ + GOTO_FAIL; \ + } while (0) + +/** + * Encode cmp insn, CMP r0, r1, r2 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the first src operand info + * @param r2 src jit register that contains the second src operand info + * + * @return true if success, false if failed + */ +static bool +lower_cmp(JitCompContext *cc, x86::Assembler &a, JitReg r0, JitReg r1, + JitReg r2) +{ + switch (jit_reg_kind(r1)) { + case JIT_REG_KIND_I32: + CMP_R_R_R(I32, int32, i32); + break; + case JIT_REG_KIND_I64: + CMP_R_R_R(I64, int64, i64); + break; + case JIT_REG_KIND_F32: + CMP_R_R_R(F32, float32, f32); + break; + case JIT_REG_KIND_F64: + CMP_R_R_R(F64, float64, f64); + break; + default: + LOG_VERBOSE("Invalid reg type of cmp: %d\n", jit_reg_kind(r1)); + GOTO_FAIL; + } + + return true; +fail: + return false; +} + +/** + * Encode select insn, SELECT r0, r1, r2, r3 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the first src operand info + * @param r2 src jit register that contains the second src operand info + * + * @return true if success, false if failed + */ +static bool +lower_select(JitCompContext *cc, x86::Assembler &a, COND_OP op, JitReg r0, + JitReg r1, JitReg r2, JitReg r3) +{ +#if 0 + char stream_mov1[128]; + char stream_mov2[128]; + char *stream1 = stream_mov1; + char *stream2 = stream_mov2; + + CHECK_NCONST(r0); + CHECK_NCONST(r1); + CHECK_KIND(r1, JIT_REG_KIND_I32); + + if (r0 == r3 && r0 != r2) { + JitReg r_tmp; + + /* Exchange r2, r3*/ + r_tmp = r2; + r2 = r3; + r3 = r_tmp; + op = not_cond(op); + } + + if (!lower_mov(cc, &stream1, r0, r2)) + GOTO_FAIL; + if (!lower_mov(cc, &stream2, r0, r3)) + GOTO_FAIL; + + if (r0 != r2) { + memcpy(stream, stream_mov1, (int32)(stream1 - stream_mov1)); + stream += (int32)(stream1 - stream_mov1); + } + + if (r3 && r0 != r3) { + stream = cmp_r_and_jmp_relative(stream, jit_reg_no(r1), op, + (int32)(stream2 - stream_mov2)); + memcpy(stream, stream_mov2, (int32)(stream2 - stream_mov2)); + stream += (int32)(stream2 - stream_mov2); + } + + return true; +fail: + return false; +#endif + return false; +} + +/** + * Encode branch insn, BEQ/BNE/../BLTU r0, r1, r2 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param r0 dst jit register that contains the dst operand info + * @param r1 src jit register that contains the first src operand info + * @param r2 src jit register that contains the second src operand info + * @param is_last_insn if current insn is the last insn of current block + * + * @return true if success, false if failed + */ +static bool +lower_branch(JitCompContext *cc, x86::Assembler &a, int32 label_src, COND_OP op, + JitReg r0, JitReg r1, JitReg r2, bool is_last_insn) +{ +#if 0 + int32 reg_no, label_dst; + + CHECK_NCONST(r0); + CHECK_KIND(r0, JIT_REG_KIND_I32); + CHECK_KIND(r1, JIT_REG_KIND_L4); + + label_dst = jit_reg_no(r1); + if (label_dst < (int32)jit_cc_label_num(cc) - 1 && is_last_insn + && label_is_neighboring(cc, label_src, label_dst)) { + JitReg r_tmp; + + r_tmp = r1; + r1 = r2; + r2 = r_tmp; + op = not_cond(op); + } + + reg_no = jit_reg_no(r0); + if (!cmp_r_and_jmp_label(cc, &stream, stream_offset, label_src, op, reg_no, + r1, r2, is_last_insn)) + GOTO_FAIL; + + return true; +fail: + return false; +#endif + return false; +} + +/** + * Encode lookupswitch with key of immediate data + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param label_src the index of src label + * @param key the entry key + * @param opnd the lookup switch operand + * @param is_last_insn if current insn is the last insn of current block + * + * @return true if success, false if failed + */ +static bool +lookupswitch_imm(JitCompContext *cc, x86::Assembler &a, int32 label_src, + int32 key, const JitOpndLookupSwitch *opnd, bool is_last_insn) +{ +#if 0 + uint32 i; + int32 label_dst; + + for (i = 0; i < opnd->match_pairs_num; i++) + if (key == opnd->match_pairs[i].value) { + label_dst = jit_reg_no(opnd->match_pairs[i].target); + if (!(is_last_insn + && label_is_neighboring(cc, label_src, label_dst))) + JMP_TO_LABEL(stream_offset, label_dst, label_src); + + return true; + } + + if (opnd->default_target) { + label_dst = jit_reg_no(opnd->default_target); + if (!(is_last_insn && label_is_neighboring(cc, label_src, label_dst))) + JMP_TO_LABEL(stream_offset, label_dst, label_src); + } + + return true; +fail: + return false; +#endif + return false; +} + +/** + * Encode detecting lookupswitch entry register and jumping to matched label + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param label_src the index of src label + * @param reg_no the no of entry register + * @param opnd the lookup switch operand + * @param is_last_insn if current insn is the last insn of current block + * + * @return true if success, false if failed + */ +static bool +lookupswitch_r(JitCompContext *cc, x86::Assembler &a, int32 label_src, + int32 reg_no, const JitOpndLookupSwitch *opnd, bool is_last_insn) +{ +#if 0 + JmpInfo *node; + Imm_Opnd imm; + uint32 i, stream_offset_new; + int32 label_dst; + + for (i = 0; i < opnd->match_pairs_num; i++) { + imm_from_sz_v_s(imm, SZ32, opnd->match_pairs[i].value, true); + alu_r_imm(cmp, regs_I32[reg_no], imm); + + label_dst = jit_reg_no(opnd->match_pairs[i].target); + imm_from_sz_v_s(imm, SZ32, label_dst, true); + + node = jit_malloc(sizeof(JmpInfo)); + if (!node) + GOTO_FAIL; + + node->type = JMP_DST_LABEL; + node->label_src = label_src; + node->dst_info.label_dst = label_dst; + node->offset = (int32)(stream + 2 - (*stream_ptr - stream_offset)); + bh_list_insert(jmp_info_list, node); + + je(imm); + } + + if (opnd->default_target) { + label_dst = jit_reg_no(opnd->default_target); + stream_offset_new = stream_offset + stream - *stream_ptr; + if (!(is_last_insn && label_is_neighboring(cc, label_src, label_dst))) + JMP_TO_LABEL(stream_offset_new, label_dst, label_src); + } + + return true; +fail: + return false; +#endif + return false; +} + +/** + * Encode lookupswitch insn, LOOKUPSWITCH opnd + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param label_src the index of src label + * @param opnd the lookup switch operand + * @param is_last_insn if current insn is the last insn of current block + * + * @return true if success, false if failed + */ +static bool +lower_lookupswitch(JitCompContext *cc, x86::Assembler &a, int32 label_src, + const JitOpndLookupSwitch *opnd, bool is_last_insn) +{ +#if 0 + JitReg r0 = opnd->value; + int32 key, reg_no; + + CHECK_KIND(r0, JIT_REG_KIND_I32); + CHECK_KIND(opnd->default_target, JIT_REG_KIND_L4); + + if (jit_reg_is_const(r0)) { + key = jit_cc_get_const_I32(cc, r0); + if (!lookupswitch_imm(cc, &stream, stream_offset, label_src, key, opnd, + is_last_insn)) + GOTO_FAIL; + } + else { + reg_no = jit_reg_no(r0); + if (!lookupswitch_r(cc, &stream, stream_offset, label_src, reg_no, opnd, + is_last_insn)) + GOTO_FAIL; + } + + return true; +fail: + return false; +#endif + return false; +} + +/** + * Encode callnative insn, CALLNATIVE r0, r1, ... + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param label_src the index of src label + * @param insn current insn info + * @param global_offset_base the base for calculating global offset + * + * @return true if success, false if failed + */ +static bool +lower_callnative(JitCompContext *cc, x86::Assembler &a, int32 label_src, + JitInsn *insn, unsigned global_offset_base) +{ + return false; +} + +/** + * Encode callbc insn, CALLBC r0, r1, r2 + * + * @param cc the compiler context + * @param a the assembler to emit the code + * @param label_src the index of src label + * @param insn current insn info + * @param global_offset_base the base for calculating global offset + * + * @return true if success, false if failed + */ +static bool +lower_callbc(JitCompContext *cc, x86::Assembler &a, int32 label_src, + JitInsn *insn, unsigned global_offset_base) +{ + return false; +} + +static bool +lower_returnbc(JitCompContext *cc, x86::Assembler &a, int32 label_src, + JitInsn *insn) +{ + return false; +} + +bool +jit_codegen_gen_native(JitCompContext *cc) +{ + jit_set_last_error(cc, "jit_codegen_gen_native failed"); + return false; +} + +bool +jit_codegen_lower(JitCompContext *cc) +{ + return true; +} + +void +jit_codegen_free_native(JitCompContext *cc) +{} + +#if WASM_ENABLE_FAST_JIT_DUMP != 0 +static void +dump_native(char *data, uint32 length) +{ + /* Initialize decoder context */ + ZydisDecoder decoder; + ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, + ZYDIS_STACK_WIDTH_64); + + /* Initialize formatter */ + ZydisFormatter formatter; + ZydisFormatterInit(&formatter, ZYDIS_FORMATTER_STYLE_INTEL); + + /* Loop over the instructions in our buffer */ + ZyanU64 runtime_address = (ZyanU64)(uintptr_t)data; + ZyanUSize offset = 0; + ZydisDecodedInstruction instruction; + ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT_VISIBLE]; + + while (ZYAN_SUCCESS(ZydisDecoderDecodeFull( + &decoder, data + offset, length - offset, &instruction, operands, + ZYDIS_MAX_OPERAND_COUNT_VISIBLE, ZYDIS_DFLAG_VISIBLE_OPERANDS_ONLY))) { + /* Print current instruction pointer */ + printf("%012" PRIX64 " ", runtime_address); + + /* Format & print the binary instruction structure to + human readable format */ + char buffer[256]; + ZydisFormatterFormatInstruction(&formatter, &instruction, operands, + instruction.operand_count_visible, + buffer, sizeof(buffer), + runtime_address); + puts(buffer); + + offset += instruction.length; + runtime_address += instruction.length; + } +} +#endif + +void +jit_codegen_dump_native(void *begin_addr, void *end_addr) +{ +#if WASM_ENABLE_FAST_JIT_DUMP != 0 + dump_native((char *)begin_addr, (char *)end_addr - (char *)begin_addr); +#endif +} + +bool +jit_codegen_init() +{ + const JitHardRegInfo *hreg_info = jit_codegen_get_hreg_info(); + char *code_buf, *stream; + uint32 code_size; + + Environment env(Arch::kX64); + CodeHolder code; + code.init(env); + x86::Assembler a(&code); + + /* push callee-save registers */ + a.push(x86::rbp); + a.push(x86::rbx); + a.push(x86::r12); + a.push(x86::r13); + a.push(x86::r14); + a.push(x86::r15); + /* push exec_env */ + a.push(x86::rdi); + /* push info */ + a.push(x86::rsi); + /* exec_env_reg = exec_env */ + a.mov(regs_i64[hreg_info->exec_env_hreg_index], x86::rdi); + /* fp_reg = info.->frame */ + a.mov(x86::ebp, x86::ptr(x86::rsi, 0)); + /* jmp target */ + a.jmp(x86::rdx); + + code_buf = (char *)code.sectionById(0)->buffer().data(); + code_size = code.sectionById(0)->buffer().size(); + stream = (char *)jit_code_cache_alloc(code_size); + if (!stream) + return false; + + bh_memcpy_s(stream, code_size, code_buf, code_size); + code_block_switch_to_jitted_from_interp = stream; + +#if 0 + dump_native(stream, code_size); +#endif + + a.setOffset(0); + /* pop info */ + a.pop(x86::rsi); + /* pop exec_env */ + a.pop(x86::rdi); + /* pop callee-save registers */ + a.pop(x86::r15); + a.pop(x86::r14); + a.pop(x86::r13); + a.pop(x86::r12); + a.pop(x86::rbx); + a.pop(x86::rbp); + + code_buf = (char *)code.sectionById(0)->buffer().data(); + code_size = code.sectionById(0)->buffer().size(); + stream = (char *)jit_code_cache_alloc(code_size); + if (!stream) { + jit_code_cache_free(code_block_switch_to_jitted_from_interp); + return false; + } + + bh_memcpy_s(stream, code_size, code_buf, code_size); + code_block_return_to_interp_from_jitted = stream; + return true; +} + +void +jit_codegen_destroy() +{ + jit_code_cache_free(code_block_switch_to_jitted_from_interp); + jit_code_cache_free(code_block_return_to_interp_from_jitted); +} + +/* clang-format off */ +static const uint8 hreg_info_I32[3][7] = { + /* ebp, eax, ebx, ecx, edx, edi, esi */ + { 1, 0, 0, 0, 0, 0, 1 }, /* fixed, esi is freely used */ + { 0, 1, 0, 1, 1, 0, 0 }, /* caller_saved_native */ + { 0, 1, 0, 1, 1, 1, 0 } /* caller_saved_jitted */ +}; + +static const uint8 hreg_info_I64[3][16] = { + /* rbp, rax, rbx, rcx, rdx, rdi, rsi, rsp, + r8, r9, r10, r11, r12, r13, r14, r15 */ + { 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 1 }, /* fixed, rsi is freely used */ + { 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 0, 0, 0, 0 }, /* caller_saved_native */ + { 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 0 }, /* caller_saved_jitted */ +}; + +static uint8 hreg_info_F32[3][16] = { + { 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1 }, /* fixed, rsi is freely used */ + { 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1 }, /* caller_saved_native */ + { 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1 }, /* caller_saved_jitted */ +}; + +static uint8 hreg_info_F64[3][16] = { + { 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0 }, /* fixed, rsi is freely used */ + { 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1 }, /* caller_saved_native */ + { 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1 }, /* caller_saved_jitted */ +}; + +static const JitHardRegInfo hreg_info = { + { + { 0, NULL, NULL, NULL }, /* VOID */ + + { sizeof(hreg_info_I32[0]), /* I32 */ + hreg_info_I32[0], + hreg_info_I32[1], + hreg_info_I32[2] }, + + { sizeof(hreg_info_I64[0]), /* I64 */ + hreg_info_I64[0], + hreg_info_I64[1], + hreg_info_I64[2] }, + + { sizeof(hreg_info_F32[0]), /* F32 */ + hreg_info_F32[0], + hreg_info_F32[1], + hreg_info_F32[2] }, + + { sizeof(hreg_info_F64[0]), /* F64 */ + hreg_info_F64[0], + hreg_info_F64[1], + hreg_info_F64[2] }, + + { 0, NULL, NULL, NULL }, /* V8 */ + { 0, NULL, NULL, NULL }, /* V16 */ + { 0, NULL, NULL, NULL } /* V32 */ + }, + /* frame pointer hreg index: rbp */ + 0, + /* exec_env hreg index: r15 */ + 15, + /* cmp hreg index: esi */ + 6 +}; +/* clang-format on */ + +const JitHardRegInfo * +jit_codegen_get_hreg_info() +{ + return &hreg_info; +} diff --git a/core/iwasm/fast-jit/fe/jit_emit_control.c b/core/iwasm/fast-jit/fe/jit_emit_control.c index e1a28db4b..f2953944e 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_control.c +++ b/core/iwasm/fast-jit/fe/jit_emit_control.c @@ -27,13 +27,14 @@ } \ } while (0) -#define BUILD_COND_BR(value_if, block_then, block_else) \ - do { \ - if (!GEN_INSN(BNE, value_if, jit_basic_block_label(block_then), \ - jit_basic_block_label(block_else))) { \ - jit_set_last_error(cc, "generate bne insn failed"); \ - goto fail; \ - } \ +#define BUILD_COND_BR(value_if, block_then, block_else) \ + do { \ + if (!GEN_INSN(CMP, cc->cmp_reg, value_if, NEW_CONST(cc, 0)) \ + || !GEN_INSN(BNE, cc->cmp_reg, jit_basic_block_label(block_then), \ + jit_basic_block_label(block_else))) { \ + jit_set_last_error(cc, "generate bne insn failed"); \ + goto fail; \ + } \ } while (0) #define SET_BUILDER_POS(basic_block) \ @@ -73,7 +74,7 @@ load_block_params(JitCompContext *cc, JitBlock *block) { JitFrame *jit_frame = cc->jit_frame; uint32 offset, i; - JitReg value; + JitReg value = 0; /* Clear jit frame's locals and stacks */ clear_values(jit_frame); @@ -181,8 +182,9 @@ push_jit_block_to_stack_and_pass_params(JitCompContext *cc, JitBlock *block, BUILD_BR(basic_block); } else { /* IF block with condition br insn */ - if (!(insn = GEN_INSN(BNE, cond, jit_basic_block_label(basic_block), - 0))) { + if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0)) + || !(insn = GEN_INSN(BNE, cc->cmp_reg, + jit_basic_block_label(basic_block), 0))) { jit_set_last_error(cc, "generate cond br failed"); goto fail; } @@ -318,7 +320,7 @@ handle_func_return(JitCompContext *cc, JitBlock *block) /* fp_reg = prev_frame */ GEN_INSN(MOV, cc->fp_reg, prev_frame); /* return 0 */ - GEN_INSN(RETURN, NEW_CONST(I32, 0)); + GEN_INSN(RETURNBC, NEW_CONST(I32, 0)); } static bool @@ -836,7 +838,9 @@ jit_compile_op_br_if(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip) clear_values(jit_frame); CREATE_BASIC_BLOCK(if_basic_block); - if (!GEN_INSN(BNE, cond, jit_basic_block_label(if_basic_block), 0)) { + if (!GEN_INSN(CMP, cc->cmp_reg, cond, NEW_CONST(I32, 0)) + || !GEN_INSN(BNE, cc->cmp_reg, jit_basic_block_label(if_basic_block), + 0)) { jit_set_last_error(cc, "generate bne insn failed"); goto fail; } diff --git a/core/iwasm/fast-jit/fe/jit_emit_exception.c b/core/iwasm/fast-jit/fe/jit_emit_exception.c index dda45ad7a..a9cba5823 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_exception.c +++ b/core/iwasm/fast-jit/fe/jit_emit_exception.c @@ -7,8 +7,72 @@ #include "../jit_frontend.h" bool -jit_emit_exception(JitCompContext *cc, int32 exception_id, bool is_cond_br, +jit_emit_exception(JitCompContext *cc, int32 exception_id, uint8 jit_opcode, JitReg cond_br_if, JitBasicBlock *cond_br_else_block) { - return false; + JitInsn *insn = NULL; + JitIncomingInsn *incoming_insn; + JitReg else_label; + + bh_assert(exception_id < EXCE_NUM); + + if (jit_opcode >= JIT_OP_BNE && jit_opcode <= JIT_OP_BLEU) { + bh_assert(cond_br_if == cc->cmp_reg); + else_label = + cond_br_else_block ? jit_basic_block_label(cond_br_else_block) : 0; + switch (jit_opcode) { + case JIT_OP_BEQ: + insn = GEN_INSN(BEQ, cond_br_if, 0, else_label); + break; + case JIT_OP_BNE: + insn = GEN_INSN(BNE, cond_br_if, 0, else_label); + break; + case JIT_OP_BGTS: + insn = GEN_INSN(BGTS, cond_br_if, 0, else_label); + break; + case JIT_OP_BGES: + insn = GEN_INSN(BGES, cond_br_if, 0, else_label); + break; + case JIT_OP_BLTS: + insn = GEN_INSN(BLTS, cond_br_if, 0, else_label); + break; + case JIT_OP_BLES: + insn = GEN_INSN(BLES, cond_br_if, 0, else_label); + break; + case JIT_OP_BGTU: + insn = GEN_INSN(BGTU, cond_br_if, 0, else_label); + break; + case JIT_OP_BGEU: + insn = GEN_INSN(BGEU, cond_br_if, 0, else_label); + break; + case JIT_OP_BLTU: + insn = GEN_INSN(BLTU, cond_br_if, 0, else_label); + break; + case JIT_OP_BLEU: + insn = GEN_INSN(BLEU, cond_br_if, 0, else_label); + break; + } + if (!insn) { + jit_set_last_error(cc, "generate cond br insn failed"); + return false; + } + } + else if (jit_opcode == JIT_OP_JMP) { + insn = GEN_INSN(JMP, 0); + if (!insn) { + jit_set_last_error(cc, "generate jmp insn failed"); + return false; + } + } + + incoming_insn = jit_calloc(sizeof(JitIncomingInsn)); + if (!incoming_insn) { + jit_set_last_error(cc, "allocate memory failed"); + return false; + } + + incoming_insn->insn = insn; + incoming_insn->next = cc->incoming_insns_for_exec_bbs[exception_id]; + cc->incoming_insns_for_exec_bbs[exception_id] = incoming_insn; + return true; } diff --git a/core/iwasm/fast-jit/fe/jit_emit_exception.h b/core/iwasm/fast-jit/fe/jit_emit_exception.h index a5739b523..7aa393b78 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_exception.h +++ b/core/iwasm/fast-jit/fe/jit_emit_exception.h @@ -13,7 +13,7 @@ extern "C" { #endif bool -jit_emit_exception(JitCompContext *cc, int32 exception_id, bool is_cond_br, +jit_emit_exception(JitCompContext *cc, int32 exception_id, uint8 jit_opcode, JitReg cond_br_if, JitBasicBlock *cond_br_else_block); #ifdef __cplusplus diff --git a/core/iwasm/fast-jit/fe/jit_emit_variable.c b/core/iwasm/fast-jit/fe/jit_emit_variable.c index 7d3763763..20c17a334 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_variable.c +++ b/core/iwasm/fast-jit/fe/jit_emit_variable.c @@ -31,7 +31,7 @@ jit_compile_op_get_local(JitCompContext *cc, uint32 local_idx) uint16 *local_offsets = wasm_func->local_offsets; uint16 local_offset; uint8 local_type; - JitReg value; + JitReg value = 0; CHECK_LOCAL(local_idx); @@ -119,7 +119,7 @@ jit_compile_op_tee_local(JitCompContext *cc, uint32 local_idx) uint16 *local_offsets = wasm_func->local_offsets; uint16 local_offset; uint8 local_type; - JitReg value; + JitReg value = 0; CHECK_LOCAL(local_idx); diff --git a/core/iwasm/fast-jit/iwasm_fast_jit.cmake b/core/iwasm/fast-jit/iwasm_fast_jit.cmake index 42e398726..b7a6b2711 100644 --- a/core/iwasm/fast-jit/iwasm_fast_jit.cmake +++ b/core/iwasm/fast-jit/iwasm_fast_jit.cmake @@ -3,16 +3,84 @@ set (IWASM_FAST_JIT_DIR ${CMAKE_CURRENT_LIST_DIR}) -add_definitions (-DWASM_ENABLE_FAST_JIT=1) +add_definitions(-DWASM_ENABLE_FAST_JIT=1) +if (WAMR_BUILD_FAST_JIT_DUMP EQUAL 1) + add_definitions(-DWASM_ENABLE_FAST_JIT_DUMP=1) +endif () include_directories (${IWASM_FAST_JIT_DIR}) +if (WAMR_BUILD_TARGET STREQUAL "X86_64" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + include(FetchContent) + FetchContent_Declare( + asmjit + GIT_REPOSITORY https://github.com/asmjit/asmjit.git + ) + FetchContent_GetProperties(asmjit) + if (NOT asmjit_POPULATED) + message ("-- Fetching asmjit ..") + FetchContent_Populate(asmjit) + add_definitions(-DASMJIT_STATIC) + add_definitions(-DASMJIT_NO_DEPRECATED) + add_definitions(-DASMJIT_NO_BUILDER) + add_definitions(-DASMJIT_NO_COMPILER) + add_definitions(-DASMJIT_NO_JIT) + add_definitions(-DASMJIT_NO_LOGGING) + add_definitions(-DASMJIT_NO_TEXT) + add_definitions(-DASMJIT_NO_VALIDATION) + add_definitions(-DASMJIT_NO_INTROSPECTION) + add_definitions(-DASMJIT_NO_INTRINSICS) + add_definitions(-DASMJIT_NO_AARCH64) + add_definitions(-DASMJIT_NO_AARCH32) + include_directories("${asmjit_SOURCE_DIR}/src") + add_subdirectory(${asmjit_SOURCE_DIR} ${asmjit_BINARY_DIR} EXCLUDE_FROM_ALL) + file (GLOB_RECURSE cpp_source_asmjit + ${asmjit_SOURCE_DIR}/src/asmjit/core/*.cpp + ${asmjit_SOURCE_DIR}/src/asmjit/x86/*.cpp + ) + endif () + if (WAMR_BUILD_FAST_JIT_DUMP EQUAL 1) + FetchContent_Declare( + zycore + GIT_REPOSITORY https://github.com/zyantific/zycore-c.git + ) + FetchContent_GetProperties(zycore) + if (NOT zycore_POPULATED) + message ("-- Fetching zycore ..") + FetchContent_Populate(zycore) + option(ZYDIS_BUILD_TOOLS "" OFF) + option(ZYDIS_BUILD_EXAMPLES "" OFF) + include_directories("${zycore_SOURCE_DIR}/include") + include_directories("${zycore_BINARY_DIR}") + add_subdirectory(${zycore_SOURCE_DIR} ${zycore_BINARY_DIR} EXCLUDE_FROM_ALL) + file (GLOB_RECURSE c_source_zycore ${zycore_SOURCE_DIR}/src/*.c) + endif () + FetchContent_Declare( + zydis + GIT_REPOSITORY https://github.com/zyantific/zydis.git + ) + FetchContent_GetProperties(zydis) + if (NOT zydis_POPULATED) + message ("-- Fetching zydis ..") + FetchContent_Populate(zydis) + option(ZYDIS_BUILD_TOOLS "" OFF) + option(ZYDIS_BUILD_EXAMPLES "" OFF) + include_directories("${zydis_BINARY_DIR}") + include_directories("${zydis_SOURCE_DIR}/include") + include_directories("${zydis_SOURCE_DIR}/src") + add_subdirectory(${zydis_SOURCE_DIR} ${zydis_BINARY_DIR} EXCLUDE_FROM_ALL) + file (GLOB_RECURSE c_source_zydis ${zydis_SOURCE_DIR}/src/*.c) + endif () + endif () +endif () + file (GLOB c_source_jit ${IWASM_FAST_JIT_DIR}/*.c ${IWASM_FAST_JIT_DIR}/fe/*.c) if (WAMR_BUILD_TARGET STREQUAL "X86_64" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") - file (GLOB_RECURSE c_source_jit_cg ${IWASM_FAST_JIT_DIR}/cg/x86-64/*.c) + file (GLOB_RECURSE cpp_source_jit_cg ${IWASM_FAST_JIT_DIR}/cg/x86-64/*.cpp) else () message (FATAL_ERROR "Fast JIT codegen for target ${WAMR_BUILD_TARGET} isn't implemented") endif () -set (IWASM_FAST_JIT_SOURCE ${c_source_jit} ${c_source_jit_cg}) +set (IWASM_FAST_JIT_SOURCE ${c_source_jit} ${cpp_source_jit_cg} + ${cpp_source_asmjit} ${c_source_zycore} ${c_source_zydis}) diff --git a/core/iwasm/fast-jit/jit_codecache.c b/core/iwasm/fast-jit/jit_codecache.c index 80525041e..7d30572df 100644 --- a/core/iwasm/fast-jit/jit_codecache.c +++ b/core/iwasm/fast-jit/jit_codecache.c @@ -41,7 +41,7 @@ jit_code_cache_destroy() } void * -jit_code_cache_malloc(uint32 size) +jit_code_cache_alloc(uint32 size) { return mem_allocator_malloc(code_cache_pool_allocator, size); } diff --git a/core/iwasm/fast-jit/jit_codegen.h b/core/iwasm/fast-jit/jit_codegen.h index f7209e5c9..341fb61aa 100644 --- a/core/iwasm/fast-jit/jit_codegen.h +++ b/core/iwasm/fast-jit/jit_codegen.h @@ -7,7 +7,7 @@ #define _JIT_CODEGEN_H_ #include "bh_platform.h" -#include "jit_ir.h" +#include "jit_compiler.h" #ifdef __cplusplus extern "C" { @@ -64,14 +64,8 @@ jit_codegen_lower(JitCompContext *cc); void jit_codegen_dump_native(void *begin_addr, void *end_addr); -/** - * Call jitted code - * - * @param exec_env the current exec_env - */ -bool -jit_codegen_call_func_jitted(void *exec_env, void *frame, void *func_inst, - void *target); +int +jit_codegen_interp_jitted_glue(void *self, JitInterpSwitchInfo *info, void *pc); #ifdef __cplusplus } diff --git a/core/iwasm/fast-jit/jit_compiler.c b/core/iwasm/fast-jit/jit_compiler.c index db0f1952a..77e50ca92 100644 --- a/core/iwasm/fast-jit/jit_compiler.c +++ b/core/iwasm/fast-jit/jit_compiler.c @@ -9,13 +9,6 @@ #include "jit_codecache.h" #include "../interpreter/wasm.h" -typedef struct JitGlobals { - /* Compiler pass sequence. The last element must be 0. */ - const uint8 *passes; - /* Code cache size. */ - uint32 code_cache_size; -} JitGlobals; - typedef struct JitCompilerPass { /* Name of the pass. */ const char *name; @@ -30,7 +23,6 @@ static JitCompilerPass compiler_passes[] = { REG_PASS(dump), REG_PASS(update_cfg), REG_PASS(frontend), - REG_PASS(lower_fe), REG_PASS(lower_cg), REG_PASS(regalloc), REG_PASS(codegen), @@ -41,20 +33,18 @@ static JitCompilerPass compiler_passes[] = { /* Number of compiler passes. */ #define COMPILER_PASS_NUM (sizeof(compiler_passes) / sizeof(compiler_passes[0])) -#define WASM_ENABLE_FAST_JIT_DUMP 1 - #if WASM_ENABLE_FAST_JIT_DUMP == 0 static const uint8 compiler_passes_without_dump[] = { - 3, 4, 5, 6, 7, 8, 0 + 3, 4, 5, 6, 7, 0 }; #else static const uint8 compiler_passes_with_dump[] = { - 3, 2, 1, 4, 1, 5, 1, 6, 1, 7, 1, 8, 0 + 3, 2, 1, 4, 1, 5, 1, 6, 1, 7, 0 }; #endif /* The exported global data of JIT compiler. */ -JitGlobals jit_globals = { +static JitGlobals jit_globals = { #if WASM_ENABLE_FAST_JIT_DUMP == 0 .passes = compiler_passes_without_dump, #else @@ -109,6 +99,12 @@ jit_compiler_destroy() jit_code_cache_destroy(); } +const JitGlobals * +jit_compiler_get_jit_globals() +{ + return &jit_globals; +} + const char * jit_compiler_get_pass_name(unsigned i) { @@ -119,6 +115,7 @@ bool jit_compiler_compile(WASMModule *module, uint32 func_idx) { JitCompContext *cc; + char *last_error; bool ret = true; /* Initialize compilation context. */ @@ -138,8 +135,10 @@ jit_compiler_compile(WASMModule *module, uint32 func_idx) || (!module->possible_memory_grow); /* Apply compiler passes. */ - if (!apply_compiler_passes(cc)) { - os_printf("fast jit compilation failed: %s\n", jit_get_last_error(cc)); + if (!apply_compiler_passes(cc) || jit_get_last_error(cc)) { + last_error = jit_get_last_error(cc); + os_printf("fast jit compilation failed: %s\n", + last_error ? last_error : "unknown error"); ret = false; } @@ -153,6 +152,7 @@ bool jit_compiler_compile_all(WASMModule *module) { JitCompContext *cc; + char *last_error; bool ret = false; uint32 i; @@ -174,9 +174,10 @@ jit_compiler_compile_all(WASMModule *module) || (!module->possible_memory_grow); /* Apply compiler passes. */ - if (!apply_compiler_passes(cc)) { + if (!apply_compiler_passes(cc) || jit_get_last_error(cc)) { + last_error = jit_get_last_error(cc); os_printf("fast jit compilation failed: %s\n", - jit_get_last_error(cc)); + last_error ? last_error : "unknown error"); ret = false; break; } @@ -188,9 +189,8 @@ jit_compiler_compile_all(WASMModule *module) return ret; } -bool -jit_interp_switch_to_jitted(void *exec_env, void *frame, - WASMFunctionInstance *func_inst, void *target) +int +jit_interp_switch_to_jitted(void *exec_env, JitInterpSwitchInfo *info, void *pc) { - return jit_codegen_call_func_jitted(exec_env, func_inst, frame, target); + return jit_codegen_interp_jitted_glue(exec_env, info, pc); } diff --git a/core/iwasm/fast-jit/jit_compiler.h b/core/iwasm/fast-jit/jit_compiler.h index d9e2e16d1..571f16ba7 100644 --- a/core/iwasm/fast-jit/jit_compiler.h +++ b/core/iwasm/fast-jit/jit_compiler.h @@ -14,12 +14,31 @@ extern "C" { #endif +typedef struct JitGlobals { + /* Compiler pass sequence, the last element must be 0 */ + const uint8 *passes; + /* Code cache size. */ + uint32 code_cache_size; +} JitGlobals; + +/** + * Information exchanged between JITed code and interpreter. + */ +typedef struct JitInterpSwitchInfo { + /* Points to the frame that is passed to JITed code and the frame + that is returned from JITed code. */ + void *frame; +} JitInterpSwitchInfo; + bool jit_compiler_init(); void jit_compiler_destroy(); +const JitGlobals * +jit_compiler_get_jit_globals(); + const char * jit_compiler_get_pass_name(unsigned i); @@ -29,9 +48,8 @@ jit_compiler_compile(WASMModule *module, uint32 func_idx); bool jit_compiler_compile_all(WASMModule *module); -bool -jit_interp_switch_to_jitted(void *exec_env, void *frame, - WASMFunctionInstance *func_inst, void *target); +int +jit_interp_switch_to_jitted(void *self, JitInterpSwitchInfo *info, void *pc); /* * Pass declarations: @@ -55,11 +73,13 @@ jit_pass_update_cfg(JitCompContext *cc); bool jit_pass_frontend(JitCompContext *cc); +#if 0 /** * Convert MIR to LIR. */ bool jit_pass_lower_fe(JitCompContext *cc); +#endif /** * Lower unsupported operations into supported ones. diff --git a/core/iwasm/fast-jit/jit_dump.c b/core/iwasm/fast-jit/jit_dump.c index 884ec8c25..b1765def4 100644 --- a/core/iwasm/fast-jit/jit_dump.c +++ b/core/iwasm/fast-jit/jit_dump.c @@ -89,6 +89,7 @@ jit_dump_insn_VReg(JitCompContext *cc, JitInsn *insn, unsigned opnd_num) os_printf("\n"); } +#if 0 static void jit_dump_insn_TableSwitch(JitCompContext *cc, JitInsn *insn, unsigned opnd_num) { @@ -107,6 +108,7 @@ jit_dump_insn_TableSwitch(JitCompContext *cc, JitInsn *insn, unsigned opnd_num) os_printf("\n"); } } +#endif static void jit_dump_insn_LookupSwitch(JitCompContext *cc, JitInsn *insn, unsigned opnd_num) @@ -307,9 +309,14 @@ jit_dump_cc(JitCompContext *cc) bool jit_pass_dump(JitCompContext *cc) { - os_printf("JIT.COMPILER.DUMP: PASS_NO=%d PREV_PASS=%s\n\n", cc->cur_pass_no, - (cc->cur_pass_no > 0 ? jit_compiler_get_pass_name(cc->cur_pass_no) - : "NULL")); + const JitGlobals *jit_globals = jit_compiler_get_jit_globals(); + const uint8 *passes = jit_globals->passes; + uint8 pass_no = cc->cur_pass_no; + const char *pass_name = + pass_no > 0 ? jit_compiler_get_pass_name(passes[pass_no - 1]) : "NULL"; + + os_printf("JIT.COMPILER.DUMP: PASS_NO=%d PREV_PASS=%s\n\n", pass_no, + pass_name); jit_dump_cc(cc); os_printf("\n"); return true; diff --git a/core/iwasm/fast-jit/jit_frontend.c b/core/iwasm/fast-jit/jit_frontend.c index 0a1379bed..3ad976d1f 100644 --- a/core/iwasm/fast-jit/jit_frontend.c +++ b/core/iwasm/fast-jit/jit_frontend.c @@ -20,6 +20,28 @@ #include "../interpreter/wasm_opcode.h" #include "../common/wasm_exec_env.h" +/* clang-format off */ +static const char *jit_exception_msgs[] = { + "unreachable", /* EXCE_UNREACHABLE */ + "allocate memory failed", /* EXCE_OUT_OF_MEMORY */ + "out of bounds memory access", /* EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS */ + "integer overflow", /* EXCE_INTEGER_OVERFLOW */ + "integer divide by zero", /* EXCE_INTEGER_DIVIDE_BY_ZERO */ + "invalid conversion to integer", /* EXCE_INVALID_CONVERSION_TO_INTEGER */ + "indirect call type mismatch", /* EXCE_INVALID_FUNCTION_TYPE_INDEX */ + "invalid function index", /* EXCE_INVALID_FUNCTION_INDEX */ + "undefined element", /* EXCE_UNDEFINED_ELEMENT */ + "uninitialized element", /* EXCE_UNINITIALIZED_ELEMENT */ + "failed to call unlinked import function", /* EXCE_CALL_UNLINKED_IMPORT_FUNC */ + "native stack overflow", /* EXCE_NATIVE_STACK_OVERFLOW */ + "unaligned atomic", /* EXCE_UNALIGNED_ATOMIC */ + "wasm auxiliary stack overflow", /* EXCE_AUX_STACK_OVERFLOW */ + "wasm auxiliary stack underflow", /* EXCE_AUX_STACK_UNDERFLOW */ + "out of bounds table access", /* EXCE_OUT_OF_BOUNDS_TABLE_ACCESS */ + "wasm operand stack overflow", /* EXCE_OPERAND_STACK_OVERFLOW */ +}; +/* clang-format on */ + JitReg gen_load_i32(JitFrame *frame, unsigned n) { @@ -124,29 +146,41 @@ gen_commit_sp_ip(JitFrame *frame) JitReg sp; if (frame->sp != frame->committed_sp) { -#if UINTPTR_MAX == UINT32_MAX - GEN_INSN_NORM(I32, sp, ADD, 0, cc->fp_reg, - NEW_CONST(I32, offset_of_local(frame->sp - frame->lp))); - GEN_INSN(STI32, sp, cc->fp_reg, +#if UINTPTR_MAX == UINT64_MAX + sp = jit_cc_new_reg_I64(cc); + GEN_INSN(ADD, sp, cc->fp_reg, + NEW_CONST(I32, offset_of_local(frame->sp - frame->lp))); + GEN_INSN(STI64, sp, cc->fp_reg, NEW_CONST(I32, offsetof(WASMInterpFrame, sp))); #else - GEN_INSN_NORM(I64, sp, ADD, 0, cc->fp_reg, - NEW_CONST(I32, offset_of_local(frame->sp - frame->lp))); - GEN_INSN(STI64, sp, cc->fp_reg, + sp = jit_cc_new_reg_I32(cc); + GEN_INSN(ADD, sp, cc->fp_reg, + NEW_CONST(I32, offset_of_local(frame->sp - frame->lp))); + GEN_INSN(STI32, sp, cc->fp_reg, NEW_CONST(I32, offsetof(WASMInterpFrame, sp))); #endif frame->committed_sp = frame->sp; } -#if 0 - if (frame->ip != frame->committed_ip) { - GEN_INSN (STI32, - NEW_REL (BCIP, NONE, offset_of_addr (frame, frame->ip), frame->ip), - cc->fp_reg, - NEW_CONST (I32, offsetof (WASMInterpFrame, ip))); - frame->committed_ip = frame->ip; - } + if (frame->ip != frame->committed_ip) { +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(STI64, NEW_CONST(I64, (uint64)(uintptr_t)frame->ip), + cc->fp_reg, NEW_CONST(I32, offsetof(WASMInterpFrame, ip))); +#else + GEN_INSN(STI32, NEW_CONST(I32, (uint32)(uintptr_t)frame->ip), + cc->fp_reg, NEW_CONST(I32, offsetof(WASMInterpFrame, ip))); #endif + frame->committed_ip = frame->ip; + } +} + +static void +jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id) +{ + if (id < EXCE_NUM) + wasm_set_exception(module_inst, jit_exception_msgs[id]); + else + wasm_set_exception(module_inst, "unknown exception"); } static bool @@ -154,7 +188,9 @@ form_and_translate_func(JitCompContext *cc) { JitBasicBlock *func_entry_basic_block; JitReg func_entry_label; - JitInsn *jmp_insn; + JitInsn *insn; + JitIncomingInsn *incoming_insn, *incoming_insn_next; + uint32 i; if (!(func_entry_basic_block = jit_frontend_translate_func(cc))) return false; @@ -165,11 +201,58 @@ form_and_translate_func(JitCompContext *cc) func_entry_label = jit_basic_block_label(func_entry_basic_block); /* Create a JMP instruction jumping to the func entry. */ - if (!(jmp_insn = jit_cc_new_insn(cc, JMP, func_entry_label))) + if (!(insn = jit_cc_new_insn(cc, JMP, func_entry_label))) return false; /* Insert the instruction into the cc entry block. */ - jit_basic_block_append_insn(jit_cc_entry_basic_block(cc), jmp_insn); + jit_basic_block_append_insn(jit_cc_entry_basic_block(cc), insn); + + /* Patch INSNs jumping to exception basic blocks. */ + for (i = 0; i < EXCE_NUM; i++) { + incoming_insn = cc->incoming_insns_for_exec_bbs[i]; + if (incoming_insn) { + if (!(cc->exce_basic_blocks[i] = jit_cc_new_basic_block(cc, 0))) { + jit_set_last_error(cc, "create basic block failed"); + return false; + } + while (incoming_insn) { + incoming_insn_next = incoming_insn->next; + insn = incoming_insn->insn; + if (insn->opcode == JIT_OP_JMP) { + *(jit_insn_opnd(insn, 0)) = + jit_basic_block_label(cc->exce_basic_blocks[i]); + } + else if (insn->opcode >= JIT_OP_BNE + && insn->opcode <= JIT_OP_BLEU) { + *(jit_insn_opnd(insn, 1)) = + jit_basic_block_label(cc->exce_basic_blocks[i]); + } + incoming_insn = incoming_insn_next; + } + cc->cur_basic_block = cc->exce_basic_blocks[i]; +#if UINTPTR_MAX == UINT64_MAX + insn = GEN_INSN( + CALLNATIVE, 0, + NEW_CONST(I64, (uint64)(uintptr_t)jit_set_exception_with_id), + 1); +#else + insn = GEN_INSN( + CALLNATIVE, 0, + NEW_CONST(I32, (uint32)(uintptr_t)jit_set_exception_with_id), + 1); +#endif + if (insn) { + *(jit_insn_opndv(insn, 2)) = NEW_CONST(I32, i); + } + GEN_INSN(RETURNBC, NEW_CONST(I32, i)); + + *(jit_annl_begin_bcip(cc, + jit_basic_block_label(cc->cur_basic_block))) = + *(jit_annl_end_bcip( + cc, jit_basic_block_label(cc->cur_basic_block))) = + cc->cur_wasm_module->load_addr; + } + } *(jit_annl_begin_bcip(cc, cc->entry_label)) = *(jit_annl_end_bcip(cc, cc->entry_label)) = @@ -199,11 +282,13 @@ jit_pass_frontend(JitCompContext *cc) return true; } +#if 0 bool jit_pass_lower_fe(JitCompContext *cc) { return true; } +#endif static JitFrame * init_func_translation(JitCompContext *cc) @@ -239,6 +324,7 @@ init_func_translation(JitCompContext *cc) jit_frame->max_locals = max_locals; jit_frame->max_stacks = max_stacks; jit_frame->sp = jit_frame->lp + max_locals; + jit_frame->ip = cur_wasm_func->code; cc->jit_frame = jit_frame; cc->cur_basic_block = jit_cc_entry_basic_block(cc); @@ -266,9 +352,14 @@ init_func_translation(JitCompContext *cc) NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.s.top_boundary))); /* frame_boundary = top + frame_size + outs_size */ GEN_INSN(ADD, frame_boundary, top, NEW_CONST(I32, frame_size + outs_size)); - GEN_INSN(CHECK_SOE, NEW_CONST(I32, 0), frame_boundary, top_boundary); + /* if frame_boundary > top_boundary, throw stack overflow exception */ + GEN_INSN(CMP, cc->cmp_reg, frame_boundary, top_boundary); + if (!jit_emit_exception(cc, EXCE_OPERAND_STACK_OVERFLOW, JIT_OP_BGTU, + cc->cmp_reg, 0)) { + return NULL; + } - /* Add first and then sub to reduce one used register. */ + /* Add first and then sub to reduce one used register */ /* new_top = frame_boundary - outs_size = top + frame_size */ GEN_INSN(SUB, new_top, frame_boundary, NEW_CONST(I32, outs_size)); /* exec_env->wasm_stack.s.top = new_top */ @@ -283,6 +374,7 @@ init_func_translation(JitCompContext *cc) /* frame->prev_frame = fp_reg */ GEN_INSN(STI64, cc->fp_reg, top, NEW_CONST(I32, offsetof(WASMInterpFrame, prev_frame))); + /* TODO: do we need to set frame->function? */ /* GEN_INSN(STI64, func_inst, top, NEW_CONST(I32, offsetof(WASMInterpFrame, function))); @@ -293,6 +385,52 @@ init_func_translation(JitCompContext *cc) /* fp_reg = top */ GEN_INSN(MOV, cc->fp_reg, top); #else + top = jit_cc_new_reg_I32(cc); + top_boundary = jit_cc_new_reg_I32(cc); + new_top = jit_cc_new_reg_I32(cc); + frame_boundary = jit_cc_new_reg_I32(cc); + frame_sp = jit_cc_new_reg_I32(cc); + + /* top = exec_env->wasm_stack.s.top */ + GEN_INSN(LDI32, top, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.s.top))); + /* top_boundary = exec_env->wasm_stack.s.top_boundary */ + GEN_INSN(LDI32, top_boundary, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.s.top_boundary))); + /* frame_boundary = top + frame_size + outs_size */ + GEN_INSN(ADD, frame_boundary, top, NEW_CONST(I32, frame_size + outs_size)); + /* if frame_boundary > top_boundary, throw stack overflow exception */ + GEN_INSN(CMP, cc->cmp_reg, frame_boundary, top_boundary); + if (!jit_emit_exception(cc, EXCE_OPERAND_STACK_OVERFLOW, JIT_OP_BGTU, + cc->cmp_reg, 0)) { + return NULL; + } + + /* Add first and then sub to reduce one used register */ + /* new_top = frame_boundary - outs_size = top + frame_size */ + GEN_INSN(SUB, new_top, frame_boundary, NEW_CONST(I32, outs_size)); + /* exec_env->wasm_stack.s.top = new_top */ + GEN_INSN(STI32, new_top, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.s.top))); + /* frame_sp = frame->lp + local_size */ + GEN_INSN(ADD, frame_sp, top, + NEW_CONST(I32, offsetof(WASMInterpFrame, lp) + local_size)); + /* frame->sp = frame_sp */ + GEN_INSN(STI32, frame_sp, top, + NEW_CONST(I32, offsetof(WASMInterpFrame, sp))); + /* frame->prev_frame = fp_reg */ + GEN_INSN(STI32, cc->fp_reg, top, + NEW_CONST(I32, offsetof(WASMInterpFrame, prev_frame))); + /* TODO: do we need to set frame->function? */ + /* + GEN_INSN(STI32, func_inst, top, + NEW_CONST(I32, offsetof(WASMInterpFrame, function))); + */ + /* exec_env->cur_frame = top */ + GEN_INSN(STI32, top, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, cur_frame))); + /* fp_reg = top */ + GEN_INSN(MOV, cc->fp_reg, top); #endif return jit_frame; @@ -1535,6 +1673,12 @@ jit_compile_func(JitCompContext *cc) jit_set_last_error(cc, "unsupported opcode"); return false; } + /* Error may occur when creating registers, basic blocks, insns, + consts and labels, in which the return value may be unchecked, + here we check again */ + if (jit_get_last_error(cc)) { + return false; + } } (void)func_idx; diff --git a/core/iwasm/fast-jit/jit_frontend.h b/core/iwasm/fast-jit/jit_frontend.h index 0f8e9d6f0..24f45f6e6 100644 --- a/core/iwasm/fast-jit/jit_frontend.h +++ b/core/iwasm/fast-jit/jit_frontend.h @@ -308,6 +308,7 @@ static inline void gen_commit_for_exception(JitFrame *frame) { gen_commit_values(frame, frame->lp, frame->lp + frame->max_locals); + gen_commit_sp_ip(frame); } /** diff --git a/core/iwasm/fast-jit/jit_ir.c b/core/iwasm/fast-jit/jit_ir.c index 986e9cb1b..4291ea0c9 100644 --- a/core/iwasm/fast-jit/jit_ir.c +++ b/core/iwasm/fast-jit/jit_ir.c @@ -315,6 +315,9 @@ JitBasicBlock * jit_basic_block_new(JitReg label, int n) { JitBasicBlock *block = jit_insn_new_PHI(label, n); + if (!block) + return NULL; + block->prev = block->next = block; return block; } @@ -380,7 +383,7 @@ jit_basic_block_succs(JitBasicBlock *block) vec._base = jit_insn_opnd(last_insn, 1); break; - case JIT_OP_LOOKUP_SWITCH: + case JIT_OP_LOOKUPSWITCH: { JitOpndLookupSwitch *opnd = jit_insn_opndls(last_insn); vec.num = opnd->match_pairs_num + 1; @@ -412,10 +415,14 @@ jit_cc_init(JitCompContext *cc, unsigned htab_size) || !(exit_block = jit_cc_new_basic_block(cc, 0))) goto fail; - if (!(cc->exception_basic_blocks = + if (!(cc->exce_basic_blocks = jit_calloc(sizeof(JitBasicBlock *) * EXCE_NUM))) goto fail; + if (!(cc->incoming_insns_for_exec_bbs = + jit_calloc(sizeof(JitIncomingInsnList) * EXCE_NUM))) + goto fail; + /* Record the entry and exit labels, whose indexes must be 0 and 1 respectively. */ cc->entry_label = jit_basic_block_label(entry_block); @@ -437,14 +444,14 @@ jit_cc_init(JitCompContext *cc, unsigned htab_size) } /* Create registers for frame pointer, exec_env and cmp. */ -#if UINTPTR_MAX == UINT32_MAX - cc->fp_reg = jit_reg_new(JIT_REG_KIND_I32, cc->hreg_info->fp_hreg_index); - cc->exec_env_reg = - jit_reg_new(JIT_REG_KIND_I32, cc->hreg_info->exec_env_hreg_index); -#else +#if UINTPTR_MAX == UINT64_MAX cc->fp_reg = jit_reg_new(JIT_REG_KIND_I64, cc->hreg_info->fp_hreg_index); cc->exec_env_reg = jit_reg_new(JIT_REG_KIND_I64, cc->hreg_info->exec_env_hreg_index); +#else + cc->fp_reg = jit_reg_new(JIT_REG_KIND_I32, cc->hreg_info->fp_hreg_index); + cc->exec_env_reg = + jit_reg_new(JIT_REG_KIND_I32, cc->hreg_info->exec_env_hreg_index); #endif cc->cmp_reg = jit_reg_new(JIT_REG_KIND_I32, cc->hreg_info->cmp_hreg_index); @@ -466,6 +473,7 @@ jit_cc_destroy(JitCompContext *cc) { unsigned i, end; JitBasicBlock *block; + JitIncomingInsn *incoming_insn, *incoming_insn_next; jit_block_stack_destroy(&cc->block_stack); @@ -476,7 +484,19 @@ jit_cc_destroy(JitCompContext *cc) /* Release the instruction hash table. */ jit_cc_disable_insn_hash(cc); - jit_free(cc->exception_basic_blocks); + jit_free(cc->exce_basic_blocks); + + if (cc->incoming_insns_for_exec_bbs) { + for (i = 0; i < EXCE_NUM; i++) { + incoming_insn = cc->incoming_insns_for_exec_bbs[i]; + while (incoming_insn) { + incoming_insn_next = incoming_insn->next; + jit_free(incoming_insn); + incoming_insn = incoming_insn_next; + } + } + jit_free(cc->incoming_insns_for_exec_bbs); + } /* Release entry and exit blocks. */ jit_basic_block_delete(jit_cc_entry_basic_block(cc)); @@ -619,6 +639,7 @@ _jit_cc_new_const(JitCompContext *cc, int kind, unsigned size, void *val) cc->_const_val._next[kind] = new_next; } else { + jit_set_last_error(cc, "create const register failed"); jit_free(new_value); jit_free(new_next); return 0; @@ -770,8 +791,10 @@ jit_cc_new_label(JitCompContext *cc) #undef ANN_LABEL #undef EMPTY_POSTFIX - if (!successful) + if (!successful) { + jit_set_last_error(cc, "create label register failed"); return 0; + } cc->_ann._label_capacity = capacity; } @@ -790,6 +813,8 @@ jit_cc_new_basic_block(JitCompContext *cc, int n) if (label && (block = jit_basic_block_new(label, n))) /* Void 0 register indicates error in creation. */ *(jit_annl_basic_block(cc, label)) = block; + else + jit_set_last_error(cc, "create basic block failed"); return block; } @@ -801,8 +826,10 @@ jit_cc_resize_basic_block(JitCompContext *cc, JitBasicBlock *block, int n) JitInsn *insn = jit_basic_block_first_insn(block); JitBasicBlock *new_block = jit_basic_block_new(label, n); - if (!new_block) + if (!new_block) { + jit_set_last_error(cc, "resize basic block failed"); return NULL; + } jit_insn_unlink(block); @@ -877,8 +904,10 @@ jit_cc_set_insn_uid(JitCompContext *cc, JitInsn *insn) #undef ANN_INSN #undef EMPTY_POSTFIX - if (!successful) + if (!successful) { + jit_set_last_error(cc, "set insn uid failed"); return NULL; + } cc->_ann._insn_capacity = capacity; } @@ -900,6 +929,7 @@ _jit_cc_set_insn_uid_for_new_insn(JitCompContext *cc, JitInsn *insn) return NULL; } +#if 0 static JitReg normalize_insn(JitCompContext *cc, JitInsn **pinsn) { @@ -1059,6 +1089,7 @@ _gen_insn_norm_1(JitCompContext *cc, JitBasicBlock *block, unsigned kind, return NULL; } +#endif JitReg jit_cc_new_reg(JitCompContext *cc, unsigned kind) @@ -1080,8 +1111,10 @@ jit_cc_new_reg(JitCompContext *cc, unsigned kind) #include "jit_ir.def" #undef ANN_REG - if (!successful) + if (!successful) { + jit_set_last_error(cc, "create register failed"); return 0; + } cc->_ann._reg_capacity[kind] = capacity; } @@ -1093,33 +1126,37 @@ jit_cc_new_reg(JitCompContext *cc, unsigned kind) #undef _JIT_REALLOC_ANN -#define ANN_LABEL(TYPE, NAME) \ - bool jit_annl_enable_##NAME(JitCompContext *cc) \ - { \ - if (cc->_ann._label_##NAME##_enabled) \ - return true; \ - \ - if (cc->_ann._label_capacity > 0 \ - && !(cc->_ann._label_##NAME = \ - jit_calloc(cc->_ann._label_capacity * sizeof(TYPE)))) \ - return false; \ - \ - cc->_ann._label_##NAME##_enabled = 1; \ - return true; \ +#define ANN_LABEL(TYPE, NAME) \ + bool jit_annl_enable_##NAME(JitCompContext *cc) \ + { \ + if (cc->_ann._label_##NAME##_enabled) \ + return true; \ + \ + if (cc->_ann._label_capacity > 0 \ + && !(cc->_ann._label_##NAME = \ + jit_calloc(cc->_ann._label_capacity * sizeof(TYPE)))) { \ + jit_set_last_error(cc, "annl enable " #NAME "failed"); \ + return false; \ + } \ + \ + cc->_ann._label_##NAME##_enabled = 1; \ + return true; \ } -#define ANN_INSN(TYPE, NAME) \ - bool jit_anni_enable_##NAME(JitCompContext *cc) \ - { \ - if (cc->_ann._insn_##NAME##_enabled) \ - return true; \ - \ - if (cc->_ann._insn_capacity > 0 \ - && !(cc->_ann._insn_##NAME = \ - jit_calloc(cc->_ann._insn_capacity * sizeof(TYPE)))) \ - return false; \ - \ - cc->_ann._insn_##NAME##_enabled = 1; \ - return true; \ +#define ANN_INSN(TYPE, NAME) \ + bool jit_anni_enable_##NAME(JitCompContext *cc) \ + { \ + if (cc->_ann._insn_##NAME##_enabled) \ + return true; \ + \ + if (cc->_ann._insn_capacity > 0 \ + && !(cc->_ann._insn_##NAME = \ + jit_calloc(cc->_ann._insn_capacity * sizeof(TYPE)))) { \ + jit_set_last_error(cc, "anni enable " #NAME "failed"); \ + return false; \ + } \ + \ + cc->_ann._insn_##NAME##_enabled = 1; \ + return true; \ } #define ANN_REG(TYPE, NAME) \ bool jit_annr_enable_##NAME(JitCompContext *cc) \ @@ -1133,6 +1170,7 @@ jit_cc_new_reg(JitCompContext *cc, unsigned kind) if (cc->_ann._reg_capacity[k] > 0 \ && !(cc->_ann._reg_##NAME[k] = jit_calloc( \ cc->_ann._reg_capacity[k] * sizeof(TYPE)))) { \ + jit_set_last_error(cc, "annr enable " #NAME "failed"); \ jit_annr_disable_##NAME(cc); \ return false; \ } \ @@ -1179,7 +1217,7 @@ jit_cc_new_reg(JitCompContext *cc, unsigned kind) char * jit_get_last_error(JitCompContext *cc) { - return cc->last_error[0] == '\0' ? "" : cc->last_error; + return cc->last_error[0] == '\0' ? NULL : cc->last_error; } void @@ -1385,8 +1423,8 @@ to_stack_value_type(uint8 type) bool jit_cc_pop_value(JitCompContext *cc, uint8 type, JitReg *p_value) { - JitValue *jit_value; - JitReg value; + JitValue *jit_value = NULL; + JitReg value = 0; if (!cc->block_stack.block_list_end) { jit_set_last_error(cc, "WASM block stack underflow"); diff --git a/core/iwasm/fast-jit/jit_ir.def b/core/iwasm/fast-jit/jit_ir.def index 8afffbbf7..21c3d984a 100644 --- a/core/iwasm/fast-jit/jit_ir.def +++ b/core/iwasm/fast-jit/jit_ir.def @@ -72,78 +72,10 @@ #define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) #endif -/* Comparison instructions */ -INSN(I32_EQZ, Reg, 3, 1) -INSN(I32_EQ, Reg, 3, 1) -INSN(I32_NE, Reg, 3, 1) -INSN(I32_LT_S, Reg, 3, 1) -INSN(I32_LT_U, Reg, 3, 1) -INSN(I32_GT_S, Reg, 3, 1) -INSN(I32_GT_U, Reg, 3, 1) -INSN(I32_LE_S, Reg, 3, 1) -INSN(I32_LE_U, Reg, 3, 1) -INSN(I32_GE_S, Reg, 3, 1) -INSN(I32_GE_U, Reg, 3, 1) - -INSN(I64_EQZ, Reg, 3, 1) -INSN(I64_EQ, Reg, 3, 1) -INSN(I64_NE, Reg, 3, 1) -INSN(I64_LT_S, Reg, 3, 1) -INSN(I64_LT_U, Reg, 3, 1) -INSN(I64_GT_S, Reg, 3, 1) -INSN(I64_GT_U, Reg, 3, 1) -INSN(I64_LE_S, Reg, 3, 1) -INSN(I64_LE_U, Reg, 3, 1) -INSN(I64_GE_S, Reg, 3, 1) -INSN(I64_GE_U, Reg, 3, 1) - -INSN(F32_EQ, Reg, 3, 1) -INSN(F32_NE, Reg, 3, 1) -INSN(F32_LT, Reg, 3, 1) -INSN(F32_GT, Reg, 3, 1) -INSN(F32_LE, Reg, 3, 1) -INSN(F32_GE, Reg, 3, 1) - -INSN(F64_EQ, Reg, 3, 1) -INSN(F64_NE, Reg, 3, 1) -INSN(F64_LT, Reg, 3, 1) -INSN(F64_GT, Reg, 3, 1) -INSN(F64_LE, Reg, 3, 1) -INSN(F64_GE, Reg, 3, 1) - -/* Select instruction */ -INSN(SELECT, Reg, 4, 1) - -/* Control instructions */ -INSN(JMP, Reg, 1, 0) -INSN(BEQ, Reg, 3, 0) -INSN(BNE, Reg, 3, 0) -INSN(BGTS, Reg, 3, 0) -INSN(BGES, Reg, 3, 0) -INSN(BLTS, Reg, 3, 0) -INSN(BLES, Reg, 3, 0) -INSN(BGTU, Reg, 3, 0) -INSN(BGEU, Reg, 3, 0) -INSN(BLTU, Reg, 3, 0) -INSN(BLEU, Reg, 3, 0) -INSN(TABLE_SWITCH, TableSwitch, 1, 0) -INSN(LOOKUP_SWITCH, LookupSwitch, 1, 0) - -/* check zero divisor */ -INSN(CHECK_DIV_ZERO, Reg, 3, 0) -/* check stack overflow */ -INSN(CHECK_SOE, Reg, 3, 0) - -/* Call and return instructions */ -INSN(CALLNATIVE, VReg, 2, 1) -INSN(CALLBC, Reg, 3, 0) -INSN(RETURN, Reg, 1, 0) - /* Move and conversion instructions that transfer values among registers of the same kind (move) or different kinds (convert) */ INSN(MOV, Reg, 2, 1) INSN(PHI, VReg, 1, 1) - INSN(I32TOI8, Reg, 2, 1) INSN(I32TOU8, Reg, 2, 1) INSN(I32TOI16, Reg, 2, 1) @@ -193,7 +125,7 @@ INSN(SELECTLTU, Reg, 4, 1) INSN(SELECTLEU, Reg, 4, 1) /* Memory access instructions: */ -INSN(LDSELF, Reg, 1, 1) +INSN(LDEXECENV, Reg, 1, 1) INSN(LDJITINFO, Reg, 1, 1) INSN(LDI8, Reg, 3, 1) INSN(LDU8, Reg, 3, 1) @@ -218,7 +150,69 @@ INSN(STV64, Reg, 3, 1) INSN(STV128, Reg, 3, 1) INSN(STV256, Reg, 3, 1) +/* Control instructions */ +INSN(JMP, Reg, 1, 0) +INSN(BEQ, Reg, 3, 0) +INSN(BNE, Reg, 3, 0) +INSN(BGTS, Reg, 3, 0) +INSN(BGES, Reg, 3, 0) +INSN(BLTS, Reg, 3, 0) +INSN(BLES, Reg, 3, 0) +INSN(BGTU, Reg, 3, 0) +INSN(BGEU, Reg, 3, 0) +INSN(BLTU, Reg, 3, 0) +INSN(BLEU, Reg, 3, 0) +INSN(LOOKUPSWITCH, LookupSwitch, 1, 0) +/* INSN(TABLESWITCH, TableSwitch, 1, 0) */ + +/* Call and return instructions */ +INSN(CALLNATIVE, VReg, 2, 1) +INSN(CALLBC, Reg, 3, 0) +INSN(RETURNBC, Reg, 1, 0) + #if 0 +/* Comparison instructions, can be translate to SELECTXXX */ +INSN(I32_EQZ, Reg, 3, 1) +INSN(I32_EQ, Reg, 3, 1) +INSN(I32_NE, Reg, 3, 1) +INSN(I32_LT_S, Reg, 3, 1) +INSN(I32_LT_U, Reg, 3, 1) +INSN(I32_GT_S, Reg, 3, 1) +INSN(I32_GT_U, Reg, 3, 1) +INSN(I32_LE_S, Reg, 3, 1) +INSN(I32_LE_U, Reg, 3, 1) +INSN(I32_GE_S, Reg, 3, 1) +INSN(I32_GE_U, Reg, 3, 1) + +INSN(I64_EQZ, Reg, 3, 1) +INSN(I64_EQ, Reg, 3, 1) +INSN(I64_NE, Reg, 3, 1) +INSN(I64_LT_S, Reg, 3, 1) +INSN(I64_LT_U, Reg, 3, 1) +INSN(I64_GT_S, Reg, 3, 1) +INSN(I64_GT_U, Reg, 3, 1) +INSN(I64_LE_S, Reg, 3, 1) +INSN(I64_LE_U, Reg, 3, 1) +INSN(I64_GE_S, Reg, 3, 1) +INSN(I64_GE_U, Reg, 3, 1) + +INSN(F32_EQ, Reg, 3, 1) +INSN(F32_NE, Reg, 3, 1) +INSN(F32_LT, Reg, 3, 1) +INSN(F32_GT, Reg, 3, 1) +INSN(F32_LE, Reg, 3, 1) +INSN(F32_GE, Reg, 3, 1) + +INSN(F64_EQ, Reg, 3, 1) +INSN(F64_NE, Reg, 3, 1) +INSN(F64_LT, Reg, 3, 1) +INSN(F64_GT, Reg, 3, 1) +INSN(F64_LE, Reg, 3, 1) +INSN(F64_GE, Reg, 3, 1) + +/* Select instruction */ +INSN(SELECT, Reg, 4, 1) + /* Memory instructions */ INSN(I32_LOAD, Reg, 2, 1) INSN(I64_LOAD, Reg, 2, 1) diff --git a/core/iwasm/fast-jit/jit_ir.h b/core/iwasm/fast-jit/jit_ir.h index 111c67d22..3baa4769f 100644 --- a/core/iwasm/fast-jit/jit_ir.h +++ b/core/iwasm/fast-jit/jit_ir.h @@ -907,9 +907,15 @@ typedef struct JitFrame { /* Max operand stack slot number. */ uint32 max_stacks; + /* Instruction pointer */ + uint8 *ip; + /* Stack top pointer */ JitValueSlot *sp; + /* Committed instruction pointer */ + uint8 *committed_ip; + /* Committed stack top pointer */ JitValueSlot *committed_sp; @@ -998,7 +1004,8 @@ typedef struct JitCompContext { be 0 and 1 respectively (see JIT_FOREACH_BLOCK). */ JitReg entry_label; JitReg exit_label; - JitBasicBlock *exception_basic_blocks; + JitBasicBlock **exce_basic_blocks; + JitIncomingInsnList *incoming_insns_for_exec_bbs; /* The current basic block to generate instructions */ JitBasicBlock *cur_basic_block; @@ -1229,6 +1236,15 @@ jit_cc_inc_ref(JitCompContext *cc) void jit_cc_delete(JitCompContext *cc); +char * +jit_get_last_error(JitCompContext *cc); + +void +jit_set_last_error(JitCompContext *cc, const char *error); + +void +jit_set_last_error_v(JitCompContext *cc, const char *format, ...); + /** * Create a I32 constant value with relocatable into the compilation * context. A constant value that has relocation info cannot be @@ -1520,6 +1536,8 @@ _gen_insn(JitCompContext *cc, JitInsn *insn) { if (insn) jit_basic_block_append_insn(cc->cur_basic_block, insn); + else + jit_set_last_error(cc, "generate insn failed"); return insn; } @@ -1529,6 +1547,7 @@ _gen_insn(JitCompContext *cc, JitInsn *insn) */ #define GEN_INSN(...) _gen_insn(cc, jit_cc_new_insn(cc, __VA_ARGS__)) +#if 0 /** * Helper function for GEN_INSN_NORM_1 * @@ -1559,6 +1578,7 @@ _gen_insn_norm_1(JitCompContext *cc, JitBasicBlock *block, unsigned kind, */ #define GEN_INSN_NORM(Type, result, ...) \ GEN_INSN_NORM_1(JIT_REG_KIND_##Type, result, __VA_ARGS__) +#endif /** * Create a constant register without relocation info. @@ -1737,15 +1757,6 @@ jit_cc_exit_basic_block(JitCompContext *cc) return *(jit_annl_basic_block(cc, cc->exit_label)); } -char * -jit_get_last_error(JitCompContext *cc); - -void -jit_set_last_error(JitCompContext *cc, const char *error); - -void -jit_set_last_error_v(JitCompContext *cc, const char *format, ...); - void jit_value_stack_push(JitValueStack *stack, JitValue *value); diff --git a/core/iwasm/fast-jit/jit_regalloc.c b/core/iwasm/fast-jit/jit_regalloc.c index c931f673f..3dec1b8d2 100644 --- a/core/iwasm/fast-jit/jit_regalloc.c +++ b/core/iwasm/fast-jit/jit_regalloc.c @@ -312,8 +312,11 @@ rc_init(RegallocContext *rc, JitCompContext *cc) const unsigned vreg_num = jit_cc_reg_num(cc, i); const unsigned hreg_num = jit_cc_hreg_num(cc, i); - if (!(rc->vregs[i] = jit_calloc(sizeof(VirtualReg) * vreg_num)) - || !(rc->hregs[i] = jit_calloc(sizeof(HardReg) * hreg_num))) + if (vreg_num > 0 + && !(rc->vregs[i] = jit_calloc(sizeof(VirtualReg) * vreg_num))) + goto fail; + if (hreg_num > 0 + && !(rc->hregs[i] = jit_calloc(sizeof(HardReg) * hreg_num))) goto fail; /* Hard registers can only be allocated to themselves. */ @@ -407,8 +410,8 @@ reload_vreg(RegallocContext *rc, JitReg vreg, JitInsn *cur_insn) JitInsn *insn = NULL; if (vreg == rc->cc->exec_env_reg) - /* Reload exec_env_reg with LDSELF. */ - insn = jit_cc_new_insn(rc->cc, LDSELF, vr->hreg); + /* Reload exec_env_reg with LDEXECENV. */ + insn = jit_cc_new_insn(rc->cc, LDEXECENV, vr->hreg); else /* Allocate spill slot if not yet and reload from there. */ { diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 0d6745f9a..0d785acd9 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -3767,8 +3767,11 @@ wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env, #if WASM_ENABLE_FAST_JIT == 0 wasm_interp_call_func_bytecode(module_inst, exec_env, function, frame); #else - jit_interp_switch_to_jitted(exec_env, frame, function, + JitInterpSwitchInfo info; + info.frame = frame; + jit_interp_switch_to_jitted(exec_env, &info, function->u.func->jitted_code); + (void)wasm_interp_call_func_bytecode; #endif } diff --git a/product-mini/platforms/linux/CMakeLists.txt b/product-mini/platforms/linux/CMakeLists.txt index b0b347792..a4efc83a1 100644 --- a/product-mini/platforms/linux/CMakeLists.txt +++ b/product-mini/platforms/linux/CMakeLists.txt @@ -123,11 +123,15 @@ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security -W if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mindirect-branch-register") # UNDEFINED BEHAVIOR, refer to https://en.cppreference.com/w/cpp/language/ub if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT WAMR_BUILD_JIT EQUAL 1) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined \ -fno-sanitize=bounds,bounds-strict,alignment \ -fno-sanitize-recover") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined \ + -fno-sanitize=bounds,bounds-strict,alignment \ + -fno-sanitize-recover") endif() else () # UNDEFINED BEHAVIOR, refer to https://en.cppreference.com/w/cpp/language/ub @@ -135,6 +139,9 @@ if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined \ -fno-sanitize=bounds,alignment \ -fno-sanitize-recover") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined \ + -fno-sanitize=bounds,alignment \ + -fno-sanitize-recover") endif() endif () endif ()