/* * Copyright (C) 2021 Intel Corporation. All rights reserved. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ #include "jit_ir.h" #include "jit_codegen.h" #include "jit_frontend.h" /** * Operand kinds of instructions. */ enum { JIT_OPND_KIND_Reg, JIT_OPND_KIND_VReg, JIT_OPND_KIND_TableSwitch, JIT_OPND_KIND_LookupSwitch }; /** * Operand kind of each instruction. */ static const uint8 insn_opnd_kind[] = { #define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) JIT_OPND_KIND_##OPND_KIND, #include "jit_ir.def" #undef INSN }; /** * Operand number of each instruction. */ static const uint8 insn_opnd_num[] = { #define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) OPND_NUM, #include "jit_ir.def" #undef INSN }; /** * Operand number of each instruction. */ static const uint8 insn_opnd_first_use[] = { #define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) FIRST_USE, #include "jit_ir.def" #undef INSN }; #define JIT_INSN_NEW_Reg(OPND_NUM) \ jit_calloc(offsetof(JitInsn, _opnd) + sizeof(JitReg) * (OPND_NUM)) #define JIT_INSN_NEW_VReg(OPND_NUM) \ jit_calloc(offsetof(JitInsn, _opnd._opnd_VReg._reg) \ + sizeof(JitReg) * (OPND_NUM)) JitInsn * _jit_insn_new_Reg_1(JitOpcode opc, JitReg r0) { JitInsn *insn = JIT_INSN_NEW_Reg(1); if (insn) { insn->opcode = opc; *jit_insn_opnd(insn, 0) = r0; } return insn; } JitInsn * _jit_insn_new_Reg_2(JitOpcode opc, JitReg r0, JitReg r1) { JitInsn *insn = JIT_INSN_NEW_Reg(2); if (insn) { insn->opcode = opc; *jit_insn_opnd(insn, 0) = r0; *jit_insn_opnd(insn, 1) = r1; } return insn; } JitInsn * _jit_insn_new_Reg_3(JitOpcode opc, JitReg r0, JitReg r1, JitReg r2) { JitInsn *insn = JIT_INSN_NEW_Reg(3); if (insn) { insn->opcode = opc; *jit_insn_opnd(insn, 0) = r0; *jit_insn_opnd(insn, 1) = r1; *jit_insn_opnd(insn, 2) = r2; } return insn; } JitInsn * _jit_insn_new_Reg_4(JitOpcode opc, JitReg r0, JitReg r1, JitReg r2, JitReg r3) { JitInsn *insn = JIT_INSN_NEW_Reg(4); if (insn) { insn->opcode = opc; *jit_insn_opnd(insn, 0) = r0; *jit_insn_opnd(insn, 1) = r1; *jit_insn_opnd(insn, 2) = r2; *jit_insn_opnd(insn, 3) = r3; } return insn; } JitInsn * _jit_insn_new_Reg_5(JitOpcode opc, JitReg r0, JitReg r1, JitReg r2, JitReg r3, JitReg r4) { JitInsn *insn = JIT_INSN_NEW_Reg(5); if (insn) { insn->opcode = opc; *jit_insn_opnd(insn, 0) = r0; *jit_insn_opnd(insn, 1) = r1; *jit_insn_opnd(insn, 2) = r2; *jit_insn_opnd(insn, 3) = r3; *jit_insn_opnd(insn, 4) = r4; } return insn; } JitInsn * _jit_insn_new_VReg_1(JitOpcode opc, JitReg r0, int n) { JitInsn *insn = JIT_INSN_NEW_VReg(1 + n); if (insn) { insn->opcode = opc; insn->_opnd._opnd_VReg._reg_num = 1 + n; *(jit_insn_opndv(insn, 0)) = r0; } return insn; } JitInsn * _jit_insn_new_VReg_2(JitOpcode opc, JitReg r0, JitReg r1, int n) { JitInsn *insn = JIT_INSN_NEW_VReg(2 + n); if (insn) { insn->opcode = opc; insn->_opnd._opnd_VReg._reg_num = 2 + n; *(jit_insn_opndv(insn, 0)) = r0; *(jit_insn_opndv(insn, 1)) = r1; } return insn; } JitInsn * _jit_insn_new_TableSwitch_1(JitOpcode opc, JitReg value, int32 low, int32 high) { JitOpndTableSwitch *opnd = NULL; JitInsn *insn = jit_calloc(offsetof(JitInsn, _opnd._opnd_TableSwitch.targets) + sizeof(opnd->targets[0]) * (high - low + 1)); if (insn) { insn->opcode = opc; opnd = jit_insn_opndts(insn); opnd->value = value; opnd->low_value = low; opnd->high_value = high; } return insn; } JitInsn * _jit_insn_new_LookupSwitch_1(JitOpcode opc, JitReg value, uint32 num) { JitOpndLookupSwitch *opnd = NULL; JitInsn *insn = jit_calloc(offsetof(JitInsn, _opnd._opnd_LookupSwitch.match_pairs) + sizeof(opnd->match_pairs[0]) * num); if (insn) { insn->opcode = opc; opnd = jit_insn_opndls(insn); opnd->value = value; opnd->match_pairs_num = num; } return insn; } #undef JIT_INSN_NEW_Reg #undef JIT_INSN_NEW_VReg void jit_insn_insert_before(JitInsn *insn1, JitInsn *insn2) { bh_assert(insn1->prev); insn1->prev->next = insn2; insn2->prev = insn1->prev; insn2->next = insn1; insn1->prev = insn2; } void jit_insn_insert_after(JitInsn *insn1, JitInsn *insn2) { bh_assert(insn1->next); insn1->next->prev = insn2; insn2->next = insn1->next; insn2->prev = insn1; insn1->next = insn2; } void jit_insn_unlink(JitInsn *insn) { bh_assert(insn->prev); insn->prev->next = insn->next; bh_assert(insn->next); insn->next->prev = insn->prev; insn->prev = insn->next = NULL; } unsigned jit_insn_hash(JitInsn *insn) { const uint8 opcode = insn->opcode; unsigned hash = opcode, i; /* Currently, only instructions with Reg kind operand require hashing. For others, simply use opcode as the hash value. */ if (insn_opnd_kind[opcode] != JIT_OPND_KIND_Reg || insn_opnd_num[opcode] < 1) return hash; /* All the instructions with hashing support must be in the assignment format, i.e. the first operand is the result (hence being ignored) and all the others are operands. This is also true for CHK instructions, whose first operand is the instruction pointer. */ for (i = 1; i < insn_opnd_num[opcode]; i++) hash = ((hash << 5) - hash) + *(jit_insn_opnd(insn, i)); return hash; } bool jit_insn_equal(JitInsn *insn1, JitInsn *insn2) { const uint8 opcode = insn1->opcode; unsigned i; if (insn2->opcode != opcode) return false; if (insn_opnd_kind[opcode] != JIT_OPND_KIND_Reg || insn_opnd_num[opcode] < 1) return false; for (i = 1; i < insn_opnd_num[opcode]; i++) if (*(jit_insn_opnd(insn1, i)) != *(jit_insn_opnd(insn2, i))) return false; return true; } JitRegVec jit_insn_opnd_regs(JitInsn *insn) { JitRegVec vec; JitOpndTableSwitch *ts; JitOpndLookupSwitch *ls; vec._stride = 1; switch (insn_opnd_kind[insn->opcode]) { case JIT_OPND_KIND_Reg: vec.num = insn_opnd_num[insn->opcode]; vec._base = jit_insn_opnd(insn, 0); break; case JIT_OPND_KIND_VReg: vec.num = jit_insn_opndv_num(insn); vec._base = jit_insn_opndv(insn, 0); break; case JIT_OPND_KIND_TableSwitch: ts = jit_insn_opndts(insn); vec.num = ts->high_value - ts->low_value + 3; vec._base = &ts->value; break; case JIT_OPND_KIND_LookupSwitch: ls = jit_insn_opndls(insn); vec.num = ls->match_pairs_num + 2; vec._base = &ls->value; vec._stride = sizeof(ls->match_pairs[0]) / sizeof(*vec._base); break; } return vec; } unsigned jit_insn_opnd_first_use(JitInsn *insn) { return insn_opnd_first_use[insn->opcode]; } JitBasicBlock * jit_basic_block_new(JitReg label, int n) { JitBasicBlock *block = jit_insn_new_PHI(label, n); if (!block) return NULL; block->prev = block->next = block; return block; } void jit_basic_block_delete(JitBasicBlock *block) { JitInsn *insn, *next_insn, *end; if (!block) return; insn = jit_basic_block_first_insn(block); end = jit_basic_block_end_insn(block); for (; insn != end; insn = next_insn) { next_insn = insn->next; jit_insn_delete(insn); } jit_insn_delete(block); } JitRegVec jit_basic_block_preds(JitBasicBlock *block) { JitRegVec vec; vec.num = jit_insn_opndv_num(block) - 1; vec._base = vec.num > 0 ? jit_insn_opndv(block, 1) : NULL; vec._stride = 1; return vec; } JitRegVec jit_basic_block_succs(JitBasicBlock *block) { JitInsn *last_insn = jit_basic_block_last_insn(block); JitRegVec vec; vec.num = 0; vec._base = NULL; vec._stride = 1; switch (last_insn->opcode) { case JIT_OP_JMP: vec.num = 1; vec._base = jit_insn_opnd(last_insn, 0); break; case JIT_OP_BEQ: case JIT_OP_BNE: case JIT_OP_BGTS: case JIT_OP_BGES: case JIT_OP_BLTS: case JIT_OP_BLES: case JIT_OP_BGTU: case JIT_OP_BGEU: case JIT_OP_BLTU: case JIT_OP_BLEU: vec.num = 2; vec._base = jit_insn_opnd(last_insn, 1); break; case JIT_OP_LOOKUPSWITCH: { JitOpndLookupSwitch *opnd = jit_insn_opndls(last_insn); vec.num = opnd->match_pairs_num + 1; vec._base = &opnd->default_target; vec._stride = sizeof(opnd->match_pairs[0]) / sizeof(*vec._base); break; } default: vec._stride = 0; } return vec; } JitCompContext * jit_cc_init(JitCompContext *cc, unsigned htab_size) { JitBasicBlock *entry_block, *exit_block; unsigned i, num; memset(cc, 0, sizeof(*cc)); cc->_reference_count = 1; jit_annl_enable_basic_block(cc); /* Create entry and exit blocks. They must be the first two blocks respectively. */ if (!(entry_block = jit_cc_new_basic_block(cc, 0)) || !(exit_block = jit_cc_new_basic_block(cc, 0))) goto fail; if (!(cc->exce_basic_blocks = jit_calloc(sizeof(JitBasicBlock *) * EXCE_NUM))) goto fail; if (!(cc->incoming_insns_for_exec_bbs = jit_calloc(sizeof(JitIncomingInsnList) * EXCE_NUM))) goto fail; /* Record the entry and exit labels, whose indexes must be 0 and 1 respectively. */ cc->entry_label = jit_basic_block_label(entry_block); cc->exit_label = jit_basic_block_label(exit_block); bh_assert(jit_reg_no(cc->entry_label) == 0 && jit_reg_no(cc->exit_label) == 1); cc->hreg_info = jit_codegen_get_hreg_info(); bh_assert(cc->hreg_info->info[JIT_REG_KIND_I32].num > 3); /* Initialize virtual registers for hard registers. */ for (i = JIT_REG_KIND_VOID; i < JIT_REG_KIND_L32; i++) { if ((num = cc->hreg_info->info[i].num)) { /* Initialize the capacity to be large enough. */ jit_cc_new_reg(cc, i); bh_assert(cc->_ann._reg_capacity[i] > num); cc->_ann._reg_num[i] = num; } } /* Create registers for frame pointer, exec_env and cmp. */ cc->fp_reg = jit_reg_new(JIT_REG_KIND_PTR, cc->hreg_info->fp_hreg_index); cc->exec_env_reg = jit_reg_new(JIT_REG_KIND_PTR, cc->hreg_info->exec_env_hreg_index); cc->cmp_reg = jit_reg_new(JIT_REG_KIND_I32, cc->hreg_info->cmp_hreg_index); cc->_const_val._hash_table_size = htab_size; if (!(cc->_const_val._hash_table = jit_calloc(htab_size * sizeof(*cc->_const_val._hash_table)))) goto fail; return cc; fail: jit_cc_destroy(cc); return NULL; } void jit_cc_destroy(JitCompContext *cc) { unsigned i, end; JitBasicBlock *block; JitIncomingInsn *incoming_insn, *incoming_insn_next; jit_block_stack_destroy(&cc->block_stack); if (cc->jit_frame) { if (cc->jit_frame->memory_regs) jit_free(cc->jit_frame->memory_regs); if (cc->jit_frame->table_regs) jit_free(cc->jit_frame->table_regs); jit_free(cc->jit_frame); } if (cc->memory_regs) jit_free(cc->memory_regs); if (cc->table_regs) jit_free(cc->table_regs); jit_free(cc->_const_val._hash_table); /* Release the instruction hash table. */ jit_cc_disable_insn_hash(cc); jit_free(cc->exce_basic_blocks); if (cc->incoming_insns_for_exec_bbs) { for (i = 0; i < EXCE_NUM; i++) { incoming_insn = cc->incoming_insns_for_exec_bbs[i]; while (incoming_insn) { incoming_insn_next = incoming_insn->next; jit_free(incoming_insn); incoming_insn = incoming_insn_next; } } jit_free(cc->incoming_insns_for_exec_bbs); } /* Release entry and exit blocks. */ jit_basic_block_delete(jit_cc_entry_basic_block(cc)); jit_basic_block_delete(jit_cc_exit_basic_block(cc)); /* clang-format off */ /* Release blocks and instructions. */ JIT_FOREACH_BLOCK(cc, i, end, block) { jit_basic_block_delete(block); } /* clang-format on */ /* Release constant values. */ for (i = JIT_REG_KIND_VOID; i < JIT_REG_KIND_L32; i++) { jit_free(cc->_const_val._value[i]); jit_free(cc->_const_val._next[i]); } /* Release storage of annotations. */ #define ANN_LABEL(TYPE, NAME) jit_annl_disable_##NAME(cc); #define ANN_INSN(TYPE, NAME) jit_anni_disable_##NAME(cc); #define ANN_REG(TYPE, NAME) jit_annr_disable_##NAME(cc); #include "jit_ir.def" #undef ANN_LABEL #undef ANN_INSN #undef ANN_REG } void jit_cc_delete(JitCompContext *cc) { if (cc && --cc->_reference_count == 0) { jit_cc_destroy(cc); jit_free(cc); } } /* * Reallocate a memory block with the new_size. * TODO: replace this with imported jit_realloc when it's available. */ static void * _jit_realloc(void *ptr, unsigned new_size, unsigned old_size) { void *new_ptr = jit_malloc(new_size); if (new_ptr) { bh_assert(new_size > old_size); if (ptr) { memcpy(new_ptr, ptr, old_size); memset((uint8 *)new_ptr + old_size, 0, new_size - old_size); jit_free(ptr); } else memset(new_ptr, 0, new_size); } return new_ptr; } static unsigned hash_of_const(unsigned kind, unsigned size, void *val) { uint8 *p = (uint8 *)val, *end = p + size; unsigned hash = kind; do hash = ((hash << 5) - hash) + *p++; while (p != end); return hash; } static inline void * address_of_const(JitCompContext *cc, JitReg reg, unsigned size) { int kind = jit_reg_kind(reg); unsigned no = jit_reg_no(reg); unsigned idx = no & ~_JIT_REG_CONST_IDX_FLAG; bh_assert(jit_reg_is_const_idx(reg) && idx < cc->_const_val._num[kind]); return cc->_const_val._value[kind] + size * idx; } static inline JitReg next_of_const(JitCompContext *cc, JitReg reg) { int kind = jit_reg_kind(reg); unsigned no = jit_reg_no(reg); unsigned idx = no & ~_JIT_REG_CONST_IDX_FLAG; bh_assert(jit_reg_is_const_idx(reg) && idx < cc->_const_val._num[kind]); return cc->_const_val._next[kind][idx]; } /** * Put a constant value into the compilation context. * * @param cc compilation context * @param kind register kind * @param size size of the value * @param val pointer to value which must be aligned * * @return a constant register containing the value */ static JitReg _jit_cc_new_const(JitCompContext *cc, int kind, unsigned size, void *val) { unsigned num = cc->_const_val._num[kind], slot; unsigned capacity = cc->_const_val._capacity[kind]; uint8 *new_value; JitReg r, *new_next; bh_assert(num <= capacity); /* Find the existing value first. */ slot = hash_of_const(kind, size, val) % cc->_const_val._hash_table_size; r = cc->_const_val._hash_table[slot]; for (; r; r = next_of_const(cc, r)) if (jit_reg_kind(r) == kind && !memcmp(val, address_of_const(cc, r, size), size)) return r; if (num == capacity) { /* Increase the space of value and next. */ capacity = capacity > 0 ? (capacity + capacity / 2) : 16; new_value = _jit_realloc(cc->_const_val._value[kind], size * capacity, size * num); new_next = _jit_realloc(cc->_const_val._next[kind], sizeof(*new_next) * capacity, sizeof(*new_next) * num); if (new_value && new_next) { cc->_const_val._value[kind] = new_value; cc->_const_val._next[kind] = new_next; } else { jit_set_last_error(cc, "create const register failed"); jit_free(new_value); jit_free(new_next); return 0; } cc->_const_val._capacity[kind] = capacity; } bh_assert(num + 1 < (uint32)_JIT_REG_CONST_IDX_FLAG); r = jit_reg_new(kind, _JIT_REG_CONST_IDX_FLAG | num); memcpy(cc->_const_val._value[kind] + size * num, val, size); cc->_const_val._next[kind][num] = cc->_const_val._hash_table[slot]; cc->_const_val._hash_table[slot] = r; cc->_const_val._num[kind] = num + 1; return r; } static inline int32 get_const_val_in_reg(JitReg reg) { int shift = 8 * sizeof(reg) - _JIT_REG_KIND_SHIFT + 1; return ((int32)(reg << shift)) >> shift; } #define _JIT_CC_NEW_CONST_HELPER(KIND, TYPE, val) \ do { \ JitReg reg = jit_reg_new( \ JIT_REG_KIND_##KIND, \ (_JIT_REG_CONST_VAL_FLAG | ((JitReg)val & ~_JIT_REG_KIND_MASK))); \ \ if ((TYPE)get_const_val_in_reg(reg) == val) \ return reg; \ return _jit_cc_new_const(cc, JIT_REG_KIND_##KIND, sizeof(val), &val); \ } while (0) JitReg jit_cc_new_const_I32_rel(JitCompContext *cc, int32 val, uint32 rel) { uint64 val64 = (uint64)(uint32)val | ((uint64)rel << 32); _JIT_CC_NEW_CONST_HELPER(I32, uint64, val64); } JitReg jit_cc_new_const_I64(JitCompContext *cc, int64 val) { _JIT_CC_NEW_CONST_HELPER(I64, int64, val); } JitReg jit_cc_new_const_F32(JitCompContext *cc, float val) { int32 float_neg_zero = 0x80000000; if (!memcmp(&val, &float_neg_zero, sizeof(float))) /* Create const -0.0f */ return _jit_cc_new_const(cc, JIT_REG_KIND_F32, sizeof(float), &val); _JIT_CC_NEW_CONST_HELPER(F32, float, val); } JitReg jit_cc_new_const_F64(JitCompContext *cc, double val) { int64 double_neg_zero = 0x8000000000000000ll; if (!memcmp(&val, &double_neg_zero, sizeof(double))) /* Create const -0.0d */ return _jit_cc_new_const(cc, JIT_REG_KIND_F64, sizeof(double), &val); _JIT_CC_NEW_CONST_HELPER(F64, double, val); } #undef _JIT_CC_NEW_CONST_HELPER #define _JIT_CC_GET_CONST_HELPER(KIND, TYPE) \ do { \ bh_assert(jit_reg_kind(reg) == JIT_REG_KIND_##KIND); \ bh_assert(jit_reg_is_const(reg)); \ \ return (jit_reg_is_const_val(reg) \ ? (TYPE)get_const_val_in_reg(reg) \ : *(TYPE *)(address_of_const(cc, reg, sizeof(TYPE)))); \ } while (0) static uint64 jit_cc_get_const_I32_helper(JitCompContext *cc, JitReg reg) { _JIT_CC_GET_CONST_HELPER(I32, uint64); } uint32 jit_cc_get_const_I32_rel(JitCompContext *cc, JitReg reg) { return (uint32)(jit_cc_get_const_I32_helper(cc, reg) >> 32); } int32 jit_cc_get_const_I32(JitCompContext *cc, JitReg reg) { return (int32)(jit_cc_get_const_I32_helper(cc, reg)); } int64 jit_cc_get_const_I64(JitCompContext *cc, JitReg reg) { _JIT_CC_GET_CONST_HELPER(I64, int64); } float jit_cc_get_const_F32(JitCompContext *cc, JitReg reg) { _JIT_CC_GET_CONST_HELPER(F32, float); } double jit_cc_get_const_F64(JitCompContext *cc, JitReg reg) { _JIT_CC_GET_CONST_HELPER(F64, double); } #undef _JIT_CC_GET_CONST_HELPER #define _JIT_REALLOC_ANN(TYPE, NAME, ANN, POSTFIX) \ if (successful && cc->_ann._##ANN##_##NAME##_enabled) { \ TYPE *ptr = _jit_realloc(cc->_ann._##ANN##_##NAME POSTFIX, \ sizeof(TYPE) * capacity, sizeof(TYPE) * num); \ if (ptr) \ cc->_ann._##ANN##_##NAME POSTFIX = ptr; \ else \ successful = false; \ } JitReg jit_cc_new_label(JitCompContext *cc) { unsigned num = cc->_ann._label_num; unsigned capacity = cc->_ann._label_capacity; bool successful = true; bh_assert(num <= capacity); if (num == capacity) { capacity = capacity > 0 ? (capacity + capacity / 2) : 16; #define EMPTY_POSTFIX #define ANN_LABEL(TYPE, NAME) _JIT_REALLOC_ANN(TYPE, NAME, label, EMPTY_POSTFIX) #include "jit_ir.def" #undef ANN_LABEL #undef EMPTY_POSTFIX if (!successful) { jit_set_last_error(cc, "create label register failed"); return 0; } cc->_ann._label_capacity = capacity; } cc->_ann._label_num = num + 1; return jit_reg_new(JIT_REG_KIND_L32, num); } JitBasicBlock * jit_cc_new_basic_block(JitCompContext *cc, int n) { JitReg label = jit_cc_new_label(cc); JitBasicBlock *block = NULL; if (label && (block = jit_basic_block_new(label, n))) /* Void 0 register indicates error in creation. */ *(jit_annl_basic_block(cc, label)) = block; else jit_set_last_error(cc, "create basic block failed"); return block; } JitBasicBlock * jit_cc_resize_basic_block(JitCompContext *cc, JitBasicBlock *block, int n) { JitReg label = jit_basic_block_label(block); JitInsn *insn = jit_basic_block_first_insn(block); JitBasicBlock *new_block = jit_basic_block_new(label, n); if (!new_block) { jit_set_last_error(cc, "resize basic block failed"); return NULL; } jit_insn_unlink(block); if (insn != block) jit_insn_insert_before(insn, new_block); bh_assert(*(jit_annl_basic_block(cc, label)) == block); *(jit_annl_basic_block(cc, label)) = new_block; jit_insn_delete(block); return new_block; } bool jit_cc_enable_insn_hash(JitCompContext *cc, unsigned n) { if (jit_anni_is_enabled__hash_link(cc)) return true; if (!jit_anni_enable__hash_link(cc)) return false; /* The table must not exist. */ bh_assert(!cc->_insn_hash_table._table); /* Integer overflow cannot happen because n << 4G (at most several times of 64K in the most extreme case). */ if (!(cc->_insn_hash_table._table = jit_calloc(n * sizeof(*cc->_insn_hash_table._table)))) { jit_anni_disable__hash_link(cc); return false; } cc->_insn_hash_table._size = n; return true; } void jit_cc_disable_insn_hash(JitCompContext *cc) { jit_anni_disable__hash_link(cc); jit_free(cc->_insn_hash_table._table); cc->_insn_hash_table._table = NULL; cc->_insn_hash_table._size = 0; } void jit_cc_reset_insn_hash(JitCompContext *cc) { if (jit_anni_is_enabled__hash_link(cc)) memset(cc->_insn_hash_table._table, 0, cc->_insn_hash_table._size * sizeof(*cc->_insn_hash_table._table)); } JitInsn * jit_cc_set_insn_uid(JitCompContext *cc, JitInsn *insn) { if (insn) { unsigned num = cc->_ann._insn_num; unsigned capacity = cc->_ann._insn_capacity; bool successful = true; bh_assert(num <= capacity); if (num == capacity) { capacity = capacity > 0 ? (capacity + capacity / 2) : 64; #define EMPTY_POSTFIX #define ANN_INSN(TYPE, NAME) _JIT_REALLOC_ANN(TYPE, NAME, insn, EMPTY_POSTFIX) #include "jit_ir.def" #undef ANN_INSN #undef EMPTY_POSTFIX if (!successful) { jit_set_last_error(cc, "set insn uid failed"); return NULL; } cc->_ann._insn_capacity = capacity; } cc->_ann._insn_num = num + 1; insn->uid = num; } return insn; } JitInsn * _jit_cc_set_insn_uid_for_new_insn(JitCompContext *cc, JitInsn *insn) { if (jit_cc_set_insn_uid(cc, insn)) return insn; jit_insn_delete(insn); return NULL; } #if 0 static JitReg normalize_insn(JitCompContext *cc, JitInsn **pinsn) { #if 0 JitInsn *insn = *pinsn; JitReg opnd1; /* TODO: impelement the rest part. See gen_array_addr. */ switch (insn->opcode) { case JIT_OP_I32TOI1: opnd1 = *(jit_insn_opnd (insn, 1)); if (jit_reg_is_const (opnd1)) return jit_cc_new_const_I32 (cc, (int8)jit_cc_get_const_I32 (cc, opnd1)); break; case JIT_OP_I32TOU1: opnd1 = *(jit_insn_opnd (insn, 1)); if (jit_reg_is_const (opnd1)) return jit_cc_new_const_I32 (cc, (uint8)jit_cc_get_const_I32 (cc, opnd1)); break; case JIT_OP_I32TOI2: opnd1 = *(jit_insn_opnd (insn, 1)); if (jit_reg_is_const (opnd1)) return jit_cc_new_const_I32 (cc, (int16)jit_cc_get_const_I32 (cc, opnd1)); break; case JIT_OP_I32TOU2: opnd1 = *(jit_insn_opnd (insn, 1)); if (jit_reg_is_const (opnd1)) return jit_cc_new_const_I32 (cc, (uint16)jit_cc_get_const_I32 (cc, opnd1)); break; case JIT_OP_I32TOI64: break; case JIT_OP_U4TOI64: break; case JIT_OP_I64TOI32: break; case JIT_OP_NEG: break; case JIT_OP_NOT: break; case JIT_OP_ADD: break; case JIT_OP_SUB: break; case JIT_OP_MUL: break; case JIT_OP_DIV: break; case JIT_OP_REM: break; case JIT_OP_SHL: break; case JIT_OP_SHRS: break; case JIT_OP_SHRU: break; case JIT_OP_OR: break; case JIT_OP_XOR: break; case JIT_OP_AND: break; case JIT_OP_CMP: break; } #endif return 0; } JitInsn * _jit_cc_new_insn_norm(JitCompContext *cc, JitReg *result, JitInsn *insn) { if (!insn) /* Creation of insn failed (due to OOM). */ { *result = 0; return NULL; } /* Try to normalize the instruction first. */ if ((*result = normalize_insn(cc, &insn))) /* It's folded to a constant, don't add the insn to cc. */ { bh_assert(jit_reg_is_const(*result)); jit_insn_delete(insn); return NULL; } /* Try to find the existing equivalent instructions. */ if (jit_anni_is_enabled__hash_link(cc)) { unsigned slot = jit_insn_hash(insn) % cc->_insn_hash_table._size; JitInsn **entry = &cc->_insn_hash_table._table[slot]; JitInsn *insn1; for (insn1 = *entry; insn1; insn1 = *(jit_anni__hash_link(cc, insn1))) if (jit_insn_equal(insn, insn1)) /* Found, don't add the insn to cc. */ { JitRegVec vec = jit_insn_opnd_regs(insn1); *result = *(jit_reg_vec_at(&vec, 0)); bh_assert(*result); jit_insn_delete(insn); return insn1; } if ((insn = _jit_cc_set_insn_uid_for_new_insn(cc, insn))) /* Add insn to the linked list of the hash table entry. */ { *(jit_anni__hash_link(cc, insn)) = *entry; *entry = insn; } } else insn = _jit_cc_set_insn_uid_for_new_insn(cc, insn); return insn; } JitInsn * _gen_insn_norm_1(JitCompContext *cc, JitBasicBlock *block, unsigned kind, JitReg *result, JitInsn *insn) { if (!*result && insn) { JitRegVec vec = jit_insn_opnd_regs(insn); JitReg *lhs = jit_reg_vec_at(&vec, 0); if (!*lhs) *lhs = jit_cc_new_reg(cc, kind); *result = *lhs; jit_basic_block_append_insn(block, insn); *(jit_annr_def_insn(cc, *lhs)) = insn; return insn; } return NULL; } #endif JitReg jit_cc_new_reg(JitCompContext *cc, unsigned kind) { unsigned num = jit_cc_reg_num(cc, kind); unsigned capacity = cc->_ann._reg_capacity[kind]; bool successful = true; bh_assert(num <= capacity); if (num == capacity) { capacity = (capacity == 0 /* Initialize the capacity to be larger than hard register number. */ ? cc->hreg_info->info[kind].num + 16 : capacity + capacity / 2); #define ANN_REG(TYPE, NAME) _JIT_REALLOC_ANN(TYPE, NAME, reg, [kind]) #include "jit_ir.def" #undef ANN_REG if (!successful) { jit_set_last_error(cc, "create register failed"); return 0; } cc->_ann._reg_capacity[kind] = capacity; } cc->_ann._reg_num[kind] = num + 1; return jit_reg_new(kind, num); } #undef _JIT_REALLOC_ANN #define ANN_LABEL(TYPE, NAME) \ bool jit_annl_enable_##NAME(JitCompContext *cc) \ { \ if (cc->_ann._label_##NAME##_enabled) \ return true; \ \ if (cc->_ann._label_capacity > 0 \ && !(cc->_ann._label_##NAME = \ jit_calloc(cc->_ann._label_capacity * sizeof(TYPE)))) { \ jit_set_last_error(cc, "annl enable " #NAME "failed"); \ return false; \ } \ \ cc->_ann._label_##NAME##_enabled = 1; \ return true; \ } #define ANN_INSN(TYPE, NAME) \ bool jit_anni_enable_##NAME(JitCompContext *cc) \ { \ if (cc->_ann._insn_##NAME##_enabled) \ return true; \ \ if (cc->_ann._insn_capacity > 0 \ && !(cc->_ann._insn_##NAME = \ jit_calloc(cc->_ann._insn_capacity * sizeof(TYPE)))) { \ jit_set_last_error(cc, "anni enable " #NAME "failed"); \ return false; \ } \ \ cc->_ann._insn_##NAME##_enabled = 1; \ return true; \ } #define ANN_REG(TYPE, NAME) \ bool jit_annr_enable_##NAME(JitCompContext *cc) \ { \ unsigned k; \ \ if (cc->_ann._reg_##NAME##_enabled) \ return true; \ \ for (k = JIT_REG_KIND_VOID; k < JIT_REG_KIND_L32; k++) \ if (cc->_ann._reg_capacity[k] > 0 \ && !(cc->_ann._reg_##NAME[k] = jit_calloc( \ cc->_ann._reg_capacity[k] * sizeof(TYPE)))) { \ jit_set_last_error(cc, "annr enable " #NAME "failed"); \ jit_annr_disable_##NAME(cc); \ return false; \ } \ \ cc->_ann._reg_##NAME##_enabled = 1; \ return true; \ } #include "jit_ir.def" #undef ANN_LABEL #undef ANN_INSN #undef ANN_REG #define ANN_LABEL(TYPE, NAME) \ void jit_annl_disable_##NAME(JitCompContext *cc) \ { \ jit_free(cc->_ann._label_##NAME); \ cc->_ann._label_##NAME = NULL; \ cc->_ann._label_##NAME##_enabled = 0; \ } #define ANN_INSN(TYPE, NAME) \ void jit_anni_disable_##NAME(JitCompContext *cc) \ { \ jit_free(cc->_ann._insn_##NAME); \ cc->_ann._insn_##NAME = NULL; \ cc->_ann._insn_##NAME##_enabled = 0; \ } #define ANN_REG(TYPE, NAME) \ void jit_annr_disable_##NAME(JitCompContext *cc) \ { \ unsigned k; \ \ for (k = JIT_REG_KIND_VOID; k < JIT_REG_KIND_L32; k++) { \ jit_free(cc->_ann._reg_##NAME[k]); \ cc->_ann._reg_##NAME[k] = NULL; \ } \ \ cc->_ann._reg_##NAME##_enabled = 0; \ } #include "jit_ir.def" #undef ANN_LABEL #undef ANN_INSN #undef ANN_REG char * jit_get_last_error(JitCompContext *cc) { return cc->last_error[0] == '\0' ? NULL : cc->last_error; } void jit_set_last_error_v(JitCompContext *cc, const char *format, ...) { va_list args; va_start(args, format); vsnprintf(cc->last_error, sizeof(cc->last_error), format, args); va_end(args); } void jit_set_last_error(JitCompContext *cc, const char *error) { if (error) snprintf(cc->last_error, sizeof(cc->last_error), "Error: %s", error); else cc->last_error[0] = '\0'; } bool jit_cc_update_cfg(JitCompContext *cc) { JitBasicBlock *block; unsigned block_index, end, succ_index, idx; JitReg *target; bool retval = false; if (!jit_annl_enable_pred_num(cc)) return false; /* Update pred_num of all blocks. */ JIT_FOREACH_BLOCK_ENTRY_EXIT(cc, block_index, end, block) { JitRegVec succs = jit_basic_block_succs(block); JIT_REG_VEC_FOREACH(succs, succ_index, target) if (jit_reg_is_kind(L32, *target)) *(jit_annl_pred_num(cc, *target)) += 1; } /* Resize predecessor vectors of body blocks. */ JIT_FOREACH_BLOCK(cc, block_index, end, block) { if (!jit_cc_resize_basic_block( cc, block, *(jit_annl_pred_num(cc, jit_basic_block_label(block))))) goto cleanup_and_return; } /* Fill in predecessor vectors all blocks. */ JIT_FOREACH_BLOCK_REVERSE_ENTRY_EXIT(cc, block_index, block) { JitRegVec succs = jit_basic_block_succs(block), preds; JIT_REG_VEC_FOREACH(succs, succ_index, target) if (jit_reg_is_kind(L32, *target)) { preds = jit_basic_block_preds(*(jit_annl_basic_block(cc, *target))); bh_assert(*(jit_annl_pred_num(cc, *target)) > 0); idx = *(jit_annl_pred_num(cc, *target)) - 1; *(jit_annl_pred_num(cc, *target)) = idx; *(jit_reg_vec_at(&preds, idx)) = jit_basic_block_label(block); } } retval = true; cleanup_and_return: jit_annl_disable_pred_num(cc); return retval; } void jit_value_stack_push(JitValueStack *stack, JitValue *value) { if (!stack->value_list_head) stack->value_list_head = stack->value_list_end = value; else { stack->value_list_end->next = value; value->prev = stack->value_list_end; stack->value_list_end = value; } } JitValue * jit_value_stack_pop(JitValueStack *stack) { JitValue *value = stack->value_list_end; bh_assert(stack->value_list_end); if (stack->value_list_head == stack->value_list_end) stack->value_list_head = stack->value_list_end = NULL; else { stack->value_list_end = stack->value_list_end->prev; stack->value_list_end->next = NULL; value->prev = NULL; } return value; } void jit_value_stack_destroy(JitValueStack *stack) { JitValue *value = stack->value_list_head, *p; while (value) { p = value->next; jit_free(value); value = p; } stack->value_list_head = NULL; stack->value_list_end = NULL; } void jit_block_stack_push(JitBlockStack *stack, JitBlock *block) { if (!stack->block_list_head) stack->block_list_head = stack->block_list_end = block; else { stack->block_list_end->next = block; block->prev = stack->block_list_end; stack->block_list_end = block; } } JitBlock * jit_block_stack_top(JitBlockStack *stack) { return stack->block_list_end; } JitBlock * jit_block_stack_pop(JitBlockStack *stack) { JitBlock *block = stack->block_list_end; bh_assert(stack->block_list_end); if (stack->block_list_head == stack->block_list_end) stack->block_list_head = stack->block_list_end = NULL; else { stack->block_list_end = stack->block_list_end->prev; stack->block_list_end->next = NULL; block->prev = NULL; } return block; } void jit_block_stack_destroy(JitBlockStack *stack) { JitBlock *block = stack->block_list_head, *p; while (block) { p = block->next; jit_value_stack_destroy(&block->value_stack); jit_block_destroy(block); block = p; } stack->block_list_head = NULL; stack->block_list_end = NULL; } bool jit_block_add_incoming_insn(JitBlock *block, JitInsn *insn, uint32 opnd_idx) { JitIncomingInsn *incoming_insn; if (!(incoming_insn = jit_calloc((uint32)sizeof(JitIncomingInsn)))) return false; incoming_insn->insn = insn; incoming_insn->opnd_idx = opnd_idx; incoming_insn->next = block->incoming_insns_for_end_bb; block->incoming_insns_for_end_bb = incoming_insn; return true; } void jit_block_destroy(JitBlock *block) { JitIncomingInsn *incoming_insn, *incoming_insn_next; jit_value_stack_destroy(&block->value_stack); if (block->param_types) jit_free(block->param_types); if (block->result_types) jit_free(block->result_types); incoming_insn = block->incoming_insns_for_end_bb; while (incoming_insn) { incoming_insn_next = incoming_insn->next; jit_free(incoming_insn); incoming_insn = incoming_insn_next; } jit_free(block); } static inline uint8 to_stack_value_type(uint8 type) { #if WASM_ENABLE_REF_TYPES != 0 if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF) return VALUE_TYPE_I32; #endif return type; } bool jit_cc_pop_value(JitCompContext *cc, uint8 type, JitReg *p_value) { JitValue *jit_value = NULL; JitReg value = 0; if (!jit_block_stack_top(&cc->block_stack)) { jit_set_last_error(cc, "WASM block stack underflow"); return false; } if (!jit_block_stack_top(&cc->block_stack)->value_stack.value_list_end) { jit_set_last_error(cc, "WASM data stack underflow"); return false; } jit_value = jit_value_stack_pop( &jit_block_stack_top(&cc->block_stack)->value_stack); bh_assert(jit_value); if (jit_value->type != to_stack_value_type(type)) { jit_set_last_error(cc, "invalid WASM stack data type"); jit_free(jit_value); return false; } switch (jit_value->type) { case VALUE_TYPE_I32: value = pop_i32(cc->jit_frame); break; case VALUE_TYPE_I64: value = pop_i64(cc->jit_frame); break; case VALUE_TYPE_F32: value = pop_f32(cc->jit_frame); break; case VALUE_TYPE_F64: value = pop_f64(cc->jit_frame); break; default: bh_assert(0); break; } bh_assert(cc->jit_frame->sp == jit_value->value); bh_assert(value == jit_value->value->reg); *p_value = value; jit_free(jit_value); return true; } bool jit_cc_push_value(JitCompContext *cc, uint8 type, JitReg value) { JitValue *jit_value; if (!jit_block_stack_top(&cc->block_stack)) { jit_set_last_error(cc, "WASM block stack underflow"); return false; } if (!(jit_value = jit_calloc(sizeof(JitValue)))) { jit_set_last_error(cc, "allocate memory failed"); return false; } bh_assert(value); jit_value->type = to_stack_value_type(type); jit_value->value = cc->jit_frame->sp; jit_value_stack_push(&jit_block_stack_top(&cc->block_stack)->value_stack, jit_value); switch (jit_value->type) { case VALUE_TYPE_I32: push_i32(cc->jit_frame, value); break; case VALUE_TYPE_I64: push_i64(cc->jit_frame, value); break; case VALUE_TYPE_F32: push_f32(cc->jit_frame, value); break; case VALUE_TYPE_F64: push_f64(cc->jit_frame, value); break; } return true; } bool _jit_insn_check_opnd_access_Reg(const JitInsn *insn, unsigned n) { unsigned opcode = insn->opcode; return (insn_opnd_kind[opcode] == JIT_OPND_KIND_Reg && n < insn_opnd_num[opcode]); } bool _jit_insn_check_opnd_access_VReg(const JitInsn *insn, unsigned n) { unsigned opcode = insn->opcode; return (insn_opnd_kind[opcode] == JIT_OPND_KIND_VReg && n < insn->_opnd._opnd_VReg._reg_num); } bool _jit_insn_check_opnd_access_TableSwitch(const JitInsn *insn) { unsigned opcode = insn->opcode; return (insn_opnd_kind[opcode] == JIT_OPND_KIND_TableSwitch); } bool _jit_insn_check_opnd_access_LookupSwitch(const JitInsn *insn) { unsigned opcode = insn->opcode; return (insn_opnd_kind[opcode] == JIT_OPND_KIND_LookupSwitch); }