mirror of
https://github.com/bytecodealliance/wasm-micro-runtime.git
synced 2025-02-06 23:15:16 +00:00
8239dd4aa7
Support to get `wasm_memory_type_t memory_type` from API `wasm_runtime_get_import_type` and `wasm_runtime_get_export_type`, and then get shared flag, initial page cout, maximum page count from the memory_type: ```C bool wasm_memory_type_get_shared(const wasm_memory_type_t memory_type); uint32_t wasm_memory_type_get_init_page_count(const wasm_memory_type_t memory_type); uint32_t wasm_memory_type_get_max_page_count(const wasm_memory_type_t memory_type); ```
1702 lines
61 KiB
C
1702 lines
61 KiB
C
/*
|
|
* Copyright (C) 2019 Intel Corporation. All rights reserved.
|
|
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
*/
|
|
|
|
#include "aot_emit_control.h"
|
|
#include "aot_compiler.h"
|
|
#include "aot_emit_exception.h"
|
|
#if WASM_ENABLE_GC != 0
|
|
#include "aot_emit_gc.h"
|
|
#endif
|
|
#include "../aot/aot_runtime.h"
|
|
#include "../interpreter/wasm_loader.h"
|
|
|
|
#if WASM_ENABLE_DEBUG_AOT != 0
|
|
#include "debug/dwarf_extractor.h"
|
|
#endif
|
|
|
|
static char *block_name_prefix[] = { "block", "loop", "if" };
|
|
static char *block_name_suffix[] = { "begin", "else", "end" };
|
|
|
|
/* clang-format off */
|
|
enum {
|
|
LABEL_BEGIN = 0,
|
|
LABEL_ELSE,
|
|
LABEL_END
|
|
};
|
|
/* clang-format on */
|
|
|
|
static void
|
|
format_block_name(char *name, uint32 name_size, uint32 block_index,
|
|
uint32 label_type, uint32 label_id)
|
|
{
|
|
if (label_type != LABEL_TYPE_FUNCTION)
|
|
snprintf(name, name_size, "%s%d%s%s", block_name_prefix[label_type],
|
|
block_index, "_", block_name_suffix[label_id]);
|
|
else
|
|
snprintf(name, name_size, "%s", "func_end");
|
|
}
|
|
|
|
#define CREATE_BLOCK(new_llvm_block, name) \
|
|
do { \
|
|
if (!(new_llvm_block = LLVMAppendBasicBlockInContext( \
|
|
comp_ctx->context, func_ctx->func, name))) { \
|
|
aot_set_last_error("add LLVM basic block failed."); \
|
|
goto fail; \
|
|
} \
|
|
} while (0)
|
|
|
|
#define CURR_BLOCK() LLVMGetInsertBlock(comp_ctx->builder)
|
|
|
|
#define MOVE_BLOCK_AFTER(llvm_block, llvm_block_after) \
|
|
LLVMMoveBasicBlockAfter(llvm_block, llvm_block_after)
|
|
|
|
#define MOVE_BLOCK_AFTER_CURR(llvm_block) \
|
|
LLVMMoveBasicBlockAfter(llvm_block, CURR_BLOCK())
|
|
|
|
#define MOVE_BLOCK_BEFORE(llvm_block, llvm_block_before) \
|
|
LLVMMoveBasicBlockBefore(llvm_block, llvm_block_before)
|
|
|
|
#define BUILD_BR(llvm_block) \
|
|
do { \
|
|
if (!LLVMBuildBr(comp_ctx->builder, llvm_block)) { \
|
|
aot_set_last_error("llvm build br failed."); \
|
|
goto fail; \
|
|
} \
|
|
} while (0)
|
|
|
|
#define BUILD_COND_BR(value_if, block_then, block_else) \
|
|
do { \
|
|
if (!LLVMBuildCondBr(comp_ctx->builder, value_if, block_then, \
|
|
block_else)) { \
|
|
aot_set_last_error("llvm build cond br failed."); \
|
|
goto fail; \
|
|
} \
|
|
} while (0)
|
|
|
|
#define SET_BUILDER_POS(llvm_block) \
|
|
LLVMPositionBuilderAtEnd(comp_ctx->builder, llvm_block)
|
|
|
|
#define CREATE_RESULT_VALUE_PHIS(block) \
|
|
do { \
|
|
if (block->result_count && !block->result_phis) { \
|
|
uint32 _i; \
|
|
uint64 _size; \
|
|
LLVMBasicBlockRef _block_curr = CURR_BLOCK(); \
|
|
/* Allocate memory */ \
|
|
_size = sizeof(LLVMValueRef) * (uint64)block->result_count; \
|
|
if (_size >= UINT32_MAX \
|
|
|| !(block->result_phis = \
|
|
wasm_runtime_malloc((uint32)_size))) { \
|
|
aot_set_last_error("allocate memory failed."); \
|
|
goto fail; \
|
|
} \
|
|
SET_BUILDER_POS(block->llvm_end_block); \
|
|
for (_i = 0; _i < block->result_count; _i++) { \
|
|
if (!(block->result_phis[_i] = LLVMBuildPhi( \
|
|
comp_ctx->builder, \
|
|
TO_LLVM_TYPE(block->result_types[_i]), "phi"))) { \
|
|
aot_set_last_error("llvm build phi failed."); \
|
|
goto fail; \
|
|
} \
|
|
} \
|
|
SET_BUILDER_POS(_block_curr); \
|
|
} \
|
|
} while (0)
|
|
|
|
#define ADD_TO_RESULT_PHIS(block, value, idx) \
|
|
do { \
|
|
LLVMBasicBlockRef _block_curr = CURR_BLOCK(); \
|
|
LLVMTypeRef phi_ty = LLVMTypeOf(block->result_phis[idx]); \
|
|
LLVMTypeRef value_ty = LLVMTypeOf(value); \
|
|
bh_assert(LLVMGetTypeKind(phi_ty) == LLVMGetTypeKind(value_ty)); \
|
|
bh_assert(LLVMGetTypeContext(phi_ty) == LLVMGetTypeContext(value_ty)); \
|
|
LLVMAddIncoming(block->result_phis[idx], &value, &_block_curr, 1); \
|
|
(void)phi_ty; \
|
|
(void)value_ty; \
|
|
} while (0)
|
|
|
|
#define BUILD_ICMP(op, left, right, res, name) \
|
|
do { \
|
|
if (!(res = \
|
|
LLVMBuildICmp(comp_ctx->builder, op, left, right, name))) { \
|
|
aot_set_last_error("llvm build icmp failed."); \
|
|
goto fail; \
|
|
} \
|
|
} while (0)
|
|
|
|
#define ADD_TO_PARAM_PHIS(block, value, idx) \
|
|
do { \
|
|
LLVMBasicBlockRef _block_curr = CURR_BLOCK(); \
|
|
LLVMAddIncoming(block->param_phis[idx], &value, &_block_curr, 1); \
|
|
} while (0)
|
|
|
|
static LLVMBasicBlockRef
|
|
find_next_llvm_end_block(AOTBlock *block)
|
|
{
|
|
block = block->prev;
|
|
while (block && !block->llvm_end_block)
|
|
block = block->prev;
|
|
return block ? block->llvm_end_block : NULL;
|
|
}
|
|
|
|
static AOTBlock *
|
|
get_target_block(AOTFuncContext *func_ctx, uint32 br_depth)
|
|
{
|
|
uint32 i = br_depth;
|
|
AOTBlock *block = func_ctx->block_stack.block_list_end;
|
|
|
|
while (i-- > 0 && block) {
|
|
block = block->prev;
|
|
}
|
|
|
|
if (!block) {
|
|
aot_set_last_error("WASM block stack underflow.");
|
|
return NULL;
|
|
}
|
|
return block;
|
|
}
|
|
|
|
static void
|
|
clear_frame_locals(AOTCompFrame *aot_frame)
|
|
{
|
|
uint32 i;
|
|
|
|
for (i = 0; i < aot_frame->max_local_cell_num; i++) {
|
|
aot_frame->lp[i].dirty = 0;
|
|
aot_frame->lp[i].value = NULL;
|
|
if (aot_frame->comp_ctx->enable_gc)
|
|
/* Mark the ref flag as committed */
|
|
aot_frame->lp[i].committed_ref = aot_frame->lp[i].ref + 1;
|
|
}
|
|
}
|
|
|
|
static void
|
|
restore_frame_sp_for_op_else(AOTBlock *block, AOTCompFrame *aot_frame)
|
|
{
|
|
uint32 all_cell_num =
|
|
aot_frame->max_local_cell_num + aot_frame->max_stack_cell_num;
|
|
AOTValueSlot *p_end = aot_frame->lp + all_cell_num, *p;
|
|
|
|
/* Reset all the value slots from current frame sp for the else
|
|
branch since they be the same as starting to translate the
|
|
if branch */
|
|
for (p = block->frame_sp_begin; p < p_end; p++) {
|
|
p->dirty = 0;
|
|
p->value = NULL;
|
|
p->type = 0;
|
|
if (aot_frame->comp_ctx->enable_gc) {
|
|
p->ref = 0;
|
|
p->committed_ref = 1;
|
|
}
|
|
}
|
|
|
|
bh_assert(aot_frame->sp >= block->frame_sp_begin);
|
|
aot_frame->sp = block->frame_sp_begin;
|
|
}
|
|
|
|
static void
|
|
restore_frame_sp_for_op_end(AOTBlock *block, AOTCompFrame *aot_frame)
|
|
{
|
|
uint32 all_cell_num =
|
|
aot_frame->max_local_cell_num + aot_frame->max_stack_cell_num;
|
|
AOTValueSlot *p_end = aot_frame->lp + all_cell_num, *p;
|
|
|
|
bh_assert(block->frame_sp_max_reached >= block->frame_sp_begin);
|
|
|
|
/* Reset all the value slots from current frame sp to be same as
|
|
starting to translate this block, except for the frame ref
|
|
flags: set the flags to uncommitted before the max frame sp
|
|
ever reached, set the flags to committed non-ref after that */
|
|
for (p = block->frame_sp_begin; p < p_end; p++) {
|
|
p->dirty = 0;
|
|
p->value = NULL;
|
|
p->type = 0;
|
|
if (aot_frame->comp_ctx->enable_gc) {
|
|
p->ref = 0;
|
|
if (p < block->frame_sp_max_reached)
|
|
p->committed_ref = 0;
|
|
else
|
|
p->committed_ref = 1;
|
|
}
|
|
}
|
|
|
|
bh_assert(aot_frame->sp >= block->frame_sp_begin);
|
|
aot_frame->sp = block->frame_sp_begin;
|
|
}
|
|
|
|
static bool
|
|
handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
AOTBlock *block = func_ctx->block_stack.block_list_end;
|
|
AOTBlock *block_prev;
|
|
AOTCompFrame *aot_frame = comp_ctx->aot_frame;
|
|
uint8 *frame_ip = NULL;
|
|
uint32 i;
|
|
AOTFuncType *func_type;
|
|
LLVMValueRef ret;
|
|
#if WASM_ENABLE_DEBUG_AOT != 0
|
|
LLVMMetadataRef return_location;
|
|
#endif
|
|
|
|
aot_checked_addr_list_destroy(func_ctx);
|
|
bh_assert(block);
|
|
|
|
#if WASM_ENABLE_DEBUG_AOT != 0
|
|
return_location = dwarf_gen_location(
|
|
comp_ctx, func_ctx,
|
|
(*p_frame_ip - 1) - comp_ctx->comp_data->wasm_module->buf_code);
|
|
#endif
|
|
|
|
if (aot_frame) {
|
|
/* Clear frame local variables since they have been committed */
|
|
clear_frame_locals(aot_frame);
|
|
}
|
|
|
|
if (block->label_type == LABEL_TYPE_IF && block->llvm_else_block
|
|
&& *p_frame_ip <= block->wasm_code_else) {
|
|
/* Clear value stack and start to translate else branch */
|
|
aot_value_stack_destroy(comp_ctx, &block->value_stack);
|
|
|
|
if (aot_frame) {
|
|
/* Restore the frame sp */
|
|
restore_frame_sp_for_op_else(block, aot_frame);
|
|
}
|
|
|
|
/* Recover parameters of else branch */
|
|
for (i = 0; i < block->param_count; i++)
|
|
PUSH(block->else_param_phis[i], block->param_types[i]);
|
|
SET_BUILDER_POS(block->llvm_else_block);
|
|
*p_frame_ip = block->wasm_code_else + 1;
|
|
return true;
|
|
}
|
|
|
|
while (block && !block->is_reachable) {
|
|
block_prev = block->prev;
|
|
block = aot_block_stack_pop(&func_ctx->block_stack);
|
|
|
|
if (block->label_type == LABEL_TYPE_IF) {
|
|
if (block->llvm_else_block && !block->skip_wasm_code_else
|
|
&& *p_frame_ip <= block->wasm_code_else) {
|
|
/* Clear value stack and start to translate else branch */
|
|
aot_value_stack_destroy(comp_ctx, &block->value_stack);
|
|
|
|
if (aot_frame) {
|
|
/* Restore the frame sp */
|
|
restore_frame_sp_for_op_else(block, aot_frame);
|
|
}
|
|
|
|
SET_BUILDER_POS(block->llvm_else_block);
|
|
*p_frame_ip = block->wasm_code_else + 1;
|
|
/* Push back the block */
|
|
aot_block_stack_push(&func_ctx->block_stack, block);
|
|
/* Recover parameters of else branch */
|
|
for (i = 0; i < block->param_count; i++)
|
|
PUSH(block->else_param_phis[i], block->param_types[i]);
|
|
return true;
|
|
}
|
|
else if (block->llvm_end_block) {
|
|
/* Remove unreachable basic block */
|
|
LLVMDeleteBasicBlock(block->llvm_end_block);
|
|
block->llvm_end_block = NULL;
|
|
}
|
|
}
|
|
|
|
frame_ip = block->wasm_code_end;
|
|
aot_block_destroy(comp_ctx, block);
|
|
block = block_prev;
|
|
}
|
|
|
|
if (!block) {
|
|
*p_frame_ip = frame_ip + 1;
|
|
return true;
|
|
}
|
|
|
|
if (block->label_type == LABEL_TYPE_IF && block->llvm_else_block
|
|
&& !block->skip_wasm_code_else
|
|
&& *p_frame_ip <= block->wasm_code_else) {
|
|
/* Clear value stack and start to translate else branch */
|
|
aot_value_stack_destroy(comp_ctx, &block->value_stack);
|
|
|
|
if (aot_frame) {
|
|
/* Restore the frame sp */
|
|
restore_frame_sp_for_op_else(block, aot_frame);
|
|
}
|
|
|
|
/* Recover parameters of else branch */
|
|
for (i = 0; i < block->param_count; i++)
|
|
PUSH(block->else_param_phis[i], block->param_types[i]);
|
|
SET_BUILDER_POS(block->llvm_else_block);
|
|
*p_frame_ip = block->wasm_code_else + 1;
|
|
return true;
|
|
}
|
|
|
|
*p_frame_ip = block->wasm_code_end + 1;
|
|
SET_BUILDER_POS(block->llvm_end_block);
|
|
|
|
/* Pop block, push its return value, and destroy the block */
|
|
block = aot_block_stack_pop(&func_ctx->block_stack);
|
|
|
|
if (aot_frame) {
|
|
/* Restore the frame sp */
|
|
restore_frame_sp_for_op_end(block, aot_frame);
|
|
}
|
|
|
|
func_type = func_ctx->aot_func->func_type;
|
|
for (i = 0; i < block->result_count; i++) {
|
|
bh_assert(block->result_phis[i]);
|
|
if (block->label_type != LABEL_TYPE_FUNCTION) {
|
|
PUSH(block->result_phis[i], block->result_types[i]);
|
|
}
|
|
else {
|
|
/* Store extra return values to function parameters */
|
|
if (i != 0) {
|
|
LLVMValueRef res;
|
|
uint32 param_index = func_type->param_count + i;
|
|
if (!(res = LLVMBuildStore(
|
|
comp_ctx->builder, block->result_phis[i],
|
|
LLVMGetParam(func_ctx->func, param_index)))) {
|
|
aot_set_last_error("llvm build store failed.");
|
|
goto fail;
|
|
}
|
|
LLVMSetAlignment(res, 1);
|
|
}
|
|
}
|
|
}
|
|
if (block->label_type == LABEL_TYPE_FUNCTION) {
|
|
if (block->result_count) {
|
|
/* Return the first return value */
|
|
if (!(ret =
|
|
LLVMBuildRet(comp_ctx->builder, block->result_phis[0]))) {
|
|
aot_set_last_error("llvm build return failed.");
|
|
goto fail;
|
|
}
|
|
#if WASM_ENABLE_DEBUG_AOT != 0
|
|
if (return_location != NULL) {
|
|
LLVMInstructionSetDebugLoc(ret, return_location);
|
|
}
|
|
#endif
|
|
}
|
|
else {
|
|
if (!(ret = LLVMBuildRetVoid(comp_ctx->builder))) {
|
|
aot_set_last_error("llvm build return void failed.");
|
|
goto fail;
|
|
}
|
|
#if WASM_ENABLE_DEBUG_AOT != 0
|
|
if (return_location != NULL) {
|
|
LLVMInstructionSetDebugLoc(ret, return_location);
|
|
}
|
|
#endif
|
|
}
|
|
}
|
|
aot_block_destroy(comp_ctx, block);
|
|
return true;
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
static bool
|
|
push_aot_block_to_stack_and_pass_params(AOTCompContext *comp_ctx,
|
|
AOTFuncContext *func_ctx,
|
|
AOTBlock *block)
|
|
{
|
|
uint32 i, param_index;
|
|
LLVMValueRef value, br_inst;
|
|
uint64 size;
|
|
char name[32];
|
|
LLVMBasicBlockRef block_curr = CURR_BLOCK();
|
|
|
|
if (block->param_count) {
|
|
size = sizeof(LLVMValueRef) * (uint64)block->param_count;
|
|
if (size >= UINT32_MAX
|
|
|| !(block->param_phis = wasm_runtime_malloc((uint32)size))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
return false;
|
|
}
|
|
|
|
if (block->label_type == LABEL_TYPE_IF && !block->skip_wasm_code_else
|
|
&& !(block->else_param_phis = wasm_runtime_malloc((uint32)size))) {
|
|
wasm_runtime_free(block->param_phis);
|
|
block->param_phis = NULL;
|
|
aot_set_last_error("allocate memory failed.");
|
|
return false;
|
|
}
|
|
|
|
/* Create param phis */
|
|
for (i = 0; i < block->param_count; i++) {
|
|
SET_BUILDER_POS(block->llvm_entry_block);
|
|
snprintf(name, sizeof(name), "%s%d_phi%d",
|
|
block_name_prefix[block->label_type], block->block_index,
|
|
i);
|
|
if (!(block->param_phis[i] = LLVMBuildPhi(
|
|
comp_ctx->builder, TO_LLVM_TYPE(block->param_types[i]),
|
|
name))) {
|
|
aot_set_last_error("llvm build phi failed.");
|
|
goto fail;
|
|
}
|
|
|
|
if (block->label_type == LABEL_TYPE_IF
|
|
&& !block->skip_wasm_code_else && block->llvm_else_block) {
|
|
/* Build else param phis */
|
|
SET_BUILDER_POS(block->llvm_else_block);
|
|
snprintf(name, sizeof(name), "else%d_phi%d", block->block_index,
|
|
i);
|
|
if (!(block->else_param_phis[i] = LLVMBuildPhi(
|
|
comp_ctx->builder,
|
|
TO_LLVM_TYPE(block->param_types[i]), name))) {
|
|
aot_set_last_error("llvm build phi failed.");
|
|
goto fail;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* At this point, the branch instruction was already built to jump to
|
|
* the new BB, to avoid generating zext instruction from the popped
|
|
* operand that would come after branch instruction, we should position
|
|
* the builder before the last branch instruction */
|
|
br_inst = LLVMGetLastInstruction(block_curr);
|
|
bh_assert(LLVMGetInstructionOpcode(br_inst) == LLVMBr);
|
|
LLVMPositionBuilderBefore(comp_ctx->builder, br_inst);
|
|
|
|
/* Pop param values from current block's
|
|
* value stack and add to param phis.
|
|
*/
|
|
for (i = 0; i < block->param_count; i++) {
|
|
param_index = block->param_count - 1 - i;
|
|
POP(value, block->param_types[param_index]);
|
|
if (block->llvm_entry_block)
|
|
/* Only add incoming phis if the entry block was created */
|
|
ADD_TO_PARAM_PHIS(block, value, param_index);
|
|
if (block->label_type == LABEL_TYPE_IF
|
|
&& !block->skip_wasm_code_else) {
|
|
if (block->llvm_else_block) {
|
|
/* has else branch, add to else param phis */
|
|
LLVMAddIncoming(block->else_param_phis[param_index], &value,
|
|
&block_curr, 1);
|
|
}
|
|
else {
|
|
/* no else branch, add to result phis */
|
|
CREATE_RESULT_VALUE_PHIS(block);
|
|
ADD_TO_RESULT_PHIS(block, value, param_index);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Push the new block to block stack */
|
|
aot_block_stack_push(&func_ctx->block_stack, block);
|
|
if (comp_ctx->aot_frame) {
|
|
block->frame_sp_begin = block->frame_sp_max_reached =
|
|
comp_ctx->aot_frame->sp;
|
|
}
|
|
|
|
/* Push param phis to the new block */
|
|
for (i = 0; i < block->param_count; i++) {
|
|
if (block->llvm_entry_block)
|
|
/* Push param phis if the entry basic block was created */
|
|
PUSH(block->param_phis[i], block->param_types[i]);
|
|
else {
|
|
bh_assert(block->label_type == LABEL_TYPE_IF
|
|
&& block->llvm_else_block && block->else_param_phis
|
|
&& !block->skip_wasm_code_else);
|
|
/* Push else param phis if we start to translate the
|
|
else branch */
|
|
PUSH(block->else_param_phis[i], block->param_types[i]);
|
|
}
|
|
}
|
|
|
|
return true;
|
|
|
|
fail:
|
|
if (block->param_phis) {
|
|
wasm_runtime_free(block->param_phis);
|
|
block->param_phis = NULL;
|
|
}
|
|
if (block->else_param_phis) {
|
|
wasm_runtime_free(block->else_param_phis);
|
|
block->else_param_phis = NULL;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip, uint8 *frame_ip_end, uint32 label_type,
|
|
uint32 param_count, uint8 *param_types,
|
|
uint32 result_count, uint8 *result_types)
|
|
{
|
|
BlockAddr block_addr_cache[BLOCK_ADDR_CACHE_SIZE][BLOCK_ADDR_CONFLICT_SIZE];
|
|
AOTBlock *block;
|
|
uint8 *else_addr, *end_addr;
|
|
LLVMValueRef value;
|
|
char name[32];
|
|
|
|
/* Check block stack */
|
|
if (!func_ctx->block_stack.block_list_end) {
|
|
aot_set_last_error("WASM block stack underflow.");
|
|
return false;
|
|
}
|
|
|
|
memset(block_addr_cache, 0, sizeof(block_addr_cache));
|
|
|
|
/* Get block info */
|
|
if (!(wasm_loader_find_block_addr(
|
|
NULL, (BlockAddr *)block_addr_cache, *p_frame_ip, frame_ip_end,
|
|
(uint8)label_type, &else_addr, &end_addr))) {
|
|
aot_set_last_error("find block end addr failed.");
|
|
return false;
|
|
}
|
|
|
|
/* Allocate memory */
|
|
if (!(block = wasm_runtime_malloc(sizeof(AOTBlock)))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
return false;
|
|
}
|
|
memset(block, 0, sizeof(AOTBlock));
|
|
if (param_count
|
|
&& !(block->param_types = wasm_runtime_malloc(param_count))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
if (result_count) {
|
|
if (!(block->result_types = wasm_runtime_malloc(result_count))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
}
|
|
|
|
/* Init aot block data */
|
|
block->label_type = label_type;
|
|
block->param_count = param_count;
|
|
if (param_count) {
|
|
bh_memcpy_s(block->param_types, param_count, param_types, param_count);
|
|
}
|
|
block->result_count = result_count;
|
|
if (result_count) {
|
|
bh_memcpy_s(block->result_types, result_count, result_types,
|
|
result_count);
|
|
}
|
|
block->wasm_code_else = else_addr;
|
|
block->wasm_code_end = end_addr;
|
|
block->block_index = func_ctx->block_stack.block_index[label_type];
|
|
func_ctx->block_stack.block_index[label_type]++;
|
|
|
|
if (comp_ctx->aot_frame) {
|
|
if (label_type != LABEL_TYPE_BLOCK && comp_ctx->enable_gc
|
|
&& !aot_gen_commit_values(comp_ctx->aot_frame)) {
|
|
goto fail;
|
|
}
|
|
}
|
|
|
|
if (label_type == LABEL_TYPE_BLOCK || label_type == LABEL_TYPE_LOOP) {
|
|
/* Create block */
|
|
format_block_name(name, sizeof(name), block->block_index, label_type,
|
|
LABEL_BEGIN);
|
|
CREATE_BLOCK(block->llvm_entry_block, name);
|
|
MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block);
|
|
/* Jump to the entry block */
|
|
BUILD_BR(block->llvm_entry_block);
|
|
if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, block))
|
|
goto fail;
|
|
/* Start to translate the block */
|
|
SET_BUILDER_POS(block->llvm_entry_block);
|
|
if (label_type == LABEL_TYPE_LOOP)
|
|
aot_checked_addr_list_destroy(func_ctx);
|
|
}
|
|
else if (label_type == LABEL_TYPE_IF) {
|
|
POP_COND(value);
|
|
|
|
if (LLVMIsUndef(value)
|
|
#if LLVM_VERSION_NUMBER >= 12
|
|
|| LLVMIsPoison(value)
|
|
#endif
|
|
) {
|
|
if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INTEGER_OVERFLOW,
|
|
false, NULL, NULL))) {
|
|
goto fail;
|
|
}
|
|
aot_block_destroy(comp_ctx, block);
|
|
return aot_handle_next_reachable_block(comp_ctx, func_ctx,
|
|
p_frame_ip);
|
|
}
|
|
|
|
if (!LLVMIsEfficientConstInt(value)) {
|
|
/* Compare value is not constant, create condition br IR */
|
|
/* Create entry block */
|
|
format_block_name(name, sizeof(name), block->block_index,
|
|
label_type, LABEL_BEGIN);
|
|
CREATE_BLOCK(block->llvm_entry_block, name);
|
|
MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block);
|
|
|
|
/* Create end block */
|
|
format_block_name(name, sizeof(name), block->block_index,
|
|
label_type, LABEL_END);
|
|
CREATE_BLOCK(block->llvm_end_block, name);
|
|
MOVE_BLOCK_AFTER(block->llvm_end_block, block->llvm_entry_block);
|
|
|
|
if (else_addr) {
|
|
/* Create else block */
|
|
format_block_name(name, sizeof(name), block->block_index,
|
|
label_type, LABEL_ELSE);
|
|
CREATE_BLOCK(block->llvm_else_block, name);
|
|
MOVE_BLOCK_AFTER(block->llvm_else_block,
|
|
block->llvm_entry_block);
|
|
/* Create condition br IR */
|
|
BUILD_COND_BR(value, block->llvm_entry_block,
|
|
block->llvm_else_block);
|
|
}
|
|
else {
|
|
/* Create condition br IR */
|
|
BUILD_COND_BR(value, block->llvm_entry_block,
|
|
block->llvm_end_block);
|
|
block->is_reachable = true;
|
|
}
|
|
if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx,
|
|
block))
|
|
goto fail;
|
|
/* Start to translate if branch of BLOCK if */
|
|
SET_BUILDER_POS(block->llvm_entry_block);
|
|
}
|
|
else {
|
|
if ((int32)LLVMConstIntGetZExtValue(value) != 0) {
|
|
/* Compare value is not 0, condition is true, else branch of
|
|
BLOCK if cannot be reached */
|
|
block->skip_wasm_code_else = true;
|
|
/* Create entry block */
|
|
format_block_name(name, sizeof(name), block->block_index,
|
|
label_type, LABEL_BEGIN);
|
|
CREATE_BLOCK(block->llvm_entry_block, name);
|
|
MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block);
|
|
/* Jump to the entry block */
|
|
BUILD_BR(block->llvm_entry_block);
|
|
if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx,
|
|
block))
|
|
goto fail;
|
|
/* Start to translate the if branch */
|
|
SET_BUILDER_POS(block->llvm_entry_block);
|
|
}
|
|
else {
|
|
/* Compare value is not 0, condition is false, if branch of
|
|
BLOCK if cannot be reached */
|
|
if (else_addr) {
|
|
/* Create else block */
|
|
format_block_name(name, sizeof(name), block->block_index,
|
|
label_type, LABEL_ELSE);
|
|
CREATE_BLOCK(block->llvm_else_block, name);
|
|
MOVE_BLOCK_AFTER_CURR(block->llvm_else_block);
|
|
/* Jump to the else block */
|
|
BUILD_BR(block->llvm_else_block);
|
|
if (!push_aot_block_to_stack_and_pass_params(
|
|
comp_ctx, func_ctx, block))
|
|
goto fail;
|
|
/* Start to translate the else branch */
|
|
SET_BUILDER_POS(block->llvm_else_block);
|
|
*p_frame_ip = else_addr + 1;
|
|
}
|
|
else {
|
|
/* skip the block */
|
|
aot_block_destroy(comp_ctx, block);
|
|
*p_frame_ip = end_addr + 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
aot_set_last_error("Invalid block type.");
|
|
goto fail;
|
|
}
|
|
|
|
return true;
|
|
fail:
|
|
aot_block_destroy(comp_ctx, block);
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
AOTBlock *block = func_ctx->block_stack.block_list_end;
|
|
LLVMValueRef value;
|
|
AOTCompFrame *aot_frame = comp_ctx->aot_frame;
|
|
char name[32];
|
|
uint32 i, result_index;
|
|
|
|
/* Check block */
|
|
if (!block) {
|
|
aot_set_last_error("WASM block stack underflow.");
|
|
return false;
|
|
}
|
|
if (block->label_type != LABEL_TYPE_IF
|
|
|| (!block->skip_wasm_code_else && !block->llvm_else_block)) {
|
|
aot_set_last_error("Invalid WASM block type.");
|
|
return false;
|
|
}
|
|
|
|
/* Create end block if needed */
|
|
if (!block->llvm_end_block) {
|
|
format_block_name(name, sizeof(name), block->block_index,
|
|
block->label_type, LABEL_END);
|
|
CREATE_BLOCK(block->llvm_end_block, name);
|
|
if (block->llvm_else_block)
|
|
MOVE_BLOCK_AFTER(block->llvm_end_block, block->llvm_else_block);
|
|
else
|
|
MOVE_BLOCK_AFTER_CURR(block->llvm_end_block);
|
|
}
|
|
|
|
block->is_reachable = true;
|
|
|
|
/* Comes from the if branch of BLOCK if */
|
|
CREATE_RESULT_VALUE_PHIS(block);
|
|
for (i = 0; i < block->result_count; i++) {
|
|
result_index = block->result_count - 1 - i;
|
|
POP(value, block->result_types[result_index]);
|
|
ADD_TO_RESULT_PHIS(block, value, result_index);
|
|
}
|
|
|
|
if (aot_frame) {
|
|
bh_assert(block->frame_sp_begin == aot_frame->sp);
|
|
if (comp_ctx->enable_gc && !aot_gen_commit_values(aot_frame)) {
|
|
goto fail;
|
|
}
|
|
}
|
|
|
|
/* Jump to end block */
|
|
BUILD_BR(block->llvm_end_block);
|
|
|
|
if (!block->skip_wasm_code_else && block->llvm_else_block) {
|
|
/* Clear value stack, recover param values
|
|
and start to translate else branch. */
|
|
aot_value_stack_destroy(comp_ctx, &block->value_stack);
|
|
|
|
if (comp_ctx->aot_frame) {
|
|
clear_frame_locals(aot_frame);
|
|
restore_frame_sp_for_op_else(block, aot_frame);
|
|
}
|
|
|
|
for (i = 0; i < block->param_count; i++)
|
|
PUSH(block->else_param_phis[i], block->param_types[i]);
|
|
SET_BUILDER_POS(block->llvm_else_block);
|
|
aot_checked_addr_list_destroy(func_ctx);
|
|
return true;
|
|
}
|
|
|
|
/* No else branch or no need to translate else branch */
|
|
block->is_reachable = true;
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_end(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
AOTBlock *block;
|
|
LLVMValueRef value;
|
|
LLVMBasicBlockRef next_llvm_end_block;
|
|
char name[32];
|
|
uint32 i, result_index;
|
|
|
|
/* Check block stack */
|
|
if (!(block = func_ctx->block_stack.block_list_end)) {
|
|
aot_set_last_error("WASM block stack underflow.");
|
|
return false;
|
|
}
|
|
|
|
/* Create the end block */
|
|
if (!block->llvm_end_block) {
|
|
format_block_name(name, sizeof(name), block->block_index,
|
|
block->label_type, LABEL_END);
|
|
CREATE_BLOCK(block->llvm_end_block, name);
|
|
if ((next_llvm_end_block = find_next_llvm_end_block(block)))
|
|
MOVE_BLOCK_BEFORE(block->llvm_end_block, next_llvm_end_block);
|
|
}
|
|
|
|
if (comp_ctx->aot_frame) {
|
|
if (block->label_type != LABEL_TYPE_FUNCTION && comp_ctx->enable_gc
|
|
&& !aot_gen_commit_values(comp_ctx->aot_frame)) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/* Handle block result values */
|
|
CREATE_RESULT_VALUE_PHIS(block);
|
|
for (i = 0; i < block->result_count; i++) {
|
|
value = NULL;
|
|
result_index = block->result_count - 1 - i;
|
|
POP(value, block->result_types[result_index]);
|
|
bh_assert(value);
|
|
ADD_TO_RESULT_PHIS(block, value, result_index);
|
|
}
|
|
|
|
if (comp_ctx->aot_frame) {
|
|
bh_assert(comp_ctx->aot_frame->sp == block->frame_sp_begin);
|
|
}
|
|
|
|
/* Jump to the end block */
|
|
BUILD_BR(block->llvm_end_block);
|
|
|
|
block->is_reachable = true;
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
bool check_terminate_and_suspend)
|
|
{
|
|
LLVMValueRef terminate_addr, terminate_flags, flag, offset, res;
|
|
LLVMBasicBlockRef terminate_block, non_terminate_block;
|
|
AOTFuncType *aot_func_type = func_ctx->aot_func->func_type;
|
|
bool is_shared_memory =
|
|
comp_ctx->comp_data->memories[0].flags & 0x02 ? true : false;
|
|
|
|
/* Only need to check the suspend flags when memory is shared since
|
|
shared memory must be enabled for multi-threading */
|
|
if (!is_shared_memory) {
|
|
return true;
|
|
}
|
|
|
|
/* Offset of suspend_flags */
|
|
offset = I32_FIVE;
|
|
|
|
if (!(terminate_addr = LLVMBuildInBoundsGEP2(
|
|
comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, &offset, 1,
|
|
"terminate_addr"))) {
|
|
aot_set_last_error("llvm build in bounds gep failed");
|
|
return false;
|
|
}
|
|
if (!(terminate_addr =
|
|
LLVMBuildBitCast(comp_ctx->builder, terminate_addr,
|
|
INT32_PTR_TYPE, "terminate_addr_ptr"))) {
|
|
aot_set_last_error("llvm build bit cast failed");
|
|
return false;
|
|
}
|
|
|
|
if (!(terminate_flags =
|
|
LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, terminate_addr,
|
|
"terminate_flags"))) {
|
|
aot_set_last_error("llvm build LOAD failed");
|
|
return false;
|
|
}
|
|
/* Set terminate_flags memory accecc to volatile, so that the value
|
|
will always be loaded from memory rather than register */
|
|
LLVMSetVolatile(terminate_flags, true);
|
|
|
|
if (!(flag = LLVMBuildAnd(comp_ctx->builder, terminate_flags, I32_ONE,
|
|
"termination_flag"))) {
|
|
aot_set_last_error("llvm build AND failed");
|
|
return false;
|
|
}
|
|
|
|
CREATE_BLOCK(non_terminate_block, "non_terminate");
|
|
MOVE_BLOCK_AFTER_CURR(non_terminate_block);
|
|
|
|
CREATE_BLOCK(terminate_block, "terminate");
|
|
MOVE_BLOCK_AFTER_CURR(terminate_block);
|
|
|
|
BUILD_ICMP(LLVMIntEQ, flag, I32_ZERO, res, "flag_terminate");
|
|
BUILD_COND_BR(res, non_terminate_block, terminate_block);
|
|
|
|
/* Move builder to terminate block */
|
|
SET_BUILDER_POS(terminate_block);
|
|
if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) {
|
|
goto fail;
|
|
}
|
|
|
|
/* Move builder to non terminate block */
|
|
SET_BUILDER_POS(non_terminate_block);
|
|
return true;
|
|
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint32 br_depth, uint8 **p_frame_ip)
|
|
{
|
|
AOTBlock *block_dst;
|
|
LLVMValueRef value_ret, value_param;
|
|
LLVMBasicBlockRef next_llvm_end_block;
|
|
char name[32];
|
|
uint32 i, param_index, result_index;
|
|
|
|
if (!(block_dst = get_target_block(func_ctx, br_depth))) {
|
|
return false;
|
|
}
|
|
|
|
if (comp_ctx->aot_frame) {
|
|
if (comp_ctx->enable_gc && !aot_gen_commit_values(comp_ctx->aot_frame))
|
|
return false;
|
|
|
|
if (block_dst->label_type == LABEL_TYPE_LOOP) {
|
|
if (comp_ctx->enable_thread_mgr) {
|
|
/* Commit sp when GC is enabled, don't commit ip */
|
|
if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame,
|
|
comp_ctx->enable_gc, false))
|
|
return false;
|
|
}
|
|
}
|
|
else {
|
|
if (comp_ctx->aot_frame->sp > block_dst->frame_sp_max_reached)
|
|
block_dst->frame_sp_max_reached = comp_ctx->aot_frame->sp;
|
|
}
|
|
}
|
|
|
|
/* Terminate or suspend current thread only when this is a backward jump */
|
|
if (comp_ctx->enable_thread_mgr
|
|
&& block_dst->label_type == LABEL_TYPE_LOOP) {
|
|
if (!check_suspend_flags(comp_ctx, func_ctx, true))
|
|
return false;
|
|
}
|
|
|
|
if (block_dst->label_type == LABEL_TYPE_LOOP) {
|
|
/* Dest block is Loop block */
|
|
/* Handle Loop parameters */
|
|
for (i = 0; i < block_dst->param_count; i++) {
|
|
param_index = block_dst->param_count - 1 - i;
|
|
POP(value_param, block_dst->param_types[param_index]);
|
|
ADD_TO_PARAM_PHIS(block_dst, value_param, param_index);
|
|
}
|
|
BUILD_BR(block_dst->llvm_entry_block);
|
|
}
|
|
else {
|
|
/* Dest block is Block/If/Function block */
|
|
/* Create the end block */
|
|
if (!block_dst->llvm_end_block) {
|
|
format_block_name(name, sizeof(name), block_dst->block_index,
|
|
block_dst->label_type, LABEL_END);
|
|
CREATE_BLOCK(block_dst->llvm_end_block, name);
|
|
if ((next_llvm_end_block = find_next_llvm_end_block(block_dst)))
|
|
MOVE_BLOCK_BEFORE(block_dst->llvm_end_block,
|
|
next_llvm_end_block);
|
|
}
|
|
|
|
block_dst->is_reachable = true;
|
|
|
|
/* Handle result values */
|
|
CREATE_RESULT_VALUE_PHIS(block_dst);
|
|
for (i = 0; i < block_dst->result_count; i++) {
|
|
result_index = block_dst->result_count - 1 - i;
|
|
POP(value_ret, block_dst->result_types[result_index]);
|
|
ADD_TO_RESULT_PHIS(block_dst, value_ret, result_index);
|
|
}
|
|
/* Jump to the end block */
|
|
BUILD_BR(block_dst->llvm_end_block);
|
|
}
|
|
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
static bool
|
|
aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint32 br_depth, LLVMValueRef value_cmp,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
AOTBlock *block_dst;
|
|
LLVMValueRef value, *values = NULL;
|
|
LLVMBasicBlockRef llvm_else_block, next_llvm_end_block;
|
|
char name[32];
|
|
uint32 i, param_index, result_index;
|
|
uint64 size;
|
|
|
|
if (!(block_dst = get_target_block(func_ctx, br_depth))) {
|
|
return false;
|
|
}
|
|
|
|
if (comp_ctx->aot_frame) {
|
|
if (comp_ctx->enable_gc && !aot_gen_commit_values(comp_ctx->aot_frame))
|
|
return false;
|
|
|
|
if (block_dst->label_type == LABEL_TYPE_LOOP) {
|
|
if (comp_ctx->enable_thread_mgr) {
|
|
/* Commit sp when GC is enabled, don't commit ip */
|
|
if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame,
|
|
comp_ctx->enable_gc, false))
|
|
return false;
|
|
}
|
|
}
|
|
else {
|
|
if (comp_ctx->aot_frame->sp > block_dst->frame_sp_max_reached)
|
|
block_dst->frame_sp_max_reached = comp_ctx->aot_frame->sp;
|
|
}
|
|
}
|
|
|
|
/* Terminate or suspend current thread only when this is
|
|
a backward jump */
|
|
if (comp_ctx->enable_thread_mgr
|
|
&& block_dst->label_type == LABEL_TYPE_LOOP) {
|
|
if (!check_suspend_flags(comp_ctx, func_ctx, true))
|
|
return false;
|
|
}
|
|
|
|
if (LLVMIsUndef(value_cmp)
|
|
#if LLVM_VERSION_NUMBER >= 12
|
|
|| LLVMIsPoison(value_cmp)
|
|
#endif
|
|
) {
|
|
if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INTEGER_OVERFLOW,
|
|
false, NULL, NULL))) {
|
|
goto fail;
|
|
}
|
|
return aot_handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
}
|
|
|
|
if (!LLVMIsEfficientConstInt(value_cmp)) {
|
|
/* Compare value is not constant, create condition br IR */
|
|
|
|
/* Create llvm else block */
|
|
CREATE_BLOCK(llvm_else_block, "br_if_else");
|
|
MOVE_BLOCK_AFTER_CURR(llvm_else_block);
|
|
|
|
if (block_dst->label_type == LABEL_TYPE_LOOP) {
|
|
/* Dest block is Loop block */
|
|
/* Handle Loop parameters */
|
|
if (block_dst->param_count) {
|
|
size = sizeof(LLVMValueRef) * (uint64)block_dst->param_count;
|
|
if (size >= UINT32_MAX
|
|
|| !(values = wasm_runtime_malloc((uint32)size))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
for (i = 0; i < block_dst->param_count; i++) {
|
|
param_index = block_dst->param_count - 1 - i;
|
|
POP(value, block_dst->param_types[param_index]);
|
|
ADD_TO_PARAM_PHIS(block_dst, value, param_index);
|
|
values[param_index] = value;
|
|
}
|
|
for (i = 0; i < block_dst->param_count; i++) {
|
|
PUSH(values[i], block_dst->param_types[i]);
|
|
}
|
|
wasm_runtime_free(values);
|
|
values = NULL;
|
|
}
|
|
|
|
BUILD_COND_BR(value_cmp, block_dst->llvm_entry_block,
|
|
llvm_else_block);
|
|
|
|
/* Move builder to else block */
|
|
SET_BUILDER_POS(llvm_else_block);
|
|
}
|
|
else {
|
|
/* Dest block is Block/If/Function block */
|
|
/* Create the end block */
|
|
if (!block_dst->llvm_end_block) {
|
|
format_block_name(name, sizeof(name), block_dst->block_index,
|
|
block_dst->label_type, LABEL_END);
|
|
CREATE_BLOCK(block_dst->llvm_end_block, name);
|
|
if ((next_llvm_end_block = find_next_llvm_end_block(block_dst)))
|
|
MOVE_BLOCK_BEFORE(block_dst->llvm_end_block,
|
|
next_llvm_end_block);
|
|
}
|
|
|
|
/* Set reachable flag and create condition br IR */
|
|
block_dst->is_reachable = true;
|
|
|
|
/* Handle result values */
|
|
if (block_dst->result_count) {
|
|
size = sizeof(LLVMValueRef) * (uint64)block_dst->result_count;
|
|
if (size >= UINT32_MAX
|
|
|| !(values = wasm_runtime_malloc((uint32)size))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
CREATE_RESULT_VALUE_PHIS(block_dst);
|
|
for (i = 0; i < block_dst->result_count; i++) {
|
|
result_index = block_dst->result_count - 1 - i;
|
|
POP(value, block_dst->result_types[result_index]);
|
|
values[result_index] = value;
|
|
ADD_TO_RESULT_PHIS(block_dst, value, result_index);
|
|
}
|
|
for (i = 0; i < block_dst->result_count; i++) {
|
|
PUSH(values[i], block_dst->result_types[i]);
|
|
}
|
|
wasm_runtime_free(values);
|
|
values = NULL;
|
|
}
|
|
|
|
/* Condition jump to end block */
|
|
BUILD_COND_BR(value_cmp, block_dst->llvm_end_block,
|
|
llvm_else_block);
|
|
|
|
/* Move builder to else block */
|
|
SET_BUILDER_POS(llvm_else_block);
|
|
}
|
|
}
|
|
else {
|
|
if ((int32)LLVMConstIntGetZExtValue(value_cmp) != 0) {
|
|
/* Compare value is not 0, condition is true, same as op_br */
|
|
return aot_compile_op_br(comp_ctx, func_ctx, br_depth, p_frame_ip);
|
|
}
|
|
else {
|
|
/* Compare value is not 0, condition is false, skip br_if */
|
|
return true;
|
|
}
|
|
}
|
|
return true;
|
|
fail:
|
|
if (values)
|
|
wasm_runtime_free(values);
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint32 br_depth, uint8 **p_frame_ip)
|
|
{
|
|
LLVMValueRef value_cmp;
|
|
|
|
POP_COND(value_cmp);
|
|
|
|
return aot_compile_conditional_br(comp_ctx, func_ctx, br_depth, value_cmp,
|
|
p_frame_ip);
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint32 *br_depths, uint32 br_count, uint8 **p_frame_ip)
|
|
{
|
|
uint32 i, j;
|
|
LLVMValueRef value_switch, value_cmp, value_case, value, *values = NULL;
|
|
LLVMBasicBlockRef default_llvm_block = NULL, target_llvm_block;
|
|
LLVMBasicBlockRef next_llvm_end_block;
|
|
AOTBlock *target_block;
|
|
uint32 br_depth, depth_idx;
|
|
uint32 param_index, result_index;
|
|
uint64 size;
|
|
char name[32];
|
|
|
|
POP_I32(value_cmp);
|
|
|
|
if (LLVMIsUndef(value_cmp)
|
|
#if LLVM_VERSION_NUMBER >= 12
|
|
|| LLVMIsPoison(value_cmp)
|
|
#endif
|
|
) {
|
|
if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INTEGER_OVERFLOW,
|
|
false, NULL, NULL))) {
|
|
goto fail;
|
|
}
|
|
return aot_handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
}
|
|
|
|
if (!LLVMIsEfficientConstInt(value_cmp)) {
|
|
if (comp_ctx->aot_frame) {
|
|
if (comp_ctx->enable_gc
|
|
&& !aot_gen_commit_values(comp_ctx->aot_frame))
|
|
return false;
|
|
|
|
if (comp_ctx->enable_thread_mgr) {
|
|
/* Commit sp when GC is enabled, don't commit ip */
|
|
if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame,
|
|
comp_ctx->enable_gc, false))
|
|
return false;
|
|
}
|
|
|
|
for (i = 0; i <= br_count; i++) {
|
|
target_block = get_target_block(func_ctx, br_depths[i]);
|
|
if (!target_block)
|
|
return false;
|
|
if (target_block->label_type != LABEL_TYPE_LOOP) {
|
|
if (comp_ctx->aot_frame->sp
|
|
> target_block->frame_sp_max_reached)
|
|
target_block->frame_sp_max_reached =
|
|
comp_ctx->aot_frame->sp;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (comp_ctx->enable_thread_mgr) {
|
|
for (i = 0; i <= br_count; i++) {
|
|
target_block = get_target_block(func_ctx, br_depths[i]);
|
|
if (!target_block)
|
|
return false;
|
|
/* Terminate or suspend current thread only when this is a
|
|
backward jump */
|
|
if (target_block->label_type == LABEL_TYPE_LOOP) {
|
|
if (!check_suspend_flags(comp_ctx, func_ctx, true))
|
|
return false;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Compare value is not constant, create switch IR */
|
|
for (i = 0; i <= br_count; i++) {
|
|
target_block = get_target_block(func_ctx, br_depths[i]);
|
|
if (!target_block)
|
|
return false;
|
|
|
|
if (target_block->label_type != LABEL_TYPE_LOOP) {
|
|
/* Dest block is Block/If/Function block */
|
|
/* Create the end block */
|
|
if (!target_block->llvm_end_block) {
|
|
format_block_name(name, sizeof(name),
|
|
target_block->block_index,
|
|
target_block->label_type, LABEL_END);
|
|
CREATE_BLOCK(target_block->llvm_end_block, name);
|
|
if ((next_llvm_end_block =
|
|
find_next_llvm_end_block(target_block)))
|
|
MOVE_BLOCK_BEFORE(target_block->llvm_end_block,
|
|
next_llvm_end_block);
|
|
}
|
|
/* Handle result values */
|
|
if (target_block->result_count) {
|
|
size = sizeof(LLVMValueRef)
|
|
* (uint64)target_block->result_count;
|
|
if (size >= UINT32_MAX
|
|
|| !(values = wasm_runtime_malloc((uint32)size))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
CREATE_RESULT_VALUE_PHIS(target_block);
|
|
for (j = 0; j < target_block->result_count; j++) {
|
|
result_index = target_block->result_count - 1 - j;
|
|
POP(value, target_block->result_types[result_index]);
|
|
values[result_index] = value;
|
|
ADD_TO_RESULT_PHIS(target_block, value, result_index);
|
|
}
|
|
for (j = 0; j < target_block->result_count; j++) {
|
|
PUSH(values[j], target_block->result_types[j]);
|
|
}
|
|
wasm_runtime_free(values);
|
|
values = NULL;
|
|
}
|
|
target_block->is_reachable = true;
|
|
if (i == br_count)
|
|
default_llvm_block = target_block->llvm_end_block;
|
|
}
|
|
else {
|
|
/* Handle Loop parameters */
|
|
if (target_block->param_count) {
|
|
size = sizeof(LLVMValueRef)
|
|
* (uint64)target_block->param_count;
|
|
if (size >= UINT32_MAX
|
|
|| !(values = wasm_runtime_malloc((uint32)size))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
for (j = 0; j < target_block->param_count; j++) {
|
|
param_index = target_block->param_count - 1 - j;
|
|
POP(value, target_block->param_types[param_index]);
|
|
values[param_index] = value;
|
|
ADD_TO_PARAM_PHIS(target_block, value, param_index);
|
|
}
|
|
for (j = 0; j < target_block->param_count; j++) {
|
|
PUSH(values[j], target_block->param_types[j]);
|
|
}
|
|
wasm_runtime_free(values);
|
|
values = NULL;
|
|
}
|
|
if (i == br_count)
|
|
default_llvm_block = target_block->llvm_entry_block;
|
|
}
|
|
}
|
|
|
|
/* Create switch IR */
|
|
if (!(value_switch = LLVMBuildSwitch(comp_ctx->builder, value_cmp,
|
|
default_llvm_block, br_count))) {
|
|
aot_set_last_error("llvm build switch failed.");
|
|
return false;
|
|
}
|
|
|
|
/* Add each case for switch IR */
|
|
for (i = 0; i < br_count; i++) {
|
|
value_case = I32_CONST(i);
|
|
CHECK_LLVM_CONST(value_case);
|
|
target_block = get_target_block(func_ctx, br_depths[i]);
|
|
if (!target_block)
|
|
return false;
|
|
target_llvm_block = target_block->label_type != LABEL_TYPE_LOOP
|
|
? target_block->llvm_end_block
|
|
: target_block->llvm_entry_block;
|
|
LLVMAddCase(value_switch, value_case, target_llvm_block);
|
|
}
|
|
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
}
|
|
else {
|
|
/* Compare value is constant, create br IR */
|
|
depth_idx = (uint32)LLVMConstIntGetZExtValue(value_cmp);
|
|
br_depth = br_depths[br_count];
|
|
if (depth_idx < br_count) {
|
|
br_depth = br_depths[depth_idx];
|
|
}
|
|
return aot_compile_op_br(comp_ctx, func_ctx, br_depth, p_frame_ip);
|
|
}
|
|
fail:
|
|
if (values)
|
|
wasm_runtime_free(values);
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_return(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
AOTBlock *block_func = func_ctx->block_stack.block_list_head;
|
|
LLVMValueRef value;
|
|
LLVMValueRef ret;
|
|
AOTFuncType *func_type;
|
|
uint32 i, param_index, result_index;
|
|
#if WASM_ENABLE_DEBUG_AOT != 0
|
|
LLVMMetadataRef return_location;
|
|
#endif
|
|
|
|
bh_assert(block_func);
|
|
func_type = func_ctx->aot_func->func_type;
|
|
|
|
#if WASM_ENABLE_DEBUG_AOT != 0
|
|
return_location = dwarf_gen_location(
|
|
comp_ctx, func_ctx,
|
|
(*p_frame_ip - 1) - comp_ctx->comp_data->wasm_module->buf_code);
|
|
#endif
|
|
|
|
if (block_func->result_count) {
|
|
/* Store extra result values to function parameters */
|
|
for (i = 0; i < block_func->result_count - 1; i++) {
|
|
LLVMValueRef res;
|
|
result_index = block_func->result_count - 1 - i;
|
|
POP(value, block_func->result_types[result_index]);
|
|
param_index = func_type->param_count + result_index;
|
|
if (!(res = LLVMBuildStore(
|
|
comp_ctx->builder, value,
|
|
LLVMGetParam(func_ctx->func, param_index)))) {
|
|
aot_set_last_error("llvm build store failed.");
|
|
goto fail;
|
|
}
|
|
LLVMSetAlignment(res, 1);
|
|
}
|
|
/* Return the first result value */
|
|
POP(value, block_func->result_types[0]);
|
|
if (!(ret = LLVMBuildRet(comp_ctx->builder, value))) {
|
|
aot_set_last_error("llvm build return failed.");
|
|
goto fail;
|
|
}
|
|
#if WASM_ENABLE_DEBUG_AOT != 0
|
|
LLVMInstructionSetDebugLoc(ret, return_location);
|
|
#endif
|
|
}
|
|
else {
|
|
if (!(ret = LLVMBuildRetVoid(comp_ctx->builder))) {
|
|
aot_set_last_error("llvm build return void failed.");
|
|
goto fail;
|
|
}
|
|
#if WASM_ENABLE_DEBUG_AOT != 0
|
|
LLVMInstructionSetDebugLoc(ret, return_location);
|
|
#endif
|
|
}
|
|
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_unreachable(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_UNREACHABLE, false, NULL,
|
|
NULL))
|
|
return false;
|
|
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
}
|
|
|
|
bool
|
|
aot_handle_next_reachable_block(AOTCompContext *comp_ctx,
|
|
AOTFuncContext *func_ctx, uint8 **p_frame_ip)
|
|
{
|
|
return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip);
|
|
}
|
|
|
|
#if WASM_ENABLE_GC != 0
|
|
static bool
|
|
commit_gc_and_check_suspend_flags(AOTCompContext *comp_ctx,
|
|
AOTFuncContext *func_ctx, uint32 br_depth)
|
|
{
|
|
AOTBlock *block_dst;
|
|
|
|
if (!(block_dst = get_target_block(func_ctx, br_depth))) {
|
|
return false;
|
|
}
|
|
|
|
if (comp_ctx->aot_frame) {
|
|
/* Note that GC is enabled, no need to check it again */
|
|
if (!aot_gen_commit_values(comp_ctx->aot_frame))
|
|
return false;
|
|
|
|
if (block_dst->label_type == LABEL_TYPE_LOOP) {
|
|
if (comp_ctx->enable_thread_mgr) {
|
|
/* Note that GC is enabled, no need to check it again */
|
|
if (!aot_gen_commit_sp_ip(comp_ctx->aot_frame, true, false))
|
|
return false;
|
|
}
|
|
}
|
|
else {
|
|
if (comp_ctx->aot_frame->sp > block_dst->frame_sp_max_reached)
|
|
block_dst->frame_sp_max_reached = comp_ctx->aot_frame->sp;
|
|
}
|
|
}
|
|
|
|
/* Terminate or suspend current thread only when this is
|
|
a backward jump */
|
|
if (comp_ctx->enable_thread_mgr
|
|
&& block_dst->label_type == LABEL_TYPE_LOOP) {
|
|
if (!check_suspend_flags(comp_ctx, func_ctx, true))
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
static bool
|
|
compile_gc_cond_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint32 br_depth, LLVMValueRef value_cmp)
|
|
{
|
|
AOTBlock *block_dst;
|
|
LLVMValueRef value, *values = NULL;
|
|
LLVMBasicBlockRef llvm_else_block, next_llvm_end_block;
|
|
char name[32];
|
|
uint32 i, param_index, result_index;
|
|
uint64 size;
|
|
|
|
if (!(block_dst = get_target_block(func_ctx, br_depth))) {
|
|
return false;
|
|
}
|
|
|
|
/* Create llvm else block */
|
|
CREATE_BLOCK(llvm_else_block, "br_if_else");
|
|
MOVE_BLOCK_AFTER_CURR(llvm_else_block);
|
|
|
|
if (block_dst->label_type == LABEL_TYPE_LOOP) {
|
|
/* Dest block is Loop block */
|
|
/* Handle Loop parameters */
|
|
if (block_dst->param_count) {
|
|
size = sizeof(LLVMValueRef) * (uint64)block_dst->param_count;
|
|
if (size >= UINT32_MAX
|
|
|| !(values = wasm_runtime_malloc((uint32)size))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
for (i = 0; i < block_dst->param_count; i++) {
|
|
param_index = block_dst->param_count - 1 - i;
|
|
POP(value, block_dst->param_types[param_index]);
|
|
ADD_TO_PARAM_PHIS(block_dst, value, param_index);
|
|
values[param_index] = value;
|
|
}
|
|
for (i = 0; i < block_dst->param_count; i++) {
|
|
PUSH(values[i], block_dst->param_types[i]);
|
|
}
|
|
wasm_runtime_free(values);
|
|
values = NULL;
|
|
}
|
|
|
|
BUILD_COND_BR(value_cmp, block_dst->llvm_entry_block, llvm_else_block);
|
|
|
|
/* Move builder to else block */
|
|
SET_BUILDER_POS(llvm_else_block);
|
|
}
|
|
else {
|
|
/* Dest block is Block/If/Function block */
|
|
/* Create the end block */
|
|
if (!block_dst->llvm_end_block) {
|
|
format_block_name(name, sizeof(name), block_dst->block_index,
|
|
block_dst->label_type, LABEL_END);
|
|
CREATE_BLOCK(block_dst->llvm_end_block, name);
|
|
if ((next_llvm_end_block = find_next_llvm_end_block(block_dst)))
|
|
MOVE_BLOCK_BEFORE(block_dst->llvm_end_block,
|
|
next_llvm_end_block);
|
|
}
|
|
|
|
/* Set reachable flag and create condition br IR */
|
|
block_dst->is_reachable = true;
|
|
|
|
/* Handle result values */
|
|
if (block_dst->result_count) {
|
|
size = sizeof(LLVMValueRef) * (uint64)block_dst->result_count;
|
|
if (size >= UINT32_MAX
|
|
|| !(values = wasm_runtime_malloc((uint32)size))) {
|
|
aot_set_last_error("allocate memory failed.");
|
|
goto fail;
|
|
}
|
|
CREATE_RESULT_VALUE_PHIS(block_dst);
|
|
for (i = 0; i < block_dst->result_count; i++) {
|
|
result_index = block_dst->result_count - 1 - i;
|
|
POP(value, block_dst->result_types[result_index]);
|
|
values[result_index] = value;
|
|
ADD_TO_RESULT_PHIS(block_dst, value, result_index);
|
|
}
|
|
for (i = 0; i < block_dst->result_count; i++) {
|
|
PUSH(values[i], block_dst->result_types[i]);
|
|
}
|
|
wasm_runtime_free(values);
|
|
values = NULL;
|
|
}
|
|
|
|
/* Condition jump to end block */
|
|
BUILD_COND_BR(value_cmp, block_dst->llvm_end_block, llvm_else_block);
|
|
|
|
/* Move builder to else block */
|
|
SET_BUILDER_POS(llvm_else_block);
|
|
}
|
|
|
|
return true;
|
|
fail:
|
|
if (values)
|
|
wasm_runtime_free(values);
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_br_on_null(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
uint32 br_depth, uint8 **p_frame_ip)
|
|
{
|
|
LLVMValueRef gc_obj, value_cmp;
|
|
|
|
if (!commit_gc_and_check_suspend_flags(comp_ctx, func_ctx, br_depth)) {
|
|
return false;
|
|
}
|
|
|
|
POP_GC_REF(gc_obj);
|
|
|
|
if (!(value_cmp =
|
|
LLVMBuildIsNull(comp_ctx->builder, gc_obj, "cmp_gc_obj"))) {
|
|
aot_set_last_error("llvm build isnull failed.");
|
|
goto fail;
|
|
}
|
|
|
|
if (!compile_gc_cond_br(comp_ctx, func_ctx, br_depth, value_cmp)) {
|
|
goto fail;
|
|
}
|
|
|
|
PUSH_GC_REF(gc_obj);
|
|
return true;
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_br_on_non_null(AOTCompContext *comp_ctx,
|
|
AOTFuncContext *func_ctx, uint32 br_depth,
|
|
uint8 **p_frame_ip)
|
|
{
|
|
LLVMValueRef gc_obj, value_cmp;
|
|
|
|
if (!commit_gc_and_check_suspend_flags(comp_ctx, func_ctx, br_depth)) {
|
|
return false;
|
|
}
|
|
|
|
GET_GC_REF_FROM_STACK(gc_obj);
|
|
|
|
if (!(value_cmp =
|
|
LLVMBuildIsNotNull(comp_ctx->builder, gc_obj, "cmp_gc_obj"))) {
|
|
aot_set_last_error("llvm build isnotnull failed.");
|
|
goto fail;
|
|
}
|
|
|
|
if (!compile_gc_cond_br(comp_ctx, func_ctx, br_depth, value_cmp)) {
|
|
goto fail;
|
|
}
|
|
|
|
POP_GC_REF(gc_obj);
|
|
return true;
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
aot_compile_op_br_on_cast(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
|
int32 heap_type, bool nullable, bool br_on_fail,
|
|
uint32 br_depth, uint8 **p_frame_ip)
|
|
{
|
|
LLVMValueRef gc_obj, is_null, castable, not_castable, br_if_phi;
|
|
LLVMBasicBlockRef block_curr, block_non_null, block_br_if;
|
|
|
|
if (!commit_gc_and_check_suspend_flags(comp_ctx, func_ctx, br_depth)) {
|
|
return false;
|
|
}
|
|
|
|
GET_GC_REF_FROM_STACK(gc_obj);
|
|
|
|
block_curr = CURR_BLOCK();
|
|
|
|
CREATE_BLOCK(block_non_null, "obj_non_null");
|
|
MOVE_BLOCK_AFTER_CURR(block_non_null);
|
|
CREATE_BLOCK(block_br_if, "br_if");
|
|
MOVE_BLOCK_AFTER(block_br_if, block_non_null);
|
|
|
|
SET_BUILDER_POS(block_br_if);
|
|
if (!(br_if_phi =
|
|
LLVMBuildPhi(comp_ctx->builder, INT1_TYPE, "br_if_phi"))) {
|
|
aot_set_last_error("llvm build phi failed.");
|
|
goto fail;
|
|
}
|
|
|
|
SET_BUILDER_POS(block_curr);
|
|
|
|
if (!(is_null = LLVMBuildIsNull(comp_ctx->builder, gc_obj, "is_null"))) {
|
|
aot_set_last_error("llvm build isnull failed.");
|
|
goto fail;
|
|
}
|
|
|
|
BUILD_COND_BR(is_null, block_br_if, block_non_null);
|
|
|
|
if ((!br_on_fail && nullable) || (br_on_fail && !nullable)) {
|
|
LLVMAddIncoming(br_if_phi, &I1_ONE, &block_curr, 1);
|
|
}
|
|
else { /* (!br_on_fail && !nullable) || (br_on_fail && nullable)) */
|
|
LLVMAddIncoming(br_if_phi, &I1_ZERO, &block_curr, 1);
|
|
}
|
|
|
|
SET_BUILDER_POS(block_non_null);
|
|
if (heap_type >= 0) {
|
|
if (!aot_call_aot_obj_is_instance_of(comp_ctx, func_ctx, gc_obj,
|
|
I32_CONST(heap_type), &castable))
|
|
goto fail;
|
|
}
|
|
else {
|
|
if (!aot_call_wasm_obj_is_type_of(comp_ctx, func_ctx, gc_obj,
|
|
I32_CONST(heap_type), &castable))
|
|
goto fail;
|
|
}
|
|
|
|
if (!br_on_fail) {
|
|
if (!(castable = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, castable,
|
|
I8_ZERO, "castable"))) {
|
|
aot_set_last_error("llvm build icmp failed.");
|
|
return false;
|
|
}
|
|
LLVMAddIncoming(br_if_phi, &castable, &block_non_null, 1);
|
|
}
|
|
else {
|
|
if (!(not_castable = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ,
|
|
castable, I8_ZERO, "castable"))) {
|
|
aot_set_last_error("llvm build icmp failed.");
|
|
return false;
|
|
}
|
|
LLVMAddIncoming(br_if_phi, ¬_castable, &block_non_null, 1);
|
|
}
|
|
BUILD_BR(block_br_if);
|
|
|
|
SET_BUILDER_POS(block_br_if);
|
|
if (!compile_gc_cond_br(comp_ctx, func_ctx, br_depth, br_if_phi)) {
|
|
goto fail;
|
|
}
|
|
|
|
return true;
|
|
fail:
|
|
return false;
|
|
}
|
|
|
|
#endif /* End of WASM_ENABLE_GC != 0 */
|