diff --git a/core/iwasm/aot/aot_intrinsic.c b/core/iwasm/aot/aot_intrinsic.c index ba02dca0b..2b4a350b9 100644 --- a/core/iwasm/aot/aot_intrinsic.c +++ b/core/iwasm/aot/aot_intrinsic.c @@ -53,7 +53,8 @@ static const aot_intrinsic g_intrinsic_mapping[] = { { "f64_convert_i64_u", "aot_intrinsic_u64_to_f64", AOT_INTRINSIC_FLAG_U64_TO_F64 }, { "f32_convert_i64_s", "aot_intrinsic_i64_to_f32", AOT_INTRINSIC_FLAG_I64_TO_F32 }, { "f32_convert_i64_u", "aot_intrinsic_u64_to_f32", AOT_INTRINSIC_FLAG_U64_TO_F32 }, - { "i32_trunc_f64_u", "aot_intrinsic_f64_to_u32", AOT_INTRINSIC_FLAG_I32_TO_F64 }, + { "i32_trunc_f64_u", "aot_intrinsic_f64_to_u32", AOT_INTRINSIC_FLAG_F64_TO_U32 }, + { "i32_trunc_f64_s", "aot_intrinsic_f64_to_i32", AOT_INTRINSIC_FLAG_F64_TO_I32 }, { "f32_demote_f64", "aot_intrinsic_f64_to_f32", AOT_INTRINSIC_FLAG_F64_TO_F32 }, { "f64_promote_f32", "aot_intrinsic_f32_to_f64", AOT_INTRINSIC_FLAG_F32_TO_F64 }, { "f32_cmp", "aot_intrinsic_f32_cmp", AOT_INTRINSIC_FLAG_F32_CMP }, diff --git a/core/iwasm/aot/aot_reloc.h b/core/iwasm/aot/aot_reloc.h index bbcac8cbf..65fccedeb 100644 --- a/core/iwasm/aot/aot_reloc.h +++ b/core/iwasm/aot/aot_reloc.h @@ -99,6 +99,7 @@ typedef struct { REG_SYM(aot_intrinsic_i64_to_f64), \ REG_SYM(aot_intrinsic_u64_to_f64), \ REG_SYM(aot_intrinsic_f64_to_f32), \ + REG_SYM(aot_intrinsic_f64_to_i32), \ REG_SYM(aot_intrinsic_f64_to_u32), \ REG_SYM(aot_intrinsic_f32_to_f64), \ REG_SYM(aot_intrinsic_f32_cmp), \ diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index aa4f24c33..0eacfa640 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -2538,6 +2538,39 @@ fail: return false; } +/* Check whether the target supports hardware atomic instructions */ +static bool +aot_require_lower_atomic_pass(AOTCompContext *comp_ctx) +{ + bool ret = false; + if (!strncmp(comp_ctx->target_arch, "riscv", 5)) { + char *feature = + LLVMGetTargetMachineFeatureString(comp_ctx->target_machine); + + if (feature) { + if (!strstr(feature, "+a")) { + ret = true; + } + LLVMDisposeMessage(feature); + } + } + return ret; +} + +/* Check whether the target needs to expand switch to if/else */ +static bool +aot_require_lower_switch_pass(AOTCompContext *comp_ctx) +{ + bool ret = false; + + /* IR switch/case will cause .rodata relocation on riscv */ + if (!strncmp(comp_ctx->target_arch, "riscv", 5)) { + ret = true; + } + + return ret; +} + bool aot_compile_wasm(AOTCompContext *comp_ctx) { @@ -2625,6 +2658,27 @@ aot_compile_wasm(AOTCompContext *comp_ctx) LLVMPassManagerBuilderDispose(pass_mgr_builder); } + if (comp_ctx->optimize && comp_ctx->is_indirect_mode) { + LLVMPassManagerRef common_pass_mgr = NULL; + + if (!(common_pass_mgr = LLVMCreatePassManager())) { + aot_set_last_error("create pass manager failed"); + return false; + } + + aot_add_expand_memory_op_pass(common_pass_mgr); + + if (aot_require_lower_atomic_pass(comp_ctx)) + LLVMAddLowerAtomicPass(common_pass_mgr); + + if (aot_require_lower_switch_pass(comp_ctx)) + LLVMAddLowerSwitchPass(common_pass_mgr); + + LLVMRunPassManager(common_pass_mgr, comp_ctx->module); + + LLVMDisposePassManager(common_pass_mgr); + } + return true; } diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 86ddb3c6f..c089f426e 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -6,6 +6,7 @@ #include "aot_emit_memory.h" #include "aot_emit_exception.h" #include "../aot/aot_runtime.h" +#include "aot_intrinsic.h" #define BUILD_ICMP(op, left, right, res, name) \ do { \ @@ -951,13 +952,66 @@ aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) if (!(dst_addr = check_bulk_memory_overflow(comp_ctx, func_ctx, dst, len))) return false; - /* TODO: lookup func ptr of "memmove" to call for XIP mode */ + if (comp_ctx->is_indirect_mode) { + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + LLVMValueRef func, params[3]; + int32 func_idx; - if (!(res = LLVMBuildMemMove(comp_ctx->builder, dst_addr, 1, src_addr, 1, - len))) { - aot_set_last_error("llvm build memmove failed."); - return false; + if (!(dst_addr = + LLVMBuildBitCast(comp_ctx->builder, dst_addr, INT32_PTR_TYPE, + "memmove dst addr cast type"))) { + aot_set_last_error("llvm cast memmove dst addr type failed."); + return false; + } + + if (!(src_addr = + LLVMBuildBitCast(comp_ctx->builder, src_addr, INT32_PTR_TYPE, + "memmove src addr cast type"))) { + aot_set_last_error("llvm cast memmove src addr type failed."); + return false; + } + + param_types[0] = INT32_PTR_TYPE; + param_types[1] = INT32_PTR_TYPE; + param_types[2] = I32_TYPE; + ret_type = INT32_PTR_TYPE; + + if (!(func_type = LLVMFunctionType(ret_type, param_types, 3, false))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function pointer type failed."); + return false; + } + + func_idx = aot_get_native_symbol_index(comp_ctx, "memmove"); + if (func_idx < 0) { + return false; + } + if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_ptr_type, func_idx))) { + return false; + } + + params[0] = dst_addr; + params[1] = src_addr; + params[2] = len; + if (!(res = LLVMBuildCall(comp_ctx->builder, func, params, 3, + "call memmove"))) { + aot_set_last_error("llvm build memmove failed."); + return false; + } } + else { + if (!(res = LLVMBuildMemMove(comp_ctx->builder, dst_addr, 1, src_addr, + 1, len))) { + aot_set_last_error("llvm build memmove failed."); + return false; + } + } + return true; fail: return false; @@ -981,11 +1035,57 @@ aot_compile_op_memory_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) return false; } - /* TODO: lookup func ptr of "memset" to call for XIP mode */ + if (comp_ctx->is_indirect_mode) { + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + LLVMValueRef func, params[3]; + int32 func_idx; - if (!(res = LLVMBuildMemSet(comp_ctx->builder, dst_addr, val, len, 1))) { - aot_set_last_error("llvm build memset failed."); - return false; + if (!(dst_addr = + LLVMBuildBitCast(comp_ctx->builder, dst_addr, INT32_PTR_TYPE, + "memset dst addr cast type"))) { + aot_set_last_error("llvm cast memset dst addr type failed."); + return false; + } + + param_types[0] = INT32_PTR_TYPE; + param_types[1] = INT8_TYPE; + param_types[2] = I32_TYPE; + ret_type = INT32_PTR_TYPE; + + if (!(func_type = LLVMFunctionType(ret_type, param_types, 3, false))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function pointer type failed."); + return false; + } + + func_idx = aot_get_native_symbol_index(comp_ctx, "memset"); + if (func_idx < 0) { + return false; + } + if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_ptr_type, func_idx))) { + return false; + } + + params[0] = dst_addr; + params[1] = val; + params[2] = len; + if (!(res = LLVMBuildCall(comp_ctx->builder, func, params, 3, + "call memset"))) { + aot_set_last_error("llvm build memset failed."); + return false; + } + } + else { + if (!(res = + LLVMBuildMemSet(comp_ctx->builder, dst_addr, val, len, 1))) { + aot_set_last_error("llvm build memset failed."); + return false; + } } return true; fail: @@ -1127,16 +1227,20 @@ aot_compile_op_atomic_cmpxchg(AOTCompContext *comp_ctx, } if (op_type == VALUE_TYPE_I32) { - if (!(result = LLVMBuildZExt(comp_ctx->builder, result, I32_TYPE, - "result_i32"))) { - goto fail; + if (LLVMTypeOf(result) != I32_TYPE) { + if (!(result = LLVMBuildZExt(comp_ctx->builder, result, I32_TYPE, + "result_i32"))) { + goto fail; + } } PUSH_I32(result); } else { - if (!(result = LLVMBuildZExt(comp_ctx->builder, result, I64_TYPE, - "result_i64"))) { - goto fail; + if (LLVMTypeOf(result) != I64_TYPE) { + if (!(result = LLVMBuildZExt(comp_ctx->builder, result, I64_TYPE, + "result_i64"))) { + goto fail; + } } PUSH_I64(result); } diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index ebd170339..dd29cae34 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -445,6 +445,9 @@ aot_get_func_from_table(const AOTCompContext *comp_ctx, LLVMValueRef base, bool aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str); +void +aot_add_expand_memory_op_pass(LLVMPassManagerRef pass); + #if WASM_ENABLE_LAZY_JIT != 0 void aot_handle_llvm_errmsg(char *error_buf, uint32 error_buf_size, diff --git a/core/iwasm/compilation/aot_llvm_extra.cpp b/core/iwasm/compilation/aot_llvm_extra.cpp index 5ca2a6173..c6a059c60 100644 --- a/core/iwasm/compilation/aot_llvm_extra.cpp +++ b/core/iwasm/compilation/aot_llvm_extra.cpp @@ -6,20 +6,29 @@ #include #include #include +#include +#include #include #include #include #include #include #include +#include #include #include #include #include #include +#include +#include +#include +#include #include #include +#include #include +#include #include using namespace llvm; @@ -33,6 +42,9 @@ WAMRCreateMCJITCompilerForModule(LLVMExecutionEngineRef *OutJIT, extern "C" bool aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str); +extern "C" void +aot_add_expand_memory_op_pass(LLVMPassManagerRef pass); + extern "C" void aot_func_disable_tce(LLVMValueRef func); @@ -106,6 +118,109 @@ WAMRCreateMCJITCompilerForModule(LLVMExecutionEngineRef *OutJIT, return 1; } +class ExpandMemoryOpPass : public llvm::ModulePass +{ + public: + static char ID; + + ExpandMemoryOpPass() + : ModulePass(ID) + {} + + bool runOnModule(Module &M) override; + + bool expandMemIntrinsicUses(Function &F); + StringRef getPassName() const override + { + return "Expand memory operation intrinsics"; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override + { + AU.addRequired(); + } +}; + +char ExpandMemoryOpPass::ID = 0; + +bool +ExpandMemoryOpPass::expandMemIntrinsicUses(Function &F) +{ + Intrinsic::ID ID = F.getIntrinsicID(); + bool Changed = false; + + for (auto I = F.user_begin(), E = F.user_end(); I != E;) { + Instruction *Inst = cast(*I); + ++I; + + switch (ID) { + case Intrinsic::memcpy: + { + auto *Memcpy = cast(Inst); + Function *ParentFunc = Memcpy->getParent()->getParent(); + const TargetTransformInfo &TTI = + getAnalysis().getTTI( + *ParentFunc); + expandMemCpyAsLoop(Memcpy, TTI); + Changed = true; + Memcpy->eraseFromParent(); + break; + } + case Intrinsic::memmove: + { + auto *Memmove = cast(Inst); + expandMemMoveAsLoop(Memmove); + Changed = true; + Memmove->eraseFromParent(); + break; + } + case Intrinsic::memset: + { + auto *Memset = cast(Inst); + expandMemSetAsLoop(Memset); + Changed = true; + Memset->eraseFromParent(); + break; + } + default: + break; + } + } + + return Changed; +} + +bool +ExpandMemoryOpPass::runOnModule(Module &M) +{ + bool Changed = false; + + for (Function &F : M) { + if (!F.isDeclaration()) + continue; + + switch (F.getIntrinsicID()) { + case Intrinsic::memcpy: + case Intrinsic::memmove: + case Intrinsic::memset: + if (expandMemIntrinsicUses(F)) + Changed = true; + break; + + default: + break; + } + } + + return Changed; +} + +void +aot_add_expand_memory_op_pass(LLVMPassManagerRef pass) +{ + unwrap(pass)->add(new ExpandMemoryOpPass()); +} + bool aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str) { diff --git a/core/iwasm/compilation/iwasm_compl.cmake b/core/iwasm/compilation/iwasm_compl.cmake index 59455de9a..4ec460304 100644 --- a/core/iwasm/compilation/iwasm_compl.cmake +++ b/core/iwasm/compilation/iwasm_compl.cmake @@ -16,3 +16,11 @@ endif() set (IWASM_COMPL_SOURCE ${source_all}) +# Disalbe rtti to works with LLVM + +if (MSVC) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-") +else() + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") +endif() +