From b21f17dd6d6df5a1b0d608d1ec9f86f6ec631abb Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 10 Jan 2024 18:37:02 +0800 Subject: [PATCH] Refine AOT/JIT code call wasm-c-api import process (#2982) Allow to invoke the quick call entry wasm_runtime_quick_invoke_c_api_import to call the wasm-c-api import functions to speedup the calling process, which reduces the data copying. Use `wamrc --invoke-c-api-import` to generate the optimized AOT code, and set `jit_options->quick_invoke_c_api_import` true in wasm_engine_new when LLVM JIT is enabled. --- core/iwasm/aot/aot_reloc.h | 1 + core/iwasm/aot/aot_runtime.c | 7 + core/iwasm/common/wasm_c_api.c | 7 + core/iwasm/common/wasm_runtime_common.c | 78 +++++- core/iwasm/common/wasm_runtime_common.h | 13 +- core/iwasm/compilation/aot_emit_function.c | 258 +++++++++++++++++- core/iwasm/compilation/aot_emit_table.c | 12 + core/iwasm/compilation/aot_emit_table.h | 5 +- core/iwasm/compilation/aot_llvm.c | 3 + core/iwasm/compilation/aot_llvm.h | 5 + core/iwasm/include/aot_export.h | 1 + core/iwasm/include/wasm_c_api.h | 1 + core/iwasm/include/wasm_export.h | 1 + core/iwasm/interpreter/wasm_loader.c | 11 +- core/iwasm/interpreter/wasm_mini_loader.c | 8 +- core/iwasm/interpreter/wasm_runtime.h | 2 +- product-mini/platforms/posix/main.c | 2 +- product-mini/platforms/windows/main.c | 2 +- .../wasm-c-api-imports/wasm/CMakeLists.txt | 2 +- wamr-compiler/main.c | 9 +- 20 files changed, 393 insertions(+), 35 deletions(-) diff --git a/core/iwasm/aot/aot_reloc.h b/core/iwasm/aot/aot_reloc.h index 98df09cb4..797eacc11 100644 --- a/core/iwasm/aot/aot_reloc.h +++ b/core/iwasm/aot/aot_reloc.h @@ -136,6 +136,7 @@ typedef struct { REG_SYM(aot_enlarge_memory), \ REG_SYM(aot_set_exception), \ REG_SYM(aot_check_app_addr_and_convert),\ + REG_SYM(wasm_runtime_quick_invoke_c_api_native),\ { "memset", (void*)aot_memset }, \ { "memmove", (void*)aot_memmove }, \ { "memcpy", (void*)aot_memmove }, \ diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index ac83d5c2d..0fa4f7939 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -47,6 +47,13 @@ bh_static_assert(sizeof(AOTMemoryInstance) == 104); bh_static_assert(offsetof(AOTTableInstance, elems) == 8); bh_static_assert(offsetof(AOTModuleInstanceExtra, stack_sizes) == 0); +bh_static_assert(offsetof(AOTModuleInstanceExtra, common.c_api_func_imports) + == sizeof(uint64)); + +bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3); + +bh_static_assert(sizeof(wasm_val_t) == 16); +bh_static_assert(offsetof(wasm_val_t, of) == 8); static void set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) diff --git a/core/iwasm/common/wasm_c_api.c b/core/iwasm/common/wasm_c_api.c index ff88fae50..41cfdf377 100644 --- a/core/iwasm/common/wasm_c_api.c +++ b/core/iwasm/common/wasm_c_api.c @@ -378,6 +378,9 @@ wasm_engine_new_internal(wasm_config_t *config) wasm_engine_t *engine = NULL; /* init runtime */ RuntimeInitArgs init_args = { 0 }; +#if WASM_ENABLE_JIT != 0 + LLVMJITOptions *jit_options = wasm_runtime_get_llvm_jit_options(); +#endif #ifndef NDEBUG bh_log_set_verbose_level(BH_LOG_LEVEL_VERBOSE); @@ -394,6 +397,10 @@ wasm_engine_new_internal(wasm_config_t *config) init_args.enable_linux_perf = config->enable_linux_perf; init_args.segue_flags = config->segue_flags; +#if WASM_ENABLE_JIT != 0 + jit_options->quick_invoke_c_api_import = true; +#endif + if (!wasm_runtime_full_init(&init_args)) { LOG_DEBUG("wasm_runtime_full_init failed"); goto failed; diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index ffc0429d7..e245cf50b 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -158,7 +158,9 @@ static JitCompOptions jit_options = { 0 }; #endif #if WASM_ENABLE_JIT != 0 -static LLVMJITOptions llvm_jit_options = { 3, 3, 0 }; +/* opt_level: 3, size_level: 3, segue-flags: 0, + quick_invoke_c_api_import: false */ +static LLVMJITOptions llvm_jit_options = { 3, 3, 0, false }; #endif static RunningMode runtime_running_mode = Mode_Default; @@ -638,10 +640,10 @@ wasm_runtime_get_default_running_mode(void) } #if WASM_ENABLE_JIT != 0 -LLVMJITOptions +LLVMJITOptions * wasm_runtime_get_llvm_jit_options(void) { - return llvm_jit_options; + return &llvm_jit_options; } #endif @@ -5675,7 +5677,7 @@ wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, wasm_val_t *params = params_buf, *results = results_buf; wasm_trap_t *trap = NULL; bool ret = false; - wasm_val_vec_t params_vec, results_vec; + wasm_val_vec_t params_vec = { 0 }, results_vec = { 0 }; if (func_type->param_count > 16) { if (!(params = @@ -5703,12 +5705,10 @@ wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, params_vec.data = params; params_vec.num_elems = func_type->param_count; params_vec.size = func_type->param_count; - params_vec.size_of_elem = sizeof(wasm_val_t); results_vec.data = results; results_vec.num_elems = 0; results_vec.size = func_type->result_count; - results_vec.size_of_elem = sizeof(wasm_val_t); if (!with_env) { wasm_func_callback_t callback = (wasm_func_callback_t)func_ptr; @@ -5744,7 +5744,6 @@ wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, wasm_runtime_set_exception(module_inst, "unsupported result type"); goto fail; } - results_vec.num_elems = func_type->result_count; ret = true; fail: @@ -5755,6 +5754,71 @@ fail: return ret; } +bool +wasm_runtime_quick_invoke_c_api_native(WASMModuleInstanceCommon *inst_comm, + CApiFuncImport *c_api_import, + wasm_val_t *params, uint32 param_count, + wasm_val_t *results, uint32 result_count) +{ + WASMModuleInstance *module_inst = (WASMModuleInstance *)inst_comm; + void *func_ptr = c_api_import->func_ptr_linked; + bool with_env_arg = c_api_import->with_env_arg, ret = true; + wasm_val_vec_t params_vec = { 0 }, results_vec = { 0 }; + wasm_trap_t *trap = NULL; + + params_vec.data = params; + params_vec.num_elems = param_count; + params_vec.size = param_count; + + results_vec.data = results; + results_vec.num_elems = 0; + results_vec.size = result_count; + + if (!func_ptr) { + wasm_set_exception_with_id(module_inst, EXCE_CALL_UNLINKED_IMPORT_FUNC); + ret = false; + goto fail; + } + + if (!with_env_arg) { + wasm_func_callback_t callback = (wasm_func_callback_t)func_ptr; + trap = callback(¶ms_vec, &results_vec); + } + else { + void *wasm_c_api_env = c_api_import->env_arg; + wasm_func_callback_with_env_t callback = + (wasm_func_callback_with_env_t)func_ptr; + trap = callback(wasm_c_api_env, ¶ms_vec, &results_vec); + } + + if (trap) { + if (trap->message->data) { + /* since trap->message->data does not end with '\0' */ + char trap_message[108] = { 0 }; + uint32 max_size_to_copy = (uint32)sizeof(trap_message) - 1; + uint32 size_to_copy = (trap->message->size < max_size_to_copy) + ? (uint32)trap->message->size + : max_size_to_copy; + bh_memcpy_s(trap_message, (uint32)sizeof(trap_message), + trap->message->data, size_to_copy); + wasm_set_exception(module_inst, trap_message); + } + else { + wasm_set_exception(module_inst, + "native function throw unknown exception"); + } + wasm_trap_delete(trap); + ret = false; + } + +fail: +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + return ret; +} + void wasm_runtime_show_app_heap_corrupted_prompt() { diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index b113ec0d7..606eea019 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -443,6 +443,7 @@ typedef struct LLVMJITOptions { uint32 opt_level; uint32 size_level; uint32 segue_flags; + bool quick_invoke_c_api_import; } LLVMJITOptions; #endif @@ -476,7 +477,7 @@ wasm_runtime_get_default_running_mode(void); #if WASM_ENABLE_JIT != 0 /* Internal API */ -LLVMJITOptions +LLVMJITOptions * wasm_runtime_get_llvm_jit_options(void); #endif @@ -1079,6 +1080,16 @@ wasm_runtime_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, uint32 argc, uint32 *argv, bool with_env, void *wasm_c_api_env); +struct CApiFuncImport; +/* A quick version of wasm_runtime_invoke_c_api_native to directly invoke + wasm-c-api import function from jitted code to improve performance */ +bool +wasm_runtime_quick_invoke_c_api_native(WASMModuleInstanceCommon *module_inst, + struct CApiFuncImport *c_api_import, + wasm_val_t *params, uint32 param_count, + wasm_val_t *results, + uint32 result_count); + void wasm_runtime_show_app_heap_corrupted_prompt(); diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index bc8cb0a55..4fb072401 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -288,6 +288,213 @@ call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } +static bool +call_aot_invoke_c_api_native(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 import_func_idx, AOTFuncType *aot_func_type, + LLVMValueRef *params) +{ + LLVMTypeRef int8_ptr_type, param_types[6], ret_type; + LLVMTypeRef value_ptr_type = NULL, value_type = NULL; + LLVMTypeRef func_type, func_ptr_type; + LLVMValueRef param_values[6], res, func, value = NULL, offset; + LLVMValueRef c_api_func_imports, c_api_func_import; + LLVMValueRef c_api_params, c_api_results, value_ret; + LLVMValueRef c_api_param_kind, c_api_param_value; + LLVMValueRef c_api_result_value; + uint32 offset_c_api_func_imports, i; + uint32 offset_param_kind, offset_param_value; + char buf[16]; + + /* `int8 **` type */ + int8_ptr_type = LLVMPointerType(INT8_PTR_TYPE, 0); + if (!int8_ptr_type) { + aot_set_last_error("create llvm pointer type failed"); + return false; + } + + param_types[0] = INT8_PTR_TYPE; /* module_inst */ + param_types[1] = INT8_PTR_TYPE; /* CApiFuncImport *c_api_import */ + param_types[2] = INT8_PTR_TYPE; /* wasm_val_t *params */ + param_types[3] = I32_TYPE; /* uint32 param_count */ + param_types[4] = INT8_PTR_TYPE; /* wasm_val_t *results */ + param_types[5] = I32_TYPE; /* uint32 result_count */ + + ret_type = INT8_TYPE; + + GET_AOT_FUNCTION(wasm_runtime_quick_invoke_c_api_native, 6); + + param_values[0] = func_ctx->aot_inst; + + /* Get module_inst->e->common.c_api_func_imports */ + offset_c_api_func_imports = + get_module_inst_extra_offset(comp_ctx) + + (comp_ctx->is_jit_mode + ? offsetof(WASMModuleInstanceExtra, common.c_api_func_imports) + /* offsetof(AOTModuleInstanceExtra, common.c_api_func_imports) */ + : sizeof(uint64)); + offset = I32_CONST(offset_c_api_func_imports); + CHECK_LLVM_CONST(offset); + c_api_func_imports = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, + &offset, 1, "c_api_func_imports_addr"); + c_api_func_imports = + LLVMBuildBitCast(comp_ctx->builder, c_api_func_imports, int8_ptr_type, + "c_api_func_imports_ptr"); + c_api_func_imports = + LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, c_api_func_imports, + "c_api_func_imports"); + + /* Get &c_api_func_imports[func_idx], note size of CApiFuncImport + is pointer_size * 3 */ + offset = I32_CONST((comp_ctx->pointer_size * 3) * import_func_idx); + CHECK_LLVM_CONST(offset); + c_api_func_import = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, c_api_func_imports, + &offset, 1, "c_api_func_import"); + + param_values[1] = c_api_func_import; + param_values[2] = c_api_params = func_ctx->argv_buf; + param_values[3] = I32_CONST(aot_func_type->param_count); + CHECK_LLVM_CONST(param_values[3]); + + /* Ensure sizeof(wasm_val_t) is 16 bytes */ + offset = I32_CONST(sizeof(wasm_val_t) * aot_func_type->param_count); + c_api_results = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->argv_buf, + &offset, 1, "results"); + param_values[4] = c_api_results; + + param_values[5] = I32_CONST(aot_func_type->result_count); + CHECK_LLVM_CONST(param_values[5]); + + /* Set each c api param */ + for (i = 0; i < aot_func_type->param_count; i++) { + /* Ensure sizeof(wasm_val_t) is 16 bytes */ + offset_param_kind = sizeof(wasm_val_t) * i; + offset = I32_CONST(offset_param_kind); + CHECK_LLVM_CONST(offset); + c_api_param_kind = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, c_api_params, + &offset, 1, "c_api_param_kind_addr"); + c_api_param_kind = + LLVMBuildBitCast(comp_ctx->builder, c_api_param_kind, INT8_PTR_TYPE, + "c_api_param_kind_ptr"); + + switch (aot_func_type->types[i]) { + case VALUE_TYPE_I32: + value = I8_CONST(WASM_I32); + break; + case VALUE_TYPE_F32: + value = I8_CONST(WASM_F32); + break; + case VALUE_TYPE_I64: + value = I8_CONST(WASM_I64); + break; + case VALUE_TYPE_F64: + value = I8_CONST(WASM_F64); + break; + default: + bh_assert(0); + break; + } + CHECK_LLVM_CONST(value); + + LLVMBuildStore(comp_ctx->builder, value, c_api_param_kind); + + /* Ensure offsetof(wasm_val_t, of) is 8 bytes */ + offset_param_value = offset_param_kind + offsetof(wasm_val_t, of); + offset = I32_CONST(offset_param_value); + CHECK_LLVM_CONST(offset); + c_api_param_value = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, c_api_params, + &offset, 1, "c_api_param_value_addr"); + + switch (aot_func_type->types[i]) { + case VALUE_TYPE_I32: + value_ptr_type = INT32_PTR_TYPE; + break; + case VALUE_TYPE_F32: + value_ptr_type = F32_PTR_TYPE; + break; + case VALUE_TYPE_I64: + value_ptr_type = INT64_PTR_TYPE; + break; + case VALUE_TYPE_F64: + value_ptr_type = F64_PTR_TYPE; + break; + default: + bh_assert(0); + break; + } + + c_api_param_value = + LLVMBuildBitCast(comp_ctx->builder, c_api_param_value, + value_ptr_type, "c_api_param_value_ptr"); + LLVMBuildStore(comp_ctx->builder, params[i], c_api_param_value); + } + + /* Call the function */ + if (!(res = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 6, "call"))) { + aot_set_last_error("LLVM build call failed."); + goto fail; + } + + /* Check whether exception was thrown when executing the function */ + if (comp_ctx->enable_bound_check + && !check_call_return(comp_ctx, func_ctx, res)) { + goto fail; + } + + for (i = 0; i < aot_func_type->result_count; i++) { + /* Ensure sizeof(wasm_val_t) is 16 bytes and + offsetof(wasm_val_t, of) is 8 bytes */ + uint32 offset_result_value = + sizeof(wasm_val_t) * i + offsetof(wasm_val_t, of); + + offset = I32_CONST(offset_result_value); + CHECK_LLVM_CONST(offset); + c_api_result_value = + LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, c_api_results, + &offset, 1, "c_api_result_value_addr"); + + switch (aot_func_type->types[aot_func_type->param_count + i]) { + case VALUE_TYPE_I32: + value_type = I32_TYPE; + value_ptr_type = INT32_PTR_TYPE; + break; + case VALUE_TYPE_F32: + value_type = F32_TYPE; + value_ptr_type = F32_PTR_TYPE; + break; + case VALUE_TYPE_I64: + value_type = I64_TYPE; + value_ptr_type = INT64_PTR_TYPE; + break; + case VALUE_TYPE_F64: + value_type = F64_TYPE; + value_ptr_type = F64_PTR_TYPE; + break; + default: + bh_assert(0); + break; + } + + c_api_result_value = + LLVMBuildBitCast(comp_ctx->builder, c_api_result_value, + value_ptr_type, "c_api_result_value_ptr"); + snprintf(buf, sizeof(buf), "%s%u", "ret", i); + value_ret = LLVMBuildLoad2(comp_ctx->builder, value_type, + c_api_result_value, buf); + + PUSH(value_ret, aot_func_type->types[aot_func_type->param_count + i]); + } + + return true; +fail: + return false; +} + #if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0) static bool call_aot_alloc_frame_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, @@ -533,6 +740,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, const char *signature = NULL; bool ret = false; char buf[32]; + bool quick_invoke_c_api_import = false; #if WASM_ENABLE_THREAD_MGR != 0 /* Insert suspend check point */ @@ -702,17 +910,43 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } if (!signature) { - /* call aot_invoke_native() */ - if (!call_aot_invoke_native_func( - comp_ctx, func_ctx, import_func_idx, func_type, - param_types + 1, param_values + 1, param_count, - param_cell_num, ret_type, wasm_ret_type, &value_ret, &res)) - goto fail; - /* Check whether there was exception thrown when executing - the function */ - if ((comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) - && !check_call_return(comp_ctx, func_ctx, res)) - goto fail; + if (comp_ctx->quick_invoke_c_api_import) { + uint32 buf_size_needed = + sizeof(wasm_val_t) * (param_count + result_count); + + /* length of exec_env->argv_buf is 64 */ + if (buf_size_needed < sizeof(uint32) * 64) { + for (i = 0; i < param_count + result_count; i++) { + /* Only support i32/i64/f32/f64 now */ + if (!(func_type->types[i] == VALUE_TYPE_I32 + || func_type->types[i] == VALUE_TYPE_I64 + || func_type->types[i] == VALUE_TYPE_F32 + || func_type->types[i] == VALUE_TYPE_F64)) + break; + } + if (i == param_count + result_count) + quick_invoke_c_api_import = true; + } + } + if (quick_invoke_c_api_import) { + if (!call_aot_invoke_c_api_native(comp_ctx, func_ctx, func_idx, + func_type, param_values + 1)) + goto fail; + } + else { + /* call aot_invoke_native() */ + if (!call_aot_invoke_native_func( + comp_ctx, func_ctx, import_func_idx, func_type, + param_types + 1, param_values + 1, param_count, + param_cell_num, ret_type, wasm_ret_type, &value_ret, + &res)) + goto fail; + /* Check whether there was exception thrown when executing + the function */ + if ((comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) + && !check_call_return(comp_ctx, func_ctx, res)) + goto fail; + } } else { /* call native func directly */ LLVMTypeRef native_func_type, func_ptr_type; @@ -869,7 +1103,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - if (func_type->result_count > 0) { + if (func_type->result_count > 0 && !quick_invoke_c_api_import) { /* Push the first result to stack */ PUSH(value_ret, func_type->types[func_type->param_count]); /* Load extra result from its address and push to stack */ diff --git a/core/iwasm/compilation/aot_emit_table.c b/core/iwasm/compilation/aot_emit_table.c index d8a5efd91..34e7d9700 100644 --- a/core/iwasm/compilation/aot_emit_table.c +++ b/core/iwasm/compilation/aot_emit_table.c @@ -46,6 +46,18 @@ get_tbl_inst_offset(const AOTCompContext *comp_ctx, return offset; } +uint32 +get_module_inst_extra_offset(AOTCompContext *comp_ctx) +{ + const AOTCompData *comp_data = comp_ctx->comp_data; + uint32 table_count = comp_data->import_table_count + comp_data->table_count; + uint64 offset = get_tbl_inst_offset(comp_ctx, NULL, table_count); + uint32 offset_32 = (uint32)offset; + bh_assert(offset <= UINT32_MAX); + offset_32 = align_uint(offset_32, 8); + return offset_32; +} + #if WASM_ENABLE_REF_TYPES != 0 LLVMValueRef diff --git a/core/iwasm/compilation/aot_emit_table.h b/core/iwasm/compilation/aot_emit_table.h index e5ab0ed48..f294cca9a 100644 --- a/core/iwasm/compilation/aot_emit_table.h +++ b/core/iwasm/compilation/aot_emit_table.h @@ -49,6 +49,9 @@ uint64 get_tbl_inst_offset(const AOTCompContext *comp_ctx, const AOTFuncContext *func_ctx, uint32 tbl_idx); +uint32 +get_module_inst_extra_offset(AOTCompContext *comp_ctx); + LLVMValueRef aot_compile_get_tbl_inst(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 tbl_idx); @@ -56,4 +59,4 @@ aot_compile_get_tbl_inst(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, #ifdef __cplusplus } /* end of extern "C" */ #endif -#endif \ No newline at end of file +#endif diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 9119998e0..6f5cddcd9 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -2398,6 +2398,9 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) if (option->enable_stack_estimation) comp_ctx->enable_stack_estimation = true; + if (option->quick_invoke_c_api_import) + comp_ctx->quick_invoke_c_api_import = true; + if (option->llvm_passes) comp_ctx->llvm_passes = option->llvm_passes; diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 0a257f165..bb7534e5c 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -357,6 +357,10 @@ typedef struct AOTCompContext { /* Enable LLVM PGO (Profile-Guided Optimization) */ bool enable_llvm_pgo; + /* Treat unknown import function as wasm-c-api import function + and allow to directly invoke it from AOT/JIT code */ + bool quick_invoke_c_api_import; + /* Use profile file collected by LLVM PGO */ char *use_prof_file; @@ -454,6 +458,7 @@ typedef struct AOTCompOption { bool disable_llvm_lto; bool enable_llvm_pgo; bool enable_stack_estimation; + bool quick_invoke_c_api_import; char *use_prof_file; uint32 opt_level; uint32 size_level; diff --git a/core/iwasm/include/aot_export.h b/core/iwasm/include/aot_export.h index 440ce03ea..e1837e64f 100644 --- a/core/iwasm/include/aot_export.h +++ b/core/iwasm/include/aot_export.h @@ -58,6 +58,7 @@ typedef struct AOTCompOption { bool disable_llvm_lto; bool enable_llvm_pgo; bool enable_stack_estimation; + bool quick_invoke_c_api_import; char *use_prof_file; uint32_t opt_level; uint32_t size_level; diff --git a/core/iwasm/include/wasm_c_api.h b/core/iwasm/include/wasm_c_api.h index d13a712c4..1d49376e0 100644 --- a/core/iwasm/include/wasm_c_api.h +++ b/core/iwasm/include/wasm_c_api.h @@ -417,6 +417,7 @@ struct wasm_ref_t; typedef struct wasm_val_t { wasm_valkind_t kind; + uint8_t __paddings[7]; union { int32_t i32; int64_t i64; diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 0934f7b48..c27b52339 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -199,6 +199,7 @@ struct wasm_ref_t; typedef struct wasm_val_t { wasm_valkind_t kind; + uint8_t __paddings[7]; union { /* also represent a function index */ int32_t i32; diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 85f402a0b..4f23542aa 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -2852,7 +2852,7 @@ static bool init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, uint32 error_buf_size) { - LLVMJITOptions llvm_jit_options = wasm_runtime_get_llvm_jit_options(); + LLVMJITOptions *llvm_jit_options = wasm_runtime_get_llvm_jit_options(); AOTCompOption option = { 0 }; char *aot_last_error; uint64 size; @@ -2892,10 +2892,11 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, option.is_jit_mode = true; - llvm_jit_options = wasm_runtime_get_llvm_jit_options(); - option.opt_level = llvm_jit_options.opt_level; - option.size_level = llvm_jit_options.size_level; - option.segue_flags = llvm_jit_options.segue_flags; + option.opt_level = llvm_jit_options->opt_level; + option.size_level = llvm_jit_options->size_level; + option.segue_flags = llvm_jit_options->segue_flags; + option.quick_invoke_c_api_import = + llvm_jit_options->quick_invoke_c_api_import; #if WASM_ENABLE_BULK_MEMORY != 0 option.enable_bulk_memory = true; diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index eb77e0c53..b6d81aa2f 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -1878,9 +1878,11 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, } option.is_jit_mode = true; - option.opt_level = llvm_jit_options.opt_level; - option.size_level = llvm_jit_options.size_level; - option.segue_flags = llvm_jit_options.segue_flags; + option.opt_level = llvm_jit_options->opt_level; + option.size_level = llvm_jit_options->size_level; + option.segue_flags = llvm_jit_options->segue_flags; + option.quick_invoke_c_api_import = + llvm_jit_options->quick_invoke_c_api_import; #if WASM_ENABLE_BULK_MEMORY != 0 option.enable_bulk_memory = true; diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index bb5fdc80b..89b92b658 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -221,8 +221,8 @@ typedef struct CApiFuncImport { /* The common part of WASMModuleInstanceExtra and AOTModuleInstanceExtra */ typedef struct WASMModuleInstanceExtraCommon { - void *contexts[WASM_MAX_INSTANCE_CONTEXTS]; CApiFuncImport *c_api_func_imports; + void *contexts[WASM_MAX_INSTANCE_CONTEXTS]; /* pointer to the exec env currently used */ WASMExecEnv *cur_exec_env; #if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 diff --git a/product-mini/platforms/posix/main.c b/product-mini/platforms/posix/main.c index acf8d81df..89aef5f91 100644 --- a/product-mini/platforms/posix/main.c +++ b/product-mini/platforms/posix/main.c @@ -767,7 +767,7 @@ main(int argc, char *argv[]) gen_prof_file = argv[0] + 16; } #endif - else if (!strncmp(argv[0], "--version", 9)) { + else if (!strcmp(argv[0], "--version")) { uint32 major, minor, patch; wasm_runtime_get_version(&major, &minor, &patch); printf("iwasm %" PRIu32 ".%" PRIu32 ".%" PRIu32 "\n", major, minor, diff --git a/product-mini/platforms/windows/main.c b/product-mini/platforms/windows/main.c index 921b77265..6461e9172 100644 --- a/product-mini/platforms/windows/main.c +++ b/product-mini/platforms/windows/main.c @@ -406,7 +406,7 @@ main(int argc, char *argv[]) ip_addr = argv[0] + 3; } #endif - else if (!strncmp(argv[0], "--version", 9)) { + else if (!strcmp(argv[0], "--version")) { uint32 major, minor, patch; wasm_runtime_get_version(&major, &minor, &patch); printf("iwasm %" PRIu32 ".%" PRIu32 ".%" PRIu32 "\n", major, minor, diff --git a/samples/wasm-c-api-imports/wasm/CMakeLists.txt b/samples/wasm-c-api-imports/wasm/CMakeLists.txt index 6b2743cb5..bf2358444 100644 --- a/samples/wasm-c-api-imports/wasm/CMakeLists.txt +++ b/samples/wasm-c-api-imports/wasm/CMakeLists.txt @@ -31,7 +31,7 @@ target_link_options(send_recv PRIVATE if(WASM_TO_AOT) # wasm -> aot add_custom_target(send_recv_aot ALL - COMMAND pwd && ${WAMRC_PATH} --enable-multi-thread -o ./send_recv.aot ./send_recv.wasm + COMMAND pwd && ${WAMRC_PATH} --invoke-c-api-import --enable-multi-thread -o ./send_recv.aot ./send_recv.wasm DEPENDS send_recv WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index 2eedebae2..f2e5efafd 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -188,6 +188,8 @@ print_help() printf(" are shared object (.so) files, for example:\n"); printf(" --native-lib=test1.so --native-lib=test2.so\n"); #endif + printf(" --invoke-c-api-import Treat unknown import function as wasm-c-api import function and\n"); + printf(" quick call it from AOT code\n"); #if WASM_ENABLE_LINUX_PERF != 0 printf(" --enable-linux-perf Enable linux perf support\n"); #endif @@ -532,12 +534,15 @@ main(int argc, char *argv[]) native_lib_list[native_lib_count++] = argv[0] + 13; } #endif + else if (!strcmp(argv[0], "--invoke-c-api-import")) { + option.quick_invoke_c_api_import = true; + } #if WASM_ENABLE_LINUX_PERF != 0 - else if (!strncmp(argv[0], "--enable-linux-perf", 19)) { + else if (!strcmp(argv[0], "--enable-linux-perf")) { enable_linux_perf = true; } #endif - else if (!strncmp(argv[0], "--version", 9)) { + else if (!strcmp(argv[0], "--version")) { uint32 major, minor, patch; wasm_runtime_get_version(&major, &minor, &patch); printf("wamrc %u.%u.%u\n", major, minor, patch);