diff --git a/.github/workflows/compilation_on_android_ubuntu_macos.yml b/.github/workflows/compilation_on_android_ubuntu_macos.yml index fd8fddc3e..627ea7460 100644 --- a/.github/workflows/compilation_on_android_ubuntu_macos.yml +++ b/.github/workflows/compilation_on_android_ubuntu_macos.yml @@ -31,8 +31,8 @@ env: AOT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" CLASSIC_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" - JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" + MC_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex" jobs: @@ -193,8 +193,8 @@ jobs: # Running mode $CLASSIC_INTERP_BUILD_OPTIONS, $FAST_INTERP_BUILD_OPTIONS, - $JIT_BUILD_OPTIONS, $LAZY_JIT_BUILD_OPTIONS, + $MC_JIT_BUILD_OPTIONS, $AOT_BUILD_OPTIONS, ] make_options_feature: [ @@ -232,11 +232,11 @@ jobs: make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" # uncompatiable mode and feature # MULTI_MODULE only on INTERP mode - - make_options_run_mode: $JIT_BUILD_OPTIONS + - make_options_run_mode: $LAZY_JIT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" - make_options_run_mode: $AOT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" - - make_options_run_mode: $LAZY_JIT_BUILD_OPTIONS + - make_options_run_mode: $MC_JIT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_MULTI_MODULE=1" # SIMD only on JIT/AOT mode - make_options_run_mode: $CLASSIC_INTERP_BUILD_OPTIONS @@ -246,10 +246,10 @@ jobs: # DEBUG_INTERP only on CLASSIC INTERP mode - make_options_run_mode: $AOT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" - - make_options_run_mode: $JIT_BUILD_OPTIONS - make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" - make_options_run_mode: $LAZY_JIT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" + - make_options_run_mode: $MC_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_DEBUG_INTERP=1" # DEBUG_AOT only on JIT/AOT mode @@ -258,17 +258,17 @@ jobs: - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" # TODO: DEBUG_AOT on JIT - - make_options_run_mode: $JIT_BUILD_OPTIONS - make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" - make_options_run_mode: $LAZY_JIT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" + - make_options_run_mode: $MC_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_DEBUG_AOT=1" # MINI_LOADER only on INTERP mode - make_options_run_mode: $AOT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" - - make_options_run_mode: $JIT_BUILD_OPTIONS - make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" - make_options_run_mode: $LAZY_JIT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + - make_options_run_mode: $MC_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" include: - os: ubuntu-18.04 light: ${{ needs.check_repo.outputs.traffic_light_on_ubuntu_1804 }} @@ -318,8 +318,8 @@ jobs: # Running mode $CLASSIC_INTERP_BUILD_OPTIONS, $FAST_INTERP_BUILD_OPTIONS, - $JIT_BUILD_OPTIONS, $LAZY_JIT_BUILD_OPTIONS, + $MC_JIT_BUILD_OPTIONS, $AOT_BUILD_OPTIONS, ] os: [ubuntu-18.04, ubuntu-20.04, macos-latest] @@ -339,11 +339,11 @@ jobs: exclude: # TODO: a .aot compatiable problem - os: macos-latest - make_options: $JIT_BUILD_OPTIONS + make_options: $LAZY_JIT_BUILD_OPTIONS - os: macos-latest make_options: $AOT_BUILD_OPTIONS - os: macos-latest - make_options: $LAZY_JIT_BUILD_OPTIONS + make_options: $MC_JIT_BUILD_OPTIONS steps: - name: light status run: echo "matrix.os=${{ matrix.os }}, light=${{ matrix.light }}" diff --git a/.github/workflows/compilation_on_sgx.yml b/.github/workflows/compilation_on_sgx.yml index f1ad8d92b..3289fa3fe 100644 --- a/.github/workflows/compilation_on_sgx.yml +++ b/.github/workflows/compilation_on_sgx.yml @@ -31,8 +31,8 @@ env: AOT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" CLASSIC_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0" - JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" + MC_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0" LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex" jobs: @@ -116,8 +116,8 @@ jobs: $CLASSIC_INTERP_BUILD_OPTIONS, $FAST_INTERP_BUILD_OPTIONS, # doesn't support - # $JIT_BUILD_OPTIONS, # $LAZY_JIT_BUILD_OPTIONS, + # $MC_JIT_BUILD_OPTIONS, $AOT_BUILD_OPTIONS, ] make_options_feature: [ @@ -208,8 +208,8 @@ jobs: $CLASSIC_INTERP_BUILD_OPTIONS, $FAST_INTERP_BUILD_OPTIONS, # doesn't support - #$JIT_BUILD_OPTIONS, #$LAZY_JIT_BUILD_OPTIONS, + #$MC_JIT_BUILD_OPTIONS, #$AOT_BUILD_OPTIONS, ] os: [ubuntu-20.04] diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 02ae865cd..ba44ceee6 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -87,7 +87,9 @@ endif () if (WAMR_BUILD_JIT EQUAL 1) if (WAMR_BUILD_AOT EQUAL 1) add_definitions("-DWASM_ENABLE_JIT=1") - if (WAMR_BUILD_LAZY_JIT EQUAL 1) + if (NOT WAMR_BUILD_LAZY_JIT EQUAL 0) + # Enable Lazy JIT by default + set (WAMR_BUILD_LAZY_JIT 1) add_definitions("-DWASM_ENABLE_LAZY_JIT=1") endif () if (NOT DEFINED LLVM_DIR) @@ -128,11 +130,10 @@ else () message (" WAMR AOT disabled") endif () if (WAMR_BUILD_JIT EQUAL 1) - message (" WAMR JIT enabled") if (WAMR_BUILD_LAZY_JIT EQUAL 1) - message (" WAMR LazyJIT enabled") + message (" WAMR Lazy JIT enabled") else () - message (" WAMR LazyJIT disabled") + message (" WAMR MC JIT enabled") endif () else () message (" WAMR JIT disabled") diff --git a/core/config.h b/core/config.h index dc11876bb..065af251b 100644 --- a/core/config.h +++ b/core/config.h @@ -81,6 +81,10 @@ #define WASM_ENABLE_LAZY_JIT 0 #endif +#ifndef WASM_LAZY_JIT_COMPILE_THREAD_NUM +#define WASM_LAZY_JIT_COMPILE_THREAD_NUM 4 +#endif + #if (WASM_ENABLE_AOT == 0) && (WASM_ENABLE_JIT != 0) /* LazyJIT or MCJIT can only be enabled when AOT is enabled */ #undef WASM_ENABLE_JIT diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index bac3b476b..b64f98b37 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -2760,6 +2760,90 @@ aot_load_from_aot_file(const uint8 *buf, uint32 size, char *error_buf, } #if WASM_ENABLE_JIT != 0 +#if WASM_ENABLE_LAZY_JIT != 0 +/* Orc JIT thread arguments */ +typedef struct OrcJitThreadArg { + AOTCompData *comp_data; + AOTCompContext *comp_ctx; + AOTModule *module; + int32 group_idx; + int32 group_stride; +} OrcJitThreadArg; + +static bool orcjit_stop_compiling = false; +static korp_tid orcjit_threads[WASM_LAZY_JIT_COMPILE_THREAD_NUM]; +static OrcJitThreadArg orcjit_thread_args[WASM_LAZY_JIT_COMPILE_THREAD_NUM]; + +static void * +orcjit_thread_callback(void *arg) +{ + LLVMErrorRef error; + LLVMOrcJITTargetAddress func_addr = 0; + OrcJitThreadArg *thread_arg = (OrcJitThreadArg *)arg; + AOTCompData *comp_data = thread_arg->comp_data; + AOTCompContext *comp_ctx = thread_arg->comp_ctx; + AOTModule *module = thread_arg->module; + char func_name[32]; + int32 i; + + /* Compile wasm functions of this group */ + for (i = thread_arg->group_idx; i < (int32)comp_data->func_count; + i += thread_arg->group_stride) { + if (!module->func_ptrs[i]) { + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + if ((error = LLVMOrcLLJITLookup(comp_ctx->orc_lazyjit, &func_addr, + func_name))) { + char *err_msg = LLVMGetErrorMessage(error); + os_printf("failed to compile orc jit function: %s", err_msg); + LLVMDisposeErrorMessage(err_msg); + break; + } + /** + * No need to lock the func_ptr[func_idx] here as it is basic + * data type, the load/store for it can be finished by one cpu + * instruction, and there can be only one cpu instruction + * loading/storing at the same time. + */ + module->func_ptrs[i] = (void *)func_addr; + } + if (orcjit_stop_compiling) { + break; + } + } + + /* Try to compile functions that haven't been compiled by other threads */ + for (i = (int32)comp_data->func_count - 1; i > 0; i--) { + if (orcjit_stop_compiling) { + break; + } + if (!module->func_ptrs[i]) { + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + if ((error = LLVMOrcLLJITLookup(comp_ctx->orc_lazyjit, &func_addr, + func_name))) { + char *err_msg = LLVMGetErrorMessage(error); + os_printf("failed to compile orc jit function: %s", err_msg); + LLVMDisposeErrorMessage(err_msg); + break; + } + module->func_ptrs[i] = (void *)func_addr; + } + } + + return NULL; +} + +static void +orcjit_stop_compile_threads() +{ + uint32 i; + + orcjit_stop_compiling = true; + for (i = 0; i < WASM_LAZY_JIT_COMPILE_THREAD_NUM; i++) { + os_thread_join(orcjit_threads[i], NULL); + } +} +#endif + static AOTModule * aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, char *error_buf, uint32 error_buf_size) @@ -2769,13 +2853,6 @@ aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, char func_name[32]; AOTModule *module; -#if WASM_ENABLE_LAZY_JIT != 0 - LLVMOrcThreadSafeModuleRef ts_module; - LLVMOrcJITDylibRef main_dylib; - LLVMErrorRef error; - LLVMOrcJITTargetAddress func_addr = 0; -#endif - /* Allocate memory for module */ if (!(module = loader_malloc(sizeof(AOTModule), error_buf, error_buf_size))) { @@ -2839,44 +2916,28 @@ aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, } #if WASM_ENABLE_LAZY_JIT != 0 - bh_assert(comp_ctx->lazy_orcjit); + /* Create threads to compile the wasm functions */ + for (i = 0; i < WASM_LAZY_JIT_COMPILE_THREAD_NUM; i++) { + orcjit_thread_args[i].comp_data = comp_data; + orcjit_thread_args[i].comp_ctx = comp_ctx; + orcjit_thread_args[i].module = module; + orcjit_thread_args[i].group_idx = (int32)i; + orcjit_thread_args[i].group_stride = WASM_LAZY_JIT_COMPILE_THREAD_NUM; + if (os_thread_create(&orcjit_threads[i], orcjit_thread_callback, + (void *)&orcjit_thread_args[i], + APP_THREAD_STACK_SIZE_DEFAULT) + != 0) { + uint32 j; - main_dylib = LLVMOrcLLLazyJITGetMainJITDylib(comp_ctx->lazy_orcjit); - if (!main_dylib) { - set_error_buf(error_buf, error_buf_size, - "failed to get dynmaic library reference"); - goto fail3; - } - - ts_module = LLVMOrcCreateNewThreadSafeModule(comp_ctx->module, - comp_ctx->ts_context); - if (!ts_module) { - set_error_buf(error_buf, error_buf_size, - "failed to create thread safe module"); - goto fail3; - } - - if ((error = LLVMOrcLLLazyJITAddLLVMIRModule(comp_ctx->lazy_orcjit, - main_dylib, ts_module))) { - /* - * If adding the ThreadSafeModule fails then we need to clean it up - * ourselves. If adding it succeeds the JIT will manage the memory. - */ - aot_handle_llvm_errmsg(error_buf, error_buf_size, - "failed to addIRModule: ", error); - goto fail4; - } - - for (i = 0; i < comp_data->func_count; i++) { - snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); - if ((error = LLVMOrcLLLazyJITLookup(comp_ctx->lazy_orcjit, &func_addr, - func_name))) { - aot_handle_llvm_errmsg(error_buf, error_buf_size, - "cannot lookup: ", error); + set_error_buf(error_buf, error_buf_size, + "create orcjit compile thread failed"); + /* Terminate the threads created */ + orcjit_stop_compiling = true; + for (j = 0; j < i; j++) { + os_thread_join(orcjit_threads[j], NULL); + } goto fail3; } - module->func_ptrs[i] = (void *)func_addr; - func_addr = 0; } #else /* Resolve function addresses */ @@ -2897,7 +2958,7 @@ aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, if (size > 0 && !(module->func_type_indexes = loader_malloc(size, error_buf, error_buf_size))) { - goto fail3; + goto fail4; } for (i = 0; i < comp_data->func_count; i++) module->func_type_indexes[i] = comp_data->funcs[i]->func_type_index; @@ -2911,6 +2972,29 @@ aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, < module->import_func_count + module->func_count); /* TODO: fix issue that start func cannot be import func */ if (comp_data->start_func_index >= module->import_func_count) { +#if WASM_ENABLE_LAZY_JIT != 0 + if (!module->func_ptrs[comp_data->start_func_index + - module->import_func_count]) { + LLVMErrorRef error; + LLVMOrcJITTargetAddress func_addr = 0; + + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, + comp_data->start_func_index + - module->import_func_count); + if ((error = LLVMOrcLLJITLookup(comp_ctx->orc_lazyjit, + &func_addr, func_name))) { + char *err_msg = LLVMGetErrorMessage(error); + set_error_buf_v(error_buf, error_buf_size, + "failed to compile orc jit function: %s", + err_msg); + LLVMDisposeErrorMessage(err_msg); + goto fail5; + } + module->func_ptrs[comp_data->start_func_index + - module->import_func_count] = + (void *)func_addr; + } +#endif module->start_function = module->func_ptrs[comp_data->start_func_index - module->import_func_count]; @@ -2945,10 +3029,16 @@ aot_load_from_comp_data(AOTCompData *comp_data, AOTCompContext *comp_ctx, return module; #if WASM_ENABLE_LAZY_JIT != 0 -fail4: - LLVMOrcDisposeThreadSafeModule(ts_module); +fail5: + if (module->func_type_indexes) + wasm_runtime_free(module->func_type_indexes); #endif +fail4: +#if WASM_ENABLE_LAZY_JIT != 0 + /* Terminate all threads before free module->func_ptrs */ + orcjit_stop_compile_threads(); +#endif fail3: if (module->func_ptrs) wasm_runtime_free(module->func_ptrs); @@ -3034,6 +3124,10 @@ void aot_unload(AOTModule *module) { #if WASM_ENABLE_JIT != 0 +#if WASM_ENABLE_LAZY_JIT != 0 + orcjit_stop_compile_threads(); +#endif + if (module->comp_data) aot_destroy_comp_data(module->comp_data); diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 6a78e1dc9..baa5ce7bc 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1419,6 +1419,17 @@ aot_call_function(WASMExecEnv *exec_env, AOTFunctionInstance *function, } argc = func_type->param_cell_num; +#if WASM_ENABLE_LAZY_JIT != 0 + if (!function->u.func.func_ptr) { + AOTModule *aot_module = (AOTModule *)module_inst->aot_module.ptr; + if (!(function->u.func.func_ptr = + aot_lookup_orcjit_func(aot_module->comp_ctx->orc_lazyjit, + module_inst, function->func_index))) { + return false; + } + } +#endif + /* set thread handle and stack boundary */ wasm_exec_env_set_thread_info(exec_env); @@ -2300,6 +2311,15 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, func_type_idx = func_type_indexes[func_idx]; func_type = aot_module->func_types[func_type_idx]; +#if WASM_ENABLE_LAZY_JIT != 0 + if (func_idx >= aot_module->import_func_count && !func_ptrs[func_idx]) { + if (!(func_ptr = aot_lookup_orcjit_func( + aot_module->comp_ctx->orc_lazyjit, module_inst, func_idx))) { + return false; + } + } +#endif + if (!(func_ptr = func_ptrs[func_idx])) { bh_assert(func_idx < aot_module->import_func_count); import_func = aot_module->import_funcs + func_idx; diff --git a/core/iwasm/common/wasm_c_api.c b/core/iwasm/common/wasm_c_api.c index fb799de19..a77f43c8d 100644 --- a/core/iwasm/common/wasm_c_api.c +++ b/core/iwasm/common/wasm_c_api.c @@ -232,7 +232,7 @@ WASM_DEFINE_VEC_OWN(store, wasm_store_delete) WASM_DEFINE_VEC_OWN(valtype, wasm_valtype_delete) /* conflicting declaration between aot_export.h and aot.h */ -#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_JIT != 0 +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0 bool aot_compile_wasm_file_init(); @@ -266,7 +266,7 @@ wasm_engine_delete_internal(wasm_engine_t *engine) wasm_runtime_free(engine); } -#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_JIT != 0 +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0 aot_compile_wasm_file_destroy(); #endif @@ -317,7 +317,7 @@ wasm_engine_new_internal(mem_alloc_type_t type, const MemAllocOption *opts) bh_log_set_verbose_level(3); #endif -#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_JIT != 0 +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0 if (!aot_compile_wasm_file_init()) { goto failed; } @@ -1820,7 +1820,7 @@ wasm_module_new(wasm_store_t *store, const wasm_byte_vec_t *binary) char error_buf[128] = { 0 }; wasm_module_ex_t *module_ex = NULL; PackageType pkg_type; -#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_JIT != 0 +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0 uint8 *aot_file_buf = NULL; uint32 aot_file_size; #endif @@ -1858,7 +1858,7 @@ wasm_module_new(wasm_store_t *store, const wasm_byte_vec_t *binary) INIT_VEC(module_ex->binary, wasm_byte_vec_new, binary->size, binary->data); -#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_JIT != 0 +#if WASM_ENABLE_AOT != 0 && WASM_ENABLE_JIT != 0 && WASM_ENABLE_LAZY_JIT == 0 if (Wasm_Module_Bytecode == pkg_type) { if (!(aot_file_buf = aot_compile_wasm_file( (uint8 *)module_ex->binary->data, diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 79668cc1e..9208b6d70 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -1307,7 +1307,7 @@ wasm_runtime_finalize_call_function(WASMExecEnv *exec_env, uint32 argv_i = 0, result_i = 0, ret_argv_i = 0; WASMType *func_type; - bh_assert(argv && ret_argv); + bh_assert((argv && ret_argv) || (argc == 0)); if (argv == ret_argv) { return true; diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index 14543885a..2807fef2f 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -2576,6 +2576,11 @@ aot_compile_wasm(AOTCompContext *comp_ctx) char *msg = NULL; bool ret; uint32 i; +#if WASM_ENABLE_LAZY_JIT != 0 + LLVMErrorRef err; + LLVMOrcJITDylibRef orc_main_dylib; + LLVMOrcThreadSafeModuleRef orc_thread_safe_module; +#endif if (!aot_validate_wasm(comp_ctx)) { return false; @@ -2607,6 +2612,7 @@ aot_compile_wasm(AOTCompContext *comp_ctx) bh_print_time("Begin to verify LLVM module"); +#if WASM_ENABLE_LAZY_JIT == 0 ret = LLVMVerifyModule(comp_ctx->module, LLVMPrintMessageAction, &msg); if (!ret && msg) { if (msg[0] != '\0') { @@ -2616,17 +2622,39 @@ aot_compile_wasm(AOTCompContext *comp_ctx) } LLVMDisposeMessage(msg); } +#else + for (i = 0; i < comp_ctx->func_ctx_count; i++) { + ret = LLVMVerifyModule(comp_ctx->modules[i], LLVMPrintMessageAction, + &msg); + if (!ret && msg) { + if (msg[0] != '\0') { + aot_set_last_error(msg); + LLVMDisposeMessage(msg); + return false; + } + LLVMDisposeMessage(msg); + } + } +#endif bh_print_time("Begin to run function optimization passes"); /* Run function pass manager */ if (comp_ctx->optimize) { +#if WASM_ENABLE_LAZY_JIT == 0 LLVMInitializeFunctionPassManager(comp_ctx->pass_mgr); - for (i = 0; i < comp_ctx->func_ctx_count; i++) + for (i = 0; i < comp_ctx->func_ctx_count; i++) { LLVMRunFunctionPassManager(comp_ctx->pass_mgr, comp_ctx->func_ctxes[i]->func); + } +#else + for (i = 0; i < comp_ctx->func_ctx_count; i++) { + LLVMRunPassManager(comp_ctx->pass_mgr, comp_ctx->modules[i]); + } +#endif /* end of WASM_ENABLE_LAZY_JIT */ } +#if WASM_ENABLE_LAZY_JIT == 0 /* Run common pass manager */ if (comp_ctx->optimize && !comp_ctx->is_jit_mode && !comp_ctx->disable_llvm_lto) { @@ -2656,6 +2684,7 @@ aot_compile_wasm(AOTCompContext *comp_ctx) LLVMDisposePassManager(common_pass_mgr); LLVMPassManagerBuilderDispose(pass_mgr_builder); } +#endif if (comp_ctx->optimize && comp_ctx->is_indirect_mode) { LLVMPassManagerRef common_pass_mgr = NULL; @@ -2673,14 +2702,58 @@ aot_compile_wasm(AOTCompContext *comp_ctx) if (aot_require_lower_switch_pass(comp_ctx)) LLVMAddLowerSwitchPass(common_pass_mgr); +#if WASM_ENABLE_LAZY_JIT == 0 LLVMRunPassManager(common_pass_mgr, comp_ctx->module); +#else + for (i = 0; i < comp_ctx->func_ctx_count; i++) + LLVMRunPassManager(common_pass_mgr, comp_ctx->modules[i]); +#endif LLVMDisposePassManager(common_pass_mgr); } +#if 0 +#if WASM_ENABLE_LAZY_JIT == 0 + LLVMDumpModule(comp_ctx->module); +#else + for (i = 0; i < comp_ctx->func_ctx_count; i++) { + LLVMDumpModule(comp_ctx->modules[i]); + os_printf("\n"); + } +#endif +#endif + +#if WASM_ENABLE_LAZY_JIT != 0 + orc_main_dylib = LLVMOrcLLJITGetMainJITDylib(comp_ctx->orc_lazyjit); + if (!orc_main_dylib) { + aot_set_last_error("failed to get orc jit main dynmaic library"); + return false; + } + + for (i = 0; i < comp_ctx->func_ctx_count; i++) { + orc_thread_safe_module = LLVMOrcCreateNewThreadSafeModule( + comp_ctx->modules[i], comp_ctx->orc_thread_safe_context); + if (!orc_thread_safe_module) { + aot_set_last_error("failed to create thread safe module"); + return false; + } + + if ((err = LLVMOrcLLJITAddLLVMIRModule(comp_ctx->orc_lazyjit, + orc_main_dylib, + orc_thread_safe_module))) { + /* If adding the ThreadSafeModule fails then we need to clean it up + by ourselves, otherwise the orc jit will manage the memory. */ + LLVMOrcDisposeThreadSafeModule(orc_thread_safe_module); + aot_handle_llvm_errmsg("failed to addIRModule", err); + return false; + } + } +#endif + return true; } +#if WASM_ENABLE_LAZY_JIT == 0 bool aot_emit_llvm_file(AOTCompContext *comp_ctx, const char *file_name) { @@ -2928,3 +3001,4 @@ fail1: return aot_file_buf; } +#endif /* end of WASM_ENABLE_LAZY_JIT == 0 */ diff --git a/core/iwasm/compilation/aot_compiler.h b/core/iwasm/compilation/aot_compiler.h index 7177de297..8637b6d82 100644 --- a/core/iwasm/compilation/aot_compiler.h +++ b/core/iwasm/compilation/aot_compiler.h @@ -345,8 +345,8 @@ check_type_compatible(uint8 src_type, uint8 dst_type) else { \ char *func_name = #name; \ /* AOT mode, delcare the function */ \ - if (!(func = LLVMGetNamedFunction(comp_ctx->module, func_name)) \ - && !(func = LLVMAddFunction(comp_ctx->module, func_name, \ + if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name)) \ + && !(func = LLVMAddFunction(func_ctx->module, func_name, \ func_type))) { \ aot_set_last_error("llvm add function failed."); \ goto fail; \ diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index e19cfaa18..d0b1d74a0 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -6,6 +6,8 @@ #include "aot_compiler.h" #include "../aot/aot_runtime.h" +#if WASM_ENABLE_LAZY_JIT == 0 + #define PUT_U64_TO_ADDR(addr, value) \ do { \ union { \ @@ -2804,3 +2806,5 @@ fail1: return ret; } + +#endif /* end of WASM_ENABLE_JIT */ diff --git a/core/iwasm/compilation/aot_emit_exception.c b/core/iwasm/compilation/aot_emit_exception.c index 9bc46be04..1ec606b47 100644 --- a/core/iwasm/compilation/aot_emit_exception.c +++ b/core/iwasm/compilation/aot_emit_exception.c @@ -83,9 +83,9 @@ aot_emit_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } else { /* Create LLVM function with external function pointer */ - if (!(func = LLVMGetNamedFunction(comp_ctx->module, + if (!(func = LLVMGetNamedFunction(func_ctx->module, "aot_set_exception_with_id")) - && !(func = LLVMAddFunction(comp_ctx->module, + && !(func = LLVMAddFunction(func_ctx->module, "aot_set_exception_with_id", func_type))) { aot_set_last_error("add LLVM function failed."); diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index fe43d6bce..c5f1379cd 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -184,9 +184,9 @@ call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } else { - if (!(func = LLVMGetNamedFunction(comp_ctx->module, func_name)) + if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name)) && !(func = - LLVMAddFunction(comp_ctx->module, func_name, func_type))) { + LLVMAddFunction(func_ctx->module, func_name, func_type))) { aot_set_last_error("add LLVM function failed."); return false; } @@ -266,6 +266,130 @@ call_aot_invoke_native_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } +#if WASM_ENABLE_LAZY_JIT != 0 +static bool +lookup_orcjit_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef func_idx, LLVMValueRef *p_func) +{ + LLVMBasicBlockRef block_curr, block_resolve_func, block_func_resolved; + LLVMValueRef param_values[3], func, value, func_ptr, cmp, phi; + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + + if (!(block_resolve_func = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "resolve_func"))) { + aot_set_last_error("llvm add basic block failed."); + return false; + } + if (!(block_func_resolved = LLVMAppendBasicBlockInContext( + comp_ctx->context, func_ctx->func, "func_resolved"))) { + aot_set_last_error("llvm add basic block failed."); + return false; + } + LLVMMoveBasicBlockAfter(block_resolve_func, block_curr); + LLVMMoveBasicBlockAfter(block_func_resolved, block_resolve_func); + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_func_resolved); + if (!(phi = LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE, "phi"))) { + aot_set_last_error("llvm build phi failed."); + return false; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); + + /* Load function pointer */ + if (!(func_ptr = + LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->func_ptrs, + &func_idx, 1, "func_ptr_tmp"))) { + aot_set_last_error("llvm build inbounds gep failed."); + return false; + } + + if (!(func = LLVMBuildLoad(comp_ctx->builder, func_ptr, "func_ptr"))) { + aot_set_last_error("llvm build load failed."); + return false; + } + + /* If func ptr is NULL, call aot_lookup_orcjit_func to resolve it */ + if (!(cmp = LLVMBuildIsNull(comp_ctx->builder, func, "cmp"))) { + aot_set_last_error("llvm build is null failed"); + return false; + } + + /* Create condition br */ + if (!LLVMBuildCondBr(comp_ctx->builder, cmp, block_resolve_func, + block_func_resolved)) { + aot_set_last_error("llvm build cond br failed."); + return false; + } + LLVMAddIncoming(phi, &func, &block_curr, 1); + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_resolve_func); + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = comp_ctx->aot_inst_type; + param_types[2] = I32_TYPE; + ret_type = INT8_PTR_TYPE; + + if (!(func_type = LLVMFunctionType(ret_type, param_types, 3, false)) + || !(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("llvm add function type failed."); + return false; + } + + if (!(value = I64_CONST((uint64)(uintptr_t)aot_lookup_orcjit_func)) + || !(func = LLVMConstIntToPtr(value, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); + return false; + } + + param_values[0] = I64_CONST((uintptr_t)comp_ctx->orc_lazyjit); + if (!param_values[0]) { + aot_set_last_error("llvm build const failed."); + return false; + } + if (!(param_values[0] = + LLVMConstIntToPtr(param_values[0], INT8_PTR_TYPE))) { + aot_set_last_error("llvm build bit cast failed."); + return false; + } + + param_values[1] = func_ctx->aot_inst; + + param_values[2] = func_idx; + if (!param_values[2]) { + aot_set_last_error("llvm build const failed."); + return false; + } + + /* Call the function */ + if (!(func = LLVMBuildCall(comp_ctx->builder, func, param_values, 3, + "call_orcjit_lookup"))) { + aot_set_last_error("LLVM build call failed."); + return false; + } + + /* Check whether exception was thrown when looking up func */ + if (!check_exception_thrown(comp_ctx, func_ctx)) { + return false; + } + + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + LLVMAddIncoming(phi, &func, &block_curr, 1); + + if (!LLVMBuildBr(comp_ctx->builder, block_func_resolved)) { + aot_set_last_error("llvm build br failed."); + return false; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_func_resolved); + + *p_func = phi; + return true; +} +#endif + #if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0) static bool call_aot_alloc_frame_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, @@ -579,8 +703,45 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } else { +#if WASM_ENABLE_LAZY_JIT == 0 func = func_ctxes[func_idx - import_func_count]->func; +#else + if (func_ctxes[func_idx - import_func_count] == func_ctx) { + /* recursive call */ + func = func_ctx->func; + } + else { + LLVMTypeRef func_ptr_type; + LLVMValueRef func_idx_const = I32_CONST(func_idx); + + if (!func_idx_const) { + aot_set_last_error("llvm build const failed."); + goto fail; + } + + /* For LAZY JIT, each function belongs to its own module, + we call aot_lookup_orcjit_func to get the func pointer */ + if (!lookup_orcjit_func(comp_ctx, func_ctx, func_idx_const, + &func)) { + goto fail; + } + + if (!(func_ptr_type = LLVMPointerType( + func_ctxes[func_idx - import_func_count]->func_type, + 0))) { + aot_set_last_error("construct func ptr type failed."); + goto fail; + } + + if (!(func = LLVMBuildBitCast(comp_ctx->builder, func, + func_ptr_type, "aot_func"))) { + aot_set_last_error("llvm bit cast failed."); + goto fail; + } + } +#endif } + aot_func = func_ctxes[func_idx - import_func_count]->aot_func; callee_cell_num = aot_func->param_cell_num + aot_func->local_cell_num + 1; @@ -704,9 +865,9 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } else { - if (!(func = LLVMGetNamedFunction(comp_ctx->module, func_name)) + if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name)) && !(func = - LLVMAddFunction(comp_ctx->module, func_name, func_type))) { + LLVMAddFunction(func_ctx->module, func_name, func_type))) { aot_set_last_error("add LLVM function failed."); return false; } @@ -1197,6 +1358,7 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + 16)) goto fail; +#if WASM_ENABLE_LAZY_JIT == 0 /* Load function pointer */ if (!(func_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->func_ptrs, @@ -1209,6 +1371,12 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("llvm build load failed."); goto fail; } +#else + /* For LAZY JIT, each function belongs to its own module, + we call aot_lookup_orcjit_func to get the func pointer */ + if (!lookup_orcjit_func(comp_ctx, func_ctx, func_idx, &func_ptr)) + goto fail; +#endif if (!(llvm_func_type = LLVMFunctionType(ret_type, param_types, total_param_count, false)) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 2c2cc9968..f1c62fd59 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -709,9 +709,9 @@ aot_compile_op_memory_grow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) else { char *func_name = "aot_enlarge_memory"; /* AOT mode, delcare the function */ - if (!(func = LLVMGetNamedFunction(comp_ctx->module, func_name)) + if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name)) && !(func = - LLVMAddFunction(comp_ctx->module, func_name, func_type))) { + LLVMAddFunction(func_ctx->module, func_name, func_type))) { aot_set_last_error("llvm add function failed."); return false; } @@ -941,6 +941,7 @@ bool aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { LLVMValueRef src, dst, src_addr, dst_addr, len, res; + bool call_aot_memmove = false; POP_I32(len); POP_I32(src); @@ -952,29 +953,23 @@ aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) if (!(dst_addr = check_bulk_memory_overflow(comp_ctx, func_ctx, dst, len))) return false; - if (comp_ctx->is_indirect_mode) { +#if WASM_ENABLE_LAZY_JIT != 0 + call_aot_memmove = true; +#endif + if (comp_ctx->is_indirect_mode) + call_aot_memmove = true; + + if (call_aot_memmove) { LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; LLVMValueRef func, params[3]; +#if WASM_ENABLE_LAZY_JIT == 0 int32 func_idx; +#endif - if (!(dst_addr = - LLVMBuildBitCast(comp_ctx->builder, dst_addr, INT32_PTR_TYPE, - "memmove dst addr cast type"))) { - aot_set_last_error("llvm cast memmove dst addr type failed."); - return false; - } - - if (!(src_addr = - LLVMBuildBitCast(comp_ctx->builder, src_addr, INT32_PTR_TYPE, - "memmove src addr cast type"))) { - aot_set_last_error("llvm cast memmove src addr type failed."); - return false; - } - - param_types[0] = INT32_PTR_TYPE; - param_types[1] = INT32_PTR_TYPE; + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INT8_PTR_TYPE; param_types[2] = I32_TYPE; - ret_type = INT32_PTR_TYPE; + ret_type = INT8_PTR_TYPE; if (!(func_type = LLVMFunctionType(ret_type, param_types, 3, false))) { aot_set_last_error("create LLVM function type failed."); @@ -986,6 +981,7 @@ aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) return false; } +#if WASM_ENABLE_LAZY_JIT == 0 func_idx = aot_get_native_symbol_index(comp_ctx, "memmove"); if (func_idx < 0) { return false; @@ -994,12 +990,19 @@ aot_compile_op_memory_copy(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) func_ptr_type, func_idx))) { return false; } +#else + if (!(func = I64_CONST((uint64)(uintptr_t)aot_memmove)) + || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); + return false; + } +#endif params[0] = dst_addr; params[1] = src_addr; params[2] = len; if (!(res = LLVMBuildCall(comp_ctx->builder, func, params, 3, - "call memmove"))) { + "call_memmove"))) { aot_set_last_error("llvm build memmove failed."); return false; } @@ -1021,6 +1024,8 @@ bool aot_compile_op_memory_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { LLVMValueRef val, dst, dst_addr, len, res; + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + LLVMValueRef func, params[3]; POP_I32(len); POP_I32(val); @@ -1029,64 +1034,57 @@ aot_compile_op_memory_fill(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) if (!(dst_addr = check_bulk_memory_overflow(comp_ctx, func_ctx, dst, len))) return false; - if (!(val = LLVMBuildIntCast2(comp_ctx->builder, val, INT8_TYPE, true, - "mem_set_value"))) { - aot_set_last_error("llvm build int cast2 failed."); + param_types[0] = INT8_PTR_TYPE; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + ret_type = INT8_PTR_TYPE; + + if (!(func_type = LLVMFunctionType(ret_type, param_types, 3, false))) { + aot_set_last_error("create LLVM function type failed."); return false; } - if (comp_ctx->is_indirect_mode) { - LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; - LLVMValueRef func, params[3]; - int32 func_idx; + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function pointer type failed."); + return false; + } - if (!(dst_addr = - LLVMBuildBitCast(comp_ctx->builder, dst_addr, INT32_PTR_TYPE, - "memset dst addr cast type"))) { - aot_set_last_error("llvm cast memset dst addr type failed."); + if (comp_ctx->is_jit_mode) { + if (!(func = I64_CONST((uint64)(uintptr_t)aot_memset)) + || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); return false; } - - param_types[0] = INT32_PTR_TYPE; - param_types[1] = INT8_TYPE; - param_types[2] = I32_TYPE; - ret_type = INT32_PTR_TYPE; - - if (!(func_type = LLVMFunctionType(ret_type, param_types, 3, false))) { - aot_set_last_error("create LLVM function type failed."); - return false; - } - - if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { - aot_set_last_error("create LLVM function pointer type failed."); - return false; - } - - func_idx = aot_get_native_symbol_index(comp_ctx, "memset"); - if (func_idx < 0) { + } + else if (comp_ctx->is_indirect_mode) { + int32 func_index; + func_index = aot_get_native_symbol_index(comp_ctx, "memset"); + if (func_index < 0) { return false; } if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, - func_ptr_type, func_idx))) { - return false; - } - - params[0] = dst_addr; - params[1] = val; - params[2] = len; - if (!(res = LLVMBuildCall(comp_ctx->builder, func, params, 3, - "call memset"))) { - aot_set_last_error("llvm build memset failed."); + func_ptr_type, func_index))) { return false; } } else { - if (!(res = - LLVMBuildMemSet(comp_ctx->builder, dst_addr, val, len, 1))) { - aot_set_last_error("llvm build memset failed."); + if (!(func = LLVMGetNamedFunction(func_ctx->module, "memset")) + && !(func = + LLVMAddFunction(func_ctx->module, "memset", func_type))) { + aot_set_last_error("llvm add function failed."); return false; } } + + params[0] = dst_addr; + params[1] = val; + params[2] = len; + if (!(res = LLVMBuildCall(comp_ctx->builder, func, params, 3, + "call_memset"))) { + aot_set_last_error("llvm build memset failed."); + return false; + } + return true; fail: return false; diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 33c7179a2..c126dcdd4 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -41,8 +41,9 @@ wasm_type_to_llvm_type(AOTLLVMTypes *llvm_types, uint8 wasm_type) * Add LLVM function */ static LLVMValueRef -aot_add_llvm_func(AOTCompContext *comp_ctx, AOTFuncType *aot_func_type, - uint32 func_index, LLVMTypeRef *p_func_type) +aot_add_llvm_func(AOTCompContext *comp_ctx, LLVMModuleRef module, + AOTFuncType *aot_func_type, uint32 func_index, + LLVMTypeRef *p_func_type) { LLVMValueRef func = NULL; LLVMTypeRef *param_types, ret_type, func_type; @@ -97,7 +98,7 @@ aot_add_llvm_func(AOTCompContext *comp_ctx, AOTFuncType *aot_func_type, /* Add LLVM function */ snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, func_index); - if (!(func = LLVMAddFunction(comp_ctx->module, func_name, func_type))) { + if (!(func = LLVMAddFunction(module, func_name, func_type))) { aot_set_last_error("add LLVM function failed."); goto fail; } @@ -616,9 +617,16 @@ aot_create_func_context(AOTCompData *comp_data, AOTCompContext *comp_ctx, memset(func_ctx, 0, (uint32)size); func_ctx->aot_func = func; +#if WASM_ENABLE_LAZY_JIT == 0 + func_ctx->module = comp_ctx->module; +#else + func_ctx->module = comp_ctx->modules[func_index]; +#endif + /* Add LLVM function */ - if (!(func_ctx->func = aot_add_llvm_func(comp_ctx, aot_func_type, - func_index, &func_ctx->func_type))) + if (!(func_ctx->func = + aot_add_llvm_func(comp_ctx, func_ctx->module, aot_func_type, + func_index, &func_ctx->func_type))) goto fail; /* Create function's first AOTBlock */ @@ -1214,152 +1222,211 @@ LLVMAddPromoteMemoryToRegisterPass(LLVMPassManagerRef PM); #if WASM_ENABLE_LAZY_JIT != 0 void -aot_handle_llvm_errmsg(char *error_buf, uint32 error_buf_size, - const char *string, LLVMErrorRef error) +aot_handle_llvm_errmsg(const char *string, LLVMErrorRef err) { - char *err_msg = LLVMGetErrorMessage(error); - if (error_buf != NULL) { - snprintf(error_buf, error_buf_size, "%s: %s", string, err_msg); - } + char *err_msg = LLVMGetErrorMessage(err); + aot_set_last_error_v("%s: %s", string, err_msg); LLVMDisposeErrorMessage(err_msg); } static bool -llvm_orcjit_create(AOTCompContext *comp_ctx) +orc_lazyjit_create(AOTCompContext *comp_ctx, uint32 func_count) { + uint32 i; char *err_msg = NULL; char *cpu = NULL; char *features = NULL; char *llvm_triple = NULL; - char buf[128] = { 0 }; - - LLVMErrorRef error; + char func_name[32] = { 0 }; + LLVMErrorRef err; LLVMTargetRef llvm_targetref = NULL; - LLVMTargetMachineRef tm_opt = NULL; - LLVMTargetMachineRef tm_opt2 = NULL; - LLVMOrcLLLazyJITRef lazy_orcjit = NULL; - LLVMOrcJITTargetMachineBuilderRef tm_builder = NULL; - LLVMOrcLLLazyJITBuilderRef lazy_orcjit_builder = NULL; -#if LLVM_VERSION_MAJOR < 12 - LLVMOrcJITDylibDefinitionGeneratorRef main_gen = NULL; -#else - LLVMOrcDefinitionGeneratorRef main_gen = NULL; -#endif + LLVMTargetMachineRef target_machine_for_orcjit = NULL; + LLVMOrcLLJITRef orc_lazyjit = NULL; + LLVMOrcJITTargetMachineBuilderRef target_machine_builder = NULL; + LLVMOrcLLJITBuilderRef orc_lazyjit_builder = NULL; + LLVMOrcMaterializationUnitRef orc_material_unit = NULL; + LLVMOrcExecutionSessionRef orc_execution_session = NULL; + LLVMOrcLazyCallThroughManagerRef orc_call_through_mgr = NULL; + LLVMOrcIndirectStubsManagerRef orc_indirect_stub_mgr = NULL; + LLVMOrcCSymbolAliasMapPair *orc_symbol_map_pairs = NULL; llvm_triple = LLVMGetDefaultTargetTriple(); if (llvm_triple == NULL) { - snprintf(buf, sizeof(buf), "failed to get default target triple."); + aot_set_last_error("failed to get default target triple."); goto fail; } if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &err_msg) != 0) { - snprintf(buf, sizeof(buf), - "failed to get target reference from triple %s.", err_msg); + aot_set_last_error_v("failed to get llvm target from triple %s.", + err_msg); LLVMDisposeMessage(err_msg); goto fail; } if (!LLVMTargetHasJIT(llvm_targetref)) { - snprintf(buf, sizeof(buf), "unspported JIT on this platform."); + aot_set_last_error("unspported JIT on this platform."); goto fail; } cpu = LLVMGetHostCPUName(); if (cpu == NULL) { - snprintf(buf, sizeof(buf), "failed to get host cpu information."); + aot_set_last_error("failed to get host cpu information."); goto fail; } features = LLVMGetHostCPUFeatures(); if (features == NULL) { - snprintf(buf, sizeof(buf), "failed to get host cpu features."); + aot_set_last_error("failed to get host cpu features."); goto fail; } LOG_VERBOSE("LLVM ORCJIT detected CPU \"%s\", with features \"%s\"\n", cpu, features); - tm_opt = LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features, - LLVMCodeGenLevelAggressive, - LLVMRelocDefault, LLVMCodeModelJITDefault); - if (!tm_opt) { - snprintf(buf, sizeof(buf), "failed to create target machine."); - goto fail; - } - - tm_opt2 = LLVMCreateTargetMachine( - llvm_targetref, llvm_triple, cpu, features, LLVMCodeGenLevelAggressive, + comp_ctx->target_machine = LLVMCreateTargetMachine( + llvm_targetref, llvm_triple, cpu, features, LLVMCodeGenLevelDefault, LLVMRelocDefault, LLVMCodeModelJITDefault); - if (!tm_opt2) { - snprintf(buf, sizeof(buf), "failed to create target machine2."); + if (!comp_ctx->target_machine) { + aot_set_last_error("failed to create target machine."); goto fail; } - /* if success, it will dispose tm_opt2 memory. */ - tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm_opt2); - if (!tm_builder) { - snprintf(buf, sizeof(buf), "failed to create target machine builder."); - goto fail; - } - tm_opt2 = NULL; - - lazy_orcjit_builder = LLVMOrcCreateLLLazyJITBuilder(); - if (!lazy_orcjit_builder) { - snprintf(buf, sizeof(buf), "failed to create lazy jit builder."); + target_machine_for_orcjit = LLVMCreateTargetMachine( + llvm_targetref, llvm_triple, cpu, features, LLVMCodeGenLevelDefault, + LLVMRelocDefault, LLVMCodeModelJITDefault); + if (!target_machine_for_orcjit) { + aot_set_last_error("failed to create target machine."); goto fail; } - LLVMOrcLLLazyJITBuilderSetJITTargetMachineBuilder(lazy_orcjit_builder, - tm_builder); - - /* if success, it will dispose lazy_orcjit_builder memory */ - error = LLVMOrcCreateLLLazyJIT(&lazy_orcjit, lazy_orcjit_builder); - if (error) { - aot_handle_llvm_errmsg(buf, sizeof(buf), - "failed to create llvm lazy orcjit instance", - error); + target_machine_builder = + LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine( + target_machine_for_orcjit); + if (!target_machine_builder) { + aot_set_last_error("failed to create target machine builder."); goto fail; } - lazy_orcjit_builder = NULL; + /* The target_machine_for_orcjit has been disposed before + LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine() returns */ + target_machine_for_orcjit = NULL; - error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess( - &main_gen, LLVMOrcLLLazyJITGetGlobalPrefix(lazy_orcjit), 0, NULL); - if (error) { - aot_handle_llvm_errmsg( - buf, sizeof(buf), - "failed to create dynmaic library search generator", error); + orc_lazyjit_builder = LLVMOrcCreateLLJITBuilder(); + if (!orc_lazyjit_builder) { + aot_set_last_error("failed to create lazy jit builder."); goto fail; } + LLVMOrcLLJITBuilderSetNumCompileThreads(orc_lazyjit_builder, + WASM_LAZY_JIT_COMPILE_THREAD_NUM); + LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(orc_lazyjit_builder, + target_machine_builder); + /* Should not dispose of the JITTargetMachineBuilder after calling + LLVMOrcLLJITBuilderSetJITTargetMachineBuilder() */ + target_machine_builder = NULL; - LLVMOrcJITDylibAddGenerator(LLVMOrcLLLazyJITGetMainJITDylib(lazy_orcjit), - main_gen); + err = LLVMOrcCreateLLJIT(&orc_lazyjit, orc_lazyjit_builder); + if (err) { + aot_handle_llvm_errmsg("failed to create llvm lazy orcjit instance", + err); + goto fail; + } + /* The orc_lazyjit_builder is managed by orc_lazyjit after calling + LLVMOrcCreateLLJIT(), here we should not dispose it again */ + orc_lazyjit_builder = NULL; + + if (func_count > 0) { + orc_execution_session = LLVMOrcLLJITGetExecutionSession(orc_lazyjit); + if (!orc_execution_session) { + aot_set_last_error("failed to get orc execution session"); + goto fail; + } + + err = LLVMOrcCreateLocalLazyCallThroughManager( + llvm_triple, orc_execution_session, 0, &orc_call_through_mgr); + if (err) { + aot_handle_llvm_errmsg("failed to create orc call through manager", + err); + goto fail; + } + + orc_indirect_stub_mgr = + LLVMOrcCreateLocalIndirectStubsManager(llvm_triple); + if (!orc_indirect_stub_mgr) { + aot_set_last_error("failed to create orc indirect stub manager"); + goto fail; + } + + if (!(orc_symbol_map_pairs = wasm_runtime_malloc( + sizeof(LLVMOrcCSymbolAliasMapPair) * func_count))) { + aot_set_last_error("failed to allocate memory"); + goto fail; + } + memset(orc_symbol_map_pairs, 0, + sizeof(LLVMOrcCSymbolAliasMapPair) * func_count); + + for (i = 0; i < func_count; i++) { + snprintf(func_name, sizeof(func_name), "orcjit_%s%d", + AOT_FUNC_PREFIX, i); + orc_symbol_map_pairs[i].Name = + LLVMOrcExecutionSessionIntern(orc_execution_session, func_name); + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, i); + orc_symbol_map_pairs[i].Entry.Name = + LLVMOrcExecutionSessionIntern(orc_execution_session, func_name); + orc_symbol_map_pairs[i].Entry.Flags.GenericFlags = + LLVMJITSymbolGenericFlagsExported + | LLVMJITSymbolGenericFlagsCallable; + orc_symbol_map_pairs[i].Entry.Flags.TargetFlags = + LLVMJITSymbolGenericFlagsExported + | LLVMJITSymbolGenericFlagsCallable; + + if (!orc_symbol_map_pairs[i].Name + || !orc_symbol_map_pairs[i].Entry.Name) { + aot_set_last_error("failed to allocate memory"); + goto fail; + } + } + + orc_material_unit = + LLVMOrcLazyReexports(orc_call_through_mgr, orc_indirect_stub_mgr, + LLVMOrcLLJITGetMainJITDylib(orc_lazyjit), + orc_symbol_map_pairs, func_count); + if (!orc_material_unit) { + aot_set_last_error("failed to orc re-exports"); + goto fail; + } + } + + comp_ctx->orc_lazyjit = orc_lazyjit; + comp_ctx->orc_material_unit = orc_material_unit; + comp_ctx->orc_symbol_map_pairs = orc_symbol_map_pairs; + comp_ctx->orc_call_through_mgr = orc_call_through_mgr; + comp_ctx->orc_indirect_stub_mgr = orc_indirect_stub_mgr; - comp_ctx->lazy_orcjit = lazy_orcjit; - comp_ctx->target_machine = tm_opt; - comp_ctx->tm_builder = tm_builder; LLVMDisposeMessage(llvm_triple); LLVMDisposeMessage(cpu); LLVMDisposeMessage(features); return true; fail: - if (lazy_orcjit) - LLVMOrcDisposeLLLazyJIT(lazy_orcjit); - if (tm_builder) - LLVMOrcDisposeJITTargetMachineBuilder(tm_builder); - if (lazy_orcjit_builder) - LLVMOrcDisposeLLLazyJITBuilder(lazy_orcjit_builder); - if (tm_opt2) - LLVMDisposeTargetMachine(tm_opt2); - if (tm_opt) - LLVMDisposeTargetMachine(tm_opt); + if (orc_symbol_map_pairs) + wasm_runtime_free(orc_symbol_map_pairs); + if (orc_call_through_mgr) + LLVMOrcDisposeLazyCallThroughManager(orc_call_through_mgr); + if (orc_indirect_stub_mgr) + LLVMOrcDisposeIndirectStubsManager(orc_indirect_stub_mgr); + if (orc_lazyjit) + LLVMOrcDisposeLLJIT(orc_lazyjit); + if (target_machine_builder) + LLVMOrcDisposeJITTargetMachineBuilder(target_machine_builder); + if (orc_lazyjit_builder) + LLVMOrcDisposeLLJITBuilder(orc_lazyjit_builder); + if (target_machine_for_orcjit) + LLVMDisposeTargetMachine(target_machine_for_orcjit); if (features) LLVMDisposeMessage(features); if (cpu) LLVMDisposeMessage(cpu); if (llvm_triple) LLVMDisposeMessage(llvm_triple); - aot_set_last_error(buf); return false; } #endif /* WASM_ENABLE_LAZY_JIT != 0 */ @@ -1371,7 +1438,6 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option) #if WASM_ENABLE_LAZY_JIT == 0 struct LLVMMCJITCompilerOptions jit_options; #endif - LLVMTargetRef target; char *triple = NULL, *triple_norm, *arch, *abi; char *cpu = NULL, *features, buf[128]; @@ -1379,7 +1445,7 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option) char *err = NULL, *fp_round = "round.tonearest", *fp_exce = "fpexcept.strict"; char triple_buf[32] = { 0 }, features_buf[128] = { 0 }; - uint32 opt_level, size_level; + uint32 opt_level, size_level, i; LLVMCodeModel code_model; LLVMTargetDataRef target_data_ref; @@ -1408,14 +1474,17 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option) /* Create LLVM context, module and builder */ #if WASM_ENABLE_LAZY_JIT != 0 - comp_ctx->ts_context = LLVMOrcCreateNewThreadSafeContext(); - if (!comp_ctx->ts_context) { + comp_ctx->orc_thread_safe_context = LLVMOrcCreateNewThreadSafeContext(); + if (!comp_ctx->orc_thread_safe_context) { aot_set_last_error("create LLVM ThreadSafeContext failed."); - return NULL; + goto fail; } - /* Get a reference to the underlying LLVMContext */ - if (!(comp_ctx->context = - LLVMOrcThreadSafeContextGetContext(comp_ctx->ts_context))) { + + /* Get a reference to the underlying LLVMContext, note: + different from non LAZY JIT mode, no need to dispose this context, + if will be disposed when the thread safe context is disposed */ + if (!(comp_ctx->context = LLVMOrcThreadSafeContextGetContext( + comp_ctx->orc_thread_safe_context))) { aot_set_last_error("get context from LLVM ThreadSafeContext failed."); goto fail; } @@ -1431,17 +1500,41 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option) goto fail; } +#if WASM_ENABLE_LAZY_JIT == 0 if (!(comp_ctx->module = LLVMModuleCreateWithNameInContext( "WASM Module", comp_ctx->context))) { aot_set_last_error("create LLVM module failed."); goto fail; } +#else + if (comp_data->func_count > 0) { + if (!(comp_ctx->modules = wasm_runtime_malloc( + sizeof(LLVMModuleRef) * comp_data->func_count))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + memset(comp_ctx->modules, 0, + sizeof(LLVMModuleRef) * comp_data->func_count); + for (i = 0; i < comp_data->func_count; i++) { + char module_name[32]; + snprintf(module_name, sizeof(module_name), "WASM Module %d", i); + /* Create individual modules for each aot function, note: + different from non LAZY JIT mode, no need to dispose them, + they will be disposed when the thread safe context is disposed */ + if (!(comp_ctx->modules[i] = LLVMModuleCreateWithNameInContext( + module_name, comp_ctx->context))) { + aot_set_last_error("create LLVM module failed."); + goto fail; + } + } + } +#endif if (BH_LIST_ERROR == bh_list_init(&comp_ctx->native_symbols)) { goto fail; } -#if WASM_ENABLE_DEBUG_AOT != 0 +#if WASM_ENABLE_DEBUG_AOT != 0 && WASM_ENABLE_LAZY_JIT == 0 if (!(comp_ctx->debug_builder = LLVMCreateDIBuilder(comp_ctx->module))) { aot_set_last_error("create LLVM Debug Infor builder failed."); goto fail; @@ -1497,10 +1590,11 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option) if (option->is_jit_mode) { char *triple_jit = NULL; + comp_ctx->is_jit_mode = true; + #if WASM_ENABLE_LAZY_JIT != 0 - /* Create LLLazyJIT Instance */ - if (!llvm_orcjit_create(comp_ctx)) { - aot_set_last_error("create LLVM Lazy JIT Compiler failed."); + /* Create LLJIT Instance */ + if (!orc_lazyjit_create(comp_ctx, comp_data->func_count)) { goto fail; } @@ -1524,7 +1618,6 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option) comp_ctx->target_machine = LLVMGetExecutionEngineTargetMachine(comp_ctx->exec_engine); #endif - comp_ctx->is_jit_mode = true; #ifndef OS_ENABLE_HW_BOUND_CHECK comp_ctx->enable_bound_check = true; @@ -1533,8 +1626,8 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option) #endif #if WASM_ENABLE_LAZY_JIT != 0 - if (!(triple_jit = (char *)LLVMOrcLLLazyJITGetTripleString( - comp_ctx->lazy_orcjit))) { + if (!(triple_jit = + (char *)LLVMOrcLLJITGetTripleString(comp_ctx->orc_lazyjit))) { aot_set_last_error("can not get triple from the target machine"); goto fail; } @@ -1768,9 +1861,17 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option) aot_set_last_error("create metadata string failed."); goto fail; } +#if WASM_ENABLE_LAZY_JIT == 0 LLVMAddModuleFlag(comp_ctx->module, LLVMModuleFlagBehaviorError, "target-abi", strlen("target-abi"), meta_target_abi); +#else + for (i = 0; i < comp_data->func_count; i++) { + LLVMAddModuleFlag(comp_ctx->modules[i], + LLVMModuleFlagBehaviorError, "target-abi", + strlen("target-abi"), meta_target_abi); + } +#endif if (!strcmp(abi, "lp64d") || !strcmp(abi, "ilp32d")) { if (features) { @@ -1878,7 +1979,9 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option) goto fail; } +#if WASM_ENABLE_LAZY_JIT == 0 LLVMSetTarget(comp_ctx->module, triple_norm); +#endif } if (option->enable_simd && strcmp(comp_ctx->target_arch, "x86_64") != 0 @@ -1921,11 +2024,18 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option) if (option->output_format == AOT_LLVMIR_UNOPT_FILE) comp_ctx->optimize = false; +#if WASM_ENABLE_LAZY_JIT == 0 if (!(comp_ctx->pass_mgr = LLVMCreateFunctionPassManagerForModule(comp_ctx->module))) { aot_set_last_error("create LLVM pass manager failed."); goto fail; } +#else + if (!(comp_ctx->pass_mgr = LLVMCreatePassManager())) { + aot_set_last_error("create LLVM pass manager failed."); + goto fail; + } +#endif LLVMAddPromoteMemoryToRegisterPass(comp_ctx->pass_mgr); LLVMAddInstructionCombiningPass(comp_ctx->pass_mgr); @@ -2012,6 +2122,7 @@ fail: if (!ret) aot_destroy_comp_context(comp_ctx); + (void)i; return ret; } @@ -2022,25 +2133,42 @@ aot_destroy_comp_context(AOTCompContext *comp_ctx) return; if (comp_ctx->pass_mgr) { +#if WASM_ENABLE_LAZY_JIT == 0 LLVMFinalizeFunctionPassManager(comp_ctx->pass_mgr); +#endif LLVMDisposePassManager(comp_ctx->pass_mgr); } #if WASM_ENABLE_LAZY_JIT != 0 - if (comp_ctx->target_machine && comp_ctx->is_jit_mode) + if (comp_ctx->orc_symbol_map_pairs) + wasm_runtime_free(comp_ctx->orc_symbol_map_pairs); + + if (comp_ctx->orc_call_through_mgr) + LLVMOrcDisposeLazyCallThroughManager(comp_ctx->orc_call_through_mgr); + + if (comp_ctx->orc_indirect_stub_mgr) + LLVMOrcDisposeIndirectStubsManager(comp_ctx->orc_indirect_stub_mgr); + + if (comp_ctx->orc_material_unit) + LLVMOrcDisposeMaterializationUnit(comp_ctx->orc_material_unit); + + if (comp_ctx->target_machine) LLVMDisposeTargetMachine(comp_ctx->target_machine); if (comp_ctx->builder) LLVMDisposeBuilder(comp_ctx->builder); - if (comp_ctx->lazy_orcjit) - LLVMOrcDisposeLLLazyJIT(comp_ctx->lazy_orcjit); + if (comp_ctx->orc_lazyjit) + LLVMOrcDisposeLLJIT(comp_ctx->orc_lazyjit); - if (comp_ctx->ts_context) - LLVMOrcDisposeThreadSafeContext(comp_ctx->ts_context); + if (comp_ctx->orc_thread_safe_context) + LLVMOrcDisposeThreadSafeContext(comp_ctx->orc_thread_safe_context); - if (comp_ctx->tm_builder) - LLVMOrcDisposeJITTargetMachineBuilder(comp_ctx->tm_builder); + if (comp_ctx->modules) + wasm_runtime_free(comp_ctx->modules); + + /* Note: don't dispose comp_ctx->context and comp_ctx->modules[i] as + they are disposed when disposing the thread safe context */ LLVMShutdown(); #else @@ -2237,7 +2365,7 @@ aot_block_stack_destroy(AOTBlockStack *stack) while (block) { p = block->next; aot_value_stack_destroy(&block->value_stack); - wasm_runtime_free(block); + aot_block_destroy(block); block = p; } } @@ -2433,7 +2561,7 @@ __call_llvm_intrinsic(const AOTCompContext *comp_ctx, } else { /* Declare llvm intrinsic function if necessary */ - if (!(func = LLVMGetNamedFunction(comp_ctx->module, name))) { + if (!(func = LLVMGetNamedFunction(func_ctx->module, name))) { if (!(func_type = LLVMFunctionType(ret_type, param_types, (uint32)param_count, false))) { aot_set_last_error( @@ -2441,7 +2569,7 @@ __call_llvm_intrinsic(const AOTCompContext *comp_ctx, return NULL; } - if (!(func = LLVMAddFunction(comp_ctx->module, name, func_type))) { + if (!(func = LLVMAddFunction(func_ctx->module, name, func_type))) { aot_set_last_error("add LLVM intrinsic function failed."); return NULL; } diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index f43df2b30..82f4b0dcb 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -20,11 +20,14 @@ #include "llvm-c/Transforms/PassManagerBuilder.h" #if WASM_ENABLE_LAZY_JIT != 0 -#include "aot_llvm_lazyjit.h" #include "llvm-c/Orc.h" #include "llvm-c/Error.h" -#include "llvm-c/Initialization.h" #include "llvm-c/Support.h" +#include "llvm-c/Initialization.h" +#include "llvm-c/TargetMachine.h" +#if LLVM_VERSION_MAJOR >= 12 +#include "llvm-c/LLJIT.h" +#endif #endif #if WASM_ENABLE_DEBUG_AOT != 0 #include "llvm-c/DebugInfo.h" @@ -128,6 +131,9 @@ typedef struct AOTFuncContext { AOTFunc *aot_func; LLVMValueRef func; LLVMTypeRef func_type; + /* LLVM module for this function, note that in LAZY JIT mode, + each aot function belongs to an individual module */ + LLVMModuleRef module; AOTBlockStack block_stack; LLVMValueRef exec_env; @@ -249,7 +255,12 @@ typedef struct AOTCompContext { /* LLVM variables required to emit LLVM IR */ LLVMContextRef context; +#if WASM_ENABLE_LAZY_JIT == 0 + /* Create one module only for non LAZY JIT mode, + for LAZY JIT mode, modules are created, each + aot function has its own module */ LLVMModuleRef module; +#endif LLVMBuilderRef builder; #if WASM_ENABLE_DEBUG_AOT LLVMDIBuilderRef debug_builder; @@ -266,12 +277,18 @@ typedef struct AOTCompContext { /* LLVM execution engine required by JIT */ #if WASM_ENABLE_LAZY_JIT != 0 - LLVMOrcLLLazyJITRef lazy_orcjit; - LLVMOrcThreadSafeContextRef ts_context; - LLVMOrcJITTargetMachineBuilderRef tm_builder; + LLVMOrcLLJITRef orc_lazyjit; + LLVMOrcMaterializationUnitRef orc_material_unit; + LLVMOrcLazyCallThroughManagerRef orc_call_through_mgr; + LLVMOrcIndirectStubsManagerRef orc_indirect_stub_mgr; + LLVMOrcCSymbolAliasMapPairs orc_symbol_map_pairs; + LLVMOrcThreadSafeContextRef orc_thread_safe_context; + /* Each aot function has its own module */ + LLVMModuleRef *modules; #else LLVMExecutionEngineRef exec_engine; #endif + bool is_jit_mode; /* AOT indirect mode flag & symbol list */ @@ -453,9 +470,19 @@ void aot_add_expand_memory_op_pass(LLVMPassManagerRef pass); #if WASM_ENABLE_LAZY_JIT != 0 +LLVMOrcJITTargetMachineBuilderRef +LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(LLVMTargetMachineRef TM); + void -aot_handle_llvm_errmsg(char *error_buf, uint32 error_buf_size, - const char *string, LLVMErrorRef error); +LLVMOrcLLJITBuilderSetNumCompileThreads(LLVMOrcLLJITBuilderRef orcjit_builder, + unsigned num_compile_threads); + +void +aot_handle_llvm_errmsg(const char *string, LLVMErrorRef err); + +void * +aot_lookup_orcjit_func(LLVMOrcLLJITRef orc_lazyjit, void *module_inst, + uint32 func_idx); #endif #ifdef __cplusplus diff --git a/core/iwasm/compilation/aot_llvm_extra.cpp b/core/iwasm/compilation/aot_llvm_extra.cpp index c6a059c60..f3aaf4ec8 100644 --- a/core/iwasm/compilation/aot_llvm_extra.cpp +++ b/core/iwasm/compilation/aot_llvm_extra.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -30,8 +31,12 @@ #include #include #include +#if WASM_ENABLE_LAZY_JIT != 0 +#include "../aot/aot_runtime.h" +#endif using namespace llvm; +using namespace llvm::orc; extern "C" LLVMBool WAMRCreateMCJITCompilerForModule(LLVMExecutionEngineRef *OutJIT, @@ -262,6 +267,63 @@ aot_check_simd_compatibility(const char *arch_c_str, const char *cpu_c_str) #endif /* WASM_ENABLE_SIMD */ } +#if LLVM_VERSION_MAJOR < 12 +LLVMOrcJITTargetMachineBuilderRef +LLVMOrcJITTargetMachineBuilderFromTargetMachine(LLVMTargetMachineRef TM); + +LLVMOrcJITTargetMachineBuilderRef +LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(LLVMTargetMachineRef TM) +{ + return LLVMOrcJITTargetMachineBuilderFromTargetMachine(TM); +} +#endif + +#if WASM_ENABLE_LAZY_JIT != 0 + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJITBuilder, LLVMOrcLLJITBuilderRef) + +void +LLVMOrcLLJITBuilderSetNumCompileThreads(LLVMOrcLLJITBuilderRef orcjit_builder, + unsigned num_compile_threads) +{ + unwrap(orcjit_builder)->setNumCompileThreads(num_compile_threads); +} + +void * +aot_lookup_orcjit_func(LLVMOrcLLJITRef orc_lazyjit, void *module_inst, + uint32 func_idx) +{ + char func_name[32], buf[128], *err_msg = NULL; + LLVMErrorRef error; + LLVMOrcJITTargetAddress func_addr = 0; + AOTModuleInstance *aot_inst = (AOTModuleInstance *)module_inst; + AOTModule *aot_module = (AOTModule *)aot_inst->aot_module.ptr; + void **func_ptrs = (void **)aot_inst->func_ptrs.ptr; + + /** + * No need to lock the func_ptr[func_idx] here as it is basic + * data type, the load/store for it can be finished by one cpu + * instruction, and there can be only one cpu instruction + * loading/storing at the same time. + */ + if (func_ptrs[func_idx]) + return func_ptrs[func_idx]; + + snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, + func_idx - aot_module->import_func_count); + if ((error = LLVMOrcLLJITLookup(orc_lazyjit, &func_addr, func_name))) { + err_msg = LLVMGetErrorMessage(error); + snprintf(buf, sizeof(buf), "failed to lookup orcjit function: %s", + err_msg); + aot_set_exception(aot_inst, buf); + LLVMDisposeErrorMessage(err_msg); + return NULL; + } + func_ptrs[func_idx] = (void *)func_addr; + return (void *)func_addr; +} +#endif + void aot_func_disable_tce(LLVMValueRef func) { diff --git a/core/iwasm/compilation/aot_llvm_lazyjit.cpp b/core/iwasm/compilation/aot_llvm_lazyjit.cpp deleted file mode 100644 index 858fbd2ab..000000000 --- a/core/iwasm/compilation/aot_llvm_lazyjit.cpp +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (C) 2019 Intel Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - */ - -#include "aot_llvm_lazyjit.h" - -LLVMOrcJITTargetMachineBuilderRef -LLVMOrcJITTargetMachineBuilderFromTargetMachine(LLVMTargetMachineRef TM); - -LLVMOrcLLJITBuilderRef -LLVMOrcCreateLLJITBuilder(void); - -void -LLVMOrcDisposeLLJITBuilder(LLVMOrcLLJITBuilderRef Builder); - -LLVMErrorRef -LLVMOrcCreateLLJIT(LLVMOrcLLJITRef *Result, LLVMOrcLLJITBuilderRef Builder); - -LLVMErrorRef -LLVMOrcDisposeLLJIT(LLVMOrcLLJITRef J); - -LLVMOrcJITDylibRef -LLVMOrcLLJITGetMainJITDylib(LLVMOrcLLJITRef J); - -const char * -LLVMOrcLLJITGetTripleString(LLVMOrcLLJITRef J); - -char -LLVMOrcLLJITGetGlobalPrefix(LLVMOrcLLJITRef J); - -LLVMErrorRef -LLVMOrcLLJITAddLLVMIRModule(LLVMOrcLLJITRef J, LLVMOrcJITDylibRef JD, - LLVMOrcThreadSafeModuleRef TSM); - -LLVMErrorRef -LLVMOrcLLJITLookup(LLVMOrcLLJITRef J, LLVMOrcJITTargetAddress *Result, - const char *Name); - -const char * -LLVMOrcLLJITGetTripleString(LLVMOrcLLJITRef J); - -void -LLVMOrcLLJITBuilderSetJITTargetMachineBuilder( - LLVMOrcLLJITBuilderRef Builder, LLVMOrcJITTargetMachineBuilderRef JTMB); - -char -LLVMOrcLLJITGetGlobalPrefix(LLVMOrcLLJITRef J); - -#if LLVM_VERSION_MAJOR < 12 -LLVMOrcJITTargetMachineBuilderRef -LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(LLVMTargetMachineRef TM) -{ - return LLVMOrcJITTargetMachineBuilderFromTargetMachine(TM); -} -#endif - -LLVMOrcJITDylibRef -LLVMOrcLLLazyJITGetMainJITDylib(LLVMOrcLLLazyJITRef J) -{ - return LLVMOrcLLJITGetMainJITDylib(J); -} - -LLVMOrcLLLazyJITBuilderRef -LLVMOrcCreateLLLazyJITBuilder(void) -{ - return LLVMOrcCreateLLJITBuilder(); -} - -void -LLVMOrcDisposeLLLazyJITBuilder(LLVMOrcLLLazyJITBuilderRef Builder) -{ - return LLVMOrcDisposeLLJITBuilder(Builder); -} - -LLVMErrorRef -LLVMOrcCreateLLLazyJIT(LLVMOrcLLLazyJITRef *Result, - LLVMOrcLLLazyJITBuilderRef Builder) -{ - return LLVMOrcCreateLLJIT(Result, Builder); -} - -LLVMErrorRef -LLVMOrcDisposeLLLazyJIT(LLVMOrcLLLazyJITRef J) -{ - return LLVMOrcDisposeLLJIT(J); -} - -LLVMErrorRef -LLVMOrcLLLazyJITAddLLVMIRModule(LLVMOrcLLLazyJITRef J, LLVMOrcJITDylibRef JD, - LLVMOrcThreadSafeModuleRef TSM) -{ - return LLVMOrcLLJITAddLLVMIRModule(J, JD, TSM); -} - -LLVMErrorRef -LLVMOrcLLLazyJITLookup(LLVMOrcLLLazyJITRef J, LLVMOrcJITTargetAddress *Result, - const char *Name) -{ - return LLVMOrcLLJITLookup(J, Result, Name); -} - -const char * -LLVMOrcLLLazyJITGetTripleString(LLVMOrcLLLazyJITRef J) -{ - return LLVMOrcLLJITGetTripleString(J); -} - -void -LLVMOrcLLLazyJITBuilderSetJITTargetMachineBuilder( - LLVMOrcLLLazyJITBuilderRef Builder, LLVMOrcJITTargetMachineBuilderRef JTMB) -{ - return LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(Builder, JTMB); -} - -char -LLVMOrcLLLazyJITGetGlobalPrefix(LLVMOrcLLLazyJITRef J) -{ - return LLVMOrcLLJITGetGlobalPrefix(J); -} diff --git a/core/iwasm/compilation/aot_llvm_lazyjit.h b/core/iwasm/compilation/aot_llvm_lazyjit.h deleted file mode 100644 index adb4b0220..000000000 --- a/core/iwasm/compilation/aot_llvm_lazyjit.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (C) 2019 Intel Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - */ - -#ifndef AOT_LLVM_LAZYJIT_H -#define AOT_LLVM_LAZYJIT_H - -#include "llvm-c/Error.h" -#include "llvm-c/Orc.h" -#include "llvm-c/TargetMachine.h" -#include "llvm-c/Types.h" -#if LLVM_VERSION_MAJOR >= 12 -#include "llvm-c/LLJIT.h" -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -typedef LLVMOrcLLJITBuilderRef LLVMOrcLLLazyJITBuilderRef; - -typedef LLVMOrcLLJITRef LLVMOrcLLLazyJITRef; - -LLVMOrcJITTargetMachineBuilderRef -LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(LLVMTargetMachineRef TM); - -LLVMOrcLLLazyJITBuilderRef -LLVMOrcCreateLLLazyJITBuilder(void); - -void -LLVMOrcDisposeLLLazyJITBuilder(LLVMOrcLLLazyJITBuilderRef Builder); - -LLVMErrorRef -LLVMOrcCreateLLLazyJIT(LLVMOrcLLLazyJITRef *Result, - LLVMOrcLLLazyJITBuilderRef Builder); - -LLVMErrorRef -LLVMOrcDisposeLLLazyJIT(LLVMOrcLLLazyJITRef J); - -LLVMOrcJITDylibRef -LLVMOrcLLLazyJITGetMainJITDylib(LLVMOrcLLLazyJITRef J); - -const char * -LLVMOrcLLLazyJITGetTripleString(LLVMOrcLLLazyJITRef J); - -char -LLVMOrcLLLazyJITGetGlobalPrefix(LLVMOrcLLLazyJITRef J); - -LLVMErrorRef -LLVMOrcLLLazyJITAddLLVMIRModule(LLVMOrcLLLazyJITRef J, LLVMOrcJITDylibRef JD, - LLVMOrcThreadSafeModuleRef TSM); - -LLVMErrorRef -LLVMOrcLLLazyJITLookup(LLVMOrcLLLazyJITRef J, LLVMOrcJITTargetAddress *Result, - const char *Name); - -const char * -LLVMOrcLLLazyJITGetTripleString(LLVMOrcLLLazyJITRef J); - -void -LLVMOrcLLLazyJITBuilderSetJITTargetMachineBuilder( - LLVMOrcLLLazyJITBuilderRef Builder, LLVMOrcJITTargetMachineBuilderRef JTMB); - -char -LLVMOrcLLLazyJITGetGlobalPrefix(LLVMOrcLLLazyJITRef J); - -#ifdef __cplusplus -} -#endif - -#endif /* end of AOT_LLVM_LAZYJIT_H */ diff --git a/core/shared/platform/common/posix/posix_thread.c b/core/shared/platform/common/posix/posix_thread.c index 4a20c10a0..0ae231c1b 100644 --- a/core/shared/platform/common/posix/posix_thread.c +++ b/core/shared/platform/common/posix/posix_thread.c @@ -32,7 +32,9 @@ os_thread_wrapper(void *arg) os_signal_handler handler = targ->signal_handler; #endif +#if 0 os_printf("THREAD CREATED %jx\n", (uintmax_t)(uintptr_t)pthread_self()); +#endif BH_FREE(targ); #ifdef OS_ENABLE_HW_BOUND_CHECK if (os_thread_signal_init(handler) != 0) diff --git a/core/shared/platform/esp-idf/espidf_thread.c b/core/shared/platform/esp-idf/espidf_thread.c index efdb1e8d2..c8d7a829d 100644 --- a/core/shared/platform/esp-idf/espidf_thread.c +++ b/core/shared/platform/esp-idf/espidf_thread.c @@ -22,7 +22,9 @@ os_thread_wrapper(void *arg) thread_start_routine_t start_func = targ->start; void *thread_arg = targ->arg; +#if 0 os_printf("THREAD CREATED %jx\n", (uintmax_t)(uintptr_t)pthread_self()); +#endif BH_FREE(targ); start_func(thread_arg); return NULL; diff --git a/core/shared/platform/linux-sgx/sgx_thread.c b/core/shared/platform/linux-sgx/sgx_thread.c index 34405a103..fbf407be0 100644 --- a/core/shared/platform/linux-sgx/sgx_thread.c +++ b/core/shared/platform/linux-sgx/sgx_thread.c @@ -18,7 +18,10 @@ os_thread_wrapper(void *arg) thread_wrapper_arg *targ = arg; thread_start_routine_t start_func = targ->start; void *thread_arg = targ->arg; + +#if 0 os_printf("THREAD CREATED %p\n", &targ); +#endif BH_FREE(targ); start_func(thread_arg); return NULL; diff --git a/core/shared/platform/windows/win_thread.c b/core/shared/platform/windows/win_thread.c index 408784d54..73b04dfa0 100644 --- a/core/shared/platform/windows/win_thread.c +++ b/core/shared/platform/windows/win_thread.c @@ -164,7 +164,9 @@ static unsigned __stdcall os_thread_wrapper(void *arg) void *retval; bool result; +#if 0 os_printf("THREAD CREATED %p\n", thread_data); +#endif os_mutex_lock(&parent->wait_lock); thread_data->thread_id = GetCurrentThreadId(); diff --git a/doc/build_wamr.md b/doc/build_wamr.md index 3ec5e327e..ff6edf9f4 100644 --- a/doc/build_wamr.md +++ b/doc/build_wamr.md @@ -41,7 +41,7 @@ cmake -DWAMR_BUILD_PLATFORM=linux -DWAMR_BUILD_TARGET=ARM - **WAMR_BUILD_AOT**=1/0, default to enable if not set - **WAMR_BUILD_JIT**=1/0, default to disable if not set -- **WAMR_BUILD_LAZY_JIT**=1/0, default to disable if not set +- **WAMR_BUILD_LAZY_JIT**=1/0, whether to use Lazy JIT mode or not when *WAMR_BUILD_JIT* is set, default to enable if not set #### **Configure LIBC** @@ -206,7 +206,7 @@ make ``` -By default in Linux, the interpreter, AOT and WASI are enabled, and JIT and LazyJIT are disabled. +By default in Linux, the interpreter, AOT and WASI are enabled, and JIT is disabled. And the build target is set to X86_64 or X86_32 depending on the platform's bitwidth. To enable WASM JIT, firstly we should build LLVM: @@ -225,10 +225,10 @@ cmake .. -DWAMR_BUILD_JIT=1 make ``` -Moreover, pass arguments `-DWAMR_BUILD_JIT=1` and `-DWAMR_BUILD_LAZY_JIT=1` together to cmake to enable WASM Lazy JIT. -If Lazy JIT is enabled, then jit function bodies in the module will not be compiled until they are first called, -so compile time reduces significantly. - +By default, the Lazy JIT is enabled to speedup the lanuching process and reduce the JIT compilation time +by creating threads to compile the WASM functions parallely, and for the main thread, the functions in the +module will not be compiled until they are firstly called and haven't been compiled by the compilation threads. +To disable it, please pass argument `-DWAMR_BUILD_LAZY_JIT=0` to cmake. Linux SGX (Intel Software Guard Extension) ------------------------- diff --git a/product-mini/platforms/android/build_jit.sh b/product-mini/platforms/android/build_jit.sh index 908d1560c..ffa440e95 100755 --- a/product-mini/platforms/android/build_jit.sh +++ b/product-mini/platforms/android/build_jit.sh @@ -6,5 +6,5 @@ rm -fr build && mkdir build cd build cmake .. -DWAMR_BUILD_JIT=1 -make +make -j ${nroc} cd .. diff --git a/product-mini/platforms/darwin/build_jit.sh b/product-mini/platforms/darwin/build_jit.sh index 908d1560c..a7d559107 100755 --- a/product-mini/platforms/darwin/build_jit.sh +++ b/product-mini/platforms/darwin/build_jit.sh @@ -6,5 +6,5 @@ rm -fr build && mkdir build cd build cmake .. -DWAMR_BUILD_JIT=1 -make +make -j ${nproc} cd .. diff --git a/product-mini/platforms/linux/build_jit.sh b/product-mini/platforms/linux/build_jit.sh index 8cfdb7369..f794a37c8 100755 --- a/product-mini/platforms/linux/build_jit.sh +++ b/product-mini/platforms/linux/build_jit.sh @@ -5,8 +5,8 @@ rm -fr build && mkdir build cd build -# Build With LazyJIT -# cmake .. -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1 +# By default LazyJIT is enabled, to disable it: +# cmake .. -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0 cmake .. -DWAMR_BUILD_JIT=1 -make +make -j ${nproc} cd ..