From f7ece614c443b00416cf895f49b0bc7adbeb0c7f Mon Sep 17 00:00:00 2001 From: Xu Jun <693788454@qq.com> Date: Mon, 20 Jun 2022 13:13:41 +0800 Subject: [PATCH] Support integrate 3rd-party toolchains into wamrc (#1237) Support integrating 3rd-party toolchain llc compiler or asm compiler into wamrc by setting environment variable WAMRC_LLC_COMPILER or WAMRC_ASM_COMPILER, wamrc will use these tools to generate object file from LLVM IR firstly, and then refactor the object file into aot file. --- core/iwasm/compilation/aot.h | 2 + core/iwasm/compilation/aot_compiler.c | 107 +++++++++++++++++++++ core/iwasm/compilation/aot_compiler.h | 4 + core/iwasm/compilation/aot_emit_aot_file.c | 35 ++++++- core/iwasm/compilation/aot_llvm.c | 45 +++++++++ core/iwasm/compilation/aot_llvm.h | 15 +++ doc/build_wasm_app.md | 30 ++++++ wamr-compiler/CMakeLists.txt | 4 + 8 files changed, 241 insertions(+), 1 deletion(-) diff --git a/core/iwasm/compilation/aot.h b/core/iwasm/compilation/aot.h index 78dc78e20..a94bf6469 100644 --- a/core/iwasm/compilation/aot.h +++ b/core/iwasm/compilation/aot.h @@ -15,7 +15,9 @@ extern "C" { #endif +#ifndef AOT_FUNC_PREFIX #define AOT_FUNC_PREFIX "aot_func#" +#endif typedef InitializerExpression AOTInitExpr; typedef WASMType AOTFuncType; diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index d1bb081ad..09c3be273 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -2886,6 +2886,36 @@ aot_compile_wasm(AOTCompContext *comp_ctx) return true; } +#if !(defined(_WIN32) || defined(_WIN32_)) +char * +aot_generate_tempfile_name(const char *prefix, const char *extension, + char *buffer, uint32 len) +{ + int fd, name_len; + + name_len = snprintf(buffer, len, "%s-XXXXXX", prefix); + + if ((fd = mkstemp(buffer)) <= 0) { + aot_set_last_error("make temp file failed."); + return NULL; + } + + /* close and remove temp file */ + close(fd); + unlink(buffer); + + /* Check if buffer length is enough */ + /* name_len + '.' + extension + '\0' */ + if (name_len + 1 + strlen(extension) + 1 > len) { + aot_set_last_error("temp file name too long."); + return NULL; + } + + snprintf(buffer + name_len, len - name_len, ".%s", extension); + return buffer; +} +#endif /* end of !(defined(_WIN32) || defined(_WIN32_)) */ + #if WASM_ENABLE_LAZY_JIT == 0 bool aot_emit_llvm_file(AOTCompContext *comp_ctx, const char *file_name) @@ -2915,6 +2945,83 @@ aot_emit_object_file(AOTCompContext *comp_ctx, char *file_name) bh_print_time("Begin to emit object file"); +#if !(defined(_WIN32) || defined(_WIN32_)) + if (comp_ctx->external_llc_compiler || comp_ctx->external_asm_compiler) { + char cmd[1024]; + int ret; + + if (comp_ctx->external_llc_compiler) { + char bc_file_name[64]; + + if (!aot_generate_tempfile_name("wamrc-bc", "bc", bc_file_name, + sizeof(bc_file_name))) { + return false; + } + + if (LLVMWriteBitcodeToFile(comp_ctx->module, bc_file_name) != 0) { + aot_set_last_error("emit llvm bitcode file failed."); + return false; + } + + snprintf(cmd, sizeof(cmd), "%s %s -o %s %s", + comp_ctx->external_llc_compiler, + comp_ctx->llc_compiler_flags ? comp_ctx->llc_compiler_flags + : "-O3 -c", + file_name, bc_file_name); + LOG_VERBOSE("invoking external LLC compiler:\n\t%s", cmd); + + ret = system(cmd); + /* remove temp bitcode file */ + unlink(bc_file_name); + + if (ret != 0) { + aot_set_last_error("failed to compile LLVM bitcode to obj file " + "with external LLC compiler."); + return false; + } + } + else if (comp_ctx->external_asm_compiler) { + char asm_file_name[64]; + + if (!aot_generate_tempfile_name("wamrc-asm", "s", asm_file_name, + sizeof(asm_file_name))) { + return false; + } + + if (LLVMTargetMachineEmitToFile(comp_ctx->target_machine, + comp_ctx->module, asm_file_name, + LLVMAssemblyFile, &err) + != 0) { + if (err) { + LLVMDisposeMessage(err); + err = NULL; + } + aot_set_last_error("emit elf to assembly file failed."); + return false; + } + + snprintf(cmd, sizeof(cmd), "%s %s -o %s %s", + comp_ctx->external_asm_compiler, + comp_ctx->asm_compiler_flags ? comp_ctx->asm_compiler_flags + : "-O3 -c", + file_name, asm_file_name); + LOG_VERBOSE("invoking external ASM compiler:\n\t%s", cmd); + + ret = system(cmd); + /* remove temp assembly file */ + unlink(asm_file_name); + + if (ret != 0) { + aot_set_last_error("failed to compile Assembly file to obj " + "file with external ASM compiler."); + return false; + } + } + + return true; + } +#endif /* end of !(defined(_WIN32) || defined(_WIN32_)) */ + if (!strncmp(LLVMGetTargetName(target), "arc", 3)) /* Emit to assmelby file instead for arc target as it cannot emit to object file */ diff --git a/core/iwasm/compilation/aot_compiler.h b/core/iwasm/compilation/aot_compiler.h index 4efd1bfb0..d5e45f28b 100644 --- a/core/iwasm/compilation/aot_compiler.h +++ b/core/iwasm/compilation/aot_compiler.h @@ -371,6 +371,10 @@ aot_emit_aot_file_buf(AOTCompContext *comp_ctx, AOTCompData *comp_data, bool aot_emit_object_file(AOTCompContext *comp_ctx, char *file_name); +char * +aot_generate_tempfile_name(const char *prefix, const char *extension, + char *buffer, uint32 len); + #ifdef __cplusplus } /* end of extern "C" */ #endif diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index 69d415779..2a46ff9be 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -2692,8 +2692,41 @@ aot_obj_data_create(AOTCompContext *comp_ctx) memset(obj_data, 0, sizeof(AOTObjectData)); bh_print_time("Begin to emit object file"); + if (comp_ctx->external_llc_compiler || comp_ctx->external_asm_compiler) { +#if defined(_WIN32) || defined(_WIN32_) + aot_set_last_error("external toolchain not supported on Windows"); + goto fail; +#else + /* Generate a temp file name */ + int ret; + char obj_file_name[64]; - if (!strncmp(LLVMGetTargetName(target), "arc", 3)) { + if (!aot_generate_tempfile_name("wamrc-obj", "o", obj_file_name, + sizeof(obj_file_name))) { + goto fail; + } + + if (!aot_emit_object_file(comp_ctx, obj_file_name)) { + goto fail; + } + + /* create memory buffer from object file */ + ret = LLVMCreateMemoryBufferWithContentsOfFile( + obj_file_name, &obj_data->mem_buf, &err); + /* remove temp object file */ + unlink(obj_file_name); + + if (ret != 0) { + if (err) { + LLVMDisposeMessage(err); + err = NULL; + } + aot_set_last_error("create mem buffer with file failed."); + goto fail; + } +#endif /* end of defined(_WIN32) || defined(_WIN32_) */ + } + else if (!strncmp(LLVMGetTargetName(target), "arc", 3)) { #if defined(_WIN32) || defined(_WIN32_) aot_set_last_error("emit object file on Windows is unsupported."); goto fail; diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 3d8f2b18d..f4449afd3 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1660,6 +1660,51 @@ aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option) opt_level = option->opt_level; size_level = option->size_level; + /* verify external llc compiler */ + comp_ctx->external_llc_compiler = getenv("WAMRC_LLC_COMPILER"); + if (comp_ctx->external_llc_compiler) { +#if defined(_WIN32) || defined(_WIN32_) + comp_ctx->external_llc_compiler = NULL; + LOG_WARNING("External LLC compiler not supported on Windows."); +#else + if (access(comp_ctx->external_llc_compiler, X_OK) != 0) { + LOG_WARNING("WAMRC_LLC_COMPILER [%s] not found, fallback to " + "default pipeline", + comp_ctx->external_llc_compiler); + comp_ctx->external_llc_compiler = NULL; + } + else { + comp_ctx->llc_compiler_flags = getenv("WAMRC_LLC_FLAGS"); + LOG_VERBOSE("Using external LLC compiler [%s]", + comp_ctx->external_llc_compiler); + } +#endif + } + + /* verify external asm compiler */ + if (!comp_ctx->external_llc_compiler) { + comp_ctx->external_asm_compiler = getenv("WAMRC_ASM_COMPILER"); + if (comp_ctx->external_asm_compiler) { +#if defined(_WIN32) || defined(_WIN32_) + comp_ctx->external_asm_compiler = NULL; + LOG_WARNING("External ASM compiler not supported on Windows."); +#else + if (access(comp_ctx->external_asm_compiler, X_OK) != 0) { + LOG_WARNING( + "WAMRC_ASM_COMPILER [%s] not found, fallback to " + "default pipeline", + comp_ctx->external_asm_compiler); + comp_ctx->external_asm_compiler = NULL; + } + else { + comp_ctx->asm_compiler_flags = getenv("WAMRC_ASM_FLAGS"); + LOG_VERBOSE("Using external ASM compiler [%s]", + comp_ctx->external_asm_compiler); + } +#endif + } + } + if (arch) { /* Add default sub-arch if not specified */ if (!strcmp(arch, "arm")) diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index ae8b31d6f..3e321f45d 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -14,6 +14,7 @@ #include "llvm-c/Object.h" #include "llvm-c/ExecutionEngine.h" #include "llvm-c/Analysis.h" +#include "llvm-c/BitWriter.h" #include "llvm-c/Transforms/Utils.h" #include "llvm-c/Transforms/Scalar.h" #include "llvm-c/Transforms/Vectorize.h" @@ -350,6 +351,20 @@ typedef struct AOTCompContext { uint32 func_ctx_count; char **custom_sections_wp; uint32 custom_sections_count; + + /* 3rd-party toolchains */ + /* External llc compiler, if specified, wamrc will emit the llvm-ir file and + * invoke the llc compiler to generate object file. + * This can be used when we want to benefit from the optimization of other + * LLVM based toolchains */ + const char *external_llc_compiler; + const char *llc_compiler_flags; + /* External asm compiler, if specified, wamrc will emit the text-based + * assembly file (.s) and invoke the llc compiler to generate object file. + * This will be useful when the upstream LLVM doesn't support to emit object + * file for some architecture (such as arc) */ + const char *external_asm_compiler; + const char *asm_compiler_flags; } AOTCompContext; enum { diff --git a/doc/build_wasm_app.md b/doc/build_wasm_app.md index a8698e11f..4034b1f5e 100644 --- a/doc/build_wasm_app.md +++ b/doc/build_wasm_app.md @@ -330,6 +330,36 @@ Examples: wamrc -o test.aot test.wasm wamrc --target=i386 --format=object -o test.o test.wasm ``` +## AoT compilation with 3rd-party toolchains + +`wamrc` uses LLVM to compile wasm bytecode to AoT file, this works for most of the architectures, but there may be circumstances where you want to use 3rd-party toolchains to take over some steps of the compilation pipeline, e.g. + +1. The upstream LLVM doesn't support generating object file for your CPU architecture (such as ARC), then we may need some other assembler to do such things. +2. You may get some other LLVM-based toolchains which may have better optimizations for the specific target, then you may want your toolchain to take over all optimization steps. + +`wamrc` provides two environment variables to achieve these: +- `WAMRC_LLC_COMPILER` + + When specified, `wamrc` will emit the optimized LLVM-IR (.bc) to a file, and invoke `$WAMRC_LLC_COMPILER` with ` -c -O3 ` to generate the object file. + + Optionally, you can use environment variable `WAMRC_LLC_FLAGS` to overwrite the default flags. + +- `WAMRC_ASM_COMPILER` + + When specified, `wamrc` will emit the text based assembly file (.s), and invoke `$WAMRC_ASM_COMPILER` with ` -c -O3 ` to generate the object file. + + Optionally, you can use environment variable `WAMRC_ASM_FLAGS` to overwrite the default flags. + +### Usage example +``` bash +WAMRC_LLC_COMPILER= ./wamrc -o test.aot test.wasm +``` + +> Note: `wamrc` will verify whether the specified file exists and executable. If verification failed, `wamrc` will report a warning and fallback to normal pipeline. Since the verification is based on file, you **must specify the absolute path to the binary** even if it's in `$PATH` + +> Note: `WAMRC_LLC_COMPILER` has higher priority than `WAMRC_ASM_COMPILER`, if `WAMRC_LLC_COMPILER` is set and verified, then `WAMRC_ASM_COMPILER` will be ignored. + +> Note: the `LLC` and `ASM` in the env name just means this compiler will be used to compile the `LLVM IR file`/`assembly file` to object file, usually passing the compiler driver is the simplest way. (e.g. for LLVM toolchain, you don't need to pass `/usr/bin/llc`, using `/usr/bin/clang` is OK) Run WASM app in WAMR mini product build ======================================= diff --git a/wamr-compiler/CMakeLists.txt b/wamr-compiler/CMakeLists.txt index 1445abab2..d38157fbb 100644 --- a/wamr-compiler/CMakeLists.txt +++ b/wamr-compiler/CMakeLists.txt @@ -41,6 +41,10 @@ if (WAMR_BUILD_LLVM_LEGACY_PM EQUAL 1) add_definitions(-DWASM_ENABLE_LLVM_LEGACY_PM=1) endif() +if (DEFINED WAMR_BUILD_AOT_FUNC_PREFIX) + add_definitions(-DAOT_FUNC_PREFIX="${WAMR_BUILD_AOT_FUNC_PREFIX}") +endif () + # Set WAMR_BUILD_TARGET, currently values supported: # "X86_64", "AMD_64", "X86_32", "ARM_32", "MIPS_32", "XTENSA_32" if (NOT WAMR_BUILD_TARGET)