Enable AOT linux perf support (#2930)

And refactor the original perf support
- use WAMR_BUILD_LINUX_PERF as the cmake compilation control
- use WASM_ENABLE_LINUX_PERF as the compiler macro
- use `wamrc --enable-linux-perf` to generate aot file which contains fp operations
- use `iwasm --enable-linux-perf` to create perf map for `perf record`

build-scripts/config_common.cmake

@@ -147,13 +147,20 @@ elseif (WAMR_BUILD_SANITIZER STREQUAL "asan")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fno-omit-frame-pointer -fsanitize=address -fno-sanitize-recover=all" )
     set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
   endif()
-elseif (WAMR_BUILD_SANITIZER STREQUAL "tsan") 
+elseif (WAMR_BUILD_SANITIZER STREQUAL "tsan")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" )
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread")
 elseif (NOT (WAMR_BUILD_SANITIZER STREQUAL "") )
   message(SEND_ERROR "Unsupported sanitizer: ${WAMR_BUILD_SANITIZER}")
 endif()
 
+if (WAMR_BUILD_LINUX_PERF EQUAL 1)
+  if (NOT WAMR_BUILD_JIT AND NOT WAMR_BUILD_AOT)
+    message(WARNING "only support perf in aot and llvm-jit")
+    set(WAMR_BUILD_LINUX_PERF 0)
+  endif ()
+endif ()
+
 ########################################
 
 message ("-- Build Configurations:")
@@ -440,3 +447,7 @@ if (WAMR_CONFIGUABLE_BOUNDS_CHECKS EQUAL 1)
   add_definitions (-DWASM_CONFIGURABLE_BOUNDS_CHECKS=1)
   message ("     Configurable bounds checks enabled")
 endif ()
+if (WAMR_BUILD_LINUX_PERF EQUAL 1)
+  add_definitions (-DWASM_ENABLE_LINUX_PERF=1)
+  message ("     Enable linux perf support")
+endif ()

core/config.h

@@ -490,4 +490,9 @@
 #define WASM_MAX_INSTANCE_CONTEXTS 8
 #endif
 
+/* linux perf support */
+#ifndef WASM_ENABLE_LINUX_PERF
+#define WASM_ENABLE_LINUX_PERF 0
+#endif
+
 #endif /* end of _CONFIG_H_ */

core/iwasm/aot/aot_loader.c

@@ -2764,6 +2764,104 @@ fail:
     return ret;
 }
 
+#if WASM_ENABLE_LINUX_PERF != 0
+struct func_info {
+    uint32 idx;
+    void *ptr;
+};
+
+static uint32
+get_func_size(const AOTModule *module, struct func_info *sorted_func_ptrs,
+              uint32 idx)
+{
+    uint32 func_sz;
+
+    if (idx == module->func_count - 1)
+        func_sz = (uintptr_t)module->code + module->code_size
+                  - (uintptr_t)(sorted_func_ptrs[idx].ptr);
+    else
+        func_sz = (uintptr_t)(sorted_func_ptrs[idx + 1].ptr)
+                  - (uintptr_t)(sorted_func_ptrs[idx].ptr);
+
+    return func_sz;
+}
+
+static int
+compare_func_ptrs(const void *f1, const void *f2)
+{
+    return (intptr_t)((struct func_info *)f1)->ptr
+           - (intptr_t)((struct func_info *)f2)->ptr;
+}
+
+static struct func_info *
+sort_func_ptrs(const AOTModule *module, char *error_buf, uint32 error_buf_size)
+{
+    uint64 content_len;
+    struct func_info *sorted_func_ptrs;
+    unsigned i;
+
+    content_len = (uint64)sizeof(struct func_info) * module->func_count;
+    sorted_func_ptrs = loader_malloc(content_len, error_buf, error_buf_size);
+    if (!sorted_func_ptrs)
+        return NULL;
+
+    for (i = 0; i < module->func_count; i++) {
+        sorted_func_ptrs[i].idx = i;
+        sorted_func_ptrs[i].ptr = module->func_ptrs[i];
+    }
+
+    qsort(sorted_func_ptrs, module->func_count, sizeof(struct func_info),
+          compare_func_ptrs);
+
+    return sorted_func_ptrs;
+}
+
+static bool
+create_perf_map(const AOTModule *module, char *error_buf, uint32 error_buf_size)
+{
+    struct func_info *sorted_func_ptrs = NULL;
+    char perf_map_info[128] = { 0 };
+    FILE *perf_map = NULL;
+    uint32 i;
+    pid_t pid = getpid();
+    bool ret = false;
+
+    sorted_func_ptrs = sort_func_ptrs(module, error_buf, error_buf_size);
+    if (!sorted_func_ptrs)
+        goto quit;
+
+    snprintf(perf_map_info, 128, "/tmp/perf-%d.map", pid);
+    perf_map = fopen(perf_map_info, "w");
+    if (!perf_map) {
+        LOG_WARNING("warning: can't create /tmp/perf-%d.map, because %s", pid,
+                    strerror(errno));
+        goto quit;
+    }
+
+    for (i = 0; i < module->func_count; i++) {
+        memset(perf_map_info, 0, 128);
+        snprintf(perf_map_info, 128, "%lx  %x  aot_func#%u\n",
+                 (uintptr_t)sorted_func_ptrs[i].ptr,
+                 get_func_size(module, sorted_func_ptrs, i),
+                 sorted_func_ptrs[i].idx);
+
+        fwrite(perf_map_info, 1, strlen(perf_map_info), perf_map);
+    }
+
+    LOG_VERBOSE("generate /tmp/perf-%d.map", pid);
+    ret = true;
+
+quit:
+    if (sorted_func_ptrs)
+        free(sorted_func_ptrs);
+
+    if (perf_map)
+        fclose(perf_map);
+
+    return ret;
+}
+#endif /* WASM_ENABLE_LINUX_PERF != 0*/
+
 static bool
 load_from_sections(AOTModule *module, AOTSection *sections,
                    bool is_load_from_file_buf, char *error_buf,
@@ -3224,6 +3322,12 @@ load(const uint8 *buf, uint32 size, AOTModule *module, char *error_buf,
     }
 #endif
 
+#if WASM_ENABLE_LINUX_PERF != 0
+    if (wasm_runtime_get_linux_perf())
+        if (!create_perf_map(module, error_buf, error_buf_size))
+            goto fail;
+#endif
+
     return ret;
 fail:
     return false;

core/iwasm/common/wasm_c_api.c

@@ -330,7 +330,7 @@ wasm_config_set_linux_perf_opt(wasm_config_t *config, bool enable)
     if (!config)
         return NULL;
 
-    config->linux_perf_support = enable;
+    config->enable_linux_perf = enable;
     return config;
 }
 
@@ -380,7 +380,8 @@ wasm_engine_new_internal(wasm_config_t *config)
     init_args.mem_alloc_type = config->mem_alloc_type;
     memcpy(&init_args.mem_alloc_option, &config->mem_alloc_option,
            sizeof(MemAllocOption));
-    init_args.linux_perf_support = config->linux_perf_support;
+
+    init_args.enable_linux_perf = config->enable_linux_perf;
 
     if (!wasm_runtime_full_init(&init_args)) {
         LOG_DEBUG("wasm_runtime_full_init failed");

core/iwasm/common/wasm_runtime_common.c

@@ -158,7 +158,7 @@ static JitCompOptions jit_options = { 0 };
 #endif
 
 #if WASM_ENABLE_JIT != 0
-static LLVMJITOptions llvm_jit_options = { 3, 3, 0, false };
+static LLVMJITOptions llvm_jit_options = { 3, 3, 0 };
 #endif
 
 static RunningMode runtime_running_mode = Mode_Default;
@@ -662,14 +662,17 @@ wasm_runtime_full_init(RuntimeInitArgs *init_args)
 #endif
 
 #if WASM_ENABLE_JIT != 0
-    LOG_DEBUG("Start LLVM_JIT, opt_sz=%u, opt_lvl=%u, segue=%s, linux_perf=%s",
-              init_args->llvm_jit_size_level, init_args->llvm_jit_opt_level,
-              init_args->segue_flags ? "Yes" : "No",
-              init_args->linux_perf_support ? "Yes" : "No");
     llvm_jit_options.size_level = init_args->llvm_jit_size_level;
     llvm_jit_options.opt_level = init_args->llvm_jit_opt_level;
     llvm_jit_options.segue_flags = init_args->segue_flags;
-    llvm_jit_options.linux_perf_support = init_args->linux_perf_support;
+#endif
+
+#if WASM_ENABLE_LINUX_PERF != 0
+    wasm_runtime_set_linux_perf(init_args->enable_linux_perf);
+#else
+    if (init_args->enable_linux_perf)
+        LOG_WARNING("warning: to enable linux perf support, please recompile "
+                    "with -DWAMR_BUILD_LINUX_PERF=1");
 #endif
 
     if (!wasm_runtime_env_init()) {
@@ -6146,3 +6149,19 @@ wasm_runtime_get_context(WASMModuleInstanceCommon *inst, void *key)
     return wasm_native_get_context(inst, key);
 }
 #endif /* WASM_ENABLE_MODULE_INST_CONTEXT != 0 */
+
+#if WASM_ENABLE_LINUX_PERF != 0
+static bool enable_linux_perf = false;
+
+bool
+wasm_runtime_get_linux_perf(void)
+{
+    return enable_linux_perf;
+}
+
+void
+wasm_runtime_set_linux_perf(bool flag)
+{
+    enable_linux_perf = flag;
+}
+#endif

core/iwasm/common/wasm_runtime_common.h

@@ -443,7 +443,6 @@ typedef struct LLVMJITOptions {
     uint32 opt_level;
     uint32 size_level;
     uint32 segue_flags;
-    bool linux_perf_support;
 } LLVMJITOptions;
 #endif
 
@@ -1105,6 +1104,14 @@ wasm_runtime_end_blocking_op(WASMExecEnv *exec_env);
 void
 wasm_runtime_interrupt_blocking_op(WASMExecEnv *exec_env);
 
+#if WASM_ENABLE_LINUX_PERF != 0
+bool
+wasm_runtime_get_linux_perf(void);
+
+void
+wasm_runtime_set_linux_perf(bool flag);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

core/iwasm/compilation/aot_llvm.c

@@ -2174,7 +2174,7 @@ jit_stack_size_callback(void *user_data, const char *name, size_t namelen,
 }
 
 static bool
-orc_jit_create(AOTCompContext *comp_ctx, bool linux_perf_support)
+orc_jit_create(AOTCompContext *comp_ctx)
 {
     LLVMErrorRef err;
     LLVMOrcLLLazyJITRef orc_jit = NULL;
@@ -2214,13 +2214,15 @@ orc_jit_create(AOTCompContext *comp_ctx, bool linux_perf_support)
     /* Ownership transfer: LLVMOrcLLJITBuilderRef -> LLVMOrcLLJITRef */
     builder = NULL;
 
-    if (linux_perf_support) {
+#if WASM_ENABLE_LINUX_PERF != 0
-        LOG_DEBUG("Enable linux perf support");
+    if (wasm_runtime_get_linux_perf()) {
+        LOG_DEBUG("Enable linux perf support in JIT");
         LLVMOrcObjectLayerRef obj_linking_layer =
             (LLVMOrcObjectLayerRef)LLVMOrcLLLazyJITGetObjLinkingLayer(orc_jit);
         LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(
             obj_linking_layer, LLVMCreatePerfJITEventListener());
     }
+#endif
 
     /* Ownership transfer: local -> AOTCompContext */
     comp_ctx->orc_jit = orc_jit;
@@ -2320,7 +2322,8 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
         goto fail;
     }
 
-    if (option->linux_perf_support) {
+#if WASM_ENABLE_LINUX_PERF != 0
+    if (wasm_runtime_get_linux_perf()) {
         /* FramePointerKind.All */
         LLVMMetadataRef val =
             LLVMValueAsMetadata(LLVMConstInt(LLVMInt32Type(), 2, false));
@@ -2330,6 +2333,7 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
 
         comp_ctx->emit_frame_pointer = true;
     }
+#endif
 
     if (BH_LIST_ERROR == bh_list_init(&comp_ctx->native_symbols)) {
         goto fail;
@@ -2434,7 +2438,7 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
             goto fail;
 
         /* Create LLJIT Instance */
-        if (!orc_jit_create(comp_ctx, option->linux_perf_support))
+        if (!orc_jit_create(comp_ctx))
             goto fail;
     }
     else {

core/iwasm/compilation/aot_llvm.h

@@ -461,7 +461,6 @@ typedef struct AOTCompOption {
     uint32 bounds_checks;
     uint32 stack_bounds_checks;
     uint32 segue_flags;
-    bool linux_perf_support;
     char **custom_sections;
     uint32 custom_sections_count;
     const char *stack_usage_file;

core/iwasm/fast-jit/jit_compiler.h

@@ -70,7 +70,6 @@ typedef struct JitInterpSwitchInfo {
 typedef struct JitCompOptions {
     uint32 code_cache_size;
     uint32 opt_level;
-    bool linux_perf_support;
 } JitCompOptions;
 
 bool

core/iwasm/include/aot_export.h

@@ -65,7 +65,6 @@ typedef struct AOTCompOption {
     uint32_t bounds_checks;
     uint32_t stack_bounds_checks;
     uint32_t segue_flags;
-    bool linux_perf_support;
     char **custom_sections;
     uint32_t custom_sections_count;
     const char *stack_usage_file;

core/iwasm/include/wasm_c_api.h

@@ -181,7 +181,7 @@ typedef union MemAllocOption {
 struct wasm_config_t {
     mem_alloc_type_t mem_alloc_type;
     MemAllocOption mem_alloc_option;
-    bool linux_perf_support;
+    bool enable_linux_perf;
     /*TODO: wasi args*/
 };
 
@@ -189,7 +189,7 @@ struct wasm_config_t {
  * by default:
  * - mem_alloc_type is Alloc_With_System_Allocator
  * - mem_alloc_option is all 0
- * - linux_perf_support is false
+ * - enable_linux_perf is false
  */
 WASM_API_EXTERN own wasm_config_t* wasm_config_new(void);
 

core/iwasm/include/wasm_export.h

@@ -172,12 +172,12 @@ typedef struct RuntimeInitArgs {
     /**
      * If enabled
      * - llvm-jit will output a jitdump file for `perf inject`
-     * - aot. TBD
+     * - aot will output a perf-${pid}.map for `perf record`
      * - fast-jit. TBD
      * - multi-tier-jit. TBD
      * - interpreter. TBD
      */
-    bool linux_perf_support;
+    bool enable_linux_perf;
 } RuntimeInitArgs;
 
 #ifndef WASM_VALKIND_T_DEFINED
@@ -243,8 +243,8 @@ WASM_RUNTIME_API_EXTERN bool
 wasm_runtime_full_init(RuntimeInitArgs *init_args);
 
 /**
- * Set the log level. To be called after the runtime is initialized. 
+ * Set the log level. To be called after the runtime is initialized.
- * 
+ *
  * @param level the log level to set
  */
 WASM_RUNTIME_API_EXTERN void

core/iwasm/interpreter/wasm_loader.c

@@ -2892,7 +2892,6 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
     option.opt_level = llvm_jit_options.opt_level;
     option.size_level = llvm_jit_options.size_level;
     option.segue_flags = llvm_jit_options.segue_flags;
-    option.linux_perf_support = llvm_jit_options.linux_perf_support;
 
 #if WASM_ENABLE_BULK_MEMORY != 0
     option.enable_bulk_memory = true;

core/iwasm/interpreter/wasm_mini_loader.c

@@ -1877,7 +1877,6 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
     option.opt_level = llvm_jit_options.opt_level;
     option.size_level = llvm_jit_options.size_level;
     option.segue_flags = llvm_jit_options.segue_flags;
-    option.linux_perf_support = llvm_jit_options.linux_perf_support;
 
 #if WASM_ENABLE_BULK_MEMORY != 0
     option.enable_bulk_memory = true;

product-mini/platforms/posix/main.c

@@ -58,7 +58,6 @@ print_help()
 #if WASM_ENABLE_JIT != 0
     printf("  --llvm-jit-size-level=n  Set LLVM JIT size level, default is 3\n");
     printf("  --llvm-jit-opt-level=n   Set LLVM JIT optimization level, default is 3\n");
-    printf("  --perf-profile           Enable linux perf support. For now, it only works in llvm-jit.\n");
 #if defined(os_writegsbase)
     printf("  --enable-segue[=<flags>] Enable using segment register GS as the base address of\n");
     printf("                           linear memory, which may improve performance, flags can be:\n");
@@ -67,6 +66,9 @@ print_help()
     printf("                           Use comma to separate, e.g. --enable-segue=i32.load,i64.store\n");
     printf("                           and --enable-segue means all flags are added.\n");
 #endif
+#endif /* WASM_ENABLE_JIT != 0*/
+#if WASM_ENABLE_LINUX_PERF != 0
+    printf("  --enable-linux-perf      Enable linux perf support. It works in aot and llvm-jit.\n");
 #endif
     printf("  --repl                   Start a very simple REPL (read-eval-print-loop) mode\n"
            "                           that runs commands in the form of \"FUNC ARG...\"\n");
@@ -561,7 +563,9 @@ main(int argc, char *argv[])
     uint32 llvm_jit_size_level = 3;
     uint32 llvm_jit_opt_level = 3;
     uint32 segue_flags = 0;
-    bool enable_linux_perf_support = false;
+#endif
+#if WASM_ENABLE_LINUX_PERF != 0
+    bool enable_linux_perf = false;
 #endif
     wasm_module_t wasm_module = NULL;
     wasm_module_inst_t wasm_module_inst = NULL;
@@ -702,9 +706,6 @@ main(int argc, char *argv[])
             if (segue_flags == (uint32)-1)
                 return print_help();
         }
-        else if (!strncmp(argv[0], "--perf-profile", 14)) {
-            enable_linux_perf_support = true;
-        }
 #endif /* end of WASM_ENABLE_JIT != 0 */
 #if BH_HAS_DLFCN
         else if (!strncmp(argv[0], "--native-lib=", 13)) {
@@ -718,6 +719,11 @@ main(int argc, char *argv[])
             native_lib_list[native_lib_count++] = argv[0] + 13;
         }
 #endif
+#if WASM_ENABLE_LINUX_PERF != 0
+        else if (!strncmp(argv[0], "--enable-linux-perf", 19)) {
+            enable_linux_perf = true;
+        }
+#endif
 #if WASM_ENABLE_MULTI_MODULE != 0
         else if (!strncmp(argv[0],
                           "--module-path=", strlen("--module-path="))) {
@@ -819,7 +825,9 @@ main(int argc, char *argv[])
     init_args.llvm_jit_size_level = llvm_jit_size_level;
     init_args.llvm_jit_opt_level = llvm_jit_opt_level;
     init_args.segue_flags = segue_flags;
-    init_args.linux_perf_support = enable_linux_perf_support;
+#endif
+#if WASM_ENABLE_LINUX_PERF != 0
+    init_args.enable_linux_perf = enable_linux_perf;
 #endif
 
 #if WASM_ENABLE_DEBUG_INTERP != 0

wamr-compiler/CMakeLists.txt

@@ -48,7 +48,11 @@ add_definitions(-DWASM_ENABLE_MODULE_INST_CONTEXT=1)
 
 if (WAMR_BUILD_LLVM_LEGACY_PM EQUAL 1)
   add_definitions(-DWASM_ENABLE_LLVM_LEGACY_PM=1)
-endif()
+endif ()
+
+if (LINUX)
+  add_definitions(-DWASM_ENABLE_LINUX_PERF=1)
+endif ()
 
 if (DEFINED WAMR_BUILD_AOT_FUNC_PREFIX)
   add_definitions(-DAOT_FUNC_PREFIX="${WAMR_BUILD_AOT_FUNC_PREFIX}")

wamr-compiler/main.c

@@ -184,9 +184,12 @@ print_help()
     printf("                            multiple names, e.g.\n");
     printf("                                --emit-custom-sections=section1,section2,sectionN\n");
 #if BH_HAS_DLFCN
-    printf("  --native-lib=<lib>       Register native libraries to the WASM module, which\n");
+    printf("  --native-lib=<lib>        Register native libraries to the WASM module, which\n");
-    printf("                           are shared object (.so) files, for example:\n");
+    printf("                            are shared object (.so) files, for example:\n");
-    printf("                             --native-lib=test1.so --native-lib=test2.so\n");
+    printf("                              --native-lib=test1.so --native-lib=test2.so\n");
+#endif
+#if WASM_ENABLE_LINUX_PERF != 0
+    printf("  --enable-linux-perf       Enable linux perf support\n");
 #endif
     printf("  -v=n                      Set log verbose level (0 to 5, default is 2), larger with more log\n");
     printf("  --version                 Show version information\n");
@@ -325,6 +328,9 @@ main(int argc, char *argv[])
     void *native_handle_list[8] = { NULL };
     uint32 native_handle_count = 0;
 #endif
+#if WASM_ENABLE_LINUX_PERF != 0
+    bool enable_linux_perf = false;
+#endif
 
     option.opt_level = 3;
     option.size_level = 3;
@@ -525,6 +531,11 @@ main(int argc, char *argv[])
             }
             native_lib_list[native_lib_count++] = argv[0] + 13;
         }
+#endif
+#if WASM_ENABLE_LINUX_PERF != 0
+        else if (!strncmp(argv[0], "--enable-linux-perf", 19)) {
+            enable_linux_perf = true;
+        }
 #endif
         else if (!strncmp(argv[0], "--version", 9)) {
             uint32 major, minor, patch;
@@ -579,6 +590,9 @@ main(int argc, char *argv[])
     init_args.mem_alloc_option.allocator.malloc_func = malloc;
     init_args.mem_alloc_option.allocator.realloc_func = realloc;
     init_args.mem_alloc_option.allocator.free_func = free;
+#if WASM_ENABLE_LINUX_PERF != 0
+    init_args.enable_linux_perf = enable_linux_perf;
+#endif
 
     /* initialize runtime environment */
     if (!wasm_runtime_full_init(&init_args)) {