Merge fbb5e21e98 into 17be90d8f0

CMakeLists.txt

@@ -119,11 +119,6 @@ if (NOT DEFINED WAMR_BUILD_REF_TYPES)
   set (WAMR_BUILD_REF_TYPES 1)
 endif ()
 
-if (NOT DEFINED WAMR_BUILD_BRANCH_HINTS)
-  # Enable branch hints by default
-  set (WAMR_BUILD_BRANCH_HINTS 1)
-endif ()
-
 set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)

build-scripts/SConscript_config

@@ -128,10 +128,6 @@ if GetDepend(['WAMR_BUILD_REF_TYPES']):
     CPPDEFINES += ['WASM_ENABLE_REF_TYPES=1']
     print('[WAMR] enable ref types')
 
-if GetDepend(['WAMR_BUILD_BRANCH_HINTS']):
-    CPPDEFINES += ['WASM_ENABLE_BRANCH_HINTS=1']
-    print('[WAMR] enable branch hints')
-
 CPPDEFINES += ['BH_MALLOC=wasm_runtime_malloc']
 CPPDEFINES += ['BH_FREE=wasm_runtime_free']
 

build-scripts/config_common.cmake

@@ -215,10 +215,6 @@ if (NOT DEFINED WAMR_BUILD_EXTENDED_CONST_EXPR)
   set (WAMR_BUILD_EXTENDED_CONST_EXPR 0)
 endif ()
 
-if (NOT DEFINED WAMR_BUILD_BRANCH_HINTS)
-  set (WAMR_BUILD_BRANCH_HINTS 1)
-endif ()
-
 ########################################
 # Compilation options to marco
 ########################################
@@ -423,12 +419,6 @@ endif ()
 if (WAMR_BUILD_REF_TYPES EQUAL 1)
   add_definitions (-DWASM_ENABLE_REF_TYPES=1)
 endif ()
-if (WAMR_BUILD_BRANCH_HINTS EQUAL 1)
-  add_definitions (-DWASM_ENABLE_BRANCH_HINTS=1)
-  message ("     branch hints enabled")
-else ()
-  message ("     branch hints disabled")
-endif ()
 if (WAMR_BUILD_GC EQUAL 1)
   if (WAMR_TEST_GC EQUAL 1)
     message("      GC testing enabled")
@@ -718,7 +708,6 @@ message (
 "       \"Tail call\" via WAMR_BUILD_TAIL_CALL: ${WAMR_BUILD_TAIL_CALL}\n"
 "       \"Threads\" via WAMR_BUILD_SHARED_MEMORY: ${WAMR_BUILD_SHARED_MEMORY}\n"
 "       \"Typed Function References\" via WAMR_BUILD_GC: ${WAMR_BUILD_GC}\n"
-"       \"Branch Hinting\" via WAMR_BUILD_BRANCH_HINTS: ${WAMR_BUILD_BRANCH_HINTS}\n"
 "     Unsupported (>= Phase4):\n"
 "       \"Custom Annotation Syntax in the Text Format\"\n"
 "       \"Exception handling\"\n"

core/iwasm/aot/aot_loader.c

@@ -505,13 +505,6 @@ check_feature_flags(char *error_buf, uint32 error_buf_size,
     }
 #endif
 
-#if WASM_ENABLE_BRANCH_HINTS == 0
-    if (feature_flags & WASM_ENABLE_BRANCH_HINTS) {
-        LOG_WARNING(
-            "branch hints not enabled, but wasm file contains branch hints");
-    }
-#endif
-
 #if WASM_ENABLE_GC == 0
     if (feature_flags & WASM_FEATURE_GARBAGE_COLLECTION) {
         set_error_buf(error_buf, error_buf_size,

core/iwasm/aot/aot_runtime.h

@@ -35,7 +35,6 @@ extern "C" {
  * and not at the beginning of each function call */
 #define WASM_FEATURE_FRAME_PER_FUNCTION (1 << 12)
 #define WASM_FEATURE_FRAME_NO_FUNC_IDX (1 << 13)
-#define WASM_FEATURE_BRANCH_HINTS (1 << 14)
 
 typedef enum AOTSectionType {
     AOT_SECTION_TYPE_TARGET_INFO = 0,
@@ -58,7 +57,6 @@ typedef enum AOTCustomSectionType {
     AOT_CUSTOM_SECTION_ACCESS_CONTROL = 2,
     AOT_CUSTOM_SECTION_NAME = 3,
     AOT_CUSTOM_SECTION_STRING_LITERAL = 4,
-    AOT_CUSTOM_SECTION_CODE_METADATA = 5,
 } AOTCustomSectionType;
 
 typedef struct AOTObjectDataSection {

core/iwasm/compilation/aot.c

@@ -392,7 +392,6 @@ aot_create_funcs(const WASMModule *module, uint32 pointer_size)
         memset(aot_func, 0, sizeof(AOTFunc));
 
         func_type = aot_func->func_type = func->func_type;
-        aot_func->func_index = i + module->import_function_count;
 
         /* Resolve function type index */
         for (j = 0; j < module->type_count; j++) {

core/iwasm/compilation/aot.h

@@ -207,7 +207,6 @@ typedef struct AOTImportFunc {
 typedef struct AOTFunc {
     AOTFuncType *func_type;
     uint32 func_type_index;
-    uint32 func_index;
     uint32 local_count;
     uint8 *local_types_wp;
     uint16 param_cell_num;

core/iwasm/compilation/aot_compiler.c

@@ -1158,16 +1158,7 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index)
 
             case WASM_OP_BR_IF:
             {
-                // ip is advanced by one byte for the opcode
-#if WASM_ENABLE_BRANCH_HINTS != 0
-                uint32 instr_offset =
-                    (frame_ip - 0x1) - (func_ctx->aot_func->code_body_begin);
-#else
-                uint32 instr_offset = 0;
-#endif
-                read_leb_uint32(frame_ip, frame_ip_end, br_depth);
-                if (!aot_compile_op_br_if(comp_ctx, func_ctx, br_depth,
-                                          &frame_ip, instr_offset))
+                if (!aot_compile_op_br_if(comp_ctx, func_ctx, &frame_ip))
                     return false;
                 break;
             }

core/iwasm/compilation/aot_emit_aot_file.c

@@ -4501,9 +4501,6 @@ aot_obj_data_create(AOTCompContext *comp_ctx)
     if (comp_ctx->enable_ref_types) {
         obj_data->target_info.feature_flags |= WASM_FEATURE_REF_TYPES;
     }
-    if (comp_ctx->enable_branch_hints) {
-        obj_data->target_info.feature_flags |= WASM_FEATURE_BRANCH_HINTS;
-    }
     if (comp_ctx->enable_gc) {
         obj_data->target_info.feature_flags |= WASM_FEATURE_GARBAGE_COLLECTION;
     }

core/iwasm/compilation/aot_emit_control.c

@@ -12,6 +12,7 @@
 #endif
 #include "../aot/aot_runtime.h"
 #include "../interpreter/wasm_loader.h"
+#include "../common/wasm_loader_common.h"
 
 #if WASM_ENABLE_DEBUG_AOT != 0
 #include "debug/dwarf_extractor.h"
@@ -87,14 +88,13 @@ format_block_name(char *name, uint32 name_size, uint32 block_index,
         }                                                             \
     } while (0)
 
-#define BUILD_COND_BR_V(value_if, block_then, block_else, instr)             \
-    do {                                                                     \
-        if (instr = LLVMBuildCondBr(comp_ctx->builder, value_if, block_then, \
-                                    block_else),                             \
-            !instr) {                                                        \
-            aot_set_last_error("llvm build cond br failed.");                \
-            goto fail;                                                       \
-        }                                                                    \
+#define BUILD_COND_BR_V(value_if, block_then, block_else, instr)               \
+    do {                                                                       \
+        if (!(instr = LLVMBuildCondBr(comp_ctx->builder, value_if, block_then, \
+                                      block_else))) {                          \
+            aot_set_last_error("llvm build cond br failed.");                  \
+            goto fail;                                                         \
+        }                                                                      \
     } while (0)
 
 #define SET_BUILDER_POS(llvm_block) \
@@ -267,34 +267,30 @@ restore_frame_sp_for_op_end(AOTBlock *block, AOTCompFrame *aot_frame)
 
 #if WASM_ENABLE_BRANCH_HINTS != 0
 static void
-aot_emit_branch_hint(LLVMContextRef ctxt, AOTFuncContext *func_ctx,
+aot_emit_branch_hint(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
                      uint32 offset, LLVMValueRef br_if_instr)
 {
-    struct WASMCompilationHintBranchHint *hint =
-        (struct WASMCompilationHintBranchHint *)func_ctx->function_hints;
+    struct WASMCompilationHint *hint = func_ctx->function_hints;
     while (hint != NULL) {
         if (hint->type == WASM_COMPILATION_BRANCH_HINT
-            && hint->offset == offset) {
+            && ((struct WASMCompilationHintBranchHint *)hint)->offset
+                   == offset) {
             break;
         }
         hint = hint->next;
     }
     if (hint != NULL) {
-        LLVMMetadataRef header = LLVMMDStringInContext2(
-            ctxt, "branch_weights", 14 /* strlen("branch_hint") */);
         // same weight llvm MDBuilder::createLikelyBranchWeights assigns
         const uint32_t likely_weight = (1U << 20) - 1;
         const uint32_t unlikely_weight = 1;
-        LLVMMetadataRef true_w = LLVMValueAsMetadata(LLVMConstInt(
-            LLVMInt32TypeInContext(ctxt),
-            hint->is_likely ? likely_weight : unlikely_weight, false));
-        LLVMMetadataRef false_w = LLVMValueAsMetadata(LLVMConstInt(
-            LLVMInt32TypeInContext(ctxt),
-            hint->is_likely ? unlikely_weight : likely_weight, false));
-        LLVMMetadataRef mds[] = { header, true_w, false_w };
-        LLVMMetadataRef md = LLVMMDNodeInContext2(ctxt, mds, 3);
-        LLVMValueRef md_val = LLVMMetadataAsValue(ctxt, md);
-        LLVMSetMetadata(br_if_instr, LLVMGetMDKindID("prof", 4), md_val);
+        aot_set_cond_br_weights(
+            comp_ctx, br_if_instr,
+            ((struct WASMCompilationHintBranchHint *)hint)->is_likely
+                ? likely_weight
+                : unlikely_weight,
+            ((struct WASMCompilationHintBranchHint *)hint)->is_likely
+                ? unlikely_weight
+                : likely_weight);
     }
 }
 #endif
@@ -723,8 +719,7 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
                                 block->llvm_else_block, br_if_val);
                 const uint32 off =
                     *p_frame_ip - func_ctx->aot_func->code_body_begin;
-                aot_emit_branch_hint(comp_ctx->context, func_ctx, off,
-                                     br_if_val);
+                aot_emit_branch_hint(comp_ctx, func_ctx, off, br_if_val);
 #else
                 BUILD_COND_BR(value, block->llvm_entry_block,
                               block->llvm_else_block);
@@ -738,8 +733,7 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
                                 block->llvm_end_block, br_if_val);
                 const uint32 off =
                     *p_frame_ip - func_ctx->aot_func->code_body_begin;
-                aot_emit_branch_hint(comp_ctx->context, func_ctx, off,
-                                     br_if_val);
+                aot_emit_branch_hint(comp_ctx, func_ctx, off, br_if_val);
 #else
                 BUILD_COND_BR(value, block->llvm_entry_block,
                               block->llvm_end_block);
@@ -1090,8 +1084,7 @@ fail:
 
 static bool
 aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
-                           uint32 br_depth, LLVMValueRef value_cmp,
-                           uint8 **p_frame_ip, uint32 off)
+                           LLVMValueRef value_cmp, uint8 **p_frame_ip)
 {
     AOTBlock *block_dst;
     LLVMValueRef value, *values = NULL;
@@ -1100,6 +1093,17 @@ aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     uint32 i, param_index, result_index;
     uint64 size;
 
+    // ip is advanced by one byte for the opcode
+#if WASM_ENABLE_BRANCH_HINTS != 0
+    uint32 instr_offset =
+        (*p_frame_ip - 0x1) - (func_ctx->aot_func->code_body_begin);
+#else
+    uint32 instr_offset = 0;
+#endif
+    uint64 br_depth;
+    if (!read_leb(p_frame_ip, *p_frame_ip + 5, 32, false, &br_depth, NULL, 0))
+        return false;
+
     if (!(block_dst = get_target_block(func_ctx, br_depth))) {
         return false;
     }
@@ -1176,7 +1180,7 @@ aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
             LLVMValueRef br_if_val = NULL;
             BUILD_COND_BR_V(value_cmp, block_dst->llvm_entry_block,
                             llvm_else_block, br_if_val);
-            aot_emit_branch_hint(comp_ctx->context, func_ctx, off, br_if_val);
+            aot_emit_branch_hint(comp_ctx, func_ctx, instr_offset, br_if_val);
 #else
             BUILD_COND_BR(value_cmp, block_dst->llvm_entry_block,
                           llvm_else_block);
@@ -1227,7 +1231,7 @@ aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
             LLVMValueRef br_if_val = NULL;
             BUILD_COND_BR_V(value_cmp, block_dst->llvm_end_block,
                             llvm_else_block, br_if_val);
-            aot_emit_branch_hint(comp_ctx->context, func_ctx, off, br_if_val);
+            aot_emit_branch_hint(comp_ctx, func_ctx, instr_offset, br_if_val);
 #else
             BUILD_COND_BR(value_cmp, block_dst->llvm_end_block,
                           llvm_else_block);
@@ -1255,14 +1259,14 @@ fail:
 
 bool
 aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
-                     uint32 br_depth, uint8 **p_frame_ip, uint32 off)
+                     uint8 **p_frame_ip)
 {
     LLVMValueRef value_cmp;
 
     POP_COND(value_cmp);
 
-    return aot_compile_conditional_br(comp_ctx, func_ctx, br_depth, value_cmp,
-                                      p_frame_ip, off);
+    return aot_compile_conditional_br(comp_ctx, func_ctx, value_cmp,
+                                      p_frame_ip);
 fail:
     return false;
 }

core/iwasm/compilation/aot_emit_control.h

@@ -32,7 +32,7 @@ aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
 
 bool
 aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
-                     uint32 br_depth, uint8 **p_frame_ip, uint32 offset);
+                     uint8 **p_frame_ip);
 
 bool
 aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,

core/iwasm/compilation/aot_llvm.c

@@ -2740,9 +2740,6 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
     if (option->enable_ref_types)
         comp_ctx->enable_ref_types = true;
 
-    if (option->enable_branch_hints)
-        comp_ctx->enable_branch_hints = true;
-
     comp_ctx->aux_stack_frame_type = option->aux_stack_frame_type;
     comp_ctx->call_stack_features = option->call_stack_features;
 

core/iwasm/compilation/aot_llvm.h

@@ -452,9 +452,6 @@ typedef struct AOTCompContext {
     /* Reference Types */
     bool enable_ref_types;
 
-    /* Branch Hinting */
-    bool enable_branch_hints;
-
     /* Disable LLVM built-in intrinsics */
     bool disable_llvm_intrinsics;
 

core/iwasm/include/aot_comp_option.h

@@ -66,7 +66,6 @@ typedef struct AOTCompOption {
     bool enable_tail_call;
     bool enable_simd;
     bool enable_ref_types;
-    bool enable_branch_hints;
     bool enable_gc;
     bool enable_aux_stack_check;
     bool enable_extended_const;

core/iwasm/interpreter/wasm.h

@@ -775,7 +775,7 @@ struct WASMCompilationHint {
     enum WASMCompilationHintType type;
 };
 struct WASMCompilationHintBranchHint {
-    void *next;
+    struct WASMCompilationHint *next;
     enum WASMCompilationHintType type;
     uint32 offset;
     bool is_likely;

core/iwasm/interpreter/wasm_loader.c

@@ -5561,8 +5561,8 @@ handle_branch_hint_section(const uint8 *buf, const uint8 *buf_end,
             goto fail;
         }
 
-        struct WASMCompilationHintBranchHint *current_hint =
-            (struct WASMCompilationHintBranchHint *)&module
+        struct WASMCompilationHint *current_hint =
+            (struct WASMCompilationHint *)&module
                 ->function_hints[func_idx - module->import_function_count];
         while (current_hint->next != NULL) {
             current_hint = current_hint->next;
@@ -5570,10 +5570,11 @@ handle_branch_hint_section(const uint8 *buf, const uint8 *buf_end,
 
         uint32 num_hints;
         read_leb_uint32(buf, buf_end, num_hints);
+        struct WASMCompilationHintBranchHint *new_hints = loader_malloc(
+            sizeof(struct WASMCompilationHintBranchHint) * num_hints, error_buf,
+            error_buf_size);
         for (uint32 j = 0; j < num_hints; ++j) {
-            struct WASMCompilationHintBranchHint *new_hint =
-                loader_malloc(sizeof(struct WASMCompilationHintBranchHint),
-                              error_buf, error_buf_size);
+            struct WASMCompilationHintBranchHint *new_hint = &new_hints[j];
             new_hint->next = NULL;
             new_hint->type = WASM_COMPILATION_BRANCH_HINT;
             read_leb_uint32(buf, buf_end, new_hint->offset);
@@ -5601,8 +5602,8 @@ handle_branch_hint_section(const uint8 *buf, const uint8 *buf_end,
                 goto fail;
             }
 
-            current_hint->next = new_hint;
-            current_hint = new_hint;
+            current_hint->next = (struct WASMCompilationHint *)new_hint;
+            current_hint = (struct WASMCompilationHint *)new_hint;
         }
     }
     if (buf != buf_end) {
@@ -5666,7 +5667,8 @@ load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
 #endif
 
 #if WASM_ENABLE_BRANCH_HINTS != 0
-    if (name_len == 25 && memcmp(p, "metadata.code.branch_hint", 25) == 0) {
+    if (name_len == 25
+        && strncmp((const char *)p, "metadata.code.branch_hint", 25) == 0) {
         p += name_len;
         if (!handle_branch_hint_section(p, p_end, module, error_buf,
                                         error_buf_size)) {
@@ -5674,6 +5676,12 @@ load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
         }
         LOG_VERBOSE("Load branch hint section success.");
     }
+#else
+    if (name_len == 25
+        && strncmp((const char *)p, "metadata.code.branch_hint", 25) == 0) {
+        LOG_VERBOSE("Found branch hint section, but branch hints are disabled "
+                    "in this build, skipping.");
+    }
 #endif
 
 #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
@@ -7463,7 +7471,16 @@ wasm_loader_unload(WASMModule *module)
     }
 #endif
 #endif
-
+#if WASM_ENABLE_BRANCH_HINTS != 0
+    for (size_t i = 0; i < module->function_count; i++) {
+        // be carefull when adding more hints. This only works as long as
+        // the hint structs have been allocated all at once as an array.
+        // With only branch-hints at the moment, this is the case.
+        if (module->function_hints[i] != NULL)
+            wasm_runtime_free(module->function_hints[i]);
+    }
+    wasm_runtime_free(module->function_hints);
+#endif
     wasm_runtime_free(module);
 }