mirror of
https://github.com/bytecodealliance/wasm-micro-runtime.git
synced 2025-07-11 06:53:16 +00:00
Compare commits
3 Commits
16a2ce284b
...
6af65b0d86
Author | SHA1 | Date | |
---|---|---|---|
![]() |
6af65b0d86 | ||
![]() |
17be90d8f0 | ||
![]() |
9ea7ff60f1 |
|
@ -119,6 +119,11 @@ if (NOT DEFINED WAMR_BUILD_REF_TYPES)
|
||||||
set (WAMR_BUILD_REF_TYPES 1)
|
set (WAMR_BUILD_REF_TYPES 1)
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
|
if (NOT DEFINED WAMR_BUILD_BRANCH_HINTS)
|
||||||
|
# Enable branch hints by default
|
||||||
|
set (WAMR_BUILD_BRANCH_HINTS 1)
|
||||||
|
endif ()
|
||||||
|
|
||||||
set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR})
|
set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR})
|
||||||
|
|
||||||
include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)
|
include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake)
|
||||||
|
|
|
@ -128,6 +128,10 @@ if GetDepend(['WAMR_BUILD_REF_TYPES']):
|
||||||
CPPDEFINES += ['WASM_ENABLE_REF_TYPES=1']
|
CPPDEFINES += ['WASM_ENABLE_REF_TYPES=1']
|
||||||
print('[WAMR] enable ref types')
|
print('[WAMR] enable ref types')
|
||||||
|
|
||||||
|
if GetDepend(['WAMR_BUILD_BRANCH_HINTS']):
|
||||||
|
CPPDEFINES += ['WASM_ENABLE_BRANCH_HINTS=1']
|
||||||
|
print('[WAMR] enable branch hints')
|
||||||
|
|
||||||
CPPDEFINES += ['BH_MALLOC=wasm_runtime_malloc']
|
CPPDEFINES += ['BH_MALLOC=wasm_runtime_malloc']
|
||||||
CPPDEFINES += ['BH_FREE=wasm_runtime_free']
|
CPPDEFINES += ['BH_FREE=wasm_runtime_free']
|
||||||
|
|
||||||
|
|
|
@ -215,6 +215,10 @@ if (NOT DEFINED WAMR_BUILD_EXTENDED_CONST_EXPR)
|
||||||
set (WAMR_BUILD_EXTENDED_CONST_EXPR 0)
|
set (WAMR_BUILD_EXTENDED_CONST_EXPR 0)
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
|
if (NOT DEFINED WAMR_BUILD_BRANCH_HINTS)
|
||||||
|
set (WAMR_BUILD_BRANCH_HINTS 1)
|
||||||
|
endif ()
|
||||||
|
|
||||||
########################################
|
########################################
|
||||||
# Compilation options to marco
|
# Compilation options to marco
|
||||||
########################################
|
########################################
|
||||||
|
@ -419,6 +423,12 @@ endif ()
|
||||||
if (WAMR_BUILD_REF_TYPES EQUAL 1)
|
if (WAMR_BUILD_REF_TYPES EQUAL 1)
|
||||||
add_definitions (-DWASM_ENABLE_REF_TYPES=1)
|
add_definitions (-DWASM_ENABLE_REF_TYPES=1)
|
||||||
endif ()
|
endif ()
|
||||||
|
if (WAMR_BUILD_BRANCH_HINTS EQUAL 1)
|
||||||
|
add_definitions (-DWASM_ENABLE_BRANCH_HINTS=1)
|
||||||
|
message (" branch hints enabled")
|
||||||
|
else ()
|
||||||
|
message (" branch hints disabled")
|
||||||
|
endif ()
|
||||||
if (WAMR_BUILD_GC EQUAL 1)
|
if (WAMR_BUILD_GC EQUAL 1)
|
||||||
if (WAMR_TEST_GC EQUAL 1)
|
if (WAMR_TEST_GC EQUAL 1)
|
||||||
message(" GC testing enabled")
|
message(" GC testing enabled")
|
||||||
|
@ -708,8 +718,8 @@ message (
|
||||||
" \"Tail call\" via WAMR_BUILD_TAIL_CALL: ${WAMR_BUILD_TAIL_CALL}\n"
|
" \"Tail call\" via WAMR_BUILD_TAIL_CALL: ${WAMR_BUILD_TAIL_CALL}\n"
|
||||||
" \"Threads\" via WAMR_BUILD_SHARED_MEMORY: ${WAMR_BUILD_SHARED_MEMORY}\n"
|
" \"Threads\" via WAMR_BUILD_SHARED_MEMORY: ${WAMR_BUILD_SHARED_MEMORY}\n"
|
||||||
" \"Typed Function References\" via WAMR_BUILD_GC: ${WAMR_BUILD_GC}\n"
|
" \"Typed Function References\" via WAMR_BUILD_GC: ${WAMR_BUILD_GC}\n"
|
||||||
|
" \"Branch Hinting\" via WAMR_BUILD_BRANCH_HINTS: ${WAMR_BUILD_BRANCH_HINTS}\n"
|
||||||
" Unsupported (>= Phase4):\n"
|
" Unsupported (>= Phase4):\n"
|
||||||
" \"Branch Hinting\"\n"
|
|
||||||
" \"Custom Annotation Syntax in the Text Format\"\n"
|
" \"Custom Annotation Syntax in the Text Format\"\n"
|
||||||
" \"Exception handling\"\n"
|
" \"Exception handling\"\n"
|
||||||
" \"Import/Export of Mutable Globals\"\n"
|
" \"Import/Export of Mutable Globals\"\n"
|
||||||
|
|
|
@ -579,6 +579,10 @@ unless used elsewhere */
|
||||||
#define WASM_ENABLE_REF_TYPES 0
|
#define WASM_ENABLE_REF_TYPES 0
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#ifndef WASM_ENABLE_BRANCH_HINTS
|
||||||
|
#define WASM_ENABLE_BRANCH_HINTS 0
|
||||||
|
#endif
|
||||||
|
|
||||||
#ifndef WASM_ENABLE_GC
|
#ifndef WASM_ENABLE_GC
|
||||||
#define WASM_ENABLE_GC 0
|
#define WASM_ENABLE_GC 0
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -505,6 +505,13 @@ check_feature_flags(char *error_buf, uint32 error_buf_size,
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS == 0
|
||||||
|
if (feature_flags & WASM_ENABLE_BRANCH_HINTS) {
|
||||||
|
LOG_WARNING(
|
||||||
|
"branch hints not enabled, but wasm file contains branch hints");
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
#if WASM_ENABLE_GC == 0
|
#if WASM_ENABLE_GC == 0
|
||||||
if (feature_flags & WASM_FEATURE_GARBAGE_COLLECTION) {
|
if (feature_flags & WASM_FEATURE_GARBAGE_COLLECTION) {
|
||||||
set_error_buf(error_buf, error_buf_size,
|
set_error_buf(error_buf, error_buf_size,
|
||||||
|
|
|
@ -35,6 +35,7 @@ extern "C" {
|
||||||
* and not at the beginning of each function call */
|
* and not at the beginning of each function call */
|
||||||
#define WASM_FEATURE_FRAME_PER_FUNCTION (1 << 12)
|
#define WASM_FEATURE_FRAME_PER_FUNCTION (1 << 12)
|
||||||
#define WASM_FEATURE_FRAME_NO_FUNC_IDX (1 << 13)
|
#define WASM_FEATURE_FRAME_NO_FUNC_IDX (1 << 13)
|
||||||
|
#define WASM_FEATURE_BRANCH_HINTS (1 << 14)
|
||||||
|
|
||||||
typedef enum AOTSectionType {
|
typedef enum AOTSectionType {
|
||||||
AOT_SECTION_TYPE_TARGET_INFO = 0,
|
AOT_SECTION_TYPE_TARGET_INFO = 0,
|
||||||
|
@ -57,6 +58,7 @@ typedef enum AOTCustomSectionType {
|
||||||
AOT_CUSTOM_SECTION_ACCESS_CONTROL = 2,
|
AOT_CUSTOM_SECTION_ACCESS_CONTROL = 2,
|
||||||
AOT_CUSTOM_SECTION_NAME = 3,
|
AOT_CUSTOM_SECTION_NAME = 3,
|
||||||
AOT_CUSTOM_SECTION_STRING_LITERAL = 4,
|
AOT_CUSTOM_SECTION_STRING_LITERAL = 4,
|
||||||
|
AOT_CUSTOM_SECTION_CODE_METADATA = 5,
|
||||||
} AOTCustomSectionType;
|
} AOTCustomSectionType;
|
||||||
|
|
||||||
typedef struct AOTObjectDataSection {
|
typedef struct AOTObjectDataSection {
|
||||||
|
|
|
@ -392,6 +392,7 @@ aot_create_funcs(const WASMModule *module, uint32 pointer_size)
|
||||||
memset(aot_func, 0, sizeof(AOTFunc));
|
memset(aot_func, 0, sizeof(AOTFunc));
|
||||||
|
|
||||||
func_type = aot_func->func_type = func->func_type;
|
func_type = aot_func->func_type = func->func_type;
|
||||||
|
aot_func->func_index = i + module->import_function_count;
|
||||||
|
|
||||||
/* Resolve function type index */
|
/* Resolve function type index */
|
||||||
for (j = 0; j < module->type_count; j++) {
|
for (j = 0; j < module->type_count; j++) {
|
||||||
|
@ -416,6 +417,9 @@ aot_create_funcs(const WASMModule *module, uint32 pointer_size)
|
||||||
aot_func->local_types_wp = func->local_types;
|
aot_func->local_types_wp = func->local_types;
|
||||||
aot_func->code = func->code;
|
aot_func->code = func->code;
|
||||||
aot_func->code_size = func->code_size;
|
aot_func->code_size = func->code_size;
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
aot_func->code_body_begin = func->code_body_begin;
|
||||||
|
#endif
|
||||||
|
|
||||||
/* Resolve local offsets */
|
/* Resolve local offsets */
|
||||||
for (j = 0; j < func_type->param_count; j++) {
|
for (j = 0; j < func_type->param_count; j++) {
|
||||||
|
@ -872,6 +876,10 @@ aot_create_comp_data(WASMModule *module, const char *target_arch,
|
||||||
comp_data->name_section_buf_end = module->name_section_buf_end;
|
comp_data->name_section_buf_end = module->name_section_buf_end;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
comp_data->function_hints = module->function_hints;
|
||||||
|
#endif
|
||||||
|
|
||||||
aot_init_aux_data(comp_data, module);
|
aot_init_aux_data(comp_data, module);
|
||||||
|
|
||||||
comp_data->wasm_module = module;
|
comp_data->wasm_module = module;
|
||||||
|
|
|
@ -207,6 +207,7 @@ typedef struct AOTImportFunc {
|
||||||
typedef struct AOTFunc {
|
typedef struct AOTFunc {
|
||||||
AOTFuncType *func_type;
|
AOTFuncType *func_type;
|
||||||
uint32 func_type_index;
|
uint32 func_type_index;
|
||||||
|
uint32 func_index;
|
||||||
uint32 local_count;
|
uint32 local_count;
|
||||||
uint8 *local_types_wp;
|
uint8 *local_types_wp;
|
||||||
uint16 param_cell_num;
|
uint16 param_cell_num;
|
||||||
|
@ -217,6 +218,9 @@ typedef struct AOTFunc {
|
||||||
/* offset of each local, including function parameters
|
/* offset of each local, including function parameters
|
||||||
and local variables */
|
and local variables */
|
||||||
uint16 *local_offsets;
|
uint16 *local_offsets;
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
uint8 *code_body_begin;
|
||||||
|
#endif
|
||||||
} AOTFunc;
|
} AOTFunc;
|
||||||
|
|
||||||
typedef struct AOTCompData {
|
typedef struct AOTCompData {
|
||||||
|
@ -296,6 +300,10 @@ typedef struct AOTCompData {
|
||||||
#if WASM_ENABLE_DEBUG_AOT != 0
|
#if WASM_ENABLE_DEBUG_AOT != 0
|
||||||
dwarf_extractor_handle_t extractor;
|
dwarf_extractor_handle_t extractor;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
struct WASMCompilationHint **function_hints;
|
||||||
|
#endif
|
||||||
} AOTCompData;
|
} AOTCompData;
|
||||||
|
|
||||||
typedef struct AOTNativeSymbol {
|
typedef struct AOTNativeSymbol {
|
||||||
|
|
|
@ -1158,9 +1158,16 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index)
|
||||||
|
|
||||||
case WASM_OP_BR_IF:
|
case WASM_OP_BR_IF:
|
||||||
{
|
{
|
||||||
|
// ip is advanced by one byte for the opcode
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
uint32 instr_offset =
|
||||||
|
(frame_ip - 0x1) - (func_ctx->aot_func->code_body_begin);
|
||||||
|
#else
|
||||||
|
uint32 instr_offset = 0;
|
||||||
|
#endif
|
||||||
read_leb_uint32(frame_ip, frame_ip_end, br_depth);
|
read_leb_uint32(frame_ip, frame_ip_end, br_depth);
|
||||||
if (!aot_compile_op_br_if(comp_ctx, func_ctx, br_depth,
|
if (!aot_compile_op_br_if(comp_ctx, func_ctx, br_depth,
|
||||||
&frame_ip))
|
&frame_ip, instr_offset))
|
||||||
return false;
|
return false;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
|
@ -4501,6 +4501,9 @@ aot_obj_data_create(AOTCompContext *comp_ctx)
|
||||||
if (comp_ctx->enable_ref_types) {
|
if (comp_ctx->enable_ref_types) {
|
||||||
obj_data->target_info.feature_flags |= WASM_FEATURE_REF_TYPES;
|
obj_data->target_info.feature_flags |= WASM_FEATURE_REF_TYPES;
|
||||||
}
|
}
|
||||||
|
if (comp_ctx->enable_branch_hints) {
|
||||||
|
obj_data->target_info.feature_flags |= WASM_FEATURE_BRANCH_HINTS;
|
||||||
|
}
|
||||||
if (comp_ctx->enable_gc) {
|
if (comp_ctx->enable_gc) {
|
||||||
obj_data->target_info.feature_flags |= WASM_FEATURE_GARBAGE_COLLECTION;
|
obj_data->target_info.feature_flags |= WASM_FEATURE_GARBAGE_COLLECTION;
|
||||||
}
|
}
|
||||||
|
|
|
@ -87,6 +87,16 @@ format_block_name(char *name, uint32 name_size, uint32 block_index,
|
||||||
} \
|
} \
|
||||||
} while (0)
|
} while (0)
|
||||||
|
|
||||||
|
#define BUILD_COND_BR_V(value_if, block_then, block_else, instr) \
|
||||||
|
do { \
|
||||||
|
if (instr = LLVMBuildCondBr(comp_ctx->builder, value_if, block_then, \
|
||||||
|
block_else), \
|
||||||
|
!instr) { \
|
||||||
|
aot_set_last_error("llvm build cond br failed."); \
|
||||||
|
goto fail; \
|
||||||
|
} \
|
||||||
|
} while (0)
|
||||||
|
|
||||||
#define SET_BUILDER_POS(llvm_block) \
|
#define SET_BUILDER_POS(llvm_block) \
|
||||||
LLVMPositionBuilderAtEnd(comp_ctx->builder, llvm_block)
|
LLVMPositionBuilderAtEnd(comp_ctx->builder, llvm_block)
|
||||||
|
|
||||||
|
@ -255,6 +265,40 @@ restore_frame_sp_for_op_end(AOTBlock *block, AOTCompFrame *aot_frame)
|
||||||
aot_frame->sp = block->frame_sp_begin;
|
aot_frame->sp = block->frame_sp_begin;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
static void
|
||||||
|
aot_emit_branch_hint(LLVMContextRef ctxt, AOTFuncContext *func_ctx,
|
||||||
|
uint32 offset, LLVMValueRef br_if_instr)
|
||||||
|
{
|
||||||
|
struct WASMCompilationHintBranchHint *hint =
|
||||||
|
(struct WASMCompilationHintBranchHint *)func_ctx->function_hints;
|
||||||
|
while (hint != NULL) {
|
||||||
|
if (hint->type == WASM_COMPILATION_BRANCH_HINT
|
||||||
|
&& hint->offset == offset) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
hint = hint->next;
|
||||||
|
}
|
||||||
|
if (hint != NULL) {
|
||||||
|
LLVMMetadataRef header = LLVMMDStringInContext2(
|
||||||
|
ctxt, "branch_weights", 14 /* strlen("branch_hint") */);
|
||||||
|
// same weight llvm MDBuilder::createLikelyBranchWeights assigns
|
||||||
|
const uint32_t likely_weight = (1U << 20) - 1;
|
||||||
|
const uint32_t unlikely_weight = 1;
|
||||||
|
LLVMMetadataRef true_w = LLVMValueAsMetadata(LLVMConstInt(
|
||||||
|
LLVMInt32TypeInContext(ctxt),
|
||||||
|
hint->is_likely ? likely_weight : unlikely_weight, false));
|
||||||
|
LLVMMetadataRef false_w = LLVMValueAsMetadata(LLVMConstInt(
|
||||||
|
LLVMInt32TypeInContext(ctxt),
|
||||||
|
hint->is_likely ? unlikely_weight : likely_weight, false));
|
||||||
|
LLVMMetadataRef mds[] = { header, true_w, false_w };
|
||||||
|
LLVMMetadataRef md = LLVMMDNodeInContext2(ctxt, mds, 3);
|
||||||
|
LLVMValueRef md_val = LLVMMetadataAsValue(ctxt, md);
|
||||||
|
LLVMSetMetadata(br_if_instr, LLVMGetMDKindID("prof", 4), md_val);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
static bool
|
static bool
|
||||||
handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
||||||
uint8 **p_frame_ip)
|
uint8 **p_frame_ip)
|
||||||
|
@ -673,13 +717,33 @@ aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
||||||
MOVE_BLOCK_AFTER(block->llvm_else_block,
|
MOVE_BLOCK_AFTER(block->llvm_else_block,
|
||||||
block->llvm_entry_block);
|
block->llvm_entry_block);
|
||||||
/* Create condition br IR */
|
/* Create condition br IR */
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
LLVMValueRef br_if_val = NULL;
|
||||||
|
BUILD_COND_BR_V(value, block->llvm_entry_block,
|
||||||
|
block->llvm_else_block, br_if_val);
|
||||||
|
const uint32 off =
|
||||||
|
*p_frame_ip - func_ctx->aot_func->code_body_begin;
|
||||||
|
aot_emit_branch_hint(comp_ctx->context, func_ctx, off,
|
||||||
|
br_if_val);
|
||||||
|
#else
|
||||||
BUILD_COND_BR(value, block->llvm_entry_block,
|
BUILD_COND_BR(value, block->llvm_entry_block,
|
||||||
block->llvm_else_block);
|
block->llvm_else_block);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
/* Create condition br IR */
|
/* Create condition br IR */
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
LLVMValueRef br_if_val = NULL;
|
||||||
|
BUILD_COND_BR_V(value, block->llvm_entry_block,
|
||||||
|
block->llvm_end_block, br_if_val);
|
||||||
|
const uint32 off =
|
||||||
|
*p_frame_ip - func_ctx->aot_func->code_body_begin;
|
||||||
|
aot_emit_branch_hint(comp_ctx->context, func_ctx, off,
|
||||||
|
br_if_val);
|
||||||
|
#else
|
||||||
BUILD_COND_BR(value, block->llvm_entry_block,
|
BUILD_COND_BR(value, block->llvm_entry_block,
|
||||||
block->llvm_end_block);
|
block->llvm_end_block);
|
||||||
|
#endif
|
||||||
block->is_reachable = true;
|
block->is_reachable = true;
|
||||||
}
|
}
|
||||||
if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx,
|
if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx,
|
||||||
|
@ -1027,7 +1091,7 @@ fail:
|
||||||
static bool
|
static bool
|
||||||
aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
||||||
uint32 br_depth, LLVMValueRef value_cmp,
|
uint32 br_depth, LLVMValueRef value_cmp,
|
||||||
uint8 **p_frame_ip)
|
uint8 **p_frame_ip, uint32 off)
|
||||||
{
|
{
|
||||||
AOTBlock *block_dst;
|
AOTBlock *block_dst;
|
||||||
LLVMValueRef value, *values = NULL;
|
LLVMValueRef value, *values = NULL;
|
||||||
|
@ -1108,8 +1172,15 @@ aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
||||||
values = NULL;
|
values = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
LLVMValueRef br_if_val = NULL;
|
||||||
|
BUILD_COND_BR_V(value_cmp, block_dst->llvm_entry_block,
|
||||||
|
llvm_else_block, br_if_val);
|
||||||
|
aot_emit_branch_hint(comp_ctx->context, func_ctx, off, br_if_val);
|
||||||
|
#else
|
||||||
BUILD_COND_BR(value_cmp, block_dst->llvm_entry_block,
|
BUILD_COND_BR(value_cmp, block_dst->llvm_entry_block,
|
||||||
llvm_else_block);
|
llvm_else_block);
|
||||||
|
#endif
|
||||||
|
|
||||||
/* Move builder to else block */
|
/* Move builder to else block */
|
||||||
SET_BUILDER_POS(llvm_else_block);
|
SET_BUILDER_POS(llvm_else_block);
|
||||||
|
@ -1152,9 +1223,15 @@ aot_compile_conditional_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Condition jump to end block */
|
/* Condition jump to end block */
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
LLVMValueRef br_if_val = NULL;
|
||||||
|
BUILD_COND_BR_V(value_cmp, block_dst->llvm_end_block,
|
||||||
|
llvm_else_block, br_if_val);
|
||||||
|
aot_emit_branch_hint(comp_ctx->context, func_ctx, off, br_if_val);
|
||||||
|
#else
|
||||||
BUILD_COND_BR(value_cmp, block_dst->llvm_end_block,
|
BUILD_COND_BR(value_cmp, block_dst->llvm_end_block,
|
||||||
llvm_else_block);
|
llvm_else_block);
|
||||||
|
#endif
|
||||||
/* Move builder to else block */
|
/* Move builder to else block */
|
||||||
SET_BUILDER_POS(llvm_else_block);
|
SET_BUILDER_POS(llvm_else_block);
|
||||||
}
|
}
|
||||||
|
@ -1178,14 +1255,14 @@ fail:
|
||||||
|
|
||||||
bool
|
bool
|
||||||
aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
||||||
uint32 br_depth, uint8 **p_frame_ip)
|
uint32 br_depth, uint8 **p_frame_ip, uint32 off)
|
||||||
{
|
{
|
||||||
LLVMValueRef value_cmp;
|
LLVMValueRef value_cmp;
|
||||||
|
|
||||||
POP_COND(value_cmp);
|
POP_COND(value_cmp);
|
||||||
|
|
||||||
return aot_compile_conditional_br(comp_ctx, func_ctx, br_depth, value_cmp,
|
return aot_compile_conditional_br(comp_ctx, func_ctx, br_depth, value_cmp,
|
||||||
p_frame_ip);
|
p_frame_ip, off);
|
||||||
fail:
|
fail:
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
|
@ -32,7 +32,7 @@ aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
||||||
|
|
||||||
bool
|
bool
|
||||||
aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
||||||
uint32 br_depth, uint8 **p_frame_ip);
|
uint32 br_depth, uint8 **p_frame_ip, uint32 offset);
|
||||||
|
|
||||||
bool
|
bool
|
||||||
aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
|
||||||
|
|
|
@ -1963,6 +1963,13 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx,
|
||||||
goto fail;
|
goto fail;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
func_ctx->function_hints =
|
||||||
|
comp_ctx->comp_data->function_hints
|
||||||
|
? comp_ctx->comp_data->function_hints[func_index]
|
||||||
|
: NULL;
|
||||||
|
#endif
|
||||||
|
|
||||||
return func_ctx;
|
return func_ctx;
|
||||||
|
|
||||||
fail:
|
fail:
|
||||||
|
@ -2733,6 +2740,9 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
|
||||||
if (option->enable_ref_types)
|
if (option->enable_ref_types)
|
||||||
comp_ctx->enable_ref_types = true;
|
comp_ctx->enable_ref_types = true;
|
||||||
|
|
||||||
|
if (option->enable_branch_hints)
|
||||||
|
comp_ctx->enable_branch_hints = true;
|
||||||
|
|
||||||
comp_ctx->aux_stack_frame_type = option->aux_stack_frame_type;
|
comp_ctx->aux_stack_frame_type = option->aux_stack_frame_type;
|
||||||
comp_ctx->call_stack_features = option->call_stack_features;
|
comp_ctx->call_stack_features = option->call_stack_features;
|
||||||
|
|
||||||
|
|
|
@ -270,6 +270,9 @@ typedef struct AOTFuncContext {
|
||||||
#if WASM_ENABLE_DEBUG_AOT != 0
|
#if WASM_ENABLE_DEBUG_AOT != 0
|
||||||
LLVMMetadataRef debug_func;
|
LLVMMetadataRef debug_func;
|
||||||
#endif
|
#endif
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
struct WASMCompilationHint *function_hints;
|
||||||
|
#endif
|
||||||
|
|
||||||
unsigned int stack_consumption_for_func_call;
|
unsigned int stack_consumption_for_func_call;
|
||||||
|
|
||||||
|
@ -449,6 +452,9 @@ typedef struct AOTCompContext {
|
||||||
/* Reference Types */
|
/* Reference Types */
|
||||||
bool enable_ref_types;
|
bool enable_ref_types;
|
||||||
|
|
||||||
|
/* Branch Hinting */
|
||||||
|
bool enable_branch_hints;
|
||||||
|
|
||||||
/* Disable LLVM built-in intrinsics */
|
/* Disable LLVM built-in intrinsics */
|
||||||
bool disable_llvm_intrinsics;
|
bool disable_llvm_intrinsics;
|
||||||
|
|
||||||
|
|
|
@ -66,6 +66,7 @@ typedef struct AOTCompOption {
|
||||||
bool enable_tail_call;
|
bool enable_tail_call;
|
||||||
bool enable_simd;
|
bool enable_simd;
|
||||||
bool enable_ref_types;
|
bool enable_ref_types;
|
||||||
|
bool enable_branch_hints;
|
||||||
bool enable_gc;
|
bool enable_gc;
|
||||||
bool enable_aux_stack_check;
|
bool enable_aux_stack_check;
|
||||||
bool enable_extended_const;
|
bool enable_extended_const;
|
||||||
|
|
|
@ -751,6 +751,10 @@ struct WASMFunction {
|
||||||
void *call_to_fast_jit_from_llvm_jit;
|
void *call_to_fast_jit_from_llvm_jit;
|
||||||
#endif
|
#endif
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
uint8 *code_body_begin;
|
||||||
|
#endif
|
||||||
};
|
};
|
||||||
|
|
||||||
#if WASM_ENABLE_TAGS != 0
|
#if WASM_ENABLE_TAGS != 0
|
||||||
|
@ -761,6 +765,23 @@ struct WASMTag {
|
||||||
};
|
};
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
enum WASMCompilationHintType {
|
||||||
|
DUMMY = 0,
|
||||||
|
WASM_COMPILATION_BRANCH_HINT = 0,
|
||||||
|
};
|
||||||
|
struct WASMCompilationHint {
|
||||||
|
struct WASMCompilationHint *next;
|
||||||
|
enum WASMCompilationHintType type;
|
||||||
|
};
|
||||||
|
struct WASMCompilationHintBranchHint {
|
||||||
|
void *next;
|
||||||
|
enum WASMCompilationHintType type;
|
||||||
|
uint32 offset;
|
||||||
|
bool is_likely;
|
||||||
|
};
|
||||||
|
#endif
|
||||||
|
|
||||||
struct WASMGlobal {
|
struct WASMGlobal {
|
||||||
WASMGlobalType type;
|
WASMGlobalType type;
|
||||||
#if WASM_ENABLE_GC != 0
|
#if WASM_ENABLE_GC != 0
|
||||||
|
@ -1049,6 +1070,10 @@ struct WASMModule {
|
||||||
const uint8 *name_section_buf_end;
|
const uint8 *name_section_buf_end;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
struct WASMCompilationHint **function_hints;
|
||||||
|
#endif
|
||||||
|
|
||||||
#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
|
#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
|
||||||
WASMCustomSection *custom_section_list;
|
WASMCustomSection *custom_section_list;
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -3882,6 +3882,9 @@ load_function_section(const uint8 *buf, const uint8 *buf_end,
|
||||||
|
|
||||||
/* Resolve local set count */
|
/* Resolve local set count */
|
||||||
p_code_end = p_code + code_size;
|
p_code_end = p_code + code_size;
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
uint8 *p_body_start = (uint8 *)p_code;
|
||||||
|
#endif
|
||||||
local_count = 0;
|
local_count = 0;
|
||||||
read_leb_uint32(p_code, buf_code_end, local_set_count);
|
read_leb_uint32(p_code, buf_code_end, local_set_count);
|
||||||
p_code_save = p_code;
|
p_code_save = p_code;
|
||||||
|
@ -3957,6 +3960,9 @@ load_function_section(const uint8 *buf, const uint8 *buf_end,
|
||||||
if (local_count > 0)
|
if (local_count > 0)
|
||||||
func->local_types = (uint8 *)func + sizeof(WASMFunction);
|
func->local_types = (uint8 *)func + sizeof(WASMFunction);
|
||||||
func->code_size = code_size;
|
func->code_size = code_size;
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
func->code_body_begin = p_body_start;
|
||||||
|
#endif
|
||||||
/*
|
/*
|
||||||
* we shall make a copy of code body [p_code, p_code + code_size]
|
* we shall make a copy of code body [p_code, p_code + code_size]
|
||||||
* when we are worrying about inappropriate releasing behaviour.
|
* when we are worrying about inappropriate releasing behaviour.
|
||||||
|
@ -5529,6 +5535,87 @@ fail:
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
static bool
|
||||||
|
handle_branch_hint_section(const uint8 *buf, const uint8 *buf_end,
|
||||||
|
WASMModule *module, char *error_buf,
|
||||||
|
uint32 error_buf_size)
|
||||||
|
{
|
||||||
|
if (module->function_hints == NULL) {
|
||||||
|
module->function_hints = loader_malloc(
|
||||||
|
sizeof(struct WASMCompilationHint) * module->function_count,
|
||||||
|
error_buf, error_buf_size);
|
||||||
|
}
|
||||||
|
uint32 numFunctionHints = 0;
|
||||||
|
read_leb_uint32(buf, buf_end, numFunctionHints);
|
||||||
|
for (uint32 i = 0; i < numFunctionHints; ++i) {
|
||||||
|
uint32 func_idx;
|
||||||
|
read_leb_uint32(buf, buf_end, func_idx);
|
||||||
|
if (!check_function_index(module, func_idx, error_buf,
|
||||||
|
error_buf_size)) {
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
if (func_idx < module->import_function_count) {
|
||||||
|
set_error_buf(error_buf, error_buf_size,
|
||||||
|
"branch hint for imported function is not allowed");
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct WASMCompilationHintBranchHint *current_hint =
|
||||||
|
(struct WASMCompilationHintBranchHint *)&module
|
||||||
|
->function_hints[func_idx - module->import_function_count];
|
||||||
|
while (current_hint->next != NULL) {
|
||||||
|
current_hint = current_hint->next;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32 num_hints;
|
||||||
|
read_leb_uint32(buf, buf_end, num_hints);
|
||||||
|
for (uint32 j = 0; j < num_hints; ++j) {
|
||||||
|
struct WASMCompilationHintBranchHint *new_hint =
|
||||||
|
loader_malloc(sizeof(struct WASMCompilationHintBranchHint),
|
||||||
|
error_buf, error_buf_size);
|
||||||
|
new_hint->next = NULL;
|
||||||
|
new_hint->type = WASM_COMPILATION_BRANCH_HINT;
|
||||||
|
read_leb_uint32(buf, buf_end, new_hint->offset);
|
||||||
|
|
||||||
|
uint32 size;
|
||||||
|
read_leb_uint32(buf, buf_end, size);
|
||||||
|
if (size != 1) {
|
||||||
|
set_error_buf_v(error_buf, error_buf_size,
|
||||||
|
"invalid branch hint size, expected 1, got %d.",
|
||||||
|
size);
|
||||||
|
wasm_runtime_free(new_hint);
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint8 data = *buf++;
|
||||||
|
if (data == 0x00)
|
||||||
|
new_hint->is_likely = false;
|
||||||
|
else if (data == 0x01)
|
||||||
|
new_hint->is_likely = true;
|
||||||
|
else {
|
||||||
|
set_error_buf_v(error_buf, error_buf_size,
|
||||||
|
"invalid branch hint, expected 0 or 1, got %d",
|
||||||
|
data);
|
||||||
|
wasm_runtime_free(new_hint);
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
|
||||||
|
current_hint->next = new_hint;
|
||||||
|
current_hint = new_hint;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (buf != buf_end) {
|
||||||
|
set_error_buf(error_buf, error_buf_size,
|
||||||
|
"invalid branch hint section, not filled until end");
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
fail:
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
static bool
|
static bool
|
||||||
load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
|
load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
|
||||||
bool is_load_from_file_buf, char *error_buf,
|
bool is_load_from_file_buf, char *error_buf,
|
||||||
|
@ -5578,6 +5665,17 @@ load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module,
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if WASM_ENABLE_BRANCH_HINTS != 0
|
||||||
|
if (name_len == 25 && memcmp(p, "metadata.code.branch_hint", 25) == 0) {
|
||||||
|
p += name_len;
|
||||||
|
if (!handle_branch_hint_section(p, p_end, module, error_buf,
|
||||||
|
error_buf_size)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
LOG_VERBOSE("Load branch hint section success.");
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
|
#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
|
||||||
{
|
{
|
||||||
WASMCustomSection *section =
|
WASMCustomSection *section =
|
||||||
|
|
|
@ -406,12 +406,11 @@ os_socket_addr_resolve(const char *host, const char *service,
|
||||||
|
|
||||||
res = result;
|
res = result;
|
||||||
while (res) {
|
while (res) {
|
||||||
if (addr_info_size > pos) {
|
|
||||||
if (!is_addrinfo_supported(res)) {
|
if (!is_addrinfo_supported(res)) {
|
||||||
res = res->ai_next;
|
res = res->ai_next;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (addr_info_size > pos) {
|
||||||
ret =
|
ret =
|
||||||
sockaddr_to_bh_sockaddr(res->ai_addr, &addr_info[pos].sockaddr);
|
sockaddr_to_bh_sockaddr(res->ai_addr, &addr_info[pos].sockaddr);
|
||||||
|
|
||||||
|
|
|
@ -46,6 +46,7 @@ add_definitions(-DWASM_ENABLE_SHARED_MEMORY=1)
|
||||||
add_definitions(-DWASM_ENABLE_THREAD_MGR=1)
|
add_definitions(-DWASM_ENABLE_THREAD_MGR=1)
|
||||||
add_definitions(-DWASM_ENABLE_TAIL_CALL=1)
|
add_definitions(-DWASM_ENABLE_TAIL_CALL=1)
|
||||||
add_definitions(-DWASM_ENABLE_REF_TYPES=1)
|
add_definitions(-DWASM_ENABLE_REF_TYPES=1)
|
||||||
|
add_definitions(-DWASM_ENABLE_BRANCH_HINTS=1)
|
||||||
add_definitions(-DWASM_ENABLE_CUSTOM_NAME_SECTION=1)
|
add_definitions(-DWASM_ENABLE_CUSTOM_NAME_SECTION=1)
|
||||||
add_definitions(-DWASM_ENABLE_AOT_STACK_FRAME=1)
|
add_definitions(-DWASM_ENABLE_AOT_STACK_FRAME=1)
|
||||||
add_definitions(-DWASM_ENABLE_DUMP_CALL_STACK=1)
|
add_definitions(-DWASM_ENABLE_DUMP_CALL_STACK=1)
|
||||||
|
|
Loading…
Reference in New Issue
Block a user