gnh1201/wasm-micro-runtime
1
0
Fork 0
mirror of https://github.com/bytecodealliance/wasm-micro-runtime.git synced 2025-02-06 06:55:07 +00:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Implement Fast JIT multi-threading feature (#2134)

Browse Source 
- Translate all the opcodes of threads spec proposal for Fast JIT
- Add the atomic flag for Fast JIT load/store IRs to support atomic load/store
- Add new atomic related Fast JIT IRs and translate them in the codegen
- Add suspend_flags check in branch opcodes and before/after call function
- Modify CI to enable Fast JIT multi-threading test

Co-authored-by: TianlongLiang <tianlong.liang@intel.com>
This commit is contained in:
Wenyong Huang 2023-04-20 10:09:34 +08:00 committed by GitHub
parent dfca21d239
commit 7e9bf9cdf5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 2290 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.github/workflows/compilation_on_android_ubuntu.yml vendored
View File

@ -125,8 +125,8 @@ jobs:
# Running mode
$CLASSIC_INTERP_BUILD_OPTIONS,
$FAST_INTERP_BUILD_OPTIONS,
$FAST_JIT_BUILD_OPTIONS
]
$FAST_JIT_BUILD_OPTIONS,
]
make_options_feature: [
# Features
"-DWAMR_BUILD_CUSTOM_NAME_SECTION=1",
@ -414,11 +414,11 @@ jobs:
os: [ubuntu-20.04, ubuntu-22.04]
wasi_sdk_release:
[
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz"
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz",
]
wabt_release:
[
"https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz"
"https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-ubuntu.tar.gz",
]
steps:
- name: checkout
@ -505,7 +505,7 @@ jobs:
build_iwasm,
build_llvm_libraries_on_ubuntu_2004,
build_llvm_libraries_on_ubuntu_2204,
build_wamrc
build_wamrc,
]
runs-on: ${{ matrix.os }}
strategy:
@ -530,7 +530,7 @@ jobs:
]
wasi_sdk_release:
[
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz"
"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz",
]
include:
- os: ubuntu-20.04
@ -551,24 +551,16 @@ jobs:
test_option: $MULTI_MODULES_TEST_OPTIONS
- running_mode: "jit"
test_option: $MULTI_MODULES_TEST_OPTIONS
# fast-jit doesn't support multi module, simd, and threads
# fast-jit doesn't support multi module, simd
- running_mode: "fast-jit"
test_option: $MULTI_MODULES_TEST_OPTIONS
- running_mode: "fast-jit"
test_option: $SIMD_TEST_OPTIONS
- running_mode: "fast-jit"
test_option: $THREADS_TEST_OPTIONS
- running_mode: "fast-jit"
test_option: $WASI_TEST_OPTIONS
# multi-tier-jit doesn't support multi module, simd, and threads
# multi-tier-jit doesn't support multi module, simd
- running_mode: "multi-tier-jit"
test_option: $MULTI_MODULES_TEST_OPTIONS
- running_mode: "multi-tier-jit"
test_option: $SIMD_TEST_OPTIONS
- running_mode: "multi-tier-jit"
test_option: $THREADS_TEST_OPTIONS
- running_mode: "multi-tier-jit"
test_option: $WASI_TEST_OPTIONS
steps:
- name: checkout
uses: actions/checkout@v3

4
core/iwasm/compilation/aot_emit_control.c
View File

@ -701,7 +701,7 @@ check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
if (!(terminate_flags =
LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, terminate_addr,
"terminate_flags"))) {
aot_set_last_error("llvm build bit cast failed");
aot_set_last_error("llvm build LOAD failed");
return false;
}
/* Set terminate_flags memory accecc to volatile, so that the value
@ -729,7 +729,7 @@ check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
goto fail;
}
/* Move builder to terminate block */
/* Move builder to non terminate block */
SET_BUILDER_POS(non_terminate_block);
return true;

1762
core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp
View File

File diff suppressed because it is too large Load Diff

55
core/iwasm/fast-jit/fe/jit_emit_control.c
View File

@ -904,6 +904,42 @@ check_copy_arities(const JitBlock *block_dst, JitFrame *jit_frame)
}
}
#if WASM_ENABLE_THREAD_MGR != 0
bool
jit_check_suspend_flags(JitCompContext *cc)
{
JitReg exec_env, suspend_flags, terminate_flag, offset;
JitBasicBlock *terminate_block, *cur_basic_block;
JitFrame *jit_frame = cc->jit_frame;
cur_basic_block = cc->cur_basic_block;
terminate_block = jit_cc_new_basic_block(cc, 0);
if (!terminate_block) {
return false;
}
gen_commit_values(jit_frame, jit_frame->lp, jit_frame->sp);
exec_env = cc->exec_env_reg;
suspend_flags = jit_cc_new_reg_I32(cc);
terminate_flag = jit_cc_new_reg_I32(cc);
offset = jit_cc_new_const_I32(cc, offsetof(WASMExecEnv, suspend_flags));
GEN_INSN(LDI32, suspend_flags, exec_env, offset);
GEN_INSN(AND, terminate_flag, suspend_flags, NEW_CONST(I32, 1));
GEN_INSN(CMP, cc->cmp_reg, terminate_flag, NEW_CONST(I32, 0));
GEN_INSN(BNE, cc->cmp_reg, jit_basic_block_label(terminate_block), 0);
cc->cur_basic_block = terminate_block;
GEN_INSN(RETURN, NEW_CONST(I32, 0));
cc->cur_basic_block = cur_basic_block;
return true;
}
#endif
static bool
handle_op_br(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip)
{
@ -986,6 +1022,13 @@ fail:
bool
jit_compile_op_br(JitCompContext *cc, uint32 br_depth, uint8 **p_frame_ip)
{
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (!jit_check_suspend_flags(cc))
return false;
#endif
return handle_op_br(cc, br_depth, p_frame_ip)
&& handle_next_reachable_block(cc, p_frame_ip);
}
@ -1105,6 +1148,12 @@ jit_compile_op_br_if(JitCompContext *cc, uint32 br_depth,
jit_insn_delete(insn_select);
}
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (!jit_check_suspend_flags(cc))
return false;
#endif
SET_BUILDER_POS(if_basic_block);
SET_BB_BEGIN_BCIP(if_basic_block, *p_frame_ip - 1);
@ -1144,6 +1193,12 @@ jit_compile_op_br_table(JitCompContext *cc, uint32 *br_depths, uint32 br_count,
uint32 i = 0;
JitOpndLookupSwitch *opnd = NULL;
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (!jit_check_suspend_flags(cc))
return false;
#endif
cur_basic_block = cc->cur_basic_block;
POP_I32(value);

31
core/iwasm/fast-jit/fe/jit_emit_function.c
View File

@ -5,6 +5,7 @@
#include "jit_emit_function.h"
#include "jit_emit_exception.h"
#include "jit_emit_control.h"
#include "../jit_frontend.h"
#include "../jit_codegen.h"
#include "../../interpreter/wasm_runtime.h"
@ -232,6 +233,12 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
bool is_pointer_arg;
bool return_value = false;
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (!jit_check_suspend_flags(cc))
goto fail;
#endif
if (func_idx < wasm_module->import_function_count) {
/* The function to call is an import function */
func_import = &wasm_module->import_functions[func_idx].u.function;
@ -275,6 +282,12 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
goto fail;
}
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (!jit_check_suspend_flags(cc))
goto fail;
#endif
return true;
}
@ -416,6 +429,12 @@ jit_compile_op_call(JitCompContext *cc, uint32 func_idx, bool tail_call)
}
}
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (!jit_check_suspend_flags(cc))
goto fail;
#endif
/* Clear part of memory regs and table regs as their values
may be changed in the function call */
if (cc->cur_wasm_module->possible_memory_grow)
@ -540,6 +559,12 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx,
GEN_INSN(STI32, func_idx, cc->exec_env_reg,
NEW_CONST(I32, offsetof(WASMExecEnv, jit_cache) + 4));
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (!jit_check_suspend_flags(cc))
goto fail;
#endif
block_import = jit_cc_new_basic_block(cc, 0);
block_nonimport = jit_cc_new_basic_block(cc, 0);
func_return = jit_cc_new_basic_block(cc, 0);
@ -742,6 +767,12 @@ jit_compile_op_call_indirect(JitCompContext *cc, uint32 type_idx,
goto fail;
}
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (!jit_check_suspend_flags(cc))
goto fail;
#endif
/* Clear part of memory regs and table regs as their values
may be changed in the function call */
if (cc->cur_wasm_module->possible_memory_grow)

374
core/iwasm/fast-jit/fe/jit_emit_memory.c
View File

@ -9,6 +9,7 @@
#include "../jit_frontend.h"
#include "../jit_codegen.h"
#include "../../interpreter/wasm_runtime.h"
#include "jit_emit_control.h"
#ifndef OS_ENABLE_HW_BOUND_CHECK
static JitReg
@ -60,6 +61,14 @@ fail:
}
#endif
#if WASM_ENABLE_SHARED_MEMORY != 0
static void
set_load_or_store_atomic(JitInsn *load_or_store_inst)
{
load_or_store_inst->flags_u8 |= 0x1;
}
#endif
#if UINTPTR_MAX == UINT64_MAX
static JitReg
check_and_seek_on_64bit_platform(JitCompContext *cc, JitReg addr, JitReg offset,
@ -177,23 +186,36 @@ fail:
return 0;
}
#define CHECK_ALIGNMENT(maddr, memory_data, offset1) \
#if UINTPTR_MAX == UINT64_MAX
#define CHECK_ALIGNMENT(offset1) \
do { \
GEN_INSN(ADD, maddr, memory_data, offset1); \
JitReg align_mask = NEW_CONST(I64, ((uint64)1 << align) - 1); \
JitReg AND_res = jit_cc_new_reg_I64(cc); \
GEN_INSN(AND, AND_res, maddr, align_mask); \
GEN_INSN(AND, AND_res, offset1, align_mask); \
GEN_INSN(CMP, cc->cmp_reg, AND_res, NEW_CONST(I64, 0)); \
if (!jit_emit_exception(cc, EXCE_UNALIGNED_ATOMIC, JIT_OP_BNE, \
cc->cmp_reg, NULL)) \
goto fail; \
} while (0)
#else
#define CHECK_ALIGNMENT(offset1) \
do { \
JitReg align_mask = NEW_CONST(I32, (1 << align) - 1); \
JitReg AND_res = jit_cc_new_reg_I32(cc); \
GEN_INSN(AND, AND_res, offset1, align_mask); \
GEN_INSN(CMP, cc->cmp_reg, AND_res, NEW_CONST(I32, 0)); \
if (!jit_emit_exception(cc, EXCE_UNALIGNED_ATOMIC, JIT_OP_BNE, \
cc->cmp_reg, NULL)) \
goto fail; \
} while (0)
#endif
bool
jit_compile_op_i32_load(JitCompContext *cc, uint32 align, uint32 offset,
uint32 bytes, bool sign, bool atomic)
{
JitReg addr, offset1, value, memory_data;
JitInsn *load_insn = NULL;
POP_I32(addr);
@ -201,6 +223,11 @@ jit_compile_op_i32_load(JitCompContext *cc, uint32 align, uint32 offset,
if (!offset1) {
goto fail;
}
#if WASM_ENABLE_SHARED_MEMORY != 0
if (atomic) {
CHECK_ALIGNMENT(offset1);
}
#endif
memory_data = get_memory_data_reg(cc->jit_frame, 0);
@ -209,30 +236,30 @@ jit_compile_op_i32_load(JitCompContext *cc, uint32 align, uint32 offset,
case 1:
{
if (sign) {
GEN_INSN(LDI8, value, memory_data, offset1);
load_insn = GEN_INSN(LDI8, value, memory_data, offset1);
}
else {
GEN_INSN(LDU8, value, memory_data, offset1);
load_insn = GEN_INSN(LDU8, value, memory_data, offset1);
}
break;
}
case 2:
{
if (sign) {
GEN_INSN(LDI16, value, memory_data, offset1);
load_insn = GEN_INSN(LDI16, value, memory_data, offset1);
}
else {
GEN_INSN(LDU16, value, memory_data, offset1);
load_insn = GEN_INSN(LDU16, value, memory_data, offset1);
}
break;
}
case 4:
{
if (sign) {
GEN_INSN(LDI32, value, memory_data, offset1);
load_insn = GEN_INSN(LDI32, value, memory_data, offset1);
}
else {
GEN_INSN(LDU32, value, memory_data, offset1);
load_insn = GEN_INSN(LDU32, value, memory_data, offset1);
}
break;
}
@ -243,6 +270,13 @@ jit_compile_op_i32_load(JitCompContext *cc, uint32 align, uint32 offset,
}
}
#if WASM_ENABLE_SHARED_MEMORY != 0
if (atomic && load_insn)
set_load_or_store_atomic(load_insn);
#else
(void)load_insn;
#endif
PUSH_I32(value);
return true;
fail:
@ -254,6 +288,7 @@ jit_compile_op_i64_load(JitCompContext *cc, uint32 align, uint32 offset,
uint32 bytes, bool sign, bool atomic)
{
JitReg addr, offset1, value, memory_data;
JitInsn *load_insn = NULL;
POP_I32(addr);
@ -261,6 +296,11 @@ jit_compile_op_i64_load(JitCompContext *cc, uint32 align, uint32 offset,
if (!offset1) {
goto fail;
}
#if WASM_ENABLE_SHARED_MEMORY != 0
if (atomic) {
CHECK_ALIGNMENT(offset1);
}
#endif
memory_data = get_memory_data_reg(cc->jit_frame, 0);
@ -269,40 +309,40 @@ jit_compile_op_i64_load(JitCompContext *cc, uint32 align, uint32 offset,
case 1:
{
if (sign) {
GEN_INSN(LDI8, value, memory_data, offset1);
load_insn = GEN_INSN(LDI8, value, memory_data, offset1);
}
else {
GEN_INSN(LDU8, value, memory_data, offset1);
load_insn = GEN_INSN(LDU8, value, memory_data, offset1);
}
break;
}
case 2:
{
if (sign) {
GEN_INSN(LDI16, value, memory_data, offset1);
load_insn = GEN_INSN(LDI16, value, memory_data, offset1);
}
else {
GEN_INSN(LDU16, value, memory_data, offset1);
load_insn = GEN_INSN(LDU16, value, memory_data, offset1);
}
break;
}
case 4:
{
if (sign) {
GEN_INSN(LDI32, value, memory_data, offset1);
load_insn = GEN_INSN(LDI32, value, memory_data, offset1);
}
else {
GEN_INSN(LDU32, value, memory_data, offset1);
load_insn = GEN_INSN(LDU32, value, memory_data, offset1);
}
break;
}
case 8:
{
if (sign) {
GEN_INSN(LDI64, value, memory_data, offset1);
load_insn = GEN_INSN(LDI64, value, memory_data, offset1);
}
else {
GEN_INSN(LDU64, value, memory_data, offset1);
load_insn = GEN_INSN(LDU64, value, memory_data, offset1);
}
break;
}
@ -313,6 +353,13 @@ jit_compile_op_i64_load(JitCompContext *cc, uint32 align, uint32 offset,
}
}
#if WASM_ENABLE_SHARED_MEMORY != 0
if (atomic && load_insn)
set_load_or_store_atomic(load_insn);
#else
(void)load_insn;
#endif
PUSH_I64(value);
return true;
fail:
@ -370,6 +417,7 @@ jit_compile_op_i32_store(JitCompContext *cc, uint32 align, uint32 offset,
uint32 bytes, bool atomic)
{
JitReg value, addr, offset1, memory_data;
JitInsn *store_insn = NULL;
POP_I32(value);
POP_I32(addr);
@ -378,23 +426,28 @@ jit_compile_op_i32_store(JitCompContext *cc, uint32 align, uint32 offset,
if (!offset1) {
goto fail;
}
#if WASM_ENABLE_SHARED_MEMORY != 0
if (atomic) {
CHECK_ALIGNMENT(offset1);
}
#endif
memory_data = get_memory_data_reg(cc->jit_frame, 0);
switch (bytes) {
case 1:
{
GEN_INSN(STI8, value, memory_data, offset1);
store_insn = GEN_INSN(STI8, value, memory_data, offset1);
break;
}
case 2:
{
GEN_INSN(STI16, value, memory_data, offset1);
store_insn = GEN_INSN(STI16, value, memory_data, offset1);
break;
}
case 4:
{
GEN_INSN(STI32, value, memory_data, offset1);
store_insn = GEN_INSN(STI32, value, memory_data, offset1);
break;
}
default:
@ -403,6 +456,12 @@ jit_compile_op_i32_store(JitCompContext *cc, uint32 align, uint32 offset,
goto fail;
}
}
#if WASM_ENABLE_SHARED_MEMORY != 0
if (atomic && store_insn)
set_load_or_store_atomic(store_insn);
#else
(void)store_insn;
#endif
return true;
fail:
@ -414,6 +473,7 @@ jit_compile_op_i64_store(JitCompContext *cc, uint32 align, uint32 offset,
uint32 bytes, bool atomic)
{
JitReg value, addr, offset1, memory_data;
JitInsn *store_insn = NULL;
POP_I64(value);
POP_I32(addr);
@ -422,6 +482,11 @@ jit_compile_op_i64_store(JitCompContext *cc, uint32 align, uint32 offset,
if (!offset1) {
goto fail;
}
#if WASM_ENABLE_SHARED_MEMORY != 0
if (atomic) {
CHECK_ALIGNMENT(offset1);
}
#endif
if (jit_reg_is_const(value) && bytes < 8) {
value = NEW_CONST(I32, (int32)jit_cc_get_const_I64(cc, value));
@ -432,22 +497,22 @@ jit_compile_op_i64_store(JitCompContext *cc, uint32 align, uint32 offset,
switch (bytes) {
case 1:
{
GEN_INSN(STI8, value, memory_data, offset1);
store_insn = GEN_INSN(STI8, value, memory_data, offset1);
break;
}
case 2:
{
GEN_INSN(STI16, value, memory_data, offset1);
store_insn = GEN_INSN(STI16, value, memory_data, offset1);
break;
}
case 4:
{
GEN_INSN(STI32, value, memory_data, offset1);
store_insn = GEN_INSN(STI32, value, memory_data, offset1);
break;
}
case 8:
{
GEN_INSN(STI64, value, memory_data, offset1);
store_insn = GEN_INSN(STI64, value, memory_data, offset1);
break;
}
default:
@ -456,6 +521,12 @@ jit_compile_op_i64_store(JitCompContext *cc, uint32 align, uint32 offset,
goto fail;
}
}
#if WASM_ENABLE_SHARED_MEMORY != 0
if (atomic && store_insn)
set_load_or_store_atomic(store_insn);
#else
(void)store_insn;
#endif
return true;
fail:
@ -774,10 +845,153 @@ fail:
#endif
#if WASM_ENABLE_SHARED_MEMORY != 0
#define GEN_AT_RMW_INSN(op, op_type, bytes, result, value, memory_data, \
offset1) \
do { \
switch (bytes) { \
case 1: \
{ \
insn = GEN_INSN(AT_##op##U8, result, value, memory_data, \
offset1); \
break; \
} \
case 2: \
{ \
insn = GEN_INSN(AT_##op##U16, result, value, memory_data, \
offset1); \
break; \
} \
case 4: \
{ \
if (op_type == VALUE_TYPE_I32) \
insn = GEN_INSN(AT_##op##I32, result, value, memory_data, \
offset1); \
else \
insn = GEN_INSN(AT_##op##U32, result, value, memory_data, \
offset1); \
break; \
} \
case 8: \
{ \
insn = GEN_INSN(AT_##op##I64, result, value, memory_data, \
offset1); \
break; \
} \
default: \
{ \
bh_assert(0); \
goto fail; \
} \
} \
} while (0)
bool
jit_compile_op_atomic_rmw(JitCompContext *cc, uint8 atomic_op, uint8 op_type,
uint32 align, uint32 offset, uint32 bytes)
{
JitReg addr, offset1, memory_data, value, result, eax_hreg, rax_hreg,
ebx_hreg, rbx_hreg;
JitInsn *insn = NULL;
bool is_i32 = op_type == VALUE_TYPE_I32;
bool is_logical_op = atomic_op == AtomicRMWBinOpAnd
|| atomic_op == AtomicRMWBinOpOr
|| atomic_op == AtomicRMWBinOpXor;
/* currently we only implement atomic rmw on x86-64 target */
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
/* For atomic logical binary ops, it implicitly uses rax in cmpxchg
* instruction and implicitly uses rbx for storing temp value in the
* generated loop */
eax_hreg = jit_codegen_get_hreg_by_name("eax");
rax_hreg = jit_codegen_get_hreg_by_name("rax");
ebx_hreg = jit_codegen_get_hreg_by_name("ebx");
rbx_hreg = jit_codegen_get_hreg_by_name("rbx");
bh_assert(op_type == VALUE_TYPE_I32 || op_type == VALUE_TYPE_I64);
if (op_type == VALUE_TYPE_I32) {
POP_I32(value);
}
else {
POP_I64(value);
}
POP_I32(addr);
offset1 = check_and_seek(cc, addr, offset, bytes);
if (!offset1) {
goto fail;
}
CHECK_ALIGNMENT(offset1);
memory_data = get_memory_data_reg(cc->jit_frame, 0);
if (op_type == VALUE_TYPE_I32)
result = jit_cc_new_reg_I32(cc);
else
result = jit_cc_new_reg_I64(cc);
switch (atomic_op) {
case AtomicRMWBinOpAdd:
{
GEN_AT_RMW_INSN(ADD, op_type, bytes, result, value, memory_data,
offset1);
break;
}
case AtomicRMWBinOpSub:
{
GEN_AT_RMW_INSN(SUB, op_type, bytes, result, value, memory_data,
offset1);
break;
}
case AtomicRMWBinOpAnd:
{
GEN_AT_RMW_INSN(AND, op_type, bytes, result, value, memory_data,
offset1);
break;
}
case AtomicRMWBinOpOr:
{
GEN_AT_RMW_INSN(OR, op_type, bytes, result, value, memory_data,
offset1);
break;
}
case AtomicRMWBinOpXor:
{
GEN_AT_RMW_INSN(XOR, op_type, bytes, result, value, memory_data,
offset1);
break;
}
case AtomicRMWBinOpXchg:
{
GEN_AT_RMW_INSN(XCHG, op_type, bytes, result, value, memory_data,
offset1);
break;
}
default:
{
bh_assert(0);
goto fail;
}
}
if (is_logical_op
&& (!insn
|| !jit_lock_reg_in_insn(cc, insn, is_i32 ? eax_hreg : rax_hreg)
|| !jit_lock_reg_in_insn(cc, insn, is_i32 ? ebx_hreg : rbx_hreg))) {
jit_set_last_error(
cc, "generate atomic logical insn or lock ra&rb hreg failed");
goto fail;
}
if (op_type == VALUE_TYPE_I32)
PUSH_I32(result);
else
PUSH_I64(result);
return true;
#endif /* defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) */
fail:
return false;
}
@ -785,6 +999,93 @@ bool
jit_compile_op_atomic_cmpxchg(JitCompContext *cc, uint8 op_type, uint32 align,
uint32 offset, uint32 bytes)
{
JitReg addr, offset1, memory_data, value, expect, result;
bool is_i32 = op_type == VALUE_TYPE_I32;
/* currently we only implement atomic cmpxchg on x86-64 target */
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)
/* cmpxchg will use register al/ax/eax/rax to store parameter expected
* value, and the read result will also be stored to al/ax/eax/rax */
JitReg eax_hreg = jit_codegen_get_hreg_by_name("eax");
JitReg rax_hreg = jit_codegen_get_hreg_by_name("rax");
JitInsn *insn = NULL;
bh_assert(op_type == VALUE_TYPE_I32 || op_type == VALUE_TYPE_I64);
if (is_i32) {
POP_I32(value);
POP_I32(expect);
result = jit_cc_new_reg_I32(cc);
}
else {
POP_I64(value);
POP_I64(expect);
result = jit_cc_new_reg_I64(cc);
}
POP_I32(addr);
offset1 = check_and_seek(cc, addr, offset, bytes);
if (!offset1) {
goto fail;
}
CHECK_ALIGNMENT(offset1);
memory_data = get_memory_data_reg(cc->jit_frame, 0);
GEN_INSN(MOV, is_i32 ? eax_hreg : rax_hreg, expect);
switch (bytes) {
case 1:
{
insn = GEN_INSN(AT_CMPXCHGU8, value, is_i32 ? eax_hreg : rax_hreg,
memory_data, offset1);
break;
}
case 2:
{
insn = GEN_INSN(AT_CMPXCHGU16, value, is_i32 ? eax_hreg : rax_hreg,
memory_data, offset1);
break;
}
case 4:
{
if (op_type == VALUE_TYPE_I32)
insn =
GEN_INSN(AT_CMPXCHGI32, value, is_i32 ? eax_hreg : rax_hreg,
memory_data, offset1);
else
insn =
GEN_INSN(AT_CMPXCHGU32, value, is_i32 ? eax_hreg : rax_hreg,
memory_data, offset1);
break;
}
case 8:
{
insn = GEN_INSN(AT_CMPXCHGI64, value, is_i32 ? eax_hreg : rax_hreg,
memory_data, offset1);
break;
}
default:
{
bh_assert(0);
goto fail;
}
}
if (!insn
|| !jit_lock_reg_in_insn(cc, insn, is_i32 ? eax_hreg : rax_hreg)) {
jit_set_last_error(cc, "generate cmpxchg insn or lock ra hreg failed");
goto fail;
}
GEN_INSN(MOV, result, is_i32 ? eax_hreg : rax_hreg);
if (is_i32)
PUSH_I32(result);
else
PUSH_I64(result);
return true;
#endif /* defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) */
fail:
return false;
}
@ -812,8 +1113,10 @@ jit_compile_op_atomic_wait(JitCompContext *cc, uint8 op_type, uint32 align,
JitReg offset1 = check_and_seek(cc, addr, offset, bytes);
if (!offset1)
goto fail;
JitReg maddr = jit_cc_new_reg_I64(cc);
CHECK_ALIGNMENT(maddr, memory_data, offset1);
CHECK_ALIGNMENT(offset1);
JitReg maddr = jit_cc_new_reg_ptr(cc);
GEN_INSN(ADD, maddr, memory_data, offset1);
// Prepare `wasm_runtime_atomic_wait` arguments
JitReg res = jit_cc_new_reg_I32(cc);
@ -835,6 +1138,12 @@ jit_compile_op_atomic_wait(JitCompContext *cc, uint8 op_type, uint32 align,
goto fail;
PUSH_I32(res);
#if WASM_ENABLE_THREAD_MGR != 0
/* Insert suspend check point */
if (!jit_check_suspend_flags(cc))
goto fail;
#endif
return true;
fail:
return false;
@ -854,8 +1163,10 @@ jit_compiler_op_atomic_notify(JitCompContext *cc, uint32 align, uint32 offset,
JitReg offset1 = check_and_seek(cc, addr, offset, bytes);
if (!offset1)
goto fail;
JitReg maddr = jit_cc_new_reg_I64(cc);
CHECK_ALIGNMENT(maddr, memory_data, offset1);
CHECK_ALIGNMENT(offset1);
JitReg maddr = jit_cc_new_reg_ptr(cc);
GEN_INSN(ADD, maddr, memory_data, offset1);
// Prepare `wasm_runtime_atomic_notify` arguments
JitReg res = jit_cc_new_reg_I32(cc);
@ -879,4 +1190,11 @@ jit_compiler_op_atomic_notify(JitCompContext *cc, uint32 align, uint32 offset,
fail:
return false;
}
bool
jit_compiler_op_atomic_fence(JitCompContext *cc)
{
GEN_INSN(FENCE);
return true;
}
#endif

3
core/iwasm/fast-jit/fe/jit_emit_memory.h
View File

@ -80,6 +80,9 @@ jit_compile_op_atomic_wait(JitCompContext *cc, uint8 op_type, uint32 align,
bool
jit_compiler_op_atomic_notify(JitCompContext *cc, uint32 align, uint32 offset,
uint32 bytes);
bool
jit_compiler_op_atomic_fence(JitCompContext *cc);
#endif
#ifdef __cplusplus

7
core/iwasm/fast-jit/jit_dump.c
View File

@ -114,7 +114,10 @@ jit_dump_insn(JitCompContext *cc, JitInsn *insn)
switch (insn->opcode) {
#define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) \
case JIT_OP_##NAME: \
os_printf(" %-15s", #NAME); \
if (insn->flags_u8 & 0x1) \
os_printf(" ATOMIC %-8s", #NAME); \
else \
os_printf(" %-15s", #NAME); \
jit_dump_insn_##OPND_KIND(cc, insn, OPND_NUM); \
break;
#include "jit_ir.def"
@ -319,7 +322,9 @@ jit_pass_dump(JitCompContext *cc)
os_printf("JIT.COMPILER.DUMP: PASS_NO=%d PREV_PASS=%s\n\n", pass_no,
pass_name);
jit_dump_cc(cc);
os_printf("\n");
return true;
}

65
core/iwasm/fast-jit/jit_frontend.c
View File

@ -223,18 +223,37 @@ get_memory_data_reg(JitFrame *frame, uint32 mem_idx)
{
JitCompContext *cc = frame->cc;
JitReg module_inst_reg = get_module_inst_reg(frame);
uint32 memory_data_offset =
(uint32)offsetof(WASMModuleInstance, global_table_data.bytes)
+ (uint32)offsetof(WASMMemoryInstance, memory_data);
uint32 memory_data_offset;
bh_assert(mem_idx == 0);
#if WASM_ENABLE_SHARED_MEMORY != 0
uint32 memories_offset = (uint32)offsetof(WASMModuleInstance, memories);
JitReg memories_addr = jit_cc_new_reg_ptr(cc);
JitReg memories_0_addr = jit_cc_new_reg_ptr(cc);
memory_data_offset = (uint32)offsetof(WASMMemoryInstance, memory_data);
if (!frame->memory_regs[mem_idx].memory_data) {
frame->memory_regs[mem_idx].memory_data =
cc->memory_regs[mem_idx].memory_data;
/* module_inst->memories */
GEN_INSN(LDPTR, memories_addr, module_inst_reg,
NEW_CONST(I32, memories_offset));
/* module_inst->memories[0] */
GEN_INSN(LDPTR, memories_0_addr, memories_addr, NEW_CONST(I32, 0));
/* memories[0]->memory_data */
GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data,
memories_0_addr, NEW_CONST(I32, memory_data_offset));
}
#else
memory_data_offset =
(uint32)offsetof(WASMModuleInstance, global_table_data.bytes)
+ (uint32)offsetof(WASMMemoryInstance, memory_data);
if (!frame->memory_regs[mem_idx].memory_data) {
frame->memory_regs[mem_idx].memory_data =
cc->memory_regs[mem_idx].memory_data;
GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data,
module_inst_reg, NEW_CONST(I32, memory_data_offset));
}
#endif
return frame->memory_regs[mem_idx].memory_data;
}
@ -1078,6 +1097,39 @@ read_leb(JitCompContext *cc, const uint8 *buf, const uint8 *buf_end,
res = (int64)res64; \
} while (0)
#if WASM_ENABLE_SHARED_MEMORY != 0
#define COMPILE_ATOMIC_RMW(OP, NAME) \
case WASM_OP_ATOMIC_RMW_I32_##NAME: \
bytes = 4; \
op_type = VALUE_TYPE_I32; \
goto OP_ATOMIC_##OP; \
case WASM_OP_ATOMIC_RMW_I64_##NAME: \
bytes = 8; \
op_type = VALUE_TYPE_I64; \
goto OP_ATOMIC_##OP; \
case WASM_OP_ATOMIC_RMW_I32_##NAME##8_U: \
bytes = 1; \
op_type = VALUE_TYPE_I32; \
goto OP_ATOMIC_##OP; \
case WASM_OP_ATOMIC_RMW_I32_##NAME##16_U: \
bytes = 2; \
op_type = VALUE_TYPE_I32; \
goto OP_ATOMIC_##OP; \
case WASM_OP_ATOMIC_RMW_I64_##NAME##8_U: \
bytes = 1; \
op_type = VALUE_TYPE_I64; \
goto OP_ATOMIC_##OP; \
case WASM_OP_ATOMIC_RMW_I64_##NAME##16_U: \
bytes = 2; \
op_type = VALUE_TYPE_I64; \
goto OP_ATOMIC_##OP; \
case WASM_OP_ATOMIC_RMW_I64_##NAME##32_U: \
bytes = 4; \
op_type = VALUE_TYPE_I64; \
OP_ATOMIC_##OP : bin_op = AtomicRMWBinOp##OP; \
goto build_atomic_rmw;
#endif
static bool
jit_compile_func(JitCompContext *cc)
{
@ -2096,6 +2148,8 @@ jit_compile_func(JitCompContext *cc)
case WASM_OP_ATOMIC_FENCE:
/* Skip memory index */
frame_ip++;
if (!jit_compiler_op_atomic_fence(cc))
return false;
break;
case WASM_OP_ATOMIC_I32_LOAD:
bytes = 4;
@ -2192,15 +2246,12 @@ jit_compile_func(JitCompContext *cc)
return false;
break;
/* TODO */
/*
COMPILE_ATOMIC_RMW(Add, ADD);
COMPILE_ATOMIC_RMW(Sub, SUB);
COMPILE_ATOMIC_RMW(And, AND);
COMPILE_ATOMIC_RMW(Or, OR);
COMPILE_ATOMIC_RMW(Xor, XOR);
COMPILE_ATOMIC_RMW(Xchg, XCHG);
*/
build_atomic_rmw:
if (!jit_compile_op_atomic_rmw(cc, bin_op, op_type,

11
core/iwasm/fast-jit/jit_frontend.h
View File

@ -108,6 +108,17 @@ typedef enum FloatArithmetic {
FLOAT_MAX,
} FloatArithmetic;
#if WASM_ENABLE_SHARED_MEMORY != 0
typedef enum AtomicRMWBinOp {
AtomicRMWBinOpAdd,
AtomicRMWBinOpSub,
AtomicRMWBinOpAnd,
AtomicRMWBinOpOr,
AtomicRMWBinOpXor,
AtomicRMWBinOpXchg
} AtomicRMWBinOp;
#endif
/**
* Translate instructions in a function. The translated block must
* end with a branch instruction whose targets are offsets relating to

18
core/iwasm/fast-jit/jit_ir.c
View File

@ -10,7 +10,11 @@
/**
* Operand kinds of instructions.
*/
enum { JIT_OPND_KIND_Reg, JIT_OPND_KIND_VReg, JIT_OPND_KIND_LookupSwitch };
enum {
JIT_OPND_KIND_Reg,
JIT_OPND_KIND_VReg,
JIT_OPND_KIND_LookupSwitch,
};
/**
* Operand kind of each instruction.
@ -45,6 +49,18 @@ static const uint8 insn_opnd_first_use[] = {
jit_calloc(offsetof(JitInsn, _opnd._opnd_VReg._reg) \
+ sizeof(JitReg) * (OPND_NUM))
JitInsn *
_jit_insn_new_Reg_0(JitOpcode opc)
{
JitInsn *insn = JIT_INSN_NEW_Reg(0);
if (insn) {
insn->opcode = opc;
}
return insn;
}
JitInsn *
_jit_insn_new_Reg_1(JitOpcode opc, JitReg r0)
{

44
core/iwasm/fast-jit/jit_ir.def
View File

@ -200,6 +200,50 @@ INSN(CALLBC, Reg, 4, 2)
INSN(RETURNBC, Reg, 3, 0)
INSN(RETURN, Reg, 1, 0)
#if WASM_ENABLE_SHARED_MEMORY != 0
/* Atomic Memory Accesses */
/* op1(replacement val) op2(expected val) op3(mem data) op4(offset)
* and in x86, the result is stored in register al/ax/eax/rax */
INSN(AT_CMPXCHGU8, Reg, 4, 0)
INSN(AT_CMPXCHGU16, Reg, 4, 0)
INSN(AT_CMPXCHGI32, Reg, 4, 0)
INSN(AT_CMPXCHGU32, Reg, 4, 0)
INSN(AT_CMPXCHGI64, Reg, 4, 0)
/* rmw operations:
* op1(read value) op2(operand value) op3(mem data) op4(offset) */
INSN(AT_ADDU8, Reg, 4, 1)
INSN(AT_ADDU16, Reg, 4, 1)
INSN(AT_ADDI32, Reg, 4, 1)
INSN(AT_ADDU32, Reg, 4, 1)
INSN(AT_ADDI64, Reg, 4, 1)
INSN(AT_SUBU8, Reg, 4, 1)
INSN(AT_SUBU16, Reg, 4, 1)
INSN(AT_SUBI32, Reg, 4, 1)
INSN(AT_SUBU32, Reg, 4, 1)
INSN(AT_SUBI64, Reg, 4, 1)
INSN(AT_ANDU8, Reg, 4, 1)
INSN(AT_ANDU16, Reg, 4, 1)
INSN(AT_ANDI32, Reg, 4, 1)
INSN(AT_ANDU32, Reg, 4, 1)
INSN(AT_ANDI64, Reg, 4, 1)
INSN(AT_ORU8, Reg, 4, 1)
INSN(AT_ORU16, Reg, 4, 1)
INSN(AT_ORI32, Reg, 4, 1)
INSN(AT_ORU32, Reg, 4, 1)
INSN(AT_ORI64, Reg, 4, 1)
INSN(AT_XORU8, Reg, 4, 1)
INSN(AT_XORU16, Reg, 4, 1)
INSN(AT_XORI32, Reg, 4, 1)
INSN(AT_XORU32, Reg, 4, 1)
INSN(AT_XORI64, Reg, 4, 1)
INSN(AT_XCHGU8, Reg, 4, 1)
INSN(AT_XCHGU16, Reg, 4, 1)
INSN(AT_XCHGI32, Reg, 4, 1)
INSN(AT_XCHGU32, Reg, 4, 1)
INSN(AT_XCHGI64, Reg, 4, 1)
INSN(FENCE, Reg, 0, 0)
#endif
#undef INSN
/**

38
core/iwasm/fast-jit/jit_ir.h
View File

@ -313,7 +313,8 @@ typedef struct JitInsn {
/* Opcode of the instruction. */
uint16 opcode;
/* Reserved field that may be used by optimizations locally. */
/* Reserved field that may be used by optimizations locally.
* bit_0(Least Significant Bit) is atomic flag for load/store */
uint8 flags_u8;
/* The unique ID of the instruction. */
@ -346,6 +347,9 @@ typedef enum JitOpcode {
* Helper functions for creating new instructions. Don't call them
* directly. Use jit_insn_new_NAME, such as jit_insn_new_MOV instead.
*/
JitInsn *
_jit_insn_new_Reg_0(JitOpcode opc);
JitInsn *
_jit_insn_new_Reg_1(JitOpcode opc, JitReg r0);
JitInsn *
@ -368,31 +372,35 @@ _jit_insn_new_LookupSwitch_1(JitOpcode opc, JitReg value, uint32 num);
* Instruction creation functions jit_insn_new_NAME, where NAME is the
* name of the instruction defined in jit_ir.def.
*/
#define ARG_DECL_Reg_0
#define ARG_LIST_Reg_0
#define ARG_DECL_Reg_1 JitReg r0
#define ARG_LIST_Reg_1 r0
#define ARG_LIST_Reg_1 , r0
#define ARG_DECL_Reg_2 JitReg r0, JitReg r1
#define ARG_LIST_Reg_2 r0, r1
#define ARG_LIST_Reg_2 , r0, r1
#define ARG_DECL_Reg_3 JitReg r0, JitReg r1, JitReg r2
#define ARG_LIST_Reg_3 r0, r1, r2
#define ARG_LIST_Reg_3 , r0, r1, r2
#define ARG_DECL_Reg_4 JitReg r0, JitReg r1, JitReg r2, JitReg r3
#define ARG_LIST_Reg_4 r0, r1, r2, r3
#define ARG_LIST_Reg_4 , r0, r1, r2, r3
#define ARG_DECL_Reg_5 JitReg r0, JitReg r1, JitReg r2, JitReg r3, JitReg r4
#define ARG_LIST_Reg_5 r0, r1, r2, r3, r4
#define ARG_LIST_Reg_5 , r0, r1, r2, r3, r4
#define ARG_DECL_VReg_1 JitReg r0, int n
#define ARG_LIST_VReg_1 r0, n
#define ARG_LIST_VReg_1 , r0, n
#define ARG_DECL_VReg_2 JitReg r0, JitReg r1, int n
#define ARG_LIST_VReg_2 r0, r1, n
#define ARG_LIST_VReg_2 , r0, r1, n
#define ARG_DECL_LookupSwitch_1 JitReg value, uint32 num
#define ARG_LIST_LookupSwitch_1 value, num
#define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) \
static inline JitInsn *jit_insn_new_##NAME( \
ARG_DECL_##OPND_KIND##_##OPND_NUM) \
{ \
return _jit_insn_new_##OPND_KIND##_##OPND_NUM( \
JIT_OP_##NAME, ARG_LIST_##OPND_KIND##_##OPND_NUM); \
#define ARG_LIST_LookupSwitch_1 , value, num
#define INSN(NAME, OPND_KIND, OPND_NUM, FIRST_USE) \
static inline JitInsn *jit_insn_new_##NAME( \
ARG_DECL_##OPND_KIND##_##OPND_NUM) \
{ \
return _jit_insn_new_##OPND_KIND##_##OPND_NUM( \
JIT_OP_##NAME ARG_LIST_##OPND_KIND##_##OPND_NUM); \
}
#include "jit_ir.def"
#undef INSN
#undef ARG_DECL_Reg_0
#undef ARG_LIST_Reg_0
#undef ARG_DECL_Reg_1
#undef ARG_LIST_Reg_1
#undef ARG_DECL_Reg_2

14
core/iwasm/fast-jit/jit_regalloc.c
View File

@ -410,6 +410,13 @@ collect_distances(RegallocContext *rc, JitBasicBlock *basic_block)
JIT_FOREACH_INSN(basic_block, insn)
{
#if WASM_ENABLE_SHARED_MEMORY != 0
/* fence insn doesn't have any operand, hence, no regs involved */
if (insn->opcode == JIT_OP_FENCE) {
continue;
}
#endif
JitRegVec regvec = jit_insn_opnd_regs(insn);
unsigned i;
JitReg *regp;
@ -737,6 +744,13 @@ allocate_for_basic_block(RegallocContext *rc, JitBasicBlock *basic_block,
JIT_FOREACH_INSN_REVERSE(basic_block, insn)
{
#if WASM_ENABLE_SHARED_MEMORY != 0
/* fence insn doesn't have any operand, hence, no regs involved */
if (insn->opcode == JIT_OP_FENCE) {
continue;
}
#endif
JitRegVec regvec = jit_insn_opnd_regs(insn);
unsigned first_use = jit_insn_opnd_first_use(insn);
unsigned i;

8
tests/wamr-test-suites/test_wamr.sh
View File

@ -444,14 +444,6 @@ function spec_test()
if [[ ${ENABLE_MULTI_THREAD} == 1 ]]; then
ARGS_FOR_SPEC_TEST+="-p "
if [[ $1 == 'fast-jit' ]]; then
echo "fast-jit doesn't support multi-thread feature yet, skip it"
return
fi
if [[ $1 == 'multi-tier-jit' ]]; then
echo "multi-tier-jit doesn't support multi-thread feature yet, skip it"
return
fi
fi
if [[ ${ENABLE_XIP} == 1 ]]; then