From 3179b10d8a12a4be2d43cfbb381c6190f80feed2 Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 16 Jan 2025 13:41:03 +0800 Subject: [PATCH 01/17] shared heap create from preallocated buffer and use as a single memory region --- .../shared-heap-preallocated/CMakeLists.txt | 127 +++++++ samples/shared-heap-preallocated/src/main.c | 335 ++++++++++++++++++ .../wasm-apps/CMakeLists.txt | 43 +++ .../wasm-apps/test1.c | 71 ++++ .../wasm-apps/test2.c | 18 + 5 files changed, 594 insertions(+) create mode 100644 samples/shared-heap-preallocated/CMakeLists.txt create mode 100644 samples/shared-heap-preallocated/src/main.c create mode 100644 samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt create mode 100644 samples/shared-heap-preallocated/wasm-apps/test1.c create mode 100644 samples/shared-heap-preallocated/wasm-apps/test2.c diff --git a/samples/shared-heap-preallocated/CMakeLists.txt b/samples/shared-heap-preallocated/CMakeLists.txt new file mode 100644 index 000000000..6346d077e --- /dev/null +++ b/samples/shared-heap-preallocated/CMakeLists.txt @@ -0,0 +1,127 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") + project (shared_heap_test) +else() + project (shared_heap_test C ASM) +endif() + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" + +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Debug) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_FAST_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) +set (WAMR_BUILD_LIBC_BUILTIN 1) +set (WAMR_BUILD_LIBC_WASI 0) +set (WAMR_BUILD_SHARED_HEAP 1) +set (WAMR_BUILD_GC_HEAP_VERIFY 1) + +if (NOT MSVC) + # linker flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif () + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib STATIC ${WAMR_RUNTIME_LIB_SOURCE}) +if (MSVC) + target_compile_definitions(vmlib PRIVATE WASM_API_EXTERN=) +endif() +target_link_libraries(vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) + +################ application related ################ +include_directories(${CMAKE_CURRENT_LIST_DIR}/src) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (shared_heap_test src/main.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (shared_heap_test PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (APPLE) + target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread) +else () + target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread -lrt) +endif () + +add_subdirectory(wasm-apps) + +if (WAMR_BUILD_AOT EQUAL 1) + set (WAMR_COMPILER_DIR ${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build) + message (CHECK_START "Detecting WAMR_COMPILER at ${WAMR_COMPILER_DIR}") + find_file (WAMR_COMPILER + wamrc + PATHS "${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build" + NO_DEFAULT_PATH + NO_CMAKE_FIND_ROOT_PATH + ) + if (WAMR_COMPILER) + message (CHECK_PASS "found") + else() + message (CHECK_FAIL "not found") + endif() + if (NOT EXISTS ${WAMR_COMPILER}) + message (FATAL_ERROR "Please build wamrc under ${WAMR_ROOT_DIR}/wamr-compiler") + else() + message (STATUS "WAMR_COMPILER is ${WAMR_COMPILER}") + endif() + + add_custom_target( + wasm_to_aot + ALL + DEPENDS wasm-apps/test1.wasm wasm-apps/test2.wasm ${WAMR_COMPILER} + COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test1.aot wasm-apps/test1.wasm + COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test2.aot wasm-apps/test2.wasm + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) +endif() diff --git a/samples/shared-heap-preallocated/src/main.c b/samples/shared-heap-preallocated/src/main.c new file mode 100644 index 000000000..9842f33c3 --- /dev/null +++ b/samples/shared-heap-preallocated/src/main.c @@ -0,0 +1,335 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_platform.h" +#include "bh_read_file.h" + +typedef struct thread_arg { + bh_queue *queue; + wasm_module_inst_t module_inst; +} thread_arg; + +static void * +thread1_callback(void *arg) +{ + thread_arg *targ = arg; + wasm_module_inst_t module_inst = targ->module_inst; + bh_queue *queue = targ->queue; + wasm_exec_env_t exec_env; + wasm_function_inst_t my_shared_heap_malloc_func; + wasm_function_inst_t my_shared_heap_free_func; + uint32 i, argv[2]; + + /* lookup wasm functions */ + if (!(my_shared_heap_malloc_func = wasm_runtime_lookup_function( + module_inst, "my_shared_heap_malloc")) + || !(my_shared_heap_free_func = wasm_runtime_lookup_function( + module_inst, "my_shared_heap_free"))) { + printf("Failed to lookup function.\n"); + } + + /* create exec env */ + if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { + printf("Failed to create exec env.\n"); + return NULL; + } + + /* allocate memory with wasm_runtime_shared_heap_malloc and send it + to wasm app2 */ + for (i = 0; i < 1; i++) { + uint8 *buf; + uint64 offset; + + offset = wasm_runtime_shared_heap_malloc(module_inst, 4096 * (i + 1), + (void **)&buf); + + if (offset == 0) { + printf("Failed to allocate memory from shared heap\n"); + break; + } + + snprintf(buf, 1024, "Hello, this is buf %u allocated from shared heap", + i + 1); + + printf("wasm app1 send buf: %s\n\n", buf); + if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + printf("Failed to post message to queue\n"); + wasm_runtime_shared_heap_free(module_inst, offset); + break; + } + } + + /* allocate memory by calling my_shared_heap_malloc function and send it + to wasm app2 */ + for (i = 1; i < 2; i++) { + uint8 *buf; + + argv[0] = 1024 * (i + 1); + argv[1] = i + 1; + wasm_runtime_call_wasm(exec_env, my_shared_heap_malloc_func, 2, argv); + + if (wasm_runtime_get_exception(module_inst)) { + printf("Failed to call 'my_shared_heap_malloc' function: %s\n", + wasm_runtime_get_exception(module_inst)); + break; + } + if (argv[0] == 0) { + printf("Failed to allocate memory from shared heap\n"); + break; + } + + buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); + + printf("wasm app1 send buf: %s\n\n", buf); + if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + printf("Failed to post message to queue\n"); + wasm_runtime_shared_heap_free(module_inst, argv[0]); + break; + } + } + + wasm_runtime_destroy_exec_env(exec_env); + + return NULL; +} + +static void +queue_callback(void *message, void *arg) +{ + bh_message_t msg = (bh_message_t)message; + wasm_exec_env_t exec_env = arg; + wasm_module_inst_t module_inst = wasm_runtime_get_module_inst(exec_env); + wasm_function_inst_t print_buf_func; + uint32 argv[2]; + + /* lookup wasm function */ + if (!(print_buf_func = + wasm_runtime_lookup_function(module_inst, "print_buf"))) { + printf("Failed to lookup function.\n"); + return; + } + + char *buf = bh_message_payload(msg); + printf("wasm app's native queue received buf: %s\n\n", buf); + + /* call wasm function */ + argv[0] = wasm_runtime_addr_native_to_app(module_inst, buf); + wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); + if (wasm_runtime_get_exception(module_inst)) { + printf("Failed to call 'print_buf' function: %s\n", + wasm_runtime_get_exception(module_inst)); + } +} + +static void * +thread2_callback(void *arg) +{ + thread_arg *targ = arg; + bh_queue *queue = targ->queue; + wasm_module_inst_t module_inst = targ->module_inst; + wasm_exec_env_t exec_env; + + /* create exec env */ + if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { + printf("Failed to create exec env.\n"); + return NULL; + } + + /* enter queue's message loop until bh_queue_exit_loop_run + is called */ + bh_queue_enter_loop_run(queue, queue_callback, exec_env); + + wasm_runtime_destroy_exec_env(exec_env); + + return NULL; +} + +static char global_heap_buf[512 * 1024]; + +int +main(int argc, char **argv) +{ + char *wasm_file1 = NULL, *wasm_file2 = NULL; + uint8 *wasm_file1_buf = NULL, *wasm_file2_buf = NULL; + uint32 wasm_file1_size, wasm_file2_size; + wasm_module_t wasm_module1 = NULL, wasm_module2 = NULL; + wasm_module_inst_t module_inst1 = NULL; + wasm_module_inst_t module_inst2 = NULL; + wasm_shared_heap_t shared_heap = NULL; + bh_queue *queue = NULL; + RuntimeInitArgs init_args; + SharedHeapInitArgs heap_init_args; + char error_buf[128] = { 0 }; + bool aot_mode = false; + int ret = -1; + +#if WASM_DISABLE_HW_BOUND_CHECK == 0 +#if defined(BUILD_TARGET_X86_64) + printf("HW BOUND CHECK\n"); +#endif +#endif + + if (argc > 1 && !strcmp(argv[1], "--aot")) + aot_mode = true; + + if (!aot_mode) + printf("Test shared heap in interpreter mode\n\n"); + else + printf("Test shared heap in AOT mode\n\n"); + + bh_log_set_verbose_level(5); + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + /* init wasm runtime */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + /* create queue */ + if (!(queue = bh_queue_create())) { + printf("Create queue failed.\n"); + goto fail; + } + + /* read wasm file */ + if (!aot_mode) + wasm_file1 = "./wasm-apps/test1.wasm"; + else + wasm_file1 = "./wasm-apps/test1.aot"; + if (!(wasm_file1_buf = + bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { + printf("Open wasm file %s failed.\n", wasm_file1); + goto fail; + } + + /* load wasm file */ + wasm_module1 = wasm_runtime_load((uint8 *)wasm_file1_buf, wasm_file1_size, + error_buf, sizeof(error_buf)); + if (!wasm_module1) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* instantiate module */ + module_inst1 = wasm_runtime_instantiate(wasm_module1, 65536, 4096, + error_buf, sizeof(error_buf)); + if (!module_inst1) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* read wasm file */ + if (!aot_mode) + wasm_file2 = "./wasm-apps/test2.wasm"; + else + wasm_file2 = "./wasm-apps/test2.aot"; + if (!(wasm_file2_buf = + bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { + printf("Open wasm file %s failed.\n", wasm_file1); + goto fail; + } + + /* load wasm file */ + wasm_module2 = wasm_runtime_load((uint8 *)wasm_file2_buf, wasm_file2_size, + error_buf, sizeof(error_buf)); + if (!wasm_module2) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* instantiate module */ + module_inst2 = wasm_runtime_instantiate(wasm_module2, 65536, 4096, + error_buf, sizeof(error_buf)); + if (!module_inst2) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* create shared heap */ + memset(&heap_init_args, 0, sizeof(heap_init_args)); + heap_init_args.size = 4096 + 1; + shared_heap = wasm_runtime_create_shared_heap(&heap_init_args); + if (!shared_heap) { + printf("Create shared heap failed. error: %s\n", error_buf); + goto fail; + } + + /* attach module instance 1 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(module_inst1, shared_heap)) { + printf("Attach shared heap failed.\n"); + goto fail; + } + + /* attach module instance 2 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(module_inst2, shared_heap)) { + printf("Attach shared heap failed.\n"); + goto fail; + } + + /* create thread 1 */ + struct thread_arg targ1 = { 0 }; + korp_tid tid1; + targ1.queue = queue; + targ1.module_inst = module_inst1; + if (os_thread_create(&tid1, thread1_callback, &targ1, + APP_THREAD_STACK_SIZE_DEFAULT)) { + printf("Failed to create thread 1\n"); + goto fail; + } + + /* create thread 2 */ + struct thread_arg targ2 = { 0 }; + korp_tid tid2; + targ2.queue = queue; + targ2.module_inst = module_inst2; + if (os_thread_create(&tid2, thread2_callback, &targ2, + APP_THREAD_STACK_SIZE_DEFAULT)) { + printf("Failed to create thread 2\n"); + os_thread_join(tid1, NULL); + goto fail; + } + + /* wait until all messages are post to wasm app2 and wasm app2 + handles all of them, then exit the queue message loop */ + usleep(10000); + bh_queue_exit_loop_run(queue); + + os_thread_join(tid1, NULL); + os_thread_join(tid2, NULL); + + ret = 0; + +fail: + if (module_inst2) + wasm_runtime_deinstantiate(module_inst2); + + if (module_inst1) + wasm_runtime_deinstantiate(module_inst1); + + if (wasm_module2) + wasm_runtime_unload(wasm_module2); + + if (wasm_module1) + wasm_runtime_unload(wasm_module1); + + if (wasm_file2_buf) + wasm_runtime_free(wasm_file2_buf); + + if (wasm_file1_buf) + wasm_runtime_free(wasm_file1_buf); + + if (queue) + bh_queue_destroy(queue); + + wasm_runtime_destroy(); + + return ret; +} diff --git a/samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt b/samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt new file mode 100644 index 000000000..a90c82e1a --- /dev/null +++ b/samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt @@ -0,0 +1,43 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(wasm-apps) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) + +if (APPLE) + set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) + set (CMAKE_C_LINK_FLAGS "") + set (CMAKE_CXX_LINK_FLAGS "") +endif () + +set (CMAKE_SYSTEM_PROCESSOR wasm32) +set (CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set (WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set (CMAKE_C_COMPILER_TARGET "wasm32") +set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") +set (CMAKE_C_FLAGS "-nostdlib -Qunused-arguments -z stack-size=32768") + +set (DEFINED_SYMBOLS "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") + +set (CMAKE_EXE_LINKER_FLAGS + "-O0 -Wl,--initial-memory=65536, \ + -Wl,--no-entry,--strip-all, \ + -Wl,--export=__heap_base,--export=__data_end \ + -Wl,--export=__wasm_call_ctors \ + -Wl,--export=my_shared_heap_malloc \ + -Wl,--export=my_shared_heap_free \ + -Wl,--export=print_buf \ + -Wl,--allow-undefined" +) + +add_executable(test1.wasm test1.c) +target_link_libraries(test1.wasm) + +add_executable(test2.wasm test2.c) +target_link_libraries(test2.wasm) diff --git a/samples/shared-heap-preallocated/wasm-apps/test1.c b/samples/shared-heap-preallocated/wasm-apps/test1.c new file mode 100644 index 000000000..2e3fd2112 --- /dev/null +++ b/samples/shared-heap-preallocated/wasm-apps/test1.c @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include + +extern void * +shared_heap_malloc(uint32_t size); +extern void +shared_heap_free(void *ptr); + +void * +my_shared_heap_malloc(uint32_t size, uint32_t index) +{ + char *buf1 = NULL, *buf2 = NULL, *buf; + + char *buf3 = NULL; + buf3 = malloc(2048); + + buf1 = shared_heap_malloc(1024); + if (!buf1) + return NULL; + + buf1[0] = 'H'; + buf1[1] = 'e'; + buf1[2] = 'l'; + buf1[3] = 'l'; + buf1[4] = 'o'; + buf1[5] = ','; + buf1[6] = ' '; + + buf2 = shared_heap_malloc(1024); + if (!buf2) { + shared_heap_free(buf1); + return NULL; + } + + snprintf(buf2, 1024, "this is buf %u allocated from shared heap", index); + + buf = shared_heap_malloc(size); + if (!buf) { + shared_heap_free(buf1); + shared_heap_free(buf2); + return NULL; + } + + memset(buf, 0, size); + memcpy(buf, buf1, strlen(buf1)); + memcpy(buf + strlen(buf1), buf2, strlen(buf2)); + + uint32_t max_value = UINT32_MAX; + char *address = (char *)max_value; + printf("The address of buf1, buf2, buf, UINT32_MAX is: %p,%p,%p,%p\n", buf1, + buf2, buf, address); + address[0] = 'c'; + printf("The content address of UINT32_MAX is: %c\n", *address); + + shared_heap_free(buf1); + shared_heap_free(buf2); + return buf; +} + +void +my_shared_heap_free(void *ptr) +{ + shared_heap_free(ptr); +} diff --git a/samples/shared-heap-preallocated/wasm-apps/test2.c b/samples/shared-heap-preallocated/wasm-apps/test2.c new file mode 100644 index 000000000..b63efcd1a --- /dev/null +++ b/samples/shared-heap-preallocated/wasm-apps/test2.c @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +#include + +extern void +shared_heap_free(void *ptr); + +void +print_buf(char *buf) +{ + printf("wasm app2's wasm func received buf: %s\n\n", buf); + shared_heap_free(buf); +} From 013306d67cf749d5971510b47fb8859ac6d840a7 Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 23 Jan 2025 18:20:01 +0800 Subject: [PATCH 02/17] first draft of shared heap enhancement in interpreter and runtime --- core/iwasm/common/wasm_memory.c | 267 ++++++++++++++----- core/iwasm/interpreter/wasm_interp_classic.c | 50 +++- 2 files changed, 232 insertions(+), 85 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 74df84e56..a24b3c32d 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -143,7 +143,7 @@ is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) #if WASM_ENABLE_SHARED_HEAP != 0 static void * -wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size); +wasm_mmap_linear_memory(uint64 map_size, uint64 commit_size); static void wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size); @@ -177,39 +177,53 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) goto fail1; } - if (!(heap->heap_handle = - runtime_malloc(mem_allocator_get_heap_struct_size()))) { - goto fail2; - } - size = align_uint(size, os_getpagesize()); heap->size = size; heap->start_off_mem64 = UINT64_MAX - heap->size + 1; heap->start_off_mem32 = UINT32_MAX - heap->size + 1; + heap->attached_count = 0; - if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { - LOG_WARNING("Invalid size of shared heap"); - goto fail3; + if (init_args->pre_allocated_addr != NULL) { + /* Create shared heap from a pre allocated buffer, its size need to + * align with system page */ + if (size != init_args->size) { + LOG_WARNING("Pre allocated size need to be aligned with system " + "page size to create shared heap"); + goto fail1; + } + + heap->heap_handle = NULL; } + else { + if (!(heap->heap_handle = + runtime_malloc(mem_allocator_get_heap_struct_size()))) { + goto fail2; + } + + if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { + LOG_WARNING("Invalid size of shared heap"); + goto fail3; + } #ifndef OS_ENABLE_HW_BOUND_CHECK - map_size = size; + map_size = size; #else - /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: - * ea = i + memarg.offset - * both i and memarg.offset are u32 in range 0 to 4G - * so the range of ea is 0 to 8G - */ - map_size = 8 * (uint64)BH_GB; + /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: + * ea = i + memarg.offset + * both i and memarg.offset are u32 in range 0 to 4G + * so the range of ea is 0 to 8G + */ + map_size = 8 * (uint64)BH_GB; #endif - if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) { - goto fail3; - } - if (!mem_allocator_create_with_struct_and_pool( - heap->heap_handle, heap_struct_size, heap->base_addr, size)) { - LOG_WARNING("init share heap failed"); - goto fail4; + if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) { + goto fail3; + } + if (!mem_allocator_create_with_struct_and_pool( + heap->heap_handle, heap_struct_size, heap->base_addr, size)) { + LOG_WARNING("init share heap failed"); + goto fail4; + } } os_mutex_lock(&shared_heap_list_lock); @@ -233,6 +247,74 @@ fail1: return NULL; } +WASMSharedHeap * +wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) +{ + WASMSharedHeap *cur; + bool heap_handle_exist = false; + + if (!head || !body) { + LOG_WARNING("Invalid shared heap to chain."); + return NULL; + } + + os_mutex_lock(&shared_heap_list_lock); + if (head->attached_count != 0 || body->attached_count != 0) { + LOG_WARNING("To create shared heap chain, all shared heap need to be " + "detached first."); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + + for (cur = head; cur; cur = cur->chain_next) { + if (cur->heap_handle && heap_handle_exist) { + LOG_WARNING( + "To create shared heap chain, only one of shared heap can " + "dynamically shared_heap_malloc and shared_heap_free, the rest " + "can only be pre-allocated shared heap"); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + if (cur->heap_handle) + heap_handle_exist = true; + } + + head->start_off_mem64 = body->start_off_mem64 - head->size + 1; + head->start_off_mem32 = body->start_off_mem32 - head->size + 1; + head->chain_next = body; + os_mutex_unlock(&shared_heap_list_lock); + return head; +} + +WASMSharedHeap * +wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain) +{ + WASMSharedHeap *cur; + + if (!head || !head->chain_next) { + LOG_WARNING("Invalid shared heap chain to disconnect the head from."); + return NULL; + } + + os_mutex_lock(&shared_heap_list_lock); + if (head->attached_count != 0) { + LOG_WARNING("To disconnect the shared heap head from the shared heap " + "chain, the shared heap chain needs to be detached first."); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + + for (cur = head; cur && cur->chain_next; cur = cur->chain_next) { + cur->start_off_mem64 = UINT64_MAX - cur->size + 1; + cur->start_off_mem32 = UINT32_MAX - cur->size + 1; + if (!entire_chain) + break; + } + + os_mutex_unlock(&shared_heap_list_lock); + return cur; +} + bool wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, WASMSharedHeap *shared_heap) @@ -303,6 +385,9 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, } #endif /* end of WASM_ENABLE_AOT != 0 */ + os_mutex_lock(&shared_heap_list_lock); + shared_heap->attached_count++; + os_mutex_unlock(&shared_heap_list_lock); return true; } @@ -324,6 +409,11 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) if (module_inst->module_type == Wasm_Module_Bytecode) { WASMModuleInstanceExtra *e = (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + if (e->shared_heap != NULL) { + os_mutex_lock(&shared_heap_list_lock); + e->shared_heap->attached_count--; + os_mutex_unlock(&shared_heap_list_lock); + } e->shared_heap = NULL; #if WASM_ENABLE_JIT != 0 #if UINTPTR_MAX == UINT64_MAX @@ -339,6 +429,11 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) if (module_inst->module_type == Wasm_Module_AoT) { AOTModuleInstanceExtra *e = (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + if (e->shared_heap != NULL) { + os_mutex_lock(&shared_heap_list_lock); + e->shared_heap->attached_count--; + os_mutex_unlock(&shared_heap_list_lock); + } e->shared_heap = NULL; #if UINTPTR_MAX == UINT64_MAX e->shared_heap_start_off.u64 = UINT64_MAX; @@ -387,9 +482,11 @@ wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) static bool is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, - bool is_memory64, uint64 app_offset, uint32 bytes) + bool is_memory64, uint64 app_offset, uint32 bytes, + WASMSharedHeap **target_heap) { - WASMSharedHeap *heap = get_shared_heap(module_inst); + WASMSharedHeap *heap = get_shared_heap(module_inst), *cur; + uint64 shared_heap_start, shared_heap_end; if (!heap) { return false; @@ -399,57 +496,66 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, bytes = 1; } - if (!is_memory64) { - if (app_offset >= heap->start_off_mem32 - && app_offset <= UINT32_MAX - bytes + 1) { - return true; - } - } - else { - if (app_offset >= heap->start_off_mem64 - && app_offset <= UINT64_MAX - bytes + 1) { + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + cur->start_off_mem64 ? is_memory64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start + cur->size; + if (app_offset >= shared_heap_start && app_offset <= shared_heap_end) { + if (target_heap) + *target_heap = cur; return true; } } + if (target_heap) + *target_heap = NULL; return false; } static bool is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, - uint8 *addr, uint32 bytes) + uint8 *addr, uint32 bytes, + WASMSharedHeap **target_heap) { - WASMSharedHeap *heap = get_shared_heap(module_inst); - uintptr_t base_addr; - uintptr_t addr_int; - uintptr_t end_addr; + WASMSharedHeap *cur, *heap_head = get_shared_heap(module_inst); + uintptr_t base_addr, addr_int, end_addr; - if (!heap) { + if (!heap_head) { return false; } - base_addr = (uintptr_t)heap->base_addr; - addr_int = (uintptr_t)addr; - if (addr_int < base_addr) { - return false; + /* Iterate through shared heap chain to find whether native addr in one of + * shared heap */ + for (cur = heap_head; cur != NULL; cur = cur->chain_next) { + base_addr = (uintptr_t)cur->base_addr; + addr_int = (uintptr_t)addr; + if (addr_int < base_addr) { + continue; + } + + end_addr = addr_int + bytes; + /* Check for overflow */ + if (end_addr <= addr_int) { + continue; + } + + if (end_addr > base_addr + cur->size) { + continue; + } + + if (target_heap) + *target_heap = cur; + return true; } - end_addr = addr_int + bytes; - /* Check for overflow */ - if (end_addr <= addr_int) { - return false; - } - - if (end_addr > base_addr + heap->size) { - return false; - } - - return true; + if (target_heap) + *target_heap = NULL; + return false; } uint64 wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, - uint64_t size, void **p_native_addr) + uint64 size, void **p_native_addr) { WASMMemoryInstance *memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); @@ -459,6 +565,14 @@ wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, if (!memory || !shared_heap) return 0; + while (shared_heap && !shared_heap->heap_handle) { + shared_heap = shared_heap->chain_next; + } + if (!shared_heap) { + LOG_WARNING("Can't allocate from pre allocated shared heap"); + return 0; + } + native_addr = mem_allocator_malloc(shared_heap->heap_handle, size); if (!native_addr) return 0; @@ -487,6 +601,14 @@ wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr) return; } + while (shared_heap && !shared_heap->heap_handle) { + shared_heap = shared_heap->chain_next; + } + if (!shared_heap) { + LOG_WARNING("The address to free is from pre allocated shared heap"); + return; + } + if (memory->is_memory64) { if (ptr < shared_heap->start_off_mem64) { /* ptr can not > UINT64_MAX */ LOG_WARNING("The address to free isn't in shared heap"); @@ -564,14 +686,16 @@ destroy_shared_heaps() while (heap) { cur = heap; heap = heap->next; - mem_allocator_destroy(cur->heap_handle); - wasm_runtime_free(cur->heap_handle); + if (cur->heap_handle) { + mem_allocator_destroy(cur->heap_handle); + wasm_runtime_free(cur->heap_handle); #ifndef OS_ENABLE_HW_BOUND_CHECK - map_size = cur->size; + map_size = cur->size; #else - map_size = 8 * (uint64)BH_GB; + map_size = 8 * (uint64)BH_GB; #endif - wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size); + wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size); + } wasm_runtime_free(cur); } os_mutex_destroy(&shared_heap_list_lock); @@ -761,7 +885,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_offset, size)) { + app_offset, size, NULL)) { return true; } #endif @@ -812,9 +936,9 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_str_offset, 1)) { - WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + app_str_offset, 1, &shared_heap)) { str = (char *)shared_heap->base_addr + (memory_inst->is_memory64 ? (app_str_offset - shared_heap->start_off_mem64) @@ -884,7 +1008,8 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size)) { + if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size, + NULL)) { return true; } #endif @@ -910,6 +1035,7 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, { WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; WASMMemoryInstance *memory_inst; + WASMSharedHeap *shared_heap; uint8 *addr; bool bounds_checks; @@ -925,8 +1051,7 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_offset, 1)) { - WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + app_offset, 1, &shared_heap)) { uint64 shared_heap_start = 0; if (memory_inst && !memory_inst->is_memory64) { @@ -985,8 +1110,9 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1)) { - WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + WASMSharedHeap *shared_heap; + if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1, + &shared_heap)) { uint64 shared_heap_start = 0; if (memory_inst && !memory_inst->is_memory64) { @@ -1112,8 +1238,7 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, memory_inst->is_memory64, app_buf_addr, - app_buf_size)) { - shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module_inst); + app_buf_size, &shared_heap)) { native_addr = shared_heap->base_addr + (memory_inst->is_memory64 ? (app_buf_addr - shared_heap->start_off_mem64) diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 98668470f..e45a4a18f 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -53,9 +53,34 @@ typedef float64 CellType_F64; #else #define is_default_memory true #endif -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) +#if WASM_ENABLE_MEMORY64 +#define get_shared_heap_start_off(shared_heap) \ + (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) +#else +#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) +#endif +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 0) \ + || ({ \ + bool in_chain = false; \ + WASMSharedHeap *cur; \ + uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ + for (cur = shared_heap; cur; cur = cur->chain_next) { \ + cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ + cur_shared_heap_end_off = \ + cur_shared_heap_start_off + cur->size - 1; \ + if ((app_addr) >= cur_shared_heap_start_off \ + && (app_addr) <= cur_shared_heap_end_off - bytes + 0) { \ + shared_heap_start_off = cur_shared_heap_start_off; \ + shared_heap_end_off = cur_shared_heap_end_off; \ + shared_heap_base_addr = cur->base_addr; \ + in_chain = true; \ + break; \ + } \ + } \ + in_chain; \ + }) #define shared_heap_addr_app_to_native(app_addr, native_addr) \ native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) @@ -1647,18 +1672,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_SHARED_HEAP != 0 WASMSharedHeap *shared_heap = module->e->shared_heap; uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; -#if WASM_ENABLE_MEMORY64 != 0 uint64 shared_heap_start_off = - shared_heap ? (is_memory64 ? shared_heap->start_off_mem64 - : shared_heap->start_off_mem32) - : 0; + shared_heap ? get_shared_heap_start_off(shared_heap) : 0; uint64 shared_heap_end_off = - shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0; -#else - uint64 shared_heap_start_off = - shared_heap ? shared_heap->start_off_mem32 : 0; - uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0; -#endif + shared_heap + ? (get_shared_heap_start_off(shared_heap) + shared_heap->size - 1) + : 0; #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_MULTI_MEMORY != 0 uint32 memidx = 0; @@ -1697,7 +1716,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - HANDLE_OP(WASM_OP_NOP) { HANDLE_OP_END(); } + HANDLE_OP(WASM_OP_NOP) + { + HANDLE_OP_END(); + } #if WASM_ENABLE_EXCE_HANDLING != 0 HANDLE_OP(WASM_OP_RETHROW) From 5c22e611cd2e4347374ce05b26351b0ae12d21c3 Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 24 Jan 2025 14:45:08 +0800 Subject: [PATCH 03/17] first draft of shared heap enhancement in interpreter and runtime: update function signature --- core/iwasm/common/wasm_memory.c | 14 ++++++++---- core/iwasm/common/wasm_memory.h | 6 +++++ core/iwasm/include/wasm_export.h | 39 +++++++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index a24b3c32d..49df9acfb 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -251,7 +251,7 @@ WASMSharedHeap * wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) { WASMSharedHeap *cur; - bool heap_handle_exist = false; + bool heap_handle_exist = head->heap_handle != NULL; if (!head || !body) { LOG_WARNING("Invalid shared heap to chain."); @@ -265,8 +265,15 @@ wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) os_mutex_unlock(&shared_heap_list_lock); return NULL; } - - for (cur = head; cur; cur = cur->chain_next) { + for (cur = shared_heap_list; cur; cur = cur->next) { + if (cur->chain_next == body) { + LOG_WARNING("To create shared heap chain, the `body` shared heap " + "can't already be in a chain"); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + } + for (cur = body; cur; cur = cur->chain_next) { if (cur->heap_handle && heap_handle_exist) { LOG_WARNING( "To create shared heap chain, only one of shared heap can " @@ -310,7 +317,6 @@ wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain) if (!entire_chain) break; } - os_mutex_unlock(&shared_heap_list_lock); return cur; } diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index bceea0ee4..88d0f3c5d 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -45,6 +45,12 @@ SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size) WASMSharedHeap * wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); +WASMSharedHeap * +wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body); + +WASMSharedHeap * +wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain); + bool wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, WASMSharedHeap *shared_heap); diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 273657246..15759f777 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -336,6 +336,7 @@ typedef enum { typedef struct SharedHeapInitArgs { uint32_t size; + void *pre_allocated_addr; } SharedHeapInitArgs; /** @@ -2258,7 +2259,37 @@ WASM_RUNTIME_API_EXTERN wasm_shared_heap_t wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); /** - * Attach a shared heap to a module instance + * This function links two shared heaps, `head` and `body`, where `head` is a + * shared heap and `body` can be shared heap chain head, into a single chain. + * The `head` heap will be the head of the chain, and the `body` heap will be + * appended to it. At most one shared heap in shared heap chain can be + * dynamically allocated, the rest have to be the pre-allocated shared heap * + * + * @param head The head of the shared heap chain. + * @param body The body of the shared heap chain to be appended. + * @return The new head of the shared heap chain. NULL if failed. + */ +WASM_RUNTIME_API_EXTERN wasm_shared_heap_t +wasm_runtime_chain_shared_heaps(wasm_shared_heap_t head, + wasm_shared_heap_t body); + +/** + * This function unchains the shared heaps from the given head. If + * `entire_chain` is true, it will unchain the entire chain of shared heaps. + * Otherwise, it will unchain only the first shared heap in the chain. + * + * @param head The head of the shared heap chain. + * @param entire_chain A boolean flag indicating whether to unchain the entire + * chain. + * @return The new head of the shared heap chain. Or the last shared heap in the + * chain if `entire_chain` is true. + */ +wasm_shared_heap_t +wasm_runtime_unchain_shared_heaps(wasm_shared_heap_t head, bool entire_chain); + +/** + * Attach a shared heap, it can be the head of shared heap chain, in that case, + * attach the shared heap chain, to a module instance * * @param module_inst the module instance * @param shared_heap the shared heap @@ -2277,7 +2308,8 @@ WASM_RUNTIME_API_EXTERN void wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst); /** - * Allocate memory from a shared heap + * Allocate memory from a shared heap, or the non-preallocated shared heap from + * the shared heap chain * * @param module_inst the module instance * @param size required memory size @@ -2294,7 +2326,8 @@ wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64_t size, void **p_native_addr); /** - * Free the memory allocated from shared heap + * Free the memory allocated from shared heap, or the non-preallocated shared + * heap from the shared heap chain * * @param module_inst the module instance * @param ptr the offset in wasm app From 1566ff1606e1d35d897b866ac25560c6acddd50f Mon Sep 17 00:00:00 2001 From: TL Date: Sun, 26 Jan 2025 11:11:40 +0800 Subject: [PATCH 04/17] first draft of shared heap enhancement in interpreter and runtime: update interpreter --- core/iwasm/common/wasm_memory.c | 2 +- core/iwasm/interpreter/wasm_interp_classic.c | 14 +++--- core/iwasm/interpreter/wasm_interp_fast.c | 45 ++++++++++++++------ core/iwasm/interpreter/wasm_runtime.h | 9 ++++ 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 49df9acfb..d3d7e2a51 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -504,7 +504,7 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, for (cur = heap; cur; cur = cur->chain_next) { shared_heap_start = - cur->start_off_mem64 ? is_memory64 : cur->start_off_mem32; + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; shared_heap_end = shared_heap_start + cur->size; if (app_offset >= shared_heap_start && app_offset <= shared_heap_end) { if (target_heap) diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index e45a4a18f..d75fc55e7 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -53,15 +53,18 @@ typedef float64 CellType_F64; #else #define is_default_memory true #endif -#if WASM_ENABLE_MEMORY64 +#if WASM_ENABLE_MEMORY64 != 0 #define get_shared_heap_start_off(shared_heap) \ (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) #else #define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) #endif +/* Check whether the app addr in the last visited shared heap, if not, check the + * shared heap chain to find which(if any) shared heap the app addr in, and + * update the last visited shared heap info if found. */ #define app_addr_in_shared_heap(app_addr, bytes) \ (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 0) \ + && (app_addr) <= shared_heap_end_off - bytes + 1) \ || ({ \ bool in_chain = false; \ WASMSharedHeap *cur; \ @@ -71,7 +74,7 @@ typedef float64 CellType_F64; cur_shared_heap_end_off = \ cur_shared_heap_start_off + cur->size - 1; \ if ((app_addr) >= cur_shared_heap_start_off \ - && (app_addr) <= cur_shared_heap_end_off - bytes + 0) { \ + && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ shared_heap_start_off = cur_shared_heap_start_off; \ shared_heap_end_off = cur_shared_heap_end_off; \ shared_heap_base_addr = cur->base_addr; \ @@ -1716,10 +1719,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - HANDLE_OP(WASM_OP_NOP) - { - HANDLE_OP_END(); - } + HANDLE_OP(WASM_OP_NOP) { HANDLE_OP_END(); } #if WASM_ENABLE_EXCE_HANDLING != 0 HANDLE_OP(WASM_OP_RETHROW) diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 359a6979c..3a9715a66 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -38,9 +38,33 @@ typedef float64 CellType_F64; #endif #if WASM_ENABLE_SHARED_HEAP != 0 -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) +/* TODO: add code for 64 bit version when memory64 is enabled for fast-interp */ +#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) +/* Check whether the app addr in the last visited shared heap, if not, check the + * shared heap chain to find which(if any) shared heap the app addr in, and + * update the last visited shared heap info if found. */ +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 1) \ + || ({ \ + bool in_chain = false; \ + WASMSharedHeap *cur; \ + uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ + for (cur = shared_heap; cur; cur = cur->chain_next) { \ + cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ + cur_shared_heap_end_off = \ + cur_shared_heap_start_off + cur->size - 1; \ + if ((app_addr) >= cur_shared_heap_start_off \ + && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ + shared_heap_start_off = cur_shared_heap_start_off; \ + shared_heap_end_off = cur_shared_heap_end_off; \ + shared_heap_base_addr = cur->base_addr; \ + in_chain = true; \ + break; \ + } \ + } \ + in_chain; \ + }) #define shared_heap_addr_app_to_native(app_addr, native_addr) \ native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) @@ -1540,19 +1564,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_SHARED_HEAP != 0 WASMSharedHeap *shared_heap = module->e ? module->e->shared_heap : NULL; uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; - /* -#if WASM_ENABLE_MEMORY64 != 0 uint64 shared_heap_start_off = - shared_heap ? (is_memory64 ? shared_heap->start_off_mem64 - : shared_heap->start_off_mem32) - : 0; + shared_heap ? get_shared_heap_start_off(shared_heap) : 0; uint64 shared_heap_end_off = - shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0; -#else - */ /* TODO: uncomment the code when memory64 is enabled for fast-interp */ - uint64 shared_heap_start_off = - shared_heap ? shared_heap->start_off_mem32 : 0; - uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0; + shared_heap + ? (get_shared_heap_start_off(shared_heap) + shared_heap->size - 1) + : 0; /* #endif */ #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 00e9ad107..b9d49ec22 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -93,12 +93,21 @@ typedef union { } MemBound; typedef struct WASMSharedHeap { + /* The global shared heap list maintained in runtime, used for runtime + * destroy */ struct WASMSharedHeap *next; + /* The logical shared heap chain the shared heap in */ + struct WASMSharedHeap *chain_next; + /* Will be null if shared heap is created from pre allocated memory chunk + * and don't need to dynamic malloc and free */ void *heap_handle; uint8 *base_addr; uint64 size; uint64 start_off_mem64; uint64 start_off_mem32; + /* The number of wasm apps it attached to, for a shared heap chain, only the + * list head need to maintain the valid attached_count */ + uint8 attached_count; } WASMSharedHeap; struct WASMMemoryInstance { From aebb391cf5ddc622b32577f656befc090df222ff Mon Sep 17 00:00:00 2001 From: TL Date: Sun, 26 Jan 2025 11:12:49 +0800 Subject: [PATCH 05/17] unify shared heap sample --- .../shared-heap-preallocated/CMakeLists.txt | 127 ------- samples/shared-heap-preallocated/src/main.c | 335 ------------------ .../wasm-apps/CMakeLists.txt | 43 --- .../wasm-apps/test1.c | 71 ---- .../wasm-apps/test2.c | 18 - 5 files changed, 594 deletions(-) delete mode 100644 samples/shared-heap-preallocated/CMakeLists.txt delete mode 100644 samples/shared-heap-preallocated/src/main.c delete mode 100644 samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt delete mode 100644 samples/shared-heap-preallocated/wasm-apps/test1.c delete mode 100644 samples/shared-heap-preallocated/wasm-apps/test2.c diff --git a/samples/shared-heap-preallocated/CMakeLists.txt b/samples/shared-heap-preallocated/CMakeLists.txt deleted file mode 100644 index 6346d077e..000000000 --- a/samples/shared-heap-preallocated/CMakeLists.txt +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -cmake_minimum_required (VERSION 3.14) - -include(CheckPIESupported) - -if (NOT WAMR_BUILD_PLATFORM STREQUAL "windows") - project (shared_heap_test) -else() - project (shared_heap_test C ASM) -endif() - -################ runtime settings ################ -string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) -if (APPLE) - add_definitions(-DBH_PLATFORM_DARWIN) -endif () - -# Reset default linker flags -set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") -set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") - -# WAMR features switch - -# Set WAMR_BUILD_TARGET, currently values supported: -# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", -# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" - -if (NOT DEFINED WAMR_BUILD_TARGET) - if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") - set (WAMR_BUILD_TARGET "AARCH64") - elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") - set (WAMR_BUILD_TARGET "RISCV64") - elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) - # Build as X86_64 by default in 64-bit platform - set (WAMR_BUILD_TARGET "X86_64") - elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) - # Build as X86_32 by default in 32-bit platform - set (WAMR_BUILD_TARGET "X86_32") - else () - message(SEND_ERROR "Unsupported build target platform!") - endif () -endif () - -if (NOT CMAKE_BUILD_TYPE) - set (CMAKE_BUILD_TYPE Debug) -endif () - -set (WAMR_BUILD_INTERP 1) -set (WAMR_BUILD_FAST_INTERP 1) -set (WAMR_BUILD_AOT 1) -set (WAMR_BUILD_JIT 0) -set (WAMR_BUILD_LIBC_BUILTIN 1) -set (WAMR_BUILD_LIBC_WASI 0) -set (WAMR_BUILD_SHARED_HEAP 1) -set (WAMR_BUILD_GC_HEAP_VERIFY 1) - -if (NOT MSVC) - # linker flags - if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") - endif () - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") - if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") - if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") - endif () - endif () -endif () - -# build out vmlib -set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) -include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) - -add_library(vmlib STATIC ${WAMR_RUNTIME_LIB_SOURCE}) -if (MSVC) - target_compile_definitions(vmlib PRIVATE WASM_API_EXTERN=) -endif() -target_link_libraries(vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthread) - -################ application related ################ -include_directories(${CMAKE_CURRENT_LIST_DIR}/src) -include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) - -add_executable (shared_heap_test src/main.c ${UNCOMMON_SHARED_SOURCE}) - -check_pie_supported() -set_target_properties (shared_heap_test PROPERTIES POSITION_INDEPENDENT_CODE ON) - -if (APPLE) - target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread) -else () - target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread -lrt) -endif () - -add_subdirectory(wasm-apps) - -if (WAMR_BUILD_AOT EQUAL 1) - set (WAMR_COMPILER_DIR ${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build) - message (CHECK_START "Detecting WAMR_COMPILER at ${WAMR_COMPILER_DIR}") - find_file (WAMR_COMPILER - wamrc - PATHS "${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build" - NO_DEFAULT_PATH - NO_CMAKE_FIND_ROOT_PATH - ) - if (WAMR_COMPILER) - message (CHECK_PASS "found") - else() - message (CHECK_FAIL "not found") - endif() - if (NOT EXISTS ${WAMR_COMPILER}) - message (FATAL_ERROR "Please build wamrc under ${WAMR_ROOT_DIR}/wamr-compiler") - else() - message (STATUS "WAMR_COMPILER is ${WAMR_COMPILER}") - endif() - - add_custom_target( - wasm_to_aot - ALL - DEPENDS wasm-apps/test1.wasm wasm-apps/test2.wasm ${WAMR_COMPILER} - COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test1.aot wasm-apps/test1.wasm - COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test2.aot wasm-apps/test2.wasm - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - ) -endif() diff --git a/samples/shared-heap-preallocated/src/main.c b/samples/shared-heap-preallocated/src/main.c deleted file mode 100644 index 9842f33c3..000000000 --- a/samples/shared-heap-preallocated/src/main.c +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright (C) 2019 Intel Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - */ - -#include "wasm_export.h" -#include "bh_platform.h" -#include "bh_read_file.h" - -typedef struct thread_arg { - bh_queue *queue; - wasm_module_inst_t module_inst; -} thread_arg; - -static void * -thread1_callback(void *arg) -{ - thread_arg *targ = arg; - wasm_module_inst_t module_inst = targ->module_inst; - bh_queue *queue = targ->queue; - wasm_exec_env_t exec_env; - wasm_function_inst_t my_shared_heap_malloc_func; - wasm_function_inst_t my_shared_heap_free_func; - uint32 i, argv[2]; - - /* lookup wasm functions */ - if (!(my_shared_heap_malloc_func = wasm_runtime_lookup_function( - module_inst, "my_shared_heap_malloc")) - || !(my_shared_heap_free_func = wasm_runtime_lookup_function( - module_inst, "my_shared_heap_free"))) { - printf("Failed to lookup function.\n"); - } - - /* create exec env */ - if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { - printf("Failed to create exec env.\n"); - return NULL; - } - - /* allocate memory with wasm_runtime_shared_heap_malloc and send it - to wasm app2 */ - for (i = 0; i < 1; i++) { - uint8 *buf; - uint64 offset; - - offset = wasm_runtime_shared_heap_malloc(module_inst, 4096 * (i + 1), - (void **)&buf); - - if (offset == 0) { - printf("Failed to allocate memory from shared heap\n"); - break; - } - - snprintf(buf, 1024, "Hello, this is buf %u allocated from shared heap", - i + 1); - - printf("wasm app1 send buf: %s\n\n", buf); - if (!bh_post_msg(queue, 1, buf, 1024 * i)) { - printf("Failed to post message to queue\n"); - wasm_runtime_shared_heap_free(module_inst, offset); - break; - } - } - - /* allocate memory by calling my_shared_heap_malloc function and send it - to wasm app2 */ - for (i = 1; i < 2; i++) { - uint8 *buf; - - argv[0] = 1024 * (i + 1); - argv[1] = i + 1; - wasm_runtime_call_wasm(exec_env, my_shared_heap_malloc_func, 2, argv); - - if (wasm_runtime_get_exception(module_inst)) { - printf("Failed to call 'my_shared_heap_malloc' function: %s\n", - wasm_runtime_get_exception(module_inst)); - break; - } - if (argv[0] == 0) { - printf("Failed to allocate memory from shared heap\n"); - break; - } - - buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); - - printf("wasm app1 send buf: %s\n\n", buf); - if (!bh_post_msg(queue, 1, buf, 1024 * i)) { - printf("Failed to post message to queue\n"); - wasm_runtime_shared_heap_free(module_inst, argv[0]); - break; - } - } - - wasm_runtime_destroy_exec_env(exec_env); - - return NULL; -} - -static void -queue_callback(void *message, void *arg) -{ - bh_message_t msg = (bh_message_t)message; - wasm_exec_env_t exec_env = arg; - wasm_module_inst_t module_inst = wasm_runtime_get_module_inst(exec_env); - wasm_function_inst_t print_buf_func; - uint32 argv[2]; - - /* lookup wasm function */ - if (!(print_buf_func = - wasm_runtime_lookup_function(module_inst, "print_buf"))) { - printf("Failed to lookup function.\n"); - return; - } - - char *buf = bh_message_payload(msg); - printf("wasm app's native queue received buf: %s\n\n", buf); - - /* call wasm function */ - argv[0] = wasm_runtime_addr_native_to_app(module_inst, buf); - wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); - if (wasm_runtime_get_exception(module_inst)) { - printf("Failed to call 'print_buf' function: %s\n", - wasm_runtime_get_exception(module_inst)); - } -} - -static void * -thread2_callback(void *arg) -{ - thread_arg *targ = arg; - bh_queue *queue = targ->queue; - wasm_module_inst_t module_inst = targ->module_inst; - wasm_exec_env_t exec_env; - - /* create exec env */ - if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { - printf("Failed to create exec env.\n"); - return NULL; - } - - /* enter queue's message loop until bh_queue_exit_loop_run - is called */ - bh_queue_enter_loop_run(queue, queue_callback, exec_env); - - wasm_runtime_destroy_exec_env(exec_env); - - return NULL; -} - -static char global_heap_buf[512 * 1024]; - -int -main(int argc, char **argv) -{ - char *wasm_file1 = NULL, *wasm_file2 = NULL; - uint8 *wasm_file1_buf = NULL, *wasm_file2_buf = NULL; - uint32 wasm_file1_size, wasm_file2_size; - wasm_module_t wasm_module1 = NULL, wasm_module2 = NULL; - wasm_module_inst_t module_inst1 = NULL; - wasm_module_inst_t module_inst2 = NULL; - wasm_shared_heap_t shared_heap = NULL; - bh_queue *queue = NULL; - RuntimeInitArgs init_args; - SharedHeapInitArgs heap_init_args; - char error_buf[128] = { 0 }; - bool aot_mode = false; - int ret = -1; - -#if WASM_DISABLE_HW_BOUND_CHECK == 0 -#if defined(BUILD_TARGET_X86_64) - printf("HW BOUND CHECK\n"); -#endif -#endif - - if (argc > 1 && !strcmp(argv[1], "--aot")) - aot_mode = true; - - if (!aot_mode) - printf("Test shared heap in interpreter mode\n\n"); - else - printf("Test shared heap in AOT mode\n\n"); - - bh_log_set_verbose_level(5); - memset(&init_args, 0, sizeof(RuntimeInitArgs)); - - init_args.mem_alloc_type = Alloc_With_Pool; - init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; - init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); - - /* init wasm runtime */ - if (!wasm_runtime_full_init(&init_args)) { - printf("Init runtime environment failed.\n"); - return -1; - } - - /* create queue */ - if (!(queue = bh_queue_create())) { - printf("Create queue failed.\n"); - goto fail; - } - - /* read wasm file */ - if (!aot_mode) - wasm_file1 = "./wasm-apps/test1.wasm"; - else - wasm_file1 = "./wasm-apps/test1.aot"; - if (!(wasm_file1_buf = - bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { - printf("Open wasm file %s failed.\n", wasm_file1); - goto fail; - } - - /* load wasm file */ - wasm_module1 = wasm_runtime_load((uint8 *)wasm_file1_buf, wasm_file1_size, - error_buf, sizeof(error_buf)); - if (!wasm_module1) { - printf("Load wasm module failed. error: %s\n", error_buf); - goto fail; - } - - /* instantiate module */ - module_inst1 = wasm_runtime_instantiate(wasm_module1, 65536, 4096, - error_buf, sizeof(error_buf)); - if (!module_inst1) { - printf("Instantiate wasm module failed. error: %s\n", error_buf); - goto fail; - } - - /* read wasm file */ - if (!aot_mode) - wasm_file2 = "./wasm-apps/test2.wasm"; - else - wasm_file2 = "./wasm-apps/test2.aot"; - if (!(wasm_file2_buf = - bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { - printf("Open wasm file %s failed.\n", wasm_file1); - goto fail; - } - - /* load wasm file */ - wasm_module2 = wasm_runtime_load((uint8 *)wasm_file2_buf, wasm_file2_size, - error_buf, sizeof(error_buf)); - if (!wasm_module2) { - printf("Load wasm module failed. error: %s\n", error_buf); - goto fail; - } - - /* instantiate module */ - module_inst2 = wasm_runtime_instantiate(wasm_module2, 65536, 4096, - error_buf, sizeof(error_buf)); - if (!module_inst2) { - printf("Instantiate wasm module failed. error: %s\n", error_buf); - goto fail; - } - - /* create shared heap */ - memset(&heap_init_args, 0, sizeof(heap_init_args)); - heap_init_args.size = 4096 + 1; - shared_heap = wasm_runtime_create_shared_heap(&heap_init_args); - if (!shared_heap) { - printf("Create shared heap failed. error: %s\n", error_buf); - goto fail; - } - - /* attach module instance 1 to the shared heap */ - if (!wasm_runtime_attach_shared_heap(module_inst1, shared_heap)) { - printf("Attach shared heap failed.\n"); - goto fail; - } - - /* attach module instance 2 to the shared heap */ - if (!wasm_runtime_attach_shared_heap(module_inst2, shared_heap)) { - printf("Attach shared heap failed.\n"); - goto fail; - } - - /* create thread 1 */ - struct thread_arg targ1 = { 0 }; - korp_tid tid1; - targ1.queue = queue; - targ1.module_inst = module_inst1; - if (os_thread_create(&tid1, thread1_callback, &targ1, - APP_THREAD_STACK_SIZE_DEFAULT)) { - printf("Failed to create thread 1\n"); - goto fail; - } - - /* create thread 2 */ - struct thread_arg targ2 = { 0 }; - korp_tid tid2; - targ2.queue = queue; - targ2.module_inst = module_inst2; - if (os_thread_create(&tid2, thread2_callback, &targ2, - APP_THREAD_STACK_SIZE_DEFAULT)) { - printf("Failed to create thread 2\n"); - os_thread_join(tid1, NULL); - goto fail; - } - - /* wait until all messages are post to wasm app2 and wasm app2 - handles all of them, then exit the queue message loop */ - usleep(10000); - bh_queue_exit_loop_run(queue); - - os_thread_join(tid1, NULL); - os_thread_join(tid2, NULL); - - ret = 0; - -fail: - if (module_inst2) - wasm_runtime_deinstantiate(module_inst2); - - if (module_inst1) - wasm_runtime_deinstantiate(module_inst1); - - if (wasm_module2) - wasm_runtime_unload(wasm_module2); - - if (wasm_module1) - wasm_runtime_unload(wasm_module1); - - if (wasm_file2_buf) - wasm_runtime_free(wasm_file2_buf); - - if (wasm_file1_buf) - wasm_runtime_free(wasm_file1_buf); - - if (queue) - bh_queue_destroy(queue); - - wasm_runtime_destroy(); - - return ret; -} diff --git a/samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt b/samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt deleted file mode 100644 index a90c82e1a..000000000 --- a/samples/shared-heap-preallocated/wasm-apps/CMakeLists.txt +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -cmake_minimum_required(VERSION 3.14) -project(wasm-apps) - -set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) - -if (APPLE) - set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) - set (CMAKE_C_LINK_FLAGS "") - set (CMAKE_CXX_LINK_FLAGS "") -endif () - -set (CMAKE_SYSTEM_PROCESSOR wasm32) -set (CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) - -if (NOT DEFINED WASI_SDK_DIR) - set (WASI_SDK_DIR "/opt/wasi-sdk") -endif () - -set (CMAKE_C_COMPILER_TARGET "wasm32") -set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") -set (CMAKE_C_FLAGS "-nostdlib -Qunused-arguments -z stack-size=32768") - -set (DEFINED_SYMBOLS "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") - -set (CMAKE_EXE_LINKER_FLAGS - "-O0 -Wl,--initial-memory=65536, \ - -Wl,--no-entry,--strip-all, \ - -Wl,--export=__heap_base,--export=__data_end \ - -Wl,--export=__wasm_call_ctors \ - -Wl,--export=my_shared_heap_malloc \ - -Wl,--export=my_shared_heap_free \ - -Wl,--export=print_buf \ - -Wl,--allow-undefined" -) - -add_executable(test1.wasm test1.c) -target_link_libraries(test1.wasm) - -add_executable(test2.wasm test2.c) -target_link_libraries(test2.wasm) diff --git a/samples/shared-heap-preallocated/wasm-apps/test1.c b/samples/shared-heap-preallocated/wasm-apps/test1.c deleted file mode 100644 index 2e3fd2112..000000000 --- a/samples/shared-heap-preallocated/wasm-apps/test1.c +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2019 Intel Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - */ - -#include -#include -#include -#include - -extern void * -shared_heap_malloc(uint32_t size); -extern void -shared_heap_free(void *ptr); - -void * -my_shared_heap_malloc(uint32_t size, uint32_t index) -{ - char *buf1 = NULL, *buf2 = NULL, *buf; - - char *buf3 = NULL; - buf3 = malloc(2048); - - buf1 = shared_heap_malloc(1024); - if (!buf1) - return NULL; - - buf1[0] = 'H'; - buf1[1] = 'e'; - buf1[2] = 'l'; - buf1[3] = 'l'; - buf1[4] = 'o'; - buf1[5] = ','; - buf1[6] = ' '; - - buf2 = shared_heap_malloc(1024); - if (!buf2) { - shared_heap_free(buf1); - return NULL; - } - - snprintf(buf2, 1024, "this is buf %u allocated from shared heap", index); - - buf = shared_heap_malloc(size); - if (!buf) { - shared_heap_free(buf1); - shared_heap_free(buf2); - return NULL; - } - - memset(buf, 0, size); - memcpy(buf, buf1, strlen(buf1)); - memcpy(buf + strlen(buf1), buf2, strlen(buf2)); - - uint32_t max_value = UINT32_MAX; - char *address = (char *)max_value; - printf("The address of buf1, buf2, buf, UINT32_MAX is: %p,%p,%p,%p\n", buf1, - buf2, buf, address); - address[0] = 'c'; - printf("The content address of UINT32_MAX is: %c\n", *address); - - shared_heap_free(buf1); - shared_heap_free(buf2); - return buf; -} - -void -my_shared_heap_free(void *ptr) -{ - shared_heap_free(ptr); -} diff --git a/samples/shared-heap-preallocated/wasm-apps/test2.c b/samples/shared-heap-preallocated/wasm-apps/test2.c deleted file mode 100644 index b63efcd1a..000000000 --- a/samples/shared-heap-preallocated/wasm-apps/test2.c +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (C) 2019 Intel Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - */ - -#include - -#include - -extern void -shared_heap_free(void *ptr); - -void -print_buf(char *buf) -{ - printf("wasm app2's wasm func received buf: %s\n\n", buf); - shared_heap_free(buf); -} From add69585b9a267de20bf9fd2cdf1b06133ab23f9 Mon Sep 17 00:00:00 2001 From: TL Date: Mon, 27 Jan 2025 14:12:22 +0800 Subject: [PATCH 06/17] fix shared heap enhancement bugs and format code --- core/iwasm/common/wasm_memory.c | 33 ++++++------- core/iwasm/interpreter/wasm_interp_classic.c | 14 ++++-- core/iwasm/interpreter/wasm_interp_fast.c | 6 +-- samples/shared-heap/src/main.c | 8 ++-- tests/unit/shared-heap/shared_heap_test.cc | 50 ++++++++++---------- 5 files changed, 60 insertions(+), 51 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index d3d7e2a51..ec57e9e67 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -183,6 +183,11 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) heap->start_off_mem32 = UINT32_MAX - heap->size + 1; heap->attached_count = 0; + if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { + LOG_WARNING("Invalid size of shared heap"); + goto fail1; + } + if (init_args->pre_allocated_addr != NULL) { /* Create shared heap from a pre allocated buffer, its size need to * align with system page */ @@ -193,6 +198,7 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) } heap->heap_handle = NULL; + heap->base_addr = init_args->pre_allocated_addr; } else { if (!(heap->heap_handle = @@ -200,11 +206,6 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) goto fail2; } - if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { - LOG_WARNING("Invalid size of shared heap"); - goto fail3; - } - #ifndef OS_ENABLE_HW_BOUND_CHECK map_size = size; #else @@ -286,8 +287,8 @@ wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) heap_handle_exist = true; } - head->start_off_mem64 = body->start_off_mem64 - head->size + 1; - head->start_off_mem32 = body->start_off_mem32 - head->size + 1; + head->start_off_mem64 = body->start_off_mem64 - head->size; + head->start_off_mem32 = body->start_off_mem32 - head->size; head->chain_next = body; os_mutex_unlock(&shared_heap_list_lock); return head; @@ -505,8 +506,9 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, for (cur = heap; cur; cur = cur->chain_next) { shared_heap_start = is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; - shared_heap_end = shared_heap_start + cur->size; - if (app_offset >= shared_heap_start && app_offset <= shared_heap_end) { + shared_heap_end = shared_heap_start - 1 + cur->size; + if (app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { if (target_heap) *target_heap = cur; return true; @@ -535,19 +537,16 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, for (cur = heap_head; cur != NULL; cur = cur->chain_next) { base_addr = (uintptr_t)cur->base_addr; addr_int = (uintptr_t)addr; - if (addr_int < base_addr) { + if (addr_int < base_addr) continue; - } end_addr = addr_int + bytes; /* Check for overflow */ - if (end_addr <= addr_int) { + if (end_addr <= addr_int) continue; - } - if (end_addr > base_addr + cur->size) { + if (end_addr > base_addr + cur->size) continue; - } if (target_heap) *target_heap = cur; @@ -1041,9 +1040,11 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, { WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; WASMMemoryInstance *memory_inst; - WASMSharedHeap *shared_heap; uint8 *addr; bool bounds_checks; +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; +#endif bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode || module_inst_comm->module_type == Wasm_Module_AoT); diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index d75fc55e7..76803cca7 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -72,7 +72,7 @@ typedef float64 CellType_F64; for (cur = shared_heap; cur; cur = cur->chain_next) { \ cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ cur_shared_heap_end_off = \ - cur_shared_heap_start_off + cur->size - 1; \ + cur_shared_heap_start_off - 1 + cur->size; \ if ((app_addr) >= cur_shared_heap_start_off \ && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ shared_heap_start_off = cur_shared_heap_start_off; \ @@ -1679,7 +1679,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, shared_heap ? get_shared_heap_start_off(shared_heap) : 0; uint64 shared_heap_end_off = shared_heap - ? (get_shared_heap_start_off(shared_heap) + shared_heap->size - 1) + ? (get_shared_heap_start_off(shared_heap) - 1 + shared_heap->size) : 0; #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_MULTI_MEMORY != 0 @@ -1719,7 +1719,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - HANDLE_OP(WASM_OP_NOP) { HANDLE_OP_END(); } + HANDLE_OP(WASM_OP_NOP) + { + HANDLE_OP_END(); + } #if WASM_ENABLE_EXCE_HANDLING != 0 HANDLE_OP(WASM_OP_RETHROW) @@ -5656,7 +5659,10 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_REINTERPRET_F32) HANDLE_OP(WASM_OP_I64_REINTERPRET_F64) HANDLE_OP(WASM_OP_F32_REINTERPRET_I32) - HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) { HANDLE_OP_END(); } + HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) + { + HANDLE_OP_END(); + } HANDLE_OP(WASM_OP_I32_EXTEND8_S) { diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 3a9715a66..6ab5d236e 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -53,7 +53,7 @@ typedef float64 CellType_F64; for (cur = shared_heap; cur; cur = cur->chain_next) { \ cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ cur_shared_heap_end_off = \ - cur_shared_heap_start_off + cur->size - 1; \ + cur_shared_heap_start_off - 1 + cur->size; \ if ((app_addr) >= cur_shared_heap_start_off \ && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ shared_heap_start_off = cur_shared_heap_start_off; \ @@ -67,7 +67,7 @@ typedef float64 CellType_F64; }) #define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) + native_addr = shared_heap_base_addr + ((app_addr) - shared_heap_start_off) #define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ if (app_addr_in_shared_heap(app_addr, bytes)) \ @@ -1568,7 +1568,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, shared_heap ? get_shared_heap_start_off(shared_heap) : 0; uint64 shared_heap_end_off = shared_heap - ? (get_shared_heap_start_off(shared_heap) + shared_heap->size - 1) + ? (get_shared_heap_start_off(shared_heap) - 1 + shared_heap->size) : 0; /* #endif */ #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ diff --git a/samples/shared-heap/src/main.c b/samples/shared-heap/src/main.c index f4024f08c..ed64cc5da 100644 --- a/samples/shared-heap/src/main.c +++ b/samples/shared-heap/src/main.c @@ -55,7 +55,7 @@ thread1_callback(void *arg) i + 1); printf("wasm app1 send buf: %s\n\n", buf); - if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + if (!bh_post_msg(queue, 1, buf, 1024 * (i + 1))) { printf("Failed to post message to queue\n"); wasm_runtime_shared_heap_free(module_inst, offset); break; @@ -84,7 +84,7 @@ thread1_callback(void *arg) buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); printf("wasm app1 send buf: %s\n\n", buf); - if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + if (!bh_post_msg(queue, 1, buf, 1024 * (i+1))) { printf("Failed to post message to queue\n"); wasm_runtime_shared_heap_free(module_inst, argv[0]); break; @@ -268,7 +268,7 @@ main(int argc, char **argv) } /* create thread 1 */ - struct thread_arg targ1 = { 0 }; + thread_arg targ1 = { 0 }; korp_tid tid1; targ1.queue = queue; targ1.module_inst = module_inst1; @@ -279,7 +279,7 @@ main(int argc, char **argv) } /* create thread 2 */ - struct thread_arg targ2 = { 0 }; + thread_arg targ2 = { 0 }; korp_tid tid2; targ2.queue = queue; targ2.module_inst = module_inst2; diff --git a/tests/unit/shared-heap/shared_heap_test.cc b/tests/unit/shared-heap/shared_heap_test.cc index deb4bbb38..76357711d 100644 --- a/tests/unit/shared-heap/shared_heap_test.cc +++ b/tests/unit/shared-heap/shared_heap_test.cc @@ -92,7 +92,9 @@ destroy_module_env(struct ret_env module_env) } } -static void test_shared_heap(WASMSharedHeap *shared_heap, const char *file, const char *func_name, uint32 argc, uint32 argv[]) +static void +test_shared_heap(WASMSharedHeap *shared_heap, const char *file, + const char *func_name, uint32 argc, uint32 argv[]) { struct ret_env tmp_module_env; WASMFunctionInstanceCommon *func_test = nullptr; @@ -101,7 +103,8 @@ static void test_shared_heap(WASMSharedHeap *shared_heap, const char *file, cons tmp_module_env = load_wasm((char *)file, 0); - if (!wasm_runtime_attach_shared_heap(tmp_module_env.wasm_module_inst, shared_heap)) { + if (!wasm_runtime_attach_shared_heap(tmp_module_env.wasm_module_inst, + shared_heap)) { printf("Failed to attach shared heap\n"); goto test_failed; } @@ -116,7 +119,8 @@ static void test_shared_heap(WASMSharedHeap *shared_heap, const char *file, cons wasm_runtime_call_wasm(tmp_module_env.exec_env, func_test, argc, argv); if (!ret) { printf("\nFailed to wasm_runtime_call_wasm!\n"); - const char *s = wasm_runtime_get_exception(tmp_module_env.wasm_module_inst); + const char *s = + wasm_runtime_get_exception(tmp_module_env.wasm_module_inst); printf("exception: %s\n", s); goto test_failed; } @@ -131,7 +135,7 @@ test_failed: TEST_F(shared_heap_test, test_shared_heap_basic) { - SharedHeapInitArgs args; + SharedHeapInitArgs args = { 0 }; WASMSharedHeap *shared_heap = nullptr; uint32 argv[1] = { 0 }; @@ -150,12 +154,11 @@ TEST_F(shared_heap_test, test_shared_heap_basic) // test aot test_shared_heap(shared_heap, "test.aot", "test", 1, argv); EXPECT_EQ(10, argv[0]); - } TEST_F(shared_heap_test, test_shared_heap_malloc_fail) { - SharedHeapInitArgs args; + SharedHeapInitArgs args = { 0 }; WASMSharedHeap *shared_heap = nullptr; uint32 argv[1] = { 0 }; @@ -177,36 +180,36 @@ TEST_F(shared_heap_test, test_shared_heap_malloc_fail) } #ifndef native_function +/* clang-format off */ #define native_function(func_name, signature) \ { #func_name, (void *)glue_##func_name, signature, NULL } - +/* clang-format on */ #endif #ifndef nitems #define nitems(_a) (sizeof(_a) / sizeof(0 [(_a)])) #endif /* nitems */ -uintptr_t glue_test_addr_conv(wasm_exec_env_t env, uintptr_t addr) +uintptr_t +glue_test_addr_conv(wasm_exec_env_t env, uintptr_t addr) { - wasm_module_inst_t module_inst = get_module_inst(env); - uintptr_t ret; - void *native_addr = (void *)addr; - uintptr_t app_addr = addr_native_to_app(native_addr); + wasm_module_inst_t module_inst = get_module_inst(env); + uintptr_t ret; + void *native_addr = (void *)addr; + uintptr_t app_addr = addr_native_to_app(native_addr); - native_addr = addr_app_to_native(app_addr); - if (native_addr != (void *)addr) - { - EXPECT_EQ(1, 0); - } - return app_addr; + native_addr = addr_app_to_native(app_addr); + if (native_addr != (void *)addr) { + EXPECT_EQ(1, 0); + } + return app_addr; } -static NativeSymbol g_test_native_symbols[] = -{ - native_function(test_addr_conv,"(*)i"), +static NativeSymbol g_test_native_symbols[] = { + native_function(test_addr_conv, "(*)i"), }; TEST_F(shared_heap_test, test_addr_conv) { - SharedHeapInitArgs args; + SharedHeapInitArgs args = { 0 }; WASMSharedHeap *shared_heap = nullptr; uint32 argv[1] = { 0 }; struct ret_env tmp_module_env; @@ -217,8 +220,7 @@ TEST_F(shared_heap_test, test_addr_conv) ret = wasm_native_register_natives("env", g_test_native_symbols, nitems(g_test_native_symbols)); - if (!ret) - { + if (!ret) { EXPECT_EQ(1, 0); return; } From e91181331799316252bf65c9a3978ee3278b2de0 Mon Sep 17 00:00:00 2001 From: TL Date: Wed, 5 Feb 2025 15:06:25 +0800 Subject: [PATCH 07/17] format and update shared heap boundary check in runtime API --- core/iwasm/common/wasm_memory.c | 34 ++++++++++++++------ core/iwasm/interpreter/wasm_interp_classic.c | 10 ++---- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index ec57e9e67..7c09ff5e3 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -496,25 +496,38 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, uint64 shared_heap_start, shared_heap_end; if (!heap) { - return false; + goto fail; } if (bytes == 0) { bytes = 1; } - for (cur = heap; cur; cur = cur->chain_next) { - shared_heap_start = - is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; - shared_heap_end = shared_heap_start - 1 + cur->size; - if (app_offset >= shared_heap_start - && app_offset <= shared_heap_end - bytes + 1) { - if (target_heap) + /* Early stop for app start address not in the shared heap(chain) at all */ + shared_heap_start = + is_memory64 ? heap->start_off_mem64 : heap->start_off_mem32; + shared_heap_end = is_memory64 ? UINT64_MAX : UINT32_MAX; + if (app_offset < shared_heap_start + || app_offset > shared_heap_end - bytes + 1) { + goto fail; + } + + /* Find the exact shared heap that app addr is in */ + if (target_heap) { + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start - 1 + cur->size; + if (app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { *target_heap = cur; - return true; + return true; + } } } + return true; +fail: if (target_heap) *target_heap = NULL; return false; @@ -529,7 +542,7 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, uintptr_t base_addr, addr_int, end_addr; if (!heap_head) { - return false; + goto fail; } /* Iterate through shared heap chain to find whether native addr in one of @@ -553,6 +566,7 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, return true; } +fail: if (target_heap) *target_heap = NULL; return false; diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 76803cca7..349e360f6 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -1719,10 +1719,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } - HANDLE_OP(WASM_OP_NOP) - { - HANDLE_OP_END(); - } + HANDLE_OP(WASM_OP_NOP) { HANDLE_OP_END(); } #if WASM_ENABLE_EXCE_HANDLING != 0 HANDLE_OP(WASM_OP_RETHROW) @@ -5659,10 +5656,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_REINTERPRET_F32) HANDLE_OP(WASM_OP_I64_REINTERPRET_F64) HANDLE_OP(WASM_OP_F32_REINTERPRET_I32) - HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) - { - HANDLE_OP_END(); - } + HANDLE_OP(WASM_OP_F64_REINTERPRET_I64) { HANDLE_OP_END(); } HANDLE_OP(WASM_OP_I32_EXTEND8_S) { From 0b0ef2d11742c9989162b7ef3890c834f3d6af2b Mon Sep 17 00:00:00 2001 From: TL Date: Wed, 5 Feb 2025 15:49:01 +0800 Subject: [PATCH 08/17] format --- core/iwasm/interpreter/wasm_interp_fast.c | 2 +- samples/shared-heap/src/main.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 6ab5d236e..6bdb03456 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -67,7 +67,7 @@ typedef float64 CellType_F64; }) #define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr) - shared_heap_start_off) + native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) #define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ if (app_addr_in_shared_heap(app_addr, bytes)) \ diff --git a/samples/shared-heap/src/main.c b/samples/shared-heap/src/main.c index ed64cc5da..1d93a6a2f 100644 --- a/samples/shared-heap/src/main.c +++ b/samples/shared-heap/src/main.c @@ -84,7 +84,7 @@ thread1_callback(void *arg) buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); printf("wasm app1 send buf: %s\n\n", buf); - if (!bh_post_msg(queue, 1, buf, 1024 * (i+1))) { + if (!bh_post_msg(queue, 1, buf, 1024 * (i + 1))) { printf("Failed to post message to queue\n"); wasm_runtime_shared_heap_free(module_inst, argv[0]); break; From 91f1234dd0c7885b0de16f39e09f54391c57c5ea Mon Sep 17 00:00:00 2001 From: TL Date: Sat, 8 Feb 2025 12:17:01 +0800 Subject: [PATCH 09/17] some refactor --- core/iwasm/common/wasm_memory.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 7c09ff5e3..32e211412 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -267,9 +267,10 @@ wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) return NULL; } for (cur = shared_heap_list; cur; cur = cur->next) { - if (cur->chain_next == body) { - LOG_WARNING("To create shared heap chain, the `body` shared heap " - "can't already be in a chain"); + if (cur->chain_next == body || cur->chain_next == head) { + LOG_WARNING( + "To create shared heap chain, both the 'head' and 'body' " + "shared heap can't already be the 'body' in another a chain"); os_mutex_unlock(&shared_heap_list_lock); return NULL; } @@ -600,12 +601,10 @@ wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, *p_native_addr = native_addr; } - if (memory->is_memory64) - return shared_heap->start_off_mem64 - + ((uint8 *)native_addr - shared_heap->base_addr); - else - return shared_heap->start_off_mem32 - + ((uint8 *)native_addr - shared_heap->base_addr); + return memory->is_memory64 + ? shared_heap->start_off_mem64 + : shared_heap->start_off_mem32 + + ((uint8 *)native_addr - shared_heap->base_addr); } void From 3296925c8a64672c310dfeeabd5b7c7248a84122 Mon Sep 17 00:00:00 2001 From: TL Date: Sat, 8 Feb 2025 12:53:58 +0800 Subject: [PATCH 10/17] some refactor --- core/iwasm/common/wasm_memory.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 32e211412..e8b8849fa 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -298,7 +298,7 @@ wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) WASMSharedHeap * wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain) { - WASMSharedHeap *cur; + WASMSharedHeap *cur, *tmp; if (!head || !head->chain_next) { LOG_WARNING("Invalid shared heap chain to disconnect the head from."); @@ -313,9 +313,13 @@ wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain) return NULL; } - for (cur = head; cur && cur->chain_next; cur = cur->chain_next) { + cur = head; + while (cur && cur->chain_next) { cur->start_off_mem64 = UINT64_MAX - cur->size + 1; cur->start_off_mem32 = UINT32_MAX - cur->size + 1; + tmp = cur; + cur = cur->chain_next; + tmp->chain_next = NULL; if (!entire_chain) break; } From a9d776eda322298eb842c3c35c79aea5ce7c9bec Mon Sep 17 00:00:00 2001 From: TL Date: Wed, 12 Feb 2025 16:20:29 +0800 Subject: [PATCH 11/17] cr suggestions and some refactor --- core/iwasm/common/wasm_memory.c | 13 ++--- core/iwasm/common/wasm_shared_memory.h | 48 +++++++++++++++++++ core/iwasm/include/wasm_export.h | 10 ++-- core/iwasm/interpreter/wasm_interp_classic.c | 50 -------------------- core/iwasm/interpreter/wasm_interp_fast.c | 46 +++--------------- 5 files changed, 66 insertions(+), 101 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index e8b8849fa..d44d1778e 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -185,7 +185,7 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { LOG_WARNING("Invalid size of shared heap"); - goto fail1; + goto fail2; } if (init_args->pre_allocated_addr != NULL) { @@ -194,7 +194,7 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) if (size != init_args->size) { LOG_WARNING("Pre allocated size need to be aligned with system " "page size to create shared heap"); - goto fail1; + goto fail2; } heap->heap_handle = NULL; @@ -252,12 +252,13 @@ WASMSharedHeap * wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) { WASMSharedHeap *cur; - bool heap_handle_exist = head->heap_handle != NULL; + bool heap_handle_exist = false; if (!head || !body) { LOG_WARNING("Invalid shared heap to chain."); return NULL; } + heap_handle_exist = head->heap_handle != NULL; os_mutex_lock(&shared_heap_list_lock); if (head->attached_count != 0 || body->attached_count != 0) { @@ -543,16 +544,16 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, uint8 *addr, uint32 bytes, WASMSharedHeap **target_heap) { - WASMSharedHeap *cur, *heap_head = get_shared_heap(module_inst); + WASMSharedHeap *cur, *heap = get_shared_heap(module_inst); uintptr_t base_addr, addr_int, end_addr; - if (!heap_head) { + if (!heap) { goto fail; } /* Iterate through shared heap chain to find whether native addr in one of * shared heap */ - for (cur = heap_head; cur != NULL; cur = cur->chain_next) { + for (cur = heap; cur != NULL; cur = cur->chain_next) { base_addr = (uintptr_t)cur->base_addr; addr_int = (uintptr_t)addr; if (addr_int < base_addr) diff --git a/core/iwasm/common/wasm_shared_memory.h b/core/iwasm/common/wasm_shared_memory.h index e1c5154a5..8b8af3efa 100644 --- a/core/iwasm/common/wasm_shared_memory.h +++ b/core/iwasm/common/wasm_shared_memory.h @@ -56,6 +56,54 @@ uint32 wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, uint32 count); +#if WASM_ENABLE_MULTI_MEMORY != 0 +/* Only enable shared heap for the default memory */ +#define is_default_memory (memidx == 0) +#else +#define is_default_memory true +#endif +#if WASM_ENABLE_MEMORY64 != 0 +#define get_shared_heap_start_off(shared_heap) \ + (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) +#else +#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) +#endif +/* Check whether the app addr in the last visited shared heap, if not, check the + * shared heap chain to find which(if any) shared heap the app addr in, and + * update the last visited shared heap info if found. */ +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 1) \ + || ({ \ + bool in_chain = false; \ + WASMSharedHeap *cur; \ + uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ + for (cur = shared_heap; cur; cur = cur->chain_next) { \ + cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ + cur_shared_heap_end_off = \ + cur_shared_heap_start_off - 1 + cur->size; \ + if ((app_addr) >= cur_shared_heap_start_off \ + && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ + shared_heap_start_off = cur_shared_heap_start_off; \ + shared_heap_end_off = cur_shared_heap_end_off; \ + shared_heap_base_addr = cur->base_addr; \ + in_chain = true; \ + break; \ + } \ + } \ + in_chain; \ + }) + +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) + +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else +#else +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) + #ifdef __cplusplus } #endif diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 15759f777..65bb0e767 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -2259,11 +2259,11 @@ WASM_RUNTIME_API_EXTERN wasm_shared_heap_t wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); /** - * This function links two shared heaps, `head` and `body`, where `head` is a - * shared heap and `body` can be shared heap chain head, into a single chain. - * The `head` heap will be the head of the chain, and the `body` heap will be - * appended to it. At most one shared heap in shared heap chain can be - * dynamically allocated, the rest have to be the pre-allocated shared heap * + * This function links two shared heap(lists), `head` and `body` in to a single + * shared heap list, where `head` becomes the new shared heap list head. The + * shared heap list remains one continuous shared heap in wasm app's point of + * view. At most one shared heap in shared heap list can be dynamically + * allocated, the rest have to be the pre-allocated shared heap. * * * @param head The head of the shared heap chain. * @param body The body of the shared heap chain to be appended. diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 349e360f6..ab1486aa9 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -46,56 +46,6 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if WASM_ENABLE_SHARED_HEAP != 0 -#if WASM_ENABLE_MULTI_MEMORY != 0 -/* Only enable shared heap for the default memory */ -#define is_default_memory (memidx == 0) -#else -#define is_default_memory true -#endif -#if WASM_ENABLE_MEMORY64 != 0 -#define get_shared_heap_start_off(shared_heap) \ - (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) -#else -#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) -#endif -/* Check whether the app addr in the last visited shared heap, if not, check the - * shared heap chain to find which(if any) shared heap the app addr in, and - * update the last visited shared heap info if found. */ -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) \ - || ({ \ - bool in_chain = false; \ - WASMSharedHeap *cur; \ - uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ - for (cur = shared_heap; cur; cur = cur->chain_next) { \ - cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ - cur_shared_heap_end_off = \ - cur_shared_heap_start_off - 1 + cur->size; \ - if ((app_addr) >= cur_shared_heap_start_off \ - && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ - shared_heap_start_off = cur_shared_heap_start_off; \ - shared_heap_end_off = cur_shared_heap_end_off; \ - shared_heap_base_addr = cur->base_addr; \ - in_chain = true; \ - break; \ - } \ - } \ - in_chain; \ - }) - -#define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) - -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ - if (app_addr_in_shared_heap(app_addr, bytes)) \ - shared_heap_addr_app_to_native(app_addr, native_addr); \ - else -#else -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) -#endif - #if WASM_ENABLE_MEMORY64 == 0 #if (!defined(OS_ENABLE_HW_BOUND_CHECK) \ diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 6bdb03456..a62b6b2de 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -37,46 +37,6 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if WASM_ENABLE_SHARED_HEAP != 0 -/* TODO: add code for 64 bit version when memory64 is enabled for fast-interp */ -#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) -/* Check whether the app addr in the last visited shared heap, if not, check the - * shared heap chain to find which(if any) shared heap the app addr in, and - * update the last visited shared heap info if found. */ -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) \ - || ({ \ - bool in_chain = false; \ - WASMSharedHeap *cur; \ - uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ - for (cur = shared_heap; cur; cur = cur->chain_next) { \ - cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ - cur_shared_heap_end_off = \ - cur_shared_heap_start_off - 1 + cur->size; \ - if ((app_addr) >= cur_shared_heap_start_off \ - && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ - shared_heap_start_off = cur_shared_heap_start_off; \ - shared_heap_end_off = cur_shared_heap_end_off; \ - shared_heap_base_addr = cur->base_addr; \ - in_chain = true; \ - break; \ - } \ - } \ - in_chain; \ - }) - -#define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) - -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ - if (app_addr_in_shared_heap(app_addr, bytes)) \ - shared_heap_addr_app_to_native(app_addr, native_addr); \ - else -#else -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) -#endif - #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 #define CHECK_MEMORY_OVERFLOW(bytes) \ @@ -1562,6 +1522,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, bool is_return_call = false; #endif #if WASM_ENABLE_SHARED_HEAP != 0 + /* TODO: currently flowing two variables are only dummy for shared heap + * boundary check, need to be updated when multi-memory or memory64 + * proposals are to be implemented */ + bool is_memory64 = false; + uint32 memidx = 0; + WASMSharedHeap *shared_heap = module->e ? module->e->shared_heap : NULL; uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; uint64 shared_heap_start_off = From 3abea0cd310173da5e3f016868c199abdcc52ae0 Mon Sep 17 00:00:00 2001 From: TL Date: Wed, 12 Feb 2025 16:54:58 +0800 Subject: [PATCH 12/17] some update --- core/iwasm/common/wasm_shared_memory.h | 48 ------------------------- core/iwasm/interpreter/wasm_loader.h | 50 ++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 48 deletions(-) diff --git a/core/iwasm/common/wasm_shared_memory.h b/core/iwasm/common/wasm_shared_memory.h index 8b8af3efa..e1c5154a5 100644 --- a/core/iwasm/common/wasm_shared_memory.h +++ b/core/iwasm/common/wasm_shared_memory.h @@ -56,54 +56,6 @@ uint32 wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, uint32 count); -#if WASM_ENABLE_MULTI_MEMORY != 0 -/* Only enable shared heap for the default memory */ -#define is_default_memory (memidx == 0) -#else -#define is_default_memory true -#endif -#if WASM_ENABLE_MEMORY64 != 0 -#define get_shared_heap_start_off(shared_heap) \ - (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) -#else -#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) -#endif -/* Check whether the app addr in the last visited shared heap, if not, check the - * shared heap chain to find which(if any) shared heap the app addr in, and - * update the last visited shared heap info if found. */ -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) \ - || ({ \ - bool in_chain = false; \ - WASMSharedHeap *cur; \ - uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ - for (cur = shared_heap; cur; cur = cur->chain_next) { \ - cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ - cur_shared_heap_end_off = \ - cur_shared_heap_start_off - 1 + cur->size; \ - if ((app_addr) >= cur_shared_heap_start_off \ - && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ - shared_heap_start_off = cur_shared_heap_start_off; \ - shared_heap_end_off = cur_shared_heap_end_off; \ - shared_heap_base_addr = cur->base_addr; \ - in_chain = true; \ - break; \ - } \ - } \ - in_chain; \ - }) - -#define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) - -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ - if (app_addr_in_shared_heap(app_addr, bytes)) \ - shared_heap_addr_app_to_native(app_addr, native_addr); \ - else -#else -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) - #ifdef __cplusplus } #endif diff --git a/core/iwasm/interpreter/wasm_loader.h b/core/iwasm/interpreter/wasm_loader.h index 676770ee2..9171d71d3 100644 --- a/core/iwasm/interpreter/wasm_loader.h +++ b/core/iwasm/interpreter/wasm_loader.h @@ -73,6 +73,56 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, uint8 block_type, uint8 **p_else_addr, uint8 **p_end_addr); +#if WASM_ENABLE_SHARED_HEAP != 0 +#if WASM_ENABLE_MULTI_MEMORY != 0 +/* Only enable shared heap for the default memory */ +#define is_default_memory (memidx == 0) +#else +#define is_default_memory true +#endif +#if WASM_ENABLE_MEMORY64 != 0 +#define get_shared_heap_start_off(shared_heap) \ + (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) +#else +#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) +#endif +/* Check whether the app addr in the last visited shared heap, if not, check the + * shared heap chain to find which(if any) shared heap the app addr in, and + * update the last visited shared heap info if found. */ +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 1) \ + || ({ \ + bool in_chain = false; \ + WASMSharedHeap *cur; \ + uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ + for (cur = shared_heap; cur; cur = cur->chain_next) { \ + cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ + cur_shared_heap_end_off = \ + cur_shared_heap_start_off - 1 + cur->size; \ + if ((app_addr) >= cur_shared_heap_start_off \ + && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ + shared_heap_start_off = cur_shared_heap_start_off; \ + shared_heap_end_off = cur_shared_heap_end_off; \ + shared_heap_base_addr = cur->base_addr; \ + in_chain = true; \ + break; \ + } \ + } \ + in_chain; \ + }) + +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) + +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else +#else +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ + #ifdef __cplusplus } #endif From 88a7f161cf4f521f46c86ba7b1e4de48821d300c Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 21 Feb 2025 11:24:14 +0800 Subject: [PATCH 13/17] some refactor --- core/iwasm/common/wasm_memory.c | 278 ++++++++++++------- core/iwasm/interpreter/wasm_interp_classic.c | 15 +- core/iwasm/interpreter/wasm_interp_fast.c | 17 +- core/iwasm/interpreter/wasm_loader.h | 47 +++- core/iwasm/interpreter/wasm_runtime.h | 5 +- 5 files changed, 232 insertions(+), 130 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index d44d1778e..69801a2f7 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -328,6 +328,132 @@ wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain) return cur; } +static uint8 * +get_last_used_shared_heap_base_addr_adj(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + return e->shared_heap_base_addr_adj; + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + return e->shared_heap_base_addr_adj; + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + return 0; +} + +static uintptr_t +get_last_used_shared_heap_start_offset(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_start_off.u64; +#else + return e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_start_off.u64; +#else + return e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + return 0; +} + +static uintptr_t +get_last_used_shared_heap_end_offset(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_end_off.u64; +#else + return e->shared_heap_end_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_end_off.u64; +#else + return e->shared_heap_end_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + return 0; +} + +static void +update_last_used_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap, bool is_memory64) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + if (is_memory64) + e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; + else + e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; + e->shared_heap_end_off.u64 = + e->shared_heap_start_off.u64 - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u64; +#else + e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; + e->shared_heap_end_off.u32[0] = + e->shared_heap_start_off.u32[0] - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + if (is_memory64) + e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; + else + e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; + e->shared_heap_end_off.u64 = + e->shared_heap_start_off.u64 - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u64; +#else + e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; + e->shared_heap_end_off.u32[0] = + e->shared_heap_start_off.u32[0] - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ +} + bool wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, WASMSharedHeap *shared_heap) @@ -358,20 +484,6 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, return false; } e->shared_heap = shared_heap; -#if WASM_ENABLE_JIT != 0 -#if UINTPTR_MAX == UINT64_MAX - if (memory->is_memory64) - e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; - else - e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; - e->shared_heap_base_addr_adj = - shared_heap->base_addr - e->shared_heap_start_off.u64; -#else - e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; - e->shared_heap_base_addr_adj = - shared_heap->base_addr - e->shared_heap_start_off.u32[0]; -#endif -#endif /* end of WASM_ENABLE_JIT != 0 */ } #endif /* end of WASM_ENABLE_INTERP != 0 */ #if WASM_ENABLE_AOT != 0 @@ -383,20 +495,9 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, return false; } e->shared_heap = shared_heap; -#if UINTPTR_MAX == UINT64_MAX - if (memory->is_memory64) - e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; - else - e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; - e->shared_heap_base_addr_adj = - shared_heap->base_addr - e->shared_heap_start_off.u64; -#else - e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; - e->shared_heap_base_addr_adj = - shared_heap->base_addr - e->shared_heap_start_off.u32[0]; -#endif } #endif /* end of WASM_ENABLE_AOT != 0 */ + update_last_used_shared_heap(module_inst, shared_heap, memory->is_memory64); os_mutex_lock(&shared_heap_list_lock); shared_heap->attached_count++; @@ -428,14 +529,14 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) os_mutex_unlock(&shared_heap_list_lock); } e->shared_heap = NULL; -#if WASM_ENABLE_JIT != 0 #if UINTPTR_MAX == UINT64_MAX e->shared_heap_start_off.u64 = UINT64_MAX; + e->shared_heap_end_off.u64 = UINT64_MAX - 1; #else e->shared_heap_start_off.u32[0] = UINT32_MAX; + e->shared_heap_end_off.u32[0] = UINT32_MAX - 1; #endif e->shared_heap_base_addr_adj = NULL; -#endif } #endif /* end of WASM_ENABLE_INTERP != 0 */ #if WASM_ENABLE_AOT != 0 @@ -450,8 +551,10 @@ wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) e->shared_heap = NULL; #if UINTPTR_MAX == UINT64_MAX e->shared_heap_start_off.u64 = UINT64_MAX; + e->shared_heap_end_off.u64 = UINT64_MAX - 1; #else e->shared_heap_start_off.u32[0] = UINT32_MAX; + e->shared_heap_end_off.u32[0] = UINT32_MAX - 1; #endif e->shared_heap_base_addr_adj = NULL; } @@ -495,8 +598,7 @@ wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) static bool is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, - bool is_memory64, uint64 app_offset, uint32 bytes, - WASMSharedHeap **target_heap) + bool is_memory64, uint64 app_offset, uint32 bytes) { WASMSharedHeap *heap = get_shared_heap(module_inst), *cur; uint64 shared_heap_start, shared_heap_end; @@ -509,6 +611,14 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, bytes = 1; } + shared_heap_start = + (uint64)get_last_used_shared_heap_start_offset(module_inst); + shared_heap_end = (uint64)get_last_used_shared_heap_end_offset(module_inst); + if (app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { + return true; + } + /* Early stop for app start address not in the shared heap(chain) at all */ shared_heap_start = is_memory64 ? heap->start_off_mem64 : heap->start_off_mem32; @@ -518,31 +628,27 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, goto fail; } - /* Find the exact shared heap that app addr is in */ - if (target_heap) { - for (cur = heap; cur; cur = cur->chain_next) { - shared_heap_start = - is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; - shared_heap_end = shared_heap_start - 1 + cur->size; - if (app_offset >= shared_heap_start - && app_offset <= shared_heap_end - bytes + 1) { - *target_heap = cur; - return true; - } + /* Find the exact shared heap that app addr is in, and update last used + * shared heap info in module inst extra */ + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start - 1 + cur->size; + if (app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { + update_last_used_shared_heap(module_inst, cur, is_memory64); + return true; } } return true; fail: - if (target_heap) - *target_heap = NULL; return false; } static bool is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, - uint8 *addr, uint32 bytes, - WASMSharedHeap **target_heap) + bool is_memory64, uint8 *addr, uint32 bytes) { WASMSharedHeap *cur, *heap = get_shared_heap(module_inst); uintptr_t base_addr, addr_int, end_addr; @@ -567,14 +673,11 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, if (end_addr > base_addr + cur->size) continue; - if (target_heap) - *target_heap = cur; + update_last_used_shared_heap(module_inst, cur, is_memory64); return true; } fail: - if (target_heap) - *target_heap = NULL; return false; } @@ -908,7 +1011,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_offset, size, NULL)) { + app_offset, size)) { return true; } #endif @@ -945,6 +1048,10 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, WASMMemoryInstance *memory_inst; uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; char *str, *str_end; +#if WASM_ENABLE_SHARED_HEAP != 0 + uintptr_t shared_heap_end_off; + char *shared_heap_base_addr_adj; +#endif bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode || module_inst_comm->module_type == Wasm_Module_AoT); @@ -959,14 +1066,14 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_str_offset, 1, &shared_heap)) { - str = (char *)shared_heap->base_addr - + (memory_inst->is_memory64 - ? (app_str_offset - shared_heap->start_off_mem64) - : (app_str_offset - shared_heap->start_off_mem32)); - str_end = (char *)shared_heap->base_addr + shared_heap->size; + app_str_offset, 1)) { + shared_heap_end_off = + get_last_used_shared_heap_end_offset(module_inst_comm); + shared_heap_base_addr_adj = + (char *)get_last_used_shared_heap_base_addr_adj(module_inst_comm); + str = shared_heap_base_addr_adj + app_str_offset; + str_end = shared_heap_base_addr_adj + shared_heap_end_off; } else #endif @@ -1031,8 +1138,8 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size, - NULL)) { + if (is_native_addr_in_shared_heap( + module_inst_comm, memory_inst->is_memory64, native_ptr, size)) { return true; } #endif @@ -1060,9 +1167,6 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, WASMMemoryInstance *memory_inst; uint8 *addr; bool bounds_checks; -#if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; -#endif bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode || module_inst_comm->module_type == Wasm_Module_AoT); @@ -1076,17 +1180,9 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_offset, 1, &shared_heap)) { - uint64 shared_heap_start = 0; - - if (memory_inst && !memory_inst->is_memory64) { - shared_heap_start = shared_heap->start_off_mem32; - } - else if (memory_inst && memory_inst->is_memory64) { - shared_heap_start = shared_heap->start_off_mem64; - } - - return shared_heap->base_addr + app_offset - shared_heap_start; + app_offset, 1)) { + return get_last_used_shared_heap_base_addr_adj(module_inst_comm) + + app_offset; } #endif @@ -1135,19 +1231,9 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; - if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1, - &shared_heap)) { - uint64 shared_heap_start = 0; - - if (memory_inst && !memory_inst->is_memory64) { - shared_heap_start = shared_heap->start_off_mem32; - } - else if (memory_inst && memory_inst->is_memory64) { - shared_heap_start = shared_heap->start_off_mem64; - } - - return shared_heap_start + (addr - shared_heap->base_addr); + if (is_native_addr_in_shared_heap(module_inst_comm, + memory_inst->is_memory64, addr, 1)) { + return addr - get_last_used_shared_heap_base_addr_adj(module_inst_comm); } #endif @@ -1249,8 +1335,8 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, uint8 *native_addr; bool bounds_checks; #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; - bool is_in_shared_heap = false; + uint8 *shared_heap_base_addr_adj = NULL; + uintptr_t shared_heap_end_off = 0; #endif bh_assert(app_buf_addr <= UINTPTR_MAX && app_buf_size <= UINTPTR_MAX); @@ -1263,12 +1349,12 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, memory_inst->is_memory64, app_buf_addr, - app_buf_size, &shared_heap)) { - native_addr = shared_heap->base_addr - + (memory_inst->is_memory64 - ? (app_buf_addr - shared_heap->start_off_mem64) - : (app_buf_addr - shared_heap->start_off_mem32)); - is_in_shared_heap = true; + app_buf_size)) { + shared_heap_base_addr_adj = get_last_used_shared_heap_base_addr_adj( + (WASMModuleInstanceCommon *)module_inst); + shared_heap_end_off = get_last_used_shared_heap_end_offset( + (WASMModuleInstanceCommon *)module_inst); + native_addr = shared_heap_base_addr_adj + app_buf_addr; } else #endif @@ -1287,12 +1373,12 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (is_in_shared_heap) { + if (shared_heap_base_addr_adj) { const char *str, *str_end; /* The whole string must be in the linear memory */ str = (const char *)native_addr; - str_end = (const char *)shared_heap->base_addr + shared_heap->size; + str_end = (const char *)shared_heap_base_addr_adj + shared_heap_end_off; while (str < str_end && *str != '\0') str++; if (str == str_end) { diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index ab1486aa9..317392f1a 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -1623,14 +1623,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, is_memory64 = memory->is_memory64; #endif #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap = module->e->shared_heap; - uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; - uint64 shared_heap_start_off = - shared_heap ? get_shared_heap_start_off(shared_heap) : 0; - uint64 shared_heap_end_off = - shared_heap - ? (get_shared_heap_start_off(shared_heap) - 1 + shared_heap->size) - : 0; + WASMModuleInstanceExtra *e = module->e; #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_MULTI_MEMORY != 0 uint32 memidx = 0; @@ -5780,13 +5773,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) - dlen = shared_heap_end_off - dst + 1; + dlen = + get_last_used_shared_heap_end_off() - dst + 1; #endif #else /* else of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) { shared_heap_addr_app_to_native((uint64)dst, mdst); - dlen = shared_heap_end_off - dst + 1; + dlen = + get_last_used_shared_heap_end_off() - dst + 1; } else #endif diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index a62b6b2de..c7c3efd6a 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -1525,17 +1525,11 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* TODO: currently flowing two variables are only dummy for shared heap * boundary check, need to be updated when multi-memory or memory64 * proposals are to be implemented */ + WASMModuleInstanceExtra *e = module->e; bool is_memory64 = false; uint32 memidx = 0; - - WASMSharedHeap *shared_heap = module->e ? module->e->shared_heap : NULL; - uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; - uint64 shared_heap_start_off = - shared_heap ? get_shared_heap_start_off(shared_heap) : 0; - uint64 shared_heap_end_off = - shared_heap - ? (get_shared_heap_start_off(shared_heap) - 1 + shared_heap->size) - : 0; + (void)is_memory64; + (void)memidx; /* #endif */ #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ @@ -5085,7 +5079,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) - dlen = shared_heap_end_off - dst + 1; + dlen = (uint64)get_last_used_shared_heap_end_off() - dst + 1; #endif #else /* else of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_SHARED_HEAP != 0 @@ -5102,7 +5096,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) { shared_heap_addr_app_to_native((uint64)dst, mdst); - dlen = shared_heap_end_off - dst + 1; + dlen = (uint64)get_last_used_shared_heap_end_off() + - dst + 1; } else #endif diff --git a/core/iwasm/interpreter/wasm_loader.h b/core/iwasm/interpreter/wasm_loader.h index 9171d71d3..7ada1a09a 100644 --- a/core/iwasm/interpreter/wasm_loader.h +++ b/core/iwasm/interpreter/wasm_loader.h @@ -74,47 +74,74 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, uint8 **p_end_addr); #if WASM_ENABLE_SHARED_HEAP != 0 + #if WASM_ENABLE_MULTI_MEMORY != 0 /* Only enable shared heap for the default memory */ #define is_default_memory (memidx == 0) #else #define is_default_memory true #endif + #if WASM_ENABLE_MEMORY64 != 0 #define get_shared_heap_start_off(shared_heap) \ (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) #else -#define get_shared_heap_start_off(shared_heap) (shared_heap->start_off_mem32) +#define get_shared_heap_start_off(shared_heap) shared_heap->start_off_mem32 #endif + +#if UINTPTR_MAX == UINT64_MAX +#define update_last_used_shared_heap(shared_heap) \ + do { \ + e->shared_heap_start_off.u64 = get_shared_heap_start_off(shared_heap); \ + e->shared_heap_end_off.u64 = \ + e->shared_heap_start_off.u64 - 1 + shared_heap->size; \ + e->shared_heap_base_addr_adj = \ + shared_heap->base_addr - e->shared_heap_start_off.u64; \ + } while (0) +#define get_last_used_shared_heap_start_off() e->shared_heap_start_off.u64 +#define get_last_used_shared_heap_end_off() e->shared_heap_end_off.u64 +#else +#define update_last_used_shared_heap(shared_heap) \ + do { \ + e->shared_heap_start_off.u32[0] = \ + (uint32)shared_heap->start_off_mem32; \ + e->shared_heap_end_off.u32[0] = \ + e->shared_heap_start_off.u32[0] - 1 + shared_heap->size; \ + e->shared_heap_base_addr_adj = \ + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; \ + } while (0) +#define get_last_used_shared_heap_start_off() \ + (uint64) e->shared_heap_start_off.u32[0] +#define get_last_used_shared_heap_end_off() \ + (uint64) e->shared_heap_end_off.u32[0] +#endif /* end of UINTPTR_MAX == UINT64_MAX */ + /* Check whether the app addr in the last visited shared heap, if not, check the * shared heap chain to find which(if any) shared heap the app addr in, and * update the last visited shared heap info if found. */ #define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) \ + (e->shared_heap && is_default_memory \ + && (app_addr) >= get_last_used_shared_heap_start_off() \ + && (app_addr) <= get_last_used_shared_heap_end_off() - bytes + 1) \ || ({ \ bool in_chain = false; \ WASMSharedHeap *cur; \ uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ - for (cur = shared_heap; cur; cur = cur->chain_next) { \ + for (cur = e->shared_heap; cur; cur = cur->chain_next) { \ cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ cur_shared_heap_end_off = \ cur_shared_heap_start_off - 1 + cur->size; \ if ((app_addr) >= cur_shared_heap_start_off \ && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ - shared_heap_start_off = cur_shared_heap_start_off; \ - shared_heap_end_off = cur_shared_heap_end_off; \ - shared_heap_base_addr = cur->base_addr; \ + update_last_used_shared_heap(cur); \ in_chain = true; \ break; \ } \ } \ in_chain; \ }) - #define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) - + native_addr = e->shared_heap_base_addr_adj + app_addr #define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ if (app_addr_in_shared_heap(app_addr, bytes)) \ shared_heap_addr_app_to_native(app_addr, native_addr); \ diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index b9d49ec22..0acb8de31 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -373,8 +373,6 @@ typedef struct WASMModuleInstanceExtra { #endif #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; -#if WASM_ENABLE_JIT != 0 /* * Adjusted shared heap based addr to simple the calculation * in the aot code. The value is: @@ -382,7 +380,8 @@ typedef struct WASMModuleInstanceExtra { */ uint8 *shared_heap_base_addr_adj; MemBound shared_heap_start_off; -#endif + MemBound shared_heap_end_off; + WASMSharedHeap *shared_heap; #endif #if WASM_ENABLE_DEBUG_INTERP != 0 \ From f173354f2eafb9887050c5991f9efa097ccc0db4 Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 21 Feb 2025 11:29:31 +0800 Subject: [PATCH 14/17] code refactor --- core/iwasm/interpreter/wasm_interp_fast.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index c7c3efd6a..7ebe6c974 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -5079,7 +5079,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) - dlen = (uint64)get_last_used_shared_heap_end_off() - dst + 1; + dlen = (uint64)get_last_used_shared_heap_end_off() + - dst + 1; #endif #else /* else of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_SHARED_HEAP != 0 From 948d981a5c42fa350dbde8092975b0df4223f11b Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 21 Feb 2025 12:32:25 +0800 Subject: [PATCH 15/17] aot data struct modification --- core/iwasm/aot/aot_runtime.c | 2 ++ core/iwasm/aot/aot_runtime.h | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 0f7b5d3d9..d3cd27ee9 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -60,6 +60,8 @@ bh_static_assert(offsetof(AOTModuleInstanceExtra, stack_sizes) == 0); bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj) == 8); bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_start_off) == 16); +bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_end_off) == 24); +bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap) == 32); bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3); diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 297b2a5b5..f841bfb6a 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -125,6 +125,8 @@ typedef struct AOTModuleInstanceExtra { */ DefPointer(uint8 *, shared_heap_base_addr_adj); MemBound shared_heap_start_off; + MemBound shared_heap_end_off; + DefPointer(WASMSharedHeap *, shared_heap); WASMModuleInstanceExtraCommon common; @@ -142,9 +144,6 @@ typedef struct AOTModuleInstanceExtra { WASMModuleInstanceCommon **import_func_module_insts; #endif -#if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; -#endif } AOTModuleInstanceExtra; #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) From 035f758734a5a14ff573f4cd4e8a192168c224da Mon Sep 17 00:00:00 2001 From: TL Date: Tue, 25 Feb 2025 15:28:36 +0800 Subject: [PATCH 16/17] code refactor --- core/iwasm/common/wasm_memory.c | 42 ++++------- core/iwasm/common/wasm_memory.h | 46 +++++++++++- core/iwasm/interpreter/wasm_interp_classic.c | 11 +-- core/iwasm/interpreter/wasm_interp_fast.c | 7 +- core/iwasm/interpreter/wasm_loader.h | 77 -------------------- 5 files changed, 64 insertions(+), 119 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 69801a2f7..b9a42ceff 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -596,7 +596,7 @@ wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) return get_shared_heap(module_inst_comm); } -static bool +bool is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, bool is_memory64, uint64 app_offset, uint32 bytes) { @@ -641,7 +641,6 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, } } - return true; fail: return false; } @@ -1220,11 +1219,6 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, bounds_checks = is_bounds_checks_enabled(module_inst_comm); -#if WASM_ENABLE_SHARED_HEAP != 0 - /* If shared heap is enabled, bounds check is always needed */ - bounds_checks = true; -#endif - memory_inst = wasm_get_default_memory(module_inst); if (!memory_inst) { return 0; @@ -1350,33 +1344,14 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, memory_inst->is_memory64, app_buf_addr, app_buf_size)) { + const char *str, *str_end; shared_heap_base_addr_adj = get_last_used_shared_heap_base_addr_adj( (WASMModuleInstanceCommon *)module_inst); shared_heap_end_off = get_last_used_shared_heap_end_offset( (WASMModuleInstanceCommon *)module_inst); native_addr = shared_heap_base_addr_adj + app_buf_addr; - } - else -#endif - { - native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; - } - bounds_checks = - is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst); - - if (!bounds_checks) { - if (app_buf_addr == 0) { - native_addr = NULL; - } - goto success; - } - -#if WASM_ENABLE_SHARED_HEAP != 0 - if (shared_heap_base_addr_adj) { - const char *str, *str_end; - - /* The whole string must be in the linear memory */ + /* The whole string must be in the shared heap */ str = (const char *)native_addr; str_end = (const char *)shared_heap_base_addr_adj + shared_heap_end_off; while (str < str_end && *str != '\0') @@ -1390,6 +1365,17 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, } #endif + native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; + bounds_checks = + is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst); + + if (!bounds_checks) { + if (app_buf_addr == 0) { + native_addr = NULL; + } + goto success; + } + /* No need to check the app_offset and buf_size if memory access boundary check with hardware trap is enabled */ #ifndef OS_ENABLE_HW_BOUND_CHECK diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index 88d0f3c5d..48bd2179f 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -41,7 +41,51 @@ SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size) #define SET_LINEAR_MEMORY_SIZE(memory, size) memory->memory_data_size = size #endif +#if WASM_ENABLE_INTERP != 0 #if WASM_ENABLE_SHARED_HEAP != 0 + +#if WASM_ENABLE_MULTI_MEMORY != 0 +/* Only enable shared heap for the default memory */ +#define is_default_memory (memidx == 0) +#else +#define is_default_memory true +#endif + +#if UINTPTR_MAX == UINT64_MAX +#define get_shared_heap_end_off() module->e->shared_heap_end_off.u64 +#else +#define get_shared_heap_end_off() \ + (uint64)(module->e->shared_heap_end_off.u32[0]) +#endif + +#if WASM_ENABLE_MEMORY64 != 0 +#define shared_heap_is_memory64 is_memory64 +#else +#define shared_heap_is_memory64 false +#endif + +#define app_addr_in_shared_heap(app_addr, bytes) \ + (is_default_memory \ + && is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module, \ + shared_heap_is_memory64, (uint64)app_addr, \ + bytes)) +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = module->e->shared_heap_base_addr_adj + app_addr +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else + +#else /* else of WASM_ENABLE_SHARED_HEAP != 0 */ +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ +#endif /* end of WASM_ENABLE_INTERP != 0 */ + +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, + bool is_memory64, uint64 app_offset, uint32 bytes); + WASMSharedHeap * wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); @@ -74,7 +118,7 @@ wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, void wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr); -#endif +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 317392f1a..9514e1e64 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -1622,9 +1622,6 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (memory) is_memory64 = memory->is_memory64; #endif -#if WASM_ENABLE_SHARED_HEAP != 0 - WASMModuleInstanceExtra *e = module->e; -#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_MULTI_MEMORY != 0 uint32 memidx = 0; uint32 memidx_cached = (uint32)-1; @@ -2399,7 +2396,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, else cur_func_type = cur_func->u.func->func_type; - /* clang-format off */ + /* clang-format off */ #if WASM_ENABLE_GC == 0 if (cur_type != cur_func_type) { wasm_set_exception(module, "indirect call type mismatch"); @@ -5773,15 +5770,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) - dlen = - get_last_used_shared_heap_end_off() - dst + 1; + dlen = get_shared_heap_end_off() - dst + 1; #endif #else /* else of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) { shared_heap_addr_app_to_native((uint64)dst, mdst); - dlen = - get_last_used_shared_heap_end_off() - dst + 1; + dlen = get_shared_heap_end_off() - dst + 1; } else #endif diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 7ebe6c974..8538d376e 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -1525,7 +1525,6 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* TODO: currently flowing two variables are only dummy for shared heap * boundary check, need to be updated when multi-memory or memory64 * proposals are to be implemented */ - WASMModuleInstanceExtra *e = module->e; bool is_memory64 = false; uint32 memidx = 0; (void)is_memory64; @@ -5079,8 +5078,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) - dlen = (uint64)get_last_used_shared_heap_end_off() - - dst + 1; + dlen = (uint64)get_shared_heap_end_off() - dst + 1; #endif #else /* else of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_SHARED_HEAP != 0 @@ -5097,8 +5095,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_SHARED_HEAP != 0 if (app_addr_in_shared_heap((uint64)dst, len)) { shared_heap_addr_app_to_native((uint64)dst, mdst); - dlen = (uint64)get_last_used_shared_heap_end_off() - - dst + 1; + dlen = (uint64)get_shared_heap_end_off() - dst + 1; } else #endif diff --git a/core/iwasm/interpreter/wasm_loader.h b/core/iwasm/interpreter/wasm_loader.h index 7ada1a09a..676770ee2 100644 --- a/core/iwasm/interpreter/wasm_loader.h +++ b/core/iwasm/interpreter/wasm_loader.h @@ -73,83 +73,6 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, uint8 block_type, uint8 **p_else_addr, uint8 **p_end_addr); -#if WASM_ENABLE_SHARED_HEAP != 0 - -#if WASM_ENABLE_MULTI_MEMORY != 0 -/* Only enable shared heap for the default memory */ -#define is_default_memory (memidx == 0) -#else -#define is_default_memory true -#endif - -#if WASM_ENABLE_MEMORY64 != 0 -#define get_shared_heap_start_off(shared_heap) \ - (is_memory64 ? shared_heap->start_off_mem64 : shared_heap->start_off_mem32) -#else -#define get_shared_heap_start_off(shared_heap) shared_heap->start_off_mem32 -#endif - -#if UINTPTR_MAX == UINT64_MAX -#define update_last_used_shared_heap(shared_heap) \ - do { \ - e->shared_heap_start_off.u64 = get_shared_heap_start_off(shared_heap); \ - e->shared_heap_end_off.u64 = \ - e->shared_heap_start_off.u64 - 1 + shared_heap->size; \ - e->shared_heap_base_addr_adj = \ - shared_heap->base_addr - e->shared_heap_start_off.u64; \ - } while (0) -#define get_last_used_shared_heap_start_off() e->shared_heap_start_off.u64 -#define get_last_used_shared_heap_end_off() e->shared_heap_end_off.u64 -#else -#define update_last_used_shared_heap(shared_heap) \ - do { \ - e->shared_heap_start_off.u32[0] = \ - (uint32)shared_heap->start_off_mem32; \ - e->shared_heap_end_off.u32[0] = \ - e->shared_heap_start_off.u32[0] - 1 + shared_heap->size; \ - e->shared_heap_base_addr_adj = \ - shared_heap->base_addr - e->shared_heap_start_off.u32[0]; \ - } while (0) -#define get_last_used_shared_heap_start_off() \ - (uint64) e->shared_heap_start_off.u32[0] -#define get_last_used_shared_heap_end_off() \ - (uint64) e->shared_heap_end_off.u32[0] -#endif /* end of UINTPTR_MAX == UINT64_MAX */ - -/* Check whether the app addr in the last visited shared heap, if not, check the - * shared heap chain to find which(if any) shared heap the app addr in, and - * update the last visited shared heap info if found. */ -#define app_addr_in_shared_heap(app_addr, bytes) \ - (e->shared_heap && is_default_memory \ - && (app_addr) >= get_last_used_shared_heap_start_off() \ - && (app_addr) <= get_last_used_shared_heap_end_off() - bytes + 1) \ - || ({ \ - bool in_chain = false; \ - WASMSharedHeap *cur; \ - uint64 cur_shared_heap_start_off, cur_shared_heap_end_off; \ - for (cur = e->shared_heap; cur; cur = cur->chain_next) { \ - cur_shared_heap_start_off = get_shared_heap_start_off(cur); \ - cur_shared_heap_end_off = \ - cur_shared_heap_start_off - 1 + cur->size; \ - if ((app_addr) >= cur_shared_heap_start_off \ - && (app_addr) <= cur_shared_heap_end_off - bytes + 1) { \ - update_last_used_shared_heap(cur); \ - in_chain = true; \ - break; \ - } \ - } \ - in_chain; \ - }) -#define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = e->shared_heap_base_addr_adj + app_addr -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ - if (app_addr_in_shared_heap(app_addr, bytes)) \ - shared_heap_addr_app_to_native(app_addr, native_addr); \ - else -#else -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) -#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ - #ifdef __cplusplus } #endif From a62d10400f2eed1b50b47e20bf622f1ca7533f72 Mon Sep 17 00:00:00 2001 From: TL Date: Tue, 25 Feb 2025 15:39:57 +0800 Subject: [PATCH 17/17] format --- core/iwasm/interpreter/wasm_interp_classic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 9514e1e64..ddf47aeee 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -2396,7 +2396,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, else cur_func_type = cur_func->u.func->func_type; - /* clang-format off */ + /* clang-format off */ #if WASM_ENABLE_GC == 0 if (cur_type != cur_func_type) { wasm_set_exception(module, "indirect call type mismatch");