From b92906464ea7d6f3a231c28e25db604cc91b59a4 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Thu, 16 Nov 2023 15:10:58 +0800 Subject: [PATCH 01/67] Fix printing ref.extern addresses in wasm_application.c (#2774) --- core/iwasm/common/wasm_application.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/iwasm/common/wasm_application.c b/core/iwasm/common/wasm_application.c index bcd0a6d0b..93c9342da 100644 --- a/core/iwasm/common/wasm_application.c +++ b/core/iwasm/common/wasm_application.c @@ -600,7 +600,7 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name, { #if UINTPTR_MAX == UINT32_MAX if (argv1[k] != 0 && argv1[k] != (uint32)-1) - os_printf("0x%" PRIxPTR ":ref.extern", (void *)argv1[k]); + os_printf("0x%" PRIxPTR ":ref.extern", (uintptr_t)argv1[k]); else os_printf("extern:ref.null"); k++; @@ -613,7 +613,7 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name, u.parts[1] = argv1[k + 1]; k += 2; if (u.val && u.val != (uintptr_t)-1LL) - os_printf("0x%" PRIxPTR ":ref.extern", (void *)u.val); + os_printf("0x%" PRIxPTR ":ref.extern", u.val); else os_printf("extern:ref.null"); #endif From 503d94ace2a59fe8978e9ec62d603039013aa838 Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Thu, 16 Nov 2023 18:32:28 +0800 Subject: [PATCH 02/67] Enable more LLVM backends for the release wamrc binary (#2778) As mentioned in https://github.com/bytecodealliance/wasm-micro-runtime/issues/2504, the current release `wamrc` only has X86 target enabled. This PR enables more targets for release `wamrc`, including AArch64, ARM, Mips and RISCV. --- .github/workflows/release_process.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_process.yml b/.github/workflows/release_process.yml index 4188b4d40..ef722acdb 100644 --- a/.github/workflows/release_process.yml +++ b/.github/workflows/release_process.yml @@ -56,21 +56,21 @@ jobs: uses: ./.github/workflows/build_llvm_libraries.yml with: os: "ubuntu-20.04" - arch: "X86" + arch: "AArch64 ARM Mips RISCV X86" build_llvm_libraries_on_ubuntu_2204: needs: [create_tag, create_release] uses: ./.github/workflows/build_llvm_libraries.yml with: os: "ubuntu-22.04" - arch: "X86" + arch: "AArch64 ARM Mips RISCV X86" build_llvm_libraries_on_macos: needs: [create_tag, create_release] uses: ./.github/workflows/build_llvm_libraries.yml with: os: "macos-latest" - arch: "X86" + arch: "AArch64 ARM Mips RISCV X86" # # WAMRC From a3349cc8c1ff97e3f138571f773deada93c8efce Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Thu, 16 Nov 2023 18:39:41 +0800 Subject: [PATCH 03/67] Fix compilation errors on zephyr platform (#2777) Add dummy korp_rwlock struct and fix os_dir_stream definition on platform zephyr/riot/rt-thread/alios to fix the compilation errors. --- core/shared/platform/alios/platform_internal.h | 10 +++++++++- core/shared/platform/riot/platform_internal.h | 10 +++++++++- core/shared/platform/rt-thread/platform_internal.h | 10 +++++++++- core/shared/platform/zephyr/platform_internal.h | 10 +++++++++- product-mini/platforms/zephyr/simple/src/main.c | 10 ++++++++++ 5 files changed, 46 insertions(+), 4 deletions(-) diff --git a/core/shared/platform/alios/platform_internal.h b/core/shared/platform/alios/platform_internal.h index dbdf74953..d2897a6b5 100644 --- a/core/shared/platform/alios/platform_internal.h +++ b/core/shared/platform/alios/platform_internal.h @@ -32,6 +32,12 @@ typedef aos_task_t *aos_tid_t; typedef aos_mutex_t korp_mutex; typedef aos_sem_t korp_sem; +/* korp_rwlock is used in platform_api_extension.h, + we just define the type to make the compiler happy */ +typedef struct { + int dummy; +} korp_rwlock; + struct os_thread_wait_node; typedef struct os_thread_wait_node *os_thread_wait_list; typedef struct korp_cond { @@ -64,8 +70,10 @@ int signbit(double x); int isnan(double x); /* clang-format on */ +/* The below types are used in platform_api_extension.h, + we just define them to make the compiler happy */ typedef int os_file_handle; -typedef DIR *os_dir_stream; +typedef void *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle diff --git a/core/shared/platform/riot/platform_internal.h b/core/shared/platform/riot/platform_internal.h index 1f71ffdb2..e88b25d40 100644 --- a/core/shared/platform/riot/platform_internal.h +++ b/core/shared/platform/riot/platform_internal.h @@ -40,6 +40,12 @@ typedef kernel_pid_t korp_tid; typedef mutex_t korp_mutex; typedef unsigned int korp_sem; +/* korp_rwlock is used in platform_api_extension.h, + we just define the type to make the compiler happy */ +typedef struct { + int dummy; +} korp_rwlock; + /* typedef sema_t korp_sem; */ struct os_thread_wait_node; @@ -52,8 +58,10 @@ typedef struct korp_cond { #define os_printf printf #define os_vprintf vprintf +/* The below types are used in platform_api_extension.h, + we just define them to make the compiler happy */ typedef int os_file_handle; -typedef DIR *os_dir_stream; +typedef void *os_dir_stream; typedef int os_raw_file_handle; #if WA_MATH diff --git a/core/shared/platform/rt-thread/platform_internal.h b/core/shared/platform/rt-thread/platform_internal.h index 0c2e1d82b..4ebdabb10 100644 --- a/core/shared/platform/rt-thread/platform_internal.h +++ b/core/shared/platform/rt-thread/platform_internal.h @@ -38,6 +38,12 @@ typedef struct rt_thread korp_cond; typedef struct rt_thread korp_thread; typedef unsigned int korp_sem; +/* korp_rwlock is used in platform_api_extension.h, + we just define the type to make the compiler happy */ +typedef struct { + int dummy; +} korp_rwlock; + typedef rt_uint8_t uint8_t; typedef rt_int8_t int8_t; typedef rt_uint16_t uint16_t; @@ -45,8 +51,10 @@ typedef rt_int16_t int16_t; typedef rt_uint64_t uint64_t; typedef rt_int64_t int64_t; +/* The below types are used in platform_api_extension.h, + we just define them to make the compiler happy */ typedef int os_file_handle; -typedef DIR *os_dir_stream; +typedef void *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle diff --git a/core/shared/platform/zephyr/platform_internal.h b/core/shared/platform/zephyr/platform_internal.h index 048387817..2306fa06b 100644 --- a/core/shared/platform/zephyr/platform_internal.h +++ b/core/shared/platform/zephyr/platform_internal.h @@ -74,6 +74,12 @@ typedef korp_thread *korp_tid; typedef struct k_mutex korp_mutex; typedef unsigned int korp_sem; +/* korp_rwlock is used in platform_api_extension.h, + we just define the type to make the compiler happy */ +typedef struct { + int dummy; +} korp_rwlock; + struct os_thread_wait_node; typedef struct os_thread_wait_node *os_thread_wait_list; typedef struct korp_cond { @@ -148,8 +154,10 @@ void set_exec_mem_alloc_func(exec_mem_alloc_func_t alloc_func, exec_mem_free_func_t free_func); +/* The below types are used in platform_api_extension.h, + we just define them to make the compiler happy */ typedef int os_file_handle; -typedef DIR *os_dir_stream; +typedef void *os_dir_stream; typedef int os_raw_file_handle; static inline os_file_handle diff --git a/product-mini/platforms/zephyr/simple/src/main.c b/product-mini/platforms/zephyr/simple/src/main.c index a6479ba75..6c0b8fc82 100644 --- a/product-mini/platforms/zephyr/simple/src/main.c +++ b/product-mini/platforms/zephyr/simple/src/main.c @@ -265,8 +265,18 @@ iwasm_init(void) iwasm_main, NULL, NULL, NULL, MAIN_THREAD_PRIORITY, 0, K_NO_WAIT); return tid ? true : false; } + +#if KERNEL_VERSION_NUMBER < 0x030400 /* version 3.4.0 */ void main(void) { iwasm_init(); } +#else +int +main(void) +{ + iwasm_init(); + return 0; +} +#endif From 2d0d4a0be9732b58d796809f5b6ad4e00d536861 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Thu, 16 Nov 2023 19:48:06 +0900 Subject: [PATCH 04/67] Remove unused JitBitmap (#2775) Fixes: https://github.com/bytecodealliance/wasm-micro-runtime/issues/2754 --- core/iwasm/fast-jit/jit_utils.c | 19 ------- core/iwasm/fast-jit/jit_utils.h | 94 --------------------------------- 2 files changed, 113 deletions(-) delete mode 100644 core/iwasm/fast-jit/jit_utils.c diff --git a/core/iwasm/fast-jit/jit_utils.c b/core/iwasm/fast-jit/jit_utils.c deleted file mode 100644 index 57a3e8f67..000000000 --- a/core/iwasm/fast-jit/jit_utils.c +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2021 Intel Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - */ - -#include "jit_utils.h" - -JitBitmap * -jit_bitmap_new(uintptr_t begin_index, unsigned bitnum) -{ - JitBitmap *bitmap; - - if ((bitmap = jit_calloc(offsetof(JitBitmap, map) + (bitnum + 7) / 8))) { - bitmap->begin_index = begin_index; - bitmap->end_index = begin_index + bitnum; - } - - return bitmap; -} diff --git a/core/iwasm/fast-jit/jit_utils.h b/core/iwasm/fast-jit/jit_utils.h index c165b7a3c..a533c70bc 100644 --- a/core/iwasm/fast-jit/jit_utils.h +++ b/core/iwasm/fast-jit/jit_utils.h @@ -12,20 +12,6 @@ extern "C" { #endif -/** - * A simple fixed size bitmap. - */ -typedef struct JitBitmap { - /* The first valid bit index. */ - uintptr_t begin_index; - - /* The last valid bit index plus one. */ - uintptr_t end_index; - - /* The bitmap. */ - uint8 map[1]; -} JitBitmap; - static inline void * jit_malloc(unsigned int size) { @@ -49,86 +35,6 @@ jit_free(void *ptr) wasm_runtime_free(ptr); } -/** - * Create a new bitmap. - * - * @param begin_index the first valid bit index - * @param bitnum maximal bit number of the bitmap. - * - * @return the new bitmap if succeeds, NULL otherwise. - */ -JitBitmap * -jit_bitmap_new(uintptr_t begin_index, unsigned bitnum); - -/** - * Delete a bitmap. - * - * @param bitmap the bitmap to be deleted - */ -static inline void -jit_bitmap_delete(JitBitmap *bitmap) -{ - jit_free(bitmap); -} - -/** - * Check whether the given index is in the range of the bitmap. - * - * @param bitmap the bitmap - * @param n the bit index - * - * @return true if the index is in range, false otherwise - */ -static inline bool -jit_bitmap_is_in_range(JitBitmap *bitmap, unsigned n) -{ - return n >= bitmap->begin_index && n < bitmap->end_index; -} - -/** - * Get a bit in the bitmap - * - * @param bitmap the bitmap - * @param n the n-th bit to be get - * - * @return value of the bit - */ -static inline int -jit_bitmap_get_bit(JitBitmap *bitmap, unsigned n) -{ - unsigned idx = n - bitmap->begin_index; - bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); - return (bitmap->map[idx / 8] >> (idx % 8)) & 1; -} - -/** - * Set a bit in the bitmap. - * - * @param bitmap the bitmap - * @param n the n-th bit to be set - */ -static inline void -jit_bitmap_set_bit(JitBitmap *bitmap, unsigned n) -{ - unsigned idx = n - bitmap->begin_index; - bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); - bitmap->map[idx / 8] |= 1 << (idx % 8); -} - -/** - * Clear a bit in the bitmap. - * - * @param bitmap the bitmap - * @param n the n-th bit to be cleared - */ -static inline void -jit_bitmap_clear_bit(JitBitmap *bitmap, unsigned n) -{ - unsigned idx = n - bitmap->begin_index; - bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); - bitmap->map[idx / 8] &= ~(1 << (idx % 8)); -} - #ifdef __cplusplus } #endif From b39fd516d3892897c1cab2d3c8c4edffa4d60b59 Mon Sep 17 00:00:00 2001 From: zoraaver <55952569+zoraaver@users.noreply.github.com> Date: Fri, 17 Nov 2023 10:40:29 +0000 Subject: [PATCH 05/67] Use next generation crypto API on Windows (#2769) CryptGenRandom is deprecated by Microsoft and may be removed in future releases. They recommend to use the next generation API instead. See https://learn.microsoft.com/en-us/windows/win32/seccng/cng-portal for more details. Also, refactor the random functions to return error codes rather than aborting the program if they fail. --- .../sandboxed-system-primitives/src/posix.c | 28 +++++-- .../sandboxed-system-primitives/src/random.c | 82 ++++++++++++------- .../sandboxed-system-primitives/src/random.h | 8 +- 3 files changed, 77 insertions(+), 41 deletions(-) diff --git a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c index 385aa15cf..8fcf455a4 100644 --- a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c +++ b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c @@ -614,14 +614,21 @@ fd_table_insert_existing(struct fd_table *ft, __wasi_fd_t in, } // Picks an unused slot from the file descriptor table. -static __wasi_fd_t -fd_table_unused(struct fd_table *ft) REQUIRES_SHARED(ft->lock) +static __wasi_errno_t +fd_table_unused(struct fd_table *ft, __wasi_fd_t *out) REQUIRES_SHARED(ft->lock) { assert(ft->size > ft->used && "File descriptor table has no free slots"); for (;;) { - __wasi_fd_t fd = (__wasi_fd_t)random_uniform(ft->size); - if (ft->entries[fd].object == NULL) - return fd; + uintmax_t random_fd = 0; + __wasi_errno_t error = random_uniform(ft->size, &random_fd); + + if (error != __WASI_ESUCCESS) + return error; + + if (ft->entries[(__wasi_fd_t)random_fd].object == NULL) { + *out = (__wasi_fd_t)random_fd; + return error; + } } } @@ -641,10 +648,14 @@ fd_table_insert(wasm_exec_env_t exec_env, struct fd_table *ft, return convert_errno(errno); } - *out = fd_table_unused(ft); + __wasi_errno_t error = fd_table_unused(ft, out); + + if (error != __WASI_ESUCCESS) + return error; + fd_table_attach(ft, *out, fo, rights_base, rights_inheriting); rwlock_unlock(&ft->lock); - return 0; + return error; } // Inserts a numerical file descriptor into the file descriptor table. @@ -2282,8 +2293,7 @@ wasmtime_ssp_poll_oneoff(wasm_exec_env_t exec_env, struct fd_table *curfds, __wasi_errno_t wasmtime_ssp_random_get(void *buf, size_t nbyte) { - random_buf(buf, nbyte); - return 0; + return random_buf(buf, nbyte); } __wasi_errno_t diff --git a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.c b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.c index b21aa197a..29c50dd87 100644 --- a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.c +++ b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.c @@ -13,14 +13,16 @@ #include "ssp_config.h" #include "bh_platform.h" +#include "libc_errno.h" #include "random.h" #if CONFIG_HAS_ARC4RANDOM_BUF -void +__wasi_errno_t random_buf(void *buf, size_t len) { arc4random_buf(buf, len); + return __WASI_ESUCCESS; } #elif CONFIG_HAS_GETRANDOM @@ -29,7 +31,7 @@ random_buf(void *buf, size_t len) #include #endif -void +__wasi_errno_t random_buf(void *buf, size_t len) { for (;;) { @@ -37,57 +39,71 @@ random_buf(void *buf, size_t len) if (x < 0) { if (errno == EINTR) continue; - os_printf("getrandom failed: %s", strerror(errno)); - abort(); + return convert_errno(errno); } if ((size_t)x == len) - return; + break; buf = (void *)((unsigned char *)buf + x); len -= (size_t)x; } + return __WASI_ESUCCESS; } #elif defined(BH_PLATFORM_WINDOWS) -#include +#include +#pragma comment(lib, "Bcrypt.lib") -void +__wasi_errno_t random_buf(void *buf, size_t len) { - static int crypt_initialized = 0; - static HCRYPTPROV provider; - if (!crypt_initialized) { - CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL, - CRYPT_VERIFYCONTEXT); - crypt_initialized = 1; - } - CryptGenRandom(provider, len, buf); + NTSTATUS ret = + BCryptGenRandom(NULL, buf, (ULONG)len, BCRYPT_USE_SYSTEM_PREFERRED_RNG); + + // Since we pass NULL for the algorithm handle, the only way BCryptGenRandom + // can fail is if one of the parameters is invalid + // (STATUS_INVALID_PARAMETER). + return ret ? __WASI_EINVAL : __WASI_ESUCCESS; } #else -static int urandom; +static int urandom = -1; +static __wasi_errno_t urandom_error = __WASI_ESUCCESS; static void open_urandom(void) { urandom = open("/dev/urandom", O_RDONLY); - if (urandom < 0) { - os_printf("Failed to open /dev/urandom\n"); - abort(); - } + if (urandom < 0) + urandom_error = convert_errno(errno); } -void +__wasi_errno_t random_buf(void *buf, size_t len) { static pthread_once_t open_once = PTHREAD_ONCE_INIT; - pthread_once(&open_once, open_urandom); + int pthread_ret = pthread_once(&open_once, open_urandom); - if ((size_t)read(urandom, buf, len) != len) { - os_printf("Short read on /dev/urandom\n"); - abort(); + if (pthread_ret != 0) + return convert_errno(pthread_ret); + + if (urandom < 0) + return urandom_error; + + size_t bytes_read = 0; + + while (bytes_read < len) { + ssize_t bytes_read_now = + read(urandom, buf + bytes_read, len - bytes_read); + + if (bytes_read_now < 0) + return convert_errno(errno); + + bytes_read += (size_t)bytes_read_now; } + + return __WASI_ESUCCESS; } #endif @@ -99,8 +115,8 @@ random_buf(void *buf, size_t len) // arc4random() until it lies within the range [2^k % upper, 2^k). As // this range has length k * upper, we can safely obtain a number // without any modulo bias. -uintmax_t -random_uniform(uintmax_t upper) +__wasi_errno_t +random_uniform(uintmax_t upper, uintmax_t *out) { // Compute 2^k % upper // == (2^k - upper) % upper @@ -108,8 +124,14 @@ random_uniform(uintmax_t upper) uintmax_t lower = -upper % upper; for (;;) { uintmax_t value; - random_buf(&value, sizeof(value)); - if (value >= lower) - return value % upper; + __wasi_errno_t error = random_buf(&value, sizeof(value)); + + if (error != __WASI_ESUCCESS) + return error; + + if (value >= lower) { + *out = value % upper; + return error; + } } } diff --git a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.h b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.h index 23c2da4db..7cd94d74d 100644 --- a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.h +++ b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/random.h @@ -14,8 +14,12 @@ #ifndef RANDOM_H #define RANDOM_H -void +#include "bh_platform.h" + +__wasi_errno_t random_buf(void *, size_t); -uintmax_t random_uniform(uintmax_t); + +__wasi_errno_t +random_uniform(uintmax_t upper, uintmax_t *out); #endif From 08c0ec74c491ae3b763beb272d2d7c356d9c11f7 Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Fri, 17 Nov 2023 19:05:00 +0800 Subject: [PATCH 06/67] More precise help info of enabled targets for wamrc (#2783) Instead of printing all support targets of wamrc, print only the targets that are included in the LLVM library with which wamrc was compiled. --- core/iwasm/compilation/aot_llvm.c | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index dacdb83d7..0818893dd 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1938,13 +1938,33 @@ static void print_supported_targets() { uint32 i; + const char *target_name; + os_printf("Supported targets:\n"); - for (i = 0; i < sizeof(valid_archs) / sizeof(ArchItem); i++) { - os_printf("%s ", valid_archs[i].arch); - if (valid_archs[i].support_eb) - os_printf("%seb ", valid_archs[i].arch); + /* over the list of all available targets */ + for (LLVMTargetRef target = LLVMGetFirstTarget(); target != NULL; + target = LLVMGetNextTarget(target)) { + target_name = LLVMGetTargetName(target); + /* Skip mipsel, aarch64_be since prefix mips, aarch64 will cover them */ + if (strcmp(target_name, "mipsel") == 0) + continue; + else if (strcmp(target_name, "aarch64_be") == 0) + continue; + + if (strcmp(target_name, "x86-64") == 0) + os_printf(" x86_64\n"); + else if (strcmp(target_name, "x86") == 0) + os_printf(" i386\n"); + else { + for (i = 0; i < sizeof(valid_archs) / sizeof(ArchItem); i++) { + /* If target_name is prefix for valid_archs[i].arch */ + if ((strncmp(target_name, valid_archs[i].arch, + strlen(target_name)) + == 0)) + os_printf(" %s\n", valid_archs[i].arch); + } + } } - os_printf("\n"); } static void From 562a5dd1b688b1517491cccc3dd635b5a58ca562 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Sat, 18 Nov 2023 09:50:16 +0900 Subject: [PATCH 07/67] Fix data/elem drop (#2747) Currently, `data.drop` instruction is implemented by directly modifying the underlying module. It breaks use cases where you have multiple instances sharing a single loaded module. `elem.drop` has the same problem too. This PR fixes the issue by keeping track of which data/elem segments have been dropped by using bitmaps for each module instances separately, and add a sample to demonstrate the issue and make the CI run it. Also add a missing check of dropped elements to the fast-jit `table.init`. Fixes: https://github.com/bytecodealliance/wasm-micro-runtime/issues/2735 Fixes: https://github.com/bytecodealliance/wasm-micro-runtime/issues/2772 --- .../compilation_on_android_ubuntu.yml | 6 + .github/workflows/compilation_on_macos.yml | 6 + .github/workflows/nightly_run.yml | 6 + core/iwasm/aot/aot_loader.c | 1 - core/iwasm/aot/aot_runtime.c | 74 +++++-- core/iwasm/compilation/aot.c | 1 - core/iwasm/compilation/aot.h | 1 - core/iwasm/compilation/aot_emit_aot_file.c | 2 +- core/iwasm/fast-jit/fe/jit_emit_memory.c | 36 +-- core/iwasm/fast-jit/fe/jit_emit_table.c | 27 ++- core/iwasm/interpreter/wasm.h | 1 - core/iwasm/interpreter/wasm_interp_classic.c | 25 ++- core/iwasm/interpreter/wasm_interp_fast.c | 27 ++- core/iwasm/interpreter/wasm_runtime.c | 59 ++++- core/iwasm/interpreter/wasm_runtime.h | 7 + core/shared/utils/bh_bitmap.c | 27 +++ core/shared/utils/bh_bitmap.h | 114 ++++++++++ product-mini/platforms/alios-things/aos.mk | 1 + product-mini/platforms/nuttx/wamr.mk | 1 + samples/shared-module/.gitignore | 1 + samples/shared-module/CMakeLists.txt | 95 ++++++++ samples/shared-module/README.md | 5 + samples/shared-module/build.sh | 63 ++++++ samples/shared-module/run.sh | 3 + samples/shared-module/src/main.c | 206 ++++++++++++++++++ samples/shared-module/wasm-apps/testapp.wat | 22 ++ 26 files changed, 745 insertions(+), 72 deletions(-) create mode 100644 core/shared/utils/bh_bitmap.c create mode 100644 core/shared/utils/bh_bitmap.h create mode 100644 samples/shared-module/.gitignore create mode 100644 samples/shared-module/CMakeLists.txt create mode 100644 samples/shared-module/README.md create mode 100755 samples/shared-module/build.sh create mode 100755 samples/shared-module/run.sh create mode 100644 samples/shared-module/src/main.c create mode 100644 samples/shared-module/wasm-apps/testapp.wat diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 6b653b501..98d346f9e 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -444,6 +444,12 @@ jobs: cmake --build . --config Release --parallel 4 ./iwasm wasm-apps/no_pthread.wasm + - name: Build Sample [shared-module] + run: | + cd samples/shared-module + ./build.sh + ./run.sh + test: needs: [ diff --git a/.github/workflows/compilation_on_macos.yml b/.github/workflows/compilation_on_macos.yml index aac16898b..12f1d73cb 100644 --- a/.github/workflows/compilation_on_macos.yml +++ b/.github/workflows/compilation_on_macos.yml @@ -327,3 +327,9 @@ jobs: cmake .. cmake --build . --config Release --parallel 4 ./iwasm wasm-apps/no_pthread.wasm + + - name: Build Sample [shared-module] + run: | + cd samples/shared-module + ./build.sh + ./run.sh diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index 7b85f2cda..8153cea99 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -501,6 +501,12 @@ jobs: cmake .. cmake --build . --config Release --parallel 4 ./iwasm wasm-apps/no_pthread.wasm + + - name: Build Sample [shared-module] + run: | + cd samples/shared-module + ./build.sh + ./run.sh test: needs: [ diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 37084ba8e..abffd6438 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -1095,7 +1095,6 @@ load_table_init_data_list(const uint8 **p_buf, const uint8 *buf_end, data_list[i]->mode = mode; data_list[i]->elem_type = elem_type; - data_list[i]->is_dropped = false; data_list[i]->table_index = table_index; data_list[i]->offset.init_expr_type = (uint8)init_expr_type; data_list[i]->offset.u.i64 = (int64)init_expr_value; diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index f747cb43f..9912f63f0 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1098,6 +1098,9 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, char *error_buf, uint32 error_buf_size) { AOTModuleInstance *module_inst; +#if WASM_ENABLE_BULK_MEMORY != 0 || WASM_ENABLE_REF_TYPES != 0 + WASMModuleInstanceExtraCommon *common; +#endif const uint32 module_inst_struct_size = offsetof(AOTModuleInstance, global_table_data.bytes); const uint64 module_inst_mem_inst_size = @@ -1164,6 +1167,32 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, } #endif +#if WASM_ENABLE_BULK_MEMORY != 0 || WASM_ENABLE_REF_TYPES != 0 + common = &((AOTModuleInstanceExtra *)module_inst->e)->common; +#endif +#if WASM_ENABLE_BULK_MEMORY != 0 + if (module->mem_init_data_count > 0) { + common->data_dropped = bh_bitmap_new(0, module->mem_init_data_count); + if (common->data_dropped == NULL) { + LOG_DEBUG("failed to allocate bitmaps"); + set_error_buf(error_buf, error_buf_size, + "failed to allocate bitmaps"); + goto fail; + } + } +#endif +#if WASM_ENABLE_REF_TYPES != 0 + if (module->table_init_data_count > 0) { + common->elem_dropped = bh_bitmap_new(0, module->table_init_data_count); + if (common->elem_dropped == NULL) { + LOG_DEBUG("failed to allocate bitmaps"); + set_error_buf(error_buf, error_buf_size, + "failed to allocate bitmaps"); + goto fail; + } + } +#endif + /* Initialize global info */ p = (uint8 *)module_inst + module_inst_struct_size + module_inst_mem_inst_size; @@ -1264,6 +1293,8 @@ fail: void aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) { + WASMModuleInstanceExtraCommon *common = + &((AOTModuleInstanceExtra *)module_inst->e)->common; if (module_inst->exec_env_singleton) { /* wasm_exec_env_destroy will call wasm_cluster_wait_for_all_except_self to wait for other @@ -1306,7 +1337,7 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) if (module_inst->func_type_indexes) wasm_runtime_free(module_inst->func_type_indexes); - if (((AOTModuleInstanceExtra *)module_inst->e)->common.c_api_func_imports) + if (common->c_api_func_imports) wasm_runtime_free(((AOTModuleInstanceExtra *)module_inst->e) ->common.c_api_func_imports); @@ -1317,6 +1348,13 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) wasm_native_call_context_dtors((WASMModuleInstanceCommon *)module_inst); } +#if WASM_ENABLE_BULK_MEMORY != 0 + bh_bitmap_delete(common->data_dropped); +#endif +#if WASM_ENABLE_REF_TYPES != 0 + bh_bitmap_delete(common->elem_dropped); +#endif + wasm_runtime_free(module_inst); } @@ -2302,13 +2340,21 @@ aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, uint32 offset, { AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); AOTModule *aot_module; - uint8 *data = NULL; + uint8 *data; uint8 *maddr; - uint64 seg_len = 0; + uint64 seg_len; - aot_module = (AOTModule *)module_inst->module; - seg_len = aot_module->mem_init_data_list[seg_index]->byte_count; - data = aot_module->mem_init_data_list[seg_index]->bytes; + if (bh_bitmap_get_bit( + ((AOTModuleInstanceExtra *)module_inst->e)->common.data_dropped, + seg_index)) { + seg_len = 0; + data = NULL; + } + else { + aot_module = (AOTModule *)module_inst->module; + seg_len = aot_module->mem_init_data_list[seg_index]->byte_count; + data = aot_module->mem_init_data_list[seg_index]->bytes; + } if (!wasm_runtime_validate_app_addr((WASMModuleInstanceCommon *)module_inst, dst, len)) @@ -2331,9 +2377,9 @@ aot_memory_init(AOTModuleInstance *module_inst, uint32 seg_index, uint32 offset, bool aot_data_drop(AOTModuleInstance *module_inst, uint32 seg_index) { - AOTModule *aot_module = (AOTModule *)module_inst->module; - - aot_module->mem_init_data_list[seg_index]->byte_count = 0; + bh_bitmap_set_bit( + ((AOTModuleInstanceExtra *)module_inst->e)->common.data_dropped, + seg_index); /* Currently we can't free the dropped data segment as the mem_init_data_count is a continuous array */ return true; @@ -2546,9 +2592,9 @@ aot_get_module_inst_mem_consumption(const AOTModuleInstance *module_inst, void aot_drop_table_seg(AOTModuleInstance *module_inst, uint32 tbl_seg_idx) { - AOTModule *module = (AOTModule *)module_inst->module; - AOTTableInitData *tbl_seg = module->table_init_data_list[tbl_seg_idx]; - tbl_seg->is_dropped = true; + bh_bitmap_set_bit( + ((AOTModuleInstanceExtra *)module_inst->e)->common.elem_dropped, + tbl_seg_idx); } void @@ -2576,7 +2622,9 @@ aot_table_init(AOTModuleInstance *module_inst, uint32 tbl_idx, return; } - if (tbl_seg->is_dropped) { + if (bh_bitmap_get_bit( + ((AOTModuleInstanceExtra *)module_inst->e)->common.elem_dropped, + tbl_seg_idx)) { aot_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); return; } diff --git a/core/iwasm/compilation/aot.c b/core/iwasm/compilation/aot.c index e836df28f..8162d006e 100644 --- a/core/iwasm/compilation/aot.c +++ b/core/iwasm/compilation/aot.c @@ -129,7 +129,6 @@ aot_create_table_init_data_list(const WASMModule *module) data_list[i]->mode = module->table_segments[i].mode; data_list[i]->elem_type = module->table_segments[i].elem_type; /* runtime control it */ - data_list[i]->is_dropped = false; data_list[i]->table_index = module->table_segments[i].table_index; bh_memcpy_s(&data_list[i]->offset, sizeof(AOTInitExpr), &module->table_segments[i].base_offset, diff --git a/core/iwasm/compilation/aot.h b/core/iwasm/compilation/aot.h index 088460636..4bee70f9c 100644 --- a/core/iwasm/compilation/aot.h +++ b/core/iwasm/compilation/aot.h @@ -143,7 +143,6 @@ typedef struct AOTTableInitData { uint32 mode; /* funcref or externref, elemkind will be considered as funcref */ uint32 elem_type; - bool is_dropped; /* optional, only for active */ uint32 table_index; /* Start address of init data */ diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index 103b2750e..80bcdc7db 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -269,7 +269,7 @@ static uint32 get_table_init_data_size(AOTTableInitData *table_init_data) { /* - * mode (4 bytes), elem_type (4 bytes), do not need is_dropped field + * mode (4 bytes), elem_type (4 bytes) * * table_index(4 bytes) + init expr type (4 bytes) + init expr value (8 * bytes) diff --git a/core/iwasm/fast-jit/fe/jit_emit_memory.c b/core/iwasm/fast-jit/fe/jit_emit_memory.c index 9635d4e57..ce3f77fae 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_memory.c +++ b/core/iwasm/fast-jit/fe/jit_emit_memory.c @@ -650,6 +650,7 @@ wasm_init_memory(WASMModuleInstance *inst, uint32 mem_idx, uint32 seg_idx, WASMDataSeg *data_segment; uint32 mem_size; uint8 *mem_addr, *data_addr; + uint32 seg_len; /* if d + n > the length of mem.data */ mem_inst = inst->memories[mem_idx]; @@ -659,13 +660,19 @@ wasm_init_memory(WASMModuleInstance *inst, uint32 mem_idx, uint32 seg_idx, /* if s + n > the length of data.data */ bh_assert(seg_idx < inst->module->data_seg_count); - data_segment = inst->module->data_segments[seg_idx]; - if (data_segment->data_length < data_offset - || data_segment->data_length - data_offset < len) + if (bh_bitmap_get_bit(inst->e->common.data_dropped, seg_idx)) { + seg_len = 0; + data_addr = NULL; + } + else { + data_segment = inst->module->data_segments[seg_idx]; + seg_len = data_segment->data_length; + data_addr = data_segment->data + data_offset; + } + if (seg_len < data_offset || seg_len - data_offset < len) goto out_of_bounds; mem_addr = mem_inst->memory_data + mem_offset; - data_addr = data_segment->data + data_offset; bh_memcpy_s(mem_addr, mem_size - mem_offset, data_addr, len); return 0; @@ -706,21 +713,22 @@ fail: return false; } +static void +wasm_data_drop(WASMModuleInstance *inst, uint32 seg_idx) +{ + bh_bitmap_set_bit(inst->e->common.data_dropped, seg_idx); +} + bool jit_compile_op_data_drop(JitCompContext *cc, uint32 seg_idx) { - JitReg module = get_module_reg(cc->jit_frame); - JitReg data_segments = jit_cc_new_reg_ptr(cc); - JitReg data_segment = jit_cc_new_reg_ptr(cc); + JitReg args[2] = { 0 }; - GEN_INSN(LDPTR, data_segments, module, - NEW_CONST(I32, offsetof(WASMModule, data_segments))); - GEN_INSN(LDPTR, data_segment, data_segments, - NEW_CONST(I32, seg_idx * sizeof(WASMDataSeg *))); - GEN_INSN(STI32, NEW_CONST(I32, 0), data_segment, - NEW_CONST(I32, offsetof(WASMDataSeg, data_length))); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, seg_idx); - return true; + return jit_emit_callnative(cc, wasm_data_drop, 0, args, + sizeof(args) / sizeof(args[0])); } static int diff --git a/core/iwasm/fast-jit/fe/jit_emit_table.c b/core/iwasm/fast-jit/fe/jit_emit_table.c index ea1b33883..b8ed6a1d5 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_table.c +++ b/core/iwasm/fast-jit/fe/jit_emit_table.c @@ -10,21 +10,22 @@ #include "../jit_frontend.h" #if WASM_ENABLE_REF_TYPES != 0 +static void +wasm_elem_drop(WASMModuleInstance *inst, uint32 tbl_seg_idx) +{ + bh_bitmap_set_bit(inst->e->common.elem_dropped, tbl_seg_idx); +} + bool jit_compile_op_elem_drop(JitCompContext *cc, uint32 tbl_seg_idx) { - JitReg module, tbl_segs; + JitReg args[2] = { 0 }; - module = get_module_reg(cc->jit_frame); + args[0] = get_module_inst_reg(cc->jit_frame); + args[1] = NEW_CONST(I32, tbl_seg_idx); - tbl_segs = jit_cc_new_reg_ptr(cc); - GEN_INSN(LDPTR, tbl_segs, module, - NEW_CONST(I32, offsetof(WASMModule, table_segments))); - - GEN_INSN(STI32, NEW_CONST(I32, true), tbl_segs, - NEW_CONST(I32, tbl_seg_idx * sizeof(WASMTableSeg) - + offsetof(WASMTableSeg, is_dropped))); - return true; + return jit_emit_callnative(cc, wasm_elem_drop, 0, args, + sizeof(args) / sizeof(args[0])); } bool @@ -105,6 +106,12 @@ wasm_init_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 elem_idx, if (offset_len_out_of_bounds(dst_offset, len, tbl_sz)) goto out_of_bounds; + if (!len) + return 0; + + if (bh_bitmap_get_bit(inst->e->common.elem_dropped, elem_idx)) + goto out_of_bounds; + bh_memcpy_s((uint8 *)tbl + offsetof(WASMTableInstance, elems) + dst_offset * sizeof(uint32), (uint32)((tbl_sz - dst_offset) * sizeof(uint32)), diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index 89b0bc741..ee537aa63 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -311,7 +311,6 @@ typedef struct WASMTableSeg { uint32 mode; /* funcref or externref, elemkind will be considered as funcref */ uint32 elem_type; - bool is_dropped; /* optional, only for active */ uint32 table_index; InitializerExpression base_offset; diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index ec28e7baf..72a81ea84 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -3160,9 +3160,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, maddr = memory->memory_data + (uint32)addr; #endif - seg_len = (uint64)module->module->data_segments[segment] - ->data_length; - data = module->module->data_segments[segment]->data; + if (bh_bitmap_get_bit(module->e->common.data_dropped, + segment)) { + seg_len = 0; + data = NULL; + } + else { + seg_len = + (uint64)module->module->data_segments[segment] + ->data_length; + data = module->module->data_segments[segment]->data; + } if (offset + bytes > seg_len) goto out_of_bounds; @@ -3175,7 +3183,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint32 segment; read_leb_uint32(frame_ip, frame_ip_end, segment); - module->module->data_segments[segment]->data_length = 0; + bh_bitmap_set_bit(module->e->common.data_dropped, + segment); break; } case WASM_OP_MEMORY_COPY: @@ -3270,8 +3279,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, break; } - if (module->module->table_segments[elem_idx] - .is_dropped) { + if (bh_bitmap_get_bit(module->e->common.elem_dropped, + elem_idx)) { wasm_set_exception(module, "out of bounds table access"); goto got_exception; @@ -3303,8 +3312,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, read_leb_uint32(frame_ip, frame_ip_end, elem_idx); bh_assert(elem_idx < module->module->table_seg_count); - module->module->table_segments[elem_idx].is_dropped = - true; + bh_bitmap_set_bit(module->e->common.elem_dropped, + elem_idx); break; } case WASM_OP_TABLE_COPY: diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 82b6e747a..b0d0c6004 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -3005,10 +3005,18 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto out_of_bounds; maddr = memory->memory_data + (uint32)addr; #endif + if (bh_bitmap_get_bit(module->e->common.data_dropped, + segment)) { + seg_len = 0; + data = NULL; + } + else { - seg_len = (uint64)module->module->data_segments[segment] - ->data_length; - data = module->module->data_segments[segment]->data; + seg_len = + (uint64)module->module->data_segments[segment] + ->data_length; + data = module->module->data_segments[segment]->data; + } if (offset + bytes > seg_len) goto out_of_bounds; @@ -3021,8 +3029,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint32 segment; segment = read_uint32(frame_ip); - - module->module->data_segments[segment]->data_length = 0; + bh_bitmap_set_bit(module->e->common.data_dropped, + segment); break; } case WASM_OP_MEMORY_COPY: @@ -3114,8 +3122,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, break; } - if (module->module->table_segments[elem_idx] - .is_dropped) { + if (bh_bitmap_get_bit(module->e->common.elem_dropped, + elem_idx)) { wasm_set_exception(module, "out of bounds table access"); goto got_exception; @@ -3144,9 +3152,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, { uint32 elem_idx = read_uint32(frame_ip); bh_assert(elem_idx < module->module->table_seg_count); - - module->module->table_segments[elem_idx].is_dropped = - true; + bh_bitmap_set_bit(module->e->common.elem_dropped, + elem_idx); break; } case WASM_OP_TABLE_COPY: diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 720a78f03..9efeb14c0 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -1666,6 +1666,31 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, } #endif +#if WASM_ENABLE_BULK_MEMORY != 0 + if (module->data_seg_count > 0) { + module_inst->e->common.data_dropped = + bh_bitmap_new(0, module->data_seg_count); + if (module_inst->e->common.data_dropped == NULL) { + LOG_DEBUG("failed to allocate bitmaps"); + set_error_buf(error_buf, error_buf_size, + "failed to allocate bitmaps"); + goto fail; + } + } +#endif +#if WASM_ENABLE_REF_TYPES != 0 + if (module->table_seg_count > 0) { + module_inst->e->common.elem_dropped = + bh_bitmap_new(0, module->table_seg_count); + if (module_inst->e->common.elem_dropped == NULL) { + LOG_DEBUG("failed to allocate bitmaps"); + set_error_buf(error_buf, error_buf_size, + "failed to allocate bitmaps"); + goto fail; + } + } +#endif + #if WASM_ENABLE_DUMP_CALL_STACK != 0 if (!(module_inst->frames = runtime_malloc((uint64)sizeof(Vector), error_buf, error_buf_size))) { @@ -2189,6 +2214,13 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst) wasm_native_call_context_dtors((WASMModuleInstanceCommon *)module_inst); } +#if WASM_ENABLE_BULK_MEMORY != 0 + bh_bitmap_delete(module_inst->e->common.data_dropped); +#endif +#if WASM_ENABLE_REF_TYPES != 0 + bh_bitmap_delete(module_inst->e->common.elem_dropped); +#endif + wasm_runtime_free(module_inst); } @@ -3148,16 +3180,23 @@ llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index, { WASMMemoryInstance *memory_inst; WASMModule *module; - uint8 *data = NULL; + uint8 *data; uint8 *maddr; - uint64 seg_len = 0; + uint64 seg_len; bh_assert(module_inst->module_type == Wasm_Module_Bytecode); memory_inst = wasm_get_default_memory(module_inst); - module = module_inst->module; - seg_len = module->data_segments[seg_index]->data_length; - data = module->data_segments[seg_index]->data; + + if (bh_bitmap_get_bit(module_inst->e->common.data_dropped, seg_index)) { + seg_len = 0; + data = NULL; + } + else { + module = module_inst->module; + seg_len = module->data_segments[seg_index]->data_length; + data = module->data_segments[seg_index]->data; + } if (!wasm_runtime_validate_app_addr((WASMModuleInstanceCommon *)module_inst, dst, len)) @@ -3182,7 +3221,7 @@ llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index) { bh_assert(module_inst->module_type == Wasm_Module_Bytecode); - module_inst->module->data_segments[seg_index]->data_length = 0; + bh_bitmap_set_bit(module_inst->e->common.data_dropped, seg_index); /* Currently we can't free the dropped data segment as they are stored in wasm bytecode */ return true; @@ -3193,12 +3232,8 @@ llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index) void llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx) { - WASMTableSeg *tbl_segs; - bh_assert(module_inst->module_type == Wasm_Module_Bytecode); - - tbl_segs = module_inst->module->table_segments; - tbl_segs[tbl_seg_idx].is_dropped = true; + bh_bitmap_set_bit(module_inst->e->common.elem_dropped, tbl_seg_idx); } void @@ -3227,7 +3262,7 @@ llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx, return; } - if (tbl_seg->is_dropped) { + if (bh_bitmap_get_bit(module_inst->e->common.elem_dropped, tbl_seg_idx)) { jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS); return; } diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 7d78df14f..1bd51551f 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -8,6 +8,7 @@ #include "wasm.h" #include "bh_atomic.h" +#include "bh_bitmap.h" #include "bh_hashmap.h" #include "../common/wasm_runtime_common.h" #include "../common/wasm_exec_env.h" @@ -223,6 +224,12 @@ typedef struct WASMModuleInstanceExtraCommon { /* Disable bounds checks or not */ bool disable_bounds_checks; #endif +#if WASM_ENABLE_BULK_MEMORY != 0 + bh_bitmap *data_dropped; +#endif +#if WASM_ENABLE_REF_TYPES != 0 + bh_bitmap *elem_dropped; +#endif } WASMModuleInstanceExtraCommon; /* Extra info of WASM module instance for interpreter/jit mode */ diff --git a/core/shared/utils/bh_bitmap.c b/core/shared/utils/bh_bitmap.c new file mode 100644 index 000000000..2ee918049 --- /dev/null +++ b/core/shared/utils/bh_bitmap.c @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "bh_bitmap.h" + +bh_bitmap * +bh_bitmap_new(uintptr_t begin_index, unsigned bitnum) +{ + bh_bitmap *bitmap; + uint32 bitmap_size = (bitnum + 7) / 8; + uint32 total_size = offsetof(bh_bitmap, map) + bitmap_size; + + if (bitnum > UINT32_MAX - 7 || total_size < offsetof(bh_bitmap, map) + || (total_size - offsetof(bh_bitmap, map)) != bitmap_size) { + return NULL; /* integer overflow */ + } + + if ((bitmap = BH_MALLOC(total_size)) != NULL) { + memset(bitmap, 0, total_size); + bitmap->begin_index = begin_index; + bitmap->end_index = begin_index + bitnum; + } + + return bitmap; +} diff --git a/core/shared/utils/bh_bitmap.h b/core/shared/utils/bh_bitmap.h new file mode 100644 index 000000000..fe7251480 --- /dev/null +++ b/core/shared/utils/bh_bitmap.h @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _BH_BITMAP_H +#define _BH_BITMAP_H + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * A simple fixed size bitmap. + */ +typedef struct bh_bitmap { + /* The first valid bit index. */ + uintptr_t begin_index; + + /* The last valid bit index plus one. */ + uintptr_t end_index; + + /* The bitmap. */ + uint8 map[1]; +} bh_bitmap; + +/** + * Create a new bitmap. + * + * @param begin_index the first valid bit index + * @param bitnum maximal bit number of the bitmap. + * + * @return the new bitmap if succeeds, NULL otherwise. + */ +bh_bitmap * +bh_bitmap_new(uintptr_t begin_index, unsigned bitnum); + +/** + * Delete a bitmap. + * + * @param bitmap the bitmap to be deleted + */ +static inline void +bh_bitmap_delete(bh_bitmap *bitmap) +{ + if (bitmap != NULL) + BH_FREE(bitmap); +} + +/** + * Check whether the given index is in the range of the bitmap. + * + * @param bitmap the bitmap + * @param n the bit index + * + * @return true if the index is in range, false otherwise + */ +static inline bool +bh_bitmap_is_in_range(bh_bitmap *bitmap, unsigned n) +{ + return n >= bitmap->begin_index && n < bitmap->end_index; +} + +/** + * Get a bit in the bitmap + * + * @param bitmap the bitmap + * @param n the n-th bit to be get + * + * @return value of the bit + */ +static inline int +bh_bitmap_get_bit(bh_bitmap *bitmap, unsigned n) +{ + unsigned idx = n - bitmap->begin_index; + bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); + return (bitmap->map[idx / 8] >> (idx % 8)) & 1; +} + +/** + * Set a bit in the bitmap. + * + * @param bitmap the bitmap + * @param n the n-th bit to be set + */ +static inline void +bh_bitmap_set_bit(bh_bitmap *bitmap, unsigned n) +{ + unsigned idx = n - bitmap->begin_index; + bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); + bitmap->map[idx / 8] |= 1 << (idx % 8); +} + +/** + * Clear a bit in the bitmap. + * + * @param bitmap the bitmap + * @param n the n-th bit to be cleared + */ +static inline void +bh_bitmap_clear_bit(bh_bitmap *bitmap, unsigned n) +{ + unsigned idx = n - bitmap->begin_index; + bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); + bitmap->map[idx / 8] &= ~(1 << (idx % 8)); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/product-mini/platforms/alios-things/aos.mk b/product-mini/platforms/alios-things/aos.mk index 383e0b239..3f25cb980 100644 --- a/product-mini/platforms/alios-things/aos.mk +++ b/product-mini/platforms/alios-things/aos.mk @@ -98,6 +98,7 @@ $(NAME)_SOURCES := ${SHARED_ROOT}/platform/alios/alios_platform.c \ ${SHARED_ROOT}/mem-alloc/ems/ems_alloc.c \ ${SHARED_ROOT}/mem-alloc/ems/ems_hmu.c \ ${SHARED_ROOT}/utils/bh_assert.c \ + ${SHARED_ROOT}/utils/bh_bitmap.c \ ${SHARED_ROOT}/utils/bh_common.c \ ${SHARED_ROOT}/utils/bh_hashmap.c \ ${SHARED_ROOT}/utils/bh_list.c \ diff --git a/product-mini/platforms/nuttx/wamr.mk b/product-mini/platforms/nuttx/wamr.mk index ef90f14da..8bd05f48d 100644 --- a/product-mini/platforms/nuttx/wamr.mk +++ b/product-mini/platforms/nuttx/wamr.mk @@ -371,6 +371,7 @@ CSRCS += nuttx_platform.c \ ems_alloc.c \ ems_hmu.c \ bh_assert.c \ + bh_bitmap.c \ bh_common.c \ bh_hashmap.c \ bh_list.c \ diff --git a/samples/shared-module/.gitignore b/samples/shared-module/.gitignore new file mode 100644 index 000000000..0fa8a76bd --- /dev/null +++ b/samples/shared-module/.gitignore @@ -0,0 +1 @@ +/out/ \ No newline at end of file diff --git a/samples/shared-module/CMakeLists.txt b/samples/shared-module/CMakeLists.txt new file mode 100644 index 000000000..c094b071c --- /dev/null +++ b/samples/shared-module/CMakeLists.txt @@ -0,0 +1,95 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +include(CheckPIESupported) + +project (shared-module) + +set (CMAKE_CXX_STANDARD 17) + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +if (APPLE) + add_definitions(-DBH_PLATFORM_DARWIN) +endif () + +# Reset default linker flags +set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") +set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + +# WAMR features switch + +# Set WAMR_BUILD_TARGET, currently values supported: +# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", +# "MIPS", "XTENSA", "RISCV64[sub]", "RISCV32[sub]" +if (NOT DEFINED WAMR_BUILD_TARGET) + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)") + set (WAMR_BUILD_TARGET "AARCH64") + elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64") + set (WAMR_BUILD_TARGET "RISCV64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 8) + # Build as X86_64 by default in 64-bit platform + set (WAMR_BUILD_TARGET "X86_64") + elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) + # Build as X86_32 by default in 32-bit platform + set (WAMR_BUILD_TARGET "X86_32") + else () + message(SEND_ERROR "Unsupported build target platform!") + endif () +endif () + +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Debug) +endif () + +set (WAMR_BUILD_INTERP 1) +set (WAMR_BUILD_AOT 1) +set (WAMR_BUILD_JIT 0) + +# fast interpreter +# set (WAMR_BUILD_FAST_INTERP 1) + +# fast-jit +# set (WAMR_BUILD_FAST_JIT 1) + +# llvm jit +# set (WAMR_BUILD_JIT 1) +# set (LLVM_DIR /usr/local/opt/llvm@14/lib/cmake/llvm) + +set (WAMR_BUILD_REF_TYPES 1) + +if (NOT MSVC) + # linker flags + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections") + endif () + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security") + if (WAMR_BUILD_TARGET MATCHES "X86_.*" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mindirect-branch-register") + endif () + endif () +endif () + +# build out vmlib +set (WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) + +################ application related ################ +include_directories(${CMAKE_CURRENT_LIST_DIR}/src) +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + +add_executable (shared-module src/main.c ${UNCOMMON_SHARED_SOURCE}) + +check_pie_supported() +set_target_properties (shared-module PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (APPLE) + target_link_libraries (shared-module vmlib -lm -ldl -lpthread ${LLVM_AVAILABLE_LIBS}) +else () + target_link_libraries (shared-module vmlib -lm -ldl -lpthread -lrt ${LLVM_AVAILABLE_LIBS}) +endif () diff --git a/samples/shared-module/README.md b/samples/shared-module/README.md new file mode 100644 index 000000000..14baa8cbf --- /dev/null +++ b/samples/shared-module/README.md @@ -0,0 +1,5 @@ +The "shared-module" sample project +================================== + +This sample demonstrates a bug described in: +https://github.com/bytecodealliance/wasm-micro-runtime/issues/2735. diff --git a/samples/shared-module/build.sh b/samples/shared-module/build.sh new file mode 100755 index 000000000..9af5b3edd --- /dev/null +++ b/samples/shared-module/build.sh @@ -0,0 +1,63 @@ +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#!/bin/bash + +CURR_DIR=$PWD +WAMR_DIR=${PWD}/../.. +OUT_DIR=${PWD}/out + +WASM_APPS=${PWD}/wasm-apps + + +rm -rf ${OUT_DIR} +mkdir ${OUT_DIR} +mkdir ${OUT_DIR}/wasm-apps + + +echo "##################### build shared-module project" +cd ${CURR_DIR} +mkdir -p cmake_build +cd cmake_build +cmake .. +make -j ${nproc} +if [ $? != 0 ];then + echo "BUILD_FAIL shared-module exit as $?\n" + exit 2 +fi + +cp -a shared-module ${OUT_DIR} + +printf "\n" + +echo "##################### build wasm apps" + +cd ${WASM_APPS} + +for i in `ls *.wat` +do +APP_SRC="$i" +OUT_FILE=${i%.*}.wasm + +# Note: the CI installs wabt in /opt/wabt +if type wat2wasm; then + WAT2WASM=${WAT2WASM:-wat2wasm} +elif [ -x /opt/wabt/bin/wat2wasm ]; then + WAT2WASM=${WAT2WASM:-/opt/wabt/bin/wat2wasm} +fi + +${WAT2WASM} -o ${OUT_DIR}/wasm-apps/${OUT_FILE} ${APP_SRC} + +# aot +# wamrc -o ${OUT_DIR}/wasm-apps/${OUT_FILE}.aot ${OUT_DIR}/wasm-apps/${OUT_FILE} +# mv ${OUT_DIR}/wasm-apps/${OUT_FILE}.aot ${OUT_DIR}/wasm-apps/${OUT_FILE} + +if [ -f ${OUT_DIR}/wasm-apps/${OUT_FILE} ]; then + echo "build ${OUT_FILE} success" +else + echo "build ${OUT_FILE} fail" +fi +done +echo "##################### build wasm apps done" diff --git a/samples/shared-module/run.sh b/samples/shared-module/run.sh new file mode 100755 index 000000000..3cb2e623e --- /dev/null +++ b/samples/shared-module/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +out/shared-module -f out/wasm-apps/testapp.wasm diff --git a/samples/shared-module/src/main.c b/samples/shared-module/src/main.c new file mode 100644 index 000000000..ebea0c6bf --- /dev/null +++ b/samples/shared-module/src/main.c @@ -0,0 +1,206 @@ + +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_read_file.h" +#include "bh_getopt.h" + +void +print_usage(void) +{ + fprintf(stdout, "Options:\r\n"); + fprintf(stdout, " -f [path of wasm file] \n"); +} + +int +main(int argc, char *argv_main[]) +{ + int exit_code = 1; + static char global_heap_buf[512 * 1024]; + char *buffer; + char error_buf[128]; + int opt; + char *wasm_path = NULL; + + const unsigned int N = 4; + wasm_module_t module = NULL; + wasm_module_inst_t module_inst[N]; + wasm_exec_env_t exec_env[N]; + const char *name_test_data_drop = "test_data_drop"; + const char *name_test_elem_drop = "test_elem_drop"; + wasm_function_inst_t func_test_data_drop[N]; + wasm_function_inst_t func_test_elem_drop[N]; + unsigned int i; + unsigned int iter; + uint32 buf_size, stack_size = 8092, heap_size = 8092; + + for (i = 0; i < N; i++) { + module_inst[i] = NULL; + exec_env[i] = NULL; + func_test_data_drop[i] = NULL; + func_test_elem_drop[i] = NULL; + } + + RuntimeInitArgs init_args; + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + while ((opt = getopt(argc, argv_main, "hf:")) != -1) { + switch (opt) { + case 'f': + wasm_path = optarg; + break; + case 'h': + print_usage(); + return 0; + case '?': + print_usage(); + return 0; + } + } + if (optind == 1) { + print_usage(); + return 0; + } + + memset(&init_args, 0, sizeof(init_args)); + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + buffer = bh_read_file_to_buffer(wasm_path, &buf_size); + + if (!buffer) { + printf("Open wasm app file [%s] failed.\n", wasm_path); + goto fail; + } + + module = wasm_runtime_load((uint8 *)buffer, buf_size, error_buf, + sizeof(error_buf)); + if (!module) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + for (i = 0; i < N; i++) { + module_inst[i] = wasm_runtime_instantiate(module, stack_size, heap_size, + error_buf, sizeof(error_buf)); + + if (!module_inst[i]) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + exec_env[i] = wasm_runtime_create_exec_env(module_inst[i], stack_size); + if (!exec_env[i]) { + printf("Create wasm execution environment failed.\n"); + goto fail; + } + + func_test_data_drop[i] = wasm_runtime_lookup_function( + module_inst[i], name_test_data_drop, NULL); + if (!func_test_data_drop[i]) { + printf("The wasm function %s is not found.\n", name_test_data_drop); + goto fail; + } + + func_test_elem_drop[i] = wasm_runtime_lookup_function( + module_inst[i], name_test_elem_drop, NULL); + if (!func_test_elem_drop[i]) { + printf("The wasm function %s is not found.\n", name_test_elem_drop); + goto fail; + } + } + + for (iter = 0; iter < 2; iter++) { + /* + * as we drop data/table in the first iteration, + * the later iterations should trap. + */ + const bool should_trap = iter > 0; + + for (i = 0; i < N; i++) { + uint32_t argv[1] = {}; + if (wasm_runtime_call_wasm(exec_env[i], func_test_data_drop[i], 0, + argv)) { + uint32_t result = argv[0]; + printf( + "Native finished calling wasm function: %s, return: %x\n", + name_test_data_drop, result); + if (result != 0x64636261) { /* "abcd" */ + printf("unexpected return value\n"); + goto fail; + } + if (should_trap) { + printf("a trap is expected\n"); + goto fail; + } + } + else if (should_trap) { + printf("call wasm function %s failed as expected. error: %s\n", + name_test_data_drop, + wasm_runtime_get_exception(module_inst[i])); + } + else { + printf("call wasm function %s failed. error: %s\n", + name_test_data_drop, + wasm_runtime_get_exception(module_inst[i])); + goto fail; + } + } + + for (i = 0; i < N; i++) { + wasm_runtime_clear_exception(module_inst[i]); + + uint32_t argv[1] = {}; + if (wasm_runtime_call_wasm(exec_env[i], func_test_elem_drop[i], 0, + argv)) { + uint32_t result = argv[0]; + printf( + "Native finished calling wasm function: %s, return: %x\n", + name_test_elem_drop, result); + if (result != 0) { + printf("unexpected return value\n"); + goto fail; + } + if (should_trap) { + printf("a trap is expected\n"); + goto fail; + } + } + else if (should_trap) { + printf("call wasm function %s failed as expected. error: %s\n", + name_test_elem_drop, + wasm_runtime_get_exception(module_inst[i])); + } + else { + printf("call wasm function %s failed. error: %s\n", + name_test_elem_drop, + wasm_runtime_get_exception(module_inst[i])); + goto fail; + } + } + } + + exit_code = 0; +fail: + for (i = 0; i < N; i++) { + if (exec_env[i]) + wasm_runtime_destroy_exec_env(exec_env[i]); + if (module_inst[i]) + wasm_runtime_deinstantiate(module_inst[i]); + } + if (module) + wasm_runtime_unload(module); + if (buffer) + BH_FREE(buffer); + wasm_runtime_destroy(); + return exit_code; +} diff --git a/samples/shared-module/wasm-apps/testapp.wat b/samples/shared-module/wasm-apps/testapp.wat new file mode 100644 index 000000000..263a87036 --- /dev/null +++ b/samples/shared-module/wasm-apps/testapp.wat @@ -0,0 +1,22 @@ +;; Copyright (C) 2023 Midokura Japan KK. All rights reserved. +;; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +(module + (func (export "test_data_drop") (result i32) + (memory.init 0 (i32.const 0) (i32.const 0) (i32.const 4)) + data.drop 0 + (i32.load (i32.const 0)) + ) + (func (export "test_elem_drop") (result i32) + (table.init 0 (i32.const 0) (i32.const 0) (i32.const 4)) + elem.drop 0 + i32.const 3 + table.get 0 + ref.is_null + ) + (func $f) + (memory 1 1) + (table 4 4 funcref) + (data "abcd") + (elem func $f $f $f $f) +) From be05af79fb8d9a6f88e8ad384675df9cfad23069 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Sat, 18 Nov 2023 08:59:58 +0800 Subject: [PATCH 08/67] Refine atomic operation flags in bh_atomic.h (#2780) --- core/shared/utils/bh_atomic.h | 104 +++++++++++++++++++++++----------- 1 file changed, 71 insertions(+), 33 deletions(-) diff --git a/core/shared/utils/bh_atomic.h b/core/shared/utils/bh_atomic.h index ac3417c30..5744a64c0 100644 --- a/core/shared/utils/bh_atomic.h +++ b/core/shared/utils/bh_atomic.h @@ -47,6 +47,19 @@ extern "C" { typedef uint32 bh_atomic_32_t; typedef uint16 bh_atomic_16_t; +/* The flag can be defined by the user if the platform + * supports atomic 32-bit operations. + * If left undefined, it will be automatically defined + * according to the platform. + */ +#ifdef WASM_UINT32_IS_ATOMIC +#define BH_ATOMIC_32_IS_ATOMIC WASM_UINT32_IS_ATOMIC +#endif /* WASM_UINT32_IS_ATOMIC */ + +#ifdef WASM_UINT16_IS_ATOMIC +#define BH_ATOMIC_16_IS_ATOMIC WASM_UINT16_IS_ATOMIC +#endif /* WASM_UINT16_IS_ATOMIC */ + #if defined(__GNUC_PREREQ) #if __GNUC_PREREQ(4, 7) #define CLANG_GCC_HAS_ATOMIC_BUILTIN @@ -58,7 +71,38 @@ typedef uint16 bh_atomic_16_t; #endif #if defined(CLANG_GCC_HAS_ATOMIC_BUILTIN) +#ifndef BH_ATOMIC_32_IS_ATOMIC #define BH_ATOMIC_32_IS_ATOMIC 1 +#endif +#ifndef BH_ATOMIC_16_IS_ATOMIC +#define BH_ATOMIC_16_IS_ATOMIC 1 +#endif +#else +#ifndef BH_ATOMIC_32_IS_ATOMIC +#define BH_ATOMIC_32_IS_ATOMIC 0 +#endif +#ifndef BH_ATOMIC_16_IS_ATOMIC +#define BH_ATOMIC_16_IS_ATOMIC 0 +#endif +#endif + +/* Force disable atomic 16-bit operations on bare-metal RISC-V + * because the 16-bit atomic operations is emulated by 32-bit + * atomic operations, which has linkage problem on current toolchain: + * in function `shared_memory_inc_reference': + * wasm_shared_memory.c:85:(.text.shared_memory_inc_reference+0x10): undefined + * reference to `__atomic_fetch_add_2' + */ +#ifndef WASM_UINT16_IS_ATOMIC +#if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) \ + && !defined(__OpenBSD__) && defined(__riscv) +#undef BH_ATOMIC_16_IS_ATOMIC +#define BH_ATOMIC_16_IS_ATOMIC 0 +#endif +#endif + +#if BH_ATOMIC_32_IS_ATOMIC != 0 + #define BH_ATOMIC_32_LOAD(v) __atomic_load_n(&(v), __ATOMIC_SEQ_CST) #define BH_ATOMIC_32_STORE(v, val) __atomic_store_n(&(v), val, __ATOMIC_SEQ_CST) #define BH_ATOMIC_32_FETCH_OR(v, val) \ @@ -70,18 +114,8 @@ typedef uint16 bh_atomic_16_t; #define BH_ATOMIC_32_FETCH_SUB(v, val) \ __atomic_fetch_sub(&(v), (val), __ATOMIC_SEQ_CST) -#define BH_ATOMIC_16_IS_ATOMIC 1 -#define BH_ATOMIC_16_LOAD(v) __atomic_load_n(&(v), __ATOMIC_SEQ_CST) -#define BH_ATOMIC_16_STORE(v, val) __atomic_store_n(&(v), val, __ATOMIC_SEQ_CST) -#define BH_ATOMIC_16_FETCH_OR(v, val) \ - __atomic_fetch_or(&(v), (val), __ATOMIC_SEQ_CST) -#define BH_ATOMIC_16_FETCH_AND(v, val) \ - __atomic_fetch_and(&(v), (val), __ATOMIC_SEQ_CST) -#define BH_ATOMIC_16_FETCH_ADD(v, val) \ - __atomic_fetch_add(&(v), (val), __ATOMIC_SEQ_CST) -#define BH_ATOMIC_16_FETCH_SUB(v, val) \ - __atomic_fetch_sub(&(v), (val), __ATOMIC_SEQ_CST) -#else /* else of defined(CLANG_GCC_HAS_ATOMIC_BUILTIN) */ +#else /* else of BH_ATOMIC_32_IS_ATOMIC != 0 */ + #define BH_ATOMIC_32_LOAD(v) (v) #define BH_ATOMIC_32_STORE(v, val) (v) = val #define BH_ATOMIC_32_FETCH_OR(v, val) nonatomic_32_fetch_or(&(v), val) @@ -89,13 +123,6 @@ typedef uint16 bh_atomic_16_t; #define BH_ATOMIC_32_FETCH_ADD(v, val) nonatomic_32_fetch_add(&(v), val) #define BH_ATOMIC_32_FETCH_SUB(v, val) nonatomic_32_fetch_sub(&(v), val) -#define BH_ATOMIC_16_LOAD(v) (v) -#define BH_ATOMIC_16_STORE(v) (v) = val -#define BH_ATOMIC_16_FETCH_OR(v, val) nonatomic_16_fetch_or(&(v), val) -#define BH_ATOMIC_16_FETCH_AND(v, val) nonatomic_16_fetch_and(&(v), val) -#define BH_ATOMIC_16_FETCH_ADD(v, val) nonatomic_16_fetch_add(&(v), val) -#define BH_ATOMIC_16_FETCH_SUB(v, val) nonatomic_16_fetch_sub(&(v), val) - static inline uint32 nonatomic_32_fetch_or(bh_atomic_32_t *p, uint32 val) { @@ -128,6 +155,31 @@ nonatomic_32_fetch_sub(bh_atomic_32_t *p, uint32 val) return old; } +#endif + +#if BH_ATOMIC_16_IS_ATOMIC != 0 + +#define BH_ATOMIC_16_IS_ATOMIC 1 +#define BH_ATOMIC_16_LOAD(v) __atomic_load_n(&(v), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_STORE(v, val) __atomic_store_n(&(v), val, __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_FETCH_OR(v, val) \ + __atomic_fetch_or(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_FETCH_AND(v, val) \ + __atomic_fetch_and(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_FETCH_ADD(v, val) \ + __atomic_fetch_add(&(v), (val), __ATOMIC_SEQ_CST) +#define BH_ATOMIC_16_FETCH_SUB(v, val) \ + __atomic_fetch_sub(&(v), (val), __ATOMIC_SEQ_CST) + +#else /* else of BH_ATOMIC_16_IS_ATOMIC != 0 */ + +#define BH_ATOMIC_16_LOAD(v) (v) +#define BH_ATOMIC_16_STORE(v) (v) = val +#define BH_ATOMIC_16_FETCH_OR(v, val) nonatomic_16_fetch_or(&(v), val) +#define BH_ATOMIC_16_FETCH_AND(v, val) nonatomic_16_fetch_and(&(v), val) +#define BH_ATOMIC_16_FETCH_ADD(v, val) nonatomic_16_fetch_add(&(v), val) +#define BH_ATOMIC_16_FETCH_SUB(v, val) nonatomic_16_fetch_sub(&(v), val) + static inline uint16 nonatomic_16_fetch_or(bh_atomic_16_t *p, uint16 val) { @@ -160,20 +212,6 @@ nonatomic_16_fetch_sub(bh_atomic_16_t *p, uint16 val) return old; } -/* The flag can be defined by the user if the platform - supports atomic access to uint32 aligned memory. */ -#ifdef WASM_UINT32_IS_ATOMIC -#define BH_ATOMIC_32_IS_ATOMIC 1 -#else /* else of WASM_UINT32_IS_ATOMIC */ -#define BH_ATOMIC_32_IS_ATOMIC 0 -#endif /* WASM_UINT32_IS_ATOMIC */ - -#ifdef WASM_UINT16_IS_ATOMIC -#define BH_ATOMIC_16_IS_ATOMIC 1 -#else /* else of WASM_UINT16_IS_ATOMIC */ -#define BH_ATOMIC_16_IS_ATOMIC 0 -#endif /* WASM_UINT16_IS_ATOMIC */ - #endif #ifdef __cplusplus From 748e7450884129fa6dcaa2cfbf198ba8c181d84a Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Sat, 18 Nov 2023 09:09:30 +0800 Subject: [PATCH 09/67] Disable FPU in NuttX spec test (#2781) Fix spec test failure on NuttX with #2780, see: https://github.com/bytecodealliance/wasm-micro-runtime/actions/runs/6910450452 --- .github/workflows/spec_test_on_nuttx.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/spec_test_on_nuttx.yml b/.github/workflows/spec_test_on_nuttx.yml index 32efc35da..d7c4246fe 100644 --- a/.github/workflows/spec_test_on_nuttx.yml +++ b/.github/workflows/spec_test_on_nuttx.yml @@ -4,6 +4,12 @@ name: spec test on nuttx on: + pull_request: + types: + - closed + branches: + - main + schedule: - cron: '0 0 * * *' @@ -104,6 +110,11 @@ jobs: find nuttx/boards -name defconfig | xargs sed -i '$a\CONFIG_INTERPRETERS_WAMR=y\nCONFIG_INTERPRETERS_WAMR_STACKSIZE=32768\nCONFIG_INTERPRETERS_WAMR_AOT=y\nCONFIG_INTERPRETERS_WAMR_FAST=y\nCONFIG_INTERPRETERS_WAMR_LOG=y\nCONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN=y\nCONFIG_INTERPRETERS_WAMR_REF_TYPES=y\nCONFIG_INTERPRETERS_WAMR_ENABLE_SPEC_TEST=y\nCONFIG_INTERPRETERS_WAMR_SHARED_MEMORY=y\nCONFIG_INTERPRETERS_WAMR_BULK_MEMORY=y\n' find nuttx/boards -name defconfig | xargs sed -i '$a\CONFIG_EOL_IS_LF=y\nCONFIG_ARM_SEMIHOSTING_HOSTFS=y\nCONFIG_ARM_SEMIHOSTING_HOSTFS_CACHE_COHERENCE=y\nCONFIG_RISCV_SEMIHOSTING_HOSTFS=y\nCONFIG_FS_HOSTFS=y\nCONFIG_LIBC_FLOATINGPOINT=y\n' + - name: Enable additional features for NuttX on RI5C-V + if: startsWith(matrix.nuttx_board_config, 'boards/risc-v') + run: | + find nuttx/boards -name defconfig | xargs sed -i '$a\# CONFIG_ARCH_FPU is not set\n' + - name: Build wamrc working-directory: apps/interpreters/wamr/wamr/wamr-compiler run: | @@ -133,7 +144,7 @@ jobs: tar xvf xpack-qemu-riscv.tar.gz export PATH=$PATH:$PWD/xpack-qemu-riscv-7.1.0-1/bin cd apps/interpreters/wamr/wamr/tests/wamr-test-suites - ./test_wamr.sh -s spec ${{ matrix.wamr_test_option }} -m RISCV32 -b -Q -P -F ${{ env.firmware }} + ./test_wamr.sh -s spec ${{ matrix.wamr_test_option }} -m riscv32 -b -Q -P -F ${{ env.firmware }} - name: Test on RISCV64 if: endsWith(matrix.nuttx_board_config, 'rv-virt/configs/nsh64') From 657fd7bb42feecf8cbd1edd82dba827b07f12adf Mon Sep 17 00:00:00 2001 From: Daniel Mangum <31777345+hasheddan@users.noreply.github.com> Date: Sun, 19 Nov 2023 21:04:45 -0500 Subject: [PATCH 10/67] Fix broken links in app-mgr README.md (#2786) Fixes a few relative path broken links in app-mgr README.md. Signed-off-by: Daniel Mangum --- core/app-mgr/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/app-mgr/README.md b/core/app-mgr/README.md index 31c705e1f..7a561897e 100644 --- a/core/app-mgr/README.md +++ b/core/app-mgr/README.md @@ -1,8 +1,8 @@ # Remote application management -The WAMR application manager supports [remote application management](../core/app-mgr) from the host environment or the cloud through any physical communications such as TCP, UPD, UART, BLE, etc. Its modular design makes it able to support application management for different managed runtimes. +The WAMR application manager supports [remote application management](../../core/app-mgr) from the host environment or the cloud through any physical communications such as TCP, UPD, UART, BLE, etc. Its modular design makes it able to support application management for different managed runtimes. -The tool [host_tool](../test-tools/host-tool) communicates to the WAMR app manager for installing/uninstalling the WASM applications on companion chip from the host system. And the [IoT App Store Demo](../test-tools/IoT-APP-Store-Demo/) shows the conception of remotely managing the device applications from the cloud. +The tool [host_tool](../../test-tools/host-tool) communicates to the WAMR app manager for installing/uninstalling the WASM applications on companion chip from the host system. And the [IoT App Store Demo](../../test-tools/IoT-APP-Store-Demo/) shows the conception of remotely managing the device applications from the cloud. From 0bb157b88a42501cd665577c3284da1a128fbf7f Mon Sep 17 00:00:00 2001 From: Daniel Mangum <31777345+hasheddan@users.noreply.github.com> Date: Sun, 19 Nov 2023 21:25:00 -0500 Subject: [PATCH 11/67] Fix comment in WAMR_MEM_DUAL_BUS_MIRROR (#2791) Fixes a small typo in the WAMR_MEM_DUAL_BUS_MIRROR description. Signed-off-by: Daniel Mangum --- core/config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config.h b/core/config.h index 14065536a..acf70ff08 100644 --- a/core/config.h +++ b/core/config.h @@ -477,7 +477,7 @@ /* Some chip cannot support external ram with rwx attr at the same time, it has to map it into 2 spaces of idbus and dbus, code in dbus can be read/written and read/executed in ibus. so there are 2 steps to execute - the code, first, copy&do relocaiton in dbus space, and second execute + the code, first, copy & do relocation in dbus space, and second execute it in ibus space, since in the 2 spaces the contents are the same, so we call it bus mirror. */ From e8c8f7fca6f590845282bb9f5e38907acce4bb3a Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Mon, 20 Nov 2023 10:36:15 +0800 Subject: [PATCH 12/67] Fix return type in wasm_loader_get_custom_section (#2794) Should return NULL instead of false. --- core/iwasm/interpreter/wasm_loader.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 9e88e2051..f831073cd 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -7026,7 +7026,7 @@ wasm_loader_get_custom_section(WASMModule *module, const char *name, section = section->next; } - return false; + return NULL; } #endif From d4184968c0f090761b57616395f240d18b3c8a8b Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Mon, 20 Nov 2023 10:45:02 +0800 Subject: [PATCH 13/67] Fix build error of libsodium benchmark (#2792) --- tests/benchmarks/libsodium/build.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/benchmarks/libsodium/build.sh b/tests/benchmarks/libsodium/build.sh index 3049f2c73..c949375b2 100755 --- a/tests/benchmarks/libsodium/build.sh +++ b/tests/benchmarks/libsodium/build.sh @@ -12,9 +12,9 @@ libsodium_CASES="aead_aes256gcm2 aead_aes256gcm aead_chacha20poly13052 aead_chac pwhash_scrypt_ll pwhash_scrypt randombytes scalarmult2 scalarmult5 \ scalarmult6 scalarmult7 scalarmult8 scalarmult_ed25519 scalarmult_ristretto255 \ scalarmult secretbox2 secretbox7 secretbox8 secretbox_easy2 secretbox_easy \ - secretbox secretstream shorthash sign siphashx24 sodium_core sodium_utils2 \ - sodium_utils3 sodium_utils sodium_version stream2 stream3 stream4 stream verify1 \ - xchacha20" + secretbox secretstream_xchacha20poly1305 shorthash sign siphashx24 sodium_core \ + sodium_utils2 sodium_utils3 sodium_utils sodium_version stream2 stream3 stream4 \ + stream verify1 xchacha20" PLATFORM=$(uname -s | tr A-Z a-z) @@ -22,16 +22,19 @@ readonly WAMRC_CMD=$PWD/../../../wamr-compiler/build/wamrc readonly OUT_DIR=$PWD/libsodium/zig-out/bin if [ ! -d libsodium ]; then - git clone -b stable https://github.com/jedisct1/libsodium.git + git clone https://github.com/jedisct1/libsodium.git + cd libsodium + git checkout 1.0.19 + cd .. fi cd libsodium echo "Build libsodium native" -zig build -Drelease-fast -Denable_benchmarks=true +zig build -Doptimize=ReleaseFast -Denable_benchmarks=true echo "Build libsodium wasm32-wasi" -zig build -Drelease-fast -Denable_benchmarks=true -Dtarget=wasm32-wasi +zig build -Doptimize=ReleaseFast -Denable_benchmarks=true -Dtarget=wasm32-wasi for case in ${libsodium_CASES} do From 73cd1969f4920f69c8e7610be41e481852b64613 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Mon, 20 Nov 2023 11:37:51 +0800 Subject: [PATCH 14/67] Add support for custom sections in nuttx (#2795) --- product-mini/platforms/nuttx/wamr.mk | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/product-mini/platforms/nuttx/wamr.mk b/product-mini/platforms/nuttx/wamr.mk index 8bd05f48d..93cd618df 100644 --- a/product-mini/platforms/nuttx/wamr.mk +++ b/product-mini/platforms/nuttx/wamr.mk @@ -317,6 +317,12 @@ endif # REVISIT: is this worth to have a Kconfig? CFLAGS += -DWASM_DISABLE_WAKEUP_BLOCKING_OP=0 +ifeq ($(CONFIG_INTERPRETERS_WAMR_LOAD_CUSTOM_SECTIONS),y) +CFLAGS += -DWASM_ENABLE_LOAD_CUSTOM_SECTION=1 +else +CFLAGS += -DWASM_ENABLE_LOAD_CUSTOM_SECTION=0 +endif + ifeq ($(CONFIG_INTERPRETERS_WAMR_CUSTOM_NAME_SECTIONS),y) CFLAGS += -DWASM_ENABLE_CUSTOM_NAME_SECTION=1 else From aebe30426fdf84bc0c6299bb8a0e000a232f4225 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Mon, 20 Nov 2023 12:11:02 +0800 Subject: [PATCH 15/67] Fix formatting in aot_dump_perf_profiling (#2796) Changes %d to %PRIu32. --- core/iwasm/aot/aot_runtime.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 9912f63f0..ba85fdc26 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -2965,12 +2965,14 @@ aot_dump_perf_profiling(const AOTModuleInstance *module_inst) func_name = get_func_name_from_index(module_inst, i); if (func_name) - os_printf(" func %s, execution time: %.3f ms, execution count: %d " - "times\n", - func_name, perf_prof->total_exec_time / 1000.0f, - perf_prof->total_exec_cnt); + os_printf( + " func %s, execution time: %.3f ms, execution count: %" PRIu32 + "times\n", + func_name, perf_prof->total_exec_time / 1000.0f, + perf_prof->total_exec_cnt); else - os_printf(" func %d, execution time: %.3f ms, execution count: %d " + os_printf(" func %" PRIu32 + ", execution time: %.3f ms, execution count: %" PRIu32 "times\n", i, perf_prof->total_exec_time / 1000.0f, perf_prof->total_exec_cnt); From a57e70016a27d593b0448275571b439bce76133f Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:25:43 +0800 Subject: [PATCH 16/67] Fix memory.init opcode issue in fast-interp (#2798) Fix fast interpreter didn't throw OOB exception correctly in some scenarios. Reported in #2797. --- core/iwasm/interpreter/wasm_interp_fast.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index b0d0c6004..330f55689 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -2989,8 +2989,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, segment = read_uint32(frame_ip); - bytes = (uint64)POP_I32(); - offset = (uint64)POP_I32(); + bytes = (uint64)(uint32)POP_I32(); + offset = (uint64)(uint32)POP_I32(); addr = POP_I32(); #if WASM_ENABLE_THREAD_MGR From 103cb8959310509feb5626871dd671076a180804 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Mon, 20 Nov 2023 17:14:10 +0800 Subject: [PATCH 17/67] aot compiler: Fix handle next reachable if block (#2793) The popped reachable block may be if block whose else branch hasn't been translated, and should push the params for the else block if there are. And use LLVMDisposeMessage to free memory allocated in is_win_platform. --- core/iwasm/compilation/aot_emit_control.c | 16 ++++++++++++++++ core/iwasm/compilation/aot_emit_function.c | 9 ++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_control.c b/core/iwasm/compilation/aot_emit_control.c index 895bf7e35..a8ee938f2 100644 --- a/core/iwasm/compilation/aot_emit_control.c +++ b/core/iwasm/compilation/aot_emit_control.c @@ -202,6 +202,9 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, *p_frame_ip = block->wasm_code_else + 1; /* Push back the block */ aot_block_stack_push(&func_ctx->block_stack, block); + /* Recover parameters of else branch */ + for (i = 0; i < block->param_count; i++) + PUSH(block->else_param_phis[i], block->param_types[i]); return true; } else if (block->llvm_end_block) { @@ -221,6 +224,19 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } + if (block->label_type == LABEL_TYPE_IF && block->llvm_else_block + && !block->skip_wasm_code_else + && *p_frame_ip <= block->wasm_code_else) { + /* Clear value stack and start to translate else branch */ + aot_value_stack_destroy(&block->value_stack); + /* Recover parameters of else branch */ + for (i = 0; i < block->param_count; i++) + PUSH(block->else_param_phis[i], block->param_types[i]); + SET_BUILDER_POS(block->llvm_else_block); + *p_frame_ip = block->wasm_code_else + 1; + return true; + } + *p_frame_ip = block->wasm_code_end + 1; SET_BUILDER_POS(block->llvm_end_block); diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index def3f7d04..bc8cb0a55 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -22,11 +22,14 @@ static bool is_win_platform(AOTCompContext *comp_ctx) { char *triple = LLVMGetTargetMachineTriple(comp_ctx->target_machine); + bool ret; bh_assert(triple); - if (strstr(triple, "win32") || strstr(triple, "win")) - return true; - return false; + ret = (strstr(triple, "win32") || strstr(triple, "win")) ? true : false; + + LLVMDisposeMessage(triple); + + return ret; } static bool From 9ad42290d888ebdd393cc4f69b1103440548fd32 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Mon, 20 Nov 2023 18:06:35 +0800 Subject: [PATCH 18/67] Fix formatting in wasm_dump_perf_profiling (#2799) Changes %d to %PRIu32. --- core/iwasm/aot/aot_runtime.c | 4 ++-- core/iwasm/interpreter/wasm_runtime.c | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index ba85fdc26..cb1d8e855 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -2967,13 +2967,13 @@ aot_dump_perf_profiling(const AOTModuleInstance *module_inst) if (func_name) os_printf( " func %s, execution time: %.3f ms, execution count: %" PRIu32 - "times\n", + " times\n", func_name, perf_prof->total_exec_time / 1000.0f, perf_prof->total_exec_cnt); else os_printf(" func %" PRIu32 ", execution time: %.3f ms, execution count: %" PRIu32 - "times\n", + " times\n", i, perf_prof->total_exec_time / 1000.0f, perf_prof->total_exec_cnt); } diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 9efeb14c0..9deced3b5 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -2419,14 +2419,16 @@ wasm_dump_perf_profiling(const WASMModuleInstance *module_inst) } if (func_name) - os_printf(" func %s, execution time: %.3f ms, execution count: %d " - "times\n", - func_name, - module_inst->e->functions[i].total_exec_time / 1000.0f, - module_inst->e->functions[i].total_exec_cnt); + os_printf( + " func %s, execution time: %.3f ms, execution count: %" PRIu32 + " times\n", + func_name, + module_inst->e->functions[i].total_exec_time / 1000.0f, + module_inst->e->functions[i].total_exec_cnt); else - os_printf(" func %d, execution time: %.3f ms, execution count: %d " - "times\n", + os_printf(" func %" PRIu32 + ", execution time: %.3f ms, execution count: %" PRIu32 + " times\n", i, module_inst->e->functions[i].total_exec_time / 1000.0f, module_inst->e->functions[i].total_exec_cnt); } From 0b29904f26f8fc12ed59deef769e08720856c3d9 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Tue, 21 Nov 2023 17:32:45 +0800 Subject: [PATCH 19/67] Fix configurable bounds checks typo (#2809) --- core/iwasm/common/wasm_memory.c | 2 +- core/iwasm/common/wasm_runtime_common.c | 2 +- core/iwasm/common/wasm_runtime_common.h | 2 +- core/iwasm/interpreter/wasm_interp_classic.c | 2 +- core/iwasm/interpreter/wasm_interp_fast.c | 2 +- core/iwasm/interpreter/wasm_runtime.h | 2 +- product-mini/platforms/nuttx/wamr.mk | 6 +++--- product-mini/platforms/posix/main.c | 8 ++++---- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index e97f92df3..c01f9d284 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -95,7 +95,7 @@ wasm_memory_init_with_allocator(void *_malloc_func, void *_realloc_func, static inline bool is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) { -#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 return wasm_runtime_is_bounds_checks_enabled(module_inst); #else return true; diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index a8b3da6dd..ddd5aa8ce 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -2581,7 +2581,7 @@ wasm_runtime_get_custom_data(WASMModuleInstanceCommon *module_inst_comm) return module_inst->custom_data; } -#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 void wasm_runtime_set_bounds_checks(WASMModuleInstanceCommon *module_inst, bool enable) diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 367ea88f9..74785d3f6 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -603,7 +603,7 @@ wasm_runtime_set_user_data(WASMExecEnv *exec_env, void *user_data); WASM_RUNTIME_API_EXTERN void * wasm_runtime_get_user_data(WASMExecEnv *exec_env); -#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 /* See wasm_export.h for description */ WASM_RUNTIME_API_EXTERN void wasm_runtime_set_bounds_checks(WASMModuleInstanceCommon *module_inst, diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 72a81ea84..e129e771e 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -1181,7 +1181,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint8 value_type; #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 -#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 bool disable_bounds_checks = !wasm_runtime_is_bounds_checks_enabled( (WASMModuleInstanceCommon *)module); #else diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 330f55689..fe5ffd1c1 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -1207,7 +1207,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint8 opcode, local_type, *global_addr; #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 -#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 bool disable_bounds_checks = !wasm_runtime_is_bounds_checks_enabled( (WASMModuleInstanceCommon *)module); #else diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 1bd51551f..b4f616f34 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -220,7 +220,7 @@ typedef struct WASMModuleInstanceExtraCommon { CApiFuncImport *c_api_func_imports; /* pointer to the exec env currently used */ WASMExecEnv *cur_exec_env; -#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 /* Disable bounds checks or not */ bool disable_bounds_checks; #endif diff --git a/product-mini/platforms/nuttx/wamr.mk b/product-mini/platforms/nuttx/wamr.mk index 93cd618df..57995c2bf 100644 --- a/product-mini/platforms/nuttx/wamr.mk +++ b/product-mini/platforms/nuttx/wamr.mk @@ -236,10 +236,10 @@ else CFLAGS += -DWASM_ENABLE_LIBC_BUILTIN=0 endif -ifeq ($(CONFIG_INTERPRETERS_WAMR_CONFIGUABLE_BOUNDS_CHECKS),y) -CFLAGS += -DWASM_CONFIGUABLE_BOUNDS_CHECKS=1 +ifeq ($(CONFIG_INTERPRETERS_WAMR_CONFIGURABLE_BOUNDS_CHECKS),y) +CFLAGS += -DWASM_CONFIGURABLE_BOUNDS_CHECKS=1 else -CFLAGS += -DWASM_CONFIGUABLE_BOUNDS_CHECKS=0 +CFLAGS += -DWASM_CONFIGURABLE_BOUNDS_CHECKS=0 endif ifeq ($(CONFIG_INTERPRETERS_WAMR_LIBC_WASI),y) diff --git a/product-mini/platforms/posix/main.c b/product-mini/platforms/posix/main.c index c225b0858..930d9db5e 100644 --- a/product-mini/platforms/posix/main.c +++ b/product-mini/platforms/posix/main.c @@ -69,7 +69,7 @@ print_help() #endif printf(" --repl Start a very simple REPL (read-eval-print-loop) mode\n" " that runs commands in the form of \"FUNC ARG...\"\n"); -#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 printf(" --disable-bounds-checks Disable bounds checks for memory accesses\n"); #endif #if WASM_ENABLE_LIBC_WASI != 0 @@ -571,7 +571,7 @@ main(int argc, char *argv[]) #endif bool is_repl_mode = false; bool is_xip_file = false; -#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 bool disable_bounds_checks = false; #endif #if WASM_ENABLE_LIBC_WASI != 0 @@ -638,7 +638,7 @@ main(int argc, char *argv[]) else if (!strcmp(argv[0], "--repl")) { is_repl_mode = true; } -#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 else if (!strcmp(argv[0], "--disable-bounds-checks")) { disable_bounds_checks = true; } @@ -886,7 +886,7 @@ main(int argc, char *argv[]) goto fail3; } -#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 +#if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0 if (disable_bounds_checks) { wasm_runtime_set_bounds_checks(wasm_module_inst, false); } From 4d5eb346fcf58dcca0ac7d9bb0a71afcf840e5a0 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 22 Nov 2023 10:38:08 +0800 Subject: [PATCH 20/67] Change is_shared_memory type from bool to uint8 (#2800) Change WASMMemoryInstance's field is_shared_memory's type from bool to uint8 whose size is fixed, so as to make WASMMemoryInstance's size and layout fixed and not break AOT ABI. See discussion in https://github.com/bytecodealliance/wasm-micro-runtime/pull/2682. --- core/iwasm/aot/aot_runtime.c | 2 +- core/iwasm/interpreter/wasm_runtime.c | 2 +- core/iwasm/interpreter/wasm_runtime.h | 11 ++++++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index cb1d8e855..076574cb7 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -605,7 +605,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, #if WASM_ENABLE_SHARED_MEMORY != 0 if (is_shared_memory) { - memory_inst->is_shared_memory = true; + memory_inst->is_shared_memory = 1; memory_inst->ref_count = 1; } #endif diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 9deced3b5..205c6d4b6 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -386,7 +386,7 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, #if WASM_ENABLE_SHARED_MEMORY != 0 if (is_shared_memory) { - memory->is_shared_memory = true; + memory->is_shared_memory = 1; memory->ref_count = 1; } #endif diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index b4f616f34..bb5fdc80b 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -81,10 +81,15 @@ struct WASMMemoryInstance { /* Module type */ uint32 module_type; - bool is_shared_memory; + /* Whether the memory is shared */ + uint8 is_shared_memory; - /* Shared memory flag */ - bh_atomic_16_t ref_count; /* 0: non-shared, > 0: reference count */ + /* One byte padding */ + uint8 __padding__; + + /* Reference count of the memory instance: + 0: non-shared memory, > 0: shared memory */ + bh_atomic_16_t ref_count; /* Number bytes per page */ uint32 num_bytes_per_page; From f9e8b9535ee47b174ea338c908c289748ab75b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A4mes=20M=C3=A9n=C3=A9trey?= Date: Wed, 22 Nov 2023 03:48:14 +0100 Subject: [PATCH 21/67] Attestation: Free JSON from the Wasm module heap (#2803) The JSON evidence is allocated on the module instance heap, but no API was given to dispose of this memory buffer. The sample mentions using the function free, which behaves differently depending on the execution context. This fix provides a new function called librats_dispose_evidence_json, enabling freeing the JSON evidence directly from the Wasm app. --- core/iwasm/libraries/lib-rats/lib_rats_wrapper.c | 16 +++++++++++++--- core/iwasm/libraries/lib-rats/lib_rats_wrapper.h | 3 +++ samples/sgx-ra/wasm-app/main.c | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/core/iwasm/libraries/lib-rats/lib_rats_wrapper.c b/core/iwasm/libraries/lib-rats/lib_rats_wrapper.c index 59d61f4c8..bdacc259c 100644 --- a/core/iwasm/libraries/lib-rats/lib_rats_wrapper.c +++ b/core/iwasm/libraries/lib-rats/lib_rats_wrapper.c @@ -17,7 +17,7 @@ #include "lib_rats_common.h" static int -librats_collect_wrapper(wasm_exec_env_t exec_env, char **evidence_json, +librats_collect_wrapper(wasm_exec_env_t exec_env, uint32_t *evidence_json, const char *buffer, uint32_t buffer_size) { wasm_module_inst_t module_inst = get_module_inst(exec_env); @@ -47,7 +47,7 @@ librats_collect_wrapper(wasm_exec_env_t exec_env, char **evidence_json, return (int)RATS_ATTESTER_ERR_NO_MEM; } bh_memcpy_s(str_ret, json_size, json, json_size); - *((int *)evidence_json) = str_ret_offset; + *evidence_json = str_ret_offset; free(json); return 0; @@ -96,6 +96,15 @@ librats_parse_evidence_wrapper(wasm_exec_env_t exec_env, return 0; } +static void +librats_dispose_evidence_json_wrapper(wasm_exec_env_t exec_env, + uint32_t evidence_json) +{ + wasm_module_inst_t module_inst = get_module_inst(exec_env); + + module_free(evidence_json); +} + /* clang-format off */ #define REG_NATIVE_FUNC(func_name, signature) \ { #func_name, func_name##_wrapper, signature, NULL } @@ -104,7 +113,8 @@ librats_parse_evidence_wrapper(wasm_exec_env_t exec_env, static NativeSymbol native_symbols_lib_rats[] = { REG_NATIVE_FUNC(librats_collect, "(**~)i"), REG_NATIVE_FUNC(librats_verify, "(*~*~)i"), - REG_NATIVE_FUNC(librats_parse_evidence, "(*~*~)i") + REG_NATIVE_FUNC(librats_parse_evidence, "(*~*~)i"), + REG_NATIVE_FUNC(librats_dispose_evidence_json, "(i)") }; uint32_t diff --git a/core/iwasm/libraries/lib-rats/lib_rats_wrapper.h b/core/iwasm/libraries/lib-rats/lib_rats_wrapper.h index e334983e9..928645108 100644 --- a/core/iwasm/libraries/lib-rats/lib_rats_wrapper.h +++ b/core/iwasm/libraries/lib-rats/lib_rats_wrapper.h @@ -41,6 +41,9 @@ librats_parse_evidence(const char *evidence_json, uint32_t json_size, evidence_json ? strlen(evidence_json) + 1 : 0, \ evidence, sizeof(rats_sgx_evidence_t)) +void +librats_dispose_evidence_json(char *evidence_json); + #ifdef __cplusplus } #endif diff --git a/samples/sgx-ra/wasm-app/main.c b/samples/sgx-ra/wasm-app/main.c index 89c4144aa..6f506e06a 100644 --- a/samples/sgx-ra/wasm-app/main.c +++ b/samples/sgx-ra/wasm-app/main.c @@ -106,7 +106,7 @@ main(int argc, char **argv) err: if (evidence_json) { - free(evidence_json); + librats_dispose_evidence_json(evidence_json); } if (evidence) { From 2175910bac4634e0da7022c2237f1927e64481d9 Mon Sep 17 00:00:00 2001 From: Daniel Mangum <31777345+hasheddan@users.noreply.github.com> Date: Wed, 22 Nov 2023 19:56:04 -0600 Subject: [PATCH 22/67] Update Zephyr support to v3.5.0 and make instructions generic to boards (#2805) Includes a number of updates to the Zephyr example, including removing a fair amount of stale configuration. This is part of moving towards supporting WAMR as a Zephyr module as described in #2782. Some functionality is removed as part of this diff (e.g. AOT XTENSA support), but it had become stale while not being actively maintained. It is anticipated that it (and AOT support for other platforms) will be added back, along with CI support for ensuring they do not become stale again. Signed-off-by: Daniel Mangum --- .../platforms/zephyr/simple/Dockerfile | 39 +-- .../platforms/zephyr/simple/Dockerfile.old | 58 ---- .../platforms/zephyr/simple/README.md | 142 ++++----- .../platforms/zephyr/simple/boards/esp32.conf | 8 - .../{nucleo767zi.conf => nucleo_f767zi.conf} | 3 - .../zephyr/simple/boards/qemu_cortex_a53.conf | 3 - .../zephyr/simple/boards/qemu_riscv32.conf | 6 - .../zephyr/simple/boards/qemu_riscv64.conf | 6 - .../zephyr/simple/boards/qemu_x86_nommu.conf | 6 - .../zephyr/simple/boards/qemu_xtensa.conf | 6 - .../platforms/zephyr/simple/build_and_run.sh | 24 +- .../zephyr/simple/esp32_custom_linker.ld | 274 ------------------ .../simple/{boards/qemu_arc.conf => prj.conf} | 0 .../platforms/zephyr/simple/src/main.c | 70 ----- 14 files changed, 86 insertions(+), 559 deletions(-) delete mode 100644 product-mini/platforms/zephyr/simple/Dockerfile.old delete mode 100644 product-mini/platforms/zephyr/simple/boards/esp32.conf rename product-mini/platforms/zephyr/simple/boards/{nucleo767zi.conf => nucleo_f767zi.conf} (72%) delete mode 100644 product-mini/platforms/zephyr/simple/boards/qemu_riscv32.conf delete mode 100644 product-mini/platforms/zephyr/simple/boards/qemu_riscv64.conf delete mode 100644 product-mini/platforms/zephyr/simple/boards/qemu_x86_nommu.conf delete mode 100644 product-mini/platforms/zephyr/simple/boards/qemu_xtensa.conf delete mode 100644 product-mini/platforms/zephyr/simple/esp32_custom_linker.ld rename product-mini/platforms/zephyr/simple/{boards/qemu_arc.conf => prj.conf} (100%) diff --git a/product-mini/platforms/zephyr/simple/Dockerfile b/product-mini/platforms/zephyr/simple/Dockerfile index 9c8e6e5a0..a4c69a8ff 100644 --- a/product-mini/platforms/zephyr/simple/Dockerfile +++ b/product-mini/platforms/zephyr/simple/Dockerfile @@ -1,49 +1,32 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -FROM ubuntu:20.04 +FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive ENV TZ=Asian/Shanghai -# Install dependencies for Zephyr and ESPRESSIF +# Install dependencies for Zephyr # hadolint ignore=DL3008 -RUN apt-get update && apt-get install -y git wget flex bison gperf python3 python3-pip python3-venv\ - python3-dev python3-setuptools python3-tk python3-wheel xz-utils file libpython3.8-dev \ - ninja-build ccache libffi-dev libssl-dev dfu-util libusb-1.0-0 device-tree-compiler \ - make gcc gcc-multilib g++-multilib libsdl2-dev libmagic1 qemu udev --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends git cmake ninja-build gperf \ + ccache dfu-util device-tree-compiler wget \ + python3-dev python3-pip python3-setuptools python3-tk python3-wheel xz-utils file \ + make gcc gcc-multilib g++-multilib libsdl2-dev libmagic1 \ && apt-get clean -y && rm -rf /var/lib/apt/lists/* -# Install recent CMake version -WORKDIR /tmp -RUN mkdir /opt/cmake \ - && wget --progress=dot:giga https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-linux-x86_64.sh \ - && sh cmake-3.22.1-linux-x86_64.sh --skip-license --prefix=/opt/cmake && rm cmake-3.22.1-linux-x86_64.sh -ENV PATH="/opt/cmake/bin:$PATH" - # Install the Zephyr Software Development Kit (SDK) WORKDIR /opt # hadolint ignore=DL4006 -RUN wget --progress=dot:giga https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.1/zephyr-sdk-0.16.1_linux-x86_64.tar.xz \ - && wget --progress=dot:giga -O - https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.1/sha256.sum | shasum --check --ignore-missing \ - && tar xvf zephyr-sdk-0.16.1_linux-x86_64.tar.xz && rm zephyr-sdk-0.16.1_linux-x86_64.tar.xz +RUN wget --progress=dot:giga https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.3/zephyr-sdk-0.16.3_linux-x86_64.tar.xz \ + && wget --progress=dot:giga -O - https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.3/sha256.sum | shasum --check --ignore-missing \ + && tar xvf zephyr-sdk-0.16.3_linux-x86_64.tar.xz && rm zephyr-sdk-0.16.3_linux-x86_64.tar.xz -WORKDIR /opt/zephyr-sdk-0.16.1 +WORKDIR /opt/zephyr-sdk-0.16.3 # hadolint ignore=DL4006 RUN yes | ./setup.sh -# Get ESP-IDF -RUN ln -s /usr/bin/python3 /usr/bin/python && mkdir -p ~/esp -WORKDIR /root/esp -RUN git clone https://github.com/espressif/esp-idf.git -WORKDIR /root/esp/esp-idf -RUN git checkout 03d4fa28694ee15ccfd5a97447575de2d1655026 \ - && git submodule update --init --recursive -# Set up the sep-idf tools -RUN ./install.sh esp32 esp32c3 - # Get Zephyr # hadolint ignore=DL3013 -RUN pip3 install --no-cache-dir west && west init -m https://github.com/zephyrproject-rtos/zephyr --mr v3.4.0 /root/zephyrproject +RUN pip3 install --no-cache-dir west && west init -m https://github.com/zephyrproject-rtos/zephyr --mr v3.5.0 /root/zephyrproject WORKDIR /root/zephyrproject RUN west update diff --git a/product-mini/platforms/zephyr/simple/Dockerfile.old b/product-mini/platforms/zephyr/simple/Dockerfile.old deleted file mode 100644 index e223f315a..000000000 --- a/product-mini/platforms/zephyr/simple/Dockerfile.old +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -FROM ubuntu:20.04 - -ARG DEBIAN_FRONTEND=noninteractive -ENV TZ=Asian/Shanghai - -# Install dependencies for Zephyr and ESPRESSIF -# hadolint ignore=DL3008 -RUN apt-get update && apt-get install -y git wget flex bison gperf python3 python3-pip python3-venv\ - python3-dev python3-setuptools python3-tk python3-wheel xz-utils file libpython3.8-dev \ - ninja-build ccache libffi-dev libssl-dev dfu-util libusb-1.0-0 device-tree-compiler \ - make gcc gcc-multilib g++-multilib libsdl2-dev libmagic1 qemu udev --no-install-recommends \ - && apt-get clean -y && rm -rf /var/lib/apt/lists/* - -# Install recent CMake version -WORKDIR /tmp -RUN mkdir /opt/cmake \ - && wget --progress=dot:giga https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-linux-x86_64.sh \ - && sh cmake-3.22.1-linux-x86_64.sh --skip-license --prefix=/opt/cmake && rm cmake-3.22.1-linux-x86_64.sh -ENV PATH="/opt/cmake/bin:$PATH" - -# Install the Zephyr Software Development Kit (SDK) -WORKDIR /opt -# hadolint ignore=DL4006 -RUN wget --progress=dot:giga https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.11.3/zephyr-sdk-0.11.3-setup.run \ - && chmod +x ./zephyr-sdk-0.11.3-setup.run \ - && ./zephyr-sdk-0.11.3-setup.run -- -d /opt/zephyr-sdk-0.11.3 -ENV ZEPHYR_TOOLCHAIN_VARIANT=zephyr -ENV ZEPHYR_SDK_INSTALL_DIR=/opt/zephyr-sdk-0.11.3 - -# Get ESP-IDF -RUN ln -s /usr/bin/python3 /usr/bin/python && mkdir -p ~/esp -WORKDIR /root/esp -RUN git clone https://github.com/espressif/esp-idf.git -WORKDIR /root/esp/esp-idf -RUN git checkout v4.0 \ - && pip install --no-cache-dir virtualenv==16.7.12 \ - && git submodule update --init --recursive \ - && ./install.sh esp32 esp32c3 - -# Get Zephyr -# hadolint ignore=DL3013 -RUN pip3 install --no-cache-dir west && west init -m https://github.com/zephyrproject-rtos/zephyr --mr v2.3.0 /root/zephyrproject - -WORKDIR /root/zephyrproject -RUN west update - -WORKDIR /root/zephyrproject/zephyr -RUN west zephyr-export && pip install --no-cache-dir -r ~/zephyrproject/zephyr/scripts/requirements.txt - -# Git clone wamr -WORKDIR /root -RUN git clone https://github.com/bytecodealliance/wasm-micro-runtime.git - -WORKDIR /root/wasm-micro-runtime/product-mini/platforms/zephyr/simple - -ENV ZEPHYR_BASE="/root/zephyrproject/zephyr" diff --git a/product-mini/platforms/zephyr/simple/README.md b/product-mini/platforms/zephyr/simple/README.md index acb955258..5f8bd41ae 100644 --- a/product-mini/platforms/zephyr/simple/README.md +++ b/product-mini/platforms/zephyr/simple/README.md @@ -1,106 +1,108 @@ # How to use WAMR with Zephyr -## Build with Docker(recommend approach) +[Zephyr](https://www.zephyrproject.org/) is an open source real-time operating +system (RTOS) with a focus on security and broad hardware support. WAMR is +compatible with Zephyr via the [Zephyr WAMR +port](../../../../core/shared/platform/zephyr). -To have a quicker start, a Docker container of the Zephyr setup can be generated. The current docker image would be considerably large(~15GB), it would take some time to build it and enough disk space to store it. +## Setup -### Build Docker images +Using WAMR with Zephyr can be accomplished by either using the provided Docker +image, or by installing Zephyr locally. Both approaches are described below. + +### Docker + +The provided Docker image sets up Zephyr and its dependencies for all +architectures, meaning that you are ready to build for any [supported +board](https://docs.zephyrproject.org/latest/boards/index.html). This comes at +the expense of building a rather large image, which both can take a long time to +build and uses up a large amount of storage (~15 GB). + +Execute the following command to build the Docker image. This may take an +extended period of time to complete. ```shell docker build -t wamr-zephyr . ``` -> PS: currently, the esp32 custom linker script only works with a lower version of Zephyr, if you want to use an esp32 board, you can build the Dockerfile with a lower version of Zephyr, Zephyr SDE, ESP-IDF. The old version of Docker image can also build other targets, but probably it's a better choice to use the new Dockerfile for other boards +Execute the following command to run the image built in the previous step. If +you are planning to flash a device after building, make sure to specify it with +[`--device`](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities). ```shell -# If you want to build on esp32 platform -docker build -f Dockerfile.old -t wamr-zephyr . +docker run -it --rm --device=/dev/ttyUSB0 wamr-zephyr ``` -### Run Docker images +### Local Environment -Adopt the device or remove if not needed. +Zephyr can also be setup locally to enable building this sample application. +This allows you have have more control over what modules and tools are +installed, which can drastically reduce the storage required compared to the +Docker image. + +Follow the steps provided in the [Zephyr Getting Started +guide](https://docs.zephyrproject.org/latest/develop/getting_started/index.html) +to setup for local development. + +## Building for a Specific Board + +With an environment setup either locally or in a Docker container, you can build +for a Zephyr suppported board using +[`west`](https://docs.zephyrproject.org/latest/develop/west/index.html). There +are already [configuaration files](./boards) for a few boards in this sample. +However, if you are using a new board, you will need to add your own file for +the board, or define configuration in the [`prj.conf](./prj.conf). After doing +so, use the following command with your board identifier to build the sample +application. ```shell -docker run -ti --device=/dev/ttyUSB0 wamr-zephyr +west build . -b --pristine -- -DWAMR_BUILD_TARGET= ``` -And then inside the docker container: +The `` can be found in the Zephyr supported boards +documentation. It must also be used as the name of the board configuration file. +You must define the architecture for WAMR to target with `WAMR_BUILD_TARGET`. +The list of supported architectures can be found in the main project +[README.md](../../../../README.md#supported-architectures-and-platforms). + +It may be necessary to define additional symbols for some boards. For example, +WAMR AOT execution may not be supported on all architectures. It and other +options can be disabled by modifying the [CMakeLists.txt](./CMakeLists.txt) +file, or by passing additional arguments at build (e.g. `-DWAMR_BUILD_AOT=0`). + +### Example Targets + +[ESP32-C3](https://docs.zephyrproject.org/latest/boards/riscv/esp32c3_devkitm/doc/index.html) +is a 32-bit RISC-V target that does not currently support AOT. ```shell -# copy the corresponding board conf file to current directory -cp boards/qemu_x86_nommu.conf prj.conf -# then build -./build_and_run.sh x86 +west build . -b esp32c3_devkitm -p always -- -DWAMR_BUILD_TARGET=RISCV32_ILP32 -DWAMR_BUILD_AOT=0 ``` -> PS: for boards esp32, need to configure some environment first +[ARM Cortex-A53 QEMU +(ARM)](https://docs.zephyrproject.org/latest/boards/arm64/qemu_cortex_a53/doc/index.html) +is a 64-bit ARM target for emulating the Cortex-A53 platform. ```shell -# configure zephyr with espressif -export ZEPHYR_TOOLCHAIN_VARIANT="espressif" -export ESPRESSIF_TOOLCHAIN_PATH="/root/.espressif/tools/xtensa-esp32-elf/esp-2019r2-8.2.0/xtensa-esp32-elf/" -export ESP_IDF_PATH="/root/esp/esp-idf" -# copy the corresponding board conf file to current directory -cp boards/esp32.conf prj.conf -# then build -./build_and_run.sh esp32 +west build . -b qemu_cortex_a53 -p always -- -DWAMR_BUILD_TARGET=AARCH64 ``` -## Build on local environment -### Dependencies installation +## Flashing or Running Image -Following the Zephyr and Espressif official document: - -1. Zephyr installation: - - - -2. ESP32 installation: - - - -And setup the Zephyr for esp32: - - - -Then Installing QEMU, for example, on Linux: +The board can be flashed with the built image with the following command. ```shell -sudo apt-get install qemu +west flash ``` -### Run the build script +`west` will automatically identify the board if it is connected to the host +machine. -Make sure you have the environment variable ready, you can use the command `env` to check: +When using emulated targets, such as those that utilize QEMU, there is no +physical device to flash, but `west` can be used to run the image under +emulation. ```shell -env -``` - -```shell -# export ZEPHYR_BASE if it's not present -export ZEPHYR_BASE=~/zephyrproject/zephyr -# and if you install zephyr in virtual environment rather than global -source ~/zephyrproject/.venv/bin/activate -``` - -For boards esp32, need to configure some extra environment first, check the following env variable whether in the env list, if not, add them like: - -> Noted: The esp32 custom linker script doesn't work with the recent version of Zephyr, if you want to use it in the local environment, please install Zephyr 2.3.0 with the corresponding SDK, and ESP-IDF 4.0 - -```shell -export ZEPHYR_TOOLCHAIN_VARIANT="espressif" -export ESPRESSIF_TOOLCHAIN_PATH="~/.espressif/tools/xtensa-esp32-elf/esp-{the version you installed}/xtensa-esp32-elf/" -export ESP_IDF_PATH="~/esp/esp-idf" -``` - -Then you can run the build script: - -```shell -# copy the corresponding board conf file to current directory -cp boards/qemu_x86_nommu.conf prj.conf -# then build -./build_and_run.sh x86 +west build -t run ``` diff --git a/product-mini/platforms/zephyr/simple/boards/esp32.conf b/product-mini/platforms/zephyr/simple/boards/esp32.conf deleted file mode 100644 index 5d6a67f9d..000000000 --- a/product-mini/platforms/zephyr/simple/boards/esp32.conf +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -CONFIG_STACK_SENTINEL=y -CONFIG_PRINTK=y -CONFIG_LOG=y -CONFIG_HAVE_CUSTOM_LINKER_SCRIPT=y -CONFIG_CUSTOM_LINKER_SCRIPT="esp32_custom_linker.ld" diff --git a/product-mini/platforms/zephyr/simple/boards/nucleo767zi.conf b/product-mini/platforms/zephyr/simple/boards/nucleo_f767zi.conf similarity index 72% rename from product-mini/platforms/zephyr/simple/boards/nucleo767zi.conf rename to product-mini/platforms/zephyr/simple/boards/nucleo_f767zi.conf index c495644b7..c920e6fd0 100644 --- a/product-mini/platforms/zephyr/simple/boards/nucleo767zi.conf +++ b/product-mini/platforms/zephyr/simple/boards/nucleo_f767zi.conf @@ -2,6 +2,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception CONFIG_ARM_MPU=y -CONFIG_STACK_SENTINEL=y -CONFIG_PRINTK=y -CONFIG_LOG=y diff --git a/product-mini/platforms/zephyr/simple/boards/qemu_cortex_a53.conf b/product-mini/platforms/zephyr/simple/boards/qemu_cortex_a53.conf index d248565fa..2cc2dd4b4 100644 --- a/product-mini/platforms/zephyr/simple/boards/qemu_cortex_a53.conf +++ b/product-mini/platforms/zephyr/simple/boards/qemu_cortex_a53.conf @@ -2,6 +2,3 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception CONFIG_ARM_MMU=n -CONFIG_STACK_SENTINEL=y -CONFIG_PRINTK=y -CONFIG_LOG=y diff --git a/product-mini/platforms/zephyr/simple/boards/qemu_riscv32.conf b/product-mini/platforms/zephyr/simple/boards/qemu_riscv32.conf deleted file mode 100644 index f3705504b..000000000 --- a/product-mini/platforms/zephyr/simple/boards/qemu_riscv32.conf +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -CONFIG_STACK_SENTINEL=y -CONFIG_PRINTK=y -CONFIG_LOG=n diff --git a/product-mini/platforms/zephyr/simple/boards/qemu_riscv64.conf b/product-mini/platforms/zephyr/simple/boards/qemu_riscv64.conf deleted file mode 100644 index f3705504b..000000000 --- a/product-mini/platforms/zephyr/simple/boards/qemu_riscv64.conf +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -CONFIG_STACK_SENTINEL=y -CONFIG_PRINTK=y -CONFIG_LOG=n diff --git a/product-mini/platforms/zephyr/simple/boards/qemu_x86_nommu.conf b/product-mini/platforms/zephyr/simple/boards/qemu_x86_nommu.conf deleted file mode 100644 index 7f4a32832..000000000 --- a/product-mini/platforms/zephyr/simple/boards/qemu_x86_nommu.conf +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -CONFIG_STACK_SENTINEL=y -CONFIG_PRINTK=y -CONFIG_LOG=y diff --git a/product-mini/platforms/zephyr/simple/boards/qemu_xtensa.conf b/product-mini/platforms/zephyr/simple/boards/qemu_xtensa.conf deleted file mode 100644 index 7f4a32832..000000000 --- a/product-mini/platforms/zephyr/simple/boards/qemu_xtensa.conf +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -CONFIG_STACK_SENTINEL=y -CONFIG_PRINTK=y -CONFIG_LOG=y diff --git a/product-mini/platforms/zephyr/simple/build_and_run.sh b/product-mini/platforms/zephyr/simple/build_and_run.sh index 921f88363..6b8fb4f87 100755 --- a/product-mini/platforms/zephyr/simple/build_and_run.sh +++ b/product-mini/platforms/zephyr/simple/build_and_run.sh @@ -5,7 +5,6 @@ X86_TARGET="x86" STM32_TARGET="stm32" -ESP32_TARGET="esp32" ESP32C3_TARGET="esp32c3" PARTICLE_ARGON_TARGET="particle_argon" QEMU_CORTEX_A53="qemu_cortex_a53" @@ -17,11 +16,10 @@ QEMU_ARC_TARGET="qemu_arc" usage () { echo "USAGE:" - echo "$0 $X86_TARGET|$STM32_TARGET|$ESP32_TARGET|$ESP32C3_TARGET|$PARTICLE_ARGON_TARGET|$QEMU_CORTEX_A53|$QEMU_XTENSA_TARGET|$QEMU_RISCV64_TARGET|$QEMU_RISCV32_TARGET|$QEMU_ARC_TARGET" + echo "$0 $X86_TARGET|$STM32_TARGET|$ESP32C3_TARGET|$PARTICLE_ARGON_TARGET|$QEMU_CORTEX_A53|$QEMU_XTENSA_TARGET|$QEMU_RISCV64_TARGET|$QEMU_RISCV32_TARGET|$QEMU_ARC_TARGET" echo "Example:" echo " $0 $X86_TARGET" echo " $0 $STM32_TARGET" - echo " $0 $ESP32_TARGET" echo " $0 $ESP32C3_TARGET" echo " $0 $PARTICLE_ARGON_TARGET" echo " $0 $QEMU_CORTEX_A53" @@ -51,27 +49,11 @@ case $TARGET in -DWAMR_BUILD_TARGET=THUMBV7 west flash ;; - $ESP32_TARGET) - export ZEPHYR_TOOLCHAIN_VARIANT="espressif" - if [[ -z "${ESPRESSIF_TOOLCHAIN_PATH}" ]]; then - echo "Set ESPRESSIF_TOOLCHAIN_PATH to your espressif toolchain" - exit 1 - fi - west build -b esp32 \ - . -p always -- \ - -DWAMR_BUILD_TARGET=XTENSA - # west flash will discover the device - west flash - ;; $ESP32C3_TARGET) - export ZEPHYR_TOOLCHAIN_VARIANT="espressif" - if [[ -z "${ESPRESSIF_TOOLCHAIN_PATH}" ]]; then - echo "Set ESPRESSIF_TOOLCHAIN_PATH to your espressif toolchain" - exit 1 - fi west build -b esp32c3_devkitm \ . -p always -- \ - -DWAMR_BUILD_TARGET=RISCV32_ILP32 + -DWAMR_BUILD_TARGET=RISCV32_ILP32 \ + -DWAMR_BUILD_AOT=0 # west flash will discover the device west flash ;; diff --git a/product-mini/platforms/zephyr/simple/esp32_custom_linker.ld b/product-mini/platforms/zephyr/simple/esp32_custom_linker.ld deleted file mode 100644 index 35932050f..000000000 --- a/product-mini/platforms/zephyr/simple/esp32_custom_linker.ld +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Copyright (c) 2016 Cadence Design Systems, Inc. - * Copyright (c) 2017 Intel Corporation - * SPDX-License-Identifier: Apache-2.0 - */ - -/** - * @file - * @brief Linker command/script file - * - * Linker script for the Xtensa platform. - */ - -#include -#include -#include -#include -#include - -#define RAMABLE_REGION dram0_0_seg :dram0_0_phdr -#define RAMABLE_REGION1 dram0_1_seg :dram0_0_phdr -#define ROMABLE_REGION iram0_0_seg :iram0_0_phdr - -PROVIDE ( __stack = 0x3ffe3f20 ); - -PROVIDE ( esp32_rom_uart_tx_one_char = 0x40009200 ); -PROVIDE ( esp32_rom_uart_rx_one_char = 0x400092d0 ); -PROVIDE ( esp32_rom_uart_attach = 0x40008fd0 ); -PROVIDE ( esp32_rom_intr_matrix_set = 0x4000681c ); -PROVIDE ( esp32_rom_gpio_matrix_in = 0x40009edc ); -PROVIDE ( esp32_rom_gpio_matrix_out = 0x40009f0c ); -PROVIDE ( esp32_rom_Cache_Flush = 0x40009a14 ); -PROVIDE ( esp32_rom_Cache_Read_Enable = 0x40009a84 ); -PROVIDE ( esp32_rom_ets_set_appcpu_boot_addr = 0x4000689c ); - -MEMORY -{ - iram0_0_seg(RX): org = 0x40080000, len = 0x20000 - iram0_2_seg(RX): org = 0x400D0018, len = 0x330000 - dram0_0_seg(RW): org = 0x3FFB0000, len = 0x30000 - dram0_1_seg(RWX):org = 0x400A0000, len = 0x20000 - drom0_0_seg(R): org = 0x3F400010, len = 0x800000 - rtc_iram_seg(RWX): org = 0x400C0000, len = 0x2000 - rtc_slow_seg(RW): org = 0x50000000, len = 0x1000 -#ifdef CONFIG_GEN_ISR_TABLES - IDT_LIST(RW): org = 0x3ebfe010, len = 0x2000 -#endif -} - -PHDRS -{ - iram0_0_phdr PT_LOAD; - dram0_0_phdr PT_LOAD; -} - -/* Default entry point: */ -PROVIDE ( _ResetVector = 0x40000400 ); -ENTRY(CONFIG_KERNEL_ENTRY) - -_rom_store_table = 0; - -PROVIDE(_memmap_vecbase_reset = 0x40000450); -PROVIDE(_memmap_reset_vector = 0x40000400); - -SECTIONS -{ - -#include - - /* RTC fast memory holds RTC wake stub code, - including from any source file named rtc_wake_stub*.c - */ - .rtc.text : - { - . = ALIGN(4); - *(.rtc.literal .rtc.text) - *rtc_wake_stub*.o(.literal .text .literal.* .text.*) - } >rtc_iram_seg - - /* RTC slow memory holds RTC wake stub - data/rodata, including from any source file - named rtc_wake_stub*.c - */ - .rtc.data : - { - _rtc_data_start = ABSOLUTE(.); - *(.rtc.data) - *(.rtc.rodata) - *rtc_wake_stub*.o(.data .rodata .data.* .rodata.* .bss .bss.*) - _rtc_data_end = ABSOLUTE(.); - } > rtc_slow_seg - - /* RTC bss, from any source file named rtc_wake_stub*.c */ - .rtc.bss (NOLOAD) : - { - _rtc_bss_start = ABSOLUTE(.); - *rtc_wake_stub*.o(.bss .bss.*) - *rtc_wake_stub*.o(COMMON) - _rtc_bss_end = ABSOLUTE(.); - } > rtc_slow_seg - - /* Send .iram0 code to iram */ - .iram0.vectors : ALIGN(4) - { - /* Vectors go to IRAM */ - _init_start = ABSOLUTE(.); - /* Vectors according to builds/RF-2015.2-win32/esp108_v1_2_s5_512int_2/config.html */ - . = 0x0; - KEEP(*(.WindowVectors.text)); - . = 0x180; - KEEP(*(.Level2InterruptVector.text)); - . = 0x1c0; - KEEP(*(.Level3InterruptVector.text)); - . = 0x200; - KEEP(*(.Level4InterruptVector.text)); - . = 0x240; - KEEP(*(.Level5InterruptVector.text)); - . = 0x280; - KEEP(*(.DebugExceptionVector.text)); - . = 0x2c0; - KEEP(*(.NMIExceptionVector.text)); - . = 0x300; - KEEP(*(.KernelExceptionVector.text)); - . = 0x340; - KEEP(*(.UserExceptionVector.text)); - . = 0x3C0; - KEEP(*(.DoubleExceptionVector.text)); - . = 0x400; - *(.*Vector.literal) - - *(.UserEnter.literal); - *(.UserEnter.text); - . = ALIGN (16); - *(.entry.text) - *(.init.literal) - *(.init) - _init_end = ABSOLUTE(.); - - /* This goes here, not at top of linker script, so addr2line finds it last, - and uses it in preference to the first symbol in IRAM */ - _iram_start = ABSOLUTE(0); - } GROUP_LINK_IN(ROMABLE_REGION) - -#include -#include - - SECTION_PROLOGUE(_TEXT_SECTION_NAME, , ALIGN(4)) - { - /* Code marked as running out of IRAM */ - _iram_text_start = ABSOLUTE(.); - *(.iram1 .iram1.*) - *(.iram0.literal .iram.literal .iram.text.literal .iram0.text .iram.text) - *(.literal .text .literal.* .text.*) - _iram_text_end = ABSOLUTE(.); - } GROUP_LINK_IN(ROMABLE_REGION) - - .dram0.text : - { - _data_start = ABSOLUTE(.); - *(.aot_code_buf) - _data_end = ABSOLUTE(.); - . = ALIGN(4); - } GROUP_LINK_IN(RAMABLE_REGION1) - - - .dram0.data : - { - _data_start = ABSOLUTE(.); - *(.data) - *(.data.*) - *(.gnu.linkonce.d.*) - *(.data1) - *(.sdata) - *(.sdata.*) - *(.gnu.linkonce.s.*) - *(.sdata2) - *(.sdata2.*) - *(.gnu.linkonce.s2.*) - KEEP(*(.jcr)) - *(.dram1 .dram1.*) - _data_end = ABSOLUTE(.); - . = ALIGN(4); - } GROUP_LINK_IN(RAMABLE_REGION) - - SECTION_PROLOGUE(_RODATA_SECTION_NAME,,ALIGN(4)) - { - _rodata_start = ABSOLUTE(.); - *(.rodata) - *(.rodata.*) - *(.gnu.linkonce.r.*) - *(.rodata1) - __XT_EXCEPTION_TABLE__ = ABSOLUTE(.); - KEEP (*(.xt_except_table)) - KEEP (*(.gcc_except_table .gcc_except_table.*)) - *(.gnu.linkonce.e.*) - *(.gnu.version_r) - KEEP (*(.eh_frame)) - /* C++ constructor and destructor tables, properly ordered: */ - KEEP (*crtbegin.o(.ctors)) - KEEP (*(EXCLUDE_FILE (*crtend.o) .ctors)) - KEEP (*(SORT(.ctors.*))) - KEEP (*(.ctors)) - KEEP (*crtbegin.o(.dtors)) - KEEP (*(EXCLUDE_FILE (*crtend.o) .dtors)) - KEEP (*(SORT(.dtors.*))) - KEEP (*(.dtors)) - /* C++ exception handlers table: */ - __XT_EXCEPTION_DESCS__ = ABSOLUTE(.); - *(.xt_except_desc) - *(.gnu.linkonce.h.*) - __XT_EXCEPTION_DESCS_END__ = ABSOLUTE(.); - *(.xt_except_desc_end) - *(.dynamic) - *(.gnu.version_d) - . = ALIGN(4); /* this table MUST be 4-byte aligned */ - _rodata_end = ABSOLUTE(.); - } GROUP_LINK_IN(RAMABLE_REGION) - - - /* Shared RAM */ - SECTION_DATA_PROLOGUE(_BSS_SECTION_NAME,(NOLOAD),) - { - . = ALIGN (8); - _bss_start = ABSOLUTE(.); - *(.dynsbss) - *(.sbss) - *(.sbss.*) - *(.gnu.linkonce.sb.*) - *(.scommon) - *(.sbss2) - *(.sbss2.*) - *(.gnu.linkonce.sb2.*) - *(.dynbss) - *(.bss) - *(.bss.*) - *(.share.mem) - *(.gnu.linkonce.b.*) - *(COMMON) - . = ALIGN (8); - _bss_end = ABSOLUTE(.); - } GROUP_LINK_IN(RAMABLE_REGION) - - - SECTION_DATA_PROLOGUE(_APP_NOINIT_SECTION_NAME, (NOLOAD),) - { - . = ALIGN (8); - *(.app_noinit) - *("app_noinit.*") - . = ALIGN (8); - _app_end = ABSOLUTE(.); - } GROUP_LINK_IN(RAMABLE_REGION) - - - SECTION_DATA_PROLOGUE(_NOINIT_SECTION_NAME, (NOLOAD),) - { - . = ALIGN (8); - *(.noinit) - *(".noinit.*") - . = ALIGN (8); - _heap_start = ABSOLUTE(.); - } GROUP_LINK_IN(RAMABLE_REGION) - -#ifdef CONFIG_GEN_ISR_TABLES -#include -#endif - -#include - - SECTION_PROLOGUE(.xtensa.info, 0,) - { - *(.xtensa.info) - } - -} diff --git a/product-mini/platforms/zephyr/simple/boards/qemu_arc.conf b/product-mini/platforms/zephyr/simple/prj.conf similarity index 100% rename from product-mini/platforms/zephyr/simple/boards/qemu_arc.conf rename to product-mini/platforms/zephyr/simple/prj.conf diff --git a/product-mini/platforms/zephyr/simple/src/main.c b/product-mini/platforms/zephyr/simple/src/main.c index 6c0b8fc82..5d209a868 100644 --- a/product-mini/platforms/zephyr/simple/src/main.c +++ b/product-mini/platforms/zephyr/simple/src/main.c @@ -104,56 +104,6 @@ app_instance_main(wasm_module_inst_t module_inst) static char global_heap_buf[CONFIG_GLOBAL_HEAP_BUF_SIZE] = { 0 }; #endif -#ifdef CONFIG_BOARD_ESP32 -#include "mem_alloc.h" -/* -esp32_technical_reference_manual: -" -The capacity of Internal SRAM 1 is 128 KB. Either CPU can read and write this -memory at addresses 0x3FFE_0000 ~ 0x3FFF_FFFF of the data bus, and also at -addresses 0x400A_0000 ~ 0x400B_FFFF of the instruction bus. -" - -The custom linker script defines dram0_1_seg and map it to 0x400A_0000 ~ -0x400B_FFFF for instruction bus access. Here we define the buffer that will be -placed to dram0_1_seg. -*/ -static char esp32_executable_memory_buf[100 * 1024] - __attribute__((section(".aot_code_buf"))) = { 0 }; - -/* the poll allocator for executable memory */ -static mem_allocator_t esp32_exec_mem_pool_allocator; - -static int -esp32_exec_mem_init() -{ - if (!(esp32_exec_mem_pool_allocator = - mem_allocator_create(esp32_executable_memory_buf, - sizeof(esp32_executable_memory_buf)))) - return -1; - - return 0; -} - -static void -esp32_exec_mem_destroy() -{ - mem_allocator_destroy(esp32_exec_mem_pool_allocator); -} - -static void * -esp32_exec_mem_alloc(unsigned int size) -{ - return mem_allocator_malloc(esp32_exec_mem_pool_allocator, size); -} - -static void -esp32_exec_mem_free(void *addr) -{ - mem_allocator_free(esp32_exec_mem_pool_allocator, addr); -} -#endif /* end of #ifdef CONFIG_BOARD_ESP32 */ - void iwasm_main(void *arg1, void *arg2, void *arg3) { @@ -189,16 +139,6 @@ iwasm_main(void *arg1, void *arg2, void *arg3) return; } -#ifdef CONFIG_BOARD_ESP32 - /* Initialize executable memory */ - if (esp32_exec_mem_init() != 0) { - printf("Init executable memory failed.\n"); - goto fail1; - } - /* Set hook functions for executable memory management */ - set_exec_mem_alloc_func(esp32_exec_mem_alloc, esp32_exec_mem_free); -#endif - #if WASM_ENABLE_LOG != 0 bh_log_set_verbose_level(log_verbose_level); #endif @@ -211,11 +151,7 @@ iwasm_main(void *arg1, void *arg2, void *arg3) if (!(wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, error_buf, sizeof(error_buf)))) { printf("%s\n", error_buf); -#ifdef CONFIG_BOARD_ESP32 - goto fail1_1; -#else goto fail1; -#endif } /* instantiate the module */ @@ -236,12 +172,6 @@ fail2: /* unload the module */ wasm_runtime_unload(wasm_module); -#ifdef CONFIG_BOARD_ESP32 -fail1_1: - /* destroy executable memory */ - esp32_exec_mem_destroy(); -#endif - fail1: /* destroy runtime environment */ wasm_runtime_destroy(); From cb44e56b4ef93f9c75a3ed576e1372cc94385e9d Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Thu, 23 Nov 2023 10:23:33 +0800 Subject: [PATCH 23/67] Fix Zifencei extension issue in RISC-V (#2807) Refer to: https://github.com/bytecodealliance/wasm-micro-runtime/pull/2805 --- core/iwasm/aot/arch/aot_reloc_riscv.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/iwasm/aot/arch/aot_reloc_riscv.c b/core/iwasm/aot/arch/aot_reloc_riscv.c index 7798d55c6..d765e0b87 100644 --- a/core/iwasm/aot/arch/aot_reloc_riscv.c +++ b/core/iwasm/aot/arch/aot_reloc_riscv.c @@ -177,7 +177,11 @@ rv_set_val(uint16 *addr, uint32 val) *addr = (val & 0xffff); *(addr + 1) = (val >> 16); +#ifdef __riscv_zifencei __asm__ volatile("fence.i"); +#else + __asm__ volatile("fence"); +#endif } /* Add a val to given address */ From 2a3c93f5024eb123014270f8268d466f06e466ef Mon Sep 17 00:00:00 2001 From: Daniel Mangum <31777345+hasheddan@users.noreply.github.com> Date: Thu, 23 Nov 2023 18:47:38 -0600 Subject: [PATCH 24/67] Fix typos in zephyr platform struct descriptions (#2818) Fixes typos in zephyr platform struct descriptions. Signed-off-by: Daniel Mangum --- core/shared/platform/zephyr/platform_internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/shared/platform/zephyr/platform_internal.h b/core/shared/platform/zephyr/platform_internal.h index 2306fa06b..a5d563a6c 100644 --- a/core/shared/platform/zephyr/platform_internal.h +++ b/core/shared/platform/zephyr/platform_internal.h @@ -131,7 +131,7 @@ float strtof(const char *nptr, char **endptr); #endif /** - * @brief Allocate executable memroy + * @brief Allocate executable memory * * @param size size of the memory to be allocated * @@ -140,7 +140,7 @@ float strtof(const char *nptr, char **endptr); typedef void *(*exec_mem_alloc_func_t)(unsigned int size); /** - * @brief Release executable memroy + * @brief Release executable memory * * @param the address of the executable memory to be released */ From 1ba4acd1c76aabe414f49e261b2fcf70f4238f87 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Fri, 24 Nov 2023 16:21:05 +0800 Subject: [PATCH 25/67] Fix wamr-test-suites script for macos (#2819) Update the wabt release binary name for macos and fix a syntax error: ```bash ./test_wamr.sh: line 356: syntax error near unexpected token `;' ./test_wamr.sh: line 356: ` ;&' ``` --- tests/wamr-test-suites/test_wamr.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index d7991ddf0..82e7a015c 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -329,12 +329,12 @@ function setup_wabt() if [ ! -f ${WAT2WASM} ]; then case ${PLATFORM} in cosmopolitan) - ;& + ;; linux) WABT_PLATFORM=ubuntu ;; darwin) - WABT_PLATFORM=macos + WABT_PLATFORM=macos-12 ;; windows) WABT_PLATFORM=windows From 5f7079f0f5800315fdb2685448d12f8fbdae6b5e Mon Sep 17 00:00:00 2001 From: Marcin Kolny Date: Fri, 24 Nov 2023 13:03:59 +0000 Subject: [PATCH 26/67] Return error when shutdown() fails (#2801) Fix issue reported in #2787. --- .../sandboxed-system-primitives/src/posix.c | 7 +-- .../common/posix/platform_api_posix.cmake | 3 +- .../platform/common/posix/posix_socket.c | 9 ++-- core/shared/platform/esp-idf/espidf_socket.c | 9 ++-- .../platform/include/platform_api_extension.h | 4 +- core/shared/platform/linux-sgx/sgx_socket.c | 8 ++- core/shared/platform/windows/win_file.c | 46 +--------------- core/shared/platform/windows/win_socket.c | 10 ++-- core/shared/platform/windows/win_util.c | 52 ++++++++++++++++++- core/shared/platform/windows/win_util.h | 6 ++- 10 files changed, 88 insertions(+), 66 deletions(-) diff --git a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c index 8fcf455a4..0f8f4b32b 100644 --- a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c +++ b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c @@ -2877,18 +2877,15 @@ wasmtime_ssp_sock_shutdown(wasm_exec_env_t exec_env, struct fd_table *curfds, { struct fd_object *fo; __wasi_errno_t error; - int ret; error = fd_object_get(curfds, &fo, sock, 0, 0); if (error != 0) return error; - ret = os_socket_shutdown(fo->file_handle); + error = os_socket_shutdown(fo->file_handle); fd_object_release(exec_env, fo); - if (BHT_OK != ret) - return convert_errno(errno); - return __WASI_ESUCCESS; + return error; } __wasi_errno_t diff --git a/core/shared/platform/common/posix/platform_api_posix.cmake b/core/shared/platform/common/posix/platform_api_posix.cmake index 258be4632..2bf9fab4a 100644 --- a/core/shared/platform/common/posix/platform_api_posix.cmake +++ b/core/shared/platform/common/posix/platform_api_posix.cmake @@ -1,6 +1,6 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - + set (PLATFORM_COMMON_POSIX_DIR ${CMAKE_CURRENT_LIST_DIR}) file (GLOB_RECURSE source_all ${PLATFORM_COMMON_POSIX_DIR}/*.c) @@ -9,6 +9,7 @@ if (NOT WAMR_BUILD_LIBC_WASI EQUAL 1) list(REMOVE_ITEM source_all ${PLATFORM_COMMON_POSIX_DIR}/posix_file.c ${PLATFORM_COMMON_POSIX_DIR}/posix_clock.c + ${PLATFORM_COMMON_POSIX_DIR}/posix_socket.c ) else() include (${CMAKE_CURRENT_LIST_DIR}/../libc-util/platform_common_libc_util.cmake) diff --git a/core/shared/platform/common/posix/posix_socket.c b/core/shared/platform/common/posix/posix_socket.c index 11c56e723..7bdcb529e 100644 --- a/core/shared/platform/common/posix/posix_socket.c +++ b/core/shared/platform/common/posix/posix_socket.c @@ -5,6 +5,7 @@ #include "platform_api_vmcore.h" #include "platform_api_extension.h" +#include "libc_errno.h" #include #include @@ -308,11 +309,13 @@ os_socket_close(bh_socket_t socket) return BHT_OK; } -int +__wasi_errno_t os_socket_shutdown(bh_socket_t socket) { - shutdown(socket, O_RDWR); - return BHT_OK; + if (shutdown(socket, O_RDWR) != 0) { + return convert_errno(errno); + } + return __WASI_ESUCCESS; } int diff --git a/core/shared/platform/esp-idf/espidf_socket.c b/core/shared/platform/esp-idf/espidf_socket.c index 83a24f435..a75d82975 100644 --- a/core/shared/platform/esp-idf/espidf_socket.c +++ b/core/shared/platform/esp-idf/espidf_socket.c @@ -5,6 +5,7 @@ #include "platform_api_vmcore.h" #include "platform_api_extension.h" +#include "libc_errno.h" #include @@ -167,11 +168,13 @@ os_socket_close(bh_socket_t socket) return BHT_OK; } -int +__wasi_errno_t os_socket_shutdown(bh_socket_t socket) { - shutdown(socket, O_RDWR); - return BHT_OK; + if (shutdown(socket, O_RDWR) != 0) { + return convert_errno(errno); + } + return __WASI_ESUCCESS; } int diff --git a/core/shared/platform/include/platform_api_extension.h b/core/shared/platform/include/platform_api_extension.h index 2795fc6ac..7c6120ba2 100644 --- a/core/shared/platform/include/platform_api_extension.h +++ b/core/shared/platform/include/platform_api_extension.h @@ -573,9 +573,9 @@ os_socket_close(bh_socket_t socket); * * @param socket the socket to be shutdown * - * @return always return 0 + * @return returns corresponding error code */ -int +__wasi_errno_t os_socket_shutdown(bh_socket_t socket); /** diff --git a/core/shared/platform/linux-sgx/sgx_socket.c b/core/shared/platform/linux-sgx/sgx_socket.c index 4d3f9a6b5..cd1a6cead 100644 --- a/core/shared/platform/linux-sgx/sgx_socket.c +++ b/core/shared/platform/linux-sgx/sgx_socket.c @@ -5,6 +5,7 @@ #include "platform_api_vmcore.h" #include "platform_api_extension.h" +#include "libc_errno.h" #ifndef SGX_DISABLE_WASI @@ -855,10 +856,13 @@ os_socket_send_to(bh_socket_t socket, const void *buf, unsigned int len, return ret; } -int +__wasi_errno_t os_socket_shutdown(bh_socket_t socket) { - return shutdown(socket, O_RDWR); + if (shutdown(socket, O_RDWR) != 0) { + return convert_errno(errno); + } + return __WASI_ESUCCESS; } int diff --git a/core/shared/platform/windows/win_file.c b/core/shared/platform/windows/win_file.c index 7e153b34e..2c4e19f8b 100644 --- a/core/shared/platform/windows/win_file.c +++ b/core/shared/platform/windows/win_file.c @@ -46,50 +46,6 @@ CHECK_VALID_FILE_HANDLE((win_dir_stream)->handle); \ } while (0) -static __wasi_errno_t -convert_winsock_error_code(int error_code) -{ - switch (error_code) { - case WSASYSNOTREADY: - case WSAEWOULDBLOCK: - return __WASI_EAGAIN; - case WSAVERNOTSUPPORTED: - return __WASI_ENOTSUP; - case WSAEINPROGRESS: - return __WASI_EINPROGRESS; - case WSAEPROCLIM: - return __WASI_EBUSY; - case WSAEFAULT: - return __WASI_EFAULT; - case WSAENETDOWN: - return __WASI_ENETDOWN; - case WSAENOTSOCK: - return __WASI_ENOTSOCK; - case WSAEINTR: - return __WASI_EINTR; - case WSAEAFNOSUPPORT: - return __WASI_EAFNOSUPPORT; - case WSAEMFILE: - return __WASI_ENFILE; - case WSAEINVAL: - return __WASI_EINVAL; - case WSAENOBUFS: - return __WASI_ENOBUFS; - case WSAEPROTONOSUPPORT: - return __WASI_EPROTONOSUPPORT; - case WSAEPROTOTYPE: - return __WASI_EPROTOTYPE; - case WSAESOCKTNOSUPPORT: - return __WASI_ENOTSUP; - case WSAEINVALIDPROCTABLE: - case WSAEINVALIDPROVIDER: - case WSAEPROVIDERFAILEDINIT: - case WSANOTINITIALISED: - default: - return __WASI_EINVAL; - } -} - static __wasi_filetype_t get_disk_filetype(DWORD attribute) { @@ -1488,4 +1444,4 @@ os_realpath(const char *path, char *resolved_path) return NULL; return resolved_path; -} \ No newline at end of file +} diff --git a/core/shared/platform/windows/win_socket.c b/core/shared/platform/windows/win_socket.c index bc05b0017..91d38fd8b 100644 --- a/core/shared/platform/windows/win_socket.c +++ b/core/shared/platform/windows/win_socket.c @@ -5,6 +5,8 @@ #include "platform_api_vmcore.h" #include "platform_api_extension.h" +#include "platform_wasi_types.h" +#include "win_util.h" /* link with Ws2_32.lib */ #pragma comment(lib, "ws2_32.lib") @@ -238,13 +240,15 @@ os_socket_close(bh_socket_t socket) return BHT_OK; } -int +__wasi_errno_t os_socket_shutdown(bh_socket_t socket) { CHECK_VALID_SOCKET_HANDLE(socket); - shutdown(socket->raw.socket, SD_BOTH); - return BHT_OK; + if (shutdown(socket->raw.socket, SD_BOTH) != 0) { + return convert_winsock_error_code(WSAGetLastError()); + } + return __WASI_ESUCCESS; } int diff --git a/core/shared/platform/windows/win_util.c b/core/shared/platform/windows/win_util.c index 50c446d69..ee0e82fb9 100644 --- a/core/shared/platform/windows/win_util.c +++ b/core/shared/platform/windows/win_util.c @@ -93,4 +93,54 @@ uwp_print_to_debugger(const char *format, va_list ap) return ret; } -#endif \ No newline at end of file +#endif + +__wasi_errno_t +convert_winsock_error_code(int error_code) +{ + switch (error_code) { + case WSASYSNOTREADY: + case WSAEWOULDBLOCK: + return __WASI_EAGAIN; + case WSAVERNOTSUPPORTED: + return __WASI_ENOTSUP; + case WSAEINPROGRESS: + return __WASI_EINPROGRESS; + case WSAEPROCLIM: + return __WASI_EBUSY; + case WSAEFAULT: + return __WASI_EFAULT; + case WSAENETDOWN: + return __WASI_ENETDOWN; + case WSAENOTSOCK: + return __WASI_ENOTSOCK; + case WSAEINTR: + return __WASI_EINTR; + case WSAEAFNOSUPPORT: + return __WASI_EAFNOSUPPORT; + case WSAEMFILE: + return __WASI_ENFILE; + case WSAEINVAL: + return __WASI_EINVAL; + case WSAENOBUFS: + return __WASI_ENOBUFS; + case WSAEPROTONOSUPPORT: + return __WASI_EPROTONOSUPPORT; + case WSAEPROTOTYPE: + return __WASI_EPROTOTYPE; + case WSAESOCKTNOSUPPORT: + return __WASI_ENOTSUP; + case WSAECONNABORTED: + return __WASI_ECONNABORTED; + case WSAECONNRESET: + return __WASI_ECONNRESET; + case WSAENOTCONN: + return __WASI_ENOTCONN; + case WSAEINVALIDPROCTABLE: + case WSAEINVALIDPROVIDER: + case WSAEPROVIDERFAILEDINIT: + case WSANOTINITIALISED: + default: + return __WASI_EINVAL; + } +} diff --git a/core/shared/platform/windows/win_util.h b/core/shared/platform/windows/win_util.h index 0733b076e..3960fe946 100644 --- a/core/shared/platform/windows/win_util.h +++ b/core/shared/platform/windows/win_util.h @@ -12,8 +12,12 @@ __wasi_timestamp_t convert_filetime_to_wasi_timestamp(LPFILETIME filetime); -// Convert a Windows error code to a WASI error code +/* Convert a Windows error code to a WASI error code */ __wasi_errno_t convert_windows_error_code(DWORD windows_error_code); +/* Convert a Winsock error code to a WASI error code */ +__wasi_errno_t +convert_winsock_error_code(int error_code); + #endif /* end of _WIN_UTIL_H */ \ No newline at end of file From 5377e186235f5780c8f26e174c849049620641b4 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Mon, 27 Nov 2023 11:16:54 +0800 Subject: [PATCH 27/67] iwasm: Print help when meeting unknown cmd options (#2824) --- product-mini/platforms/common/libc_wasi.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/product-mini/platforms/common/libc_wasi.c b/product-mini/platforms/common/libc_wasi.c index 4bcb1d279..2064c4431 100644 --- a/product-mini/platforms/common/libc_wasi.c +++ b/product-mini/platforms/common/libc_wasi.c @@ -156,6 +156,9 @@ libc_wasi_parse(char *arg, libc_wasi_parse_context_t *ctx) } ctx->ns_lookup_pool[ctx->ns_lookup_pool_size++] = arg + 16; } + else { + return LIBC_WASI_PARSE_RESULT_NEED_HELP; + } return LIBC_WASI_PARSE_RESULT_OK; } From d7608690c08d2755bf10965539d0354b320e3038 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Mon, 27 Nov 2023 15:24:02 +0800 Subject: [PATCH 28/67] Run spec test for classic/fast-interp in NuttX CI (#2817) --- .github/workflows/spec_test_on_nuttx.yml | 125 ++++++++++++------ product-mini/platforms/nuttx/wamr.mk | 2 + .../wamr-test-suites/spec-test-script/all.py | 47 ++++--- .../spec-test-script/runtest.py | 66 +++++---- tests/wamr-test-suites/test_wamr.sh | 10 +- 5 files changed, 155 insertions(+), 95 deletions(-) diff --git a/.github/workflows/spec_test_on_nuttx.yml b/.github/workflows/spec_test_on_nuttx.yml index d7c4246fe..12269b562 100644 --- a/.github/workflows/spec_test_on_nuttx.yml +++ b/.github/workflows/spec_test_on_nuttx.yml @@ -18,6 +18,8 @@ on: env: LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex" WASI_SDK_PATH: "/opt/wasi-sdk" + WAMR_COMMON_OPTION: + "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_STACKSIZE=32768\\nCONFIG_INTERPRETERS_WAMR_LOG=y\\nCONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN=y\\nCONFIG_INTERPRETERS_WAMR_REF_TYPES=y\\nCONFIG_INTERPRETERS_WAMR_ENABLE_SPEC_TEST=y\\nCONFIG_INTERPRETERS_WAMR_SHARED_MEMORY=y\\nCONFIG_INTERPRETERS_WAMR_BULK_MEMORY=y\\nCONFIG_EOL_IS_LF=y\\nCONFIG_ARM_SEMIHOSTING_HOSTFS=y\\nCONFIG_ARM_SEMIHOSTING_HOSTFS_CACHE_COHERENCE=y\\nCONFIG_RISCV_SEMIHOSTING_HOSTFS=y\\nCONFIG_FS_HOSTFS=y\\nCONFIG_LIBC_FLOATINGPOINT=y\\n" jobs: build_llvm_libraries: @@ -27,36 +29,78 @@ jobs: arch: "ARM RISCV AArch64" spec_test_on_qemu: - runs-on: ${{ matrix.os }} + runs-on: ubuntu-22.04 needs: [build_llvm_libraries] strategy: matrix: - os: [ubuntu-22.04] - nuttx_board_config: [ - # cortex-a9 - "boards/arm/imx6/sabre-6quad/configs/nsh", - # riscv32imac - "boards/risc-v/qemu-rv/rv-virt/configs/nsh", - # riscv64imac - # "boards/risc-v/qemu-rv/rv-virt/configs/nsh64", + target_config: [ + # { + # config: "boards/arm64/qemu/qemu-armv8a/configs/nsh", + # target: "aarch64_vfp", + # use_fpu: true + # }, + # { + # config: "boards/arm/imx6/sabre-6quad/configs/nsh", + # target: "thumbv7", + # use_fpu: false + # }, + { + config: "boards/arm/imx6/sabre-6quad/configs/nsh", + target: "thumbv7_vfp", + use_fpu: true + }, + { + config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh", + target: "riscv32", + use_fpu: false + }, + # { + # config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh", + # target: "riscv32_ilp32d", + # use_fpu: true + # }, + # { + # config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh64", + # target: "riscv64", + # use_fpu: false + # }, ] + wamr_test_option: [ - # "-t fast-interp", - "-t aot", - "-t aot -X" + { + mode: "-t aot", + option: "CONFIG_INTERPRETERS_WAMR_AOT=y\\n" + }, + { + mode: "-t aot -X", + option: "CONFIG_INTERPRETERS_WAMR_AOT=y\\n" + }, + { + mode: "-t classic-interp", + option: "CONFIG_INTERPRETERS_WAMR_CLASSIC=y\\n" + }, + { + mode: "-t fast-interp", + option: "CONFIG_INTERPRETERS_WAMR_FAST=y\\n" + }, ] - llvm_cache_key: [ "${{ needs.build_llvm_libraries.outputs.cache_key }}" ] + steps: - name: Install Utilities run: | sudo apt install -y kconfig-frontends-nox genromfs - name: Install ARM Compilers - if: contains(matrix.nuttx_board_config, 'arm') - run: sudo apt install -y gcc-arm-none-eabi + if: startsWith(matrix.target_config.config, 'boards/arm') + run: | + sudo apt install -y gcc-arm-none-eabi + wget --quiet https://developer.arm.com/-/media/Files/downloads/gnu/11.2-2022.02/binrel/gcc-arm-11.2-2022.02-x86_64-aarch64-none-elf.tar.xz + xz -d gcc-arm-11.2-2022.02-x86_64-aarch64-none-elf.tar.xz + tar xf gcc-arm-11.2-2022.02-x86_64-aarch64-none-elf.tar + echo "$PWD/gcc-arm-11.2-2022.02-x86_64-aarch64-none-elf/bin" >> $GITHUB_PATH - name: Install RISC-V Compilers - if: contains(matrix.nuttx_board_config, 'risc-v') + if: startsWith(matrix.target_config.config, 'boards/risc-v') run: | curl -L https://github.com/xpack-dev-tools/riscv-none-elf-gcc-xpack/releases/download/v12.3.0-1/xpack-riscv-none-elf-gcc-12.3.0-1-linux-x64.tar.gz > riscv.tar.gz tar xvf riscv.tar.gz @@ -87,6 +131,7 @@ jobs: path: apps/interpreters/wamr/wamr - name: Get LLVM libraries + if: contains(matrix.wamr_test_option.mode, 'aot') id: retrieve_llvm_libs uses: actions/cache@v3 with: @@ -96,61 +141,59 @@ jobs: ./core/deps/llvm/build/lib ./core/deps/llvm/build/libexec ./core/deps/llvm/build/share - key: ${{ matrix.llvm_cache_key }} + key: ${{ needs.build_llvm_libraries.outputs.cache_key }} - name: Quit if cache miss - if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' + if: contains(matrix.wamr_test_option.mode, 'aot') && steps.retrieve_llvm_libs.outputs.cache-hit != 'true' run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - name: Copy LLVM + if: contains(matrix.wamr_test_option.mode, 'aot') run: cp -r core/deps/llvm apps/interpreters/wamr/wamr/core/deps/llvm - name: Enable WAMR for NuttX run: | - find nuttx/boards -name defconfig | xargs sed -i '$a\CONFIG_INTERPRETERS_WAMR=y\nCONFIG_INTERPRETERS_WAMR_STACKSIZE=32768\nCONFIG_INTERPRETERS_WAMR_AOT=y\nCONFIG_INTERPRETERS_WAMR_FAST=y\nCONFIG_INTERPRETERS_WAMR_LOG=y\nCONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN=y\nCONFIG_INTERPRETERS_WAMR_REF_TYPES=y\nCONFIG_INTERPRETERS_WAMR_ENABLE_SPEC_TEST=y\nCONFIG_INTERPRETERS_WAMR_SHARED_MEMORY=y\nCONFIG_INTERPRETERS_WAMR_BULK_MEMORY=y\n' - find nuttx/boards -name defconfig | xargs sed -i '$a\CONFIG_EOL_IS_LF=y\nCONFIG_ARM_SEMIHOSTING_HOSTFS=y\nCONFIG_ARM_SEMIHOSTING_HOSTFS_CACHE_COHERENCE=y\nCONFIG_RISCV_SEMIHOSTING_HOSTFS=y\nCONFIG_FS_HOSTFS=y\nCONFIG_LIBC_FLOATINGPOINT=y\n' + find nuttx/boards -name defconfig | xargs sed -i '$a\${{ env.WAMR_COMMON_OPTION }}' + + - name: Enable WAMR Interpreter for NuttX + run: | + find nuttx/boards -name defconfig | xargs sed -i '$a\${{ matrix.wamr_test_option.option }}' - - name: Enable additional features for NuttX on RI5C-V - if: startsWith(matrix.nuttx_board_config, 'boards/risc-v') + - name: Disable FPU for NuttX + if: matrix.target_config.use_fpu== false run: | find nuttx/boards -name defconfig | xargs sed -i '$a\# CONFIG_ARCH_FPU is not set\n' - name: Build wamrc + if: contains(matrix.wamr_test_option.mode, 'aot') working-directory: apps/interpreters/wamr/wamr/wamr-compiler run: | cmake -Bbuild . cmake --build build - name: Build + id: build_firmware run: | cd nuttx - tools/configure.sh ${{ matrix.nuttx_board_config }} + tools/configure.sh ${{ matrix.target_config.config }} make -j$(nproc) - echo "firmware=$PWD/nuttx" >> $GITHUB_ENV + echo "firmware=$PWD/nuttx" >> $GITHUB_OUTPUT - - name: Test on ARM - if: endsWith(matrix.nuttx_board_config, 'sabre-6quad/configs/nsh') + - name: Install QEMU for ARM + if: startsWith(matrix.target_config.config, 'boards/arm') run: | curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v7.1.0-1/xpack-qemu-arm-7.1.0-1-linux-x64.tar.gz > xpack-qemu-arm.tar.gz tar xvf xpack-qemu-arm.tar.gz - export PATH=$PATH:$PWD/xpack-qemu-arm-7.1.0-1/bin - cd apps/interpreters/wamr/wamr/tests/wamr-test-suites - ./test_wamr.sh -s spec ${{ matrix.wamr_test_option }} -m thumbv7_vfp -b -Q -P -F ${{ env.firmware }} + echo $PWD/xpack-qemu-arm-7.1.0-1/bin >> $GITHUB_PATH - - name: Test on RISCV32 - if: endsWith(matrix.nuttx_board_config, 'rv-virt/configs/nsh') + - name: Install QEMU for RISC-V + if: startsWith(matrix.target_config.config, 'boards/risc-v') run: | curl -L https://github.com/xpack-dev-tools/qemu-riscv-xpack/releases/download/v7.1.0-1/xpack-qemu-riscv-7.1.0-1-linux-x64.tar.gz > xpack-qemu-riscv.tar.gz tar xvf xpack-qemu-riscv.tar.gz - export PATH=$PATH:$PWD/xpack-qemu-riscv-7.1.0-1/bin - cd apps/interpreters/wamr/wamr/tests/wamr-test-suites - ./test_wamr.sh -s spec ${{ matrix.wamr_test_option }} -m riscv32 -b -Q -P -F ${{ env.firmware }} - - - name: Test on RISCV64 - if: endsWith(matrix.nuttx_board_config, 'rv-virt/configs/nsh64') + echo PATH=$PATH:$PWD/xpack-qemu-riscv-7.1.0-1/bin >> $GITHUB_PATH + + - name: Test run: | - curl -L https://github.com/xpack-dev-tools/qemu-riscv-xpack/releases/download/v7.1.0-1/xpack-qemu-riscv-7.1.0-1-linux-x64.tar.gz > xpack-qemu-riscv.tar.gz - tar xvf xpack-qemu-riscv.tar.gz - export PATH=$PATH:$PWD/xpack-qemu-riscv-7.1.0-1/bin cd apps/interpreters/wamr/wamr/tests/wamr-test-suites - ./test_wamr.sh -s spec ${{ matrix.wamr_test_option }} -m riscv64 -b -Q -P -F ${{ env.firmware }} + ./test_wamr.sh -s spec ${{ matrix.wamr_test_option.mode }} -m ${{ matrix.target_config.target }} -b -Q -P -F ${{ steps.build_firmware.outputs.firmware }} diff --git a/product-mini/platforms/nuttx/wamr.mk b/product-mini/platforms/nuttx/wamr.mk index 57995c2bf..b91fac9a8 100644 --- a/product-mini/platforms/nuttx/wamr.mk +++ b/product-mini/platforms/nuttx/wamr.mk @@ -13,6 +13,8 @@ else ifeq ($(CONFIG_ARCH_ARMV7M),y) WAMR_BUILD_TARGET := THUMBV7EM else ifeq ($(CONFIG_ARCH_ARMV8M),y) WAMR_BUILD_TARGET := THUMBV8M +else ifeq ($(CONFIG_ARCH_ARM64),y) +WAMR_BUILD_TARGET := AARCH64 else ifeq ($(CONFIG_ARCH_X86),y) WAMR_BUILD_TARGET := X86_32 else ifeq ($(CONFIG_ARCH_X86_64),y) diff --git a/tests/wamr-test-suites/spec-test-script/all.py b/tests/wamr-test-suites/spec-test-script/all.py index 36d5c9bcd..551a3176c 100644 --- a/tests/wamr-test-suites/spec-test-script/all.py +++ b/tests/wamr-test-suites/spec-test-script/all.py @@ -50,26 +50,23 @@ SPEC_TEST_DIR = "spec/test/core" WAST2WASM_CMD = exe_file_path("./wabt/out/gcc/Release/wat2wasm") SPEC_INTERPRETER_CMD = "spec/interpreter/wasm" WAMRC_CMD = "../../../wamr-compiler/build/wamrc" - - -class TargetAction(argparse.Action): - TARGET_MAP = { - "ARMV7_VFP": "armv7", - "RISCV32": "riscv32_ilp32", - "RISCV32_ILP32": "riscv32_ilp32", - "RISCV32_ILP32D": "riscv32_ilp32d", - "RISCV64": "riscv64_lp64", - "RISCV64_LP64": "riscv64_lp64", - "RISCV64_LP64D": "riscv64_lp64", - "THUMBV7_VFP": "thumbv7", - "X86_32": "i386", - "X86_64": "x86_64", - "AARCH64": "arm64" - } - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, self.TARGET_MAP.get(values, "x86_64")) - +AVAILABLE_TARGETS = [ + "I386", + "X86_32", + "X86_64", + "AARCH64", + "AARCH64_VFP", + "ARMV7", + "ARMV7_VFP", + "RISCV32", + "RISCV32_ILP32F", + "RISCV32_ILP32D", + "RISCV64", + "RISCV64_LP64F", + "RISCV64_LP64D", + "THUMBV7", + "THUMBV7_VFP", +] def ignore_the_case( case_name, @@ -404,8 +401,7 @@ def main(): ) parser.add_argument( "-m", - action=TargetAction, - choices=list(TargetAction.TARGET_MAP.keys()), + choices=AVAILABLE_TARGETS, type=str, dest="target", default="X86_64", @@ -505,6 +501,13 @@ def main(): options = parser.parse_args() + # Convert target to lower case for internal use, e.g. X86_64 -> x86_64 + # target is always exist, so no need to check it + options.target = options.target.lower() + + if options.target == "x86_32": + options.target = "i386" + if not preflight_check(options.aot_flag): return False diff --git a/tests/wamr-test-suites/spec-test-script/runtest.py b/tests/wamr-test-suites/spec-test-script/runtest.py index 0eab6aeed..ca7dfcd9a 100755 --- a/tests/wamr-test-suites/spec-test-script/runtest.py +++ b/tests/wamr-test-suites/spec-test-script/runtest.py @@ -27,7 +27,9 @@ else: IS_PY_3 = True test_aot = False -# "x86_64", "i386", "aarch64", "armv7", "thumbv7", "riscv32_ilp32", "riscv32_ilp32d", "riscv32_lp64", "riscv64_lp64d" +# Available targets: +# "aarch64" "aarch64_vfp" "armv7" "armv7_vfp" "thumbv7" "thumbv7_vfp" +# "riscv32" "riscv32_ilp32f" "riscv32_ilp32d" "riscv64" "riscv64_lp64f" "riscv64_lp64d" test_target = "x86_64" debug_file = None @@ -39,6 +41,25 @@ temp_file_repo = [] # to save the mapping of module files in /tmp by name temp_module_table = {} +# AOT compilation options mapping +aot_target_options_map = { + "i386": ["--target=i386"], + "x86_32": ["--target=i386"], + "x86_64": ["--target=x86_64", "--cpu=skylake"], + "aarch64": ["--target=aarch64", "--target-abi=eabi", "--cpu=cortex-a53"], + "aarch64_vfp": ["--target=aarch64", "--target-abi=gnueabihf", "--cpu=cortex-a53"], + "armv7": ["--target=armv7", "--target-abi=eabi", "--cpu=cortex-a9", "--cpu-features=-neon"], + "armv7_vfp": ["--target=armv7", "--target-abi=gnueabihf", "--cpu=cortex-a9"], + "thumbv7": ["--target=thumbv7", "--target-abi=eabi", "--cpu=cortex-a9", "--cpu-features=-neon,-vfpv3"], + "thumbv7_vfp": ["--target=thumbv7", "--target-abi=gnueabihf", "--cpu=cortex-a9", "--cpu-features=-neon"], + "riscv32": ["--target=riscv32", "--target-abi=ilp32", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c"], + "riscv32_ilp32f": ["--target=riscv32", "--target-abi=ilp32f", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c,+f"], + "riscv32_ilp32d": ["--target=riscv32", "--target-abi=ilp32d", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c,+f,+d"], + "riscv64": ["--target=riscv64", "--target-abi=lp64", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c"], + "riscv64_lp64f": ["--target=riscv64", "--target-abi=lp64f", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c,+f"], + "riscv64_lp64d": ["--target=riscv64", "--target-abi=lp64d", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c,+f,+d"], +} + def debug(data): if debug_file: debug_file.write(data) @@ -1025,27 +1046,8 @@ def compile_wasm_to_aot(wasm_tempfile, aot_tempfile, runner, opts, r, output = ' log("Compiling AOT to '%s'" % aot_tempfile) cmd = [opts.aot_compiler] - if test_target == "x86_64": - cmd.append("--target=x86_64") - cmd.append("--cpu=skylake") - elif test_target == "i386": - cmd.append("--target=i386") - elif test_target == "aarch64": - cmd += ["--target=aarch64", "--cpu=cortex-a57"] - elif test_target == "armv7": - cmd += ["--target=armv7", "--target-abi=gnueabihf"] - elif test_target == "thumbv7": - cmd += ["--target=thumbv7", "--target-abi=gnueabihf", "--cpu=cortex-a9", "--cpu-features=-neon"] - elif test_target == "riscv32_ilp32": - cmd += ["--target=riscv32", "--target-abi=ilp32", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c"] - elif test_target == "riscv32_ilp32d": - cmd += ["--target=riscv32", "--target-abi=ilp32d", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c"] - elif test_target == "riscv64_lp64": - cmd += ["--target=riscv64", "--target-abi=lp64", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c"] - elif test_target == "riscv64_lp64d": - cmd += ["--target=riscv64", "--target-abi=lp64d", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c"] - else: - pass + if test_target in aot_target_options_map: + cmd += aot_target_options_map[test_target] if opts.sgx: cmd.append("-sgx") @@ -1094,12 +1096,20 @@ def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r): if opts.qemu_firmware == '': raise Exception("QEMU firmware missing") - if opts.target == "thumbv7": - cmd = ["qemu-system-arm", "-semihosting", "-M", "sabrelite", "-m", "1024", "-smp", "4", "-nographic", "-kernel", opts.qemu_firmware] - elif opts.target == "riscv32_ilp32": - cmd = ["qemu-system-riscv32", "-semihosting", "-M", "virt,aclint=on", "-cpu", "rv32", "-smp", "8", "-nographic", "-bios", "none", "-kernel", opts.qemu_firmware] - elif opts.target == "riscv64_lp64": - cmd = ["qemu-system-riscv64", "-semihosting", "-M", "virt,aclint=on", "-cpu", "rv64", "-smp", "8", "-nographic", "-bios", "none", "-kernel", opts.qemu_firmware] + if opts.target.startswith("aarch64"): + cmd = "qemu-system-aarch64 -cpu cortex-a53 -nographic -machine virt,virtualization=on,gic-version=3 -net none -chardev stdio,id=con,mux=on -serial chardev:con -mon chardev=con,mode=readline -kernel".split() + cmd.append(opts.qemu_firmware) + elif opts.target.startswith("thumbv7"): + cmd = "qemu-system-arm -semihosting -M sabrelite -m 1024 -smp 4 -nographic -kernel".split() + cmd.append(opts.qemu_firmware) + elif opts.target.startswith("riscv32"): + cmd = "qemu-system-riscv32 -semihosting -M virt,aclint=on -cpu rv32 -smp 8 -nographic -bios none -kernel".split() + cmd.append(opts.qemu_firmware) + elif opts.target.startswith("riscv64"): + cmd = "qemu-system-riscv64 -semihosting -M virt,aclint=on -cpu rv64 -smp 8 -nographic -bios none -kernel".split() + cmd.append(opts.qemu_firmware) + else: + raise Exception("Unknwon target for QEMU: %s" % opts.target) else: cmd = cmd_iwasm diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index 82e7a015c..f21e52fdb 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -15,7 +15,9 @@ function help() echo "test_wamr.sh [options]" echo "-c clean previous test results, not start test" echo "-s {suite_name} test only one suite (spec|wasi_certification|wamr_compiler)" - echo "-m set compile target of iwasm(x86_64|x86_32|armv7_vfp|thumbv7_vfp|riscv64_lp64d|riscv64_lp64|aarch64)" + echo "-m set compile target of iwasm(x86_64|x86_32|armv7|armv7_vfp|thumbv7|thumbv7_vfp|" + echo " riscv32|riscv32_ilp32f|riscv32_ilp32d|riscv64|" + echo " riscv64_lp64f|riscv64_lp64d|aarch64|aarch64_vfp)" echo "-t set compile type of iwasm(classic-interp|fast-interp|jit|aot|fast-jit|multi-tier-jit)" echo "-M enable multi module feature" echo "-p enable multi thread feature" @@ -65,6 +67,8 @@ ENABLE_QEMU=0 QEMU_FIRMWARE="" # prod/testsuite-all branch WASI_TESTSUITE_COMMIT="ee807fc551978490bf1c277059aabfa1e589a6c2" +TARGET_LIST=("AARCH64" "AARCH64_VFP" "ARMV7" "ARMV7_VFP" "THUMBV7" "THUMBV7_VFP" \ + "RISCV32" "RISCV32_ILP32F" "RISCV32_ILP32D" "RISCV64" "RISCV64_LP64F" "RISCV64_LP64D") while getopts ":s:cabgvt:m:MCpSXxwPGQF:j:T:" opt do @@ -727,9 +731,7 @@ function build_iwasm_with_cfg() function build_wamrc() { - if [[ $TARGET == "ARMV7_VFP" || $TARGET == "THUMBV7_VFP" - || $TARGET == "RISCV32" || $TARGET == "RISCV32_ILP32" || $TARGET == "RISCV32_ILP32D" - || $TARGET == "RISCV64" || $TARGET == "RISCV64_LP64D" || $TARGET == "RISCV64_LP64" ]];then + if [[ "${TARGET_LIST[*]}" =~ "${TARGET}" ]]; then echo "suppose wamrc is already built" return fi From 8aa813f44a43de1e11bd64f49722ce1f4444e0b6 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Mon, 27 Nov 2023 15:42:00 +0800 Subject: [PATCH 29/67] Generate jitdump to support linux perf for LLVM JIT (#2788) --- build-scripts/build_llvm.py | 8 +-- core/iwasm/common/wasm_runtime_common.c | 7 +- core/iwasm/common/wasm_runtime_common.h | 1 + core/iwasm/compilation/aot_llvm.c | 36 +++++++++- core/iwasm/compilation/aot_llvm.h | 5 ++ core/iwasm/compilation/aot_orc_extra.cpp | 11 ++- core/iwasm/compilation/aot_orc_extra.h | 3 + core/iwasm/fast-jit/jit_compiler.h | 1 + core/iwasm/include/aot_export.h | 2 + core/iwasm/include/wasm_export.h | 14 +++- core/iwasm/interpreter/wasm_loader.c | 1 + core/iwasm/interpreter/wasm_mini_loader.c | 1 + doc/perf_tune.md | 83 ++++++++++++++++++++++- product-mini/platforms/posix/main.c | 6 ++ 14 files changed, 165 insertions(+), 14 deletions(-) diff --git a/build-scripts/build_llvm.py b/build-scripts/build_llvm.py index d70915c3b..e48f4e257 100755 --- a/build-scripts/build_llvm.py +++ b/build-scripts/build_llvm.py @@ -67,8 +67,8 @@ def build_llvm(llvm_dir, platform, backends, projects, use_clang=False, extra_fl "-DLLVM_INCLUDE_EXAMPLES:BOOL=OFF", "-DLLVM_INCLUDE_UTILS:BOOL=OFF", "-DLLVM_INCLUDE_TESTS:BOOL=OFF", - "-DLLVM_BUILD_TESTS:BOOL=OFF", "-DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON", + "-DLLVM_USE_PERF:BOOL=ON", ] # use clang/clang++/lld. but macos doesn't support lld @@ -255,7 +255,7 @@ def main(): "branch": "release/15.x", }, "xtensa": { - "repo": "https://github.com/espressif/llvm-project.git", + "repo": "https://github.com/espressif/llvm-project.git", "repo_ssh": "git@github.com:espressif/llvm-project.git", "branch": "xtensa_release_15.x", }, @@ -281,13 +281,13 @@ def main(): commit_hash = query_llvm_version(llvm_info) print(commit_hash) return commit_hash is not None - + repo_addr = llvm_info["repo"] if os.environ.get('USE_GIT_SSH') == "true": repo_addr = llvm_info["repo_ssh"] else: print("To use ssh for git clone, run: export USE_GIT_SSH=true") - + llvm_dir = clone_llvm(deps_dir, repo_addr, llvm_info["branch"]) if ( build_llvm( diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index ddd5aa8ce..5b6a8ce59 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -158,7 +158,7 @@ static JitCompOptions jit_options = { 0 }; #endif #if WASM_ENABLE_JIT != 0 -static LLVMJITOptions llvm_jit_options = { 3, 3, 0 }; +static LLVMJITOptions llvm_jit_options = { 3, 3, 0, false }; #endif static RunningMode runtime_running_mode = Mode_Default; @@ -662,9 +662,14 @@ wasm_runtime_full_init(RuntimeInitArgs *init_args) #endif #if WASM_ENABLE_JIT != 0 + LOG_DEBUG("Start LLVM_JIT, opt_sz=%u, opt_lvl=%u, segue=%s, linux_perf=%s", + init_args->llvm_jit_size_level, init_args->llvm_jit_opt_level, + init_args->segue_flags ? "Yes" : "No", + init_args->linux_perf_support ? "Yes" : "No"); llvm_jit_options.size_level = init_args->llvm_jit_size_level; llvm_jit_options.opt_level = init_args->llvm_jit_opt_level; llvm_jit_options.segue_flags = init_args->segue_flags; + llvm_jit_options.linux_perf_support = init_args->linux_perf_support; #endif if (!wasm_runtime_env_init()) { diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 74785d3f6..2908cafb5 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -430,6 +430,7 @@ typedef struct LLVMJITOptions { uint32 opt_level; uint32 size_level; uint32 segue_flags; + bool linux_perf_support; } LLVMJITOptions; #endif diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 0818893dd..f15cfd829 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -653,6 +653,19 @@ aot_add_llvm_func(AOTCompContext *comp_ctx, LLVMModuleRef module, attr_no_jump_tables); } + /* spread fp.all to every function */ + if (comp_ctx->emit_frame_pointer) { + const char *key = "frame-pointer"; + const char *val = "all"; + LLVMAttributeRef no_omit_fp = LLVMCreateStringAttribute( + comp_ctx->context, key, strlen(key), val, strlen(val)); + if (!no_omit_fp) { + aot_set_last_error("create LLVM attribute (frame-pointer) failed."); + goto fail; + } + LLVMAddAttributeAtIndex(func, LLVMAttributeFunctionIndex, no_omit_fp); + } + if (need_precheck) { if (!comp_ctx->is_jit_mode) LLVMSetLinkage(func, LLVMInternalLinkage); @@ -2160,7 +2173,7 @@ jit_stack_size_callback(void *user_data, const char *name, size_t namelen, } static bool -orc_jit_create(AOTCompContext *comp_ctx) +orc_jit_create(AOTCompContext *comp_ctx, bool linux_perf_support) { LLVMErrorRef err; LLVMOrcLLLazyJITRef orc_jit = NULL; @@ -2200,6 +2213,14 @@ orc_jit_create(AOTCompContext *comp_ctx) /* Ownership transfer: LLVMOrcLLJITBuilderRef -> LLVMOrcLLJITRef */ builder = NULL; + if (linux_perf_support) { + LOG_DEBUG("Enable linux perf support"); + LLVMOrcObjectLayerRef obj_linking_layer = + (LLVMOrcObjectLayerRef)LLVMOrcLLLazyJITGetObjLinkingLayer(orc_jit); + LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener( + obj_linking_layer, LLVMCreatePerfJITEventListener()); + } + /* Ownership transfer: local -> AOTCompContext */ comp_ctx->orc_jit = orc_jit; orc_jit = NULL; @@ -2298,6 +2319,17 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) goto fail; } + if (option->linux_perf_support) { + /* FramePointerKind.All */ + LLVMMetadataRef val = + LLVMValueAsMetadata(LLVMConstInt(LLVMInt32Type(), 2, false)); + const char *key = "frame-pointer"; + LLVMAddModuleFlag(comp_ctx->module, LLVMModuleFlagBehaviorWarning, key, + strlen(key), val); + + comp_ctx->emit_frame_pointer = true; + } + if (BH_LIST_ERROR == bh_list_init(&comp_ctx->native_symbols)) { goto fail; } @@ -2401,7 +2433,7 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) goto fail; /* Create LLJIT Instance */ - if (!orc_jit_create(comp_ctx)) + if (!orc_jit_create(comp_ctx, option->linux_perf_support)) goto fail; } else { diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index b46ac3bd0..32d7bbeba 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -12,6 +12,7 @@ #include "llvm-c/Target.h" #include "llvm-c/Core.h" #include "llvm-c/Object.h" +#include "llvm-c/OrcEE.h" #include "llvm-c/ExecutionEngine.h" #include "llvm-c/Analysis.h" #include "llvm-c/BitWriter.h" @@ -422,6 +423,8 @@ typedef struct AOTCompContext { char stack_usage_temp_file[64]; const char *llvm_passes; const char *builtin_intrinsics; + + bool emit_frame_pointer; } AOTCompContext; enum { @@ -431,6 +434,7 @@ enum { AOT_LLVMIR_OPT_FILE, }; +/* always sync it with AOTCompOption in aot_export.h */ typedef struct AOTCompOption { bool is_jit_mode; bool is_indirect_mode; @@ -457,6 +461,7 @@ typedef struct AOTCompOption { uint32 bounds_checks; uint32 stack_bounds_checks; uint32 segue_flags; + bool linux_perf_support; char **custom_sections; uint32 custom_sections_count; const char *stack_usage_file; diff --git a/core/iwasm/compilation/aot_orc_extra.cpp b/core/iwasm/compilation/aot_orc_extra.cpp index 51d61bf3c..ad8c41c3f 100644 --- a/core/iwasm/compilation/aot_orc_extra.cpp +++ b/core/iwasm/compilation/aot_orc_extra.cpp @@ -12,11 +12,13 @@ #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #endif +#include "llvm/ExecutionEngine/JITEventListener.h" +#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" -#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/Support/CBindingWrapping.h" @@ -108,6 +110,7 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ObjectTransformLayer, LLVMOrcObjectTransformLayerRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcV2CAPIHelper::PoolEntry, LLVMOrcSymbolStringPoolEntryRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ObjectLayer, LLVMOrcObjectLayerRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SymbolStringPool, LLVMOrcSymbolStringPoolRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ThreadSafeModule, LLVMOrcThreadSafeModuleRef) @@ -322,3 +325,9 @@ LLVMOrcLLLazyJITGetObjTransformLayer(LLVMOrcLLLazyJITRef J) { return wrap(&unwrap(J)->getObjTransformLayer()); } + +LLVMOrcObjectLayerRef +LLVMOrcLLLazyJITGetObjLinkingLayer(LLVMOrcLLLazyJITRef J) +{ + return wrap(&unwrap(J)->getObjLinkingLayer()); +} diff --git a/core/iwasm/compilation/aot_orc_extra.h b/core/iwasm/compilation/aot_orc_extra.h index 44c2cd7a7..32ece4de4 100644 --- a/core/iwasm/compilation/aot_orc_extra.h +++ b/core/iwasm/compilation/aot_orc_extra.h @@ -76,5 +76,8 @@ LLVMOrcLLJITBuilderSetCompileFuncitonCreatorWithStackSizesCallback( LLVMOrcLLLazyJITBuilderRef Builder, void (*cb)(void *, const char *, size_t, size_t), void *cb_data); +LLVMOrcObjectLayerRef +LLVMOrcLLLazyJITGetObjLinkingLayer(LLVMOrcLLLazyJITRef J); + LLVM_C_EXTERN_C_END #endif diff --git a/core/iwasm/fast-jit/jit_compiler.h b/core/iwasm/fast-jit/jit_compiler.h index 9a49cffdd..dee2631d1 100644 --- a/core/iwasm/fast-jit/jit_compiler.h +++ b/core/iwasm/fast-jit/jit_compiler.h @@ -70,6 +70,7 @@ typedef struct JitInterpSwitchInfo { typedef struct JitCompOptions { uint32 code_cache_size; uint32 opt_level; + bool linux_perf_support; } JitCompOptions; bool diff --git a/core/iwasm/include/aot_export.h b/core/iwasm/include/aot_export.h index ce8ae81fe..f2184033a 100644 --- a/core/iwasm/include/aot_export.h +++ b/core/iwasm/include/aot_export.h @@ -38,6 +38,7 @@ enum { AOT_LLVMIR_OPT_FILE, }; +/* always sync it with AOTCompOption in compilation/aot_llvm.h */ typedef struct AOTCompOption { bool is_jit_mode; bool is_indirect_mode; @@ -64,6 +65,7 @@ typedef struct AOTCompOption { uint32_t bounds_checks; uint32_t stack_bounds_checks; uint32_t segue_flags; + bool linux_perf_support; char **custom_sections; uint32_t custom_sections_count; const char *stack_usage_file; diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 86f4573c7..cb6de7631 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -169,6 +169,15 @@ typedef struct RuntimeInitArgs { uint32_t llvm_jit_size_level; /* Segue optimization flags for LLVM JIT */ uint32_t segue_flags; + /** + * If enabled + * - llvm-jit will output a jitdump file for `perf inject` + * - aot. TBD + * - fast-jit. TBD + * - multi-tier-jit. TBD + * - interpreter. TBD + */ + bool linux_perf_support; } RuntimeInitArgs; #ifndef WASM_VALKIND_T_DEFINED @@ -945,7 +954,7 @@ wasm_runtime_get_custom_data(wasm_module_inst_t module_inst); /** * Set the memory bounds checks flag of a WASM module instance. - * + * * @param module_inst the WASM module instance * @param enable the flag to enable/disable the memory bounds checks */ @@ -954,9 +963,8 @@ wasm_runtime_set_bounds_checks(wasm_module_inst_t module_inst, bool enable); /** * Check if the memory bounds checks flag is enabled for a WASM module instance. - * - * @param module_inst the WASM module instance * + * @param module_inst the WASM module instance * @return true if the memory bounds checks flag is enabled, false otherwise */ WASM_RUNTIME_API_EXTERN bool diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index f831073cd..09e3f47ed 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -2878,6 +2878,7 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, option.opt_level = llvm_jit_options.opt_level; option.size_level = llvm_jit_options.size_level; option.segue_flags = llvm_jit_options.segue_flags; + option.linux_perf_support = llvm_jit_options.linux_perf_support; #if WASM_ENABLE_BULK_MEMORY != 0 option.enable_bulk_memory = true; diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 44aab2654..2d5bf57a9 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -1877,6 +1877,7 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf, option.opt_level = llvm_jit_options.opt_level; option.size_level = llvm_jit_options.size_level; option.segue_flags = llvm_jit_options.segue_flags; + option.linux_perf_support = llvm_jit_options.linux_perf_support; #if WASM_ENABLE_BULK_MEMORY != 0 option.enable_bulk_memory = true; diff --git a/doc/perf_tune.md b/doc/perf_tune.md index 2cc428984..ca175d476 100644 --- a/doc/perf_tune.md +++ b/doc/perf_tune.md @@ -2,7 +2,7 @@ Normally there are some methods to tune the performance: -## 1. Use `wasm-opt` tool +## 1. Use `wasm-opt` tool Download the [binaryen release](https://github.com/WebAssembly/binaryen/releases), and use the `wasm-opt` tool in it to optimize the wasm file, for example: @@ -23,16 +23,19 @@ emcc -msimd128 -O3 -o ## 3. Enable segue optimization for wamrc when generating the aot file [Segue](https://plas2022.github.io/files/pdf/SegueColorGuard.pdf) is an optimization technology which uses x86 segment register to store the WebAssembly linear memory base address, so as to remove most of the cost of SFI (Software-based Fault Isolation) base addition and free up a general purpose register, by this way it may: + - Improve the performance of JIT/AOT - Reduce the footprint of JIT/AOT, the JIT/AOT code generated is smaller - Reduce the compilation time of JIT/AOT Currently it is supported on linux x86-64, developer can use `--enable-segue=[]` for wamrc: + ```bash wamrc --enable-segue -o aot_file wasm_file # or wamrc --enable-segue=[] -o aot_file wasm_file ``` + `flags` can be: i32.load, i64.load, f32.load, f64.load, v128.load, i32.store, i64.store, f32.store, f64.store and v128.store, use comma to separate them, e.g. `--enable-segue=i32.load,i64.store`, and `--enable-segue` means all flags are added. > Note: Normally for most cases, using `--enable-segue` is enough, but for some cases, using `--enable-segue=` may be better, for example for CoreMark benchmark, `--enable-segue=i32.store` may lead to better performance than `--enable-segue`. @@ -40,7 +43,8 @@ wamrc --enable-segue=[] -o aot_file wasm_file ## 4. Enable segue optimization for iwasm when running wasm file Similar to segue optimization for wamrc, run: -``` bash + +```bash iwasm --enable-segue wasm_file (iwasm is built with llvm-jit enabled) # or iwasm --enable-segue=[] wasm_file @@ -55,6 +59,7 @@ LLVM PGO (Profile-Guided Optimization) allows the compiler to better optimize co 2. Compile iwasm with `cmake -DWAMR_BUILD_STATIC_PGO=1` and run `iwasm --gen-prof-file= ` to generate the raw profile file. > Note: Directly dumping raw profile data to file system may be unsupported in some environments, developer can dump the profile data into memory buffer instead and try outputting it through network (e.g. uart or socket): + ```C uint32_t wasm_runtime_get_pgo_prof_data_size(wasm_module_inst_t module_inst); @@ -84,6 +89,78 @@ Please notice that this method is not a general solution since it may lead to se 3. Run the AOT module by iwasm with `--disable-bounds-checks` option. > Note: The size of AOT file will be much smaller than the default, and some tricks are possible such as let the wasm application access the memory of host os directly. -Please notice that if this option is enabled, the wasm spec test will fail since it requires the memory boundary check. For example, the runtime will crash when accessing the memory out of the boundary in some cases instead of throwing an exception as the spec requires. +> Please notice that if this option is enabled, the wasm spec test will fail since it requires the memory boundary check. For example, the runtime will crash when accessing the memory out of the boundary in some cases instead of throwing an exception as the spec requires. You should only use this method for well tested wasm applications and make sure the memory access is safe. + +## 7. Use linux-perf + +Linux perf is a powerful tool to analyze the performance of a program, developer can use it to find the hot functions and optimize them. It is one profiler supported by WAMR. In order to use it, you need to add `--perf-profile` while running _iwasm_. By default, it is disabled. + +> [!CAUTION] +> For now, only llvm-jit mode supports linux-perf. + +Here is a basic example, if there is a Wasm application _foo.wasm_, you'll execute. + +``` +$ perf record --output=perf.data.raw -- iwasm --perf-profile foo.wasm +``` + +This will create a _perf.data_ and a _jit-xxx.dump_ under _~/.debug.jit/_ folder. This extra file is WAMR generated at runtime, and it contains the mapping between the JIT code and the original Wasm function names. + +The next thing need to do is to merge _jit-xxx.dump_ file into the _perf.data_. + +``` +$ perf inject --jit --input=perf.data.raw --output=perf.data +``` + +This step will create a lot of _jitted-xxxx-N.so_ which are ELF images for all JIT functions created at runtime. + +> [!TIP] +> add `-v` and check if there is output likes _write ELF image ..._. If yes, it means above merge is successful. + +Finally, you can use _perf report_ to analyze the performance. + +``` +$ perf report --input=perf.data +``` + +> [!CAUTION] +> Using release build of llvm and iwasm will produce "[unknown]" functions in the call graph. It is not only because +> of the missing debug information, but also because of removing frame pointers. To get the complete result, please +> use debug build of llvm and iwasm. +> +> Wasm functions will not be affected. + +### 7.1 Flamegraph + +[Flamegraph](https://www.brendangregg.com/flamegraphs.html) is a powerful tool to visualize stack traces of profiled software so that the most frequent code-paths can be identified quickly and accurately. In order to use it, you need to record call graphs when running `perf record` + +``` +$ perf record -k mono --call-graph=fp --output=perf.data.raw -- iwasm --perf-profile foo.wasm +``` + + +merge the _jit-xxx.dump_ file into the _perf.data.raw_. + +``` +$ perf inject --jit --input=perf.data.raw --output=perf.data +``` + +generate the stack trace file. + +``` +$ perf script > out.perf +``` + +[fold stacks](https://github.com/brendangregg/FlameGraph#2-fold-stacks). + +``` +$ ./FlameGraph/stackcollapse-perf.pl out.perf > out.folded +``` + +[render a flamegraph](https://github.com/brendangregg/FlameGraph#3-flamegraphpl) + +``` +$ ./FlameGraph/flamegraph.pl out.folded > perf.foo.wasm.svg +``` diff --git a/product-mini/platforms/posix/main.c b/product-mini/platforms/posix/main.c index 930d9db5e..851855497 100644 --- a/product-mini/platforms/posix/main.c +++ b/product-mini/platforms/posix/main.c @@ -58,6 +58,7 @@ print_help() #if WASM_ENABLE_JIT != 0 printf(" --llvm-jit-size-level=n Set LLVM JIT size level, default is 3\n"); printf(" --llvm-jit-opt-level=n Set LLVM JIT optimization level, default is 3\n"); + printf(" --perf-profile Enable linux perf support. For now, it only works in llvm-jit.\n"); #if defined(os_writegsbase) printf(" --enable-segue[=] Enable using segment register GS as the base address of\n"); printf(" linear memory, which may improve performance, flags can be:\n"); @@ -560,6 +561,7 @@ main(int argc, char *argv[]) uint32 llvm_jit_size_level = 3; uint32 llvm_jit_opt_level = 3; uint32 segue_flags = 0; + bool enable_linux_perf_support = false; #endif wasm_module_t wasm_module = NULL; wasm_module_inst_t wasm_module_inst = NULL; @@ -700,6 +702,9 @@ main(int argc, char *argv[]) if (segue_flags == (uint32)-1) return print_help(); } + else if (!strncmp(argv[0], "--perf-profile", 14)) { + enable_linux_perf_support = true; + } #endif /* end of WASM_ENABLE_JIT != 0 */ #if BH_HAS_DLFCN else if (!strncmp(argv[0], "--native-lib=", 13)) { @@ -814,6 +819,7 @@ main(int argc, char *argv[]) init_args.llvm_jit_size_level = llvm_jit_size_level; init_args.llvm_jit_opt_level = llvm_jit_opt_level; init_args.segue_flags = segue_flags; + init_args.linux_perf_support = enable_linux_perf_support; #endif #if WASM_ENABLE_DEBUG_INTERP != 0 From 1d0f7897541e91e40f325637f97584e4120b919b Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Mon, 27 Nov 2023 15:53:37 +0800 Subject: [PATCH 30/67] Fix typos in libsodium workload (#2825) --- tests/benchmarks/libsodium/run_aot.sh | 4 ++-- tests/benchmarks/libsodium/test_pgo.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/benchmarks/libsodium/run_aot.sh b/tests/benchmarks/libsodium/run_aot.sh index 8859d0634..3623deb2c 100755 --- a/tests/benchmarks/libsodium/run_aot.sh +++ b/tests/benchmarks/libsodium/run_aot.sh @@ -12,8 +12,8 @@ libsodium_CASES="aead_aes256gcm2 aead_aes256gcm aead_chacha20poly13052 aead_chac pwhash_scrypt_ll pwhash_scrypt randombytes scalarmult2 scalarmult5 \ scalarmult6 scalarmult7 scalarmult8 scalarmult_ed25519 scalarmult_ristretto255 \ scalarmult secretbox2 secretbox7 secretbox8 secretbox_easy2 secretbox_easy \ - secretbox secretstream shorthash sign siphashx24 sodium_core sodium_utils2 \ - sodium_utils stream2 stream3 stream4 stream verify1 xchacha20" + secretbox secretstream_xchacha20poly1305 shorthash sign siphashx24 sodium_core \ + sodium_utils2 sodium_utils stream2 stream3 stream4 stream verify1 xchacha20" PLATFORM=$(uname -s | tr A-Z a-z) diff --git a/tests/benchmarks/libsodium/test_pgo.sh b/tests/benchmarks/libsodium/test_pgo.sh index 2dda6a5f9..3f9ee2eae 100755 --- a/tests/benchmarks/libsodium/test_pgo.sh +++ b/tests/benchmarks/libsodium/test_pgo.sh @@ -12,8 +12,8 @@ libsodium_CASES="aead_aes256gcm2 aead_aes256gcm aead_chacha20poly13052 aead_chac pwhash_scrypt_ll pwhash_scrypt randombytes scalarmult2 scalarmult5 \ scalarmult6 scalarmult7 scalarmult8 scalarmult_ed25519 scalarmult_ristretto255 \ scalarmult secretbox2 secretbox7 secretbox8 secretbox_easy2 secretbox_easy \ - secretbox secretstream shorthash sign siphashx24 sodium_core sodium_utils2 \ - sodium_utils stream2 stream3 stream4 stream verify1 xchacha20" + secretbox secretstream_xchacha20poly1305 shorthash sign siphashx24 sodium_core \ + sodium_utils2 sodium_utils stream2 stream3 stream4 stream verify1 xchacha20" PLATFORM=$(uname -s | tr A-Z a-z) From b81abd01ef7200aa38c127925da3fca17cc0fad7 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Tue, 28 Nov 2023 13:46:14 +0800 Subject: [PATCH 31/67] Add a user tip about flamegraph (#2827) --- doc/perf_tune.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/doc/perf_tune.md b/doc/perf_tune.md index ca175d476..f57abe068 100644 --- a/doc/perf_tune.md +++ b/doc/perf_tune.md @@ -126,21 +126,24 @@ $ perf report --input=perf.data ``` > [!CAUTION] -> Using release build of llvm and iwasm will produce "[unknown]" functions in the call graph. It is not only because +> Using release builds of llvm and iwasm will produce "[unknown]" functions in the call graph. It is not only because > of the missing debug information, but also because of removing frame pointers. To get the complete result, please -> use debug build of llvm and iwasm. +> use debug builds of both llvm and iwasm. > -> Wasm functions will not be affected. +> Wasm functions names are stored in _the custom name section_. Toolchains always generate the custom name section in both debug and release builds. However, the custom name section is stripped to pursue smallest size in release build. So, if you want to get a understandable result, please search the manual of toolchain to look for a way to keep the custom name section. +> +> For example, with EMCC, you can add `-g2`. +> +> If not able to get the context of the custom name section, WAMR will use `aot_func#N` to represent the function name. `N` is from 0. `aot_func#0` represents the first *not imported wasm function*. ### 7.1 Flamegraph -[Flamegraph](https://www.brendangregg.com/flamegraphs.html) is a powerful tool to visualize stack traces of profiled software so that the most frequent code-paths can be identified quickly and accurately. In order to use it, you need to record call graphs when running `perf record` +[Flamegraph](https://www.brendangregg.com/flamegraphs.html) is a powerful tool to visualize stack traces of profiled software so that the most frequent code-paths can be identified quickly and accurately. In order to use it, you need to [capture graphs](https://github.com/brendangregg/FlameGraph#1-capture-stacks) when running `perf record` ``` $ perf record -k mono --call-graph=fp --output=perf.data.raw -- iwasm --perf-profile foo.wasm ``` - merge the _jit-xxx.dump_ file into the _perf.data.raw_. ``` @@ -164,3 +167,13 @@ $ ./FlameGraph/stackcollapse-perf.pl out.perf > out.folded ``` $ ./FlameGraph/flamegraph.pl out.folded > perf.foo.wasm.svg ``` + +> [!TIP] +> use `grep` to pick up folded stacks you are interested in or filter out something. +> +> For example, if just want to see stacks of wasm functions, you can use +> +> ```bash +> # only jitted functions +> $ grep "wasm_runtime_invoke_native" out.folded | ./FlameGraph/flamegraph.pl > perf.foo.wasm.only.svg +> ``` From 0455071fc1cf32e3f70c61883d61ce03d008fcda Mon Sep 17 00:00:00 2001 From: Enrico Loparco Date: Wed, 29 Nov 2023 12:27:17 +0000 Subject: [PATCH 32/67] Access linear memory size atomically (#2834) Fixes: https://github.com/bytecodealliance/wasm-micro-runtime/issues/2804 --- core/iwasm/common/wasm_memory.c | 4 ++-- core/iwasm/common/wasm_memory.h | 10 ++++++++++ core/iwasm/interpreter/wasm_interp_classic.c | 20 +++++++++++++------- core/iwasm/interpreter/wasm_interp_fast.c | 20 +++++++++++++------- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index c01f9d284..3941c224a 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -686,7 +686,7 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count) memory->num_bytes_per_page = num_bytes_per_page; memory->cur_page_count = total_page_count; memory->max_page_count = max_page_count; - memory->memory_data_size = (uint32)total_size_new; + SET_LINEAR_MEMORY_SIZE(memory, (uint32)total_size_new); memory->memory_data_end = memory->memory_data + (uint32)total_size_new; wasm_runtime_set_mem_bound_check_bytes(memory, total_size_new); @@ -844,7 +844,7 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count) memory->num_bytes_per_page = num_bytes_per_page; memory->cur_page_count = total_page_count; memory->max_page_count = max_page_count; - memory->memory_data_size = (uint32)total_size_new; + SET_LINEAR_MEMORY_SIZE(memory, (uint32)total_size_new); memory->memory_data_end = memory->memory_data + (uint32)total_size_new; wasm_runtime_set_mem_bound_check_bytes(memory, total_size_new); diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index c46b32ea7..9b74db526 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -14,6 +14,16 @@ extern "C" { #endif +#if WASM_ENABLE_SHARED_MEMORY != 0 +#define GET_LINEAR_MEMORY_SIZE(memory) \ + BH_ATOMIC_32_LOAD(memory->memory_data_size) +#define SET_LINEAR_MEMORY_SIZE(memory, size) \ + BH_ATOMIC_32_STORE(memory->memory_data_size, size) +#else +#define GET_LINEAR_MEMORY_SIZE(memory) memory->memory_data_size +#define SET_LINEAR_MEMORY_SIZE(memory, size) memory->memory_data_size = size +#endif + bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, const MemAllocOption *alloc_option); diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index e129e771e..e27b16544 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -36,7 +36,7 @@ typedef float64 CellType_F64; * multi-threading mode since it may be changed by other * threads in memory.grow */ -#define get_linear_mem_size() memory->memory_data_size +#define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ @@ -1155,7 +1155,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 - uint32 linear_mem_size = memory ? memory->memory_data_size : 0; + uint32 linear_mem_size = 0; + if (memory) +#if WASM_ENABLE_THREAD_MGR == 0 + linear_mem_size = memory->memory_data_size; +#else + linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); +#endif #endif WASMType **wasm_types = module->module->types; WASMGlobalInstance *globals = module->e->globals, *global; @@ -2143,7 +2149,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 - linear_mem_size = memory->memory_data_size; + linear_mem_size = get_linear_mem_size(); #endif } @@ -3148,7 +3154,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, addr = (uint32)POP_I32(); #if WASM_ENABLE_THREAD_MGR != 0 - linear_mem_size = memory->memory_data_size; + linear_mem_size = get_linear_mem_size(); #endif #ifndef OS_ENABLE_HW_BOUND_CHECK @@ -3199,7 +3205,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, dst = POP_I32(); #if WASM_ENABLE_THREAD_MGR != 0 - linear_mem_size = memory->memory_data_size; + linear_mem_size = get_linear_mem_size(); #endif #ifndef OS_ENABLE_HW_BOUND_CHECK @@ -3230,7 +3236,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, dst = POP_I32(); #if WASM_ENABLE_THREAD_MGR != 0 - linear_mem_size = memory->memory_data_size; + linear_mem_size = get_linear_mem_size(); #endif #ifndef OS_ENABLE_HW_BOUND_CHECK @@ -3907,7 +3913,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 if (memory) - linear_mem_size = memory->memory_data_size; + linear_mem_size = get_linear_mem_size(); #endif if (wasm_copy_exception(module, NULL)) goto got_exception; diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index fe5ffd1c1..d3cebc334 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -27,7 +27,7 @@ typedef float64 CellType_F64; * multi-threading mode since it may be changed by other * threads in memory.grow */ -#define get_linear_mem_size() memory->memory_data_size +#define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ @@ -1180,7 +1180,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 - uint32 linear_mem_size = memory ? memory->memory_data_size : 0; + uint32 linear_mem_size = 0; + if (memory) +#if WASM_ENABLE_THREAD_MGR == 0 + linear_mem_size = memory->memory_data_size; +#else + linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); +#endif #endif WASMGlobalInstance *globals = module->e ? module->e->globals : NULL; WASMGlobalInstance *global; @@ -1911,7 +1917,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 - linear_mem_size = memory->memory_data_size; + linear_mem_size = get_linear_mem_size(); #endif } @@ -2994,7 +3000,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, addr = POP_I32(); #if WASM_ENABLE_THREAD_MGR - linear_mem_size = memory->memory_data_size; + linear_mem_size = get_linear_mem_size(); #endif #ifndef OS_ENABLE_HW_BOUND_CHECK @@ -3043,7 +3049,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, dst = POP_I32(); #if WASM_ENABLE_THREAD_MGR - linear_mem_size = memory->memory_data_size; + linear_mem_size = get_linear_mem_size(); #endif #ifndef OS_ENABLE_HW_BOUND_CHECK @@ -3073,7 +3079,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, dst = POP_I32(); #if WASM_ENABLE_THREAD_MGR - linear_mem_size = memory->memory_data_size; + linear_mem_size = get_linear_mem_size(); #endif #ifndef OS_ENABLE_HW_BOUND_CHECK @@ -3848,7 +3854,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 if (memory) - linear_mem_size = memory->memory_data_size; + linear_mem_size = get_linear_mem_size(); #endif if (wasm_copy_exception(module, NULL)) goto got_exception; From 873558c40edc60731fd9091722681e5e54bbe95c Mon Sep 17 00:00:00 2001 From: Enrico Loparco Date: Thu, 30 Nov 2023 00:49:58 +0000 Subject: [PATCH 33/67] Get rid of compilation warnings and minor doc fix (#2839) --- core/iwasm/common/wasm_runtime_common.c | 2 +- core/shared/platform/common/posix/posix_file.c | 8 ++++---- samples/multi-module/README.md | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 5b6a8ce59..4a632543b 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -5873,7 +5873,7 @@ wasm_runtime_register_sub_module(const WASMModuleCommon *parent_module, { /* register sub_module into its parent sub module list */ WASMRegisteredModule *node = NULL; - bh_list_status ret; + bh_list_status ret = BH_LIST_ERROR; if (wasm_runtime_search_sub_module(parent_module, sub_module_name)) { LOG_DEBUG("%s has been registered in its parent", sub_module_name); diff --git a/core/shared/platform/common/posix/posix_file.c b/core/shared/platform/common/posix/posix_file.c index a16509760..8c4f7aa9d 100644 --- a/core/shared/platform/common/posix/posix_file.c +++ b/core/shared/platform/common/posix/posix_file.c @@ -479,7 +479,7 @@ os_preadv(os_file_handle handle, const struct __wasi_iovec_t *iov, int iovcnt, // Allocate a single buffer to fit all data. size_t totalsize = 0; - for (size_t i = 0; i < iovcnt; ++i) + for (int i = 0; i < iovcnt; ++i) totalsize += iov[i].buf_len; char *buf = BH_MALLOC(totalsize); @@ -498,7 +498,7 @@ os_preadv(os_file_handle handle, const struct __wasi_iovec_t *iov, int iovcnt, // Copy data back to vectors. size_t bufoff = 0; - for (size_t i = 0; i < iovcnt; ++i) { + for (int i = 0; i < iovcnt; ++i) { if (bufoff + iov[i].buf_len < (size_t)len) { memcpy(iov[i].buf, buf + bufoff, iov[i].buf_len); bufoff += iov[i].buf_len; @@ -533,14 +533,14 @@ os_pwritev(os_file_handle handle, const struct __wasi_ciovec_t *iov, int iovcnt, else { // Allocate a single buffer to fit all data. size_t totalsize = 0; - for (size_t i = 0; i < iovcnt; ++i) + for (int i = 0; i < iovcnt; ++i) totalsize += iov[i].buf_len; char *buf = BH_MALLOC(totalsize); if (buf == NULL) { return __WASI_ENOMEM; } size_t bufoff = 0; - for (size_t i = 0; i < iovcnt; ++i) { + for (int i = 0; i < iovcnt; ++i) { memcpy(buf + bufoff, iov[i].buf, iov[i].buf_len); bufoff += iov[i].buf_len; } diff --git a/samples/multi-module/README.md b/samples/multi-module/README.md index 1ec92620c..731ba62c2 100644 --- a/samples/multi-module/README.md +++ b/samples/multi-module/README.md @@ -9,12 +9,12 @@ $ mkdir build $ cd build $ cmake .. $ make -$ # It will build multi-module runtime and +$ # It will build multi_module runtime and $ # wasm file under the ./build . $ # If you have built wamrc, $ # aot file will also genrate. -$ ./multi-module mC.wasm +$ ./multi_module mC.wasm $ ... -$ ./multi-module mC.aot +$ ./multi_module mC.aot $ ... From 718f0671e7e62eeab3b57899c43a9530e89aff63 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Fri, 1 Dec 2023 11:14:13 +0800 Subject: [PATCH 34/67] Output warning and quit if import/export name contains '\00' (#2806) Leave it as a limitation when import/export name contains '\00' in wasm file. p.s. https://github.com/bytecodealliance/wasm-micro-runtime/issues/2789 --- core/iwasm/interpreter/wasm_loader.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 09e3f47ed..61e30be78 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -301,7 +301,13 @@ check_utf8_str(const uint8 *str, uint32 len) while (p < p_end) { chr = *p; - if (chr < 0x80) { + + if (chr == 0) { + LOG_WARNING( + "LIMITATION: a string which contains '\\00' is unsupported"); + return false; + } + else if (chr < 0x80) { p++; } else if (chr >= 0xC2 && chr <= 0xDF && p + 1 < p_end) { From 162a97700615e5df1d7ec9e9c8bf90f8c78f9803 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Fri, 1 Dec 2023 12:34:11 +0800 Subject: [PATCH 35/67] Use wasm_config_t to pass private configuration to wasm_engine_new (#2837) Support new a wasm_config_t, set allocation and linux_perf_support options to it, and then pass it to wasm_engine_new_with_config to new an engine with private configuration. --- core/iwasm/common/wasm_c_api.c | 100 ++++++++++++++++++-------------- core/iwasm/include/wasm_c_api.h | 80 ++++++++++++++++--------- 2 files changed, 110 insertions(+), 70 deletions(-) diff --git a/core/iwasm/common/wasm_c_api.c b/core/iwasm/common/wasm_c_api.c index 2da733874..57c3ebd0d 100644 --- a/core/iwasm/common/wasm_c_api.c +++ b/core/iwasm/common/wasm_c_api.c @@ -292,13 +292,46 @@ WASM_DEFINE_VEC_OWN(valtype, wasm_valtype_delete) own wasm_config_t * wasm_config_new(void) { - return NULL; + /* since wasm_runtime_malloc is not ready */ + wasm_config_t *config = os_malloc(sizeof(wasm_config_t)); + if (!config) + return NULL; + + memset(config, 0, sizeof(wasm_config_t)); + config->mem_alloc_type = Alloc_With_System_Allocator; + return config; } void wasm_config_delete(own wasm_config_t *config) { - (void)config; + if (config) + os_free(config); +} + +wasm_config_t * +wasm_config_set_mem_alloc_opt(wasm_config_t *config, + mem_alloc_type_t mem_alloc_type, + MemAllocOption *mem_alloc_option) +{ + if (!config) + return NULL; + + config->mem_alloc_type = mem_alloc_type; + if (mem_alloc_option) + memcpy(&config->mem_alloc_option, mem_alloc_option, + sizeof(MemAllocOption)); + return config; +} + +wasm_config_t * +wasm_config_set_linux_perf_opt(wasm_config_t *config, bool enable) +{ + if (!config) + return NULL; + + config->linux_perf_support = enable; + return config; } static void @@ -329,12 +362,11 @@ wasm_engine_delete_internal(wasm_engine_t *engine) } static wasm_engine_t * -wasm_engine_new_internal(mem_alloc_type_t type, const MemAllocOption *opts) +wasm_engine_new_internal(wasm_config_t *config) { wasm_engine_t *engine = NULL; /* init runtime */ RuntimeInitArgs init_args = { 0 }; - init_args.mem_alloc_type = type; #ifndef NDEBUG bh_log_set_verbose_level(BH_LOG_LEVEL_VERBOSE); @@ -344,34 +376,11 @@ wasm_engine_new_internal(mem_alloc_type_t type, const MemAllocOption *opts) WASM_C_DUMP_PROC_MEM(); - if (type == Alloc_With_Pool) { - if (!opts) { - return NULL; - } - - init_args.mem_alloc_option.pool.heap_buf = opts->pool.heap_buf; - init_args.mem_alloc_option.pool.heap_size = opts->pool.heap_size; - } - else if (type == Alloc_With_Allocator) { - if (!opts) { - return NULL; - } - - init_args.mem_alloc_option.allocator.malloc_func = - opts->allocator.malloc_func; - init_args.mem_alloc_option.allocator.free_func = - opts->allocator.free_func; - init_args.mem_alloc_option.allocator.realloc_func = - opts->allocator.realloc_func; -#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 - init_args.mem_alloc_option.allocator.user_data = - opts->allocator.user_data; -#endif - } - else { - init_args.mem_alloc_option.pool.heap_buf = NULL; - init_args.mem_alloc_option.pool.heap_size = 0; - } + /* wasm_config_t->MemAllocOption -> RuntimeInitArgs->MemAllocOption */ + init_args.mem_alloc_type = config->mem_alloc_type; + memcpy(&init_args.mem_alloc_option, &config->mem_alloc_option, + sizeof(MemAllocOption)); + init_args.linux_perf_support = config->linux_perf_support; if (!wasm_runtime_full_init(&init_args)) { LOG_DEBUG("wasm_runtime_full_init failed"); @@ -418,14 +427,23 @@ static korp_mutex engine_lock = OS_THREAD_MUTEX_INITIALIZER; #endif own wasm_engine_t * -wasm_engine_new_with_args(mem_alloc_type_t type, const MemAllocOption *opts) +wasm_engine_new() +{ + wasm_config_t config = { 0 }; + wasm_config_set_mem_alloc_opt(&config, Alloc_With_System_Allocator, NULL); + wasm_engine_t *engine = wasm_engine_new_with_config(&config); + return engine; +} + +own wasm_engine_t * +wasm_engine_new_with_config(wasm_config_t *config) { #if defined(OS_THREAD_MUTEX_INITIALIZER) os_mutex_lock(&engine_lock); #endif if (!singleton_engine) - singleton_engine = wasm_engine_new_internal(type, opts); + singleton_engine = wasm_engine_new_internal(config); else singleton_engine->ref_count++; @@ -437,16 +455,12 @@ wasm_engine_new_with_args(mem_alloc_type_t type, const MemAllocOption *opts) } own wasm_engine_t * -wasm_engine_new() +wasm_engine_new_with_args(mem_alloc_type_t type, const MemAllocOption *opts) { - return wasm_engine_new_with_args(Alloc_With_System_Allocator, NULL); -} - -own wasm_engine_t * -wasm_engine_new_with_config(own wasm_config_t *config) -{ - (void)config; - return wasm_engine_new_with_args(Alloc_With_System_Allocator, NULL); + wasm_config_t config = { 0 }; + config.mem_alloc_type = type; + memcpy(&config.mem_alloc_option, opts, sizeof(MemAllocOption)); + return wasm_engine_new_with_config(&config); } void diff --git a/core/iwasm/include/wasm_c_api.h b/core/iwasm/include/wasm_c_api.h index 324a43bd5..eaad941e4 100644 --- a/core/iwasm/include/wasm_c_api.h +++ b/core/iwasm/include/wasm_c_api.h @@ -21,6 +21,15 @@ #endif #endif +#if defined(__GNUC__) || defined(__clang__) +#define DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) +#define DEPRECATED __declspec(deprecated) +#else +#pragma message("WARNING: You need to implement DEPRECATED for this compiler") +#define DEPRECATED +#endif + #ifdef __cplusplus extern "C" { #endif @@ -136,31 +145,6 @@ static inline void wasm_name_new_from_string_nt( WASM_DECLARE_OWN(config) -WASM_API_EXTERN own wasm_config_t* wasm_config_new(void); - -// Embedders may provide custom functions for manipulating configs. - - -// Engine - -WASM_DECLARE_OWN(engine) - -/** - * Create a new engine - * - * Note: for the engine new/delete operations, including this, - * wasm_engine_new_with_config, wasm_engine_new_with_args, and - * wasm_engine_delete, if the platform has mutex initializer, - * then they are thread-safe: we use a global lock to lock the - * operations of the engine. Otherwise they are not thread-safe: - * when there are engine new/delete operations happening - * simultaneously in multiple threads, developer must create - * the lock by himself, and add the lock when calling these - * functions. - */ -WASM_API_EXTERN own wasm_engine_t* wasm_engine_new(void); -WASM_API_EXTERN own wasm_engine_t* wasm_engine_new_with_config(own wasm_config_t*); - #ifndef MEM_ALLOC_OPTION_DEFINED #define MEM_ALLOC_OPTION_DEFINED /* same definition from wasm_export.h */ @@ -191,9 +175,51 @@ typedef union MemAllocOption { void *user_data; } allocator; } MemAllocOption; -#endif +#endif /* MEM_ALLOC_OPTION_DEFINED */ -WASM_API_EXTERN own wasm_engine_t * +/* Runtime configration */ +struct wasm_config_t { + mem_alloc_type_t mem_alloc_type; + MemAllocOption mem_alloc_option; + bool linux_perf_support; + /*TODO: wasi args*/ +}; + +/* + * by default: + * - mem_alloc_type is Alloc_With_System_Allocator + * - mem_alloc_option is all 0 + * - linux_perf_support is false + */ +WASM_API_EXTERN own wasm_config_t* wasm_config_new(void); + +// Embedders may provide custom functions for manipulating configs. +WASM_API_EXTERN own wasm_config_t* +wasm_config_set_mem_alloc_opt(wasm_config_t *, mem_alloc_type_t, MemAllocOption *); + +WASM_API_EXTERN own wasm_config_t* +wasm_config_set_linux_perf_opt(wasm_config_t *, bool); + +// Engine + +WASM_DECLARE_OWN(engine) + +/** + * Create a new engine + * + * Note: for the engine new/delete operations, including this, + * wasm_engine_new_with_config, wasm_engine_new_with_args, and + * wasm_engine_delete, if the platform has mutex initializer, + * then they are thread-safe: we use a global lock to lock the + * operations of the engine. Otherwise they are not thread-safe: + * when there are engine new/delete operations happening + * simultaneously in multiple threads, developer must create + * the lock by himself, and add the lock when calling these + * functions. + */ +WASM_API_EXTERN own wasm_engine_t* wasm_engine_new(void); +WASM_API_EXTERN own wasm_engine_t* wasm_engine_new_with_config(wasm_config_t*); +DEPRECATED WASM_API_EXTERN own wasm_engine_t * wasm_engine_new_with_args(mem_alloc_type_t type, const MemAllocOption *opts); // Store From 3d0342fbc87d963ebd638166207a6821b96376b6 Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Fri, 1 Dec 2023 17:41:24 +0800 Subject: [PATCH 36/67] Fix fast-jit accessing shared memory's fields issue (#2841) For shared memory, runtime should get the memories pointer from module_inst first, then get memory instance from memories array, and then get the fields of the memory instance. --- core/iwasm/fast-jit/fe/jit_emit_memory.c | 34 +- core/iwasm/fast-jit/jit_frontend.c | 408 +++++++++++++++++++---- core/iwasm/fast-jit/jit_frontend.h | 6 + core/iwasm/fast-jit/jit_ir.h | 2 + 4 files changed, 358 insertions(+), 92 deletions(-) diff --git a/core/iwasm/fast-jit/fe/jit_emit_memory.c b/core/iwasm/fast-jit/fe/jit_emit_memory.c index ce3f77fae..420b4dd8e 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_memory.c +++ b/core/iwasm/fast-jit/fe/jit_emit_memory.c @@ -137,6 +137,7 @@ check_and_seek(JitCompContext *cc, JitReg addr, uint32 offset, uint32 bytes) { JitReg memory_boundary = 0, offset1; #ifndef OS_ENABLE_HW_BOUND_CHECK + JitReg cur_page_count; /* the default memory */ uint32 mem_idx = 0; #endif @@ -146,16 +147,10 @@ check_and_seek(JitCompContext *cc, JitReg addr, uint32 offset, uint32 bytes) /* 1. shortcut if the memory size is 0 */ if (cc->cur_wasm_module->memories != NULL && 0 == cc->cur_wasm_module->memories[mem_idx].init_page_count) { - JitReg module_inst, cur_page_count; - uint32 cur_page_count_offset = - (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) - + (uint32)offsetof(WASMMemoryInstance, cur_page_count); + + cur_page_count = get_cur_page_count_reg(cc->jit_frame, mem_idx); /* if (cur_mem_page_count == 0) goto EXCEPTION */ - module_inst = get_module_inst_reg(cc->jit_frame); - cur_page_count = jit_cc_new_reg_I32(cc); - GEN_INSN(LDI32, cur_page_count, module_inst, - NEW_CONST(I32, cur_page_count_offset)); GEN_INSN(CMP, cc->cmp_reg, cur_page_count, NEW_CONST(I32, 0)); if (!jit_emit_exception(cc, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, JIT_OP_BEQ, cc->cmp_reg, NULL)) { @@ -580,15 +575,9 @@ fail: bool jit_compile_op_memory_size(JitCompContext *cc, uint32 mem_idx) { - JitReg module_inst, cur_page_count; - uint32 cur_page_count_offset = - (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) - + (uint32)offsetof(WASMMemoryInstance, cur_page_count); + JitReg cur_page_count; - module_inst = get_module_inst_reg(cc->jit_frame); - cur_page_count = jit_cc_new_reg_I32(cc); - GEN_INSN(LDI32, cur_page_count, module_inst, - NEW_CONST(I32, cur_page_count_offset)); + cur_page_count = get_cur_page_count_reg(cc->jit_frame, mem_idx); PUSH_I32(cur_page_count); @@ -600,18 +589,11 @@ fail: bool jit_compile_op_memory_grow(JitCompContext *cc, uint32 mem_idx) { - JitReg module_inst, grow_res, res; + JitReg grow_res, res; JitReg prev_page_count, inc_page_count, args[2]; - /* Get current page count */ - uint32 cur_page_count_offset = - (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) - + (uint32)offsetof(WASMMemoryInstance, cur_page_count); - - module_inst = get_module_inst_reg(cc->jit_frame); - prev_page_count = jit_cc_new_reg_I32(cc); - GEN_INSN(LDI32, prev_page_count, module_inst, - NEW_CONST(I32, cur_page_count_offset)); + /* Get current page count as prev_page_count */ + prev_page_count = get_cur_page_count_reg(cc->jit_frame, mem_idx); /* Call wasm_enlarge_memory */ POP_I32(inc_page_count); diff --git a/core/iwasm/fast-jit/jit_frontend.c b/core/iwasm/fast-jit/jit_frontend.c index b24657246..c9c22e0ad 100644 --- a/core/iwasm/fast-jit/jit_frontend.c +++ b/core/iwasm/fast-jit/jit_frontend.c @@ -218,42 +218,149 @@ get_aux_stack_bottom_reg(JitFrame *frame) return frame->aux_stack_bottom_reg; } +#if WASM_ENABLE_SHARED_MEMORY != 0 +static bool +is_shared_memory(WASMModule *module, uint32 mem_idx) +{ + WASMMemory *memory; + WASMMemoryImport *memory_import; + bool is_shared; + + if (mem_idx < module->import_memory_count) { + memory_import = &(module->import_memories[mem_idx].u.memory); + is_shared = memory_import->flags & 0x02 ? true : false; + } + else { + memory = &module->memories[mem_idx - module->import_memory_count]; + is_shared = memory->flags & 0x02 ? true : false; + } + return is_shared; +} +#endif + +JitReg +get_memory_inst_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + uint32 memory_inst_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memories_addr; + uint32 memories_offset; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].memory_inst) + return frame->memory_regs[mem_idx].memory_inst; + + frame->memory_regs[mem_idx].memory_inst = + cc->memory_regs[mem_idx].memory_inst; + + bh_assert(mem_idx == 0); +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memories_addr = jit_cc_new_reg_ptr(cc); + memories_offset = (uint32)offsetof(WASMModuleInstance, memories); + /* module_inst->memories */ + GEN_INSN(LDPTR, memories_addr, module_inst_reg, + NEW_CONST(I32, memories_offset)); + /* module_inst->memories[mem_idx], mem_idx can only be 0 now */ + GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_inst, memories_addr, + NEW_CONST(I32, mem_idx)); + } + else +#endif + { + memory_inst_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes); + GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_inst, + module_inst_reg, NEW_CONST(I32, memory_inst_offset)); + } + + return frame->memory_regs[mem_idx].memory_inst; +} + +JitReg +get_cur_page_count_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg; + uint32 cur_page_count_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].cur_page_count) + return frame->memory_regs[mem_idx].cur_page_count; + + frame->memory_regs[mem_idx].cur_page_count = + cc->memory_regs[mem_idx].cur_page_count; + + /* Get current page count */ + bh_assert(mem_idx == 0); +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + cur_page_count_offset = + (uint32)offsetof(WASMMemoryInstance, cur_page_count); + /* memories[mem_idx]->cur_page_count_offset */ + GEN_INSN(LDI32, frame->memory_regs[mem_idx].cur_page_count, + memory_inst_reg, NEW_CONST(I32, cur_page_count_offset)); + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + cur_page_count_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, cur_page_count); + GEN_INSN(LDI32, frame->memory_regs[mem_idx].cur_page_count, + module_inst_reg, NEW_CONST(I32, cur_page_count_offset)); + } + + return frame->memory_regs[mem_idx].cur_page_count; +} + JitReg get_memory_data_reg(JitFrame *frame, uint32 mem_idx) { JitCompContext *cc = frame->cc; - JitReg module_inst_reg = get_module_inst_reg(frame); + JitReg module_inst_reg; uint32 memory_data_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].memory_data) + return frame->memory_regs[mem_idx].memory_data; + + frame->memory_regs[mem_idx].memory_data = + cc->memory_regs[mem_idx].memory_data; bh_assert(mem_idx == 0); #if WASM_ENABLE_SHARED_MEMORY != 0 - uint32 memories_offset = (uint32)offsetof(WASMModuleInstance, memories); - JitReg memories_addr = jit_cc_new_reg_ptr(cc); - JitReg memories_0_addr = jit_cc_new_reg_ptr(cc); - memory_data_offset = (uint32)offsetof(WASMMemoryInstance, memory_data); - if (!frame->memory_regs[mem_idx].memory_data) { - frame->memory_regs[mem_idx].memory_data = - cc->memory_regs[mem_idx].memory_data; - /* module_inst->memories */ - GEN_INSN(LDPTR, memories_addr, module_inst_reg, - NEW_CONST(I32, memories_offset)); - /* module_inst->memories[0] */ - GEN_INSN(LDPTR, memories_0_addr, memories_addr, NEW_CONST(I32, 0)); - /* memories[0]->memory_data */ + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + memory_data_offset = (uint32)offsetof(WASMMemoryInstance, memory_data); + /* memories[mem_idx]->memory_data */ GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data, - memories_0_addr, NEW_CONST(I32, memory_data_offset)); + memory_inst_reg, NEW_CONST(I32, memory_data_offset)); } -#else - memory_data_offset = - (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) - + (uint32)offsetof(WASMMemoryInstance, memory_data); - if (!frame->memory_regs[mem_idx].memory_data) { - frame->memory_regs[mem_idx].memory_data = - cc->memory_regs[mem_idx].memory_data; + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + memory_data_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, memory_data); GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data, module_inst_reg, NEW_CONST(I32, memory_data_offset)); } -#endif return frame->memory_regs[mem_idx].memory_data; } @@ -261,16 +368,37 @@ JitReg get_memory_data_end_reg(JitFrame *frame, uint32 mem_idx) { JitCompContext *cc = frame->cc; - JitReg module_inst_reg = get_module_inst_reg(frame); - uint32 memory_data_end_offset = - (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) - + (uint32)offsetof(WASMMemoryInstance, memory_data_end); + JitReg module_inst_reg; + uint32 memory_data_end_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].memory_data_end) + return frame->memory_regs[mem_idx].memory_data_end; + + frame->memory_regs[mem_idx].memory_data_end = + cc->memory_regs[mem_idx].memory_data_end; bh_assert(mem_idx == 0); - - if (!frame->memory_regs[mem_idx].memory_data_end) { - frame->memory_regs[mem_idx].memory_data_end = - cc->memory_regs[mem_idx].memory_data_end; +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + memory_data_end_offset = + (uint32)offsetof(WASMMemoryInstance, memory_data_end); + /* memories[mem_idx]->memory_data_end */ + GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data_end, + memory_inst_reg, NEW_CONST(I32, memory_data_end_offset)); + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + memory_data_end_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, memory_data_end); GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data_end, module_inst_reg, NEW_CONST(I32, memory_data_end_offset)); } @@ -281,16 +409,43 @@ JitReg get_mem_bound_check_1byte_reg(JitFrame *frame, uint32 mem_idx) { JitCompContext *cc = frame->cc; - JitReg module_inst_reg = get_module_inst_reg(frame); - uint32 mem_bound_check_1byte_offset = - (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) - + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_1byte); + JitReg module_inst_reg; + uint32 mem_bound_check_1byte_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].mem_bound_check_1byte) + return frame->memory_regs[mem_idx].mem_bound_check_1byte; + + frame->memory_regs[mem_idx].mem_bound_check_1byte = + cc->memory_regs[mem_idx].mem_bound_check_1byte; bh_assert(mem_idx == 0); - if (!frame->memory_regs[mem_idx].mem_bound_check_1byte) { - frame->memory_regs[mem_idx].mem_bound_check_1byte = - cc->memory_regs[mem_idx].mem_bound_check_1byte; +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + mem_bound_check_1byte_offset = + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_1byte); + /* memories[mem_idx]->mem_bound_check_1byte */ +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_1byte, + memory_inst_reg, NEW_CONST(I32, mem_bound_check_1byte_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_1byte, + memory_inst_reg, NEW_CONST(I32, mem_bound_check_1byte_offset)); +#endif + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + mem_bound_check_1byte_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_1byte); #if UINTPTR_MAX == UINT64_MAX GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_1byte, module_inst_reg, NEW_CONST(I32, mem_bound_check_1byte_offset)); @@ -306,16 +461,45 @@ JitReg get_mem_bound_check_2bytes_reg(JitFrame *frame, uint32 mem_idx) { JitCompContext *cc = frame->cc; - JitReg module_inst_reg = get_module_inst_reg(frame); - uint32 mem_bound_check_2bytes_offset = - (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) - + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_2bytes); + JitReg module_inst_reg; + uint32 mem_bound_check_2bytes_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].mem_bound_check_2bytes) + return frame->memory_regs[mem_idx].mem_bound_check_2bytes; + + frame->memory_regs[mem_idx].mem_bound_check_2bytes = + cc->memory_regs[mem_idx].mem_bound_check_2bytes; bh_assert(mem_idx == 0); - if (!frame->memory_regs[mem_idx].mem_bound_check_2bytes) { - frame->memory_regs[mem_idx].mem_bound_check_2bytes = - cc->memory_regs[mem_idx].mem_bound_check_2bytes; +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + mem_bound_check_2bytes_offset = + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_2bytes); + /* memories[mem_idx]->mem_bound_check_2bytes */ +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_2bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_2bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_2bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_2bytes_offset)); +#endif + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + mem_bound_check_2bytes_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_2bytes); #if UINTPTR_MAX == UINT64_MAX GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_2bytes, module_inst_reg, @@ -333,16 +517,45 @@ JitReg get_mem_bound_check_4bytes_reg(JitFrame *frame, uint32 mem_idx) { JitCompContext *cc = frame->cc; - JitReg module_inst_reg = get_module_inst_reg(frame); - uint32 mem_bound_check_4bytes_offset = - (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) - + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_4bytes); + JitReg module_inst_reg; + uint32 mem_bound_check_4bytes_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].mem_bound_check_4bytes) + return frame->memory_regs[mem_idx].mem_bound_check_4bytes; + + frame->memory_regs[mem_idx].mem_bound_check_4bytes = + cc->memory_regs[mem_idx].mem_bound_check_4bytes; bh_assert(mem_idx == 0); - if (!frame->memory_regs[mem_idx].mem_bound_check_4bytes) { - frame->memory_regs[mem_idx].mem_bound_check_4bytes = - cc->memory_regs[mem_idx].mem_bound_check_4bytes; +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + mem_bound_check_4bytes_offset = + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_4bytes); + /* memories[mem_idx]->mem_bound_check_4bytes */ +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_4bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_4bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_4bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_4bytes_offset)); +#endif + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + mem_bound_check_4bytes_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_4bytes); #if UINTPTR_MAX == UINT64_MAX GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_4bytes, module_inst_reg, @@ -360,16 +573,45 @@ JitReg get_mem_bound_check_8bytes_reg(JitFrame *frame, uint32 mem_idx) { JitCompContext *cc = frame->cc; - JitReg module_inst_reg = get_module_inst_reg(frame); - uint32 mem_bound_check_8bytes_offset = - (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) - + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_8bytes); + JitReg module_inst_reg; + uint32 mem_bound_check_8bytes_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].mem_bound_check_8bytes) + return frame->memory_regs[mem_idx].mem_bound_check_8bytes; + + frame->memory_regs[mem_idx].mem_bound_check_8bytes = + cc->memory_regs[mem_idx].mem_bound_check_8bytes; bh_assert(mem_idx == 0); - if (!frame->memory_regs[mem_idx].mem_bound_check_8bytes) { - frame->memory_regs[mem_idx].mem_bound_check_8bytes = - cc->memory_regs[mem_idx].mem_bound_check_8bytes; +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + mem_bound_check_8bytes_offset = + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_8bytes); + /* memories[mem_idx]->mem_bound_check_8bytes */ +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_8bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_8bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_8bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_8bytes_offset)); +#endif + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + mem_bound_check_8bytes_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_8bytes); #if UINTPTR_MAX == UINT64_MAX GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_8bytes, module_inst_reg, @@ -387,16 +629,45 @@ JitReg get_mem_bound_check_16bytes_reg(JitFrame *frame, uint32 mem_idx) { JitCompContext *cc = frame->cc; - JitReg module_inst_reg = get_module_inst_reg(frame); - uint32 mem_bound_check_16bytes_offset = - (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) - + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_16bytes); + JitReg module_inst_reg; + uint32 mem_bound_check_16bytes_offset; +#if WASM_ENABLE_SHARED_MEMORY != 0 + JitReg memory_inst_reg; + bool is_shared; +#endif + + if (frame->memory_regs[mem_idx].mem_bound_check_16bytes) + return frame->memory_regs[mem_idx].mem_bound_check_16bytes; + + frame->memory_regs[mem_idx].mem_bound_check_16bytes = + cc->memory_regs[mem_idx].mem_bound_check_16bytes; bh_assert(mem_idx == 0); - if (!frame->memory_regs[mem_idx].mem_bound_check_16bytes) { - frame->memory_regs[mem_idx].mem_bound_check_16bytes = - cc->memory_regs[mem_idx].mem_bound_check_16bytes; +#if WASM_ENABLE_SHARED_MEMORY != 0 + is_shared = is_shared_memory(cc->cur_wasm_module, mem_idx); + if (is_shared) { + memory_inst_reg = get_memory_inst_reg(frame, mem_idx); + mem_bound_check_16bytes_offset = + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_16bytes); + /* memories[mem_idx]->mem_bound_check_16bytes */ +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_16bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_16bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_16bytes, + memory_inst_reg, + NEW_CONST(I32, mem_bound_check_16bytes_offset)); +#endif + } + else +#endif + { + module_inst_reg = get_module_inst_reg(frame); + mem_bound_check_16bytes_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_16bytes); #if UINTPTR_MAX == UINT64_MAX GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_16bytes, module_inst_reg, @@ -462,6 +733,8 @@ clear_fixed_virtual_regs(JitFrame *frame) count = module->import_memory_count + module->memory_count; for (i = 0; i < count; i++) { + frame->memory_regs[i].memory_inst = 0; + frame->memory_regs[i].cur_page_count = 0; frame->memory_regs[i].memory_data = 0; frame->memory_regs[i].memory_data_end = 0; frame->memory_regs[i].mem_bound_check_1byte = 0; @@ -486,6 +759,7 @@ clear_memory_regs(JitFrame *frame) count = module->import_memory_count + module->memory_count; for (i = 0; i < count; i++) { + frame->memory_regs[i].cur_page_count = 0; frame->memory_regs[i].memory_data = 0; frame->memory_regs[i].memory_data_end = 0; frame->memory_regs[i].mem_bound_check_1byte = 0; @@ -654,6 +928,8 @@ create_fixed_virtual_regs(JitCompContext *cc) } for (i = 0; i < count; i++) { + cc->memory_regs[i].memory_inst = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].cur_page_count = jit_cc_new_reg_I32(cc); cc->memory_regs[i].memory_data = jit_cc_new_reg_ptr(cc); cc->memory_regs[i].memory_data_end = jit_cc_new_reg_ptr(cc); cc->memory_regs[i].mem_bound_check_1byte = jit_cc_new_reg_ptr(cc); diff --git a/core/iwasm/fast-jit/jit_frontend.h b/core/iwasm/fast-jit/jit_frontend.h index 7aa460fd9..9065d23ec 100644 --- a/core/iwasm/fast-jit/jit_frontend.h +++ b/core/iwasm/fast-jit/jit_frontend.h @@ -192,6 +192,12 @@ get_aux_stack_bound_reg(JitFrame *frame); JitReg get_aux_stack_bottom_reg(JitFrame *frame); +JitReg +get_memory_inst_reg(JitFrame *frame, uint32 mem_idx); + +JitReg +get_cur_page_count_reg(JitFrame *frame, uint32 mem_idx); + JitReg get_memory_data_reg(JitFrame *frame, uint32 mem_idx); diff --git a/core/iwasm/fast-jit/jit_ir.h b/core/iwasm/fast-jit/jit_ir.h index e13a41d1d..bef3fcc0c 100644 --- a/core/iwasm/fast-jit/jit_ir.h +++ b/core/iwasm/fast-jit/jit_ir.h @@ -866,6 +866,8 @@ typedef struct JitValueSlot { typedef struct JitMemRegs { /* The following registers should be re-loaded after memory.grow, callbc and callnative */ + JitReg memory_inst; + JitReg cur_page_count; JitReg memory_data; JitReg memory_data_end; JitReg mem_bound_check_1byte; From fbd9a760e770acc01e87023cd6d24995e51aa260 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Fri, 1 Dec 2023 19:12:55 +0900 Subject: [PATCH 37/67] test_wamr.sh: Don't bother to build shared library (#2844) This script only uses the iwasm command. --- tests/wamr-test-suites/test_wamr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index f21e52fdb..76b235d49 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -706,7 +706,7 @@ function build_iwasm_with_cfg() && if [ -d build ]; then rm -rf build/*; else mkdir build; fi \ && cd build \ && cmake $* .. \ - && cmake --build . -j 4 --config RelWithDebInfo + && cmake --build . -j 4 --config RelWithDebInfo --target iwasm fi if [ "$?" != 0 ];then From 73914caa9b05cd702232aa719410b44143385061 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Fri, 1 Dec 2023 19:29:15 +0900 Subject: [PATCH 38/67] core/iwasm/interpreter/wasm_loader.c: remove an extra validation (#2845) * Empty names are spec-wise valid. * As we ignore unknown custom sections anyway, it's safe to accept empty names here. * Currently, the problem is not exposed on our CI because the wabt version used there is a bit old. --- core/iwasm/interpreter/wasm_loader.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 61e30be78..1286aa0ad 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -2708,7 +2708,7 @@ load_user_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, read_leb_uint32(p, p_end, name_len); - if (name_len == 0 || p + name_len > p_end) { + if (p + name_len > p_end) { set_error_buf(error_buf, error_buf_size, "unexpected end"); return false; } From 48e24032ba81dbad8597e4b920d57da63db0697d Mon Sep 17 00:00:00 2001 From: Daniel Mangum <31777345+hasheddan@users.noreply.github.com> Date: Sun, 3 Dec 2023 20:05:26 -0600 Subject: [PATCH 39/67] doc/build_wamr.md: Fix links to RISC-V named ABIs (#2852) Fixes links to RISC-V application binary interface names. Signed-off-by: Daniel Mangum --- doc/build_wamr.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/build_wamr.md b/doc/build_wamr.md index 9938eadcb..12cbb4125 100644 --- a/doc/build_wamr.md +++ b/doc/build_wamr.md @@ -27,8 +27,8 @@ The script `runtime_lib.cmake` defines a number of variables for configuring the - **WAMR_BUILD_TARGET**: set the target CPU architecture. Current supported targets are: X86_64, X86_32, AARCH64, ARM, THUMB, XTENSA, ARC, RISCV32, RISCV64 and MIPS. - For ARM and THUMB, the format is \\[\]\[_VFP], where \ is the ARM sub-architecture and the "_VFP" suffix means using VFP coprocessor registers s0-s15 (d0-d7) for passing arguments or returning results in standard procedure-call. Both \ and "_VFP" are optional, e.g. ARMV7, ARMV7_VFP, THUMBV7, THUMBV7_VFP and so on. - For AARCH64, the format is\[\], VFP is enabled by default. \ is optional, e.g. AARCH64, AARCH64V8, AARCH64V8.1 and so on. - - For RISCV64, the format is \[_abi], where "_abi" is optional, currently the supported formats are RISCV64, RISCV64_LP64D and RISCV64_LP64: RISCV64 and RISCV64_LP64D are identical, using [LP64D](https://github.com/riscv/riscv-elf-psabi-doc/blob/master/riscv-elf.md#-named-abis) as abi (LP64 with hardware floating-point calling convention for FLEN=64). And RISCV64_LP64 uses [LP64](https://github.com/riscv/riscv-elf-psabi-doc/blob/master/riscv-elf.md#-named-abis) as abi (Integer calling-convention only, and hardware floating-point calling convention is not used). - - For RISCV32, the format is \[_abi], where "_abi" is optional, currently the supported formats are RISCV32, RISCV32_ILP32D and RISCV32_ILP32: RISCV32 and RISCV32_ILP32D are identical, using [ILP32D](https://github.com/riscv/riscv-elf-psabi-doc/blob/master/riscv-elf.md#-named-abis) as abi (ILP32 with hardware floating-point calling convention for FLEN=64). And RISCV32_ILP32 uses [ILP32](https://github.com/riscv/riscv-elf-psabi-doc/blob/master/riscv-elf.md#-named-abis) as abi (Integer calling-convention only, and hardware floating-point calling convention is not used). + - For RISCV64, the format is \[_abi], where "_abi" is optional, currently the supported formats are RISCV64, RISCV64_LP64D and RISCV64_LP64: RISCV64 and RISCV64_LP64D are identical, using [LP64D](https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#named-abis) as abi (LP64 with hardware floating-point calling convention for FLEN=64). And RISCV64_LP64 uses [LP64](https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#named-abis) as abi (Integer calling-convention only, and hardware floating-point calling convention is not used). + - For RISCV32, the format is \[_abi], where "_abi" is optional, currently the supported formats are RISCV32, RISCV32_ILP32D and RISCV32_ILP32: RISCV32 and RISCV32_ILP32D are identical, using [ILP32D](https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#named-abis) as abi (ILP32 with hardware floating-point calling convention for FLEN=64). And RISCV32_ILP32 uses [ILP32](https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#named-abis) as abi (Integer calling-convention only, and hardware floating-point calling convention is not used). ```bash cmake -DWAMR_BUILD_PLATFORM=linux -DWAMR_BUILD_TARGET=ARM From 157c289d075bab41683d3bcca8b54e13b4b0ae1b Mon Sep 17 00:00:00 2001 From: Daniel Mangum <31777345+hasheddan@users.noreply.github.com> Date: Sun, 3 Dec 2023 21:38:54 -0600 Subject: [PATCH 40/67] Fix typos of CIDR in docs and help text (#2851) Fixes typos of CIDR in socket documentation and libc WASI help text. Signed-off-by: Daniel Mangum --- doc/socket_api.md | 2 +- product-mini/platforms/common/libc_wasi.c | 2 +- product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/socket_api.md b/doc/socket_api.md index 9e65d33cd..eff937617 100644 --- a/doc/socket_api.md +++ b/doc/socket_api.md @@ -58,7 +58,7 @@ enabled. _iwasm_ accepts address ranges via an option, `--addr-pool`, to implement the capability control. All IP address the WebAssembly application may need to `bind()` or `connect()` -should be announced first. Every IP address should be in CIRD notation. +should be announced first. Every IP address should be in CIDR notation. ```bash $ iwasm --addr-pool=1.2.3.4/15,2.3.4.6/16 socket_example.wasm diff --git a/product-mini/platforms/common/libc_wasi.c b/product-mini/platforms/common/libc_wasi.c index 2064c4431..84e133bc0 100644 --- a/product-mini/platforms/common/libc_wasi.c +++ b/product-mini/platforms/common/libc_wasi.c @@ -47,7 +47,7 @@ libc_wasi_print_help() "--map-dir=\n"); printf(" --addr-pool= Grant wasi access to the given network " "addresses in\n"); - printf(" CIRD notation to the program, seperated " + printf(" CIDR notation to the program, seperated " "with ',',\n"); printf(" for example:\n"); printf(" --addr-pool=1.2.3.4/15,2.3.4.5/16\n"); diff --git a/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp b/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp index 2b5300ff3..9f24cdaa0 100644 --- a/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp +++ b/product-mini/platforms/linux-sgx/enclave-sample/App/App.cpp @@ -228,7 +228,7 @@ print_help() printf(" to the program, for example:\n"); printf(" --dir= --dir=\n"); printf(" --addr-pool= Grant wasi access to the given network addresses in\n"); - printf(" CIRD notation to the program, seperated with ',',\n"); + printf(" CIDR notation to the program, seperated with ',',\n"); printf(" for example:\n"); printf(" --addr-pool=1.2.3.4/15,2.3.4.5/16\n"); printf(" --max-threads=n Set maximum thread number per cluster, default is 4\n"); From 453a29a9d4d5ad311609032678de1c8787bd7f1e Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Mon, 4 Dec 2023 14:22:47 +0800 Subject: [PATCH 41/67] Enable spectest on riscv64 (#2843) Fix relocation issues on riscv and update test scripts and CI to enable test spec cases on riscv QEMU. --- .github/workflows/spec_test_on_nuttx.yml | 17 ++- core/iwasm/aot/arch/aot_reloc_riscv.c | 116 ++++++++++-------- .../spec-test-script/runtest.py | 11 ++ 3 files changed, 89 insertions(+), 55 deletions(-) diff --git a/.github/workflows/spec_test_on_nuttx.yml b/.github/workflows/spec_test_on_nuttx.yml index 12269b562..be209ec2b 100644 --- a/.github/workflows/spec_test_on_nuttx.yml +++ b/.github/workflows/spec_test_on_nuttx.yml @@ -59,11 +59,11 @@ jobs: # target: "riscv32_ilp32d", # use_fpu: true # }, - # { - # config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh64", - # target: "riscv64", - # use_fpu: false - # }, + { + config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh64", + target: "riscv64", + use_fpu: false + }, ] wamr_test_option: [ @@ -85,6 +85,11 @@ jobs: }, ] + exclude: + # XIP is not fully supported yet on RISCV64, some relocations can not be resolved + - target_config: { config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh64" } + wamr_test_option: { mode: "-t aot -X" } + steps: - name: Install Utilities run: | @@ -160,7 +165,7 @@ jobs: find nuttx/boards -name defconfig | xargs sed -i '$a\${{ matrix.wamr_test_option.option }}' - name: Disable FPU for NuttX - if: matrix.target_config.use_fpu== false + if: matrix.target_config.use_fpu == false run: | find nuttx/boards -name defconfig | xargs sed -i '$a\# CONFIG_ARCH_FPU is not set\n' diff --git a/core/iwasm/aot/arch/aot_reloc_riscv.c b/core/iwasm/aot/arch/aot_reloc_riscv.c index d765e0b87..045cab641 100644 --- a/core/iwasm/aot/arch/aot_reloc_riscv.c +++ b/core/iwasm/aot/arch/aot_reloc_riscv.c @@ -11,6 +11,7 @@ #define R_RISCV_CALL_PLT 19 #define R_RISCV_PCREL_HI20 23 #define R_RISCV_PCREL_LO12_I 24 +#define R_RISCV_PCREL_LO12_S 25 #define R_RISCV_HI20 26 #define R_RISCV_LO12_I 27 #define R_RISCV_LO12_S 28 @@ -101,20 +102,20 @@ static SymbolMap target_sym_map[] = { REG_SYM(__truncdfsf2), REG_SYM(__unorddf2), REG_SYM(__unordsf2), -#if __riscv_xlen == 32 + REG_SYM(__mulsf3), + REG_SYM(__floatundidf), REG_SYM(__fixdfdi), + REG_SYM(__floatsidf), + REG_SYM(__floatunsidf), +#if __riscv_xlen == 32 REG_SYM(__fixdfsi), REG_SYM(__fixsfdi), REG_SYM(__fixsfsi), REG_SYM(__floatdidf), REG_SYM(__floatdisf), - REG_SYM(__floatsidf), REG_SYM(__floatsisf), - REG_SYM(__floatundidf), REG_SYM(__floatundisf), - REG_SYM(__floatunsidf), REG_SYM(__floatunsisf), - REG_SYM(__mulsf3), REG_SYM(__mulsi3), #endif #endif @@ -269,9 +270,10 @@ typedef struct RelocTypeStrMap { } static RelocTypeStrMap reloc_type_str_maps[] = { - RELOC_TYPE_MAP(R_RISCV_32), RELOC_TYPE_MAP(R_RISCV_CALL), - RELOC_TYPE_MAP(R_RISCV_CALL_PLT), RELOC_TYPE_MAP(R_RISCV_PCREL_HI20), - RELOC_TYPE_MAP(R_RISCV_PCREL_LO12_I), RELOC_TYPE_MAP(R_RISCV_HI20), + RELOC_TYPE_MAP(R_RISCV_32), RELOC_TYPE_MAP(R_RISCV_64), + RELOC_TYPE_MAP(R_RISCV_CALL), RELOC_TYPE_MAP(R_RISCV_CALL_PLT), + RELOC_TYPE_MAP(R_RISCV_PCREL_HI20), RELOC_TYPE_MAP(R_RISCV_PCREL_LO12_I), + RELOC_TYPE_MAP(R_RISCV_PCREL_LO12_S), RELOC_TYPE_MAP(R_RISCV_HI20), RELOC_TYPE_MAP(R_RISCV_LO12_I), RELOC_TYPE_MAP(R_RISCV_LO12_S), }; @@ -327,21 +329,37 @@ apply_relocation(AOTModule *module, uint8 *target_section_addr, rv_set_val((uint16 *)addr, val_32); break; } + +#if __riscv_xlen == 64 case R_RISCV_64: { uint64 val_64 = - (uint64)((uintptr_t)symbol_addr + (intptr_t)reloc_addend); + (uint64)((intptr_t)symbol_addr + (intptr_t)reloc_addend); + CHECK_RELOC_OFFSET(sizeof(uint64)); + if (val_64 + != (uint64)((intptr_t)symbol_addr + (intptr_t)reloc_addend)) { + goto fail_addr_out_of_range; + } + bh_memcpy_s(addr, 8, &val_64, 8); +#ifdef __riscv_zifencei + __asm__ volatile("fence.i"); +#else + __asm__ volatile("fence"); +#endif break; } +#endif + case R_RISCV_CALL: case R_RISCV_CALL_PLT: + case R_RISCV_PCREL_HI20: /* S + A - P */ { - val = (int32)(intptr_t)((uint8 *)symbol_addr - addr); + val = (int32)(intptr_t)((uint8 *)symbol_addr + reloc_addend - addr); CHECK_RELOC_OFFSET(sizeof(uint32)); - if (val != (intptr_t)((uint8 *)symbol_addr - addr)) { + if (val != (intptr_t)((uint8 *)symbol_addr + reloc_addend - addr)) { if (symbol_index >= 0) { /* Call runtime function by plt code */ symbol_addr = (uint8 *)module->code + module->code_size @@ -351,7 +369,7 @@ apply_relocation(AOTModule *module, uint8 *target_section_addr, } } - if (val != (intptr_t)((uint8 *)symbol_addr - addr)) { + if (val != (intptr_t)((uint8 *)symbol_addr + reloc_addend - addr)) { goto fail_addr_out_of_range; } @@ -372,32 +390,15 @@ apply_relocation(AOTModule *module, uint8 *target_section_addr, break; } - case R_RISCV_HI20: /* S + A */ - case R_RISCV_PCREL_HI20: /* S + A - P */ + case R_RISCV_HI20: /* S + A */ { - if (reloc_type == R_RISCV_PCREL_HI20) { - val = (int32)((intptr_t)symbol_addr + (intptr_t)reloc_addend - - (intptr_t)addr); - } - else { - val = (int32)((intptr_t)symbol_addr + (intptr_t)reloc_addend); - } + val = (int32)((intptr_t)symbol_addr + (intptr_t)reloc_addend); CHECK_RELOC_OFFSET(sizeof(uint32)); - if (reloc_type == R_RISCV_PCREL_HI20) { - if (val - != ((intptr_t)symbol_addr + (intptr_t)reloc_addend - - (intptr_t)addr)) { - goto fail_addr_out_of_range; - } - } - else { - if (val != ((intptr_t)symbol_addr + (intptr_t)reloc_addend)) { - goto fail_addr_out_of_range; - } + if (val != ((intptr_t)symbol_addr + (intptr_t)reloc_addend)) { + goto fail_addr_out_of_range; } - addr = target_section_addr + reloc_offset; insn = rv_get_val((uint16 *)addr); rv_calc_imm(val, &imm_hi, &imm_lo); insn = (insn & 0x00000fff) | (imm_hi << 12); @@ -405,27 +406,44 @@ apply_relocation(AOTModule *module, uint8 *target_section_addr, break; } - case R_RISCV_LO12_I: /* S + A */ case R_RISCV_PCREL_LO12_I: /* S - P */ + case R_RISCV_PCREL_LO12_S: /* S - P */ { - if (reloc_type == R_RISCV_PCREL_LO12_I) { - /* A = 0 */ - val = (int32)((intptr_t)symbol_addr - (intptr_t)addr); - } - else { - val = (int32)((intptr_t)symbol_addr + (intptr_t)reloc_addend); + /* Already handled in R_RISCV_PCREL_HI20, it should be skipped for + * most cases. But it is still needed for some special cases, e.g. + * ``` + * label: + * auipc t0, %pcrel_hi(symbol) # R_RISCV_PCREL_HI20 (symbol) + * lui t1, 1 + * lw t2, t0, %pcrel_lo(label) # R_RISCV_PCREL_LO12_I (label) + * add t2, t2, t1 + * sw t2, t0, %pcrel_lo(label) # R_RISCV_PCREL_LO12_S (label) + * ``` + * In this case, the R_RISCV_PCREL_LO12_I/S relocation should be + * handled after R_RISCV_PCREL_HI20 relocation. + * + * So, if the R_RISCV_PCREL_LO12_I/S relocation is not followed by + * R_RISCV_PCREL_HI20 relocation, it should be handled here but + * not implemented yet. + */ + + if ((uintptr_t)addr - (uintptr_t)symbol_addr + - (uintptr_t)reloc_addend + != 4) { + goto fail_addr_out_of_range; } + break; + } + + case R_RISCV_LO12_I: /* S + A */ + { + + val = (int32)((intptr_t)symbol_addr + (intptr_t)reloc_addend); CHECK_RELOC_OFFSET(sizeof(uint32)); - if (reloc_type == R_RISCV_PCREL_LO12_I) { - if (val != (intptr_t)symbol_addr - (intptr_t)addr) { - goto fail_addr_out_of_range; - } - } - else { - if (val != (intptr_t)symbol_addr + (intptr_t)reloc_addend) { - goto fail_addr_out_of_range; - } + + if (val != (intptr_t)symbol_addr + (intptr_t)reloc_addend) { + goto fail_addr_out_of_range; } addr = target_section_addr + reloc_offset; diff --git a/tests/wamr-test-suites/spec-test-script/runtest.py b/tests/wamr-test-suites/spec-test-script/runtest.py index ca7dfcd9a..6ca1df0d4 100755 --- a/tests/wamr-test-suites/spec-test-script/runtest.py +++ b/tests/wamr-test-suites/spec-test-script/runtest.py @@ -1072,6 +1072,17 @@ def compile_wasm_to_aot(wasm_tempfile, aot_tempfile, runner, opts, r, output = ' # exception isn't thrown in several cases cmd.append("--disable-llvm-lto") + # Bounds checks is disabled by default for 64-bit targets, to + # use the hardware based bounds checks. But it is not supported + # in QEMU with NuttX. + # Enable bounds checks explicitly for all targets if running in QEMU. + if opts.qemu: + cmd.append("--bounds-checks=1") + + # RISCV64 requires -mcmodel=medany, which can be set by --size-level=1 + if test_target.startswith("riscv64"): + cmd.append("--size-level=1") + cmd += ["-o", aot_tempfile, wasm_tempfile] log("Running: %s" % " ".join(cmd)) From b0d5b8df1dd45bb7c1c22d097501b0f2daa89354 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Mon, 4 Dec 2023 16:40:54 +0800 Subject: [PATCH 42/67] Fix issues of build/run with llvm-17 (#2853) - Fix compilation error of using PGOOptions - Fix LLVM JIT run error due to `llvm_orc_registerEHFrameSectionWrapper` symbol not found --- build-scripts/config_common.cmake | 5 +++++ core/iwasm/compilation/aot_llvm_extra.cpp | 23 +++++++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 210f2d788..e73ebc85f 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -122,6 +122,11 @@ if (WAMR_BUILD_JIT EQUAL 1) if (CXX_SUPPORTS_REDUNDANT_MOVE_FLAG) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-redundant-move") endif () + # Enable exporting symbols after llvm-17, or LLVM JIT may run failed + # with `llvm_orc_registerEHFrameSectionWrapper` symbol not found error + if (${LLVM_PACKAGE_VERSION} VERSION_GREATER_EQUAL "17.0.0") + set (CMAKE_ENABLE_EXPORTS 1) + endif () endif () else () unset (LLVM_AVAILABLE_LIBS) diff --git a/core/iwasm/compilation/aot_llvm_extra.cpp b/core/iwasm/compilation/aot_llvm_extra.cpp index 72e163fa2..ed0205c3c 100644 --- a/core/iwasm/compilation/aot_llvm_extra.cpp +++ b/core/iwasm/compilation/aot_llvm_extra.cpp @@ -36,6 +36,7 @@ #include #if LLVM_VERSION_MAJOR >= 17 #include +#include #endif #include #include @@ -203,19 +204,27 @@ aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module) Optional PGO = llvm::None; #endif -// TODO -#if LLVM_VERSION_MAJOR < 17 if (comp_ctx->enable_llvm_pgo) { /* Disable static counter allocation for value profiler, it will be allocated by runtime */ const char *argv[] = { "", "-vp-static-alloc=false" }; cl::ParseCommandLineOptions(2, argv); +#if LLVM_VERSION_MAJOR < 17 PGO = PGOOptions("", "", "", PGOOptions::IRInstr); +#else + auto FS = vfs::getRealFileSystem(); + PGO = PGOOptions("", "", "", "", FS, PGOOptions::IRInstr); +#endif } else if (comp_ctx->use_prof_file) { +#if LLVM_VERSION_MAJOR < 17 PGO = PGOOptions(comp_ctx->use_prof_file, "", "", PGOOptions::IRUse); - } +#else + auto FS = vfs::getRealFileSystem(); + PGO = PGOOptions(comp_ctx->use_prof_file, "", "", "", FS, + PGOOptions::IRUse); #endif + } #ifdef DEBUG_PASS PassInstrumentationCallbacks PIC; @@ -343,7 +352,13 @@ aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module) ExitOnErr(PB.parsePassPipeline(MPM, comp_ctx->llvm_passes)); } - if (OptimizationLevel::O0 == OL) { + if ( +#if LLVM_VERSION_MAJOR <= 13 + PassBuilder::OptimizationLevel::O0 == OL +#else + OptimizationLevel::O0 == OL +#endif + ) { MPM.addPass(PB.buildO0DefaultPipeline(OL)); } else { From e350e65b12b3b7f72d57a11bb14fbcd52dded792 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Mon, 4 Dec 2023 17:00:21 +0800 Subject: [PATCH 43/67] Don't add "+d" to riscv cpu features if already given (#2855) --- core/iwasm/compilation/aot_llvm.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index f15cfd829..adfc1d0bd 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -2708,13 +2708,14 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) meta_target_abi); if (!strcmp(abi, "lp64d") || !strcmp(abi, "ilp32d")) { - if (features) { + if (features && !strstr(features, "+d")) { snprintf(features_buf, sizeof(features_buf), "%s%s", features, ",+d"); features = features_buf; } - else + else if (!features) { features = "+d"; + } } } From 7308b1eb006803c110ca9bbe764a6bd5392aafc6 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Mon, 4 Dec 2023 18:57:18 +0800 Subject: [PATCH 44/67] Update FPU configuration in spec_test_on_nuttx.yml (#2856) --- .github/workflows/spec_test_on_nuttx.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/spec_test_on_nuttx.yml b/.github/workflows/spec_test_on_nuttx.yml index be209ec2b..bba376a82 100644 --- a/.github/workflows/spec_test_on_nuttx.yml +++ b/.github/workflows/spec_test_on_nuttx.yml @@ -37,32 +37,32 @@ jobs: # { # config: "boards/arm64/qemu/qemu-armv8a/configs/nsh", # target: "aarch64_vfp", - # use_fpu: true + # fpu_type: "fp" # }, # { # config: "boards/arm/imx6/sabre-6quad/configs/nsh", # target: "thumbv7", - # use_fpu: false + # fpu_type: "none" # }, { config: "boards/arm/imx6/sabre-6quad/configs/nsh", target: "thumbv7_vfp", - use_fpu: true + fpu_type: "dp" }, { config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh", target: "riscv32", - use_fpu: false + fpu_type: "none" }, # { # config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh", # target: "riscv32_ilp32d", - # use_fpu: true + # fpu_type: "dp" # }, { config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh64", target: "riscv64", - use_fpu: false + fpu_type: "none" }, ] @@ -165,10 +165,15 @@ jobs: find nuttx/boards -name defconfig | xargs sed -i '$a\${{ matrix.wamr_test_option.option }}' - name: Disable FPU for NuttX - if: matrix.target_config.use_fpu == false + if: matrix.target_config.fpu_type == 'none' run: | find nuttx/boards -name defconfig | xargs sed -i '$a\# CONFIG_ARCH_FPU is not set\n' + - name: Disable DPFPU for NuttX + if: matrix.target_config.fpu_type == 'fp' + run: | + find nuttx/boards -name defconfig | xargs sed -i '$a\# CONFIG_ARCH_DPFPU is not set\n' + - name: Build wamrc if: contains(matrix.wamr_test_option.mode, 'aot') working-directory: apps/interpreters/wamr/wamr/wamr-compiler From 23c1343fb3840390e6afd6cc449fe6fd91cb6415 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Tue, 5 Dec 2023 16:59:13 +0800 Subject: [PATCH 45/67] Fix wasm loader handle op_br_table and op_drop (#2864) - Fix op_br_table arity type check when the dest block is loop block - Fix op_drop issue when the stack is polymorphic and it is to drop an ANY type value in the stack --- core/iwasm/interpreter/wasm_loader.c | 9 ++++++++- core/iwasm/interpreter/wasm_mini_loader.c | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 1286aa0ad..dc1614a57 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -7451,6 +7451,9 @@ re_scan: if (frame_csp_tmp->label_type != LABEL_TYPE_LOOP) ret_count = block_type_get_result_types( &frame_csp_tmp->block_type, &ret_types); + else + ret_count = block_type_get_param_types( + &frame_csp_tmp->block_type, &ret_types); } else { uint8 *tmp_ret_types = NULL; @@ -7461,6 +7464,9 @@ re_scan: if (frame_csp_tmp->label_type != LABEL_TYPE_LOOP) tmp_ret_count = block_type_get_result_types( &frame_csp_tmp->block_type, &tmp_ret_types); + else + tmp_ret_count = block_type_get_param_types( + &frame_csp_tmp->block_type, &tmp_ret_types); if (ret_count != tmp_ret_count || (ret_count @@ -7753,7 +7759,8 @@ re_scan: } if (available_stack_cell > 0) { - if (is_32bit_type(*(loader_ctx->frame_ref - 1))) { + if (is_32bit_type(*(loader_ctx->frame_ref - 1)) + || *(loader_ctx->frame_ref - 1) == VALUE_TYPE_ANY) { loader_ctx->frame_ref--; loader_ctx->stack_cell_num--; #if WASM_ENABLE_FAST_INTERP != 0 diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 2d5bf57a9..85da017d6 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -6100,7 +6100,8 @@ re_scan: && !cur_block->is_stack_polymorphic)); if (available_stack_cell > 0) { - if (is_32bit_type(*(loader_ctx->frame_ref - 1))) { + if (is_32bit_type(*(loader_ctx->frame_ref - 1)) + || *(loader_ctx->frame_ref - 1) == VALUE_TYPE_ANY) { loader_ctx->frame_ref--; loader_ctx->stack_cell_num--; #if WASM_ENABLE_FAST_INTERP != 0 From 53c3fa27d44710701693a62044f2f0fe882ebca0 Mon Sep 17 00:00:00 2001 From: Xu Jun Date: Tue, 5 Dec 2023 17:09:05 +0800 Subject: [PATCH 46/67] Fix block with type issue in fast interp (#2866) Reported in https://github.com/bytecodealliance/wasm-micro-runtime/issues/2863. --- core/iwasm/interpreter/wasm_loader.c | 11 +++++------ core/iwasm/interpreter/wasm_mini_loader.c | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index dc1614a57..2a06f42e3 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -7199,10 +7199,7 @@ re_scan: } #if WASM_ENABLE_FAST_INTERP != 0 - if (opcode == WASM_OP_BLOCK) { - skip_label(); - } - else if (opcode == WASM_OP_LOOP) { + if (opcode == WASM_OP_BLOCK || opcode == WASM_OP_LOOP) { skip_label(); if (BLOCK_HAS_PARAM(block_type)) { /* Make sure params are in dynamic space */ @@ -7210,8 +7207,10 @@ re_scan: loader_ctx, false, error_buf, error_buf_size)) goto fail; } - (loader_ctx->frame_csp - 1)->code_compiled = - loader_ctx->p_code_compiled; + if (opcode == WASM_OP_LOOP) { + (loader_ctx->frame_csp - 1)->code_compiled = + loader_ctx->p_code_compiled; + } } else if (opcode == WASM_OP_IF) { /* If block has parameters, we should make sure they are in diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 85da017d6..47ec549ee 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -5627,10 +5627,7 @@ re_scan: } #if WASM_ENABLE_FAST_INTERP != 0 - if (opcode == WASM_OP_BLOCK) { - skip_label(); - } - else if (opcode == WASM_OP_LOOP) { + if (opcode == WASM_OP_BLOCK || opcode == WASM_OP_LOOP) { skip_label(); if (BLOCK_HAS_PARAM(block_type)) { /* Make sure params are in dynamic space */ @@ -5638,8 +5635,10 @@ re_scan: loader_ctx, false, error_buf, error_buf_size)) goto fail; } - (loader_ctx->frame_csp - 1)->code_compiled = - loader_ctx->p_code_compiled; + if (opcode == WASM_OP_LOOP) { + (loader_ctx->frame_csp - 1)->code_compiled = + loader_ctx->p_code_compiled; + } } else if (opcode == WASM_OP_IF) { /* If block has parameters, we should make sure they are in From 47993ec72d17448a46a9e05714f2741bda63a0a8 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Wed, 6 Dec 2023 09:46:50 +0800 Subject: [PATCH 47/67] Don't launch extra cores in QEMU to save resource (#2867) --- tests/wamr-test-suites/spec-test-script/runtest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/wamr-test-suites/spec-test-script/runtest.py b/tests/wamr-test-suites/spec-test-script/runtest.py index 6ca1df0d4..dcc0c3361 100755 --- a/tests/wamr-test-suites/spec-test-script/runtest.py +++ b/tests/wamr-test-suites/spec-test-script/runtest.py @@ -1111,13 +1111,13 @@ def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r): cmd = "qemu-system-aarch64 -cpu cortex-a53 -nographic -machine virt,virtualization=on,gic-version=3 -net none -chardev stdio,id=con,mux=on -serial chardev:con -mon chardev=con,mode=readline -kernel".split() cmd.append(opts.qemu_firmware) elif opts.target.startswith("thumbv7"): - cmd = "qemu-system-arm -semihosting -M sabrelite -m 1024 -smp 4 -nographic -kernel".split() + cmd = "qemu-system-arm -semihosting -M sabrelite -m 1024 -smp 1 -nographic -kernel".split() cmd.append(opts.qemu_firmware) elif opts.target.startswith("riscv32"): - cmd = "qemu-system-riscv32 -semihosting -M virt,aclint=on -cpu rv32 -smp 8 -nographic -bios none -kernel".split() + cmd = "qemu-system-riscv32 -semihosting -M virt,aclint=on -cpu rv32 -smp 1 -nographic -bios none -kernel".split() cmd.append(opts.qemu_firmware) elif opts.target.startswith("riscv64"): - cmd = "qemu-system-riscv64 -semihosting -M virt,aclint=on -cpu rv64 -smp 8 -nographic -bios none -kernel".split() + cmd = "qemu-system-riscv64 -semihosting -M virt,aclint=on -cpu rv64 -smp 1 -nographic -bios none -kernel".split() cmd.append(opts.qemu_firmware) else: raise Exception("Unknwon target for QEMU: %s" % opts.target) From 49cd2e011bee65a77f2acacda50e5633482fa04c Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Wed, 6 Dec 2023 11:30:07 +0800 Subject: [PATCH 48/67] Fix float argument handling for riscv32 ilp32d (#2871) --- core/iwasm/common/wasm_runtime_common.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 4a632543b..567e77bec 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -4023,16 +4023,14 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, if (n_stacks & 1) n_stacks++; if (func_type->types[i] == VALUE_TYPE_F32) { - *(float32 *)&stacks[n_stacks] = *(float32 *)argv_src++; - /* NaN boxing, the upper bits of a valid NaN-boxed - value must be all 1s. */ - stacks[n_stacks + 1] = 0xFFFFFFFF; + *(float32 *)&stacks[n_stacks++] = + *(float32 *)argv_src++; } else { *(float64 *)&stacks[n_stacks] = *(float64 *)argv_src; argv_src += 2; + n_stacks += 2; } - n_stacks += 2; } break; } From 39d0fabda3e6ce26bbe7b92571d200f05beaa92b Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Wed, 6 Dec 2023 16:09:17 +0800 Subject: [PATCH 49/67] Using stable branch for NuttX relative CI (#2872) --- .github/workflows/compilation_on_nuttx.yml | 2 ++ .github/workflows/spec_test_on_nuttx.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/compilation_on_nuttx.yml b/.github/workflows/compilation_on_nuttx.yml index 6125c1f56..77b3da75a 100644 --- a/.github/workflows/compilation_on_nuttx.yml +++ b/.github/workflows/compilation_on_nuttx.yml @@ -108,12 +108,14 @@ jobs: uses: actions/checkout@v3 with: repository: apache/incubator-nuttx + ref: releases/12.3 path: nuttx - name: Checkout NuttX Apps uses: actions/checkout@v3 with: repository: apache/incubator-nuttx-apps + ref: releases/12.3 path: apps - name: Checkout WAMR diff --git a/.github/workflows/spec_test_on_nuttx.yml b/.github/workflows/spec_test_on_nuttx.yml index bba376a82..48fff2d23 100644 --- a/.github/workflows/spec_test_on_nuttx.yml +++ b/.github/workflows/spec_test_on_nuttx.yml @@ -121,12 +121,14 @@ jobs: uses: actions/checkout@v3 with: repository: apache/incubator-nuttx + ref: releases/12.3 path: nuttx - name: Checkout NuttX Apps uses: actions/checkout@v3 with: repository: apache/incubator-nuttx-apps + ref: releases/12.3 path: apps - name: Checkout WAMR From 67a887e2d3a2d52e76ce884d8da6b6107a8c4270 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 6 Dec 2023 16:12:33 +0800 Subject: [PATCH 50/67] Fix compilation warnings on Windows (#2868) --- build-scripts/build_llvm.py | 8 +++++--- core/iwasm/compilation/aot_llvm.c | 3 ++- core/shared/utils/bh_bitmap.h | 14 +++++++------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/build-scripts/build_llvm.py b/build-scripts/build_llvm.py index e48f4e257..a4b92b05c 100755 --- a/build-scripts/build_llvm.py +++ b/build-scripts/build_llvm.py @@ -55,8 +55,6 @@ def build_llvm(llvm_dir, platform, backends, projects, use_clang=False, extra_fl "-DLLVM_APPEND_VC_REV:BOOL=ON", "-DLLVM_BUILD_EXAMPLES:BOOL=OFF", "-DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF", - "-DLLVM_BUILD_TESTS:BOOL=OFF", - "-DLLVM_CCACHE_BUILD:BOOL=ON", "-DLLVM_ENABLE_BINDINGS:BOOL=OFF", "-DLLVM_ENABLE_IDE:BOOL=OFF", "-DLLVM_ENABLE_LIBEDIT=OFF", @@ -68,9 +66,13 @@ def build_llvm(llvm_dir, platform, backends, projects, use_clang=False, extra_fl "-DLLVM_INCLUDE_UTILS:BOOL=OFF", "-DLLVM_INCLUDE_TESTS:BOOL=OFF", "-DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON", - "-DLLVM_USE_PERF:BOOL=ON", ] + # ccache and perf support are not available on Windows + if not "windows" == platform: + LLVM_COMPILE_OPTIONS.append("-DLLVM_CCACHE_BUILD:BOOL=ON") + LLVM_COMPILE_OPTIONS.append("-DLLVM_USE_PERF:BOOL=ON") + # use clang/clang++/lld. but macos doesn't support lld if not sys.platform.startswith("darwin") and use_clang: if shutil.which("clang") and shutil.which("clang++") and shutil.which("lld"): diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index adfc1d0bd..bbf16f55c 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -658,7 +658,8 @@ aot_add_llvm_func(AOTCompContext *comp_ctx, LLVMModuleRef module, const char *key = "frame-pointer"; const char *val = "all"; LLVMAttributeRef no_omit_fp = LLVMCreateStringAttribute( - comp_ctx->context, key, strlen(key), val, strlen(val)); + comp_ctx->context, key, (unsigned)strlen(key), val, + (unsigned)strlen(val)); if (!no_omit_fp) { aot_set_last_error("create LLVM attribute (frame-pointer) failed."); goto fail; diff --git a/core/shared/utils/bh_bitmap.h b/core/shared/utils/bh_bitmap.h index fe7251480..c0e56cb99 100644 --- a/core/shared/utils/bh_bitmap.h +++ b/core/shared/utils/bh_bitmap.h @@ -58,7 +58,7 @@ bh_bitmap_delete(bh_bitmap *bitmap) * @return true if the index is in range, false otherwise */ static inline bool -bh_bitmap_is_in_range(bh_bitmap *bitmap, unsigned n) +bh_bitmap_is_in_range(bh_bitmap *bitmap, uintptr_t n) { return n >= bitmap->begin_index && n < bitmap->end_index; } @@ -72,9 +72,9 @@ bh_bitmap_is_in_range(bh_bitmap *bitmap, unsigned n) * @return value of the bit */ static inline int -bh_bitmap_get_bit(bh_bitmap *bitmap, unsigned n) +bh_bitmap_get_bit(bh_bitmap *bitmap, uintptr_t n) { - unsigned idx = n - bitmap->begin_index; + uintptr_t idx = n - bitmap->begin_index; bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); return (bitmap->map[idx / 8] >> (idx % 8)) & 1; } @@ -86,9 +86,9 @@ bh_bitmap_get_bit(bh_bitmap *bitmap, unsigned n) * @param n the n-th bit to be set */ static inline void -bh_bitmap_set_bit(bh_bitmap *bitmap, unsigned n) +bh_bitmap_set_bit(bh_bitmap *bitmap, uintptr_t n) { - unsigned idx = n - bitmap->begin_index; + uintptr_t idx = n - bitmap->begin_index; bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); bitmap->map[idx / 8] |= 1 << (idx % 8); } @@ -100,9 +100,9 @@ bh_bitmap_set_bit(bh_bitmap *bitmap, unsigned n) * @param n the n-th bit to be cleared */ static inline void -bh_bitmap_clear_bit(bh_bitmap *bitmap, unsigned n) +bh_bitmap_clear_bit(bh_bitmap *bitmap, uintptr_t n) { - unsigned idx = n - bitmap->begin_index; + uintptr_t idx = n - bitmap->begin_index; bh_assert(n >= bitmap->begin_index && n < bitmap->end_index); bitmap->map[idx / 8] &= ~(1 << (idx % 8)); } From ac602bda32618018e101a4f117914e7af2af85c4 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 6 Dec 2023 16:27:21 +0800 Subject: [PATCH 51/67] Update version number to 1.3.0 and update release notes (#2821) And refine the code format for wasm_export.h. --- RELEASE_NOTES.md | 165 +++++++++++++++++++++++++++++++ core/iwasm/include/wasm_export.h | 22 +++-- core/version.h | 4 +- 3 files changed, 181 insertions(+), 10 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 5c7f598da..9f4d76d77 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,168 @@ +## WAMR-1.3.0 + +### Breaking Changes +- Abstract POSIX filesystem functions (#2585) + - Change API wasm_runtime_set_wasi_args_ex's arguments + `int stdinfd/stdoutfd/stderrfd` to `int64_t stdinfd/stdoutfd/stderrfd` +- core/iwasm: Support mapped file system access on non-libuv WASI (#2628) + - Enable mapping host directories to guest directories by parsing + the `map_dir_list` argument in API `wasm_runtime_init_wasi` for libc-wasi +- Support muti-module for AOT mode (#2482) + - Add argument `package_type_t module_type` for module_reader callback +- Generate jitdump to support linux perf for LLVM JIT (#2788) + - Add a field `bool linux_perf_support` in RuntimeInitArgs +- Remove provision of unnecessary fd rights (#2579) +- libc-wasi: Conditionally support SYNC flags (#2581) + +### New Features +- Support muti-module for AOT mode (#2482) +- Implement libc-wasi for Windows platform (#2740) +- Implement module instance context APIs (#2436) +- Implement async termination of blocking thread (#2516) +- Generate jitdump to support linux perf for LLVM JIT (#2788) +- Add Cosmopolitan Libc Platform (#2598) + +### Bug Fixes +- sgx-ra: Disable the building of samples (#2507) +- Handle a return from wasi _start function correctly (#2529) +- fd_object_release: Preserve errno (#2535) +- Fix build error with ancient GCC (4.8) (#2553) +- Fix compiling error for RT-Thread (#2569) +- Fix potential unaligned store issue when extra return value is v128 (#2583) +- Fix loader push_pop_frame_ref_offset (#2590) +- Fix compilation error on Android platform (#2594) +- Ignore handling SIG_DFL/SIG_IGN for previous sig action (#2589) +- Fix nightly run sanitizer error in Fast JIT (#2601) +- Check ValueKind before extracting a constant int value (#2595) +- Patch implementations of vfbinop(min,max,pmin,pax) (#2584) +- Improve stack trace dump and fix coding guideline CI (#2599) +- aot_resolve_stack_sizes: Disable the size check for now (#2608) +- Remove module instance from hashmap in wasi_nn_destroy (#2613) +- Fix label index out-of-range references in op_br_table_cache (#2615) +- Fix compilation of shift opcodes on x86_64 and i386 architectures (#2619) +- Fix potential issue in aot compiler when translating block opcodes (#2622) +- Use another default pipeline when opt-level is 0 (#2624) +- Fix AOT shift operations for indirect constants (#2627) +- Fix fast-interp "pre-compiled label offset out of range" issue (#2659) +- Revert "Strip static and shared libraries of iwasm to reduce the binary size (#2431)" (#2669) +- Fix windows compilation on C++20 (#2670) +- Fix fast-jit f32/f64 truncate to i32/i64 (#2671) +- Fix use getrandom on cosmopolitan libc (#2674) +- Fix repeatedly initialize shared memory data and protect the memory's fields (#2673) +- Minor fixes for Go bindings (#2676) +- Fix issues reported by Coverity (#2681) +- Add more buffer boundary checks in wasm loader (#2734) +- Grab cluster->lock when modifying exec_env->module_inst (#2685) +- Fix CMSIS import with Zephyr 3.4+ (#2744) +- Fix log messages in Zephyr example (#2761) +- Fix fast-jit callnative translation (#2765) +- aot compiler: Disable musttail for thumb (#2771) +- Fix data/elem drop (#2747) +- Fix formatting in aot_dump_perf_profiling (#2796) +- Fix formatting in wasm_dump_perf_profiling (#2799) +- Fix memory.init opcode issue in fast-interp (#2798) +- aot compiler: Fix handle next reachable if block (#2793) +- Fix configurable bounds checks typo (#2809) +- Attestation: Free JSON from the Wasm module heap (#2803) +- Update Zephyr support to v3.5.0 and make instructions generic to boards (#2805) +- Return error when shutdown() fails (#2801) +- iwasm: Print help when meeting unknown cmd options (#2824) +- Fix fast-jit accessing shared memory's fields issue (#2841) +- Fix wasm loader handle op_br_table and op_drop (#2864) +- Fix block with type issue in fast interp (#2866) + +### Enhancements +- Implement strict validation of thread IDs according to the specification (#2521) +- Stop abusing shared memory lock to protect exception (#2509) +- Implement os_usleep for posix (#2517) +- set_exception_visitor: Remove the special case for wasi proc exit (#2525) +- Revert "Return error when exception was raised after main thread finishes" (#2524) +- libc-wasi: Remove unused code (#2528) +- Add callback to handle memory.grow failures (#2522) +- Add context to enlarge memory error callback (#2546) +- Add ARM aeabi symbol for clearing memory content in a specific range (#2531) +- Unifdef -U WASMTIME_SSP_STATIC_CURFDS (#2533) +- Fix typo for IP address buffer (#2532) +- Add an API to terminate instance (#2538) +- Add user to enlarge memory error callback (#2546) +- runtest.py: Show accurate case amount in summary (#2549) +- Allow using custom signal handler from non-main thread (#2551) +- Return __WASI_EINVAL from fd_prestat_dir_name (#2580) +- Support AOT compiler with LLVM 17 (#2567) +- Add support for closing/renumbering preopen fds (#2578) +- Enable AOT usage on M1 mac (#2618) +- core/iwasm: Support mapped file system access on non-libuv WASI (#2628) +- Enable MASM automatically in runtime_lib.cmake (#2634) +- Abstract POSIX filesystem functions (#2585) +- Implement wasi clock_time/clock_res get (#2637) +- Fix several typo/warning/unused-code issues (#2655) +- Partial windows filesystem implementation (#2657) +- Apply no_sanitize_address for clang compiler in several places (#2663) +- Refactor clock functions to use WASI types (#2666) +- Refine lock/unlock shared memory (#2682) +- Fix several AOT compiler issues (#2697) +- Fix AOT compiler simd shift opcodes (#2715) +- Fix invalid use of jit_reg_is_const_val in fast-jit (#2718) +- Use user defined malloc/free functions for user defined memory allocator (#2717) +- Move WASI types into separate header (#2724) +- Provide default vprintf on UWP (#2725) +- Fix typo in Zephyr simple example (#2738) +- Fix switch-case fallthrough compilation warning (#2753) +- Add eabihf ABI support and set vendor-sys of bare-metal targets (#2745) +- Return uint32 from WASI functions (#2749) +- Add compilation flag to enable/disable heap corruption check (#2766) +- Extend os_mmap to support map file from fd (#2763) +- Fix printing ref.extern addresses in wasm_application.c (#2774) +- Remove unused JitBitmap (#2775) +- Use next generation crypto API on Windows (#2769) +- More precise help info of enabled targets for wamrc (#2783) +- Refine atomic operation flags in bh_atomic.h (#2780) +- Fix comment in WAMR_MEM_DUAL_BUS_MIRROR (#2791) +- Fix return type in wasm_loader_get_custom_section (#2794) +- Add support for custom sections in nuttx (#2795) +- Change is_shared_memory type from bool to uint8 (#2800) +- Fix typos in zephyr platform struct descriptions (#2818) +- Access linear memory size atomically (#2834) +- Output warning and quit if import/export name contains '\00' (#2806) +- Use wasm_config_t to pass private configuration to wasm_engine_new (#2837) +- core/iwasm/interpreter/wasm_loader.c: remove an extra validation (#2845) +- Don't add "+d" to riscv cpu features if already given (#2855) + +### Others +- Add mutex stress test (#2472) +- Add unit tests for the tid allocator (#2519) +- Add support for running tests on apple M1 macs (#2554) +- export_native_api.md: Add a note about thread termination (#2572) +- test_wamr.sh: Print a bit more meaningful message (#2574) +- run_wasi_tests.sh: Provide stdin by ourselves (#2576) +- Fix a few issues in "run_wasi_tests.sh: provide stdin by ourselves" (#2582) +- Fix compile error of tsf benchmark (#2588) +- test_wamr.sh: Bump wasi-testsuite version (#2568) +- samples/inst-context-threads: Add a brief explanation (#2592) +- doc/memory_tune.md: "remove malloc" hack is not relevant to wasi-threads (#2603) +- Refactor stress tests to make them runnable in reactor mode (#2614) +- Run rust tests from wasi-testsuite (#2484) +- spec-test-script: Fix NaN comparision between v128 values (#2605) +- CI: Enable testing AOT multi-module feature (#2621) +- Vote for nomination of Woods, Chris and Trenner, Thomas as TSC members (#2638) +- Add tsan for fast interp and aot (#2679) +- Enable WASI tests on Windows CI (#2699) +- docs: Fix typo in export native APIs doc (#2750) +- Update RISC-V compilers in Nuttx compilation CI and spec test CI (#2756) +- Enable more LLVM backends for the release wamrc binary (#2778) +- Disable FPU in NuttX spec test (#2781) +- Fix broken links in app-mgr README.md (#2786) +- Fix build error of libsodium benchmark (#2792) +- Fix wamr-test-suites script for macos (#2819) +- Run spec test for classic/fast-interp in NuttX CI (#2817) +- test_wamr.sh: Don't bother to build shared library (#2844) +- doc/build_wamr.md: Fix links to RISC-V named ABIs (#2852) +- Fix typos of CIDR in docs and help text (#2851) +- Enable spectest on riscv64 (#2843) +- Update FPU configuration in spec_test_on_nuttx.yml (#2856) + +--- + ## WAMR-1.2.3 ### Breaking Changes diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index cb6de7631..d8e761e83 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -118,7 +118,7 @@ typedef union MemAllocOption { void *realloc_func; void *free_func; /* allocator user data, only used when - WASM_MEM_ALLOC_WITH_USER_DATA is defined */ + WASM_MEM_ALLOC_WITH_USER_DATA is defined */ void *user_data; } allocator; } MemAllocOption; @@ -149,7 +149,7 @@ typedef struct RuntimeInitArgs { uint32_t n_native_symbols; /* maximum thread number, only used when - WASM_ENABLE_THREAD_MGR is defined */ + WASM_ENABLE_THREAD_MGR is defined */ uint32_t max_thread_num; /* Debug settings, only used when @@ -320,7 +320,8 @@ wasm_runtime_is_xip_file(const uint8_t *buf, uint32_t size); /** * Callback to load a module file into a buffer in multi-module feature */ -typedef bool (*module_reader)(package_type_t module_type,const char *module_name, +typedef bool (*module_reader)(package_type_t module_type, + const char *module_name, uint8_t **p_buffer, uint32_t *p_size); /** @@ -880,6 +881,7 @@ wasm_application_execute_main(wasm_module_inst_t module_inst, WASM_RUNTIME_API_EXTERN bool wasm_application_execute_func(wasm_module_inst_t module_inst, const char *name, int32_t argc, char *argv[]); + /** * Get exception info of the WASM module instance. * @@ -942,6 +944,7 @@ wasm_runtime_terminate(wasm_module_inst_t module_inst); WASM_RUNTIME_API_EXTERN void wasm_runtime_set_custom_data(wasm_module_inst_t module_inst, void *custom_data); + /** * Get the custom data within a WASM module instance. * @@ -961,6 +964,7 @@ wasm_runtime_get_custom_data(wasm_module_inst_t module_inst); WASM_RUNTIME_API_EXTERN void wasm_runtime_set_bounds_checks(wasm_module_inst_t module_inst, bool enable); + /** * Check if the memory bounds checks flag is enabled for a WASM module instance. * @@ -970,6 +974,7 @@ wasm_runtime_set_bounds_checks(wasm_module_inst_t module_inst, WASM_RUNTIME_API_EXTERN bool wasm_runtime_is_bounds_checks_enabled( wasm_module_inst_t module_inst); + /** * Allocate memory from the heap of WASM module instance * @@ -1344,7 +1349,8 @@ wasm_externref_objdel(wasm_module_inst_t module_inst, void *extern_obj); * * @param module_inst the WASM module instance that the extern object * belongs to - * @param extern_obj the external object to which to set the `extern_obj_cleanup` cleanup callback. + * @param extern_obj the external object to which to set the + * `extern_obj_cleanup` cleanup callback. * @param extern_obj_cleanup a callback to release `extern_obj` * * @return true if success, false otherwise @@ -1552,11 +1558,11 @@ WASM_RUNTIME_API_EXTERN void wasm_runtime_destroy_context_key(void *key); WASM_RUNTIME_API_EXTERN void -wasm_runtime_set_context(wasm_module_inst_t inst, void *key, - void *ctx); +wasm_runtime_set_context(wasm_module_inst_t inst, void *key, void *ctx); + WASM_RUNTIME_API_EXTERN void -wasm_runtime_set_context_spread(wasm_module_inst_t inst, void *key, - void *ctx); +wasm_runtime_set_context_spread(wasm_module_inst_t inst, void *key, void *ctx); + WASM_RUNTIME_API_EXTERN void * wasm_runtime_get_context(wasm_module_inst_t inst, void *key); diff --git a/core/version.h b/core/version.h index 925779b31..9bb76b262 100644 --- a/core/version.h +++ b/core/version.h @@ -6,6 +6,6 @@ #ifndef _WAMR_VERSION_H_ #define _WAMR_VERSION_H_ #define WAMR_VERSION_MAJOR 1 -#define WAMR_VERSION_MINOR 2 -#define WAMR_VERSION_PATCH 3 +#define WAMR_VERSION_MINOR 3 +#define WAMR_VERSION_PATCH 0 #endif From 0b332d89874c780086da785306010764dff2ef0d Mon Sep 17 00:00:00 2001 From: Yage Hu Date: Wed, 6 Dec 2023 17:52:53 -0800 Subject: [PATCH 52/67] Portably handle `fd_advise` on directory fd (#2875) This commit adds a check to `fd_advise`. If the fd is a directory, return `ebadf`. This brings iwasm in line with Wasmtime's behavior. WASI folks have stated that fd_advise should not work on directories as this is a Linux-specific behavior: https://github.com/bytecodealliance/wasmtime/issues/6505#issuecomment-1574122949 --- .../libc-wasi/sandboxed-system-primitives/src/posix.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c index 0f8f4b32b..b59035f34 100644 --- a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c +++ b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c @@ -1155,6 +1155,11 @@ wasmtime_ssp_fd_advise(wasm_exec_env_t exec_env, struct fd_table *curfds, if (error != 0) return error; + if (fo->type == __WASI_FILETYPE_DIRECTORY) { + fd_object_release(exec_env, fo); + return __WASI_EBADF; + } + error = os_fadvise(fo->file_handle, offset, len, advice); fd_object_release(exec_env, fo); From f42ffe1d9db8f2db7fadc81c68fc4253728995d5 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Thu, 7 Dec 2023 10:26:28 +0800 Subject: [PATCH 53/67] Fix sample basic intToStr was called with wrong length (#2876) Reported in https://github.com/bytecodealliance/wasm-micro-runtime/issues/2874. --- RELEASE_NOTES.md | 4 ++++ samples/basic/wasm-apps/testapp.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 9f4d76d77..fbb97f5d7 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -70,6 +70,9 @@ - Fix fast-jit accessing shared memory's fields issue (#2841) - Fix wasm loader handle op_br_table and op_drop (#2864) - Fix block with type issue in fast interp (#2866) +- Fix float argument handling for riscv32 ilp32d (#2871) +- Portably handle fd_advise on directory fd (#2875) +- Fix sample basic intToStr was called with wrong length (#2876) ### Enhancements - Implement strict validation of thread IDs according to the specification (#2521) @@ -127,6 +130,7 @@ - Use wasm_config_t to pass private configuration to wasm_engine_new (#2837) - core/iwasm/interpreter/wasm_loader.c: remove an extra validation (#2845) - Don't add "+d" to riscv cpu features if already given (#2855) +- Fix compilation warnings on Windows (#2868) ### Others - Add mutex stress test (#2472) diff --git a/samples/basic/wasm-apps/testapp.c b/samples/basic/wasm-apps/testapp.c index ea575e20c..8db293071 100644 --- a/samples/basic/wasm-apps/testapp.c +++ b/samples/basic/wasm-apps/testapp.c @@ -58,7 +58,7 @@ float_to_string(float n, char *res, int res_size, int afterpoint) // is needed to handle cases like 233.007 fpart = fpart * get_pow(10, afterpoint); - intToStr((int)fpart, res + i + 1, sizeof(res + i + 1), afterpoint); + intToStr((int)fpart, res + i + 1, res_size - i - 1, afterpoint); } } From 78be22159489c691f9162d4f09d9e5307fbb0851 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Thu, 7 Dec 2023 11:09:36 +0800 Subject: [PATCH 54/67] Fix build llvm on macos (#2877) Perf support is available on Linux only. --- build-scripts/build_llvm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build-scripts/build_llvm.py b/build-scripts/build_llvm.py index a4b92b05c..e5036e5ca 100755 --- a/build-scripts/build_llvm.py +++ b/build-scripts/build_llvm.py @@ -68,9 +68,11 @@ def build_llvm(llvm_dir, platform, backends, projects, use_clang=False, extra_fl "-DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON", ] - # ccache and perf support are not available on Windows + # ccache is not available on Windows if not "windows" == platform: LLVM_COMPILE_OPTIONS.append("-DLLVM_CCACHE_BUILD:BOOL=ON") + # perf support is available on Linux only + if "linux" == platform: LLVM_COMPILE_OPTIONS.append("-DLLVM_USE_PERF:BOOL=ON") # use clang/clang++/lld. but macos doesn't support lld From a58e5969951188ef90b0fc2e852f896878756fd8 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Thu, 7 Dec 2023 18:03:07 +0800 Subject: [PATCH 55/67] compilation_on_nuttx.yml: Use docker image to simplify env setup (#2878) --- .github/workflows/compilation_on_nuttx.yml | 38 +++++----------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/.github/workflows/compilation_on_nuttx.yml b/.github/workflows/compilation_on_nuttx.yml index 77b3da75a..73ed296bd 100644 --- a/.github/workflows/compilation_on_nuttx.yml +++ b/.github/workflows/compilation_on_nuttx.yml @@ -50,7 +50,10 @@ env: jobs: build_iwasm_on_nuttx: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/apache/nuttx/apache-nuttx-ci-linux@sha256:4b4cbf0b70512e61ada9cdcb76b97e90ad478b85e4d0774d05a95fa32caa8c39 + strategy: matrix: nuttx_board_config: [ @@ -60,12 +63,12 @@ jobs: "boards/arm/rp2040/raspberrypi-pico/configs/nsh", # cortex-m7 "boards/arm/stm32h7/nucleo-h743zi/configs/nsh", - # riscv32imac + # riscv32imc + "boards/risc-v/espressif/esp32c3-generic/configs/nsh", + # riscv32gc "boards/risc-v/qemu-rv/rv-virt/configs/nsh", - # riscv64imac - "boards/risc-v/qemu-rv/rv-virt/configs/nsh64", # riscv64gc - "boards/risc-v/k210/maix-bit/configs/nsh", + "boards/risc-v/qemu-rv/rv-virt/configs/nsh64", ] wamr_config_option: [ "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_AOT=y\\nCONFIG_INTERPRETERS_WAMR_FAST=y\\n", @@ -81,29 +84,6 @@ jobs: ] steps: - - name: Install Utilities - run: | - sudo apt install -y kconfig-frontends-nox genromfs - pip3 install pyelftools - pip3 install cxxfilt - - - name: Install ARM Compilers - if: contains(matrix.nuttx_board_config, 'arm') - run: sudo apt install -y gcc-arm-none-eabi - - - name: Install RISC-V Compilers - if: contains(matrix.nuttx_board_config, 'risc-v') - run: | - curl -L https://github.com/xpack-dev-tools/riscv-none-elf-gcc-xpack/releases/download/v12.3.0-1/xpack-riscv-none-elf-gcc-12.3.0-1-linux-x64.tar.gz > riscv.tar.gz - tar xvf riscv.tar.gz - echo "$PWD/xpack-riscv-none-elf-gcc-12.3.0-1/bin" >> $GITHUB_PATH - - - name: Install WASI-SDK - run: | - curl -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz > wasi-sdk.tar.gz - tar xvf wasi-sdk.tar.gz - sudo mv wasi-sdk-* /opt/wasi-sdk - - name: Checkout NuttX uses: actions/checkout@v3 with: @@ -126,7 +106,7 @@ jobs: - name: Enable WAMR for NuttX run: | - find nuttx/boards -name defconfig | xargs sed -i '$a\CONFIG_EOL_IS_LF=y\nCONFIG_PSEUDOFS_SOFTLINKS=y\n${{ matrix.wamr_config_option }}' + find nuttx/boards -name defconfig | xargs sed -i '$a\CONFIG_EOL_IS_LF=y\n${{ matrix.wamr_config_option }}' find nuttx/boards/sim -name defconfig | xargs sed -i '$a\CONFIG_LIBM=y\n' - name: Build From df83aef101a2e267630fa16c79eeb9015b2cbcaf Mon Sep 17 00:00:00 2001 From: Stephen Berard Date: Fri, 8 Dec 2023 10:16:16 +0100 Subject: [PATCH 56/67] Corrects Zephyr include files for current versions of Zephyr (#2881) This fixes bug #2880. Zephyr 3.2 made changes to how headers are reference (see [release notes](https://docs.zephyrproject.org/latest/releases/release-notes-3.2.html)). Work item [49578](https://github.com/zephyrproject-rtos/zephyr/issues/49578) deprecated the old headers names. The current WAMR codebase references these old headers, thus causing compile errors with current versions of Zephyr. This update adds #ifdefs around the header names. With this change, compiling with Zephyr 3.2.0 and above will use the new header files. Prior versions will use the existing code. --- core/app-mgr/app-manager/platform/zephyr/app_mgr_zephyr.c | 6 ++++++ .../gui/wasm-runtime-wgl/src/platform/zephyr/XPT2046.c | 8 ++++++-- .../src/platform/zephyr/display_ili9340.h | 5 +++++ .../gui/wasm-runtime-wgl/src/platform/zephyr/iwasm_main.c | 5 +++++ .../vgl-wasm-runtime/src/platform/zephyr/XPT2046.c | 8 ++++++-- .../src/platform/zephyr/display_ili9340.h | 5 +++++ .../vgl-wasm-runtime/src/platform/zephyr/iwasm_main.c | 5 +++++ 7 files changed, 38 insertions(+), 4 deletions(-) diff --git a/core/app-mgr/app-manager/platform/zephyr/app_mgr_zephyr.c b/core/app-mgr/app-manager/platform/zephyr/app_mgr_zephyr.c index 650e536f1..68147c062 100644 --- a/core/app-mgr/app-manager/platform/zephyr/app_mgr_zephyr.c +++ b/core/app-mgr/app-manager/platform/zephyr/app_mgr_zephyr.c @@ -6,8 +6,14 @@ #include "app_manager.h" #include "bh_platform.h" #include + +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ #include #include +#else +#include +#endif + #if 0 #include #endif diff --git a/samples/gui/wasm-runtime-wgl/src/platform/zephyr/XPT2046.c b/samples/gui/wasm-runtime-wgl/src/platform/zephyr/XPT2046.c index cdad9dac2..5502cfd89 100644 --- a/samples/gui/wasm-runtime-wgl/src/platform/zephyr/XPT2046.c +++ b/samples/gui/wasm-runtime-wgl/src/platform/zephyr/XPT2046.c @@ -10,8 +10,12 @@ #include #include "drivers/spi.h" -#include "zephyr.h" -#include "kernel.h" +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ +#include +#include +#else +#include +#endif #if USE_XPT2046 diff --git a/samples/gui/wasm-runtime-wgl/src/platform/zephyr/display_ili9340.h b/samples/gui/wasm-runtime-wgl/src/platform/zephyr/display_ili9340.h index f72279d69..ddc396e1d 100644 --- a/samples/gui/wasm-runtime-wgl/src/platform/zephyr/display_ili9340.h +++ b/samples/gui/wasm-runtime-wgl/src/platform/zephyr/display_ili9340.h @@ -7,7 +7,12 @@ #define ZEPHYR_DRIVERS_DISPLAY_DISPLAY_ILI9340_H_ #include "board_config.h" #include + +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ #include +#else +#include +#endif #define ILI9340_CMD_ENTER_SLEEP 0x10 #define ILI9340_CMD_EXIT_SLEEP 0x11 diff --git a/samples/gui/wasm-runtime-wgl/src/platform/zephyr/iwasm_main.c b/samples/gui/wasm-runtime-wgl/src/platform/zephyr/iwasm_main.c index 6a88f8007..9ce0c11cd 100644 --- a/samples/gui/wasm-runtime-wgl/src/platform/zephyr/iwasm_main.c +++ b/samples/gui/wasm-runtime-wgl/src/platform/zephyr/iwasm_main.c @@ -31,7 +31,12 @@ xpt2046_init(void); extern void wgl_init(); +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ #include +#else +#include +#endif + #include #include diff --git a/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/XPT2046.c b/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/XPT2046.c index d59b6c9b7..e3948e2f2 100644 --- a/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/XPT2046.c +++ b/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/XPT2046.c @@ -10,8 +10,12 @@ #include #include "drivers/spi.h" -#include "zephyr.h" -#include "kernel.h" +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ +#include +#include +#else +#include +#endif #if USE_XPT2046 diff --git a/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/display_ili9340.h b/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/display_ili9340.h index 4eb11e2d1..39257a29a 100644 --- a/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/display_ili9340.h +++ b/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/display_ili9340.h @@ -7,7 +7,12 @@ #define ZEPHYR_DRIVERS_DISPLAY_DISPLAY_ILI9340_H_ #include "board_config.h" #include + +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ #include +#else +#include +#endif #define ILI9340_CMD_ENTER_SLEEP 0x10 #define ILI9340_CMD_EXIT_SLEEP 0x11 diff --git a/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/iwasm_main.c b/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/iwasm_main.c index dceb5786b..10f498026 100644 --- a/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/iwasm_main.c +++ b/samples/littlevgl/vgl-wasm-runtime/src/platform/zephyr/iwasm_main.c @@ -16,7 +16,12 @@ #include "connection_native_api.h" #include "display_indev.h" +#if KERNEL_VERSION_NUMBER < 0x030200 /* version 3.2.0 */ #include +#else +#include +#endif + #include #include From b008ab4fbaf09e6957013d0d786ad09c1211eda2 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Fri, 8 Dec 2023 17:28:57 +0800 Subject: [PATCH 57/67] Fix possible dead lock in wasm_cluster_spawn_exec_env (#2882) Fix a deadlock issue like: wasm_runtime_spawn_exec_env wasm_cluster_spawn_exec_env (hold cluster->lock) wasm_runtime_instantiate_internal wasm_instantiate execute_post_instantiate_functions wasm_exec_env_set_module_inst (grab the lock again) Refer to #2879. --- .../libraries/thread-mgr/thread_manager.c | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index dbf002468..f8bdf02bc 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -488,10 +488,25 @@ wasm_cluster_spawn_exec_env(WASMExecEnv *exec_env) return NULL; } + if (!(new_module_inst = wasm_runtime_instantiate_internal( + module, module_inst, exec_env, stack_size, 0, NULL, 0))) { + return NULL; + } + + /* Set custom_data to new module instance */ + wasm_runtime_set_custom_data_internal( + new_module_inst, wasm_runtime_get_custom_data(module_inst)); + + wasm_native_inherit_contexts(new_module_inst, module_inst); + + if (!(wasm_cluster_dup_c_api_imports(new_module_inst, module_inst))) { + goto fail1; + } + os_mutex_lock(&cluster->lock); if (cluster->has_exception || cluster->processing) { - goto fail1; + goto fail2; } #if WASM_ENABLE_INTERP != 0 @@ -508,21 +523,11 @@ wasm_cluster_spawn_exec_env(WASMExecEnv *exec_env) } #endif - if (!(new_module_inst = wasm_runtime_instantiate_internal( - module, module_inst, exec_env, stack_size, 0, NULL, 0))) { - goto fail1; - } - - /* Set custom_data to new module instance */ - wasm_runtime_set_custom_data_internal( - new_module_inst, wasm_runtime_get_custom_data(module_inst)); - - wasm_native_inherit_contexts(new_module_inst, module_inst); - new_exec_env = wasm_exec_env_create_internal(new_module_inst, exec_env->wasm_stack_size); - if (!new_exec_env) + if (!new_exec_env) { goto fail2; + } if (!allocate_aux_stack(exec_env, &aux_stack_start, &aux_stack_size)) { LOG_ERROR("thread manager error: " @@ -539,8 +544,9 @@ wasm_cluster_spawn_exec_env(WASMExecEnv *exec_env) /* Inherit suspend_flags of parent thread */ new_exec_env->suspend_flags.flags = exec_env->suspend_flags.flags; - if (!wasm_cluster_add_exec_env(cluster, new_exec_env)) + if (!wasm_cluster_add_exec_env(cluster, new_exec_env)) { goto fail4; + } os_mutex_unlock(&cluster->lock); @@ -552,9 +558,9 @@ fail4: fail3: wasm_exec_env_destroy_internal(new_exec_env); fail2: - wasm_runtime_deinstantiate_internal(new_module_inst, true); -fail1: os_mutex_unlock(&cluster->lock); +fail1: + wasm_runtime_deinstantiate_internal(new_module_inst, true); return NULL; } From 9c3581c179fc0ee8493f9e29d14113812748867f Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Fri, 8 Dec 2023 19:12:37 +0900 Subject: [PATCH 58/67] samples/spawn-thread: Disable libc and pthread (#2883) because this sample doesn't really require them. --- samples/spawn-thread/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/spawn-thread/CMakeLists.txt b/samples/spawn-thread/CMakeLists.txt index 7b76311d7..29c4dc429 100644 --- a/samples/spawn-thread/CMakeLists.txt +++ b/samples/spawn-thread/CMakeLists.txt @@ -45,9 +45,9 @@ endif () set(WAMR_BUILD_INTERP 1) set(WAMR_BUILD_AOT 1) set(WAMR_BUILD_JIT 0) -set(WAMR_BUILD_LIBC_BUILTIN 1) set(WAMR_BUILD_FAST_INTERP 1) -set(WAMR_BUILD_LIB_PTHREAD 1) +set(WAMR_BUILD_THREAD_MGR 1) +set(WAMR_BUILD_SHARED_MEMORY 1) # compiling and linking flags if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) From 2fb3dc3ec0523ffe18c0c05e37ec0853cf94c751 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Fri, 8 Dec 2023 18:21:47 +0800 Subject: [PATCH 59/67] Add arm64 to nuttx compilation test (#2886) --- .github/workflows/compilation_on_nuttx.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/compilation_on_nuttx.yml b/.github/workflows/compilation_on_nuttx.yml index 73ed296bd..8e1f4aad0 100644 --- a/.github/workflows/compilation_on_nuttx.yml +++ b/.github/workflows/compilation_on_nuttx.yml @@ -69,6 +69,8 @@ jobs: "boards/risc-v/qemu-rv/rv-virt/configs/nsh", # riscv64gc "boards/risc-v/qemu-rv/rv-virt/configs/nsh64", + # arm64 + "boards/arm64/qemu/qemu-armv8a/configs/nsh", ] wamr_config_option: [ "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_AOT=y\\nCONFIG_INTERPRETERS_WAMR_FAST=y\\n", From 1dbae404b430d0b055f069a4cf821419875c6898 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Fri, 8 Dec 2023 19:35:40 +0900 Subject: [PATCH 60/67] samples/spawn-thread: Tweak to expose a bug (#2888) this would expose the deadlock bug, which has been fixed by https://github.com/bytecodealliance/wasm-micro-runtime/pull/2882 --- samples/spawn-thread/wasm-apps/CMakeLists.txt | 1 + samples/spawn-thread/wasm-apps/sum.c | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/samples/spawn-thread/wasm-apps/CMakeLists.txt b/samples/spawn-thread/wasm-apps/CMakeLists.txt index 52ee7d752..0996d5841 100644 --- a/samples/spawn-thread/wasm-apps/CMakeLists.txt +++ b/samples/spawn-thread/wasm-apps/CMakeLists.txt @@ -29,6 +29,7 @@ set (DEFINED_SYMBOLS set (CMAKE_EXE_LINKER_FLAGS "-Wl,--shared-memory,--max-memory=131072, \ -Wl,--no-entry,--strip-all,--export=sum, \ + -Wl,--export=return_bss, \ -Wl,--export=__heap_base,--export=__data_end \ -Wl,--export=__wasm_call_ctors \ -Wl,--allow-undefined-file=${DEFINED_SYMBOLS}" diff --git a/samples/spawn-thread/wasm-apps/sum.c b/samples/spawn-thread/wasm-apps/sum.c index 0bcb8918d..6f8b8f218 100644 --- a/samples/spawn-thread/wasm-apps/sum.c +++ b/samples/spawn-thread/wasm-apps/sum.c @@ -3,6 +3,17 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ +/* + * have something in bss so that llvm synthesizes + * wasm start function for this module. + */ +char * +return_bss() +{ + static char bss[4096]; + return bss; +} + int sum(int start, int length) { @@ -13,4 +24,4 @@ sum(int start, int length) } return sum; -} \ No newline at end of file +} From 63696ba6034a110ea5e60fcf8af130734d0a8d25 Mon Sep 17 00:00:00 2001 From: Maks Litskevich Date: Mon, 11 Dec 2023 01:16:30 +0000 Subject: [PATCH 61/67] Fix typo in CI config and suppress STORE_U8 in TSAN (#2802) This typo prevented sanitizers to work in the CI. --- .github/workflows/nightly_run.yml | 2 +- core/iwasm/common/wasm_runtime_common.h | 13 +++++++++++++ core/iwasm/interpreter/wasm_interp_fast.c | 4 ++-- tests/wamr-test-suites/tsan_suppressions.txt | 1 + 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index 8153cea99..1bd0c4856 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -666,7 +666,7 @@ jobs: - name: run tests timeout-minutes: 40 - run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} -T %{{matrix.sanitizer}} + run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} -T "${{ matrix.sanitizer }}" working-directory: ./tests/wamr-test-suites #only install x32 support libraries when to run x86_32 cases diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 2908cafb5..f2baf7bde 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -54,6 +54,12 @@ STORE_U16(void *addr, uint16_t value) { *(uint16_t *)(addr) = (uint16_t)(value); } +static inline void +STORE_U8(void *addr, uint8_t value) +{ + *(uint8 *)addr = value; +} + /* For LOAD opcodes */ #define LOAD_I64(addr) (*(int64 *)(addr)) #define LOAD_F64(addr) (*(float64 *)(addr)) @@ -173,6 +179,13 @@ STORE_U32(void *addr, uint32_t value) } } } + +static inline void +STORE_U8(void *addr, uint8_t value) +{ + *(uint8 *)addr = value; +} + static inline void STORE_U16(void *addr, uint16_t value) { diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index d3cebc334..380a5b61e 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -1692,7 +1692,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, frame_ip += 2; addr_ret = GET_OFFSET(); CHECK_MEMORY_OVERFLOW(1); - frame_lp[addr_ret] = (uint32)(*(uint8 *)maddr); + frame_lp[addr_ret] = (uint32)(*(uint8 *)(maddr)); HANDLE_OP_END(); } @@ -1817,7 +1817,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, addr = GET_OPERAND(uint32, I32, 2); frame_ip += 4; CHECK_MEMORY_OVERFLOW(1); - *(uint8 *)maddr = (uint8)sval; + STORE_U8(maddr, (uint8_t)sval); HANDLE_OP_END(); } diff --git a/tests/wamr-test-suites/tsan_suppressions.txt b/tests/wamr-test-suites/tsan_suppressions.txt index d79eafddc..dd903a605 100644 --- a/tests/wamr-test-suites/tsan_suppressions.txt +++ b/tests/wamr-test-suites/tsan_suppressions.txt @@ -1,6 +1,7 @@ # Proposing to accept this risk for now. It might be wasi-libc related. # https://github.com/bytecodealliance/wasm-micro-runtime/pull/1963#issuecomment-1455342931 race:STORE_U32 +race:STORE_U8 # Suppressing signal-unsafe inside of a signal for AOT mode # see https://github.com/bytecodealliance/wasm-micro-runtime/issues/2248#issuecomment-1630189656 From c6848e45a1cc2fd32b9a0352e9b7c8729f80feb4 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Mon, 11 Dec 2023 09:48:59 +0800 Subject: [PATCH 62/67] Using docker image for nuttx spectest (#2887) And disable running the nuttx spectest CI every time PR merged since it will consume too much runners to slow down the other checks. --- .github/workflows/build_llvm_libraries.yml | 10 ++++- .github/workflows/spec_test_on_nuttx.yml | 52 ++++------------------ 2 files changed, 17 insertions(+), 45 deletions(-) diff --git a/.github/workflows/build_llvm_libraries.yml b/.github/workflows/build_llvm_libraries.yml index 7ef1dd63f..a24e340fa 100644 --- a/.github/workflows/build_llvm_libraries.yml +++ b/.github/workflows/build_llvm_libraries.yml @@ -11,6 +11,9 @@ on: arch: required: true type: string + container_image: + required: false + type: string outputs: cache_key: description: "A cached key of LLVM libraries" @@ -19,6 +22,10 @@ on: jobs: build_llvm_libraries: runs-on: ${{ inputs.os }} + # Using given container image if it is specified. + # Otherwise, it will be ignored by the runner. + container: + image: ${{ inputs.container_image }} outputs: key: ${{ steps.create_lib_cache_key.outputs.key}} @@ -71,8 +78,9 @@ jobs: 0-ccache-${{ inputs.os }} if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && inputs.os == 'ubuntu-22.04' + # Don't install dependencies if the cache is hit or running in docker container - run: sudo apt install -y ccache ninja-build - if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && startsWith(inputs.os, 'ubuntu') + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' && startsWith(inputs.os, 'ubuntu') && inputs.container_image == '' - uses: actions/cache@v3 with: diff --git a/.github/workflows/spec_test_on_nuttx.yml b/.github/workflows/spec_test_on_nuttx.yml index 48fff2d23..957f82f34 100644 --- a/.github/workflows/spec_test_on_nuttx.yml +++ b/.github/workflows/spec_test_on_nuttx.yml @@ -6,9 +6,10 @@ name: spec test on nuttx on: pull_request: types: - - closed - branches: - - main + - opened + - synchronize + paths: + - ".github/workflows/spec_test_on_nuttx.yml" schedule: - cron: '0 0 * * *' @@ -27,10 +28,13 @@ jobs: with: os: "ubuntu-22.04" arch: "ARM RISCV AArch64" + container_image: ghcr.io/apache/nuttx/apache-nuttx-ci-linux@sha256:4b4cbf0b70512e61ada9cdcb76b97e90ad478b85e4d0774d05a95fa32caa8c39 spec_test_on_qemu: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest needs: [build_llvm_libraries] + container: + image: ghcr.io/apache/nuttx/apache-nuttx-ci-linux@sha256:4b4cbf0b70512e61ada9cdcb76b97e90ad478b85e4d0774d05a95fa32caa8c39 strategy: matrix: target_config: [ @@ -91,32 +95,6 @@ jobs: wamr_test_option: { mode: "-t aot -X" } steps: - - name: Install Utilities - run: | - sudo apt install -y kconfig-frontends-nox genromfs - - - name: Install ARM Compilers - if: startsWith(matrix.target_config.config, 'boards/arm') - run: | - sudo apt install -y gcc-arm-none-eabi - wget --quiet https://developer.arm.com/-/media/Files/downloads/gnu/11.2-2022.02/binrel/gcc-arm-11.2-2022.02-x86_64-aarch64-none-elf.tar.xz - xz -d gcc-arm-11.2-2022.02-x86_64-aarch64-none-elf.tar.xz - tar xf gcc-arm-11.2-2022.02-x86_64-aarch64-none-elf.tar - echo "$PWD/gcc-arm-11.2-2022.02-x86_64-aarch64-none-elf/bin" >> $GITHUB_PATH - - - name: Install RISC-V Compilers - if: startsWith(matrix.target_config.config, 'boards/risc-v') - run: | - curl -L https://github.com/xpack-dev-tools/riscv-none-elf-gcc-xpack/releases/download/v12.3.0-1/xpack-riscv-none-elf-gcc-12.3.0-1-linux-x64.tar.gz > riscv.tar.gz - tar xvf riscv.tar.gz - echo "$PWD/xpack-riscv-none-elf-gcc-12.3.0-1/bin" >> $GITHUB_PATH - - - name: Install WASI-SDK - run: | - curl -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz > wasi-sdk.tar.gz - tar xvf wasi-sdk.tar.gz - sudo mv wasi-sdk-* /opt/wasi-sdk - - name: Checkout NuttX uses: actions/checkout@v3 with: @@ -190,20 +168,6 @@ jobs: tools/configure.sh ${{ matrix.target_config.config }} make -j$(nproc) echo "firmware=$PWD/nuttx" >> $GITHUB_OUTPUT - - - name: Install QEMU for ARM - if: startsWith(matrix.target_config.config, 'boards/arm') - run: | - curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v7.1.0-1/xpack-qemu-arm-7.1.0-1-linux-x64.tar.gz > xpack-qemu-arm.tar.gz - tar xvf xpack-qemu-arm.tar.gz - echo $PWD/xpack-qemu-arm-7.1.0-1/bin >> $GITHUB_PATH - - - name: Install QEMU for RISC-V - if: startsWith(matrix.target_config.config, 'boards/risc-v') - run: | - curl -L https://github.com/xpack-dev-tools/qemu-riscv-xpack/releases/download/v7.1.0-1/xpack-qemu-riscv-7.1.0-1-linux-x64.tar.gz > xpack-qemu-riscv.tar.gz - tar xvf xpack-qemu-riscv.tar.gz - echo PATH=$PATH:$PWD/xpack-qemu-riscv-7.1.0-1/bin >> $GITHUB_PATH - name: Test run: | From 6cb2ea49353fd1c1b0acfde387955b644cf0c712 Mon Sep 17 00:00:00 2001 From: Enrico Loparco Date: Mon, 11 Dec 2023 03:10:26 +0100 Subject: [PATCH 63/67] fix command-reactor: Look for _initialize only if _start not found (#2891) A wasm module can be either a command or a reactor, so it can export either `_start` or `_initialize`. Currently, if a command module is run, `iwasm` still looks for `_initialize`, resulting in the warning: `can not find an export 0 named _initialize in the module`. Change to look for `_initialize` only if `_start` not found to resolve the issue. --- core/iwasm/interpreter/wasm_loader.c | 30 ++++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 2a06f42e3..19a8ad8a4 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -3994,19 +3994,23 @@ check_wasi_abi_compatibility(const WASMModule *module, return false; } } - - /* (func (export "_initialize") (...) */ - initialize = wasm_loader_find_export( - module, "", "_initialize", EXPORT_KIND_FUNC, error_buf, error_buf_size); - if (initialize) { - WASMType *func_type = - module->functions[initialize->index - module->import_function_count] - ->func_type; - if (func_type->param_count || func_type->result_count) { - set_error_buf( - error_buf, error_buf_size, - "the signature of builtin _initialize function is wrong"); - return false; + else { + /* (func (export "_initialize") (...) */ + initialize = + wasm_loader_find_export(module, "", "_initialize", EXPORT_KIND_FUNC, + error_buf, error_buf_size); + if (initialize) { + WASMType *func_type = + module + ->functions[initialize->index + - module->import_function_count] + ->func_type; + if (func_type->param_count || func_type->result_count) { + set_error_buf( + error_buf, error_buf_size, + "the signature of builtin _initialize function is wrong"); + return false; + } } } From bc2d8959ddb74d4cc5930fa28b394d8f53db1079 Mon Sep 17 00:00:00 2001 From: Yage Hu Date: Sun, 10 Dec 2023 22:16:58 -0800 Subject: [PATCH 64/67] Handle ambiguous fstflags on fd_filestat_set_times (#2892) It's possible to set both `atim` and `atim_now` in the `fstflags` parameter. Same goes for `mtin` and `mtim_now`. However, it's ambiguous which time should be set in these two cases. This commit checks this and returns `EINVAL`. --- .../libc-wasi/sandboxed-system-primitives/src/posix.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c index b59035f34..9819c92fd 100644 --- a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c +++ b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/posix.c @@ -1891,7 +1891,13 @@ wasmtime_ssp_fd_filestat_set_times(wasm_exec_env_t exec_env, if ((fstflags & ~(__WASI_FILESTAT_SET_ATIM | __WASI_FILESTAT_SET_ATIM_NOW | __WASI_FILESTAT_SET_MTIM | __WASI_FILESTAT_SET_MTIM_NOW)) - != 0) + != 0 + || (fstflags + & (__WASI_FILESTAT_SET_ATIM | __WASI_FILESTAT_SET_ATIM_NOW)) + == (__WASI_FILESTAT_SET_ATIM | __WASI_FILESTAT_SET_ATIM_NOW) + || (fstflags + & (__WASI_FILESTAT_SET_MTIM | __WASI_FILESTAT_SET_MTIM_NOW)) + == (__WASI_FILESTAT_SET_MTIM | __WASI_FILESTAT_SET_MTIM_NOW)) return __WASI_EINVAL; struct fd_object *fo; From 269b695f85968b023dd947b2d75c35fea4dfebc9 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Mon, 11 Dec 2023 14:44:41 +0800 Subject: [PATCH 65/67] Refactor reloc symbols for riscv (#2894) --- core/iwasm/aot/arch/aot_reloc_riscv.c | 124 ++++++++++++++++++-------- 1 file changed, 85 insertions(+), 39 deletions(-) diff --git a/core/iwasm/aot/arch/aot_reloc_riscv.c b/core/iwasm/aot/arch/aot_reloc_riscv.c index 045cab641..31830f780 100644 --- a/core/iwasm/aot/arch/aot_reloc_riscv.c +++ b/core/iwasm/aot/arch/aot_reloc_riscv.c @@ -18,15 +18,45 @@ #define RV_OPCODE_SW 0x23 +#undef NEED_SOFT_FP +#undef NEED_SOFT_DP +#undef NEED_SOFT_I32_MUL +#undef NEED_SOFT_I32_DIV +#undef NEED_SOFT_I64_MUL +#undef NEED_SOFT_I64_DIV + +#ifdef __riscv_flen +#if __riscv_flen == 32 +#define NEED_SOFT_DP +#endif +#else +#define NEED_SOFT_FP +#define NEED_SOFT_DP +#endif + +#ifndef __riscv_mul +#define NEED_SOFT_I32_MUL +#define NEED_SOFT_I64_MUL +#elif __riscv_xlen == 32 +#define NEED_SOFT_I64_MUL +#endif + +#ifndef __riscv_div +#define NEED_SOFT_I32_DIV +#define NEED_SOFT_I64_DIV +#elif __riscv_xlen == 32 +#define NEED_SOFT_I64_DIV +#endif + /* clang-format off */ void __adddf3(); void __addsf3(); -void __divdi3(); -void __divsi3(); void __divdf3(); +void __divdi3(); void __divsf3(); -void __eqsf2(); +void __divsi3(); void __eqdf2(); +void __eqsf2(); void __extendsfdf2(); void __fixdfdi(); void __fixdfsi(); @@ -38,12 +68,12 @@ void __fixunssfdi(); void __fixunssfsi(); void __floatdidf(); void __floatdisf(); -void __floatsisf(); void __floatsidf(); +void __floatsisf(); void __floatundidf(); void __floatundisf(); -void __floatunsisf(); void __floatunsidf(); +void __floatunsisf(); void __gedf2(); void __gesf2(); void __gtdf2(); @@ -59,6 +89,8 @@ void __muldi3(); void __mulsf3(); void __mulsi3(); void __nedf2(); +void __negdf2(); +void __negsf2(); void __nesf2(); void __subdf3(); void __subsf3(); @@ -74,60 +106,74 @@ void __unordsf2(); static SymbolMap target_sym_map[] = { /* clang-format off */ REG_COMMON_SYMBOLS -#ifndef __riscv_flen - REG_SYM(__adddf3), +#ifdef NEED_SOFT_FP REG_SYM(__addsf3), - REG_SYM(__divdf3), REG_SYM(__divsf3), - REG_SYM(__eqdf2), REG_SYM(__eqsf2), - REG_SYM(__extendsfdf2), - REG_SYM(__fixunsdfdi), - REG_SYM(__fixunsdfsi), + REG_SYM(__fixsfdi), REG_SYM(__fixunssfdi), REG_SYM(__fixunssfsi), - REG_SYM(__gedf2), + REG_SYM(__floatsidf), REG_SYM(__gesf2), - REG_SYM(__gtdf2), REG_SYM(__gtsf2), - REG_SYM(__ledf2), REG_SYM(__lesf2), - REG_SYM(__ltdf2), - REG_SYM(__ltsf2), + REG_SYM(__mulsf3), + REG_SYM(__negsf2), + REG_SYM(__nesf2), + REG_SYM(__subsf3), + REG_SYM(__unordsf2), +#elif __riscv_xlen == 32 + /* rv32f, support FP instruction but need soft routines + * to convert float and long long + */ + REG_SYM(__floatundisf), +#endif +#ifdef NEED_SOFT_DP + REG_SYM(__adddf3), + REG_SYM(__divdf3), + REG_SYM(__eqdf2), + REG_SYM(__extendsfdf2), + REG_SYM(__fixdfdi), + REG_SYM(__fixunsdfdi), + REG_SYM(__fixunsdfsi), + REG_SYM(__floatdidf), + REG_SYM(__floatsidf), + REG_SYM(__floatundidf), + REG_SYM(__floatunsidf), + REG_SYM(__gedf2), + REG_SYM(__gtdf2), + REG_SYM(__ledf2), REG_SYM(__muldf3), REG_SYM(__nedf2), - REG_SYM(__nesf2), + REG_SYM(__negdf2), REG_SYM(__subdf3), - REG_SYM(__subsf3), REG_SYM(__truncdfsf2), REG_SYM(__unorddf2), - REG_SYM(__unordsf2), - REG_SYM(__mulsf3), - REG_SYM(__floatundidf), +#elif __riscv_xlen == 32 + /* rv32d, support DP instruction but need soft routines + * to convert double and long long + */ REG_SYM(__fixdfdi), - REG_SYM(__floatsidf), - REG_SYM(__floatunsidf), -#if __riscv_xlen == 32 - REG_SYM(__fixdfsi), - REG_SYM(__fixsfdi), - REG_SYM(__fixsfsi), - REG_SYM(__floatdidf), - REG_SYM(__floatdisf), - REG_SYM(__floatsisf), - REG_SYM(__floatundisf), - REG_SYM(__floatunsisf), + REG_SYM(__floatundidf), +#endif +#ifdef NEED_SOFT_I32_MUL REG_SYM(__mulsi3), #endif -#endif - REG_SYM(__divdi3), +#ifdef NEED_SOFT_I32_DIV REG_SYM(__divsi3), - REG_SYM(__moddi3), REG_SYM(__modsi3), - REG_SYM(__muldi3), - REG_SYM(__udivdi3), REG_SYM(__udivsi3), - REG_SYM(__umoddi3), REG_SYM(__umodsi3), +#endif +#ifdef NEED_SOFT_I64_MUL + REG_SYM(__muldi3), +#endif +#ifdef NEED_SOFT_I64_DIV + REG_SYM(__divdi3), + REG_SYM(__moddi3), + REG_SYM(__udivdi3), + REG_SYM(__umoddi3), +#endif /* clang-format on */ }; From 4aee3cf14e471eff3f0a1e731423ad35243b9991 Mon Sep 17 00:00:00 2001 From: Enrico Loparco Date: Mon, 11 Dec 2023 15:17:31 +0100 Subject: [PATCH 66/67] Avoid memory import failure when wasi-threads is enabled (#2893) According to the specification: ``` When instantiating a module which is expected to run with `wasi-threads`, the WASI host must first allocate shared memories to satisfy the module's imports. ``` Currently, if a test from the spec is executed while having the `multi-module` feature enabled, WAMR fails with `WASM module load failed: unknown import`. That happens because spec tests use memory like this: `(memory (export "memory") (import "foo" "bar") 1 1 shared)` and WAMR tries to find a registered module named `foo`. At the moment, there is no specific module name that can be used to identify that the memory is imported because using WASI threads: https://github.com/WebAssembly/wasi-threads/issues/33, so this PR only avoids treating the submodule dependency not being found as a failure. --- core/iwasm/interpreter/wasm_loader.c | 34 +++++++++++++++++----------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 19a8ad8a4..e5460af7f 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -1169,23 +1169,31 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end, (WASMModuleCommon *)parent_module, sub_module_name, error_buf, error_buf_size); if (!sub_module) { +#if WASM_ENABLE_LIB_WASI_THREADS != 0 + /* Avoid memory import failure when wasi-threads is enabled + and the memory is shared */ + if (!(declare_max_page_count_flag & 2)) + return false; +#else return false; +#endif /* WASM_ENABLE_LIB_WASI_THREADS */ } + else { + linked_memory = wasm_loader_resolve_memory( + sub_module_name, memory_name, declare_init_page_count, + declare_max_page_count, error_buf, error_buf_size); + if (!linked_memory) { + return false; + } - linked_memory = wasm_loader_resolve_memory( - sub_module_name, memory_name, declare_init_page_count, - declare_max_page_count, error_buf, error_buf_size); - if (!linked_memory) { - return false; + /** + * reset with linked memory limit + */ + memory->import_module = sub_module; + memory->import_memory_linked = linked_memory; + declare_init_page_count = linked_memory->init_page_count; + declare_max_page_count = linked_memory->max_page_count; } - - /** - * reset with linked memory limit - */ - memory->import_module = sub_module; - memory->import_memory_linked = linked_memory; - declare_init_page_count = linked_memory->init_page_count; - declare_max_page_count = linked_memory->max_page_count; } #endif From ef0cd221192e1fc2cd2936ffd3f306d6d7d63742 Mon Sep 17 00:00:00 2001 From: Yage Hu Date: Mon, 11 Dec 2023 16:36:59 -0800 Subject: [PATCH 67/67] Fix memory size not updating after growing in interpreter (#2898) This commit fixes linear memory size not updating after growing. This causes `memory.fill` to throw an exception after `memory.grow`. --- core/iwasm/interpreter/wasm_interp_classic.c | 6 +++++- core/iwasm/interpreter/wasm_interp_fast.c | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index e27b16544..d875c23d6 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -2149,7 +2149,11 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 - linear_mem_size = get_linear_mem_size(); +#if WASM_ENABLE_THREAD_MGR == 0 + linear_mem_size = memory->memory_data_size; +#else + linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); +#endif #endif } diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 380a5b61e..374185129 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -1917,7 +1917,11 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ || WASM_ENABLE_BULK_MEMORY != 0 - linear_mem_size = get_linear_mem_size(); +#if WASM_ENABLE_THREAD_MGR == 0 + linear_mem_size = memory->memory_data_size; +#else + linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); +#endif #endif }