From 9c8551cf759102521c72ab5a8baf65a853890e90 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Mon, 1 Apr 2024 11:26:05 +0800 Subject: [PATCH 01/36] Add cmake flag to control aot intrinsics (#3261) Add cmake variable `-DWAMR_BUILD_AOT_INTRINSICS=1/0` to enable/disable the aot intrinsic functions, which are normally used by AOT XIP feature, and can be disabled to reduce the aot runtime binary size. And refactor the code in aot_intrinsics.h/.c. --- build-scripts/config_common.cmake | 16 +++ core/config.h | 8 ++ core/iwasm/aot/aot_intrinsic.c | 164 +++++++++++++++--------------- core/iwasm/aot/aot_intrinsic.h | 2 +- core/iwasm/aot/aot_reloc.h | 6 +- doc/build_wamr.md | 4 + 6 files changed, 116 insertions(+), 84 deletions(-) diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 4569648a1..6c211d213 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -529,3 +529,19 @@ else () # Disable quick aot/jit entries for interp and fast-jit add_definitions (-DWASM_ENABLE_QUICK_AOT_ENTRY=0) endif () +if (WAMR_BUILD_AOT EQUAL 1) + if (NOT DEFINED WAMR_BUILD_AOT_INTRINSICS) + # Enable aot intrinsics by default + set (WAMR_BUILD_AOT_INTRINSICS 1) + endif () + if (WAMR_BUILD_AOT_INTRINSICS EQUAL 1) + add_definitions (-DWASM_ENABLE_AOT_INTRINSICS=1) + message (" AOT intrinsics enabled") + else () + add_definitions (-DWASM_ENABLE_AOT_INTRINSICS=0) + message (" AOT intrinsics disabled") + endif () +else () + # Disable aot intrinsics for interp, fast-jit and llvm-jit + add_definitions (-DWASM_ENABLE_AOT_INTRINSICS=0) +endif () diff --git a/core/config.h b/core/config.h index 616c1f6e7..7cb8ce7ed 100644 --- a/core/config.h +++ b/core/config.h @@ -570,6 +570,14 @@ #define WASM_ENABLE_QUICK_AOT_ENTRY 1 #endif +/* Support AOT intrinsic functions which can be called from the AOT code + when `--disable-llvm-intrinsics` flag or + `--enable-builtin-intrinsics=` is used by wamrc to + generate the AOT file */ +#ifndef WASM_ENABLE_AOT_INTRINSICS +#define WASM_ENABLE_AOT_INTRINSICS 1 +#endif + #ifndef WASM_TABLE_MAX_SIZE #define WASM_TABLE_MAX_SIZE 1024 #endif diff --git a/core/iwasm/aot/aot_intrinsic.c b/core/iwasm/aot/aot_intrinsic.c index 189b43b09..7b455cbb0 100644 --- a/core/iwasm/aot/aot_intrinsic.c +++ b/core/iwasm/aot/aot_intrinsic.c @@ -5,86 +5,6 @@ #include "aot_intrinsic.h" -typedef struct { - const char *llvm_intrinsic; - const char *native_intrinsic; - uint64 flag; -} aot_intrinsic; - -/* clang-format off */ -static const aot_intrinsic g_intrinsic_mapping[] = { - { "llvm.experimental.constrained.fadd.f32", "aot_intrinsic_fadd_f32", AOT_INTRINSIC_FLAG_F32_FADD }, - { "llvm.experimental.constrained.fadd.f64", "aot_intrinsic_fadd_f64", AOT_INTRINSIC_FLAG_F64_FADD }, - { "llvm.experimental.constrained.fsub.f32", "aot_intrinsic_fsub_f32", AOT_INTRINSIC_FLAG_F32_FSUB }, - { "llvm.experimental.constrained.fsub.f64", "aot_intrinsic_fsub_f64", AOT_INTRINSIC_FLAG_F64_FSUB }, - { "llvm.experimental.constrained.fmul.f32", "aot_intrinsic_fmul_f32", AOT_INTRINSIC_FLAG_F32_FMUL }, - { "llvm.experimental.constrained.fmul.f64", "aot_intrinsic_fmul_f64", AOT_INTRINSIC_FLAG_F64_FMUL }, - { "llvm.experimental.constrained.fdiv.f32", "aot_intrinsic_fdiv_f32", AOT_INTRINSIC_FLAG_F32_FDIV }, - { "llvm.experimental.constrained.fdiv.f64", "aot_intrinsic_fdiv_f64", AOT_INTRINSIC_FLAG_F64_FDIV }, - { "llvm.fabs.f32", "aot_intrinsic_fabs_f32", AOT_INTRINSIC_FLAG_F32_FABS }, - { "llvm.fabs.f64", "aot_intrinsic_fabs_f64", AOT_INTRINSIC_FLAG_F64_FABS }, - { "llvm.ceil.f32", "aot_intrinsic_ceil_f32", AOT_INTRINSIC_FLAG_F32_CEIL }, - { "llvm.ceil.f64", "aot_intrinsic_ceil_f64", AOT_INTRINSIC_FLAG_F64_CEIL }, - { "llvm.floor.f32", "aot_intrinsic_floor_f32", AOT_INTRINSIC_FLAG_F32_FLOOR }, - { "llvm.floor.f64", "aot_intrinsic_floor_f64", AOT_INTRINSIC_FLAG_F64_FLOOR }, - { "llvm.trunc.f32", "aot_intrinsic_trunc_f32", AOT_INTRINSIC_FLAG_F32_TRUNC }, - { "llvm.trunc.f64", "aot_intrinsic_trunc_f64", AOT_INTRINSIC_FLAG_F64_TRUNC }, - { "llvm.rint.f32", "aot_intrinsic_rint_f32", AOT_INTRINSIC_FLAG_F32_RINT }, - { "llvm.rint.f64", "aot_intrinsic_rint_f64", AOT_INTRINSIC_FLAG_F64_RINT }, - { "llvm.sqrt.f32", "aot_intrinsic_sqrt_f32", AOT_INTRINSIC_FLAG_F32_SQRT }, - { "llvm.sqrt.f64", "aot_intrinsic_sqrt_f64", AOT_INTRINSIC_FLAG_F64_SQRT }, - { "llvm.copysign.f32", "aot_intrinsic_copysign_f32", AOT_INTRINSIC_FLAG_F32_COPYSIGN }, - { "llvm.copysign.f64", "aot_intrinsic_copysign_f64", AOT_INTRINSIC_FLAG_F64_COPYSIGN }, - { "llvm.minnum.f32", "aot_intrinsic_fmin_f32", AOT_INTRINSIC_FLAG_F32_MIN }, - { "llvm.minnum.f64", "aot_intrinsic_fmin_f64", AOT_INTRINSIC_FLAG_F64_MIN }, - { "llvm.maxnum.f32", "aot_intrinsic_fmax_f32", AOT_INTRINSIC_FLAG_F32_MAX }, - { "llvm.maxnum.f64", "aot_intrinsic_fmax_f64", AOT_INTRINSIC_FLAG_F64_MAX }, - { "llvm.ctlz.i32", "aot_intrinsic_clz_i32", AOT_INTRINSIC_FLAG_I32_CLZ }, - { "llvm.ctlz.i64", "aot_intrinsic_clz_i64", AOT_INTRINSIC_FLAG_I64_CLZ }, - { "llvm.cttz.i32", "aot_intrinsic_ctz_i32", AOT_INTRINSIC_FLAG_I32_CTZ }, - { "llvm.cttz.i64", "aot_intrinsic_ctz_i64", AOT_INTRINSIC_FLAG_I64_CTZ }, - { "llvm.ctpop.i32", "aot_intrinsic_popcnt_i32", AOT_INTRINSIC_FLAG_I32_POPCNT }, - { "llvm.ctpop.i64", "aot_intrinsic_popcnt_i64", AOT_INTRINSIC_FLAG_I64_POPCNT }, - { "f64_convert_i32_s", "aot_intrinsic_i32_to_f64", AOT_INTRINSIC_FLAG_I32_TO_F64 }, - { "f64_convert_i32_u", "aot_intrinsic_u32_to_f64", AOT_INTRINSIC_FLAG_U32_TO_F64 }, - { "f32_convert_i32_s", "aot_intrinsic_i32_to_f32", AOT_INTRINSIC_FLAG_I32_TO_F32 }, - { "f32_convert_i32_u", "aot_intrinsic_u32_to_f32", AOT_INTRINSIC_FLAG_U32_TO_F32 }, - { "f64_convert_i64_s", "aot_intrinsic_i64_to_f64", AOT_INTRINSIC_FLAG_I32_TO_F64 }, - { "f64_convert_i64_u", "aot_intrinsic_u64_to_f64", AOT_INTRINSIC_FLAG_U64_TO_F64 }, - { "f32_convert_i64_s", "aot_intrinsic_i64_to_f32", AOT_INTRINSIC_FLAG_I64_TO_F32 }, - { "f32_convert_i64_u", "aot_intrinsic_u64_to_f32", AOT_INTRINSIC_FLAG_U64_TO_F32 }, - { "i32_trunc_f32_u", "aot_intrinsic_f32_to_u32", AOT_INTRINSIC_FLAG_F32_TO_U32 }, - { "i32_trunc_f32_s", "aot_intrinsic_f32_to_i32", AOT_INTRINSIC_FLAG_F32_TO_I32 }, - { "i32_trunc_f64_u", "aot_intrinsic_f64_to_u32", AOT_INTRINSIC_FLAG_F64_TO_U32 }, - { "i32_trunc_f64_s", "aot_intrinsic_f64_to_i32", AOT_INTRINSIC_FLAG_F64_TO_I32 }, - { "i64_trunc_f64_u", "aot_intrinsic_f64_to_u64", AOT_INTRINSIC_FLAG_F64_TO_U64 }, - { "i64_trunc_f32_s", "aot_intrinsic_f32_to_i64", AOT_INTRINSIC_FLAG_F32_TO_I64 }, - { "i64_trunc_f32_u", "aot_intrinsic_f32_to_u64", AOT_INTRINSIC_FLAG_F32_TO_U64 }, - { "i64_trunc_f64_s", "aot_intrinsic_f64_to_i64", AOT_INTRINSIC_FLAG_F64_TO_I64 }, - { "f32_demote_f64", "aot_intrinsic_f64_to_f32", AOT_INTRINSIC_FLAG_F64_TO_F32 }, - { "f64_promote_f32", "aot_intrinsic_f32_to_f64", AOT_INTRINSIC_FLAG_F32_TO_F64 }, - { "f32_cmp", "aot_intrinsic_f32_cmp", AOT_INTRINSIC_FLAG_F32_CMP }, - { "f64_cmp", "aot_intrinsic_f64_cmp", AOT_INTRINSIC_FLAG_F64_CMP }, - { "i32.const", NULL, AOT_INTRINSIC_FLAG_I32_CONST }, - { "i64.const", NULL, AOT_INTRINSIC_FLAG_I64_CONST }, - { "f32.const", NULL, AOT_INTRINSIC_FLAG_F32_CONST }, - { "f64.const", NULL, AOT_INTRINSIC_FLAG_F64_CONST }, - { "i64.div_s", "aot_intrinsic_i64_div_s", AOT_INTRINSIC_FLAG_I64_DIV_S}, - { "i32.div_s", "aot_intrinsic_i32_div_s", AOT_INTRINSIC_FLAG_I32_DIV_S}, - { "i32.div_u", "aot_intrinsic_i32_div_u", AOT_INTRINSIC_FLAG_I32_DIV_U}, - { "i32.rem_s", "aot_intrinsic_i32_rem_s", AOT_INTRINSIC_FLAG_I32_REM_S}, - { "i32.rem_u", "aot_intrinsic_i32_rem_u", AOT_INTRINSIC_FLAG_I32_REM_U}, - { "i64.div_u", "aot_intrinsic_i64_div_u", AOT_INTRINSIC_FLAG_I64_DIV_U}, - { "i64.rem_s", "aot_intrinsic_i64_rem_s", AOT_INTRINSIC_FLAG_I64_REM_S}, - { "i64.rem_u", "aot_intrinsic_i64_rem_u", AOT_INTRINSIC_FLAG_I64_REM_U}, - { "i64.or", "aot_intrinsic_i64_bit_or", AOT_INTRINSIC_FLAG_I64_BIT_OR}, - { "i64.and", "aot_intrinsic_i64_bit_and", AOT_INTRINSIC_FLAG_I64_BIT_AND}, -}; -/* clang-format on */ - -static const uint32 g_intrinsic_count = - sizeof(g_intrinsic_mapping) / sizeof(aot_intrinsic); - float32 aot_intrinsic_fadd_f32(float32 a, float32 b) { @@ -565,6 +485,88 @@ aot_intrinsic_i64_bit_and(uint64 l, uint64 r) return l & r; } +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 + +typedef struct { + const char *llvm_intrinsic; + const char *native_intrinsic; + uint64 flag; +} aot_intrinsic; + +/* clang-format off */ +static const aot_intrinsic g_intrinsic_mapping[] = { + { "llvm.experimental.constrained.fadd.f32", "aot_intrinsic_fadd_f32", AOT_INTRINSIC_FLAG_F32_FADD }, + { "llvm.experimental.constrained.fadd.f64", "aot_intrinsic_fadd_f64", AOT_INTRINSIC_FLAG_F64_FADD }, + { "llvm.experimental.constrained.fsub.f32", "aot_intrinsic_fsub_f32", AOT_INTRINSIC_FLAG_F32_FSUB }, + { "llvm.experimental.constrained.fsub.f64", "aot_intrinsic_fsub_f64", AOT_INTRINSIC_FLAG_F64_FSUB }, + { "llvm.experimental.constrained.fmul.f32", "aot_intrinsic_fmul_f32", AOT_INTRINSIC_FLAG_F32_FMUL }, + { "llvm.experimental.constrained.fmul.f64", "aot_intrinsic_fmul_f64", AOT_INTRINSIC_FLAG_F64_FMUL }, + { "llvm.experimental.constrained.fdiv.f32", "aot_intrinsic_fdiv_f32", AOT_INTRINSIC_FLAG_F32_FDIV }, + { "llvm.experimental.constrained.fdiv.f64", "aot_intrinsic_fdiv_f64", AOT_INTRINSIC_FLAG_F64_FDIV }, + { "llvm.fabs.f32", "aot_intrinsic_fabs_f32", AOT_INTRINSIC_FLAG_F32_FABS }, + { "llvm.fabs.f64", "aot_intrinsic_fabs_f64", AOT_INTRINSIC_FLAG_F64_FABS }, + { "llvm.ceil.f32", "aot_intrinsic_ceil_f32", AOT_INTRINSIC_FLAG_F32_CEIL }, + { "llvm.ceil.f64", "aot_intrinsic_ceil_f64", AOT_INTRINSIC_FLAG_F64_CEIL }, + { "llvm.floor.f32", "aot_intrinsic_floor_f32", AOT_INTRINSIC_FLAG_F32_FLOOR }, + { "llvm.floor.f64", "aot_intrinsic_floor_f64", AOT_INTRINSIC_FLAG_F64_FLOOR }, + { "llvm.trunc.f32", "aot_intrinsic_trunc_f32", AOT_INTRINSIC_FLAG_F32_TRUNC }, + { "llvm.trunc.f64", "aot_intrinsic_trunc_f64", AOT_INTRINSIC_FLAG_F64_TRUNC }, + { "llvm.rint.f32", "aot_intrinsic_rint_f32", AOT_INTRINSIC_FLAG_F32_RINT }, + { "llvm.rint.f64", "aot_intrinsic_rint_f64", AOT_INTRINSIC_FLAG_F64_RINT }, + { "llvm.sqrt.f32", "aot_intrinsic_sqrt_f32", AOT_INTRINSIC_FLAG_F32_SQRT }, + { "llvm.sqrt.f64", "aot_intrinsic_sqrt_f64", AOT_INTRINSIC_FLAG_F64_SQRT }, + { "llvm.copysign.f32", "aot_intrinsic_copysign_f32", AOT_INTRINSIC_FLAG_F32_COPYSIGN }, + { "llvm.copysign.f64", "aot_intrinsic_copysign_f64", AOT_INTRINSIC_FLAG_F64_COPYSIGN }, + { "llvm.minnum.f32", "aot_intrinsic_fmin_f32", AOT_INTRINSIC_FLAG_F32_MIN }, + { "llvm.minnum.f64", "aot_intrinsic_fmin_f64", AOT_INTRINSIC_FLAG_F64_MIN }, + { "llvm.maxnum.f32", "aot_intrinsic_fmax_f32", AOT_INTRINSIC_FLAG_F32_MAX }, + { "llvm.maxnum.f64", "aot_intrinsic_fmax_f64", AOT_INTRINSIC_FLAG_F64_MAX }, + { "llvm.ctlz.i32", "aot_intrinsic_clz_i32", AOT_INTRINSIC_FLAG_I32_CLZ }, + { "llvm.ctlz.i64", "aot_intrinsic_clz_i64", AOT_INTRINSIC_FLAG_I64_CLZ }, + { "llvm.cttz.i32", "aot_intrinsic_ctz_i32", AOT_INTRINSIC_FLAG_I32_CTZ }, + { "llvm.cttz.i64", "aot_intrinsic_ctz_i64", AOT_INTRINSIC_FLAG_I64_CTZ }, + { "llvm.ctpop.i32", "aot_intrinsic_popcnt_i32", AOT_INTRINSIC_FLAG_I32_POPCNT }, + { "llvm.ctpop.i64", "aot_intrinsic_popcnt_i64", AOT_INTRINSIC_FLAG_I64_POPCNT }, + { "f64_convert_i32_s", "aot_intrinsic_i32_to_f64", AOT_INTRINSIC_FLAG_I32_TO_F64 }, + { "f64_convert_i32_u", "aot_intrinsic_u32_to_f64", AOT_INTRINSIC_FLAG_U32_TO_F64 }, + { "f32_convert_i32_s", "aot_intrinsic_i32_to_f32", AOT_INTRINSIC_FLAG_I32_TO_F32 }, + { "f32_convert_i32_u", "aot_intrinsic_u32_to_f32", AOT_INTRINSIC_FLAG_U32_TO_F32 }, + { "f64_convert_i64_s", "aot_intrinsic_i64_to_f64", AOT_INTRINSIC_FLAG_I32_TO_F64 }, + { "f64_convert_i64_u", "aot_intrinsic_u64_to_f64", AOT_INTRINSIC_FLAG_U64_TO_F64 }, + { "f32_convert_i64_s", "aot_intrinsic_i64_to_f32", AOT_INTRINSIC_FLAG_I64_TO_F32 }, + { "f32_convert_i64_u", "aot_intrinsic_u64_to_f32", AOT_INTRINSIC_FLAG_U64_TO_F32 }, + { "i32_trunc_f32_u", "aot_intrinsic_f32_to_u32", AOT_INTRINSIC_FLAG_F32_TO_U32 }, + { "i32_trunc_f32_s", "aot_intrinsic_f32_to_i32", AOT_INTRINSIC_FLAG_F32_TO_I32 }, + { "i32_trunc_f64_u", "aot_intrinsic_f64_to_u32", AOT_INTRINSIC_FLAG_F64_TO_U32 }, + { "i32_trunc_f64_s", "aot_intrinsic_f64_to_i32", AOT_INTRINSIC_FLAG_F64_TO_I32 }, + { "i64_trunc_f64_u", "aot_intrinsic_f64_to_u64", AOT_INTRINSIC_FLAG_F64_TO_U64 }, + { "i64_trunc_f32_s", "aot_intrinsic_f32_to_i64", AOT_INTRINSIC_FLAG_F32_TO_I64 }, + { "i64_trunc_f32_u", "aot_intrinsic_f32_to_u64", AOT_INTRINSIC_FLAG_F32_TO_U64 }, + { "i64_trunc_f64_s", "aot_intrinsic_f64_to_i64", AOT_INTRINSIC_FLAG_F64_TO_I64 }, + { "f32_demote_f64", "aot_intrinsic_f64_to_f32", AOT_INTRINSIC_FLAG_F64_TO_F32 }, + { "f64_promote_f32", "aot_intrinsic_f32_to_f64", AOT_INTRINSIC_FLAG_F32_TO_F64 }, + { "f32_cmp", "aot_intrinsic_f32_cmp", AOT_INTRINSIC_FLAG_F32_CMP }, + { "f64_cmp", "aot_intrinsic_f64_cmp", AOT_INTRINSIC_FLAG_F64_CMP }, + { "i32.const", NULL, AOT_INTRINSIC_FLAG_I32_CONST }, + { "i64.const", NULL, AOT_INTRINSIC_FLAG_I64_CONST }, + { "f32.const", NULL, AOT_INTRINSIC_FLAG_F32_CONST }, + { "f64.const", NULL, AOT_INTRINSIC_FLAG_F64_CONST }, + { "i64.div_s", "aot_intrinsic_i64_div_s", AOT_INTRINSIC_FLAG_I64_DIV_S}, + { "i32.div_s", "aot_intrinsic_i32_div_s", AOT_INTRINSIC_FLAG_I32_DIV_S}, + { "i32.div_u", "aot_intrinsic_i32_div_u", AOT_INTRINSIC_FLAG_I32_DIV_U}, + { "i32.rem_s", "aot_intrinsic_i32_rem_s", AOT_INTRINSIC_FLAG_I32_REM_S}, + { "i32.rem_u", "aot_intrinsic_i32_rem_u", AOT_INTRINSIC_FLAG_I32_REM_U}, + { "i64.div_u", "aot_intrinsic_i64_div_u", AOT_INTRINSIC_FLAG_I64_DIV_U}, + { "i64.rem_s", "aot_intrinsic_i64_rem_s", AOT_INTRINSIC_FLAG_I64_REM_S}, + { "i64.rem_u", "aot_intrinsic_i64_rem_u", AOT_INTRINSIC_FLAG_I64_REM_U}, + { "i64.or", "aot_intrinsic_i64_bit_or", AOT_INTRINSIC_FLAG_I64_BIT_OR}, + { "i64.and", "aot_intrinsic_i64_bit_and", AOT_INTRINSIC_FLAG_I64_BIT_AND}, +}; +/* clang-format on */ + +static const uint32 g_intrinsic_count = + sizeof(g_intrinsic_mapping) / sizeof(aot_intrinsic); + const char * aot_intrinsic_get_symbol(const char *llvm_intrinsic) { @@ -577,8 +579,6 @@ aot_intrinsic_get_symbol(const char *llvm_intrinsic) return NULL; } -#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 - static void add_intrinsic_capability(AOTCompContext *comp_ctx, uint64 flag) { diff --git a/core/iwasm/aot/aot_intrinsic.h b/core/iwasm/aot/aot_intrinsic.h index 2123058b9..6a456efda 100644 --- a/core/iwasm/aot/aot_intrinsic.h +++ b/core/iwasm/aot/aot_intrinsic.h @@ -287,10 +287,10 @@ aot_intrinsic_i64_bit_or(uint64 l, uint64 r); uint64 aot_intrinsic_i64_bit_and(uint64 l, uint64 r); +#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 const char * aot_intrinsic_get_symbol(const char *llvm_intrinsic); -#if WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0 bool aot_intrinsic_check_capability(const AOTCompContext *comp_ctx, const char *llvm_intrinsic); diff --git a/core/iwasm/aot/aot_reloc.h b/core/iwasm/aot/aot_reloc.h index c250f20a7..293e2fc79 100644 --- a/core/iwasm/aot/aot_reloc.h +++ b/core/iwasm/aot/aot_reloc.h @@ -62,6 +62,7 @@ typedef struct { #define REG_AOT_TRACE_SYM() #endif +#if WASM_ENABLE_AOT_INTRINSICS != 0 #define REG_INTRINSIC_SYM() \ REG_SYM(aot_intrinsic_fabs_f32), \ REG_SYM(aot_intrinsic_fabs_f64), \ @@ -124,7 +125,10 @@ typedef struct { REG_SYM(aot_intrinsic_i32_div_s), \ REG_SYM(aot_intrinsic_i32_div_u), \ REG_SYM(aot_intrinsic_i32_rem_s), \ - REG_SYM(aot_intrinsic_i32_rem_u), \ + REG_SYM(aot_intrinsic_i32_rem_u), +#else +#define REG_INTRINSIC_SYM() +#endif #if WASM_ENABLE_STATIC_PGO != 0 #define REG_LLVM_PGO_SYM() \ diff --git a/doc/build_wamr.md b/doc/build_wamr.md index a9ffca204..0d372e0d7 100644 --- a/doc/build_wamr.md +++ b/doc/build_wamr.md @@ -264,6 +264,10 @@ Currently we only profile the memory consumption of module, module_instance and - **WAMR_BUILD_QUICK_AOT_ENTRY**=1/0, enable registering quick call entries to speedup the aot/jit func call process, default to enable if not set > Note: See [Refine callings to AOT/JIT functions from host native](./perf_tune.md#83-refine-callings-to-aotjit-functions-from-host-native) for more details. +#### **Enable AOT intrinsics** +- **WAMR_BUILD_AOT_INTRINSICS**=1/0, enable the AOT intrinsic functions, default to enable if not set. These functions can be called from the AOT code when `--disable-llvm-intrinsics` flag or `--enable-builtin-intrinsics=` flag is used by wamrc to generate the AOT file. +> Note: See [Tuning the XIP intrinsic functions](./xip.md#tuning-the-xip-intrinsic-functions) for more details. + #### **Configurale memory access boundary check** - **WAMR_CONFIGUABLE_BOUNDS_CHECKS**=1/0, default to disable if not set > Note: If it is enabled, allow to run `iwasm --disable-bounds-checks` to disable the memory access boundary checks for interpreter mode. From ec15b6bbadf72606ef9c9f8817048a6a1f245566 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Mon, 1 Apr 2024 12:39:46 +0800 Subject: [PATCH 02/36] Add lock and ref_count for runtime init (#3263) Some environment may call wasm_runtime_full_init/wasm_runtime_init multiple times without knowing that runtime is initialized or not, it is better to add lock and increase reference count during initialization. ps. https://github.com/bytecodealliance/wasm-micro-runtime/discussions/3253. --- core/iwasm/common/wasm_runtime_common.c | 92 +++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 7 deletions(-) diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index b3ee45091..a3c8f39eb 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -564,8 +564,20 @@ wasm_runtime_exec_env_check(WASMExecEnv *exec_env) && exec_env->wasm_stack.top <= exec_env->wasm_stack.top_boundary; } -bool -wasm_runtime_init() +#if defined(OS_THREAD_MUTEX_INITIALIZER) +/** + * lock for wasm_runtime_init/wasm_runtime_full_init and runtime_ref_count + * Note: if the platform has mutex initializer, we use a global lock to + * lock the operations of runtime init/full_init, otherwise when there are + * operations happening simultaneously in multiple threads, developer + * must create the lock by himself, and use it to lock the operations + */ +static korp_mutex runtime_lock = OS_THREAD_MUTEX_INITIALIZER; +#endif +static int32 runtime_ref_count = 0; + +static bool +wasm_runtime_init_internal() { if (!wasm_runtime_memory_init(Alloc_With_System_Allocator, NULL)) return false; @@ -578,8 +590,32 @@ wasm_runtime_init() return true; } -void -wasm_runtime_destroy() +bool +wasm_runtime_init() +{ + bool ret = true; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&runtime_lock); +#endif + + bh_assert(runtime_ref_count >= 0); + if (runtime_ref_count == 0) { + ret = wasm_runtime_init_internal(); + } + if (ret) { + runtime_ref_count++; + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&runtime_lock); +#endif + + return ret; +} + +static void +wasm_runtime_destroy_internal() { #if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 wasm_externref_map_destroy(); @@ -640,6 +676,24 @@ wasm_runtime_destroy() wasm_runtime_memory_destroy(); } +void +wasm_runtime_destroy() +{ +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&runtime_lock); +#endif + + bh_assert(runtime_ref_count > 0); + runtime_ref_count--; + if (runtime_ref_count == 0) { + wasm_runtime_destroy_internal(); + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&runtime_lock); +#endif +} + RunningMode wasm_runtime_get_default_running_mode(void) { @@ -662,8 +716,8 @@ wasm_runtime_get_gc_heap_size_default(void) } #endif -bool -wasm_runtime_full_init(RuntimeInitArgs *init_args) +static bool +wasm_runtime_full_init_internal(RuntimeInitArgs *init_args) { if (!wasm_runtime_memory_init(init_args->mem_alloc_type, &init_args->mem_alloc_option)) @@ -725,6 +779,30 @@ wasm_runtime_full_init(RuntimeInitArgs *init_args) return true; } +bool +wasm_runtime_full_init(RuntimeInitArgs *init_args) +{ + bool ret = true; + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_lock(&runtime_lock); +#endif + + bh_assert(runtime_ref_count >= 0); + if (runtime_ref_count == 0) { + ret = wasm_runtime_full_init_internal(init_args); + } + if (ret) { + runtime_ref_count++; + } + +#if defined(OS_THREAD_MUTEX_INITIALIZER) + os_mutex_unlock(&runtime_lock); +#endif + + return ret; +} + void wasm_runtime_set_log_level(log_level_t level) { @@ -6625,4 +6703,4 @@ wasm_runtime_get_module_name(wasm_module_t module) #endif return ""; -} \ No newline at end of file +} From b1502258ead6b69150f9da6ddd7025048eacd22d Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Mon, 1 Apr 2024 15:27:15 +0800 Subject: [PATCH 03/36] Fix CI error when install packages for macos-14 (#3270) MacOS CI ran failed with "error: externally-managed-environment" reported when installing dependencies. Add argument "--break-system-packages" to fix it. ps. https://github.com/bytecodealliance/wasm-micro-runtime/actions/runs/8503917001/job/23291537189 https://github.com/bytecodealliance/wasm-micro-runtime/actions/runs/8502961539/job/23289867170 --- .github/workflows/build_llvm_libraries.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_llvm_libraries.yml b/.github/workflows/build_llvm_libraries.yml index 18b90e568..67eaf614d 100644 --- a/.github/workflows/build_llvm_libraries.yml +++ b/.github/workflows/build_llvm_libraries.yml @@ -33,10 +33,16 @@ jobs: - name: checkout uses: actions/checkout@v4 - - name: install dependencies + - name: install dependencies for non macos-14 + if: inputs.os != 'macos-14' run: /usr/bin/env python3 -m pip install -r requirements.txt working-directory: build-scripts + - name: install dependencies for macos-14 + if: inputs.os == 'macos-14' + run: /usr/bin/env python3 -m pip install -r requirements.txt --break-system-packages + working-directory: build-scripts + - name: retrive the last commit ID id: get_last_commit run: echo "last_commit=$(GH_TOKEN=${{ secrets.GITHUB_TOKEN }} /usr/bin/env python3 ./build_llvm.py --llvm-ver)" >> $GITHUB_OUTPUT From 6ee71000f93daeb97ba130671d2273a8dbc5993e Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Mon, 1 Apr 2024 15:40:20 +0800 Subject: [PATCH 04/36] Fix dynamic offset not updated in op_br for block with ret type (#3269) The PR #3259 reverted PR #3192, it fixes #3210 but makes #3170 failed again. The workaround is that we should update `ctx->dynamic_offset` only for opcode br and should not update it for opcode br_if. This PR fixes both issue #3170 and #3210. --- core/iwasm/interpreter/wasm_loader.c | 44 ++++++++++++----------- core/iwasm/interpreter/wasm_mini_loader.c | 36 ++++++++++--------- 2 files changed, 42 insertions(+), 38 deletions(-) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index c440ba586..b602df877 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -8448,11 +8448,11 @@ fail: wasm_loader_emit_ptr(loader_ctx, NULL); \ } while (0) -#define emit_br_info(frame_csp) \ - do { \ - if (!wasm_loader_emit_br_info(loader_ctx, frame_csp, error_buf, \ - error_buf_size)) \ - goto fail; \ +#define emit_br_info(frame_csp, is_br) \ + do { \ + if (!wasm_loader_emit_br_info(loader_ctx, frame_csp, is_br, error_buf, \ + error_buf_size)) \ + goto fail; \ } while (0) #define LAST_OP_OUTPUT_I32() \ @@ -8846,7 +8846,7 @@ apply_label_patch(WASMLoaderContext *ctx, uint8 depth, uint8 patch_type) static bool wasm_loader_emit_br_info(WASMLoaderContext *ctx, BranchBlock *frame_csp, - char *error_buf, uint32 error_buf_size) + bool is_br, char *error_buf, uint32 error_buf_size) { /* br info layout: * a) arity of target block @@ -8907,6 +8907,8 @@ wasm_loader_emit_br_info(WASMLoaderContext *ctx, BranchBlock *frame_csp, /* Part e */ dynamic_offset = frame_csp->dynamic_offset + wasm_get_cell_num(types, arity); + if (is_br) + ctx->dynamic_offset = dynamic_offset; for (i = (int32)arity - 1; i >= 0; i--) { cell = (uint8)wasm_value_type_cell_num(types[i]); dynamic_offset -= cell; @@ -9774,8 +9776,8 @@ check_memory_align_equal(uint8 opcode, uint32 align, char *error_buf, #endif /* end of WASM_ENABLE_SHARED_MEMORY */ static bool -wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, - bool is_br_table, char *error_buf, uint32 error_buf_size) +wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, uint8 opcode, + char *error_buf, uint32 error_buf_size) { BranchBlock *target_block, *cur_block; BlockType *target_block_type; @@ -9872,7 +9874,7 @@ wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, /* Backup stack data since it may be changed in the below push operations, and the stack data may be used when checking other target blocks of opcode br_table */ - if (is_br_table) { + if (opcode == WASM_OP_BR_TABLE) { uint64 total_size; frame_ref_after_popped = loader_ctx->frame_ref; @@ -9936,13 +9938,13 @@ wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, } #if WASM_ENABLE_FAST_INTERP != 0 - emit_br_info(target_block); + emit_br_info(target_block, opcode == WASM_OP_BR); #endif /* Restore the stack data, note that frame_ref_bottom, frame_reftype_map_bottom, frame_offset_bottom may be re-allocated in the above push operations */ - if (is_br_table) { + if (opcode == WASM_OP_BR_TABLE) { uint32 total_size; /* The stack operand num should not be smaller than before @@ -10033,7 +10035,7 @@ wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, } #if WASM_ENABLE_FAST_INTERP != 0 - emit_br_info(target_block); + emit_br_info(target_block, opcode == WASM_OP_BR); #endif ret = true; @@ -10056,14 +10058,14 @@ fail: static BranchBlock * check_branch_block(WASMLoaderContext *loader_ctx, uint8 **p_buf, uint8 *buf_end, - bool is_br_table, char *error_buf, uint32 error_buf_size) + uint8 opcode, char *error_buf, uint32 error_buf_size) { uint8 *p = *p_buf, *p_end = buf_end; BranchBlock *frame_csp_tmp; uint32 depth; read_leb_uint32(p, p_end, depth); - if (!wasm_loader_check_br(loader_ctx, depth, is_br_table, error_buf, + if (!wasm_loader_check_br(loader_ctx, depth, opcode, error_buf, error_buf_size)) { goto fail; } @@ -10980,7 +10982,7 @@ re_scan: /* check the target catching block: LABEL_TYPE_CATCH */ if (!(frame_csp_tmp = - check_branch_block(loader_ctx, &p, p_end, false, + check_branch_block(loader_ctx, &p, p_end, opcode, error_buf, error_buf_size))) goto fail; @@ -11267,7 +11269,7 @@ re_scan: case WASM_OP_BR: { if (!(frame_csp_tmp = - check_branch_block(loader_ctx, &p, p_end, false, + check_branch_block(loader_ctx, &p, p_end, opcode, error_buf, error_buf_size))) goto fail; @@ -11281,7 +11283,7 @@ re_scan: POP_I32(); if (!(frame_csp_tmp = - check_branch_block(loader_ctx, &p, p_end, false, + check_branch_block(loader_ctx, &p, p_end, opcode, error_buf, error_buf_size))) goto fail; @@ -11351,7 +11353,7 @@ re_scan: } if (!(frame_csp_tmp = - check_branch_block(loader_ctx, &p, p_end, true, + check_branch_block(loader_ctx, &p, p_end, opcode, error_buf, error_buf_size))) { goto fail; } @@ -12256,7 +12258,7 @@ re_scan: if (opcode == WASM_OP_BR_ON_NULL) { if (!(frame_csp_tmp = - check_branch_block(loader_ctx, &p, p_end, false, + check_branch_block(loader_ctx, &p, p_end, opcode, error_buf, error_buf_size))) { goto fail; } @@ -12303,7 +12305,7 @@ re_scan: PUSH_REF(type); } if (!(frame_csp_tmp = - check_branch_block(loader_ctx, &p, p_end, false, + check_branch_block(loader_ctx, &p, p_end, opcode, error_buf, error_buf_size))) { goto fail; } @@ -13785,7 +13787,7 @@ re_scan: } PUSH_REF(type_tmp); if (!(frame_csp_tmp = check_branch_block( - loader_ctx, &p, p_end, false, error_buf, + loader_ctx, &p, p_end, opcode, error_buf, error_buf_size))) { goto fail; } diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 7d37006a8..51dbbe003 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -4260,11 +4260,11 @@ wasm_loader_pop_frame_csp(WASMLoaderContext *ctx, char *error_buf, wasm_loader_emit_ptr(loader_ctx, NULL); \ } while (0) -#define emit_br_info(frame_csp) \ - do { \ - if (!wasm_loader_emit_br_info(loader_ctx, frame_csp, error_buf, \ - error_buf_size)) \ - goto fail; \ +#define emit_br_info(frame_csp, is_br) \ + do { \ + if (!wasm_loader_emit_br_info(loader_ctx, frame_csp, is_br, error_buf, \ + error_buf_size)) \ + goto fail; \ } while (0) #define LAST_OP_OUTPUT_I32() \ @@ -4649,7 +4649,7 @@ apply_label_patch(WASMLoaderContext *ctx, uint8 depth, uint8 patch_type) static bool wasm_loader_emit_br_info(WASMLoaderContext *ctx, BranchBlock *frame_csp, - char *error_buf, uint32 error_buf_size) + bool is_br, char *error_buf, uint32 error_buf_size) { /* br info layout: * a) arity of target block @@ -4698,6 +4698,8 @@ wasm_loader_emit_br_info(WASMLoaderContext *ctx, BranchBlock *frame_csp, /* Part e */ dynamic_offset = frame_csp->dynamic_offset + wasm_get_cell_num(types, arity); + if (is_br) + ctx->dynamic_offset = dynamic_offset; for (i = (int32)arity - 1; i >= 0; i--) { cell = (uint8)wasm_value_type_cell_num(types[i]); dynamic_offset -= cell; @@ -5382,8 +5384,8 @@ fail: } while (0) static bool -wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, - bool is_br_table, char *error_buf, uint32 error_buf_size) +wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, uint8 opcode, + char *error_buf, uint32 error_buf_size) { BranchBlock *target_block, *cur_block; BlockType *target_block_type; @@ -5441,7 +5443,7 @@ wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, /* Backup stack data since it may be changed in the below push operations, and the stack data may be used when checking other target blocks of opcode br_table */ - if (is_br_table) { + if (opcode == WASM_OP_BR_TABLE) { uint64 total_size; frame_ref_after_popped = loader_ctx->frame_ref; @@ -5479,13 +5481,13 @@ wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, } #if WASM_ENABLE_FAST_INTERP != 0 - emit_br_info(target_block); + emit_br_info(target_block, opcode == WASM_OP_BR); #endif /* Restore the stack data, note that frame_ref_bottom, frame_reftype_map_bottom, frame_offset_bottom may be re-allocated in the above push operations */ - if (is_br_table) { + if (opcode == WASM_OP_BR_TABLE) { uint32 total_size; /* The stack operand num should not be smaller than before @@ -5529,7 +5531,7 @@ wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, } #if WASM_ENABLE_FAST_INTERP != 0 - emit_br_info(target_block); + emit_br_info(target_block, opcode == WASM_OP_BR); #endif ret = true; @@ -5548,14 +5550,14 @@ fail: static BranchBlock * check_branch_block(WASMLoaderContext *loader_ctx, uint8 **p_buf, uint8 *buf_end, - bool is_br_table, char *error_buf, uint32 error_buf_size) + uint8 opcode, char *error_buf, uint32 error_buf_size) { uint8 *p = *p_buf, *p_end = buf_end; BranchBlock *frame_csp_tmp; uint32 depth; read_leb_uint32(p, p_end, depth); - if (!wasm_loader_check_br(loader_ctx, depth, is_br_table, error_buf, + if (!wasm_loader_check_br(loader_ctx, depth, opcode, error_buf, error_buf_size)) { goto fail; } @@ -6179,7 +6181,7 @@ re_scan: case WASM_OP_BR: { if (!(frame_csp_tmp = - check_branch_block(loader_ctx, &p, p_end, false, + check_branch_block(loader_ctx, &p, p_end, opcode, error_buf, error_buf_size))) goto fail; @@ -6193,7 +6195,7 @@ re_scan: POP_I32(); if (!(frame_csp_tmp = - check_branch_block(loader_ctx, &p, p_end, false, + check_branch_block(loader_ctx, &p, p_end, opcode, error_buf, error_buf_size))) goto fail; @@ -6223,7 +6225,7 @@ re_scan: #endif for (i = 0; i <= count; i++) { if (!(frame_csp_tmp = - check_branch_block(loader_ctx, &p, p_end, true, + check_branch_block(loader_ctx, &p, p_end, opcode, error_buf, error_buf_size))) goto fail; From bad9a2a231d26fce6cee9515a757833157a57dbf Mon Sep 17 00:00:00 2001 From: Xin Xu <31881278+xuxin930@users.noreply.github.com> Date: Tue, 2 Apr 2024 08:53:30 +0800 Subject: [PATCH 05/36] nuttx: Migrate NuttX CMake build for WAMR (#3256) Add NuttX CMake build support for platforms. Signed-off-by: xuxin19 --- .../platform/nuttx/shared_platform.cmake | 9 +- product-mini/platforms/nuttx/CMakeLists.txt | 199 ++++++++++++++++++ 2 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 product-mini/platforms/nuttx/CMakeLists.txt diff --git a/core/shared/platform/nuttx/shared_platform.cmake b/core/shared/platform/nuttx/shared_platform.cmake index ff70cc031..d691068f2 100644 --- a/core/shared/platform/nuttx/shared_platform.cmake +++ b/core/shared/platform/nuttx/shared_platform.cmake @@ -8,6 +8,10 @@ add_definitions(-DBH_PLATFORM_NUTTX) include_directories(${PLATFORM_SHARED_DIR}) include_directories(${PLATFORM_SHARED_DIR}/../include) +include (${CMAKE_CURRENT_LIST_DIR}/../common/posix/platform_api_posix.cmake) + +include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) + file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c) if (WAMR_BUILD_LIBC_WASI EQUAL 1) @@ -16,6 +20,7 @@ if (WAMR_BUILD_LIBC_WASI EQUAL 1) set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE}) endif () -include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake) -set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE} ${PLATFORM_COMMON_MEMORY_SOURCE}) +set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE} ${PLATFORM_COMMON_POSIX_SOURCE} ${UNCOMMON_SHARED_SOURCE}) +# remove posix_memmap.c for NuttX +list(REMOVE_ITEM ${PLATFORM_SHARED_SOURCE} ${PLATFORM_COMMON_POSIX_DIR}/posix_memmap.c) diff --git a/product-mini/platforms/nuttx/CMakeLists.txt b/product-mini/platforms/nuttx/CMakeLists.txt new file mode 100644 index 000000000..f83e79916 --- /dev/null +++ b/product-mini/platforms/nuttx/CMakeLists.txt @@ -0,0 +1,199 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# select platform configuration setting WAMR_BUILD_TARGET +set(WAMR_BUILD_PLATFORM nuttx) + +if(CONFIG_ARCH_ARMV6M) + set(WAMR_BUILD_TARGET THUMBV6M) +elseif(CONFIG_ARCH_ARMV7A) + set(WAMR_BUILD_TARGET THUMBV7) +elseif(CONFIG_ARCH_ARMV7M) + set(WAMR_BUILD_TARGET THUMBV7EM) +elseif(CONFIG_ARCH_ARMV8M) + set(WAMR_BUILD_TARGET THUMBV8M) +elseif(CONFIG_ARCH_X86) + set(WAMR_BUILD_TARGET X86_32) +elseif(CONFIG_ARCH_X86_64) + set(WAMR_BUILD_TARGET X86_64) +elseif(CONFIG_ARCH_XTENSA) + set(WAMR_BUILD_TARGET XTENSA) +elseif(CONFIG_ARCH_RV64GC OR CONFIG_ARCH_RV64) + set(WAMR_BUILD_TARGET RISCV64) +elseif(CONFIG_ARCH_RV32IM OR CONFIG_ARCH_RV32) + set(WAMR_BUILD_TARGET RISCV32) +elseif(CONFIG_ARCH_SIM) + if(CONFIG_SIM_M32 OR CONFIG_HOST_X86) + set(WAMR_BUILD_TARGET X86_32) + elseif(CONFIG_HOST_ARM) + set(WAMR_BUILD_TARGET ARM) + elseif(CONFIG_HOST_ARM64) + set(WAMR_BUILD_TARGET AARCH64) + else() + set(WAMR_BUILD_TARGET X86_64) + endif() + if(CONFIG_HOST_MACOS) + add_definitions(-DBH_PLATFORM_DARWIN) + endif() +endif() + +if(CONFIG_INTERPRETERS_WAMR_LOG) + add_definitions(-DWASM_ENABLE_LOG=1) +else() + add_definitions(-DWASM_ENABLE_LOG=0) +endif() + +if(CONFIG_INTERPRETERS_WAMR_AOT_WORD_ALIGN_READ) + add_definitions(-DWASM_ENABLE_WORD_ALIGN_READ=1) +else() + add_definitions(-DWASM_ENABLE_WORD_ALIGN_READ=0) +endif() + +if(CONFIG_INTERPRETERS_WAMR_STACK_GUARD_SIZE) + add_definitions(-DWASM_STACK_GUARD_SIZE=0) +else() + add_definitions( + -DWASM_STACK_GUARD_SIZE=${CONFIG_INTERPRETERS_WAMR_STACK_GUARD_SIZE}) +endif() + +if(CONFIG_INTERPRETERS_WAMR_MEMORY_TRACING) + add_definitions(-DWASM_ENABLE_MEMORY_TRACING=1) +else() + add_definitions(-DWASM_ENABLE_MEMORY_TRACING=0) +endif() + +if(CONFIG_INTERPRETERS_WAMR_SHARED_MEMORY) + set(WAMR_BUILD_SHARED_MEMORY 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_BULK_MEMORY) + set(WAMR_BUILD_BULK_MEMORY 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_AOT_STACK_FRAME) + set(WAMR_BUILD_AOT_STACK_FRAME 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_PERF_PROFILING) + set(WAMR_BUILD_PERF_PROFILING 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_GC) + set(WAMR_BUILD_GC 1) + set(WAMR_BUILD_STRINGREF 1) + set(WAMR_BUILD_REF_TYPES 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_GC_MANUALLY) + add_definitions(-DWASM_GC_MANUALLY=1) +else() + add_definitions(-DWASM_GC_MANUALLY=0) +endif() + +if(CONFIG_INTERPRETERS_WAMR_GC_PERF_PROFILING) + set(WAMR_BUILD_GC_PERF_PROFILING 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_ENABLE_EXCE_HANDLING) + set(WAMR_BUILD_EXCE_HANDLING 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_TAIL_CALL) + set(WAMR_BUILD_TAIL_CALL 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_MEMORY_PROFILING) + set(WAMR_BUILD_MEMORY_PROFILING 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_MULTI_MODULE) + set(WAMR_BUILD_MULTI_MODULE 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_LIB_PTHREAD_SEMAPHORE) + set(WAMR_BUILD_LIB_PTHREAD_SEMAPHORE 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_DISABLE_HW_BOUND_CHECK) + set(WAMR_DISABLE_HW_BOUND_CHECK 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_CUSTOM_NAME_SECTIONS) + set(WAMR_BUILD_CUSTOM_NAME_SECTION 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_GLOBAL_HEAP_POOL) + set(WAMR_BUILD_GLOBAL_HEAP_POOL 1) + math(EXPR _HEAP_SIZE_ + "${CONFIG_INTERPRETERS_WAMR_GLOBAL_HEAP_POOL_SIZE} * 1024") + set(WAMR_BUILD_GLOBAL_HEAP_SIZE ${_HEAP_SIZE_}) +endif() + +if(CONFIG_INTERPRETERS_WAMR_ENABLE_SPEC_TEST) + set(WAMR_BUILD_SPEC_TEST 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_REF_TYPES) + set(WAMR_BUILD_REF_TYPES 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_CLASSIC) + # include iwasm_interp.cmake + set(WAMR_BUILD_INTERP 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_FAST) + # enable iwasm_interp.cmake + set(WAMR_BUILD_FAST_INTERP 1) +endif() + +if((CONFIG_INTERPRETERS_WAMR_FAST OR CONFIG_INTERPRETERS_WAMR_CLASSIC) + AND CONFIG_INTERPRETERS_WAMR_MINILOADER) + # enable iwasm_interp.cmake + set(WAMR_BUILD_MINI_LOADER 1) +else() + set(WAMR_BUILD_MINI_LOADER 0) +endif() + +if(CONFIG_INTERPRETERS_WAMR_AOT) + # include iwasm_aot.cmake + set(WAMR_BUILD_AOT 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_DEBUG_INTERP) + # include debug_engine.cmake + set(WAMR_BUILD_DEBUG_INTERP 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN) + # include libc_builtin.cmake + set(WAMR_BUILD_LIBC_BUILTIN 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_LIBC_WASI) + # include libc_wasi.cmake + set(WAMR_BUILD_LIBC_WASI 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_THREAD_MGR) + # include thread_mgr.cmake + set(WAMR_BUILD_THREAD_MGR 1) +endif() + +if(CONFIG_INTERPRETERS_WAMR_LIB_PTHREAD) + # include lib_pthread.cmake + set(WAMR_BUILD_LIB_PTHREAD 1) +endif() + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../..) + +# enable WAMR build system +include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) + +# NuttX wamr lib complie required: `WAMR_SOURCES` `WAMR_CFLAGS` `WAMR_INCDIRS` +# `WAMR_DEFINITIONS` +set(WAMR_SOURCES ${WAMR_RUNTIME_LIB_SOURCE}) +set(WAMR_CFLAGS -Wno-strict-prototypes -Wno-shadow -Wno-unused-variable + -Wno-int-conversion -Wno-implicit-function-declaration) +get_directory_property(WAMR_INCDIRS INCLUDE_DIRECTORIES) +get_directory_property(WAMR_DEFINITIONS COMPILE_DEFINITIONS) From 250829c0cce2fdd5690b33803329354621959631 Mon Sep 17 00:00:00 2001 From: dongsheng28849455 <68947925+dongsheng28849455@users.noreply.github.com> Date: Tue, 2 Apr 2024 14:17:30 +0800 Subject: [PATCH 06/36] Set is_vram_word_align as false in aot_const_str_set_insert function (#3271) Set `is_vram_word_align` as false in aot_const_str_set_insert function when `const char *name` is not from vram required word_align reading. --- core/iwasm/aot/aot_runtime.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 1489b6260..aeecc8606 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -4722,6 +4722,9 @@ aot_set_module_name(AOTModule *module, const char *name, char *error_buf, module->name = aot_const_str_set_insert((const uint8 *)name, strlen(name) + 1, module, +#if (WASM_ENABLE_WORD_ALIGN_READ != 0) + false, +#endif error_buf, error_buf_size); return module->name != NULL; } From 6b0b5de1c50656a5ce8504a31ff8d4ff6f05770f Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 2 Apr 2024 15:30:08 +0900 Subject: [PATCH 07/36] aot debug: Fix a few NULL dereferences on errors (#3273) --- core/iwasm/compilation/aot_compiler.c | 4 +++- core/iwasm/compilation/aot_emit_control.c | 8 ++++++-- core/iwasm/compilation/debug/dwarf_extractor.cpp | 2 ++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index 81ad9b7a3..9740cd0d1 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -966,7 +966,9 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) location = dwarf_gen_location( comp_ctx, func_ctx, (frame_ip - 1) - comp_ctx->comp_data->wasm_module->buf_code); - LLVMSetCurrentDebugLocation2(comp_ctx->builder, location); + if (location != NULL) { + LLVMSetCurrentDebugLocation2(comp_ctx->builder, location); + } #endif switch (opcode) { diff --git a/core/iwasm/compilation/aot_emit_control.c b/core/iwasm/compilation/aot_emit_control.c index 8b24bcab8..4e28babc3 100644 --- a/core/iwasm/compilation/aot_emit_control.c +++ b/core/iwasm/compilation/aot_emit_control.c @@ -374,7 +374,9 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } #if WASM_ENABLE_DEBUG_AOT != 0 - LLVMInstructionSetDebugLoc(ret, return_location); + if (return_location != NULL) { + LLVMInstructionSetDebugLoc(ret, return_location); + } #endif } else { @@ -383,7 +385,9 @@ handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } #if WASM_ENABLE_DEBUG_AOT != 0 - LLVMInstructionSetDebugLoc(ret, return_location); + if (return_location != NULL) { + LLVMInstructionSetDebugLoc(ret, return_location); + } #endif } } diff --git a/core/iwasm/compilation/debug/dwarf_extractor.cpp b/core/iwasm/compilation/debug/dwarf_extractor.cpp index 06618ad70..da33fc432 100644 --- a/core/iwasm/compilation/debug/dwarf_extractor.cpp +++ b/core/iwasm/compilation/debug/dwarf_extractor.cpp @@ -491,6 +491,8 @@ dwarf_gen_location(const AOTCompContext *comp_ctx, dwarf_extractor *extractor; AOTFunc *func = func_ctx->aot_func; + if (func_ctx->debug_func == NULL) + return NULL; if (!(extractor = TO_EXTACTOR(comp_ctx->comp_data->extractor))) return NULL; From a23fa9f86c36e0fa2795fa0a950c95c2263d560e Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Tue, 2 Apr 2024 15:22:07 +0800 Subject: [PATCH 08/36] Implement memory64 for classic interpreter (#3266) Adding a new cmake flag (cache variable) `WAMR_BUILD_MEMORY64` to enable the memory64 feature, it can only be enabled on the 64-bit platform/target and can only use software boundary check. And when it is enabled, it can support both i32 and i64 linear memory types. The main modifications are: - wasm loader & mini-loader: loading and bytecode validating process - wasm runtime: memory instantiating process - classic-interpreter: wasm code executing process - Support memory64 memory in related runtime APIs - Modify main function type check when it's memory64 wasm file - Modify `wasm_runtime_invoke_native` and `wasm_runtime_invoke_native_raw` to handle registered native function pointer argument when memory64 is enabled - memory64 classic-interpreter spec test in `test_wamr.sh` and in CI Currently, it supports memory64 memory wasm file that uses core spec (including bulk memory proposal) opcodes and threads opcodes. ps. https://github.com/bytecodealliance/wasm-micro-runtime/issues/3091 https://github.com/bytecodealliance/wasm-micro-runtime/pull/3240 https://github.com/bytecodealliance/wasm-micro-runtime/pull/3260 --- .../compilation_on_android_ubuntu.yml | 41 +- .github/workflows/nightly_run.yml | 22 + build-scripts/config_common.cmake | 9 + core/config.h | 9 +- core/iwasm/aot/aot_runtime.c | 7 +- core/iwasm/common/wasm_application.c | 33 +- core/iwasm/common/wasm_memory.c | 51 ++- core/iwasm/common/wasm_memory.h | 5 +- core/iwasm/common/wasm_runtime_common.c | 103 ++++- core/iwasm/common/wasm_runtime_common.h | 4 +- core/iwasm/interpreter/wasm.h | 17 + core/iwasm/interpreter/wasm_interp_classic.c | 391 ++++++++++++------ core/iwasm/interpreter/wasm_loader.c | 302 ++++++++++---- core/iwasm/interpreter/wasm_mini_loader.c | 232 ++++++++--- core/iwasm/interpreter/wasm_runtime.c | 85 +++- core/iwasm/interpreter/wasm_runtime.h | 2 +- .../libc-builtin/libc_builtin_wrapper.c | 2 +- .../platform/common/posix/posix_memmap.c | 4 +- .../wamr-test-suites/spec-test-script/all.py | 25 +- .../spec-test-script/memory64.patch | 28 ++ .../spec-test-script/runtest.py | 8 +- tests/wamr-test-suites/test_wamr.sh | 46 ++- 22 files changed, 1084 insertions(+), 342 deletions(-) create mode 100644 tests/wamr-test-suites/spec-test-script/memory64.patch diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 481b06676..f1e437774 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -65,6 +65,7 @@ env: WASI_TEST_OPTIONS: "-s wasi_certification -w" WAMR_COMPILER_TEST_OPTIONS: "-s wamr_compiler -S -b -P" GC_TEST_OPTIONS: "-s spec -G -b -P" + MEMORY64_TEST_OPTIONS: "-s spec -W -b -P" jobs: build_llvm_libraries_on_ubuntu_2204: @@ -144,6 +145,7 @@ jobs: "-DWAMR_BUILD_SIMD=1", "-DWAMR_BUILD_TAIL_CALL=1", "-DWAMR_DISABLE_HW_BOUND_CHECK=1", + "-DWAMR_BUILD_MEMORY64=1", ] os: [ubuntu-22.04] platform: [android, linux] @@ -202,6 +204,21 @@ jobs: make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + # Memory64 only on CLASSIC INTERP mode, and only on 64-bit platform + - make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + platform: android + - make_options_run_mode: $AOT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" # Fast-JIT and Multi-Tier-JIT mode don't support android - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS platform: android @@ -503,6 +520,7 @@ jobs: $THREADS_TEST_OPTIONS, $WASI_TEST_OPTIONS, $GC_TEST_OPTIONS, + $MEMORY64_TEST_OPTIONS, ] wasi_sdk_release: [ @@ -541,19 +559,30 @@ jobs: test_option: $GC_TEST_OPTIONS - running_mode: "multi-tier-jit" test_option: $GC_TEST_OPTIONS + # aot, fast-interp, fast-jit, llvm-jit, multi-tier-jit don't support Memory64 + - running_mode: "aot" + test_option: $MEMORY64_TEST_OPTIONS + - running_mode: "fast-interp" + test_option: $MEMORY64_TEST_OPTIONS + - running_mode: "fast-jit" + test_option: $MEMORY64_TEST_OPTIONS + - running_mode: "jit" + test_option: $MEMORY64_TEST_OPTIONS + - running_mode: "multi-tier-jit" + test_option: $MEMORY64_TEST_OPTIONS steps: - name: checkout uses: actions/checkout@v4 - name: Set-up OCaml uses: ocaml/setup-ocaml@v2 - if: matrix.test_option == '$GC_TEST_OPTIONS' + if: matrix.test_option == '$GC_TEST_OPTIONS' || matrix.test_option == '$MEMORY64_TEST_OPTIONS' with: ocaml-compiler: 4.13 - name: Set-up Ocamlbuild - if: matrix.test_option == '$GC_TEST_OPTIONS' - run: opam install ocamlbuild dune + if: matrix.test_option == '$GC_TEST_OPTIONS' || matrix.test_option == '$MEMORY64_TEST_OPTIONS' + run: opam install ocamlbuild dune menhir - name: download and install wasi-sdk if: matrix.test_option == '$WASI_TEST_OPTIONS' @@ -617,13 +646,13 @@ jobs: - name: run tests timeout-minutes: 30 - if: matrix.test_option != '$GC_TEST_OPTIONS' + if: matrix.test_option != '$GC_TEST_OPTIONS' && matrix.test_option != '$MEMORY64_TEST_OPTIONS' run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} working-directory: ./tests/wamr-test-suites - - name: run gc tests + - name: run gc or memory64 tests timeout-minutes: 20 - if: matrix.test_option == '$GC_TEST_OPTIONS' + if: matrix.test_option == '$GC_TEST_OPTIONS' || matrix.test_option == '$MEMORY64_TEST_OPTIONS' run: | eval $(opam env) ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index 9d13e41e2..341194df8 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -130,6 +130,7 @@ jobs: "-DWAMR_BUILD_SIMD=1", "-DWAMR_BUILD_TAIL_CALL=1", "-DWAMR_DISABLE_HW_BOUND_CHECK=1", + "-DWAMR_BUILD_MEMORY64=1", ] os: [ubuntu-20.04] platform: [android, linux] @@ -188,6 +189,21 @@ jobs: make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + # Memory64 only on CLASSIC INTERP mode, and only on 64-bit platform + - make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + platform: android + - make_options_run_mode: $AOT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" # Fast-JIT and Multi-Tier-JIT mode don't support android - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS platform: android @@ -271,6 +287,7 @@ jobs: "-DWAMR_BUILD_SIMD=1", "-DWAMR_BUILD_TAIL_CALL=1", "-DWAMR_DISABLE_HW_BOUND_CHECK=1", + "-DWAMR_BUILD_MEMORY64=1", ] exclude: # uncompatiable feature and platform @@ -299,6 +316,11 @@ jobs: # MINI_LOADER only on INTERP mode - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" + # Memory64 only on CLASSIC INTERP mode + - make_options_run_mode: $FAST_INTERP_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" + - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS + make_options_feature: "-DWAMR_BUILD_MEMORY64=1" steps: - name: checkout uses: actions/checkout@v3 diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 6c211d213..8422b060b 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -248,6 +248,15 @@ if (WAMR_BUILD_SHARED_MEMORY EQUAL 1) else () add_definitions (-DWASM_ENABLE_SHARED_MEMORY=0) endif () +if (WAMR_BUILD_MEMORY64 EQUAL 1) + # if native is 32-bit or cross-compiled to 32-bit + if (NOT WAMR_BUILD_TARGET MATCHES ".*64.*") + message (FATAL_ERROR "-- Memory64 is only available on the 64-bit platform/target") + endif() + add_definitions (-DWASM_ENABLE_MEMORY64=1) + set (WAMR_DISABLE_HW_BOUND_CHECK 1) + message (" Memory64 memory enabled") +endif () if (WAMR_BUILD_THREAD_MGR EQUAL 1) message (" Thread manager enabled") endif () diff --git a/core/config.h b/core/config.h index 7cb8ce7ed..d84ed3f36 100644 --- a/core/config.h +++ b/core/config.h @@ -415,7 +415,7 @@ #else #define DEFAULT_WASM_STACK_SIZE (12 * 1024) #endif -/* Min auxilliary stack size of each wasm thread */ +/* Min auxiliary stack size of each wasm thread */ #define WASM_THREAD_AUX_STACK_SIZE_MIN (256) /* Default/min native stack size of each app thread */ @@ -564,7 +564,7 @@ #endif /* Support registering quick AOT/JIT function entries of some func types - to speedup the calling process of invoking the AOT/JIT functions of + to speed up the calling process of invoking the AOT/JIT functions of these types from the host embedder */ #ifndef WASM_ENABLE_QUICK_AOT_ENTRY #define WASM_ENABLE_QUICK_AOT_ENTRY 1 @@ -578,6 +578,11 @@ #define WASM_ENABLE_AOT_INTRINSICS 1 #endif +/* Disable memory64 by default */ +#ifndef WASM_ENABLE_MEMORY64 +#define WASM_ENABLE_MEMORY64 0 +#endif + #ifndef WASM_TABLE_MAX_SIZE #define WASM_TABLE_MAX_SIZE 1024 #endif diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index aeecc8606..d10db89af 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -914,9 +914,10 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, bh_assert(max_memory_data_size <= MAX_LINEAR_MEMORY_SIZE); (void)max_memory_data_size; - if (wasm_allocate_linear_memory(&p, is_shared_memory, num_bytes_per_page, - init_page_count, max_page_count, - &memory_data_size) + /* TODO: memory64 uses is_memory64 flag */ + if (wasm_allocate_linear_memory(&p, is_shared_memory, false, + num_bytes_per_page, init_page_count, + max_page_count, &memory_data_size) != BHT_OK) { set_error_buf(error_buf, error_buf_size, "allocate linear memory failed"); diff --git a/core/iwasm/common/wasm_application.c b/core/iwasm/common/wasm_application.c index 7aa4a563a..13ad2b1a6 100644 --- a/core/iwasm/common/wasm_application.c +++ b/core/iwasm/common/wasm_application.c @@ -61,7 +61,7 @@ static union { * Implementation of wasm_application_execute_main() */ static bool -check_main_func_type(const WASMFuncType *type) +check_main_func_type(const WASMFuncType *type, bool is_memory64) { if (!(type->param_count == 0 || type->param_count == 2) || type->result_count > 1) { @@ -72,7 +72,8 @@ check_main_func_type(const WASMFuncType *type) if (type->param_count == 2 && !(type->types[0] == VALUE_TYPE_I32 - && type->types[1] == VALUE_TYPE_I32)) { + && type->types[1] + == (is_memory64 ? VALUE_TYPE_I64 : VALUE_TYPE_I32))) { LOG_ERROR( "WASM execute application failed: invalid main function type.\n"); return false; @@ -94,14 +95,18 @@ execute_main(WASMModuleInstanceCommon *module_inst, int32 argc, char *argv[]) WASMFunctionInstanceCommon *func; WASMFuncType *func_type = NULL; WASMExecEnv *exec_env = NULL; - uint32 argc1 = 0, argv1[2] = { 0 }; + uint32 argc1 = 0, argv1[3] = { 0 }; uint32 total_argv_size = 0; uint64 total_size; uint64 argv_buf_offset = 0; int32 i; char *argv_buf, *p, *p_end; uint32 *argv_offsets, module_type; - bool ret, is_import_func = true; + bool ret, is_import_func = true, is_memory64 = false; +#if WASM_ENABLE_MEMORY64 != 0 + WASMModuleInstance *wasm_module_inst = (WASMModuleInstance *)module_inst; + is_memory64 = wasm_module_inst->memories[0]->is_memory64; +#endif exec_env = wasm_runtime_get_exec_env_singleton(module_inst); if (!exec_env) { @@ -187,7 +192,7 @@ execute_main(WASMModuleInstanceCommon *module_inst, int32 argc, char *argv[]) return false; } - if (!check_main_func_type(func_type)) { + if (!check_main_func_type(func_type, is_memory64)) { wasm_runtime_set_exception(module_inst, "invalid function type of main function"); return false; @@ -218,11 +223,21 @@ execute_main(WASMModuleInstanceCommon *module_inst, int32 argc, char *argv[]) p += strlen(argv[i]) + 1; } - argc1 = 2; argv1[0] = (uint32)argc; - /* TODO: memory64 uint64 when the memory idx is i64 */ - argv1[1] = - (uint32)wasm_runtime_addr_native_to_app(module_inst, argv_offsets); +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) { + argc1 = 3; + uint64 app_addr = + wasm_runtime_addr_native_to_app(module_inst, argv_offsets); + PUT_I64_TO_ADDR(&argv[1], app_addr); + } + else +#endif + { + argc1 = 2; + argv1[1] = (uint32)wasm_runtime_addr_native_to_app(module_inst, + argv_offsets); + } } ret = wasm_runtime_call_wasm(exec_env, func, argc1, argv1); diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 381e6b447..1d2cd1677 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -286,6 +286,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, { WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; WASMMemoryInstance *memory_inst; + uint64 max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode || module_inst_comm->module_type == Wasm_Module_AoT); @@ -299,9 +300,13 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, goto fail; } +#if WASM_ENABLE_MEMORY64 != 0 + if (memory_inst->is_memory64) + max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; +#endif /* boundary overflow check */ - if (size > MAX_LINEAR_MEMORY_SIZE - || app_offset > MAX_LINEAR_MEMORY_SIZE - size) { + if (size > max_linear_memory_size + || app_offset > max_linear_memory_size - size) { goto fail; } @@ -324,7 +329,7 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, uint64 app_str_offset) { WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; - uint64 app_end_offset; + uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; char *str, *str_end; bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode @@ -338,10 +343,14 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, &app_end_offset)) goto fail; +#if WASM_ENABLE_MEMORY64 != 0 + if (module_inst->memories[0]->is_memory64) + max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; +#endif /* boundary overflow check, max start offset can only be size - 1, while end * offset can be size */ - if (app_str_offset >= MAX_LINEAR_MEMORY_SIZE - || app_end_offset > MAX_LINEAR_MEMORY_SIZE) + if (app_str_offset >= max_linear_memory_size + || app_end_offset > max_linear_memory_size) goto fail; str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset); @@ -364,6 +373,7 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; WASMMemoryInstance *memory_inst; uint8 *addr = (uint8 *)native_ptr; + uint64 max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode || module_inst_comm->module_type == Wasm_Module_AoT); @@ -377,8 +387,12 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, goto fail; } +#if WASM_ENABLE_MEMORY64 != 0 + if (memory_inst->is_memory64) + max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; +#endif /* boundary overflow check */ - if (size > MAX_LINEAR_MEMORY_SIZE || (uintptr_t)addr > UINTPTR_MAX - size) { + if (size > max_linear_memory_size || (uintptr_t)addr > UINTPTR_MAX - size) { goto fail; } @@ -748,12 +762,13 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count) goto return_func; } - bh_assert(total_size_new <= MAX_LINEAR_MEMORY_SIZE); + bh_assert(total_size_new + <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); if (full_size_mmaped) { #ifdef BH_PLATFORM_WINDOWS if (!os_mem_commit(memory->memory_data_end, - (uint32)(total_size_new - total_size_old), + (mem_offset_t)(total_size_new - total_size_old), MMAP_PROT_READ | MMAP_PROT_WRITE)) { ret = false; goto return_func; @@ -761,12 +776,12 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count) #endif if (os_mprotect(memory->memory_data_end, - (uint32)(total_size_new - total_size_old), + (mem_offset_t)(total_size_new - total_size_old), MMAP_PROT_READ | MMAP_PROT_WRITE) != 0) { #ifdef BH_PLATFORM_WINDOWS os_mem_decommit(memory->memory_data_end, - (uint32)(total_size_new - total_size_old)); + (mem_offset_t)(total_size_new - total_size_old)); #endif ret = false; goto return_func; @@ -895,8 +910,9 @@ wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst) int wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory, - uint64 num_bytes_per_page, uint64 init_page_count, - uint64 max_page_count, uint64 *memory_data_size) + bool is_memory64, uint64 num_bytes_per_page, + uint64 init_page_count, uint64 max_page_count, + uint64 *memory_data_size) { uint64 map_size, page_size; @@ -925,7 +941,16 @@ wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory, page_size = os_getpagesize(); *memory_data_size = init_page_count * num_bytes_per_page; - bh_assert(*memory_data_size <= MAX_LINEAR_MEMORY_SIZE); + +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) { + bh_assert(*memory_data_size <= MAX_LINEAR_MEM64_MEMORY_SIZE); + } + else +#endif + { + bh_assert(*memory_data_size <= MAX_LINEAR_MEMORY_SIZE); + } align_as_and_cast(*memory_data_size, page_size); if (map_size > 0) { diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index 2ada2e618..a5dfefae9 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -64,8 +64,9 @@ wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst); int wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory, - uint64 num_bytes_per_page, uint64 init_page_count, - uint64 max_page_count, uint64 *memory_data_size); + bool is_memory64, uint64 num_bytes_per_page, + uint64 init_page_count, uint64 max_page_count, + uint64 *memory_data_size); #ifdef __cplusplus } diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index a3c8f39eb..e3b4ca7b5 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -3746,7 +3746,7 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env); typedef void (*NativeRawFuncPtr)(WASMExecEnv *, uint64 *); NativeRawFuncPtr invoke_native_raw = (NativeRawFuncPtr)func_ptr; - uint64 argv_buf[16] = { 0 }, *argv1 = argv_buf, *argv_dst, size; + uint64 argv_buf[16] = { 0 }, *argv1 = argv_buf, *argv_dst, size, arg_i64; uint32 *argv_src = argv, i, argc1, ptr_len; uint32 arg_i32; bool ret = false; @@ -3770,9 +3770,11 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, case VALUE_TYPE_FUNCREF: #endif { - /* TODO: memory64 the data type of ptr_len and argc depends on - * mem idx type */ *(uint32 *)argv_dst = arg_i32 = *argv_src++; + /* TODO: memory64 if future there is a way for supporting + * wasm64 and wasm32 in libc at the same time, remove the + * macro control */ +#if WASM_ENABLE_MEMORY64 == 0 if (signature) { if (signature[i + 1] == '*') { /* param is a pointer */ @@ -3802,9 +3804,49 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, module, (uint64)arg_i32); } } +#endif break; } case VALUE_TYPE_I64: +#if WASM_ENABLE_MEMORY64 != 0 + { + PUT_I64_TO_ADDR((uint32 *)argv_dst, + GET_I64_FROM_ADDR(argv_src)); + argv_src += 2; + arg_i64 = *argv_dst; + if (signature) { + /* TODO: memory64 pointer with length need a new symbol + * to represent type i64, with '~' still represent i32 + * length */ + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv_src; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr(module, arg_i64, + (uint64)ptr_len)) + goto fail; + + *argv_dst = (uint64)wasm_runtime_addr_app_to_native( + module, arg_i64); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr(module, + arg_i64)) + goto fail; + + *argv_dst = (uint64)wasm_runtime_addr_app_to_native( + module, arg_i64); + } + } + break; + } +#endif case VALUE_TYPE_F64: bh_memcpy_s(argv_dst, sizeof(uint64), argv_src, sizeof(uint32) * 2); @@ -3933,6 +3975,9 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr, fail: if (argv1 != argv_buf) wasm_runtime_free(argv1); +#if WASM_ENABLE_MEMORY64 == 0 + (void)arg_i64; +#endif return ret; } @@ -4195,8 +4240,6 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, { arg_i32 = *argv_src++; - /* TODO: memory64 the data type of ptr_len and argc depends on - * mem idx type */ if (signature) { if (signature[i + 1] == '*') { /* param is a pointer */ @@ -4572,8 +4615,6 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, { arg_i32 = *argv++; - /* TODO: memory64 the data type of ptr_len and argc depends on - * mem idx type */ if (signature) { if (signature[i + 1] == '*') { /* param is a pointer */ @@ -4889,8 +4930,10 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, { arg_i32 = *argv_src++; arg_i64 = arg_i32; - /* TODO: memory64 the data type of ptr_len and argc depends on - * mem idx type */ + /* TODO: memory64 if future there is a way for supporting + * wasm64 and wasm32 in libc at the same time, remove the + * macro control */ +#if WASM_ENABLE_MEMORY64 == 0 if (signature) { if (signature[i + 1] == '*') { /* param is a pointer */ @@ -4918,6 +4961,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, module, (uint64)arg_i32); } } +#endif if (n_ints < MAX_REG_INTS) ints[n_ints++] = arg_i64; else @@ -4925,6 +4969,47 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr, break; } case VALUE_TYPE_I64: +#if WASM_ENABLE_MEMORY64 != 0 + { + arg_i64 = GET_I64_FROM_ADDR(argv_src); + argv_src += 2; + if (signature) { + /* TODO: memory64 pointer with length need a new symbol + * to represent type i64, with '~' still represent i32 + * length */ + if (signature[i + 1] == '*') { + /* param is a pointer */ + if (signature[i + 2] == '~') + /* pointer with length followed */ + ptr_len = *argv_src; + else + /* pointer without length followed */ + ptr_len = 1; + + if (!wasm_runtime_validate_app_addr(module, arg_i64, + (uint64)ptr_len)) + goto fail; + + arg_i64 = (uint64)wasm_runtime_addr_app_to_native( + module, arg_i64); + } + else if (signature[i + 1] == '$') { + /* param is a string */ + if (!wasm_runtime_validate_app_str_addr(module, + arg_i64)) + goto fail; + + arg_i64 = (uint64)wasm_runtime_addr_app_to_native( + module, arg_i64); + } + } + if (n_ints < MAX_REG_INTS) + ints[n_ints++] = arg_i64; + else + stacks[n_stacks++] = arg_i64; + break; + } +#endif #if WASM_ENABLE_GC != 0 case REF_TYPE_FUNCREF: case REF_TYPE_EXTERNREF: diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index aaf04ec2f..62c35473a 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -373,7 +373,7 @@ typedef struct WASMModuleCommon { /* The following uint8[1] member is a dummy just to indicate some module_type dependent members follow. - Typically it should be accessed by casting to the corresponding + Typically, it should be accessed by casting to the corresponding actual module_type dependent structure, not via this member. */ uint8 module_data[1]; } WASMModuleCommon; @@ -389,7 +389,7 @@ typedef struct WASMModuleInstanceCommon { /* The following uint8[1] member is a dummy just to indicate some module_type dependent members follow. - Typically it should be accessed by casting to the corresponding + Typically, it should be accessed by casting to the corresponding actual module_type dependent structure, not via this member. */ uint8 module_inst_data[1]; } WASMModuleInstanceCommon; diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index 56c8ebe75..5fd86b572 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -90,11 +90,22 @@ extern "C" { */ #define VALUE_TYPE_GC_REF 0x43 +#define MAX_PAGE_COUNT_FLAG 0x01 +#define SHARED_MEMORY_FLAG 0x02 +#define MEMORY64_FLAG 0x04 + #define DEFAULT_NUM_BYTES_PER_PAGE 65536 #define DEFAULT_MAX_PAGES 65536 +#define DEFAULT_MEM64_MAX_PAGES UINT32_MAX /* Max size of linear memory */ #define MAX_LINEAR_MEMORY_SIZE (4 * (uint64)BH_GB) +/* Roughly 274 TB */ +#define MAX_LINEAR_MEM64_MEMORY_SIZE \ + (DEFAULT_MEM64_MAX_PAGES * (uint64)64 * (uint64)BH_KB) +/* Macro to check memory flag and return appropriate memory size */ +#define GET_MAX_LINEAR_MEMORY_SIZE(is_memory64) \ + (is_memory64 ? MAX_LINEAR_MEM64_MEMORY_SIZE : MAX_LINEAR_MEMORY_SIZE) #if WASM_ENABLE_GC == 0 typedef uintptr_t table_elem_type_t; @@ -484,6 +495,12 @@ typedef struct WASMTable { #endif } WASMTable; +#if WASM_ENABLE_MEMORY64 != 0 +typedef uint64 mem_offset_t; +#else +typedef uint32 mem_offset_t; +#endif + typedef struct WASMMemory { uint32 flags; uint32 num_bytes_per_page; diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index d8fd2d708..ca972fd4b 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -46,8 +46,10 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ - || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 +#if WASM_ENABLE_MEMORY64 == 0 + +#if (!defined(OS_ENABLE_HW_BOUND_CHECK) \ + || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0) #define CHECK_MEMORY_OVERFLOW(bytes) \ do { \ uint64 offset1 = (uint64)offset + (uint64)addr; \ @@ -69,7 +71,8 @@ typedef float64 CellType_F64; else \ goto out_of_bounds; \ } while (0) -#else +#else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ + WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ #define CHECK_MEMORY_OVERFLOW(bytes) \ do { \ uint64 offset1 = (uint64)offset + (uint64)addr; \ @@ -80,8 +83,37 @@ typedef float64 CellType_F64; do { \ maddr = memory->memory_data + (uint32)(start); \ } while (0) -#endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \ - || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ +#endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ + WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ + +#else /* else of WASM_ENABLE_MEMORY64 == 0 */ + +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + /* If memory64 is enabled, offset1, offset1 + bytes can overflow */ \ + if (disable_bounds_checks \ + || (offset1 >= offset && offset1 + bytes >= offset1 \ + && offset1 + bytes <= get_linear_mem_size())) \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ + } while (0) +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + uint64 offset1 = (uint64)(start); \ + /* If memory64 is enabled, offset1 + bytes can overflow */ \ + if (disable_bounds_checks \ + || (offset1 + bytes >= offset1 \ + && offset1 + bytes <= get_linear_mem_size())) \ + /* App heap space is not valid space for \ + bulk memory operation */ \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ + } while (0) + +#endif /* end of WASM_ENABLE_MEMORY64 == 0 */ #define CHECK_ATOMIC_MEMORY_ACCESS() \ do { \ @@ -472,6 +504,23 @@ wasm_interp_get_frame_ref(WASMInterpFrame *frame) #define SET_LABEL_TYPE(_label_type) (void)0 #endif +#if WASM_ENABLE_MEMORY64 != 0 +#define PUSH_MEM_OFFSET(value) \ + do { \ + if (is_memory64) { \ + PUT_I64_TO_ADDR(frame_sp, value); \ + frame_sp += 2; \ + } \ + else { \ + *(int32 *)frame_sp++ = (int32)(value); \ + } \ + } while (0) +#else +#define PUSH_MEM_OFFSET(value) PUSH_I32(value) +#endif + +#define PUSH_PAGE_COUNT(value) PUSH_MEM_OFFSET(value) + #define PUSH_CSP(_label_type, param_cell_num, cell_num, _target_addr) \ do { \ bh_assert(frame_csp < frame->csp_boundary); \ @@ -501,6 +550,14 @@ wasm_interp_get_frame_ref(WASMInterpFrame *frame) GET_REF_FROM_ADDR(frame_sp)) #endif +#if WASM_ENABLE_MEMORY64 != 0 +#define POP_MEM_OFFSET() (is_memory64 ? POP_I64() : POP_I32()) +#else +#define POP_MEM_OFFSET() POP_I32() +#endif + +#define POP_PAGE_COUNT() POP_MEM_OFFSET() + #define POP_CSP_CHECK_OVERFLOW(n) \ do { \ bh_assert(frame_csp - n >= frame->csp_bottom); \ @@ -567,51 +624,73 @@ wasm_interp_get_frame_ref(WASMInterpFrame *frame) frame_csp = frame->csp; \ } while (0) -#define read_leb_int64(p, p_end, res) \ - do { \ - uint8 _val = *p; \ - if (!(_val & 0x80)) { \ - res = (int64)_val; \ - if (_val & 0x40) \ - /* sign extend */ \ - res |= 0xFFFFFFFFFFFFFF80LL; \ - p++; \ - break; \ - } \ - uint32 _off = 0; \ - res = (int64)read_leb(p, &_off, 64, true); \ - p += _off; \ +#define read_leb_int64(p, p_end, res) \ + do { \ + uint8 _val = *p; \ + if (!(_val & 0x80)) { \ + res = (int64)_val; \ + if (_val & 0x40) \ + /* sign extend */ \ + res |= 0xFFFFFFFFFFFFFF80LL; \ + p++; \ + } \ + else { \ + uint32 _off = 0; \ + res = (int64)read_leb(p, &_off, 64, true); \ + p += _off; \ + } \ } while (0) -#define read_leb_uint32(p, p_end, res) \ - do { \ - uint8 _val = *p; \ - if (!(_val & 0x80)) { \ - res = _val; \ - p++; \ - break; \ - } \ - uint32 _off = 0; \ - res = (uint32)read_leb(p, &_off, 32, false); \ - p += _off; \ +#define read_leb_uint32(p, p_end, res) \ + do { \ + uint8 _val = *p; \ + if (!(_val & 0x80)) { \ + res = _val; \ + p++; \ + } \ + else { \ + uint32 _off = 0; \ + res = (uint32)read_leb(p, &_off, 32, false); \ + p += _off; \ + } \ } while (0) -#define read_leb_int32(p, p_end, res) \ - do { \ - uint8 _val = *p; \ - if (!(_val & 0x80)) { \ - res = (int32)_val; \ - if (_val & 0x40) \ - /* sign extend */ \ - res |= 0xFFFFFF80; \ - p++; \ - break; \ - } \ - uint32 _off = 0; \ - res = (int32)read_leb(p, &_off, 32, true); \ - p += _off; \ +#define read_leb_int32(p, p_end, res) \ + do { \ + uint8 _val = *p; \ + if (!(_val & 0x80)) { \ + res = (int32)_val; \ + if (_val & 0x40) \ + /* sign extend */ \ + res |= 0xFFFFFF80; \ + p++; \ + } \ + else { \ + uint32 _off = 0; \ + res = (int32)read_leb(p, &_off, 32, true); \ + p += _off; \ + } \ } while (0) +#if WASM_ENABLE_MEMORY64 != 0 +#define read_leb_mem_offset(p, p_end, res) \ + do { \ + uint8 _val = *p; \ + if (!(_val & 0x80)) { \ + res = (mem_offset_t)_val; \ + p++; \ + } \ + else { \ + uint32 _off = 0; \ + res = (mem_offset_t)read_leb(p, &_off, is_memory64 ? 64 : 32, \ + false); \ + p += _off; \ + } \ + } while (0) +#else +#define read_leb_mem_offset(p, p_end, res) read_leb_uint32(p, p_end, res) +#endif + #if WASM_ENABLE_LABELS_AS_VALUES == 0 #define RECOVER_FRAME_IP_END() frame_ip_end = wasm_get_func_code_end(cur_func) #else @@ -872,7 +951,7 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min, uint32 readv, sval; \ \ sval = POP_I32(); \ - addr = POP_I32(); \ + addr = POP_MEM_OFFSET(); \ \ if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U) { \ CHECK_MEMORY_OVERFLOW(1); \ @@ -912,7 +991,7 @@ trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min, uint64 readv, sval; \ \ sval = (uint64)POP_I64(); \ - addr = POP_I32(); \ + addr = POP_MEM_OFFSET(); \ \ if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U) { \ CHECK_MEMORY_OVERFLOW(1); \ @@ -1430,6 +1509,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMStringviewIterObjectRef stringview_iter_obj; #endif #endif +#if WASM_ENABLE_MEMORY64 != 0 + /* TODO: multi-memories for now assuming the memory idx type is consistent + * across multi-memories */ + bool is_memory64 = false; + if (memory) + is_memory64 = memory->is_memory64; +#endif #if WASM_ENABLE_DEBUG_INTERP != 0 uint8 *frame_ip_orig = NULL; @@ -4087,8 +4173,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, bh_assert(global_idx < module->e->global_count); global = globals + global_idx; global_addr = get_global_addr(global_data, global); - /* TODO: Memory64 the data type depends on mem idx type */ - aux_stack_top = (uint64)(*(uint32 *)(frame_sp - 1)); +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) { + aux_stack_top = *(uint64 *)(frame_sp - 2); + } + else +#endif + { + aux_stack_top = (uint64)(*(uint32 *)(frame_sp - 1)); + } if (aux_stack_top <= (uint64)exec_env->aux_stack_boundary) { wasm_set_exception(module, "wasm auxiliary stack overflow"); goto got_exception; @@ -4098,8 +4191,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, "wasm auxiliary stack underflow"); goto got_exception; } - *(int32 *)global_addr = aux_stack_top; - frame_sp--; +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) { + *(uint64 *)global_addr = aux_stack_top; + frame_sp -= 2; + } + else +#endif + { + *(uint32 *)global_addr = aux_stack_top; + frame_sp--; + } #if WASM_ENABLE_MEMORY_PROFILING != 0 if (module->module->aux_stack_top_global_index != (uint32)-1) { uint32 aux_stack_used = @@ -4126,11 +4228,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_LOAD) HANDLE_OP(WASM_OP_F32_LOAD) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); - addr = POP_I32(); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(4); PUSH_I32(LOAD_I32(maddr)); CHECK_READ_WATCHPOINT(addr, offset); @@ -4141,11 +4244,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I64_LOAD) HANDLE_OP(WASM_OP_F64_LOAD) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); - addr = POP_I32(); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(8); PUSH_I64(LOAD_I64(maddr)); CHECK_READ_WATCHPOINT(addr, offset); @@ -4155,11 +4259,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_LOAD8_S) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); - addr = POP_I32(); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(1); PUSH_I32(sign_ext_8_32(*(int8 *)maddr)); CHECK_READ_WATCHPOINT(addr, offset); @@ -4169,11 +4274,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_LOAD8_U) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); - addr = POP_I32(); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(1); PUSH_I32((uint32)(*(uint8 *)maddr)); CHECK_READ_WATCHPOINT(addr, offset); @@ -4183,11 +4289,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_LOAD16_S) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); - addr = POP_I32(); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(2); PUSH_I32(sign_ext_16_32(LOAD_I16(maddr))); CHECK_READ_WATCHPOINT(addr, offset); @@ -4197,11 +4304,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_LOAD16_U) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); - addr = POP_I32(); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(2); PUSH_I32((uint32)(LOAD_U16(maddr))); CHECK_READ_WATCHPOINT(addr, offset); @@ -4211,11 +4319,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I64_LOAD8_S) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); - addr = POP_I32(); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(1); PUSH_I64(sign_ext_8_64(*(int8 *)maddr)); CHECK_READ_WATCHPOINT(addr, offset); @@ -4225,11 +4334,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I64_LOAD8_U) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); - addr = POP_I32(); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(1); PUSH_I64((uint64)(*(uint8 *)maddr)); CHECK_READ_WATCHPOINT(addr, offset); @@ -4239,11 +4349,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I64_LOAD16_S) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); - addr = POP_I32(); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(2); PUSH_I64(sign_ext_16_64(LOAD_I16(maddr))); CHECK_READ_WATCHPOINT(addr, offset); @@ -4253,11 +4364,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I64_LOAD16_U) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); - addr = POP_I32(); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(2); PUSH_I64((uint64)(LOAD_U16(maddr))); CHECK_READ_WATCHPOINT(addr, offset); @@ -4267,12 +4379,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I64_LOAD32_S) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; opcode = *(frame_ip - 1); read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); - addr = POP_I32(); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(4); PUSH_I64(sign_ext_32_64(LOAD_I32(maddr))); CHECK_READ_WATCHPOINT(addr, offset); @@ -4282,11 +4395,12 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I64_LOAD32_U) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); - addr = POP_I32(); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(4); PUSH_I64((uint64)(LOAD_U32(maddr))); CHECK_READ_WATCHPOINT(addr, offset); @@ -4298,14 +4412,23 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_STORE) HANDLE_OP(WASM_OP_F32_STORE) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); frame_sp--; - addr = POP_I32(); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(4); - STORE_U32(maddr, frame_sp[1]); +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) { + STORE_U32(maddr, frame_sp[2]); + } + else +#endif + { + STORE_U32(maddr, frame_sp[1]); + } CHECK_WRITE_WATCHPOINT(addr, offset); (void)flags; HANDLE_OP_END(); @@ -4314,15 +4437,26 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I64_STORE) HANDLE_OP(WASM_OP_F64_STORE) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); frame_sp -= 2; - addr = POP_I32(); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(8); - PUT_I64_TO_ADDR((uint32 *)maddr, - GET_I64_FROM_ADDR(frame_sp + 1)); + +#if WASM_ENABLE_MEMORY64 != 0 + if (is_memory64) { + PUT_I64_TO_ADDR((mem_offset_t *)maddr, + GET_I64_FROM_ADDR(frame_sp + 2)); + } + else +#endif + { + PUT_I64_TO_ADDR((uint32 *)maddr, + GET_I64_FROM_ADDR(frame_sp + 1)); + } CHECK_WRITE_WATCHPOINT(addr, offset); (void)flags; HANDLE_OP_END(); @@ -4331,14 +4465,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I32_STORE8) HANDLE_OP(WASM_OP_I32_STORE16) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; uint32 sval; opcode = *(frame_ip - 1); read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); sval = (uint32)POP_I32(); - addr = POP_I32(); + addr = POP_MEM_OFFSET(); if (opcode == WASM_OP_I32_STORE8) { CHECK_MEMORY_OVERFLOW(1); @@ -4357,14 +4492,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, HANDLE_OP(WASM_OP_I64_STORE16) HANDLE_OP(WASM_OP_I64_STORE32) { - uint32 offset, flags, addr; + uint32 flags; + mem_offset_t offset, addr; uint64 sval; opcode = *(frame_ip - 1); read_leb_uint32(frame_ip, frame_ip_end, flags); - read_leb_uint32(frame_ip, frame_ip_end, offset); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); sval = (uint64)POP_I64(); - addr = POP_I32(); + addr = POP_MEM_OFFSET(); if (opcode == WASM_OP_I64_STORE8) { CHECK_MEMORY_OVERFLOW(1); @@ -4388,7 +4524,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, { uint32 reserved; read_leb_uint32(frame_ip, frame_ip_end, reserved); - PUSH_I32(memory->cur_page_count); + PUSH_PAGE_COUNT(memory->cur_page_count); (void)reserved; HANDLE_OP_END(); } @@ -4399,15 +4535,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, prev_page_count = memory->cur_page_count; read_leb_uint32(frame_ip, frame_ip_end, reserved); - delta = (uint32)POP_I32(); + delta = (uint32)POP_PAGE_COUNT(); if (!wasm_enlarge_memory(module, delta)) { /* failed to memory.grow, return -1 */ - PUSH_I32(-1); + PUSH_PAGE_COUNT(-1); } else { /* success, return previous page count */ - PUSH_I32(prev_page_count); + PUSH_PAGE_COUNT(prev_page_count); /* update memory size, no need to update memory ptr as it isn't changed in wasm_enlarge_memory */ #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ @@ -5407,7 +5543,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_BULK_MEMORY != 0 case WASM_OP_MEMORY_INIT: { - uint32 addr, segment; + uint32 segment; + mem_offset_t addr; uint64 bytes, offset, seg_len; uint8 *data; @@ -5417,7 +5554,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, bytes = (uint64)(uint32)POP_I32(); offset = (uint64)(uint32)POP_I32(); - addr = (uint32)POP_I32(); + addr = (mem_offset_t)POP_MEM_OFFSET(); #if WASM_ENABLE_THREAD_MGR != 0 linear_mem_size = get_linear_mem_size(); @@ -5460,14 +5597,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } case WASM_OP_MEMORY_COPY: { - uint32 dst, src, len; + mem_offset_t dst, src, len; uint8 *mdst, *msrc; frame_ip += 2; - - len = POP_I32(); - src = POP_I32(); - dst = POP_I32(); + len = POP_MEM_OFFSET(); + src = POP_MEM_OFFSET(); + dst = POP_MEM_OFFSET(); #if WASM_ENABLE_THREAD_MGR != 0 linear_mem_size = get_linear_mem_size(); @@ -5493,13 +5629,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } case WASM_OP_MEMORY_FILL: { - uint32 dst, len; + mem_offset_t dst, len; uint8 fill_val, *mdst; frame_ip++; - len = POP_I32(); + len = POP_MEM_OFFSET(); fill_val = POP_I32(); - dst = POP_I32(); + dst = POP_MEM_OFFSET(); #if WASM_ENABLE_THREAD_MGR != 0 linear_mem_size = get_linear_mem_size(); @@ -5729,7 +5865,8 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_SHARED_MEMORY != 0 HANDLE_OP(WASM_OP_ATOMIC_PREFIX) { - uint32 offset = 0, align = 0, addr; + mem_offset_t offset = 0, addr; + uint32 align = 0; uint32 opcode1; read_leb_uint32(frame_ip, frame_ip_end, opcode1); @@ -5739,7 +5876,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (opcode != WASM_OP_ATOMIC_FENCE) { read_leb_uint32(frame_ip, frame_ip_end, align); - read_leb_uint32(frame_ip, frame_ip_end, offset); + read_leb_mem_offset(frame_ip, frame_ip_end, offset); } switch (opcode) { @@ -5748,7 +5885,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint32 notify_count, ret; notify_count = POP_I32(); - addr = POP_I32(); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(4); CHECK_ATOMIC_MEMORY_ACCESS(); @@ -5768,7 +5905,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, timeout = POP_I64(); expect = POP_I32(); - addr = POP_I32(); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(4); CHECK_ATOMIC_MEMORY_ACCESS(); @@ -5792,7 +5929,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, timeout = POP_I64(); expect = POP_I64(); - addr = POP_I32(); + addr = POP_MEM_OFFSET(); CHECK_MEMORY_OVERFLOW(8); CHECK_ATOMIC_MEMORY_ACCESS(); @@ -5823,7 +5960,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, { uint32 readv; - addr = POP_I32(); + addr = POP_MEM_OFFSET(); if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) { CHECK_MEMORY_OVERFLOW(1); @@ -5858,7 +5995,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, { uint64 readv; - addr = POP_I32(); + addr = POP_MEM_OFFSET(); if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) { CHECK_MEMORY_OVERFLOW(1); @@ -5900,7 +6037,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint32 sval; sval = (uint32)POP_I32(); - addr = POP_I32(); + addr = POP_MEM_OFFSET(); if (opcode == WASM_OP_ATOMIC_I32_STORE8) { CHECK_MEMORY_OVERFLOW(1); @@ -5934,7 +6071,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, uint64 sval; sval = (uint64)POP_I64(); - addr = POP_I32(); + addr = POP_MEM_OFFSET(); if (opcode == WASM_OP_ATOMIC_I64_STORE8) { CHECK_MEMORY_OVERFLOW(1); @@ -5961,7 +6098,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, CHECK_MEMORY_OVERFLOW(8); CHECK_ATOMIC_MEMORY_ACCESS(); shared_memory_lock(memory); - PUT_I64_TO_ADDR((uint32 *)maddr, sval); + STORE_I64(maddr, sval); shared_memory_unlock(memory); } break; @@ -5975,7 +6112,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, sval = POP_I32(); expect = POP_I32(); - addr = POP_I32(); + addr = POP_MEM_OFFSET(); if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U) { CHECK_MEMORY_OVERFLOW(1); @@ -6021,7 +6158,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, sval = (uint64)POP_I64(); expect = (uint64)POP_I64(); - addr = POP_I32(); + addr = POP_MEM_OFFSET(); if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U) { CHECK_MEMORY_OVERFLOW(1); diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index b602df877..51384cb69 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -45,6 +45,16 @@ set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) } } +#if WASM_ENABLE_MEMORY64 != 0 +static void +set_error_buf_mem_offset_out_of_range(char *error_buf, uint32 error_buf_size) +{ + if (error_buf != NULL) { + snprintf(error_buf, error_buf_size, "offset out of range"); + } +} +#endif + static void set_error_buf_v(char *error_buf, uint32 error_buf_size, const char *format, ...) { @@ -102,6 +112,7 @@ check_buf1(const uint8 *buf, const uint8 *buf_end, uint32 length, #define skip_leb_int64(p, p_end) skip_leb(p) #define skip_leb_uint32(p, p_end) skip_leb(p) #define skip_leb_int32(p, p_end) skip_leb(p) +#define skip_leb_mem_offset(p, p_end) skip_leb(p) static bool read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign, @@ -139,7 +150,7 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign, } else if (sign && maxbits == 32) { if (shift < maxbits) { - /* Sign extend, second highest bit is the sign bit */ + /* Sign extend, second-highest bit is the sign bit */ if ((uint8)byte & 0x40) result |= (~((uint64)0)) << shift; } @@ -154,7 +165,7 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign, } else if (sign && maxbits == 64) { if (shift < maxbits) { - /* Sign extend, second highest bit is the sign bit */ + /* Sign extend, second-highest bit is the sign bit */ if ((uint8)byte & 0x40) result |= (~((uint64)0)) << shift; } @@ -191,6 +202,21 @@ fail: res = (int64)res64; \ } while (0) +#if WASM_ENABLE_MEMORY64 != 0 +#define read_leb_mem_offset(p, p_end, res) \ + do { \ + uint64 res64; \ + if (!read_leb((uint8 **)&p, p_end, is_memory64 ? 64 : 32, false, \ + &res64, error_buf, error_buf_size)) { \ + set_error_buf_mem_offset_out_of_range(error_buf, error_buf_size); \ + goto fail; \ + } \ + res = (mem_offset_t)res64; \ + } while (0) +#else +#define read_leb_mem_offset(p, p_end, res) read_leb_uint32(p, p_end, res) +#endif + #define read_leb_uint32(p, p_end, res) \ do { \ uint64 res64; \ @@ -2582,31 +2608,92 @@ fail: } static bool -check_memory_init_size(uint32 init_size, char *error_buf, uint32 error_buf_size) +check_memory_init_size(bool is_memory64, uint32 init_size, char *error_buf, + uint32 error_buf_size) { - if (init_size > DEFAULT_MAX_PAGES) { + uint32 default_max_size = + is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; + + if (!is_memory64 && init_size > default_max_size) { set_error_buf(error_buf, error_buf_size, "memory size must be at most 65536 pages (4GiB)"); return false; } +#if WASM_ENABLE_MEMORY64 != 0 + else if (is_memory64 && init_size > default_max_size) { + set_error_buf( + error_buf, error_buf_size, + "memory size must be at most 4,294,967,295 pages (274 Terabyte)"); + return false; + } +#endif return true; } static bool -check_memory_max_size(uint32 init_size, uint32 max_size, char *error_buf, - uint32 error_buf_size) +check_memory_max_size(bool is_memory64, uint32 init_size, uint32 max_size, + char *error_buf, uint32 error_buf_size) { + uint32 default_max_size = + is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; + if (max_size < init_size) { set_error_buf(error_buf, error_buf_size, "size minimum must not be greater than maximum"); return false; } - if (max_size > DEFAULT_MAX_PAGES) { + if (!is_memory64 && max_size > default_max_size) { set_error_buf(error_buf, error_buf_size, "memory size must be at most 65536 pages (4GiB)"); return false; } +#if WASM_ENABLE_MEMORY64 != 0 + else if (is_memory64 && max_size > default_max_size) { + set_error_buf( + error_buf, error_buf_size, + "memory size must be at most 4,294,967,295 pages (274 Terabyte)"); + return false; + } +#endif + + return true; +} + +static bool +check_memory_flag(const uint8 mem_flag, char *error_buf, uint32 error_buf_size) +{ + /* Check whether certain features indicated by mem_flag are enabled in + * runtime */ + if (mem_flag > MAX_PAGE_COUNT_FLAG) { +#if WASM_ENABLE_SHARED_MEMORY == 0 + if (mem_flag & SHARED_MEMORY_FLAG) { + LOG_VERBOSE("shared memory flag was found, please enable shared " + "memory, lib-pthread or lib-wasi-threads"); + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } +#endif +#if WASM_ENABLE_MEMORY64 == 0 + if (mem_flag & MEMORY64_FLAG) { + LOG_VERBOSE("memory64 flag was found, please enable memory64"); + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } +#endif + } + + if (mem_flag > MAX_PAGE_COUNT_FLAG + SHARED_MEMORY_FLAG + MEMORY64_FLAG) { + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } + else if ((mem_flag & SHARED_MEMORY_FLAG) + && !(mem_flag & MAX_PAGE_COUNT_FLAG)) { + set_error_buf(error_buf, error_buf_size, + "shared memory must have maximum"); + return false; + } + return true; } @@ -2616,15 +2703,16 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end, const char *memory_name, WASMMemoryImport *memory, char *error_buf, uint32 error_buf_size) { - const uint8 *p = *p_buf, *p_end = buf_end; + const uint8 *p = *p_buf, *p_end = buf_end, *p_org; #if WASM_ENABLE_APP_FRAMEWORK != 0 uint32 pool_size = wasm_runtime_memory_pool_size(); uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT / DEFAULT_NUM_BYTES_PER_PAGE; #else - uint32 max_page_count = DEFAULT_MAX_PAGES; + uint32 max_page_count; #endif /* WASM_ENABLE_APP_FRAMEWORK */ - uint32 declare_max_page_count_flag = 0; + uint32 mem_flag = 0; + bool is_memory64 = false; uint32 declare_init_page_count = 0; uint32 declare_max_page_count = 0; #if WASM_ENABLE_MULTI_MODULE != 0 @@ -2632,16 +2720,31 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *linked_memory = NULL; #endif - read_leb_uint32(p, p_end, declare_max_page_count_flag); + p_org = p; + read_leb_uint32(p, p_end, mem_flag); + is_memory64 = mem_flag & MEMORY64_FLAG; + if (p - p_org > 1) { + LOG_VERBOSE("integer representation too long"); + set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + return false; + } + + if (!check_memory_flag(mem_flag, error_buf, error_buf_size)) { + return false; + } + read_leb_uint32(p, p_end, declare_init_page_count); - if (!check_memory_init_size(declare_init_page_count, error_buf, + if (!check_memory_init_size(is_memory64, declare_init_page_count, error_buf, error_buf_size)) { return false; } - if (declare_max_page_count_flag & 1) { +#if WASM_ENABLE_APP_FRAMEWORK == 0 + max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; +#endif + if (mem_flag & MAX_PAGE_COUNT_FLAG) { read_leb_uint32(p, p_end, declare_max_page_count); - if (!check_memory_max_size(declare_init_page_count, + if (!check_memory_max_size(is_memory64, declare_init_page_count, declare_max_page_count, error_buf, error_buf_size)) { return false; @@ -2664,7 +2767,7 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end, #if WASM_ENABLE_LIB_WASI_THREADS != 0 /* Avoid memory import failure when wasi-threads is enabled and the memory is shared */ - if (!(declare_max_page_count_flag & 2)) + if (!(mem_flag & SHARED_MEMORY_FLAG)) return false; #else return false; @@ -2712,7 +2815,7 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end, } /* now we believe all declaration are ok */ - memory->flags = declare_max_page_count_flag; + memory->flags = mem_flag; memory->init_page_count = declare_init_page_count; memory->max_page_count = declare_max_page_count; memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; @@ -3013,53 +3116,34 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT / DEFAULT_NUM_BYTES_PER_PAGE; #else - uint32 max_page_count = DEFAULT_MAX_PAGES; + uint32 max_page_count; #endif + bool is_memory64 = false; p_org = p; read_leb_uint32(p, p_end, memory->flags); -#if WASM_ENABLE_SHARED_MEMORY == 0 - if (p - p_org > 1) { - set_error_buf(error_buf, error_buf_size, - "integer representation too long"); - return false; - } - if (memory->flags > 1) { - if (memory->flags & 2) { - set_error_buf(error_buf, error_buf_size, - "shared memory flag was found, " - "please enable shared memory, lib-pthread " - "or lib-wasi-threads"); - } - else { - set_error_buf(error_buf, error_buf_size, "invalid memory flags"); - } - return false; - } -#else + is_memory64 = memory->flags & MEMORY64_FLAG; if (p - p_org > 1) { + LOG_VERBOSE("integer representation too long"); set_error_buf(error_buf, error_buf_size, "invalid limits flags"); return false; } - if (memory->flags > 3) { - set_error_buf(error_buf, error_buf_size, "invalid limits flags"); + + if (!check_memory_flag(memory->flags, error_buf, error_buf_size)) { return false; } - else if (memory->flags == 2) { - set_error_buf(error_buf, error_buf_size, - "shared memory must have maximum"); - return false; - } -#endif read_leb_uint32(p, p_end, memory->init_page_count); - if (!check_memory_init_size(memory->init_page_count, error_buf, + if (!check_memory_init_size(is_memory64, memory->init_page_count, error_buf, error_buf_size)) return false; +#if WASM_ENABLE_APP_FRAMEWORK == 0 + max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; +#endif if (memory->flags & 1) { read_leb_uint32(p, p_end, memory->max_page_count); - if (!check_memory_max_size(memory->init_page_count, + if (!check_memory_max_size(is_memory64, memory->init_page_count, memory->max_page_count, error_buf, error_buf_size)) return false; @@ -4450,6 +4534,7 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end, bool is_passive = false; uint32 mem_flag; #endif + uint8 mem_offset_type; read_leb_uint32(p, p_end, data_seg_count); @@ -4515,11 +4600,35 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end, } #endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_BULK_MEMORY != 0 + if (!is_passive) +#endif + { +#if WASM_ENABLE_MEMORY64 != 0 + /* This memory_flag is from memory instead of data segment */ + uint8 memory_flag; + if (module->import_memory_count > 0) { + memory_flag = + module->import_memories[mem_index].u.memory.flags; + } + else { + memory_flag = + module + ->memories[mem_index - module->import_memory_count] + .flags; + } + mem_offset_type = memory_flag & MEMORY64_FLAG ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#else + mem_offset_type = VALUE_TYPE_I32; +#endif + } + #if WASM_ENABLE_BULK_MEMORY != 0 if (!is_passive) #endif if (!load_init_expr(module, &p, p_end, &init_expr, - VALUE_TYPE_I32, NULL, error_buf, + mem_offset_type, NULL, error_buf, error_buf_size)) return false; @@ -6979,8 +7088,8 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, case WASM_OP_I64_STORE8: case WASM_OP_I64_STORE16: case WASM_OP_I64_STORE32: - skip_leb_uint32(p, p_end); /* align */ - skip_leb_uint32(p, p_end); /* offset */ + skip_leb_uint32(p, p_end); /* align */ + skip_leb_mem_offset(p, p_end); /* offset */ break; case WASM_OP_MEMORY_SIZE: @@ -7326,6 +7435,7 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, #if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) case WASM_OP_SIMD_PREFIX: { + /* TODO: memory64 offset type changes */ uint32 opcode1; read_leb_uint32(p, p_end, opcode1); @@ -7422,6 +7532,7 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, #if WASM_ENABLE_SHARED_MEMORY != 0 case WASM_OP_ATOMIC_PREFIX: { + /* TODO: memory64 offset type changes */ uint32 opcode1; /* atomic_op (u32_leb) + memarg (2 u32_leb) */ @@ -7431,8 +7542,8 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, opcode = (uint8)opcode1; if (opcode != WASM_OP_ATOMIC_FENCE) { - skip_leb_uint32(p, p_end); /* align */ - skip_leb_uint32(p, p_end); /* offset */ + skip_leb_uint32(p, p_end); /* align */ + skip_leb_mem_offset(p, p_end); /* offset */ } else { /* atomic.fence doesn't have memarg */ @@ -9334,6 +9445,8 @@ fail: #define PUSH_EXTERNREF() TEMPLATE_PUSH(EXTERNREF) #define PUSH_REF(Type) TEMPLATE_PUSH_REF(Type) #define POP_REF(Type) TEMPLATE_POP_REF(Type) +#define PUSH_MEM_OFFSET() TEMPLATE_PUSH_REF(mem_offset_type) +#define PUSH_PAGE_COUNT() PUSH_MEM_OFFSET() #define POP_I32() TEMPLATE_POP(I32) #define POP_F32() TEMPLATE_POP(F32) @@ -9343,6 +9456,7 @@ fail: #define POP_FUNCREF() TEMPLATE_POP(FUNCREF) #define POP_EXTERNREF() TEMPLATE_POP(EXTERNREF) #define POP_STRINGREF() TEMPLATE_POP(STRINGREF) +#define POP_MEM_OFFSET() TEMPLATE_POP_REF(mem_offset_type) #if WASM_ENABLE_FAST_INTERP != 0 @@ -10510,11 +10624,12 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, { uint8 *p = func->code, *p_end = func->code + func->code_size, *p_org; uint32 param_count, local_count, global_count; - uint8 *param_types, *local_types, local_type, global_type; + uint8 *param_types, *local_types, local_type, global_type, mem_offset_type; BlockType func_block_type; uint16 *local_offsets, local_offset; uint32 type_idx, func_idx, local_idx, global_idx, table_idx; - uint32 table_seg_idx, data_seg_idx, count, align, mem_offset, i; + uint32 table_seg_idx, data_seg_idx, count, align, i; + mem_offset_t mem_offset; int32 i32_const = 0; int64 i64_const; uint8 opcode; @@ -10539,6 +10654,19 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, LOG_OP("\nProcessing func | [%d] params | [%d] locals | [%d] return\n", func->param_cell_num, func->local_cell_num, func->ret_cell_num); #endif +#if WASM_ENABLE_MEMORY64 != 0 + bool is_memory64 = false; + /* TODO: multi-memories for now assuming the memory idx type is consistent + * across multi-memories */ + if (module->import_memory_count > 0) + is_memory64 = module->import_memories[0].u.memory.flags & MEMORY64_FLAG; + else if (module->memory_count > 0) + is_memory64 = module->memories[0].flags & MEMORY64_FLAG; + + mem_offset_type = is_memory64 ? VALUE_TYPE_I64 : VALUE_TYPE_I32; +#else + mem_offset_type = VALUE_TYPE_I32; +#endif global_count = module->import_global_count + module->global_count; @@ -12730,8 +12858,8 @@ re_scan: } #endif CHECK_MEMORY(); - read_leb_uint32(p, p_end, align); /* align */ - read_leb_uint32(p, p_end, mem_offset); /* offset */ + read_leb_uint32(p, p_end, align); /* align */ + read_leb_mem_offset(p, p_end, mem_offset); /* offset */ if (!check_memory_access_align(opcode, align, error_buf, error_buf_size)) { goto fail; @@ -12749,7 +12877,7 @@ re_scan: case WASM_OP_I32_LOAD8_U: case WASM_OP_I32_LOAD16_S: case WASM_OP_I32_LOAD16_U: - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32); break; case WASM_OP_I64_LOAD: case WASM_OP_I64_LOAD8_S: @@ -12758,35 +12886,35 @@ re_scan: case WASM_OP_I64_LOAD16_U: case WASM_OP_I64_LOAD32_S: case WASM_OP_I64_LOAD32_U: - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64); + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64); break; case WASM_OP_F32_LOAD: - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F32); + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F32); break; case WASM_OP_F64_LOAD: - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F64); + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F64); break; /* store */ case WASM_OP_I32_STORE: case WASM_OP_I32_STORE8: case WASM_OP_I32_STORE16: POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); break; case WASM_OP_I64_STORE: case WASM_OP_I64_STORE8: case WASM_OP_I64_STORE16: case WASM_OP_I64_STORE32: POP_I64(); - POP_I32(); + POP_MEM_OFFSET(); break; case WASM_OP_F32_STORE: POP_F32(); - POP_I32(); + POP_MEM_OFFSET(); break; case WASM_OP_F64_STORE: POP_F64(); - POP_I32(); + POP_MEM_OFFSET(); break; default: break; @@ -12802,7 +12930,7 @@ re_scan: "zero byte expected"); goto fail; } - PUSH_I32(); + PUSH_PAGE_COUNT(); module->possible_memory_grow = true; #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 @@ -12818,7 +12946,7 @@ re_scan: "zero byte expected"); goto fail; } - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + POP_AND_PUSH(mem_offset_type, mem_offset_type); module->possible_memory_grow = true; #if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ @@ -14179,7 +14307,7 @@ re_scan: POP_I32(); POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 func->has_memory_operations = true; #endif @@ -14222,9 +14350,9 @@ re_scan: && module->memory_count == 0) goto fail_unknown_memory; - POP_I32(); - POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); + POP_MEM_OFFSET(); + POP_MEM_OFFSET(); #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 func->has_memory_operations = true; #endif @@ -14242,10 +14370,9 @@ re_scan: && module->memory_count == 0) { goto fail_unknown_memory; } - - POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); POP_I32(); + POP_MEM_OFFSET(); #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 func->has_memory_operations = true; #endif @@ -14491,6 +14618,7 @@ re_scan: #if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) case WASM_OP_SIMD_PREFIX: { + /* TODO: memory64 offset type changes */ uint32 opcode1; #if WASM_ENABLE_WAMR_COMPILER != 0 @@ -15167,8 +15295,8 @@ re_scan: #endif if (opcode1 != WASM_OP_ATOMIC_FENCE) { CHECK_MEMORY(); - read_leb_uint32(p, p_end, align); /* align */ - read_leb_uint32(p, p_end, mem_offset); /* offset */ + read_leb_uint32(p, p_end, align); /* align */ + read_leb_mem_offset(p, p_end, mem_offset); /* offset */ if (!check_memory_align_equal(opcode1, align, error_buf, error_buf_size)) { goto fail; @@ -15182,18 +15310,20 @@ re_scan: #endif switch (opcode1) { case WASM_OP_ATOMIC_NOTIFY: - POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); break; case WASM_OP_ATOMIC_WAIT32: POP_I64(); POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); PUSH_I32(); break; case WASM_OP_ATOMIC_WAIT64: POP_I64(); POP_I64(); - POP_I32(); + POP_MEM_OFFSET(); PUSH_I32(); break; case WASM_OP_ATOMIC_FENCE: @@ -15207,26 +15337,26 @@ re_scan: case WASM_OP_ATOMIC_I32_LOAD: case WASM_OP_ATOMIC_I32_LOAD8_U: case WASM_OP_ATOMIC_I32_LOAD16_U: - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32); break; case WASM_OP_ATOMIC_I32_STORE: case WASM_OP_ATOMIC_I32_STORE8: case WASM_OP_ATOMIC_I32_STORE16: POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); break; case WASM_OP_ATOMIC_I64_LOAD: case WASM_OP_ATOMIC_I64_LOAD8_U: case WASM_OP_ATOMIC_I64_LOAD16_U: case WASM_OP_ATOMIC_I64_LOAD32_U: - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64); + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64); break; case WASM_OP_ATOMIC_I64_STORE: case WASM_OP_ATOMIC_I64_STORE8: case WASM_OP_ATOMIC_I64_STORE16: case WASM_OP_ATOMIC_I64_STORE32: POP_I64(); - POP_I32(); + POP_MEM_OFFSET(); break; case WASM_OP_ATOMIC_RMW_I32_ADD: case WASM_OP_ATOMIC_RMW_I32_ADD8_U: @@ -15246,7 +15376,9 @@ re_scan: case WASM_OP_ATOMIC_RMW_I32_XCHG: case WASM_OP_ATOMIC_RMW_I32_XCHG8_U: case WASM_OP_ATOMIC_RMW_I32_XCHG16_U: - POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); break; case WASM_OP_ATOMIC_RMW_I64_ADD: case WASM_OP_ATOMIC_RMW_I64_ADD8_U: @@ -15273,7 +15405,7 @@ re_scan: case WASM_OP_ATOMIC_RMW_I64_XCHG16_U: case WASM_OP_ATOMIC_RMW_I64_XCHG32_U: POP_I64(); - POP_I32(); + POP_MEM_OFFSET(); PUSH_I64(); break; case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: @@ -15281,7 +15413,7 @@ re_scan: case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: POP_I32(); POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); PUSH_I32(); break; case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: @@ -15290,7 +15422,7 @@ re_scan: case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: POP_I64(); POP_I64(); - POP_I32(); + POP_MEM_OFFSET(); PUSH_I64(); break; default: diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 51dbbe003..f1023fa01 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -47,6 +47,7 @@ set_error_buf(char *error_buf, uint32 error_buf_size, const char *string) #define skip_leb_int64(p, p_end) skip_leb(p) #define skip_leb_uint32(p, p_end) skip_leb(p) #define skip_leb_int32(p, p_end) skip_leb(p) +#define skip_leb_mem_offset(p, p_end) skip_leb(p) static bool is_32bit_type(uint8 type) @@ -116,7 +117,7 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign, } else if (sign && maxbits == 32) { if (shift < maxbits) { - /* Sign extend, second highest bit is the sign bit */ + /* Sign extend, second-highest bit is the sign bit */ if ((uint8)byte & 0x40) result |= (~((uint64)0)) << shift; } @@ -132,7 +133,7 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign, } else if (sign && maxbits == 64) { if (shift < maxbits) { - /* Sign extend, second highest bit is the sign bit */ + /* Sign extend, second-highest bit is the sign bit */ if ((uint8)byte & 0x40) result |= (~((uint64)0)) << shift; } @@ -180,6 +181,18 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign, res = (int32)res64; \ } while (0) +#if WASM_ENABLE_MEMORY64 != 0 +#define read_leb_mem_offset(p, p_end, res) \ + do { \ + uint64 res64; \ + read_leb((uint8 **)&p, p_end, is_memory64 ? 64 : 32, false, &res64, \ + error_buf, error_buf_size); \ + res = (mem_offset_t)res64; \ + } while (0) +#else +#define read_leb_mem_offset(p, p_end, res) read_leb_uint32(p, p_end, res) +#endif + static void * loader_malloc(uint64 size, char *error_buf, uint32 error_buf_size) { @@ -683,6 +696,38 @@ load_table_import(const uint8 **p_buf, const uint8 *buf_end, return true; } +static bool +check_memory_flag(const uint8 mem_flag) +{ + /* Check whether certain features indicated by mem_flag are enabled in + * runtime */ + if (mem_flag > MAX_PAGE_COUNT_FLAG) { +#if WASM_ENABLE_SHARED_MEMORY == 0 + if (mem_flag & SHARED_MEMORY_FLAG) { + LOG_VERBOSE("shared memory flag was found, please enable shared " + "memory, lib-pthread or lib-wasi-threads"); + return false; + } +#endif +#if WASM_ENABLE_MEMORY64 == 0 + if (mem_flag & MEMORY64_FLAG) { + LOG_VERBOSE("memory64 flag was found, please enable memory64"); + return false; + } +#endif + } + + if (mem_flag > MAX_PAGE_COUNT_FLAG + SHARED_MEMORY_FLAG + MEMORY64_FLAG) { + return false; + } + else if ((mem_flag & SHARED_MEMORY_FLAG) + && !(mem_flag & MAX_PAGE_COUNT_FLAG)) { + return false; + } + + return true; +} + static bool load_memory_import(const uint8 **p_buf, const uint8 *buf_end, WASMModule *parent_module, const char *sub_module_name, @@ -695,20 +740,28 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end, uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT / DEFAULT_NUM_BYTES_PER_PAGE; #else - uint32 max_page_count = DEFAULT_MAX_PAGES; + uint32 max_page_count; #endif /* WASM_ENABLE_APP_FRAMEWORK */ - uint32 declare_max_page_count_flag = 0; + uint32 mem_flag = 0; + bool is_memory64 = false; uint32 declare_init_page_count = 0; uint32 declare_max_page_count = 0; - read_leb_uint32(p, p_end, declare_max_page_count_flag); - read_leb_uint32(p, p_end, declare_init_page_count); - bh_assert(declare_init_page_count <= 65536); + read_leb_uint32(p, p_end, mem_flag); + bh_assert(check_memory_flag(mem_flag)); - if (declare_max_page_count_flag & 1) { +#if WASM_ENABLE_APP_FRAMEWORK == 0 + is_memory64 = mem_flag & MEMORY64_FLAG; + max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; +#endif + + read_leb_uint32(p, p_end, declare_init_page_count); + bh_assert(declare_init_page_count <= max_page_count); + + if (mem_flag & MAX_PAGE_COUNT_FLAG) { read_leb_uint32(p, p_end, declare_max_page_count); bh_assert(declare_init_page_count <= declare_max_page_count); - bh_assert(declare_max_page_count <= 65536); + bh_assert(declare_max_page_count <= max_page_count); if (declare_max_page_count > max_page_count) { declare_max_page_count = max_page_count; } @@ -719,12 +772,13 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end, } /* now we believe all declaration are ok */ - memory->flags = declare_max_page_count_flag; + memory->flags = mem_flag; memory->init_page_count = declare_init_page_count; memory->max_page_count = declare_max_page_count; memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; *p_buf = p; + (void)check_memory_flag; return true; } @@ -811,26 +865,28 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT / DEFAULT_NUM_BYTES_PER_PAGE; #else - uint32 max_page_count = DEFAULT_MAX_PAGES; + uint32 max_page_count; + bool is_memory64 = false; #endif p_org = p; read_leb_uint32(p, p_end, memory->flags); bh_assert(p - p_org <= 1); (void)p_org; -#if WASM_ENABLE_SHARED_MEMORY == 0 - bh_assert(memory->flags <= 1); -#else - bh_assert(memory->flags <= 3 && memory->flags != 2); + bh_assert(check_memory_flag(memory->flags)); + +#if WASM_ENABLE_APP_FRAMEWORK == 0 + is_memory64 = memory->flags & MEMORY64_FLAG; + max_page_count = is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; #endif read_leb_uint32(p, p_end, memory->init_page_count); - bh_assert(memory->init_page_count <= 65536); + bh_assert(memory->init_page_count <= max_page_count); if (memory->flags & 1) { read_leb_uint32(p, p_end, memory->max_page_count); bh_assert(memory->init_page_count <= memory->max_page_count); - bh_assert(memory->max_page_count <= 65536); + bh_assert(memory->max_page_count <= max_page_count); if (memory->max_page_count > max_page_count) memory->max_page_count = max_page_count; } @@ -842,6 +898,7 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, memory->num_bytes_per_page = DEFAULT_NUM_BYTES_PER_PAGE; *p_buf = p; + (void)check_memory_flag; return true; } @@ -1704,6 +1761,7 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end, bool is_passive = false; uint32 mem_flag; #endif + uint8 mem_offset_type; read_leb_uint32(p, p_end, data_seg_count); @@ -1750,11 +1808,35 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end, < module->import_memory_count + module->memory_count); #endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_BULK_MEMORY != 0 + if (!is_passive) +#endif /* WASM_ENABLE_BULK_MEMORY */ + { +#if WASM_ENABLE_MEMORY64 != 0 + /* This memory_flag is from memory instead of data segment */ + uint8 memory_flag; + if (module->import_memory_count > 0) { + memory_flag = + module->import_memories[mem_index].u.memory.flags; + } + else { + memory_flag = + module + ->memories[mem_index - module->import_memory_count] + .flags; + } + mem_offset_type = memory_flag & MEMORY64_FLAG ? VALUE_TYPE_I64 + : VALUE_TYPE_I32; +#else + mem_offset_type = VALUE_TYPE_I32; +#endif /* WASM_ENABLE_MEMORY64 */ + } + #if WASM_ENABLE_BULK_MEMORY != 0 if (!is_passive) #endif if (!load_init_expr(module, &p, p_end, &init_expr, - VALUE_TYPE_I32, error_buf, error_buf_size)) + mem_offset_type, error_buf, error_buf_size)) return false; read_leb_uint32(p, p_end, data_seg_len); @@ -3532,8 +3614,8 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, case WASM_OP_I64_STORE8: case WASM_OP_I64_STORE16: case WASM_OP_I64_STORE32: - skip_leb_uint32(p, p_end); /* align */ - skip_leb_uint32(p, p_end); /* offset */ + skip_leb_uint32(p, p_end); /* align */ + skip_leb_mem_offset(p, p_end); /* offset */ break; case WASM_OP_MEMORY_SIZE: @@ -3748,6 +3830,7 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, #if WASM_ENABLE_SHARED_MEMORY != 0 case WASM_OP_ATOMIC_PREFIX: { + /* TODO: memory64 offset type changes */ uint32 opcode1; /* atomic_op (u32_leb) + memarg (2 u32_leb) */ @@ -3757,8 +3840,8 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, opcode = (uint8)opcode1; if (opcode != WASM_OP_ATOMIC_FENCE) { - skip_leb_uint32(p, p_end); /* align */ - skip_leb_uint32(p, p_end); /* offset */ + skip_leb_uint32(p, p_end); /* align */ + skip_leb_mem_offset(p, p_end); /* offset */ } else { /* atomic.fence doesn't have memarg */ @@ -5075,6 +5158,11 @@ fail: goto fail; \ } while (0) +#define PUSH_MEM_OFFSET() PUSH_OFFSET_TYPE(mem_offset_type) +#define PUSH_PAGE_COUNT() PUSH_MEM_OFFSET() + +#define POP_MEM_OFFSET() POP_OFFSET_TYPE(mem_offset_type) + #define POP_AND_PUSH(type_pop, type_push) \ do { \ if (!(wasm_loader_push_pop_frame_ref_offset( \ @@ -5129,6 +5217,15 @@ fail: goto fail; \ } while (0) +#define PUSH_MEM_OFFSET() \ + do { \ + if (!(wasm_loader_push_frame_ref(loader_ctx, mem_offset_type, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + +#define PUSH_PAGE_COUNT() PUSH_MEM_OFFSET() + #define POP_I32() \ do { \ if (!(wasm_loader_pop_frame_ref(loader_ctx, VALUE_TYPE_I32, error_buf, \ @@ -5164,6 +5261,13 @@ fail: goto fail; \ } while (0) +#define POP_MEM_OFFSET() \ + do { \ + if (!(wasm_loader_pop_frame_ref(loader_ctx, mem_offset_type, \ + error_buf, error_buf_size))) \ + goto fail; \ + } while (0) + #define POP_AND_PUSH(type_pop, type_push) \ do { \ if (!(wasm_loader_push_pop_frame_ref(loader_ctx, 1, type_push, \ @@ -5774,10 +5878,11 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, { uint8 *p = func->code, *p_end = func->code + func->code_size, *p_org; uint32 param_count, local_count, global_count; - uint8 *param_types, *local_types, local_type, global_type; + uint8 *param_types, *local_types, local_type, global_type, mem_offset_type; BlockType func_block_type; uint16 *local_offsets, local_offset; - uint32 count, local_idx, global_idx, u32, align, mem_offset, i; + uint32 count, local_idx, global_idx, u32, align, i; + mem_offset_t mem_offset; int32 i32, i32_const = 0; int64 i64_const; uint8 opcode, u8; @@ -5799,6 +5904,19 @@ wasm_loader_prepare_bytecode(WASMModule *module, WASMFunction *func, LOG_OP("\nProcessing func | [%d] params | [%d] locals | [%d] return\n", func->param_cell_num, func->local_cell_num, func->ret_cell_num); #endif +#if WASM_ENABLE_MEMORY64 != 0 + bool is_memory64 = false; + /* TODO: multi-memories for now assuming the memory idx type is consistent + * across multi-memories */ + if (module->import_memory_count > 0) + is_memory64 = module->import_memories[0].u.memory.flags & MEMORY64_FLAG; + else if (module->memory_count > 0) + is_memory64 = module->memories[0].flags & MEMORY64_FLAG; + + mem_offset_type = is_memory64 ? VALUE_TYPE_I64 : VALUE_TYPE_I32; +#else + mem_offset_type = VALUE_TYPE_I32; +#endif global_count = module->import_global_count + module->global_count; @@ -7107,8 +7225,8 @@ re_scan: } #endif CHECK_MEMORY(); - read_leb_uint32(p, p_end, align); /* align */ - read_leb_uint32(p, p_end, mem_offset); /* offset */ + read_leb_uint32(p, p_end, align); /* align */ + read_leb_mem_offset(p, p_end, mem_offset); /* offset */ #if WASM_ENABLE_FAST_INTERP != 0 emit_uint32(loader_ctx, mem_offset); #endif @@ -7122,7 +7240,7 @@ re_scan: case WASM_OP_I32_LOAD8_U: case WASM_OP_I32_LOAD16_S: case WASM_OP_I32_LOAD16_U: - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32); break; case WASM_OP_I64_LOAD: case WASM_OP_I64_LOAD8_S: @@ -7131,35 +7249,35 @@ re_scan: case WASM_OP_I64_LOAD16_U: case WASM_OP_I64_LOAD32_S: case WASM_OP_I64_LOAD32_U: - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64); + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64); break; case WASM_OP_F32_LOAD: - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F32); + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F32); break; case WASM_OP_F64_LOAD: - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_F64); + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_F64); break; /* store */ case WASM_OP_I32_STORE: case WASM_OP_I32_STORE8: case WASM_OP_I32_STORE16: POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); break; case WASM_OP_I64_STORE: case WASM_OP_I64_STORE8: case WASM_OP_I64_STORE16: case WASM_OP_I64_STORE32: POP_I64(); - POP_I32(); + POP_MEM_OFFSET(); break; case WASM_OP_F32_STORE: POP_F32(); - POP_I32(); + POP_MEM_OFFSET(); break; case WASM_OP_F64_STORE: POP_F64(); - POP_I32(); + POP_MEM_OFFSET(); break; default: break; @@ -7172,7 +7290,7 @@ re_scan: /* reserved byte 0x00 */ bh_assert(*p == 0x00); p++; - PUSH_I32(); + PUSH_PAGE_COUNT(); module->possible_memory_grow = true; #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 @@ -7185,7 +7303,7 @@ re_scan: /* reserved byte 0x00 */ bh_assert(*p == 0x00); p++; - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + POP_AND_PUSH(mem_offset_type, mem_offset_type); module->possible_memory_grow = true; #if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ @@ -7536,7 +7654,7 @@ re_scan: POP_I32(); POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 func->has_memory_operations = true; #endif @@ -7565,9 +7683,9 @@ re_scan: + module->memory_count > 0); - POP_I32(); - POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); + POP_MEM_OFFSET(); + POP_MEM_OFFSET(); #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 func->has_memory_operations = true; #endif @@ -7582,9 +7700,9 @@ re_scan: + module->memory_count > 0); + POP_MEM_OFFSET(); POP_I32(); - POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 func->has_memory_operations = true; #endif @@ -7748,8 +7866,8 @@ re_scan: #endif if (opcode1 != WASM_OP_ATOMIC_FENCE) { CHECK_MEMORY(); - read_leb_uint32(p, p_end, align); /* align */ - read_leb_uint32(p, p_end, mem_offset); /* offset */ + read_leb_uint32(p, p_end, align); /* align */ + read_leb_mem_offset(p, p_end, mem_offset); /* offset */ #if WASM_ENABLE_FAST_INTERP != 0 emit_uint32(loader_ctx, mem_offset); #endif @@ -7759,18 +7877,20 @@ re_scan: #endif switch (opcode1) { case WASM_OP_ATOMIC_NOTIFY: - POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); break; case WASM_OP_ATOMIC_WAIT32: POP_I64(); POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); PUSH_I32(); break; case WASM_OP_ATOMIC_WAIT64: POP_I64(); POP_I64(); - POP_I32(); + POP_MEM_OFFSET(); PUSH_I32(); break; case WASM_OP_ATOMIC_FENCE: @@ -7781,26 +7901,26 @@ re_scan: case WASM_OP_ATOMIC_I32_LOAD: case WASM_OP_ATOMIC_I32_LOAD8_U: case WASM_OP_ATOMIC_I32_LOAD16_U: - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I32); break; case WASM_OP_ATOMIC_I32_STORE: case WASM_OP_ATOMIC_I32_STORE8: case WASM_OP_ATOMIC_I32_STORE16: POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); break; case WASM_OP_ATOMIC_I64_LOAD: case WASM_OP_ATOMIC_I64_LOAD8_U: case WASM_OP_ATOMIC_I64_LOAD16_U: case WASM_OP_ATOMIC_I64_LOAD32_U: - POP_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I64); + POP_AND_PUSH(mem_offset_type, VALUE_TYPE_I64); break; case WASM_OP_ATOMIC_I64_STORE: case WASM_OP_ATOMIC_I64_STORE8: case WASM_OP_ATOMIC_I64_STORE16: case WASM_OP_ATOMIC_I64_STORE32: POP_I64(); - POP_I32(); + POP_MEM_OFFSET(); break; case WASM_OP_ATOMIC_RMW_I32_ADD: case WASM_OP_ATOMIC_RMW_I32_ADD8_U: @@ -7820,7 +7940,9 @@ re_scan: case WASM_OP_ATOMIC_RMW_I32_XCHG: case WASM_OP_ATOMIC_RMW_I32_XCHG8_U: case WASM_OP_ATOMIC_RMW_I32_XCHG16_U: - POP2_AND_PUSH(VALUE_TYPE_I32, VALUE_TYPE_I32); + POP_I32(); + POP_MEM_OFFSET(); + PUSH_I32(); break; case WASM_OP_ATOMIC_RMW_I64_ADD: case WASM_OP_ATOMIC_RMW_I64_ADD8_U: @@ -7847,7 +7969,7 @@ re_scan: case WASM_OP_ATOMIC_RMW_I64_XCHG16_U: case WASM_OP_ATOMIC_RMW_I64_XCHG32_U: POP_I64(); - POP_I32(); + POP_MEM_OFFSET(); PUSH_I64(); break; case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: @@ -7855,7 +7977,7 @@ re_scan: case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: POP_I32(); POP_I32(); - POP_I32(); + POP_MEM_OFFSET(); PUSH_I32(); break; case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: @@ -7864,7 +7986,7 @@ re_scan: case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: POP_I64(); POP_I64(); - POP_I32(); + POP_MEM_OFFSET(); PUSH_I64(); break; default: diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index f80d182df..71e7d54ee 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -162,7 +162,7 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, char *error_buf, uint32 error_buf_size) { WASMModule *module = module_inst->module; - uint32 inc_page_count, global_idx; + uint32 inc_page_count, global_idx, default_max_page; uint32 bytes_of_last_page, bytes_to_page_end; uint64 aux_heap_base, heap_offset = (uint64)num_bytes_per_page * init_page_count; @@ -171,7 +171,7 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, bool is_shared_memory = false; #if WASM_ENABLE_SHARED_MEMORY != 0 - is_shared_memory = flags & 0x02 ? true : false; + is_shared_memory = flags & SHARED_MEMORY_FLAG ? true : false; /* shared memory */ if (is_shared_memory && parent != NULL) { @@ -186,6 +186,14 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, (void)flags; #endif /* end of WASM_ENABLE_SHARED_MEMORY */ +#if WASM_ENABLE_MEMORY64 != 0 + if (flags & MEMORY64_FLAG) { + memory->is_memory64 = 1; + } +#endif + default_max_page = + memory->is_memory64 ? DEFAULT_MEM64_MAX_PAGES : DEFAULT_MAX_PAGES; + if (heap_size > 0 && module_inst->module->malloc_function != (uint32)-1 && module_inst->module->free_function != (uint32)-1) { /* Disable app heap, use malloc/free function exported @@ -195,7 +203,8 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, /* If initial memory is the largest size allowed, disallowing insert host * managed heap */ - if (heap_size > 0 && heap_offset == MAX_LINEAR_MEMORY_SIZE) { + if (heap_size > 0 + && heap_offset == GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)) { set_error_buf(error_buf, error_buf_size, "failed to insert app heap into linear memory, " "try using `--heap-size=0` option"); @@ -253,8 +262,18 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, && global_idx < module_inst->e->global_count); global_addr = module_inst->global_data + module_inst->e->globals[global_idx].data_offset; - *(uint32 *)global_addr = (uint32)aux_heap_base; - LOG_VERBOSE("Reset __heap_base global to %u", aux_heap_base); +#if WASM_ENABLE_MEMORY64 != 0 + if (memory->is_memory64) { + /* For memory64, the global value should be i64 */ + *(uint64 *)global_addr = aux_heap_base; + } + else +#endif + { + /* For memory32, the global value should be i32 */ + *(uint32 *)global_addr = (uint32)aux_heap_base; + } + LOG_VERBOSE("Reset __heap_base global to %lu", aux_heap_base); } else { /* Insert app heap before new page */ @@ -267,14 +286,15 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, } init_page_count += inc_page_count; max_page_count += inc_page_count; - if (init_page_count > DEFAULT_MAX_PAGES) { + if (init_page_count > default_max_page) { set_error_buf(error_buf, error_buf_size, "failed to insert app heap into linear memory, " "try using `--heap-size=0` option"); return NULL; } - if (max_page_count > DEFAULT_MAX_PAGES) - max_page_count = DEFAULT_MAX_PAGES; + + if (max_page_count > default_max_page) + max_page_count = default_max_page; } LOG_VERBOSE("Memory instantiate:"); @@ -283,14 +303,16 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, LOG_VERBOSE(" heap offset: %u, heap size: %d\n", heap_offset, heap_size); max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; - bh_assert(max_memory_data_size <= MAX_LINEAR_MEMORY_SIZE); + bh_assert(max_memory_data_size + <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); (void)max_memory_data_size; bh_assert(memory != NULL); if (wasm_allocate_linear_memory(&memory->memory_data, is_shared_memory, - num_bytes_per_page, init_page_count, - max_page_count, &memory_data_size) + memory->is_memory64, num_bytes_per_page, + init_page_count, max_page_count, + &memory_data_size) != BHT_OK) { set_error_buf(error_buf, error_buf_size, "allocate linear memory failed"); @@ -1947,7 +1969,8 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, WASMGlobalInstance *globals = NULL, *global; WASMTableInstance *first_table; uint32 global_count, i; - uint32 base_offset, length, extra_info_offset; + uint32 length, extra_info_offset; + mem_offset_t base_offset; uint32 module_inst_struct_size = offsetof(WASMModuleInstance, global_table_data.bytes); uint64 module_inst_mem_inst_size; @@ -2305,10 +2328,12 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, (uint64)memory->num_bytes_per_page * memory->cur_page_count; bh_assert(memory_data || memory_size == 0); - bh_assert(data_seg->base_offset.init_expr_type - == INIT_EXPR_TYPE_I32_CONST - || data_seg->base_offset.init_expr_type - == INIT_EXPR_TYPE_GET_GLOBAL); + bh_assert( + data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL + || (data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_I32_CONST + && !memory->is_memory64) + || (data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_I64_CONST + && memory->is_memory64)); if (data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { if (!check_global_init_expr(module, @@ -2319,17 +2344,37 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, if (!globals || globals[data_seg->base_offset.u.global_index].type - != VALUE_TYPE_I32) { + != (memory->is_memory64 ? VALUE_TYPE_I64 + : VALUE_TYPE_I32)) { set_error_buf(error_buf, error_buf_size, "data segment does not fit"); goto fail; } - base_offset = - globals[data_seg->base_offset.u.global_index].initial_value.i32; +#if WASM_ENABLE_MEMORY64 != 0 + if (memory->is_memory64) { + base_offset = + (uint64)globals[data_seg->base_offset.u.global_index] + .initial_value.i64; + } + else +#endif + { + base_offset = + (uint32)globals[data_seg->base_offset.u.global_index] + .initial_value.i32; + } } else { - base_offset = (uint32)data_seg->base_offset.u.i32; +#if WASM_ENABLE_MEMORY64 != 0 + if (memory->is_memory64) { + base_offset = (uint64)data_seg->base_offset.u.i64; + } + else +#endif + { + base_offset = (uint32)data_seg->base_offset.u.i32; + } } /* check offset */ diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 5933e6a9c..3b01f05cd 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -103,7 +103,7 @@ struct WASMMemoryInstance { /* Whether the memory is shared */ uint8 is_shared_memory; - /* TODO: Memory64 whether the memory has 64-bit memory addresses */ + /* Whether the memory has 64-bit memory addresses */ uint8 is_memory64; /* Reference count of the memory instance: diff --git a/core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c b/core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c index 7aa3444f9..2ac381033 100644 --- a/core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c +++ b/core/iwasm/libraries/libc-builtin/libc_builtin_wrapper.c @@ -17,7 +17,7 @@ void wasm_runtime_set_exception(wasm_module_inst_t module, const char *exception); uint32 -wasm_runtime_module_realloc(wasm_module_inst_t module, uint32 ptr, uint32 size, +wasm_runtime_module_realloc(wasm_module_inst_t module, uint64 ptr, uint64 size, void **p_native_addr); /* clang-format off */ diff --git a/core/shared/platform/common/posix/posix_memmap.c b/core/shared/platform/common/posix/posix_memmap.c index c9a7e5897..c76abf137 100644 --- a/core/shared/platform/common/posix/posix_memmap.c +++ b/core/shared/platform/common/posix/posix_memmap.c @@ -65,9 +65,11 @@ os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) /* integer overflow */ return NULL; +#if WASM_ENABLE_MEMORY64 == 0 if (request_size > 16 * (uint64)UINT32_MAX) - /* at most 16 G is allowed */ + /* at most 64 G is allowed */ return NULL; +#endif if (prot & MMAP_PROT_READ) map_prot |= PROT_READ; diff --git a/tests/wamr-test-suites/spec-test-script/all.py b/tests/wamr-test-suites/spec-test-script/all.py index 8027abde0..98f5c1e63 100644 --- a/tests/wamr-test-suites/spec-test-script/all.py +++ b/tests/wamr-test-suites/spec-test-script/all.py @@ -14,7 +14,7 @@ import time """ The script itself has to be put under the same directory with the "spec". -To run a single non-GC case with interpreter mode: +To run a single non-GC and non-memory64 case with interpreter mode: cd workspace python3 runtest.py --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \ spec/test/core/xxx.wast @@ -22,7 +22,7 @@ To run a single non-GC case with aot mode: cd workspace python3 runtest.py --aot --wast2wasm wabt/bin/wat2wasm --interpreter iwasm \ --aot-compiler wamrc spec/test/core/xxx.wast -To run a single GC case: +To run a single GC case or single memory64 case: cd workspace python3 runtest.py --wast2wasm spec/interpreter/wasm --interpreter iwasm \ --aot-compiler wamrc --gc spec/test/core/xxx.wast @@ -78,6 +78,7 @@ def ignore_the_case( multi_thread_flag=False, simd_flag=False, gc_flag=False, + memory64_flag=False, xip_flag=False, eh_flag=False, qemu_flag=False, @@ -162,6 +163,7 @@ def test_case( clean_up_flag=True, verbose_flag=True, gc_flag=False, + memory64_flag=False, qemu_flag=False, qemu_firmware="", log="", @@ -169,7 +171,7 @@ def test_case( ): CMD = [sys.executable, "runtest.py"] CMD.append("--wast2wasm") - CMD.append(WAST2WASM_CMD if not gc_flag else SPEC_INTERPRETER_CMD) + CMD.append(WAST2WASM_CMD if not gc_flag and not memory64_flag else SPEC_INTERPRETER_CMD) CMD.append("--interpreter") if sgx_flag: CMD.append(IWASM_SGX_CMD) @@ -217,6 +219,9 @@ def test_case( if gc_flag: CMD.append("--gc") + if memory64_flag: + CMD.append("--memory64") + if log != "": CMD.append("--log-dir") CMD.append(log) @@ -283,6 +288,7 @@ def test_suite( clean_up_flag=True, verbose_flag=True, gc_flag=False, + memory64_flag=False, parl_flag=False, qemu_flag=False, qemu_firmware="", @@ -325,6 +331,7 @@ def test_suite( multi_thread_flag, simd_flag, gc_flag, + memory64_flag, xip_flag, eh_flag, qemu_flag, @@ -357,6 +364,7 @@ def test_suite( clean_up_flag, verbose_flag, gc_flag, + memory64_flag, qemu_flag, qemu_firmware, log, @@ -382,6 +390,7 @@ def test_suite( else: print(f"----- Run the whole spec test suite -----") for case_path in case_list: + print(case_path) try: test_case( str(case_path), @@ -396,6 +405,7 @@ def test_suite( clean_up_flag, verbose_flag, gc_flag, + memory64_flag, qemu_flag, qemu_firmware, log, @@ -521,6 +531,13 @@ def main(): dest="gc_flag", help="Running with GC feature", ) + parser.add_argument( + "--memory64", + action="store_true", + default=False, + dest="memory64_flag", + help="Running with memory64 feature", + ) parser.add_argument( "cases", metavar="path_to__case", @@ -563,6 +580,7 @@ def main(): options.clean_up_flag, options.verbose_flag, options.gc_flag, + options.memory64_flag, options.parl_flag, options.qemu_flag, options.qemu_firmware, @@ -589,6 +607,7 @@ def main(): options.clean_up_flag, options.verbose_flag, options.gc_flag, + options.memory64_flag, options.qemu_flag, options.qemu_firmware, options.log, diff --git a/tests/wamr-test-suites/spec-test-script/memory64.patch b/tests/wamr-test-suites/spec-test-script/memory64.patch new file mode 100644 index 000000000..739a1df60 --- /dev/null +++ b/tests/wamr-test-suites/spec-test-script/memory64.patch @@ -0,0 +1,28 @@ +diff --git a/test/core/memory.wast b/test/core/memory.wast +index 1dd5b84..497b69f 100644 +--- a/test/core/memory.wast ++++ b/test/core/memory.wast +@@ -76,17 +76,17 @@ + "memory size must be at most 65536 pages (4GiB)" + ) + +-(assert_invalid ++(assert_malformed + (module quote "(memory 0x1_0000_0000)") +- "memory size must be at most 65536 pages (4GiB)" ++ "i32 constant out of range" + ) +-(assert_invalid ++(assert_malformed + (module quote "(memory 0x1_0000_0000 0x1_0000_0000)") +- "memory size must be at most 65536 pages (4GiB)" ++ "i32 constant out of range" + ) +-(assert_invalid ++(assert_malformed + (module quote "(memory 0 0x1_0000_0000)") +- "memory size must be at most 65536 pages (4GiB)" ++ "i32 constant out of range" + ) + + (module diff --git a/tests/wamr-test-suites/spec-test-script/runtest.py b/tests/wamr-test-suites/spec-test-script/runtest.py index 344e4fd44..13229d977 100755 --- a/tests/wamr-test-suites/spec-test-script/runtest.py +++ b/tests/wamr-test-suites/spec-test-script/runtest.py @@ -313,6 +313,9 @@ parser.add_argument('--multi-thread', default=False, action='store_true', parser.add_argument('--gc', default=False, action='store_true', help='Test with GC') +parser.add_argument('--memory64', default=False, action='store_true', + help='Test with Memory64') + parser.add_argument('--qemu', default=False, action='store_true', help="Enable QEMU") @@ -1071,7 +1074,7 @@ def compile_wast_to_wasm(form, wast_tempfile, wasm_tempfile, opts): log("Compiling WASM to '%s'" % wasm_tempfile) # default arguments - if opts.gc: + if opts.gc or opts.memory64: cmd = [opts.wast2wasm, "-u", "-d", wast_tempfile, "-o", wasm_tempfile] elif opts.eh: cmd = [opts.wast2wasm, "--enable-thread", "--no-check", "--enable-exceptions", "--enable-tail-call", wast_tempfile, "-o", wasm_tempfile ] @@ -1116,6 +1119,9 @@ def compile_wasm_to_aot(wasm_tempfile, aot_tempfile, runner, opts, r, output = ' cmd.append("--enable-gc") cmd.append("--enable-tail-call") + if opts.memory64: + cmd.append("--enable-memory64") + if output == 'object': cmd.append("--format=object") elif output == 'ir': diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index 933583816..0c56acac4 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -23,6 +23,7 @@ function help() echo "-p enable multi thread feature" echo "-S enable SIMD feature" echo "-G enable GC feature" + echo "-W enable memory64 feature" echo "-X enable XIP feature" echo "-e enable exception handling" echo "-x test SGX" @@ -50,6 +51,7 @@ ENABLE_MULTI_THREAD=0 COLLECT_CODE_COVERAGE=0 ENABLE_SIMD=0 ENABLE_GC=0 +ENABLE_MEMORY64=0 ENABLE_XIP=0 ENABLE_EH=0 ENABLE_DEBUG_VERSION=0 @@ -72,7 +74,7 @@ WASI_TESTSUITE_COMMIT="ee807fc551978490bf1c277059aabfa1e589a6c2" TARGET_LIST=("AARCH64" "AARCH64_VFP" "ARMV7" "ARMV7_VFP" "THUMBV7" "THUMBV7_VFP" \ "RISCV32" "RISCV32_ILP32F" "RISCV32_ILP32D" "RISCV64" "RISCV64_LP64F" "RISCV64_LP64D") -while getopts ":s:cabgvt:m:MCpSXexwPGQF:j:T:" opt +while getopts ":s:cabgvt:m:MCpSXexwWPGQF:j:T:" opt do OPT_PARSED="TRUE" case $opt in @@ -131,6 +133,10 @@ do echo "enable multi module feature" ENABLE_MULTI_MODULE=1 ;; + W) + echo "enable wasm64(memory64) feature" + ENABLE_MEMORY64=1 + ;; C) echo "enable code coverage" COLLECT_CODE_COVERAGE=1 @@ -478,6 +484,29 @@ function spec_test() popd fi + # update memory64 cases + if [[ ${ENABLE_MEMORY64} == 1 ]]; then + echo "checkout spec for memory64 proposal" + + popd + rm -fr spec + # check spec test cases for memory64 + git clone -b main --single-branch https://github.com/WebAssembly/memory64.git spec + pushd spec + + git restore . && git clean -ffd . + # Reset to commit: "Merge remote-tracking branch 'upstream/main' into merge2" + git reset --hard 48e69f394869c55b7bbe14ac963c09f4605490b6 + git checkout 044d0d2e77bdcbe891f7e0b9dd2ac01d56435f0b -- test/core/elem.wast + git apply ../../spec-test-script/ignore_cases.patch + git apply ../../spec-test-script/memory64.patch + + echo "compile the reference intepreter" + pushd interpreter + make + popd + fi + popd echo $(pwd) @@ -488,7 +517,7 @@ function spec_test() local ARGS_FOR_SPEC_TEST="" - # multi-module only enable in interp mode + # multi-module only enable in interp mode and aot mode if [[ 1 == ${ENABLE_MULTI_MODULE} ]]; then if [[ $1 == 'classic-interp' || $1 == 'fast-interp' || $1 == 'aot' ]]; then ARGS_FOR_SPEC_TEST+="-M " @@ -537,6 +566,13 @@ function spec_test() ARGS_FOR_SPEC_TEST+="--gc " fi + # wasm64(memory64) is only enabled in interp and aot mode + if [[ 1 == ${ENABLE_MEMORY64} ]]; then + if [[ $1 == 'classic-interp' || $1 == 'aot' ]]; then + ARGS_FOR_SPEC_TEST+="--memory64 " + fi + fi + if [[ ${ENABLE_QEMU} == 1 ]]; then ARGS_FOR_SPEC_TEST+="--qemu " ARGS_FOR_SPEC_TEST+="--qemu-firmware ${QEMU_FIRMWARE} " @@ -833,6 +869,12 @@ function trigger() EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_MULTI_MODULE=0" fi + if [[ ${ENABLE_MEMORY64} == 1 ]];then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_MEMORY64=1" + else + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_MEMORY64=0" + fi + if [[ ${ENABLE_MULTI_THREAD} == 1 ]];then EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_LIB_PTHREAD=1" fi From 4806e4e298c42bee9d35c542d8a7b1ceaef86c1c Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 2 Apr 2024 17:00:43 +0900 Subject: [PATCH 09/36] LLVM 19: Switch to debug records (#3272) References: https://llvm.org/docs/RemoveDIsDebugInfo.html https://github.com/llvm/llvm-project/pull/86529 --- core/iwasm/compilation/aot_llvm.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 79a39d06a..dfb12a7f4 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -2547,6 +2547,9 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) aot_set_last_error("create LLVM module failed."); goto fail; } +#if LLVM_VERSION_MAJOR >= 19 + LLVMSetIsNewDbgInfoFormat(comp_ctx->module, true); +#endif #if WASM_ENABLE_LINUX_PERF != 0 if (wasm_runtime_get_linux_perf()) { From dae09c0e03d44e8a19d49b503e9c5ba5f0fb564e Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Wed, 3 Apr 2024 18:10:46 +0900 Subject: [PATCH 10/36] aot debug: Fix a NULL dereference (#3274) It happens on eg. a C function taking a structure argument. --- core/iwasm/compilation/debug/dwarf_extractor.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/iwasm/compilation/debug/dwarf_extractor.cpp b/core/iwasm/compilation/debug/dwarf_extractor.cpp index da33fc432..99b4e7f5e 100644 --- a/core/iwasm/compilation/debug/dwarf_extractor.cpp +++ b/core/iwasm/compilation/debug/dwarf_extractor.cpp @@ -313,6 +313,17 @@ lldb_function_to_function_dbi(const AOTCompContext *comp_ctx, if (function_arg_type.IsValid()) { ParamTypes[function_arg_idx + 1] = lldb_type_to_type_dbi(comp_ctx, function_arg_type); + if (ParamTypes[function_arg_idx + 1] == NULL) { + LOG_WARNING( + "func %s arg %" PRIu32 + " has a type not implemented by lldb_type_to_type_dbi", + function_name, function_arg_idx); + } + } + else { + LOG_WARNING("func %s arg %" PRIu32 ": GetTypeAtIndex failed", + function_name, function_arg_idx); + ParamTypes[function_arg_idx + 1] = NULL; } } @@ -399,7 +410,7 @@ lldb_function_to_function_dbi(const AOTCompContext *comp_ctx, ++function_arg_idx) { uint32_t variable_idx = variable_offset + function_arg_idx; SBValue variable(variable_list.GetValueAtIndex(variable_idx)); - if (variable.IsValid()) { + if (variable.IsValid() && ParamTypes[function_arg_idx + 1] != NULL) { SBDeclaration dec(variable.GetDeclaration()); auto valtype = variable.GetType(); LLVMMetadataRef ParamLocation = LLVMDIBuilderCreateDebugLocation( From b4cab84e5f6006185f44db2c66fe389850e5f4b9 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Wed, 3 Apr 2024 19:03:31 +0900 Subject: [PATCH 11/36] aot debug: Process lldb_function_to_function_dbi only for C (#3278) This is a workaroud for: https://github.com/bytecodealliance/wasm-micro-runtime/issues/3187 https://github.com/bytecodealliance/wasm-micro-runtime/issues/3163 --- .../compilation/debug/dwarf_extractor.cpp | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/core/iwasm/compilation/debug/dwarf_extractor.cpp b/core/iwasm/compilation/debug/dwarf_extractor.cpp index 99b4e7f5e..5a44206e1 100644 --- a/core/iwasm/compilation/debug/dwarf_extractor.cpp +++ b/core/iwasm/compilation/debug/dwarf_extractor.cpp @@ -295,6 +295,28 @@ lldb_function_to_function_dbi(const AOTCompContext *comp_ctx, const size_t num_function_args = function_args.GetSize(); dwarf_extractor *extractor; + /* + * Process only known languages. + * We have a few assumptions which might not be true for non-C functions. + * + * At least it's known broken for C++ and Rust: + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/3187 + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/3163 + */ + LanguageType language_type = function.GetLanguage(); + switch (language_type) { + case eLanguageTypeC89: + case eLanguageTypeC: + case eLanguageTypeC99: + case eLanguageTypeC11: + case eLanguageTypeC17: + break; + default: + LOG_WARNING("func %s has unsuppoted language_type 0x%x", + function_name, (int)language_type); + return NULL; + } + if (!(extractor = TO_EXTACTOR(comp_ctx->comp_data->extractor))) return NULL; From 202abc19377d6f3dd5648a5504205988c6ec4d93 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Thu, 4 Apr 2024 18:32:53 +0800 Subject: [PATCH 12/36] Small enhancement on addr2line.py (#3276) - If can't parse function name with dwarf information, use "name section" instead - Add introduction about using custom section --- samples/debug-tools/README.md | 21 ++++++++++++++- test-tools/addr2line/addr2line.py | 44 ++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/samples/debug-tools/README.md b/samples/debug-tools/README.md index 304778596..634d31197 100644 --- a/samples/debug-tools/README.md +++ b/samples/debug-tools/README.md @@ -63,7 +63,7 @@ The output should be something like: at wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:17:12 3: main at wasm-micro-runtime/samples/debug-tools/wasm-apps/trap.c:24:5 -4: +4: __main_void at unknown:?:? 5: _start ``` @@ -79,3 +79,22 @@ $ python3 ../../../test-tools/addr2line/addr2line.py \ --wasm-file wasm-apps/trap.wasm \ call_stack.txt --no-addr ``` + +### Another approach + +If the wasm file is with "name" section, it is able to output function name in the stack trace. To achieve that, need to enable `WAMR_BUILD_LOAD_CUSTOM_SECTION` and `WAMR_BUILD_CUSTOM_NAME_SECTION`. If using .aot file, need to add `--emit-custom-sections=name` into wamrc command line options. + +Then the output should be something like + +```text +#00: 0x0159 - c +#01: 0x01b2 - b +#02: 0x0200 - a +#03: 0x026b - main +#04: 0x236b - __main_void +#05: 0x011f - _start + +Exception: unreachable +``` + +Also, it is able to use *addr2line.py* to add file and line info to the stack trace. diff --git a/test-tools/addr2line/addr2line.py b/test-tools/addr2line/addr2line.py index 4502d5fec..594f8e19f 100644 --- a/test-tools/addr2line/addr2line.py +++ b/test-tools/addr2line/addr2line.py @@ -178,9 +178,13 @@ def parse_call_stack_line(line: str) -> tuple[str, str, str]: #00: 0x0a04 - $f18 => (00, 0x0a04, $f18) Old format: #00 $f18 => (00, _, $f18) + Text format (-DWAMR_BUILD_LOAD_CUSTOM_SECTION=1 -DWAMR_BUILD_CUSTOM_NAME_SECTION=1): + #02: 0x0200 - a => (02, 0x0200, a) + _start (always): + #05: 0x011f - _start => (05, 0x011f, _start) """ - # New format + # New format and Text format and _start PATTERN = r"#([0-9]+): 0x([0-9a-f]+) - (\S+)" m = re.match(PATTERN, line) if m is not None: @@ -261,8 +265,7 @@ def main(): if code_section_start == -1: return -1 - if args.no_addr: - function_index_to_name = parse_module_functions(wasm_objdump, args.wasm_file) + function_index_to_name = parse_module_functions(wasm_objdump, args.wasm_file) assert args.call_stack_file.exists() with open(args.call_stack_file, "rt", encoding="ascii") as f: @@ -272,15 +275,17 @@ def main(): continue splitted = parse_call_stack_line(line) - assert splitted is not None + if splitted is None: + print(f"{line}") + continue _, offset, index = splitted - if not index.startswith("$f"): # E.g. _start - print(f"{i}: {index}") - continue - index = index[2:] - if args.no_addr: + if not index.startswith("$f"): # E.g. _start or Text format + print(f"{i}: {index}") + continue + index = index[2:] + if index not in function_index_to_name: print(f"{i}: {line}") continue @@ -289,21 +294,30 @@ def main(): llvm_dwarf_dump, args.wasm_file, function_index_to_name[index] ) - _, funciton_file, function_line = line_info + _, function_file, function_line = line_info function_name = demangle(llvm_cxxfilt, function_index_to_name[index]) print(f"{i}: {function_name}") - print(f"\tat {funciton_file}:{function_line}") + print(f"\tat {function_file}:{function_line}") else: offset = int(offset, 16) offset = offset - code_section_start - line_info = get_line_info_from_function_addr( - llvm_dwarf_dump, args.wasm_file, offset + function_name, function_file, function_line, function_column = ( + get_line_info_from_function_addr( + llvm_dwarf_dump, args.wasm_file, offset + ) ) - function_name, funciton_file, function_line, function_column = line_info + # if can't parse function_name, use name section or + if function_name == "": + if index.startswith("$f"): + function_name = function_index_to_name.get(index[2:], index) + else: + function_name = index + function_name = demangle(llvm_cxxfilt, function_name) + print(f"{i}: {function_name}") - print(f"\tat {funciton_file}:{function_line}:{function_column}") + print(f"\tat {function_file}:{function_line}:{function_column}") return 0 From 53f0941ffae72106644db7e45af01123f3a7047b Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Sat, 6 Apr 2024 16:00:48 +0900 Subject: [PATCH 13/36] Revert "lldb_function_to_function_dbi: A hack to avoid crashing on C++ methods (#3190)" (#3281) This reverts commit 0e8d949440441b2b18d166934747d595a8f696ee. Because it doesn't make much sense anymore after we disabled debug info processing on C++ functions in: "aot debug: process lldb_function_to_function_dbi only for C". --- .../compilation/debug/dwarf_extractor.cpp | 35 +++++-------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/core/iwasm/compilation/debug/dwarf_extractor.cpp b/core/iwasm/compilation/debug/dwarf_extractor.cpp index 5a44206e1..e2e515ba0 100644 --- a/core/iwasm/compilation/debug/dwarf_extractor.cpp +++ b/core/iwasm/compilation/debug/dwarf_extractor.cpp @@ -387,27 +387,10 @@ lldb_function_to_function_dbi(const AOTCompContext *comp_ctx, LLVMDIBuilderCreateExpression(DIB, NULL, 0); auto variable_list = function.GetBlock().GetVariables(extractor->target, true, false, false); - unsigned int variable_offset = 0; if (num_function_args != variable_list.GetSize()) { - // A hack to detect C++ "this" pointer. - // - // REVISIT: is there a more reliable way? - // At the DWARF level, we can probably look at DW_AT_object_pointer - // and DW_AT_artificial. I'm not sure how it can be done via the - // LLDB API though. - if (num_function_args + 1 == variable_list.GetSize()) { - SBValue variable(variable_list.GetValueAtIndex(0)); - const char *varname = variable.GetName(); - if (varname != NULL && !strcmp(varname, "this")) { - variable_offset = 1; - } - } - if (!variable_offset) { - LOG_ERROR("function args number dismatch!:function %s %s value " - "number=%d, function args=%d", - function_name, function.GetMangledName(), - variable_list.GetSize(), num_function_args); - } + LOG_ERROR( + "function args number dismatch!:value number=%d, function args=%d", + variable_list.GetSize(), num_function_args); } LLVMMetadataRef ParamLocation = LLVMDIBuilderCreateDebugLocation( @@ -428,10 +411,9 @@ lldb_function_to_function_dbi(const AOTCompContext *comp_ctx, LLVMDIBuilderInsertDbgValueAtEnd(DIB, Param, ParamVar, ParamExpression, ParamLocation, block_curr); - for (uint32_t function_arg_idx = 0; function_arg_idx < num_function_args; - ++function_arg_idx) { - uint32_t variable_idx = variable_offset + function_arg_idx; - SBValue variable(variable_list.GetValueAtIndex(variable_idx)); + for (uint32_t function_arg_idx = 0; + function_arg_idx < variable_list.GetSize(); ++function_arg_idx) { + SBValue variable(variable_list.GetValueAtIndex(function_arg_idx)); if (variable.IsValid() && ParamTypes[function_arg_idx + 1] != NULL) { SBDeclaration dec(variable.GetDeclaration()); auto valtype = variable.GetType(); @@ -441,11 +423,12 @@ lldb_function_to_function_dbi(const AOTCompContext *comp_ctx, const char *varname = variable.GetName(); LLVMMetadataRef ParamVar = LLVMDIBuilderCreateParameterVariable( DIB, FunctionMetadata, varname, varname ? strlen(varname) : 0, - variable_idx + 1 + 1, + function_arg_idx + 1 + 1, File, // starts form 1, and 1 is exenv, dec.GetLine(), ParamTypes[function_arg_idx + 1], true, LLVMDIFlagZero); - LLVMValueRef Param = LLVMGetParam(func_ctx->func, variable_idx + 1); + LLVMValueRef Param = + LLVMGetParam(func_ctx->func, function_arg_idx + 1); LLVMDIBuilderInsertDbgValueAtEnd(DIB, Param, ParamVar, ParamExpression, ParamLocation, block_curr); From 2013f1f7d7499ae4a6891a40a69e11635dc04c60 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Sun, 7 Apr 2024 11:57:31 +0800 Subject: [PATCH 14/36] Fix warnings/issues reported in Windows and by CodeQL/Coverity (#3275) Fix the warnings and issues reported: - in Windows platform - by CodeQL static code analyzing - by Coverity static code analyzing And update CodeQL script to build exception handling and memory features. --- .github/workflows/codeql_buildscript.sh | 20 +++++ core/iwasm/aot/aot_loader.c | 19 ++++- core/iwasm/aot/aot_runtime.c | 39 +++++----- core/iwasm/common/gc/gc_type.c | 6 +- core/iwasm/common/wasm_c_api.c | 4 +- core/iwasm/common/wasm_memory.c | 8 +- core/iwasm/common/wasm_runtime_common.c | 15 ++-- core/iwasm/compilation/aot_compiler.c | 8 +- core/iwasm/compilation/aot_emit_control.c | 2 + core/iwasm/compilation/aot_llvm.c | 3 +- .../fast-jit/cg/x86-64/jit_codegen_x86_64.cpp | 6 +- core/iwasm/fast-jit/fe/jit_emit_memory.c | 10 ++- core/iwasm/interpreter/wasm.h | 4 +- core/iwasm/interpreter/wasm_interp_fast.c | 2 +- core/iwasm/interpreter/wasm_loader.c | 75 +++++++++++-------- core/iwasm/interpreter/wasm_mini_loader.c | 23 +++--- core/iwasm/interpreter/wasm_runtime.c | 31 +++++--- core/iwasm/libraries/debug-engine/handler.c | 4 +- .../lib-pthread/lib_pthread_wrapper.c | 3 +- .../src/blocking_op.h | 5 ++ core/shared/mem-alloc/ems/ems_alloc.c | 4 +- core/shared/mem-alloc/ems/ems_gc.c | 16 ++-- core/shared/mem-alloc/mem_alloc.c | 4 +- core/shared/utils/runtime_timer.c | 2 +- product-mini/platforms/posix/main.c | 3 +- product-mini/platforms/windows/main.c | 4 +- 26 files changed, 202 insertions(+), 118 deletions(-) diff --git a/.github/workflows/codeql_buildscript.sh b/.github/workflows/codeql_buildscript.sh index 34e0ddd79..ed717734e 100755 --- a/.github/workflows/codeql_buildscript.sh +++ b/.github/workflows/codeql_buildscript.sh @@ -101,6 +101,26 @@ if [[ $? != 0 ]]; then exit 1; fi +# build iwasm with exception handling enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_EXCE_HANDLING=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with exception handling enabled!" + exit 1; +fi + +# build iwasm with memory64 enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MEMORY64=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with memory64 enabled!" + exit 1; +fi + # build iwasm with hardware boundary check disabled cd ${WAMR_DIR}/product-mini/platforms/linux rm -rf build && mkdir build && cd build diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 1634a8977..9789b1744 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -1430,9 +1430,20 @@ load_table_init_data_list(const uint8 **p_buf, const uint8 *buf_end, read_uint64(buf, buf_end, init_expr_value); #if WASM_ENABLE_GC != 0 if (wasm_is_type_multi_byte_type(elem_type)) { - /* TODO: check ref_type */ - read_uint16(buf, buf_end, reftype.ref_ht_common.ref_type); - read_uint16(buf, buf_end, reftype.ref_ht_common.nullable); + uint16 ref_type, nullable; + read_uint16(buf, buf_end, ref_type); + if (elem_type != ref_type) { + set_error_buf(error_buf, error_buf_size, "invalid elem type"); + return false; + } + reftype.ref_ht_common.ref_type = (uint8)ref_type; + read_uint16(buf, buf_end, nullable); + if (nullable != 0 && nullable != 1) { + set_error_buf(error_buf, error_buf_size, + "invalid nullable value"); + return false; + } + reftype.ref_ht_common.nullable = (uint8)nullable; read_uint32(buf, buf_end, reftype.ref_ht_common.heap_type); } else @@ -4379,7 +4390,7 @@ aot_unload(AOTModule *module) } if (module->string_literal_ptrs) { - wasm_runtime_free(module->string_literal_ptrs); + wasm_runtime_free((void *)module->string_literal_ptrs); } } #endif diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index d10db89af..f8757fcc6 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -880,7 +880,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, global_addr = module_inst->global_data + module->globals[global_idx].data_offset; *(uint32 *)global_addr = (uint32)aux_heap_base; - LOG_VERBOSE("Reset __heap_base global to %u", aux_heap_base); + LOG_VERBOSE("Reset __heap_base global to %" PRIu64, aux_heap_base); } else { /* Insert app heap before new page */ @@ -906,9 +906,10 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, LOG_VERBOSE("Memory instantiate:"); LOG_VERBOSE(" page bytes: %u, init pages: %u, max pages: %u", num_bytes_per_page, init_page_count, max_page_count); - LOG_VERBOSE(" data offset: %u, stack size: %d", module->aux_data_end, - module->aux_stack_size); - LOG_VERBOSE(" heap offset: %u, heap size: %d\n", heap_offset, heap_size); + LOG_VERBOSE(" data offset: %" PRIu64 ", stack size: %d", + module->aux_data_end, module->aux_stack_size); + LOG_VERBOSE(" heap offset: %" PRIu64 ", heap size: %d\n", heap_offset, + heap_size); max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; bh_assert(max_memory_data_size <= MAX_LINEAR_MEMORY_SIZE); @@ -1070,8 +1071,8 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, /* Check memory data */ /* check offset since length might negative */ if (base_offset > memory_inst->memory_data_size) { - LOG_DEBUG("base_offset(%d) > memory_data_size(%d)", base_offset, - memory_inst->memory_data_size); + LOG_DEBUG("base_offset(%d) > memory_data_size(%" PRIu64 ")", + base_offset, memory_inst->memory_data_size); #if WASM_ENABLE_REF_TYPES != 0 set_error_buf(error_buf, error_buf_size, "out of bounds memory access"); @@ -1085,7 +1086,8 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, /* check offset + length(could be zero) */ length = data_seg->byte_count; if (base_offset + length > memory_inst->memory_data_size) { - LOG_DEBUG("base_offset(%d) + length(%d) > memory_data_size(%d)", + LOG_DEBUG("base_offset(%d) + length(%d) > memory_data_size(%" PRIu64 + ")", base_offset, length, memory_inst->memory_data_size); #if WASM_ENABLE_REF_TYPES != 0 set_error_buf(error_buf, error_buf_size, @@ -2523,7 +2525,8 @@ aot_module_malloc_internal(AOTModuleInstance *module_inst, aot_set_exception(module_inst, "app heap corrupted"); } else { - LOG_WARNING("warning: allocate %u bytes memory failed", size); + LOG_WARNING("warning: allocate %" PRIu64 " bytes memory failed", + size); } return 0; } @@ -2806,7 +2809,7 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, } #if WASM_ENABLE_GC == 0 - func_idx = tbl_elem_val; + func_idx = (uint32)tbl_elem_val; #else func_idx = wasm_func_obj_get_func_idx_bound((WASMFuncObjectRef)tbl_elem_val); @@ -3697,14 +3700,14 @@ aot_create_call_stack(struct WASMExecEnv *exec_env) frame.instance = module_inst; frame.module_offset = 0; - frame.func_index = cur_frame->func_index; - frame.func_offset = cur_frame->ip_offset; - frame.func_name_wp = - get_func_name_from_index(module_inst, cur_frame->func_index); + frame.func_index = (uint32)cur_frame->func_index; + frame.func_offset = (uint32)cur_frame->ip_offset; + frame.func_name_wp = get_func_name_from_index( + module_inst, (uint32)cur_frame->func_index); if (cur_frame->func_index >= module->import_func_count) { uint32 aot_func_idx = - cur_frame->func_index - module->import_func_count; + (uint32)(cur_frame->func_index - module->import_func_count); max_local_cell_num = module->max_local_cell_nums[aot_func_idx]; max_stack_cell_num = module->max_stack_cell_nums[aot_func_idx]; } @@ -4721,12 +4724,12 @@ aot_set_module_name(AOTModule *module, const char *name, char *error_buf, if (!name) return false; - module->name = - aot_const_str_set_insert((const uint8 *)name, strlen(name) + 1, module, + module->name = aot_const_str_set_insert((const uint8 *)name, + (uint32)(strlen(name) + 1), module, #if (WASM_ENABLE_WORD_ALIGN_READ != 0) - false, + false, #endif - error_buf, error_buf_size); + error_buf, error_buf_size); return module->name != NULL; } diff --git a/core/iwasm/common/gc/gc_type.c b/core/iwasm/common/gc/gc_type.c index 0c9271c87..60f0e7e7a 100644 --- a/core/iwasm/common/gc/gc_type.c +++ b/core/iwasm/common/gc/gc_type.c @@ -148,7 +148,7 @@ wasm_dump_func_type(const WASMFuncType *type) os_printf("] -> ["); - for (; i < type->param_count + type->result_count; i++) { + for (; i < (uint32)(type->param_count + type->result_count); i++) { if (wasm_is_type_multi_byte_type(type->types[i])) { bh_assert(j < type->ref_type_map_count); bh_assert(i == type->ref_type_maps[j].index); @@ -264,7 +264,7 @@ wasm_func_type_equal(const WASMFuncType *type1, const WASMFuncType *type2, || type1->ref_type_map_count != type2->ref_type_map_count) return false; - for (i = 0; i < type1->param_count + type1->result_count; i++) { + for (i = 0; i < (uint32)(type1->param_count + type1->result_count); i++) { if (type1->types[i] != type2->types[i]) return false; @@ -399,7 +399,7 @@ wasm_func_type_is_subtype_of(const WASMFuncType *type1, } } - for (; i < type1->param_count + type1->result_count; i++) { + for (; i < (uint32)(type1->param_count + type1->result_count); i++) { if (wasm_is_type_multi_byte_type(type1->types[i])) { bh_assert(j1 < type1->ref_type_map_count); ref_type1 = type1->ref_type_maps[j1++].ref_type; diff --git a/core/iwasm/common/wasm_c_api.c b/core/iwasm/common/wasm_c_api.c index 10ceb7583..29da8e22f 100644 --- a/core/iwasm/common/wasm_c_api.c +++ b/core/iwasm/common/wasm_c_api.c @@ -3987,7 +3987,7 @@ wasm_table_get(const wasm_table_t *table, wasm_table_size_t index) if (index >= table_interp->cur_size) { return NULL; } - ref_idx = table_interp->elems[index]; + ref_idx = (uint32)table_interp->elems[index]; } #endif @@ -3998,7 +3998,7 @@ wasm_table_get(const wasm_table_t *table, wasm_table_size_t index) if (index >= table_aot->cur_size) { return NULL; } - ref_idx = table_aot->elems[index]; + ref_idx = (uint32)table_aot->elems[index]; } #endif diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 1d2cd1677..50ee917ed 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -41,12 +41,12 @@ static void (*free_func)(void *ptr) = NULL; static unsigned int global_pool_size; -static uint32 +static uint64 align_as_and_cast(uint64 size, uint64 alignment) { uint64 aligned_size = (size + alignment - 1) & ~(alignment - 1); - return aligned_size > UINT32_MAX ? UINT32_MAX : (uint32)aligned_size; + return aligned_size; } static bool @@ -951,7 +951,7 @@ wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory, { bh_assert(*memory_data_size <= MAX_LINEAR_MEMORY_SIZE); } - align_as_and_cast(*memory_data_size, page_size); + *memory_data_size = align_as_and_cast(*memory_data_size, page_size); if (map_size > 0) { if (!(*data = wasm_mmap_linear_memory(map_size, *memory_data_size))) { @@ -960,4 +960,4 @@ wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory, } return BHT_OK; -} \ No newline at end of file +} diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index e3b4ca7b5..c7906edbe 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -275,11 +275,11 @@ decode_insn(uint8 *insn) buffer, sizeof(buffer), runtime_address); +#if 0 /* Print current instruction */ - /* os_printf("%012" PRIX64 " ", runtime_address); puts(buffer); - */ +#endif return instruction.length; } @@ -1043,7 +1043,7 @@ wasm_runtime_register_module_internal(const char *module_name, /* module hasn't been registered */ node = runtime_malloc(sizeof(WASMRegisteredModule), NULL, NULL, 0); if (!node) { - LOG_DEBUG("malloc WASMRegisteredModule failed. SZ=%d", + LOG_DEBUG("malloc WASMRegisteredModule failed. SZ=%zu", sizeof(WASMRegisteredModule)); return false; } @@ -1780,7 +1780,7 @@ wasm_runtime_dump_mem_consumption(WASMExecEnv *exec_env) wasm_runtime_dump_module_inst_mem_consumption(module_inst_common); wasm_runtime_dump_exec_env_mem_consumption(exec_env); os_printf("\nTotal memory consumption of module, module inst and " - "exec env: %u\n", + "exec env: %" PRIu64 "\n", total_size); os_printf("Total interpreter stack used: %u\n", exec_env->max_wasm_stack_used); @@ -5488,6 +5488,7 @@ wasm_externref_set_cleanup(WASMModuleInstanceCommon *module_inst, if (lookup_user_data.found) { void *key = (void *)(uintptr_t)lookup_user_data.externref_idx; ExternRefMapNode *node = bh_hash_map_find(externref_map, key); + bh_assert(node); node->cleanup = extern_obj_cleanup; ok = true; } @@ -6539,12 +6540,12 @@ wasm_runtime_load_depended_module(const WASMModuleCommon *parent_module, if (!ret) { LOG_DEBUG("read the file of %s failed", sub_module_name); set_error_buf_v(parent_module, error_buf, error_buf_size, - "unknown import", sub_module_name); + "unknown import %s", sub_module_name); goto delete_loading_module; } if (get_package_type(buffer, buffer_size) != parent_module->module_type) { LOG_DEBUG("moudle %s type error", sub_module_name); - goto delete_loading_module; + goto destroy_file_buffer; } if (get_package_type(buffer, buffer_size) == Wasm_Module_Bytecode) { #if WASM_ENABLE_INTERP != 0 @@ -6650,7 +6651,7 @@ wasm_runtime_sub_module_instantiate(WASMModuleCommon *module, sub_module_inst_list_node = loader_malloc(sizeof(WASMSubModInstNode), error_buf, error_buf_size); if (!sub_module_inst_list_node) { - LOG_DEBUG("Malloc WASMSubModInstNode failed, SZ:%d", + LOG_DEBUG("Malloc WASMSubModInstNode failed, SZ: %zu", sizeof(WASMSubModInstNode)); if (sub_module_inst) wasm_runtime_deinstantiate_internal(sub_module_inst, false); diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index 9740cd0d1..ef3931b34 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -330,7 +330,7 @@ aot_gen_commit_values(AOTCompFrame *frame) if (!p->dirty) continue; - n = p - frame->lp; + n = (uint32)(p - frame->lp); /* Commit reference flag */ if (comp_ctx->enable_gc) { @@ -432,7 +432,7 @@ aot_gen_commit_values(AOTCompFrame *frame) continue; p->dirty = 0; - n = p - frame->lp; + n = (uint32)(p - frame->lp); /* Commit values */ switch (p->type) { @@ -538,7 +538,7 @@ aot_gen_commit_values(AOTCompFrame *frame) /* Clear reference flags for unused stack slots. */ for (p = frame->sp; p < end; p++) { bh_assert(!p->ref); - n = p - frame->lp; + n = (uint32)(p - frame->lp); /* Commit reference flag. */ if (p->ref != p->committed_ref - 1) { @@ -621,7 +621,7 @@ aot_gen_commit_sp_ip(AOTCompFrame *frame, bool commit_sp, bool commit_ip) } if (commit_sp) { - n = sp - frame->lp; + n = (uint32)(sp - frame->lp); value = I32_CONST(offset_of_local(comp_ctx, n)); if (!value) { aot_set_last_error("llvm build const failed"); diff --git a/core/iwasm/compilation/aot_emit_control.c b/core/iwasm/compilation/aot_emit_control.c index 4e28babc3..24511ffd0 100644 --- a/core/iwasm/compilation/aot_emit_control.c +++ b/core/iwasm/compilation/aot_emit_control.c @@ -1269,6 +1269,7 @@ aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, PUSH(values[j], target_block->result_types[j]); } wasm_runtime_free(values); + values = NULL; } target_block->is_reachable = true; if (i == br_count) @@ -1294,6 +1295,7 @@ aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, PUSH(values[j], target_block->param_types[j]); } wasm_runtime_free(values); + values = NULL; } if (i == br_count) default_llvm_block = target_block->llvm_entry_block; diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index dfb12a7f4..d4d1cff02 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -674,7 +674,8 @@ aot_add_llvm_func(AOTCompContext *comp_ctx, LLVMModuleRef module, uint32 backend_thread_num, compile_thread_num; /* Check function parameter types and result types */ - for (i = 0; i < aot_func_type->param_count + aot_func_type->result_count; + for (i = 0; + i < (uint32)(aot_func_type->param_count + aot_func_type->result_count); i++) { if (!check_wasm_type(comp_ctx, aot_func_type->types[i])) return NULL; diff --git a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp index f5605b6f2..53761e70a 100644 --- a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp +++ b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp @@ -7511,7 +7511,7 @@ at_rmw_xor_r_base_r_offset_r(x86::Assembler &a, uint32 bytes_dst, CHECK_KIND(r3, JIT_REG_KIND_I64); \ } \ /* r0: read/return value r2: memory base addr can't be const */ \ - /* already check it's not const in LOAD_4ARGS(); */ \ + /* already check it's not const in LOAD_4ARGS() */ \ reg_no_dst = jit_reg_no(r0); \ CHECK_REG_NO(reg_no_dst, jit_reg_kind(r0)); \ /* mem_data base address has to be non-const */ \ @@ -9419,7 +9419,7 @@ static uint8 hreg_info_F64[3][16] = { 1, 1, 1, 1, 1, 1, 1, 0 }, /* caller_saved_jitted */ }; -static const JitHardRegInfo hreg_info = { +static const JitHardRegInfo g_hreg_info = { { { 0, NULL, NULL, NULL }, /* VOID */ @@ -9459,7 +9459,7 @@ static const JitHardRegInfo hreg_info = { const JitHardRegInfo * jit_codegen_get_hreg_info() { - return &hreg_info; + return &g_hreg_info; } static const char *reg_names_i32[] = { diff --git a/core/iwasm/fast-jit/fe/jit_emit_memory.c b/core/iwasm/fast-jit/fe/jit_emit_memory.c index 0a977c1d6..ea245ba34 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_memory.c +++ b/core/iwasm/fast-jit/fe/jit_emit_memory.c @@ -636,7 +636,7 @@ wasm_init_memory(WASMModuleInstance *inst, uint32 mem_idx, uint32 seg_idx, /* if d + n > the length of mem.data */ mem_inst = inst->memories[mem_idx]; - mem_size = mem_inst->cur_page_count * mem_inst->num_bytes_per_page; + mem_size = mem_inst->cur_page_count * (uint64)mem_inst->num_bytes_per_page; if (mem_size < mem_offset || mem_size - mem_offset < len) goto out_of_bounds; @@ -724,8 +724,10 @@ wasm_copy_memory(WASMModuleInstance *inst, uint32 src_mem_idx, src_mem = inst->memories[src_mem_idx]; dst_mem = inst->memories[dst_mem_idx]; - src_mem_size = src_mem->cur_page_count * src_mem->num_bytes_per_page; - dst_mem_size = dst_mem->cur_page_count * dst_mem->num_bytes_per_page; + src_mem_size = + src_mem->cur_page_count * (uint64)src_mem->num_bytes_per_page; + dst_mem_size = + dst_mem->cur_page_count * (uint64)dst_mem->num_bytes_per_page; /* if s + n > the length of mem.data */ if (src_mem_size < src_offset || src_mem_size - src_offset < len) @@ -788,7 +790,7 @@ wasm_fill_memory(WASMModuleInstance *inst, uint32 mem_idx, uint32 len, uint8 *dst_addr; mem_inst = inst->memories[mem_idx]; - mem_size = mem_inst->cur_page_count * mem_inst->num_bytes_per_page; + mem_size = mem_inst->cur_page_count * (uint64)mem_inst->num_bytes_per_page; if (mem_size < dst || mem_size - dst < len) goto out_of_bounds; diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index 5fd86b572..80ce67b8e 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -1329,8 +1329,8 @@ block_type_get_param_types(BlockType *block_type, uint8 **p_param_types, param_count = func_type->param_count; #if WASM_ENABLE_GC != 0 *p_param_reftype_maps = func_type->ref_type_maps; - *p_param_reftype_map_count = - func_type->result_ref_type_maps - func_type->ref_type_maps; + *p_param_reftype_map_count = (uint32)(func_type->result_ref_type_maps + - func_type->ref_type_maps); #endif } else { diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 21412046e..004371163 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -1693,7 +1693,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, /* clang-format off */ #if WASM_ENABLE_GC == 0 - fidx = tbl_inst->elems[val]; + fidx = (uint32)tbl_inst->elems[val]; if (fidx == (uint32)-1) { wasm_set_exception(module, "uninitialized element"); goto got_exception; diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 51384cb69..d8ceb714a 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -1521,7 +1521,7 @@ resolve_func_type(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, #endif #if WASM_ENABLE_WAMR_COMPILER != 0 - for (i = 0; i < type->param_count + type->result_count; i++) { + for (i = 0; i < (uint32)(type->param_count + type->result_count); i++) { if (type->types[i] == VALUE_TYPE_V128) module->is_simd_used = true; } @@ -1929,8 +1929,8 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, } #else /* else of WASM_ENABLE_GC == 0 */ for (i = 0; i < type_count; i++) { - uint32 super_type_count = 0, parent_type_idx = (uint32)-1, - rec_count = 1, j; + uint32 super_type_count = 0, parent_type_idx = (uint32)-1; + uint32 rec_count = 1, j; bool is_sub_final = true; CHECK_BUF(p, p_end, 1); @@ -1942,10 +1942,22 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, if (rec_count > 1) { uint64 new_total_size; + /* integer overflow */ + if (rec_count - 1 > UINT32_MAX - module->type_count) { + set_error_buf(error_buf, error_buf_size, + "recursive type count too large"); + return false; + } module->type_count += rec_count - 1; new_total_size = sizeof(WASMFuncType *) * (uint64)module->type_count; - MEM_REALLOC(module->types, total_size, new_total_size); + if (new_total_size > UINT32_MAX) { + set_error_buf(error_buf, error_buf_size, + "allocate memory failed"); + return false; + } + MEM_REALLOC(module->types, (uint32)total_size, + (uint32)new_total_size); total_size = new_total_size; } @@ -5574,8 +5586,8 @@ load_from_sections(WASMModule *module, WASMSection *sections, *buf_func = NULL, *buf_func_end = NULL; WASMGlobal *aux_data_end_global = NULL, *aux_heap_base_global = NULL; WASMGlobal *aux_stack_top_global = NULL, *global; - uint64 aux_data_end = (uint64)-1, aux_heap_base = (uint64)-1, - aux_stack_top = (uint64)-1; + uint64 aux_data_end = (uint64)-1LL, aux_heap_base = (uint64)-1LL, + aux_stack_top = (uint64)-1LL; uint32 global_index, func_index, i; uint32 aux_data_end_global_index = (uint32)-1; uint32 aux_heap_base_global_index = (uint32)-1; @@ -5715,7 +5727,7 @@ load_from_sections(WASMModule *module, WASMSection *sections, aux_heap_base_global = global; aux_heap_base = (uint64)(uint32)global->init_expr.u.i32; aux_heap_base_global_index = export->index; - LOG_VERBOSE("Found aux __heap_base global, value: %d", + LOG_VERBOSE("Found aux __heap_base global, value: %" PRIu64, aux_heap_base); } } @@ -5728,7 +5740,7 @@ load_from_sections(WASMModule *module, WASMSection *sections, aux_data_end_global = global; aux_data_end = (uint64)(uint32)global->init_expr.u.i32; aux_data_end_global_index = export->index; - LOG_VERBOSE("Found aux __data_end global, value: %d", + LOG_VERBOSE("Found aux __data_end global, value: %" PRIu64, aux_data_end); aux_data_end = align_uint64(aux_data_end, 16); @@ -5778,10 +5790,11 @@ load_from_sections(WASMModule *module, WASMSection *sections, aux_stack_top > aux_data_end ? (uint32)(aux_stack_top - aux_data_end) : (uint32)aux_stack_top; - LOG_VERBOSE("Found aux stack top global, value: %d, " - "global index: %d, stack size: %d", - aux_stack_top, global_index, - module->aux_stack_size); + LOG_VERBOSE( + "Found aux stack top global, value: %" PRIu64 ", " + "global index: %d, stack size: %d", + aux_stack_top, global_index, + module->aux_stack_size); break; } } @@ -5929,9 +5942,10 @@ load_from_sections(WASMModule *module, WASMSection *sections, * memory_import->init_page_count; if (shrunk_memory_size <= init_memory_size) { /* Reset memory info to decrease memory usage */ - memory_import->num_bytes_per_page = shrunk_memory_size; + memory_import->num_bytes_per_page = + (uint32)shrunk_memory_size; memory_import->init_page_count = 1; - LOG_VERBOSE("Shrink import memory size to %d", + LOG_VERBOSE("Shrink import memory size to %" PRIu64, shrunk_memory_size); } } @@ -5942,9 +5956,9 @@ load_from_sections(WASMModule *module, WASMSection *sections, * memory->init_page_count; if (shrunk_memory_size <= init_memory_size) { /* Reset memory info to decrease memory usage */ - memory->num_bytes_per_page = shrunk_memory_size; + memory->num_bytes_per_page = (uint32)shrunk_memory_size; memory->init_page_count = 1; - LOG_VERBOSE("Shrink memory size to %d", + LOG_VERBOSE("Shrink memory size to %" PRIu64, shrunk_memory_size); } } @@ -6654,7 +6668,7 @@ wasm_loader_unload(WASMModule *module) #if WASM_ENABLE_STRINGREF != 0 if (module->string_literal_ptrs) { - wasm_runtime_free(module->string_literal_ptrs); + wasm_runtime_free((void *)module->string_literal_ptrs); } if (module->string_literal_lengths) { wasm_runtime_free(module->string_literal_lengths); @@ -8356,12 +8370,12 @@ wasm_loader_pop_nullable_ht(WASMLoaderContext *ctx, uint8 *p_type, } /* Convert to related (ref ht) and return */ - if ((type >= REF_TYPE_EQREF && type <= REF_TYPE_FUNCREF) - || (type >= REF_TYPE_NULLREF && type <= REF_TYPE_I31REF)) { - /* Return (ref func/extern/any/eq/i31/nofunc/noextern/struct/array/none) + if (type >= REF_TYPE_ARRAYREF && type <= REF_TYPE_NULLFUNCREF) { + /* Return (ref array/struct/i31/eq/any/extern/func/none/noextern/nofunc) */ wasm_set_refheaptype_common(&ref_ht_ret->ref_ht_common, false, - HEAP_TYPE_FUNC + (type - REF_TYPE_FUNCREF)); + HEAP_TYPE_ARRAY + + (type - REF_TYPE_ARRAYREF)); type = ref_ht_ret->ref_type; } else if (wasm_is_reftype_htref_nullable(type) @@ -10067,8 +10081,8 @@ wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, uint8 opcode, loader_ctx->stack_cell_num = stack_cell_num_old; loader_ctx->frame_ref = loader_ctx->frame_ref_bottom + stack_cell_num_old; - total_size = (uint32)sizeof(uint8) - * (frame_ref_old - frame_ref_after_popped); + total_size = (uint32)(sizeof(uint8) + * (frame_ref_old - frame_ref_after_popped)); bh_memcpy_s((uint8 *)loader_ctx->frame_ref - total_size, total_size, frame_ref_buf, total_size); @@ -10079,9 +10093,9 @@ wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, uint8 opcode, loader_ctx->reftype_map_num = reftype_map_num_old; loader_ctx->frame_reftype_map = loader_ctx->frame_reftype_map_bottom + reftype_map_num_old; - total_size = - (uint32)sizeof(WASMRefTypeMap) - * (frame_reftype_map_old - frame_reftype_map_after_popped); + total_size = (uint32)(sizeof(WASMRefTypeMap) + * (frame_reftype_map_old + - frame_reftype_map_after_popped)); bh_memcpy_s((uint8 *)loader_ctx->frame_reftype_map - total_size, total_size, frame_reftype_map_buf, total_size); #endif @@ -10089,8 +10103,9 @@ wasm_loader_check_br(WASMLoaderContext *loader_ctx, uint32 depth, uint8 opcode, #if WASM_ENABLE_FAST_INTERP != 0 loader_ctx->frame_offset = loader_ctx->frame_offset_bottom + stack_cell_num_old; - total_size = (uint32)sizeof(int16) - * (frame_offset_old - frame_offset_after_popped); + total_size = + (uint32)(sizeof(int16) + * (frame_offset_old - frame_offset_after_popped)); bh_memcpy_s((uint8 *)loader_ctx->frame_offset - total_size, total_size, frame_offset_buf, total_size); (loader_ctx->frame_csp - 1)->dynamic_offset = dynamic_offset_old; @@ -10164,7 +10179,7 @@ fail: #endif #if WASM_ENABLE_FAST_INTERP != 0 if (frame_offset_buf && frame_offset_buf != frame_offset_tmp) - wasm_runtime_free(frame_offset_tmp); + wasm_runtime_free(frame_offset_buf); #endif return ret; @@ -10220,7 +10235,7 @@ check_branch_block_for_delegate(WASMLoaderContext *loader_ctx, uint8 **p_buf, } frame_csp_tmp = loader_ctx->frame_csp - depth - 2; #if WASM_ENABLE_FAST_INTERP != 0 - emit_br_info(frame_csp_tmp); + emit_br_info(frame_csp_tmp, false); #endif *p_buf = p; diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index f1023fa01..f0859e96e 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -2567,8 +2567,8 @@ load_from_sections(WASMModule *module, WASMSection *sections, *buf_func = NULL, *buf_func_end = NULL; WASMGlobal *aux_data_end_global = NULL, *aux_heap_base_global = NULL; WASMGlobal *aux_stack_top_global = NULL, *global; - uint64 aux_data_end = (uint64)-1, aux_heap_base = (uint64)-1, - aux_stack_top = (uint64)-1; + uint64 aux_data_end = (uint64)-1LL, aux_heap_base = (uint64)-1LL, + aux_stack_top = (uint64)-1LL; uint32 global_index, func_index, i; uint32 aux_data_end_global_index = (uint32)-1; uint32 aux_heap_base_global_index = (uint32)-1; @@ -2689,7 +2689,7 @@ load_from_sections(WASMModule *module, WASMSection *sections, aux_heap_base_global = global; aux_heap_base = (uint64)(uint32)global->init_expr.u.i32; aux_heap_base_global_index = export->index; - LOG_VERBOSE("Found aux __heap_base global, value: %d", + LOG_VERBOSE("Found aux __heap_base global, value: %" PRIu64, aux_heap_base); } } @@ -2702,7 +2702,7 @@ load_from_sections(WASMModule *module, WASMSection *sections, aux_data_end_global = global; aux_data_end = (uint64)(uint32)global->init_expr.u.i32; aux_data_end_global_index = export->index; - LOG_VERBOSE("Found aux __data_end global, value: %d", + LOG_VERBOSE("Found aux __data_end global, value: %" PRIu64, aux_data_end); aux_data_end = align_uint64(aux_data_end, 16); } @@ -2751,10 +2751,11 @@ load_from_sections(WASMModule *module, WASMSection *sections, aux_stack_top > aux_data_end ? (uint32)(aux_stack_top - aux_data_end) : (uint32)aux_stack_top; - LOG_VERBOSE("Found aux stack top global, value: %d, " - "global index: %d, stack size: %d", - aux_stack_top, global_index, - module->aux_stack_size); + LOG_VERBOSE( + "Found aux stack top global, value: %" PRIu64 ", " + "global index: %d, stack size: %d", + aux_stack_top, global_index, + module->aux_stack_size); break; } } @@ -2901,7 +2902,7 @@ load_from_sections(WASMModule *module, WASMSection *sections, /* Reset memory info to decrease memory usage */ memory_import->num_bytes_per_page = shrunk_memory_size; memory_import->init_page_count = 1; - LOG_VERBOSE("Shrink import memory size to %d", + LOG_VERBOSE("Shrink import memory size to %" PRIu64, shrunk_memory_size); } } @@ -2914,7 +2915,7 @@ load_from_sections(WASMModule *module, WASMSection *sections, /* Reset memory info to decrease memory usage */ memory->num_bytes_per_page = shrunk_memory_size; memory->init_page_count = 1; - LOG_VERBOSE("Shrink memory size to %d", + LOG_VERBOSE("Shrink memory size to %" PRIu64, shrunk_memory_size); } } @@ -5646,7 +5647,7 @@ fail: wasm_runtime_free(frame_ref_buf); #if WASM_ENABLE_FAST_INTERP != 0 if (frame_offset_buf && frame_offset_buf != frame_offset_tmp) - wasm_runtime_free(frame_offset_tmp); + wasm_runtime_free(frame_offset_buf); #endif return ret; diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 71e7d54ee..a216b4e29 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -273,7 +273,7 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, /* For memory32, the global value should be i32 */ *(uint32 *)global_addr = (uint32)aux_heap_base; } - LOG_VERBOSE("Reset __heap_base global to %lu", aux_heap_base); + LOG_VERBOSE("Reset __heap_base global to %" PRIu64, aux_heap_base); } else { /* Insert app heap before new page */ @@ -300,7 +300,8 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, LOG_VERBOSE("Memory instantiate:"); LOG_VERBOSE(" page bytes: %u, init pages: %u, max pages: %u", num_bytes_per_page, init_page_count, max_page_count); - LOG_VERBOSE(" heap offset: %u, heap size: %d\n", heap_offset, heap_size); + LOG_VERBOSE(" heap offset: %" PRIu64 ", heap size: %u\n", heap_offset, + heap_size); max_memory_data_size = (uint64)num_bytes_per_page * max_page_count; bh_assert(max_memory_data_size @@ -2379,8 +2380,13 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, /* check offset */ if (base_offset > memory_size) { - LOG_DEBUG("base_offset(%d) > memory_size(%d)", base_offset, +#if WASM_ENABLE_MEMORY64 != 0 + LOG_DEBUG("base_offset(%" PRIu64 ") > memory_size(%" PRIu64 ")", + base_offset, memory_size); +#else + LOG_DEBUG("base_offset(%u) > memory_size(%" PRIu64 ")", base_offset, memory_size); +#endif #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 set_error_buf(error_buf, error_buf_size, "out of bounds memory access"); @@ -2394,8 +2400,14 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, /* check offset + length(could be zero) */ length = data_seg->data_length; if ((uint64)base_offset + length > memory_size) { - LOG_DEBUG("base_offset(%d) + length(%d) > memory_size(%d)", +#if WASM_ENABLE_MEMORY64 != 0 + LOG_DEBUG("base_offset(%" PRIu64 + ") + length(%d) > memory_size(%" PRIu64 ")", base_offset, length, memory_size); +#else + LOG_DEBUG("base_offset(%u) + length(%d) > memory_size(%" PRIu64 ")", + base_offset, length, memory_size); +#endif #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 set_error_buf(error_buf, error_buf_size, "out of bounds memory access"); @@ -3356,7 +3368,8 @@ wasm_module_malloc_internal(WASMModuleInstance *module_inst, wasm_set_exception(module_inst, "app heap corrupted"); } else { - LOG_WARNING("warning: allocate %u bytes memory failed", size); + LOG_WARNING("warning: allocate %" PRIu64 " bytes memory failed", + size); } return 0; } @@ -3555,7 +3568,7 @@ call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 tbl_elem_idx, } #if WASM_ENABLE_GC == 0 - func_idx = tbl_elem_val; + func_idx = (uint32)tbl_elem_val; #else func_idx = wasm_func_obj_get_func_idx_bound((WASMFuncObjectRef)tbl_elem_val); @@ -4586,8 +4599,8 @@ wasm_set_module_name(WASMModule *module, const char *name, char *error_buf, return false; module->name = - wasm_const_str_list_insert((const uint8 *)name, strlen(name), module, - false, error_buf, error_buf_size); + wasm_const_str_list_insert((const uint8 *)name, (uint32)strlen(name), + module, false, error_buf, error_buf_size); return module->name != NULL; } @@ -4595,4 +4608,4 @@ const char * wasm_get_module_name(WASMModule *module) { return module->name; -} \ No newline at end of file +} diff --git a/core/iwasm/libraries/debug-engine/handler.c b/core/iwasm/libraries/debug-engine/handler.c index 8d451b1a3..905ca2f7c 100644 --- a/core/iwasm/libraries/debug-engine/handler.c +++ b/core/iwasm/libraries/debug-engine/handler.c @@ -309,9 +309,11 @@ handle_general_query(WASMGDBServer *server, char *payload) } if (!strcmp(name, "WasmData")) { + write_packet(server, ""); } if (!strcmp(name, "WasmMem")) { + write_packet(server, ""); } if (!strcmp(name, "Symbol")) { @@ -447,7 +449,7 @@ send_thread_stop_status(WASMGDBServer *server, uint32 status, korp_tid tid) "thread-pcs:%" PRIx64 ";00:%s;reason:%s;", pc, pc_string, "trace"); } - else if (status > 0) { + else { /* status > 0 (== 0 is checked at the function beginning) */ len += snprintf(tmpbuf + len, MAX_PACKET_SIZE - len, "thread-pcs:%" PRIx64 ";00:%s;reason:%s;", pc, pc_string, "signal"); diff --git a/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c b/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c index 30055e634..b3fa57d72 100644 --- a/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c +++ b/core/iwasm/libraries/lib-pthread/lib_pthread_wrapper.c @@ -1123,7 +1123,8 @@ posix_memalign_wrapper(wasm_exec_env_t exec_env, void **memptr, int32 align, wasm_module_inst_t module_inst = get_module_inst(exec_env); void *p = NULL; - *((int32 *)memptr) = module_malloc(size, (void **)&p); + /* TODO: for memory 64, module_malloc may return uint64 offset */ + *((uint32 *)memptr) = (uint32)module_malloc(size, (void **)&p); if (!p) return -1; diff --git a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/blocking_op.h b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/blocking_op.h index 9c36d7df6..a32e5d662 100644 --- a/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/blocking_op.h +++ b/core/iwasm/libraries/libc-wasi/sandboxed-system-primitives/src/blocking_op.h @@ -3,6 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ +#ifndef _BLOCKING_OP_H_ +#define _BLOCKING_OP_H_ + #include "bh_platform.h" #include "wasm_export.h" @@ -57,3 +60,5 @@ __wasi_errno_t blocking_op_poll(wasm_exec_env_t exec_env, struct pollfd *pfds, nfds_t nfds, int timeout, int *retp); #endif + +#endif /* end of _BLOCKING_OP_H_ */ diff --git a/core/shared/mem-alloc/ems/ems_alloc.c b/core/shared/mem-alloc/ems/ems_alloc.c index b667fbe9f..4863527d6 100644 --- a/core/shared/mem-alloc/ems/ems_alloc.c +++ b/core/shared/mem-alloc/ems/ems_alloc.c @@ -785,8 +785,8 @@ gc_alloc_wo_internal(void *vheap, gc_size_t size, const char *file, int line) if (!hmu) goto finish; - /* Do we need to memset the memory to 0? */ - /* memset((char *)hmu + sizeof(*hmu), 0, tot_size - sizeof(*hmu)); */ + /* Don't memset the memory to improve performance, the caller should + decide whether to memset it or not */ bh_assert(hmu_get_size(hmu) >= tot_size); /* the total size allocated may be larger than diff --git a/core/shared/mem-alloc/ems/ems_gc.c b/core/shared/mem-alloc/ems/ems_gc.c index b0f14772b..26e83a975 100644 --- a/core/shared/mem-alloc/ems/ems_gc.c +++ b/core/shared/mem-alloc/ems/ems_gc.c @@ -114,8 +114,8 @@ sweep_instance_heap(gc_heap_t *heap) else { /* current block is still live */ if (last) { - tot_free += (char *)cur - (char *)last; - gci_add_fc(heap, last, (char *)cur - (char *)last); + tot_free += (gc_size_t)((char *)cur - (char *)last); + gci_add_fc(heap, last, (gc_size_t)((char *)cur - (char *)last)); hmu_mark_pinuse(last); last = NULL; } @@ -132,8 +132,8 @@ sweep_instance_heap(gc_heap_t *heap) bh_assert(cur == end); if (last) { - tot_free += (char *)cur - (char *)last; - gci_add_fc(heap, last, (char *)cur - (char *)last); + tot_free += (gc_size_t)((char *)cur - (char *)last); + gci_add_fc(heap, last, (gc_size_t)((char *)cur - (char *)last)); hmu_mark_pinuse(last); } @@ -449,7 +449,9 @@ gci_gc_heap(void *h) LOG_VERBOSE("#reclaim instance heap %p", heap); - gct_vm_gc_prepare(); + /* TODO: get exec_env of current thread when GC multi-threading + is enabled, and pass it to runtime */ + gct_vm_gc_prepare(NULL); gct_vm_mutex_lock(&heap->lock); heap->is_doing_reclaim = 1; @@ -459,7 +461,9 @@ gci_gc_heap(void *h) heap->is_doing_reclaim = 0; gct_vm_mutex_unlock(&heap->lock); - gct_vm_gc_finished(); + /* TODO: get exec_env of current thread when GC multi-threading + is enabled, and pass it to runtime */ + gct_vm_gc_finished(NULL); LOG_VERBOSE("#reclaim instance heap %p done", heap); diff --git a/core/shared/mem-alloc/mem_alloc.c b/core/shared/mem-alloc/mem_alloc.c index 1f9e03d5a..df1a4de4c 100644 --- a/core/shared/mem-alloc/mem_alloc.c +++ b/core/shared/mem-alloc/mem_alloc.c @@ -77,13 +77,13 @@ mem_allocator_free_with_gc(mem_allocator_t allocator, void *ptr) void mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *exec_env) { - return gc_enable_gc_reclaim((gc_handle_t)allocator, exec_env); + gc_enable_gc_reclaim((gc_handle_t)allocator, exec_env); } #else void mem_allocator_enable_gc_reclaim(mem_allocator_t allocator, void *cluster) { - return gc_enable_gc_reclaim((gc_handle_t)allocator, cluster); + gc_enable_gc_reclaim((gc_handle_t)allocator, cluster); } #endif diff --git a/core/shared/utils/runtime_timer.c b/core/shared/utils/runtime_timer.c index b9ace567f..9d390c214 100644 --- a/core/shared/utils/runtime_timer.c +++ b/core/shared/utils/runtime_timer.c @@ -394,7 +394,7 @@ handle_expired_timers(timer_ctx_t ctx, app_timer_t *expired) operation may change expired->next */ expired = expired->next; if (t->is_periodic) { - /* if it is repeating, then reschedule it; */ + /* if it is repeating, then reschedule it */ reschedule_timer(ctx, t); } else { diff --git a/product-mini/platforms/posix/main.c b/product-mini/platforms/posix/main.c index 37ee0cb87..217eb20cb 100644 --- a/product-mini/platforms/posix/main.c +++ b/product-mini/platforms/posix/main.c @@ -851,7 +851,8 @@ main(int argc, char *argv[]) #if WASM_ENABLE_DEBUG_INTERP != 0 init_args.instance_port = instance_port; if (ip_addr) - strcpy(init_args.ip_addr, ip_addr); + /* ensure that init_args.ip_addr is null terminated */ + strncpy(init_args.ip_addr, ip_addr, sizeof(init_args.ip_addr) - 1); #endif /* initialize runtime environment */ diff --git a/product-mini/platforms/windows/main.c b/product-mini/platforms/windows/main.c index 6461e9172..35a489721 100644 --- a/product-mini/platforms/windows/main.c +++ b/product-mini/platforms/windows/main.c @@ -464,7 +464,9 @@ main(int argc, char *argv[]) #if WASM_ENABLE_DEBUG_INTERP != 0 init_args.instance_port = instance_port; if (ip_addr) - strcpy(init_args.ip_addr, ip_addr); + /* ensure that init_args.ip_addr is null terminated */ + strncpy_s(init_args.ip_addr, sizeof(init_args.ip_addr) - 1, ip_addr, + strlen(ip_addr)); #endif /* initialize runtime environment */ From cee9b826a504cc36917d3d1de6ac3dcdc67de5b6 Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Sun, 7 Apr 2024 14:54:02 +0800 Subject: [PATCH 15/36] Update document for GC, exception handling and memory64 features (#3284) Add GC, exception handling and memory64 to README.md post MVP feature list, and update build_wamr.md for how to build them. And remove `Non-trapping float-to-int conversions`, `Sign-extension operators`, `Multi-value` links in README since they are in wasm MVP and very common now. --- README.md | 5 ++--- doc/build_wamr.md | 13 +++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d38a3208e..cb91c22ce 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,8 @@ WebAssembly Micro Runtime (WAMR) is a lightweight standalone WebAssembly (Wasm) - [wasm-c-api](https://github.com/WebAssembly/wasm-c-api), ref to [document](doc/wasm_c_api.md) and [sample](samples/wasm-c-api) - [128-bit SIMD](https://github.com/WebAssembly/simd), ref to [samples/workload](samples/workload) - [Reference Types](https://github.com/WebAssembly/reference-types), ref to [document](doc/ref_types.md) and [sample](samples/ref-types) -- [Non-trapping float-to-int conversions](https://github.com/WebAssembly/nontrapping-float-to-int-conversions) -- [Sign-extension operators](https://github.com/WebAssembly/sign-extension-ops), [Bulk memory operations](https://github.com/WebAssembly/bulk-memory-operations) -- [Multi-value](https://github.com/WebAssembly/multi-value), [Tail-call](https://github.com/WebAssembly/tail-call), [Shared memory](https://github.com/WebAssembly/threads/blob/main/proposals/threads/Overview.md#shared-linear-memory) +- [Bulk memory operations](https://github.com/WebAssembly/bulk-memory-operations), [Shared memory](https://github.com/WebAssembly/threads/blob/main/proposals/threads/Overview.md#shared-linear-memory), [Memory64](https://github.com/WebAssembly/memory64) +- [Tail-call](https://github.com/WebAssembly/tail-call), [Garbage Collection](https://github.com/WebAssembly/gc), [Exception Handling](https://github.com/WebAssembly/exception-handling) ### Supported architectures and platforms The WAMR VMcore supports the following architectures: diff --git a/doc/build_wamr.md b/doc/build_wamr.md index 0d372e0d7..b0a8ea35f 100644 --- a/doc/build_wamr.md +++ b/doc/build_wamr.md @@ -76,6 +76,11 @@ cmake -DWAMR_BUILD_PLATFORM=linux -DWAMR_BUILD_TARGET=ARM #### **Enable bulk memory feature** - **WAMR_BUILD_BULK_MEMORY**=1/0, default to disable if not set +#### **Enable memory64 feature** +- **WAMR_BUILD_MEMORY64**=1/0, default to disable if not set + +> Note: Currently, the memory64 feature is only supported in classic interpreter running mode. + #### **Enable thread manager** - **WAMR_BUILD_THREAD_MGR**=1/0, default to disable if not set @@ -129,6 +134,14 @@ cmake -DWAMR_BUILD_PLATFORM=linux -DWAMR_BUILD_TARGET=ARM - **WAMR_BUILD_SIMD**=1/0, default to enable if not set > Note: only supported in AOT mode x86-64 target. +#### **Enable Exception Handling** +- **WAMR_BUILD_EXCE_HANDLING**=1/0, default to disable if not set + +> Note: Currently, the exception handling feature is only supported in classic interpreter running mode. + +#### **Enable Garbage Collection** +- **WAMR_BUILD_GC**=1/0, default to disable if not set + #### **Configure Debug** - **WAMR_BUILD_CUSTOM_NAME_SECTION**=1/0, load the function name from custom name section, default to disable if not set From 4ef724bbfffab1c7761b2674c483a8e7d7c840b8 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Sun, 7 Apr 2024 15:04:35 +0800 Subject: [PATCH 16/36] Enhance wasm loading with LoadArgs and support module names (#3265) - Add new API wasm_runtime_load_ex() in wasm_export.h and wasm_module_new_ex in wasm_c_api.h - Put aot_create_perf_map() into a separated file aot_perf_map.c - In perf.map, function names include user specified module name - Enhance the script to help flamegraph generations --- core/iwasm/aot/aot_loader.c | 116 +- core/iwasm/aot/aot_perf_map.c | 120 ++ core/iwasm/aot/aot_perf_map.h | 15 + core/iwasm/aot/aot_runtime.h | 4 +- core/iwasm/common/wasm_c_api.c | 15 +- core/iwasm/common/wasm_runtime_common.c | 34 +- core/iwasm/compilation/aot_llvm.c | 2 +- core/iwasm/include/wasm_c_api.h | 11 + core/iwasm/include/wasm_export.h | 15 + core/iwasm/interpreter/wasm_loader.c | 10 +- core/iwasm/interpreter/wasm_loader.h | 2 +- core/iwasm/interpreter/wasm_mini_loader.c | 10 +- core/iwasm/interpreter/wasm_runtime.c | 4 +- core/iwasm/interpreter/wasm_runtime.h | 2 +- samples/linux-perf/CMakeLists.txt | 63 + samples/linux-perf/README.md | 90 ++ samples/linux-perf/cmake/FindWAMRC.cmake | 14 + samples/linux-perf/cmake/FindWASISDK.cmake | 23 + samples/linux-perf/host/demo.c | 198 +++ samples/linux-perf/pics/perf.ackermann.svg | 1349 +++++++++++++++++ samples/linux-perf/pics/perf.fib.svg | 605 ++++++++ samples/linux-perf/pics/perf.png | Bin 0 -> 92293 bytes samples/linux-perf/wasm/CMakeLists.txt | 42 + samples/linux-perf/wasm/ackermann.c | 38 + samples/linux-perf/wasm/fib.c | 32 + test-tools/flame-graph-helper/.gitignore | 2 + .../flame-graph-helper/process_folded_data.py | 325 ++++ .../trans_wasm_func_name.py | 213 --- 28 files changed, 3008 insertions(+), 346 deletions(-) create mode 100644 core/iwasm/aot/aot_perf_map.c create mode 100644 core/iwasm/aot/aot_perf_map.h create mode 100644 samples/linux-perf/CMakeLists.txt create mode 100644 samples/linux-perf/README.md create mode 100644 samples/linux-perf/cmake/FindWAMRC.cmake create mode 100644 samples/linux-perf/cmake/FindWASISDK.cmake create mode 100644 samples/linux-perf/host/demo.c create mode 100644 samples/linux-perf/pics/perf.ackermann.svg create mode 100644 samples/linux-perf/pics/perf.fib.svg create mode 100755 samples/linux-perf/pics/perf.png create mode 100644 samples/linux-perf/wasm/CMakeLists.txt create mode 100644 samples/linux-perf/wasm/ackermann.c create mode 100644 samples/linux-perf/wasm/fib.c create mode 100644 test-tools/flame-graph-helper/.gitignore create mode 100644 test-tools/flame-graph-helper/process_folded_data.py delete mode 100644 test-tools/trans-jitted-func-name/trans_wasm_func_name.py diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 9789b1744..df487039c 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -16,6 +16,10 @@ #include "debug/jit_debug.h" #endif +#if WASM_ENABLE_LINUX_PERF != 0 +#include "aot_perf_map.h" +#endif + #define YMM_PLT_PREFIX "__ymm@" #define XMM_PLT_PREFIX "__xmm@" #define REAL_PLT_PREFIX "__real@" @@ -3601,104 +3605,6 @@ fail: return ret; } -#if WASM_ENABLE_LINUX_PERF != 0 -struct func_info { - uint32 idx; - void *ptr; -}; - -static uint32 -get_func_size(const AOTModule *module, struct func_info *sorted_func_ptrs, - uint32 idx) -{ - uint32 func_sz; - - if (idx == module->func_count - 1) - func_sz = (uintptr_t)module->code + module->code_size - - (uintptr_t)(sorted_func_ptrs[idx].ptr); - else - func_sz = (uintptr_t)(sorted_func_ptrs[idx + 1].ptr) - - (uintptr_t)(sorted_func_ptrs[idx].ptr); - - return func_sz; -} - -static int -compare_func_ptrs(const void *f1, const void *f2) -{ - return (intptr_t)((struct func_info *)f1)->ptr - - (intptr_t)((struct func_info *)f2)->ptr; -} - -static struct func_info * -sort_func_ptrs(const AOTModule *module, char *error_buf, uint32 error_buf_size) -{ - uint64 content_len; - struct func_info *sorted_func_ptrs; - unsigned i; - - content_len = (uint64)sizeof(struct func_info) * module->func_count; - sorted_func_ptrs = loader_malloc(content_len, error_buf, error_buf_size); - if (!sorted_func_ptrs) - return NULL; - - for (i = 0; i < module->func_count; i++) { - sorted_func_ptrs[i].idx = i; - sorted_func_ptrs[i].ptr = module->func_ptrs[i]; - } - - qsort(sorted_func_ptrs, module->func_count, sizeof(struct func_info), - compare_func_ptrs); - - return sorted_func_ptrs; -} - -static bool -create_perf_map(const AOTModule *module, char *error_buf, uint32 error_buf_size) -{ - struct func_info *sorted_func_ptrs = NULL; - char perf_map_info[128] = { 0 }; - FILE *perf_map = NULL; - uint32 i; - pid_t pid = getpid(); - bool ret = false; - - sorted_func_ptrs = sort_func_ptrs(module, error_buf, error_buf_size); - if (!sorted_func_ptrs) - goto quit; - - snprintf(perf_map_info, 128, "/tmp/perf-%d.map", pid); - perf_map = fopen(perf_map_info, "w"); - if (!perf_map) { - LOG_WARNING("warning: can't create /tmp/perf-%d.map, because %s", pid, - strerror(errno)); - goto quit; - } - - for (i = 0; i < module->func_count; i++) { - memset(perf_map_info, 0, 128); - snprintf(perf_map_info, 128, "%lx %x aot_func#%u\n", - (uintptr_t)sorted_func_ptrs[i].ptr, - get_func_size(module, sorted_func_ptrs, i), - sorted_func_ptrs[i].idx); - - fwrite(perf_map_info, 1, strlen(perf_map_info), perf_map); - } - - LOG_VERBOSE("generate /tmp/perf-%d.map", pid); - ret = true; - -quit: - if (sorted_func_ptrs) - free(sorted_func_ptrs); - - if (perf_map) - fclose(perf_map); - - return ret; -} -#endif /* WASM_ENABLE_LINUX_PERF != 0*/ - static bool load_from_sections(AOTModule *module, AOTSection *sections, bool is_load_from_file_buf, char *error_buf, @@ -3889,7 +3795,7 @@ load_from_sections(AOTModule *module, AOTSection *sections, } static AOTModule * -create_module(char *error_buf, uint32 error_buf_size) +create_module(char *name, char *error_buf, uint32 error_buf_size) { AOTModule *module = loader_malloc(sizeof(AOTModule), error_buf, error_buf_size); @@ -3901,7 +3807,7 @@ create_module(char *error_buf, uint32 error_buf_size) module->module_type = Wasm_Module_AoT; - module->name = ""; + module->name = name; #if WASM_ENABLE_MULTI_MODULE != 0 module->import_module_list = &module->import_module_list_head; @@ -3937,7 +3843,7 @@ AOTModule * aot_load_from_sections(AOTSection *section_list, char *error_buf, uint32 error_buf_size) { - AOTModule *module = create_module(error_buf, error_buf_size); + AOTModule *module = create_module("", error_buf, error_buf_size); if (!module) return NULL; @@ -4183,7 +4089,7 @@ load(const uint8 *buf, uint32 size, AOTModule *module, char *error_buf, #if WASM_ENABLE_LINUX_PERF != 0 if (wasm_runtime_get_linux_perf()) - if (!create_perf_map(module, error_buf, error_buf_size)) + if (!aot_create_perf_map(module, error_buf, error_buf_size)) goto fail; #endif @@ -4193,10 +4099,10 @@ fail: } AOTModule * -aot_load_from_aot_file(const uint8 *buf, uint32 size, char *error_buf, - uint32 error_buf_size) +aot_load_from_aot_file(const uint8 *buf, uint32 size, const LoadArgs *args, + char *error_buf, uint32 error_buf_size) { - AOTModule *module = create_module(error_buf, error_buf_size); + AOTModule *module = create_module(args->name, error_buf, error_buf_size); if (!module) return NULL; diff --git a/core/iwasm/aot/aot_perf_map.c b/core/iwasm/aot/aot_perf_map.c new file mode 100644 index 000000000..22700dcdd --- /dev/null +++ b/core/iwasm/aot/aot_perf_map.c @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_perf_map.h" +#include "bh_log.h" +#include "bh_platform.h" + +#if WASM_ENABLE_LINUX_PERF != 0 +struct func_info { + uint32 idx; + void *ptr; +}; + +static uint32 +get_func_size(const AOTModule *module, struct func_info *sorted_func_ptrs, + uint32 idx) +{ + uint32 func_sz; + + if (idx == module->func_count - 1) + func_sz = (uintptr_t)module->code + module->code_size + - (uintptr_t)(sorted_func_ptrs[idx].ptr); + else + func_sz = (uintptr_t)(sorted_func_ptrs[idx + 1].ptr) + - (uintptr_t)(sorted_func_ptrs[idx].ptr); + + return func_sz; +} + +static int +compare_func_ptrs(const void *f1, const void *f2) +{ + return (intptr_t)((struct func_info *)f1)->ptr + - (intptr_t)((struct func_info *)f2)->ptr; +} + +static struct func_info * +sort_func_ptrs(const AOTModule *module, char *error_buf, uint32 error_buf_size) +{ + uint64 content_len; + struct func_info *sorted_func_ptrs; + unsigned i; + + content_len = (uint64)sizeof(struct func_info) * module->func_count; + sorted_func_ptrs = wasm_runtime_malloc(content_len); + if (!sorted_func_ptrs) { + snprintf(error_buf, error_buf_size, + "allocate memory failed when creating perf map"); + return NULL; + } + + for (i = 0; i < module->func_count; i++) { + sorted_func_ptrs[i].idx = i; + sorted_func_ptrs[i].ptr = module->func_ptrs[i]; + } + + qsort(sorted_func_ptrs, module->func_count, sizeof(struct func_info), + compare_func_ptrs); + + return sorted_func_ptrs; +} + +bool +aot_create_perf_map(const AOTModule *module, char *error_buf, + uint32 error_buf_size) +{ + struct func_info *sorted_func_ptrs = NULL; + char perf_map_path[64] = { 0 }; + char perf_map_info[128] = { 0 }; + FILE *perf_map = NULL; + uint32 i; + pid_t pid = getpid(); + bool ret = false; + + sorted_func_ptrs = sort_func_ptrs(module, error_buf, error_buf_size); + if (!sorted_func_ptrs) + goto quit; + + snprintf(perf_map_path, sizeof(perf_map_path) - 1, "/tmp/perf-%d.map", pid); + perf_map = fopen(perf_map_path, "a"); + if (!perf_map) { + LOG_WARNING("warning: can't create /tmp/perf-%d.map, because %s", pid, + strerror(errno)); + goto quit; + } + + const char *module_name = aot_get_module_name((AOTModule *)module); + for (i = 0; i < module->func_count; i++) { + memset(perf_map_info, 0, 128); + if (strlen(module_name) > 0) + snprintf(perf_map_info, 128, "%lx %x [%s]#aot_func#%u\n", + (uintptr_t)sorted_func_ptrs[i].ptr, + get_func_size(module, sorted_func_ptrs, i), module_name, + sorted_func_ptrs[i].idx); + else + snprintf(perf_map_info, 128, "%lx %x aot_func#%u\n", + (uintptr_t)sorted_func_ptrs[i].ptr, + get_func_size(module, sorted_func_ptrs, i), + sorted_func_ptrs[i].idx); + + /* fwrite() is thread safe */ + fwrite(perf_map_info, 1, strlen(perf_map_info), perf_map); + } + + LOG_VERBOSE("write map information from %s into /tmp/perf-%d.map", + module_name, pid); + ret = true; + +quit: + if (sorted_func_ptrs) + wasm_runtime_free(sorted_func_ptrs); + + if (perf_map) + fclose(perf_map); + + return ret; +} +#endif /* WASM_ENABLE_LINUX_PERF != 0 */ \ No newline at end of file diff --git a/core/iwasm/aot/aot_perf_map.h b/core/iwasm/aot/aot_perf_map.h new file mode 100644 index 000000000..3e6583c5c --- /dev/null +++ b/core/iwasm/aot/aot_perf_map.h @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _AOT_PERF_MAP_H_ +#define _AOT_PERF_MAP_H_ + +#include "aot_runtime.h" + +bool +aot_create_perf_map(const AOTModule *module, char *error_buf, + uint32 error_buf_size); + +#endif /* _AOT_PERF_MAP_H_ */ \ No newline at end of file diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 519c1edc1..2d3013467 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -444,8 +444,8 @@ typedef struct LLVMProfileData_64 { * @return return AOT module loaded, NULL if failed */ AOTModule * -aot_load_from_aot_file(const uint8 *buf, uint32 size, char *error_buf, - uint32 error_buf_size); +aot_load_from_aot_file(const uint8 *buf, uint32 size, const LoadArgs *args, + char *error_buf, uint32 error_buf_size); /** * Load a AOT module from a specified AOT section list. diff --git a/core/iwasm/common/wasm_c_api.c b/core/iwasm/common/wasm_c_api.c index 29da8e22f..456ce505e 100644 --- a/core/iwasm/common/wasm_c_api.c +++ b/core/iwasm/common/wasm_c_api.c @@ -2234,7 +2234,8 @@ quit: #endif /* WASM_ENABLE_WASM_CACHE != 0 */ wasm_module_t * -wasm_module_new(wasm_store_t *store, const wasm_byte_vec_t *binary) +wasm_module_new_ex(wasm_store_t *store, const wasm_byte_vec_t *binary, + const LoadArgs *args) { char error_buf[128] = { 0 }; wasm_module_ex_t *module_ex = NULL; @@ -2290,8 +2291,8 @@ wasm_module_new(wasm_store_t *store, const wasm_byte_vec_t *binary) if (!module_ex->binary->data) goto free_binary; - module_ex->module_comm_rt = wasm_runtime_load( - (uint8 *)module_ex->binary->data, (uint32)module_ex->binary->size, + module_ex->module_comm_rt = wasm_runtime_load_ex( + (uint8 *)module_ex->binary->data, (uint32)module_ex->binary->size, args, error_buf, (uint32)sizeof(error_buf)); if (!(module_ex->module_comm_rt)) { LOG_ERROR("%s", error_buf); @@ -2337,6 +2338,14 @@ quit: return NULL; } +wasm_module_t * +wasm_module_new(wasm_store_t *store, const wasm_byte_vec_t *binary) +{ + LoadArgs args = { 0 }; + args.name = ""; + return wasm_module_new_ex(store, binary, &args); +} + bool wasm_module_validate(wasm_store_t *store, const wasm_byte_vec_t *binary) { diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index c7906edbe..d93bb682e 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -65,7 +65,7 @@ #if WASM_ENABLE_MULTI_MODULE != 0 /** * A safety insurance to prevent - * circular depencies which leads stack overflow + * circular dependencies which leads stack overflow * try to break early */ typedef struct LoadingModule { @@ -1333,11 +1333,15 @@ register_module_with_null_name(WASMModuleCommon *module_common, char *error_buf, } WASMModuleCommon * -wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf, - uint32 error_buf_size) +wasm_runtime_load_ex(uint8 *buf, uint32 size, const LoadArgs *args, + char *error_buf, uint32 error_buf_size) { WASMModuleCommon *module_common = NULL; + if (!args) { + return NULL; + } + if (get_package_type(buf, size) == Wasm_Module_Bytecode) { #if WASM_ENABLE_INTERP != 0 module_common = @@ -1345,13 +1349,13 @@ wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf, #if WASM_ENABLE_MULTI_MODULE != 0 true, #endif - error_buf, error_buf_size); + args, error_buf, error_buf_size); #endif } else if (get_package_type(buf, size) == Wasm_Module_AoT) { #if WASM_ENABLE_AOT != 0 module_common = (WASMModuleCommon *)aot_load_from_aot_file( - buf, size, error_buf, error_buf_size); + buf, size, args, error_buf, error_buf_size); #endif } else { @@ -1367,10 +1371,21 @@ wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf, LOG_DEBUG("WASM module load failed"); return NULL; } + + /*TODO: use file name as name and register with name? */ return register_module_with_null_name(module_common, error_buf, error_buf_size); } +WASMModuleCommon * +wasm_runtime_load(uint8 *buf, uint32 size, char *error_buf, + uint32 error_buf_size) +{ + LoadArgs args = { 0 }; + args.name = ""; + return wasm_runtime_load_ex(buf, size, &args, error_buf, error_buf_size); +} + WASMModuleCommon * wasm_runtime_load_from_sections(WASMSection *section_list, bool is_aot, char *error_buf, uint32 error_buf_size) @@ -6501,6 +6516,7 @@ wasm_runtime_load_depended_module(const WASMModuleCommon *parent_module, bool ret = false; uint8 *buffer = NULL; uint32 buffer_size = 0; + LoadArgs args = { 0 }; /* check the registered module list of the parent */ sub_module = wasm_runtime_search_sub_module(parent_module, sub_module_name); @@ -6547,16 +6563,18 @@ wasm_runtime_load_depended_module(const WASMModuleCommon *parent_module, LOG_DEBUG("moudle %s type error", sub_module_name); goto destroy_file_buffer; } + + args.name = (char *)sub_module_name; if (get_package_type(buffer, buffer_size) == Wasm_Module_Bytecode) { #if WASM_ENABLE_INTERP != 0 - sub_module = (WASMModuleCommon *)wasm_load(buffer, buffer_size, false, - error_buf, error_buf_size); + sub_module = (WASMModuleCommon *)wasm_load( + buffer, buffer_size, false, &args, error_buf, error_buf_size); #endif } else if (get_package_type(buffer, buffer_size) == Wasm_Module_AoT) { #if WASM_ENABLE_AOT != 0 sub_module = (WASMModuleCommon *)aot_load_from_aot_file( - buffer, buffer_size, error_buf, error_buf_size); + buffer, buffer_size, &args, error_buf, error_buf_size); #endif } if (!sub_module) { diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index d4d1cff02..3af56e8b6 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -85,7 +85,7 @@ aot_add_llvm_func1(const AOTCompContext *comp_ctx, LLVMModuleRef module, uint32 func_index, uint32 param_count, LLVMTypeRef func_type, const char *prefix) { - char func_name[48]; + char func_name[48] = { 0 }; LLVMValueRef func; LLVMValueRef local_value; uint32 i, j; diff --git a/core/iwasm/include/wasm_c_api.h b/core/iwasm/include/wasm_c_api.h index 606b9ff82..0d62c2751 100644 --- a/core/iwasm/include/wasm_c_api.h +++ b/core/iwasm/include/wasm_c_api.h @@ -517,10 +517,21 @@ struct WASMModuleCommon; typedef struct WASMModuleCommon *wasm_module_t; #endif +#ifndef LOAD_ARGS_OPTION_DEFINED +#define LOAD_ARGS_OPTION_DEFINED +typedef struct LoadArgs { + char *name; + /* TODO: more fields? */ +} LoadArgs; +#endif /* LOAD_ARGS_OPTION_DEFINED */ WASM_API_EXTERN own wasm_module_t* wasm_module_new( wasm_store_t*, const wasm_byte_vec_t* binary); +// please refer to wasm_runtime_load_ex(...) in core/iwasm/include/wasm_export.h +WASM_API_EXTERN own wasm_module_t* wasm_module_new_ex( + wasm_store_t*, const wasm_byte_vec_t* binary, const LoadArgs *args); + WASM_API_EXTERN void wasm_module_delete(own wasm_module_t*); WASM_API_EXTERN bool wasm_module_validate(wasm_store_t*, const wasm_byte_vec_t* binary); diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index b40a3440a..e40e94885 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -183,6 +183,14 @@ typedef struct RuntimeInitArgs { bool enable_linux_perf; } RuntimeInitArgs; +#ifndef LOAD_ARGS_OPTION_DEFINED +#define LOAD_ARGS_OPTION_DEFINED +typedef struct LoadArgs { + char *name; + /* TODO: more fields? */ +} LoadArgs; +#endif /* LOAD_ARGS_OPTION_DEFINED */ + #ifndef INSTANTIATION_ARGS_OPTION_DEFINED #define INSTANTIATION_ARGS_OPTION_DEFINED /* WASM module instantiation arguments */ @@ -419,6 +427,13 @@ WASM_RUNTIME_API_EXTERN wasm_module_t wasm_runtime_load(uint8_t *buf, uint32_t size, char *error_buf, uint32_t error_buf_size); +/** + * Load a WASM module with specified load argument. + */ +WASM_RUNTIME_API_EXTERN wasm_module_t +wasm_runtime_load_ex(uint8_t *buf, uint32_t size, const LoadArgs *args, + char *error_buf, uint32_t error_buf_size); + /** * Load a WASM module from a specified WASM or AOT section list. * diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index d8ceb714a..a07ce5866 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -6043,7 +6043,7 @@ load_from_sections(WASMModule *module, WASMSection *sections, } static WASMModule * -create_module(char *error_buf, uint32 error_buf_size) +create_module(char *name, char *error_buf, uint32 error_buf_size) { WASMModule *module = loader_malloc(sizeof(WASMModule), error_buf, error_buf_size); @@ -6058,7 +6058,7 @@ create_module(char *error_buf, uint32 error_buf_size) /* Set start_function to -1, means no start function */ module->start_function = (uint32)-1; - module->name = ""; + module->name = name; #if WASM_ENABLE_FAST_INTERP == 0 module->br_table_cache_list = &module->br_table_cache_list_head; @@ -6138,7 +6138,7 @@ WASMModule * wasm_loader_load_from_sections(WASMSection *section_list, char *error_buf, uint32 error_buf_size) { - WASMModule *module = create_module(error_buf, error_buf_size); + WASMModule *module = create_module("", error_buf, error_buf_size); if (!module) return NULL; @@ -6479,9 +6479,9 @@ wasm_loader_load(uint8 *buf, uint32 size, #if WASM_ENABLE_MULTI_MODULE != 0 bool main_module, #endif - char *error_buf, uint32 error_buf_size) + const LoadArgs *args, char *error_buf, uint32 error_buf_size) { - WASMModule *module = create_module(error_buf, error_buf_size); + WASMModule *module = create_module(args->name, error_buf, error_buf_size); if (!module) { return NULL; } diff --git a/core/iwasm/interpreter/wasm_loader.h b/core/iwasm/interpreter/wasm_loader.h index 8b0dc77d6..676770ee2 100644 --- a/core/iwasm/interpreter/wasm_loader.h +++ b/core/iwasm/interpreter/wasm_loader.h @@ -28,7 +28,7 @@ wasm_loader_load(uint8 *buf, uint32 size, #if WASM_ENABLE_MULTI_MODULE != 0 bool main_module, #endif - char *error_buf, uint32 error_buf_size); + const LoadArgs *args, char *error_buf, uint32 error_buf_size); /** * Load a WASM module from a specified WASM section list. diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index f0859e96e..dc96a194d 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -2994,7 +2994,7 @@ load_from_sections(WASMModule *module, WASMSection *sections, } static WASMModule * -create_module(char *error_buf, uint32 error_buf_size) +create_module(char *name, char *error_buf, uint32 error_buf_size) { WASMModule *module = loader_malloc(sizeof(WASMModule), error_buf, error_buf_size); @@ -3009,7 +3009,7 @@ create_module(char *error_buf, uint32 error_buf_size) /* Set start_function to -1, means no start function */ module->start_function = (uint32)-1; - module->name = ""; + module->name = name; #if WASM_ENABLE_FAST_INTERP == 0 module->br_table_cache_list = &module->br_table_cache_list_head; @@ -3035,7 +3035,7 @@ WASMModule * wasm_loader_load_from_sections(WASMSection *section_list, char *error_buf, uint32 error_buf_size) { - WASMModule *module = create_module(error_buf, error_buf_size); + WASMModule *module = create_module("", error_buf, error_buf_size); if (!module) return NULL; @@ -3206,10 +3206,10 @@ load(const uint8 *buf, uint32 size, WASMModule *module, char *error_buf, } WASMModule * -wasm_loader_load(uint8 *buf, uint32 size, char *error_buf, +wasm_loader_load(uint8 *buf, uint32 size, const LoadArgs *args, char *error_buf, uint32 error_buf_size) { - WASMModule *module = create_module(error_buf, error_buf_size); + WASMModule *module = create_module(args->name, error_buf, error_buf_size); if (!module) { return NULL; } diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index a216b4e29..688f1c2c1 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -60,13 +60,13 @@ wasm_load(uint8 *buf, uint32 size, #if WASM_ENABLE_MULTI_MODULE != 0 bool main_module, #endif - char *error_buf, uint32 error_buf_size) + const LoadArgs *name, char *error_buf, uint32 error_buf_size) { return wasm_loader_load(buf, size, #if WASM_ENABLE_MULTI_MODULE != 0 main_module, #endif - error_buf, error_buf_size); + name, error_buf, error_buf_size); } WASMModule * diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 3b01f05cd..13b738f9e 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -508,7 +508,7 @@ wasm_load(uint8 *buf, uint32 size, #if WASM_ENABLE_MULTI_MODULE != 0 bool main_module, #endif - char *error_buf, uint32 error_buf_size); + const LoadArgs *args, char *error_buf, uint32 error_buf_size); WASMModule * wasm_load_from_sections(WASMSection *section_list, char *error_buf, diff --git a/samples/linux-perf/CMakeLists.txt b/samples/linux-perf/CMakeLists.txt new file mode 100644 index 000000000..e9882c835 --- /dev/null +++ b/samples/linux-perf/CMakeLists.txt @@ -0,0 +1,63 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(linux_perf_sample) + +if(NOT CMAKE_HOST_LINUX) + message(FATAL_ERROR "This sample only works on linux") +endif() + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +set(CMAKE_CXX_STANDARD 17) + +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) +find_package(WASISDK REQUIRED) + +################ runtime settings ################ +string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) +include(CheckPIESupported) + +# AOT and JIT byd default +set(WAMR_BUILD_AOT 1) +set(WAMR_BUILD_INTERP 0) +set(WAMR_BUILD_JIT 1) +# wasm32-wasi +set(WAMR_BUILD_LIBC_BUILTIN 0) +set(WAMR_BUILD_LIBC_WASI 1) +# mvp +set(WAMR_BUILD_BULK_MEMORY 1) +set(WAMR_BUILD_REF_TYPES 1) +set(WAMR_BUILD_SIMD 1) +set(WAMR_BUILD_TAIL_CALL 1) +# trap information +set(WAMR_BUILD_DUMP_CALL_STACK 1) +# linux perf +set(WAMR_BUILD_LINUX_PERF 1) +# +#set(WAMR_BUILD_THREAD_MGR 0) + +# vmlib +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_LIST_DIR}/../..) +include(${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) +add_library(vmlib SHARED ${WAMR_RUNTIME_LIB_SOURCE}) +target_include_directories(vmlib INTERFACE ${WAMR_ROOT_DIR}/core/iwasm/include) +target_link_libraries (vmlib ${LLVM_AVAILABLE_LIBS} -lm -ldl) + +################ host ################ +add_executable(${PROJECT_NAME} host/demo.c) +target_link_libraries(${PROJECT_NAME} vmlib) + +################ aot + wasm ################ +include(ExternalProject) +ExternalProject_Add(wasm + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/wasm" + CONFIGURE_COMMAND ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/wasm -B build + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + BUILD_COMMAND ${CMAKE_COMMAND} --build build + INSTALL_COMMAND ${CMAKE_COMMAND} --install build --prefix ${CMAKE_CURRENT_BINARY_DIR} +) \ No newline at end of file diff --git a/samples/linux-perf/README.md b/samples/linux-perf/README.md new file mode 100644 index 000000000..5a8dc578f --- /dev/null +++ b/samples/linux-perf/README.md @@ -0,0 +1,90 @@ +# linux perf sample introduction + +This is a sample to show how to use the Linux perf tool to profile the execution of a WebAssembly application. And how to use the [Flamegraph](https://www.brendangregg.com/flamegraphs.html) tool to visualize the profiling result. + +## Build and run the sample + +There are two Wasm modules and their instance will be created and run in the sample. [The first module](./wasm/fib.c) is a simple Wasm module that calculates the Fibonacci number. [The second module](./wasm/ackermann.c) is a simple Wasm module that execute the Ackermann function. The target is enable to profile the execution of both two modules separately. + +```bash +$ cmake -S . -B build +$ cmake --build build +``` + +### Profile the execution + +```bash +$ cd build +$ perf record -k mono -g --output=perf.data -- ./linux_perf_sample +``` + +Enable to use `perf report --stdio` to do a quick analysis of the profiling result. + +### Visualize the profiling result + +Need to download Flamegraph tool from [Flamegraph](https://github.com/brendangregg/FlameGraph/releases/tag/v1.0) firstly. + +```bash +$ perf script > out.perf +$ ./FlameGraph/stackcollapse-perf.pl out.perf > out.folded +$ ./FlameGraph/flamegraph.pl out.folded > perf.svg +``` + +In this result, you'll see two modules's profiling result and all wasm functions are named as "aot_func#N" which is a little hard to distinguish. + +![perf.png](./pics/perf.png) + +### Separate profiling result + +[process_folded_data.py](../../test-tools/flame-graph-helper/process_folded_data.py) is a script can a) translate "aot_func#N" into its original function name in name sections, b) separate the profiling result of different modules. + +In this sample, we want to separate `fib` and `ackermann` profiling data from _out.folded_. In [demo](host/demo.c), we decide to name the module of `fib1.wasm` as `fib2` and the module of `ackermann1.wasm` as `ackermann2`. + +```bash +$ python process_folded_data.py --wabt_home /opt/wabt --wasm_names fib2=./fib1.wasm,ackermann2=./ackermann1.wasm out.folded +-> write into out.fib2.translated +-> write into out.ackermann2.translated +-> write into out.translated +``` + +More scenarios: + +if only using one wasm during profiling, the script can be used like this: + +```bash +$ python process_folded_data.py --wabt_home /opt/wabt --wasm --folded +``` + +if only using one wasm during profiling and specify the module name via APIs, the script can be used like this: + +```bash +$ python process_folded_data.py --wabt_home /opt/wabt --wasm_names = --folded +``` + +if only using one wasm during profiling and specify the module name, which is same with the basename of wasm file, via APIs, the script can be used like this: + +```bash +$ python process_folded_data.py --wabt_home /opt/wabt --wasm --folded +``` + +if using multiple wasm during profiling and specify module names, which are same with basename of wasm files, via APIs, the script can be used like this: + +```bash +$ python process_folded_data.py --wabt_home /opt/wabt --wasm --wasm --wasm --folded +``` + +if using multiple wasm during profiling and specify module names via APIs, the script can be used like this: + +```bash +$ python process_folded_data.py --wabt_home /opt/wabt --wasm_names =,=,= --folded +``` + +Now we have two flame-graphs for two wasm modules: + +![fib.svg](./pics/perf.fib.svg) + +![ackermann.svg](./pics/perf.ackermann.svg) + +## Reference + +- [perf_tune](../../doc/perf_tune.md) diff --git a/samples/linux-perf/cmake/FindWAMRC.cmake b/samples/linux-perf/cmake/FindWAMRC.cmake new file mode 100644 index 000000000..586263edd --- /dev/null +++ b/samples/linux-perf/cmake/FindWAMRC.cmake @@ -0,0 +1,14 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FindPackageHandleStandardArgs) + +find_file(WAMRC_BIN + NAMES wamrc + DOC "search wamrc" + HINTS ${CMAKE_CURRENT_SOURCE_DIR}/../../../wamr-compiler/build + REQUIRED +) + +find_package_handle_standard_args(WAMRC REQUIRED_VARS WAMRC_BIN) +mark_as_advanced(WAMRC_BIN) \ No newline at end of file diff --git a/samples/linux-perf/cmake/FindWASISDK.cmake b/samples/linux-perf/cmake/FindWASISDK.cmake new file mode 100644 index 000000000..5cdfea41e --- /dev/null +++ b/samples/linux-perf/cmake/FindWASISDK.cmake @@ -0,0 +1,23 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FindPackageHandleStandardArgs) + +file(GLOB WASISDK_SEARCH_PATH "/opt/wasi-sdk-*") +find_path(WASISDK_HOME + NAMES share/wasi-sysroot + PATHS ${WASISDK_SEARCH_PATH} + NO_DEFAULT_PATH + REQUIRED +) + +string(REGEX MATCH [0-9]+\.[0-9]+\.*[0-9]* WASISDK_VERSION ${WASISDK_HOME}) + +find_package_handle_standard_args(WASISDK REQUIRED_VARS WASISDK_HOME VERSION_VAR WASISDK_VERSION) + +if(WASISDK_FOUND) + set(WASISDK_CC_COMMAND ${WASISDK_HOME}/bin/clang) + set(WASISDK_CXX_COMMAND ${WASISDK_HOME}/bin/clang++) + set(WASISDK_TOOLCHAIN ${WASISDK_HOME}/share/cmake/wasi-sdk.cmake) + set(WASISDK_SYSROOT ${WASISDK_HOME}/share/wasi-sysroot) +endif() diff --git a/samples/linux-perf/host/demo.c b/samples/linux-perf/host/demo.c new file mode 100644 index 000000000..8ea446ed4 --- /dev/null +++ b/samples/linux-perf/host/demo.c @@ -0,0 +1,198 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include +#include + +#include "wasm_c_api.h" + +#define own + +/* return a copy of the file stem of a file path */ +static own char * +stem(const char *file_path) +{ + char *base_name = basename(file_path); + char *s = strdup(base_name); + char *dot = strchr(s, '.'); + assert(dot); + *dot = '\0'; + return s; +} + +static void +guest_i32_to_wasm_i32_array(int *args, unsigned argc, wasm_val_t *data, + unsigned datac) +{ + for (unsigned i = 0; i < argc && i < datac; i++) { + memset(&data[i], 0, sizeof(wasm_val_t)); + data[i].kind = WASM_I32; + data[i].of.i32 = args[i]; + } +} + +int +load_run_wasm_file(wasm_engine_t *engine, const char *file_path, int *args, + unsigned argc) +{ + wasm_store_t *store = wasm_store_new(engine); + // Load binary. + printf("Loading binary...\n"); + FILE *file = fopen(file_path, "rb"); + assert(file); + + int ret = fseek(file, 0L, SEEK_END); + assert(ret == 0); + + long file_size = ftell(file); + assert(file_size != -1); + + ret = fseek(file, 0L, SEEK_SET); + assert(ret == 0); + + wasm_byte_vec_t binary = { 0 }; + wasm_byte_vec_new_uninitialized(&binary, file_size); + + size_t nread = fread(binary.data, file_size, 1, file); + fclose(file); + + // Compile. + printf("Compiling module...\n"); + + // Use its file name as the module name + char *file_name = stem(file_path); + assert(file_name); + + LoadArgs load_args = { 0 }; + load_args.name = file_name; + own wasm_module_t *module = wasm_module_new_ex(store, &binary, &load_args); + wasm_byte_vec_delete(&binary); + assert(module); + + // Use export type to find the function index to call later + wasm_exporttype_vec_t export_types = { 0 }; + wasm_module_exports(module, &export_types); + int func_to_call = -1; + for (unsigned i = 0; i < export_types.num_elems; i++) { + const wasm_name_t *name = wasm_exporttype_name(export_types.data[i]); + if (strncmp(name->data, "run", 3) == 0) { + func_to_call = i; + break; + } + } + assert(func_to_call != -1); + + // Instantiate. + printf("Instantiating module...\n"); + wasm_extern_vec_t imports = WASM_EMPTY_VEC; + own wasm_instance_t *instance = wasm_instance_new_with_args( + store, module, &imports, NULL, 16 * 1024 * 1024, 1 * 1024 * 1024); + assert(instance); + + // Extract export. + printf("Extracting export...\n"); + own wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + assert(exports.size); + + assert(wasm_extern_kind(exports.data[func_to_call]) == WASM_EXTERN_FUNC); + const wasm_func_t *run_func = + wasm_extern_as_func(exports.data[func_to_call]); + assert(run_func); + + wasm_module_delete(module); + wasm_instance_delete(instance); + + // Call. + printf("Calling export...\n"); + wasm_val_t as[4] = { 0 }; + guest_i32_to_wasm_i32_array(args, argc, as, 4); + + wasm_val_vec_t params = WASM_ARRAY_VEC(as); + wasm_val_t rs[1] = { WASM_I32_VAL(0) }; + wasm_val_vec_t results = WASM_ARRAY_VEC(rs); + wasm_trap_t *trap = wasm_func_call(run_func, ¶ms, &results); + assert(!trap); + + wasm_extern_vec_delete(&exports); + free(file_name); + wasm_store_delete(store); + + { + nread = nread; + ret = ret; + trap = trap; + } + return 0; +} + +void * +load_run_fib_wasm(void *arg) +{ + wasm_engine_t *engine = (wasm_engine_t *)arg; + int args[] = { 40 }; + load_run_wasm_file(engine, "./fib1.wasm", args, 1); + return NULL; +} + +void * +load_run_fib_aot(void *arg) +{ + wasm_engine_t *engine = (wasm_engine_t *)arg; + int args[] = { 40 }; + load_run_wasm_file(engine, "./fib2.aot", args, 1); + return NULL; +} + +void * +load_run_ackermann_wasm(void *arg) +{ + wasm_engine_t *engine = (wasm_engine_t *)arg; + int args[] = { 3, 12 }; + load_run_wasm_file(engine, "./ackermann1.wasm", args, 2); + return NULL; +} + +void * +load_run_ackermann_aot(void *arg) +{ + wasm_engine_t *engine = (wasm_engine_t *)arg; + int args[] = { 3, 12 }; + load_run_wasm_file(engine, "./ackermann2.aot", args, 2); + return NULL; +} + +int +main(int argc, const char *argv[]) +{ + // Initialize. + printf("Initializing...\n"); + wasm_config_t *config = wasm_config_new(); + wasm_config_set_linux_perf_opt(config, true); + wasm_engine_t *engine = wasm_engine_new_with_config(config); + + pthread_t tid[4] = { 0 }; + /* FIXME: uncomment when it is able to run two modules with llvm-jit */ + // pthread_create(&tid[0], NULL, load_run_fib_wasm, (void *)engine); + // pthread_create(&tid[2], NULL, load_run_ackermann_wasm, (void *)engine); + + pthread_create(&tid[1], NULL, load_run_fib_aot, (void *)engine); + pthread_create(&tid[3], NULL, load_run_ackermann_aot, (void *)engine); + + for (unsigned i = 0; i < sizeof(tid) / sizeof(tid[0]); i++) + pthread_join(tid[i], NULL); + + // Shut down. + printf("Shutting down...\n"); + wasm_engine_delete(engine); + + // All done. + printf("Done.\n"); + return 0; +} diff --git a/samples/linux-perf/pics/perf.ackermann.svg b/samples/linux-perf/pics/perf.ackermann.svg new file mode 100644 index 000000000..c2e1d8747 --- /dev/null +++ b/samples/linux-perf/pics/perf.ackermann.svg @@ -0,0 +1,1349 @@ + + + + + + + + + + + + + + +Flame Graph + +Reset Zoom +Search +ic + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +load_run_wasm_file (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +invoke_ii_i (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +load_run_ackermann_aot (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] run (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +wasm_runtime_call_wasm (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +all (11,485,868,643 samples, 100%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +aot_call_function (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +linux_perf_samp (11,485,868,643 samples, 100.00%) +linux_perf_samp + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +wasm_func_call (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +invoke_native_with_hw_bound_check (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +start_thread (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (11,484,299,081 samples, 99.99%) +[Wasm] [ackermann2] ackermann + + +[Wasm] [ackermann2] ackermann (1,569,562 samples, 0.01%) + + + + diff --git a/samples/linux-perf/pics/perf.fib.svg b/samples/linux-perf/pics/perf.fib.svg new file mode 100644 index 000000000..a1db059a7 --- /dev/null +++ b/samples/linux-perf/pics/perf.fib.svg @@ -0,0 +1,605 @@ + + + + + + + + + + + + + + +Flame Graph + +Reset Zoom +Search +ic + + + +[Wasm] [fib2] fibonacci (1,321,095,222 samples, 93.80%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (382,407,564 samples, 27.15%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (15,340,273 samples, 1.09%) + + + +[Wasm] [fib2] fibonacci (1,359,552,763 samples, 96.53%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,408,481,525 samples, 100.00%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (27,274,310 samples, 1.94%) +[.. + + +[Wasm] [fib2] fibonacci (62,450,767 samples, 4.43%) +[Wasm.. + + +[Wasm] [fib2] fibonacci (1,408,481,525 samples, 100.00%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,388,674,508 samples, 98.59%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,170,751,868 samples, 83.12%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,408,481,525 samples, 100.00%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (120,820,158 samples, 8.58%) +[Wasm] [fib2.. + + +invoke_i_i (1,408,481,525 samples, 100.00%) +invoke_i_i + + +[Wasm] [fib2] fibonacci (1,375,872,224 samples, 97.68%) +[Wasm] [fib2] fibonacci + + +load_run_wasm_file (1,408,481,525 samples, 100.00%) +load_run_wasm_file + + +load_run_fib_aot (1,408,481,525 samples, 100.00%) +load_run_fib_aot + + +[Wasm] [fib2] run (1,408,481,525 samples, 100.00%) +[Wasm] [fib2] run + + +[Wasm] [fib2] fibonacci (42,420,273 samples, 3.01%) +[Wa.. + + +[Wasm] [fib2] fibonacci (1,266,323,684 samples, 89.91%) +[Wasm] [fib2] fibonacci + + +linux_perf_samp (1,408,481,525 samples, 100.00%) +linux_perf_samp + + +[Wasm] [fib2] fibonacci (280,259,464 samples, 19.90%) +[Wasm] [fib2] fibonacci + + +start_thread (1,408,481,525 samples, 100.00%) +start_thread + + +[Wasm] [fib2] fibonacci (2,334,521 samples, 0.17%) + + + +[Wasm] [fib2] fibonacci (666,394,609 samples, 47.31%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (943,121,736 samples, 66.96%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,169,581 samples, 0.08%) + + + +[Wasm] [fib2] fibonacci (1,169,581 samples, 0.08%) + + + +[Wasm] [fib2] fibonacci (194,755,877 samples, 13.83%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,406,148,966 samples, 99.83%) +[Wasm] [fib2] fibonacci + + +all (1,408,481,525 samples, 100%) + + + +wasm_func_call (1,408,481,525 samples, 100.00%) +wasm_func_call + + +[Wasm] [fib2] fibonacci (5,943,602 samples, 0.42%) + + + +[Wasm] [fib2] fibonacci (1,408,481,525 samples, 100.00%) +[Wasm] [fib2] fibonacci + + +wasm_runtime_call_wasm (1,408,481,525 samples, 100.00%) +wasm_runtime_call_wasm + + +[Wasm] [fib2] fibonacci (1,401,486,191 samples, 99.50%) +[Wasm] [fib2] fibonacci + + +aot_call_function (1,408,481,525 samples, 100.00%) +aot_call_function + + +[Wasm] [fib2] fibonacci (531,941,563 samples, 37.77%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,406,148,966 samples, 99.83%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,061,055,435 samples, 75.33%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,408,481,525 samples, 100.00%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (1,403,816,880 samples, 99.67%) +[Wasm] [fib2] fibonacci + + +[Wasm] [fib2] fibonacci (800,646,766 samples, 56.84%) +[Wasm] [fib2] fibonacci + + +invoke_native_with_hw_bound_check (1,408,481,525 samples, 100.00%) +invoke_native_with_hw_bound_check + + + diff --git a/samples/linux-perf/pics/perf.png b/samples/linux-perf/pics/perf.png new file mode 100755 index 0000000000000000000000000000000000000000..fc4c0930fc289d0df96fea22d05dee3e0737e1b8 GIT binary patch literal 92293 zcmX_nWl&pP7cSBQ#VIK+1&S4dyM*GUxD@xG#ft@p0>Qmlu?DBb-Q9yjk>c*|e$)56 z_x{Laa%M8K&pB(a_1F$mQIf^MB*R2PLc)>zD5Zvkg!}~w3B?x!4RPg%`=@in7qYXO ztOQc&2*obq#|ukwMR6pgiWsa1lb49!uN*(>I3pq9cK!Px_c;`rAt62I$w`T8KnxFB z(EaJ>43_RcH#Ru`$V_+RJAQ8&Pr$W0chW9ktWX_*j%EUpVpJe9E=Hg2H&%e~9j96* z1R=>W6}xGXGj8u@jx^kFsWj~QJU3n+xm;h^QdxVt`2P6uOUQNPdFrB5%jkK3N>H~* zBOIpU-I{ts=>2@jzcSNyh7t2j(x&by9IfO-=DhT_^^SZw5XiAHu!>|SFcl&J6Ir@BFCaE$1p+&WGGeJw$4Gp%NP-ua>M&5Qq z<9!D93o`|i7(Kc#`!C-nFml)nK>D=&2#+W$g!prABpAbKt$rlg*3m}KPu!Zi8UlkQ zkds{3pQ$cK3}H9T=V4jThTiAyk4X8Skd1RIo>yonxJK7d{hTqC1!Yfb{A{S%jFeX+ zs6-+Mij-H&KoBarUHmMN7rpiJ>}%<6~ekxoT%Kkj?9Q1sp*9OfAWN7|_1@Z2MPR|FTioc-K`yjS>VaOf0 zB$1ia7g8Y>Vf&1+CN@ZNkL&94{c#HO2zFm{*LyP_SeTt3J(PpH)Tq?K#L_NnlV zJDQK|$q}k3JB8TSW6?1-zt?_OMfCoOazuP;oz4>BO|CPnX?Q4nJg#!=3oUZ|XamhdVz%6uLg@&alUq7Z=gSSA)wd+{6(i@VlEE>IW|0HN3Kw zkY#ri2Z8>23u-D+zSwRRCMhZDNLK?t6GHK%meBfxx*3l=mvuRcIYZ3E98GU=6Jlbg zuU%bUN1_wVs!T^BwcCY>Dl7oZ9!jWZYM}}fpeeFj)yDT#?;bTw(|tUF|EB|)-RnjI z6nWXZ&!0R=y2~3jC&>1OF0)U6_THfueB@i$)I?2- zXUFh82*6)H>gj>s12BWzuIHHrZb#-aMx%@HT3m68;LE`UY7hzorIBQj+0M+!_gTt)x?K$`y8gx#0zeh?(jv>J%z!mu93OIf)|bzAJTb zb8zq*JUq$)<1dxLZynZ7Vxw_J+dF)5ezSl)CqW~c+>pN2E4`U>t+aj|Ge@&r;87-J zHf)zfj?;~DHf_dn6LWt->w^PJw;FwWUtCNKup{ET_@==k=Zo)}iPjv#Kt$m0$r&Mpj{)p5{N%KlCVzF!X zCK1=)H6*ImOk0RdQ`oNme#4q6KY=}9k?mkr|?DArboDiqptBd`EGQ@@+ z>MV4Ho3gXo>i0G{js{h|U>{;X(j}DP-i@oU*sZ6eU4INgBma#XUJLaR6)HuvIqnqk z!B;Ew1a%}0#Q`XXr`-xzpRdFQLDvA63zaCwq}_>2z|zw+g$(n=^EsGTp$x73&u`UW z38&=o5!HF_j|R7f#U(XNty^|q^EjtjXrOoR(4R~xyUTZTr5%9sf#ysR#VSmOSPzO% ztwyrxZDr2B+g0FbA=XjYS)5;RUU*Ce)z1tX`}HP&_#IDW8Ro53h{<>s@-+F!wLThn zQ5$pb0&5rTNTq8N=A3=p<1zbzh=}ywr-u}NigLqtAH3twa!nGj`T*!t*`qnhe2@JWxfzvQZ@*d||vGe%& z$T13SM&dlXdrykVIoDY0VnL?i{ z6p%AZkw)23v}}J4c@j3;^O~}%*B8J!F#(@bl}~wy2vleN%nRnW#iNm&LtJoNZsu?=XgFQ5sY|tI>X9Ep8 z`kB#}B*0T11N#pFkC;N!{u1wZBLbSCU4VhEz;=~rrKB0Jov+Reqh0~1f==sXGU$*J zS;++2Lk^4JY^7o8B+k~o6sY^*%D>lx@=0i_?&VCmYeR++WG<(J?8(F3_J}l*nM|RE*UP zB>n{j-$K*c=;M_LS`z0~x(*R7&PJT!1KGpXGqUqWrqTeS4GHd99SR*hUq1@8Y^<${ zGN%A!$7KoR3yyM<0zmf6muWdBcs@`y_M5g=_&?sLSW2-Z6 zw|p5GeEaqT=TwD5x5UoaCvu`+ZvJ#W3?o1<4(8H6uQ@HjP7LYhL5R+9pE-LJ`!e{Q z>HWdCz`#HvPaq{Pqf`-5$NFhzWUhMkhWVN^7ATrx=mSGXI@3MG@m-!F!+StD(Cm32 z#F)azYx`hiTiS>Lb2B_*H$H^>4W{L%kYJ2$P4DmfR1Cjpu8{{_X0(23*fy@gA_Jh? zYqxLkwr3=Y#|ZdakRtN%eOhCfr~SUDn%1oIr-4Qi^@nEL7viIsYtw#yl#lj;dMX|I zbP(7e1_;eGS)}?Yr?|TT%bm6(IJ^$}8;ehgx6>@tu*fV2VCzX(^xq5CqqrQ%!^ay} zgnPCPYDhkiZWlUhs8b04ZXUqgb%oOCgz1?` zyT>(RtklPvara8dEKA8ChXrq^e7_a?)C?t)*6sZuoEaGv-LBr~NJ*!0gm%n0#bd=8 zugXNWJ{TcKI)8YwWY*Rl5^C{w0EDfgO##%Jnggv+SJA0g6xx-FAGbV z6O$2(&62k|JA0l-|9N?|3s|^4V44!Y#r%|Y<0j(V)03u@n-HVfO?vN(ZMi7zArOel zj0qgwtU4M=6AN9A4_aYW&K zY#S30cqrOvR7C}mG0=gg=dqoGKJJ_FgiYs#ghC?ItY#!~c(pe_6$zTmF}$;|VXOr3 zOOZMRs+itzM3cbnI2s#mlK`p&18gzNs64gLs&3+Q>?(YYenmN9dMoSp>5liUarHF+ z1lPxi-84soKq8r`7$-ZAIu~S|Z0sJ8l@Gct3K~oRJ(8P=WvMS0CI5P1vUjR>E{Bos zRk@Pt^sce1?$)JR=oiuU#6166O-YjvvQb_-W-BT&VPoef8*k7fYA`sZc_V2B_&Cg< z^`QL~_2llS+V%MwBzayu@&ITNi=qM$-MDIlyPIrtE_TBiJuuv+hahR%hn1g}$J}W^ z7lZL1n&Q(9b+|JVxfel~9#&=?`Z) zGi8e>_E>;>m}+j4?f0)W$u&g!5e1vIn28%sCtP4CGyfehp>5=50pZ5zj^*N((X+IC z-0RM5mpO=VqC0~B!JjfUyM4POUsooF-GFtqV9&zofq$w^QJt%l`{1@U`v$Y;$ZQ4f zYja%yTH7OA!1Ke0tygQw&K0|4hj#w{ELNM0KcO$>1>a*~e^WJo)wAb%9Mf!O5G--} zG8B5j{|KmdQQm$Sv9yG>enoq7e*lWip)4gy$A^<-C)$tG!yOxQM0$QVQbeF3>@;D0A!yZD6I-k26-ePJ@lvV-|n|Rue{zj0MCiky{?Pl_m4UI z`#DKW9d(NY9L%X6$y^lai&>ugbM)knb{{^Kr!}|bA!sRPd8<;NiT%@_4n=I}%6wY5 z%j<>%4C=TWt!(Wk=y+z00aC%G6e|J5c>lmu=^DeiPPVq;8biafhf}Kt$f=wh(7~Ju z-)R;Fcn)ymjfumBs)fo#C210c)_&0!+6Ib%luaJx4Otk+=eaK~D@2q{GGK*9U?bAr z^gI$hTfm9X^Sf{GFa3^9v=So{a%mfzD0Mmw{Izj$)I7U4U^mCO3KBJ#3e(a1zeOF@az5=;(~*jILP;ymjPU!*h=dxOM-5-`gq6=HBy^xvD+vOew_42SnSDc7bH`D=*_7y;{GU~=yxL;1*g$M+GA z1AjkT%%WT_Pwm=OX_7Ti?pHoj97%w}fC2oXkX$^dxd=QEcJg7VU4zcw!Ad~lav6h{ z;-j_pdJUCmx8QQOun^y!_o6I~`P$nuxYz)eDG$W@T^J>p9jsr9-hZvVlcNGp*+aMP zGOy2+fT_5=^1Hjn7rPtzH+0=#`5R%sY7&RD_g$H*dFQ)A;K+L-cuc)9pS_D!%C(s5 zN0ap*q**$%N?)er*O!KU@T8e;c>K)SzC24{afqbsP+!O|GQ{m74PxI04KFRc?Uobn z{)W!O^q_8zr_6&Y@O>>Ia^?VJ78(f}U&EKiJQ?^AA~uh4qR-r}KFmRS%IR5PuE zNb{xpbhi5if>`2gYU%VL)Q;tQ6S<${yEd2fwLdO=vJb5k_%hw5 zFus(=7rPeNX(F~Z?F@}Zxo=y-R9P>Ht{8O!nR(tEr{JTPl0s&FhH;AaQkUJOm}3*z z`~!DsRt2Cf2H5k?a-U~QFahSlu;z4tHcW*O+TSkH|G^)l zNe(1I^+qbLm5AIPzG8)F>;wG#=exo`xCe`1OAL9n!xl;`yAwP>w33o5FC(q1jWiL> zR+@NksJ-qXizCEslBYlzqlVx4&2O>uLjfaPC%>|=hWwb-;$XBMWt+d+7CHA5ALejl zIbBtRVXEGAK&7qc9eBQqvxs(nFbC#JpU~YJJ_`&gDo7Z8EI9+z5Ty<|W7cPxtA?ZD zs{1EX8jJjfS%viPQ?YsIG<*F74p~M3BZ;|o3QF7pF>v^UO7=-=OuwT56Bd#RK|QTg?CbEX3NH8rs$?c4LJ?`2mPjnbEoCp?)VXjh+CwjQb89Fr`$U@sVoI z(%9)-Jw|DKZT%k=BiM5@lrm)4d*Fr5a5?EicWaow?NYRwz6<0?>`>9|Gal(AIEuN_ z5#b-t;(8r-$g3E+fkQDwLNcH|JDJvD48lofi9>@(#8 z7ZS!f_oW4rMomA&s9Z}l9^?BoYjR*?w&6we$$9}Pb}Be*ceJ-8*Nq*JjofpI(bYMX z5>Z~5%x3yY{cC@;e+BnyJZvBOsKpaX1?9(E4DzRNP0b8t9t!Ddy)71My;PY1MsKZz zWR{-AQ@n6pF%eJLH?`jFWun*vUwT}R-L=(#$f$0-A6mL+q-W-{ibod{%6pa}dj{s5 z^iP|>Uw=VypG~&(IB_qm>lOZii&df&eI>1enI4dPNT(37} z+T?t0)1oNQF}VHp);XzByv`TVJEHFF{^r%S>Cbu}UcT2t^kQ5bpXZf2<3pe7e~dLy z#i9FYLh!rq{%2aTBk{6+Y7{T<<}0k*4&xUZWPK`pXg2sC|FEH>e^W zVgv7cuUqL^KV*@7N%x%&XPr)nmUO|^s!)&smvcIJ6T?EctHRJYzx&E3(r(i3oQ7j{ z_vST-RjM@8DALjRL(b1CU|BL#FDOQ~>f;@$JSrnGErp#HS$4j7u|mnHsqeMObEW1?L+l zfgISTw18rrw+CIX=ZnlfZDv#-jk{FOAWI_V&WyZz-C>MmrowZ;@ogRkw~aUci2z)i zOD{`ff6w;Jl*4QYZU3`Q*()9o2NsvzwO&T%2}^QL6p#pCxCz&5vk02cOn~qvf5qrK z8%Lg$r;Hz&Im% zQ7T=bV(_HjnoOj3;M_adUJrb~+MH{TP^s3?P?tZ+`j}k{?MW5p48A6#e&Ro|)r0ac zQ)#{brd*fNjd+!tK#)1P!eqwPry1(*1ZQBgogeHlSFv&H?KcDK z?5UYRNn5(A8J)D(?OZQ^HIH#`(2-0lI2c~nGezj=cYp6_!P_m_ z#XvN(uN^kgzm0S8{g*(($U3DD04&z4JJ<7xFUn=E&$taY>KiCtz6(=CVT$QUGm-|= zdZCSd1&fFp z^({SVaLITz92vttawa?deQ6wect3=p%z6WXuTou{b~wVU)>q@2|JOWY?-<*V*8AB;x6Gi~G&H>Gr{*BZ!E|mn~m;SW-t7_xH`PMu86)KjKpzzH&eN2{i!7$HIq?Z?xJ@qSjAw)(zMg8&o@iq({tgYPIWuh-XpMcm6y z44^YE!p5^VrZkHFvVFweb@yO3;;bFEx&JNM@R!XdrXxv}T#fDY9ShfIH^*GDw8(KK zp_P{nZ0zszqNkF}=;VjrlKLhq>+P&dN|&DES@$Uk9Zn^mPE{rkq~Ir%!M@mOMR;0AsqZesAJnVE^b`$Zj; zGyo6Yc*p}c&(;htWZns5x>h5E3cN=;ksACJ;5yMS-^kU-=zPMC5>F;vGe=f>>O86b zDiKxbr_&wNO^$?#gg?4x)|5_wjxZj8b_oDwPN372rBi#Uy(S9DVK$*trY8i>w@!+q zI8FJ{TJ9i{z@&rQF_*hwPhy};2wUVmJhdu|BXspl?R8pG)A9=<4`XcI@*gIZD;=+h zT01QR5a0wp;5+~S^$8-uz1r|?R?4^`Y6F*(Q$cDhz*;`*@0tyhye*TczEJ~*)zKN0 zhaP(3UTZRlf~bzJ`cOfoz>JZ1|nkHViF;RUmhejJMb?96qDQUX_CkQ zE+7nse9H6%WABhnb3E>}=`~%e9ugk9@47T2Y{EAYzu6FG(CI?sC^|}T!|JLv2uBSD zg{5Z>(SObPS1K+ItC|i6!O7}ZYL`vTnU{66SM|=bp!Zq^`Xw)7%Vf5GnQ(P0O*7__ z_OgzeV=O^pkPR#*;A8^d^8%w#MA|kOaU0veI#uVFg5Tokr`Md0dtiDnj#F@t81u;} zcXS_w2k!sDz!ndPj{QW6u~?pwuw_9;#%1ebx7fo)S)ItIvhrwE%%PVv@1W9;j%`+h z;pf>J0N=#7zt8$d9E9J#ugf?zPm_M^fe2OvCF{A_RI6EwMr)SBF0Z61)tElz>5iR%DgSw?-xu3yNLm_JgWreJ0fby61Q* zMfE#}IVujXCt|Zn$LP;Tf770UXAL85l_XnU!|JjI$-6PEn}#W+w9{Scp8F$IT>YPx z^vc#rxO{EDghH|^-)ZHaR}3xov3Q}dAgZ72?tB^5;P^0o`Jawr$P@5q`g(QfoQld* zAS$m0|Nc6)kkt!gTX(j;jJogMf8twAD<_N2xe99;I3)BDBe;U+6ARm2pBmr_38*|l z#hFCb?kFzVutqbJ(7E?5V*T2J)4eq;cEt0U3YA{3@1{I@tvN(ux^GjB$f2>UWJPO) zheSByloOS+XYqH4)t(TPi!+IZv-gfiUu;h5XjpW|T1I>B#PQpyjx++i7#hF3$Lc3p zD>P!KW9+r|D$W=*q?`}-Ovvrw0A`r(WP=R47^$3%Efs@GV=Sr4z~KCVsLGg#tm2gl zYwE0tlL7u%9F89>h>NAL|Nt+-s!gB6iY6t#p?Nied`;3l6a z$$Nz0cvg8vzviX%{oyqY?y?FyAwnIo`Ox~A3i$9|?M^f!UO0==dCJ$Cpr`HT%}}zz z!C_6UH^*xMcjOSV}kKNEs#N`^{USp_h3_%Kc?A-y*a&dRPm8A$P?(6Txz^4 zxf~two8X9&i^B+uBZYh1MRMmexjgF2g{+r@mq^x_FnRhv%&lu)<|BBixkb|#q_a}^ z1MlR>Jl8RP2S85^38VML`B^m%-=-d2?1iHO?N%IPqVpKa3uGmWsAwG^^#^9Br4_Xt zSq7CA?r#-O|E$~*kPTz-^+0$i8I2eQlcR`P5hr*8-9flc3Ja!{o|gUnGW}F zD`l4_5Z;zGmaC->>(_)wTONbr&$c2}L~ql0jn$&Fvg%Ze1huW8+Pdyei!n}L=44qS zjZ;0je0DVkx^|`U68KcCJ?dKz-Y&Y7OZ5pi5YT1W3vXJ$dv>z}Q)PlKaSV;VI>Lyu zLrg_X`Ub)|4Kfl_mAl4dqfo?p_$;R-i~D7{qVVZlc4m_IAdM7nyTd`M3mh)>WPwQPK%988bKpq2E2fX{ieQg&o#qp?d ze<<@1O^=wA--twv^*f1-5nTb0?OJb;>qG@4DoQi8pBVvD(wpj!Z^7D6dJ)_+_?lZY&vTJl8CGjFhYVsY^;$<;0*q^Pn z17Z#Az5LimEDuU^QWm1Q;&#KkFREOo{VcL^icuGv&59S3U>}r2iux$hdL&&RgGa`j zt^ALH-0(k>PdLmg{*FDq;i}mG98gJZ9RAH+dIHe=1tI2TYHj4bg-aFd0eY-G0%R*F zV1YF`_ytTUTFi~~pKK#&TVzxZqrgKq1es1uof(&lF;MUC=CQZF|Pv3NE2g`*7omIv~nKsCTq$vjJ=RbyyuL$}$ z)Apl{f-@(XayqUmi#6F%htzFjCRF8pbi6q6$G0!3&=Rc+or{%Q&@VH6&bPEJLr-ln zA@}=^z|{F4JqX_lKD-`Qt(T)KfBTl2Vo((^^@g$iLZk0Wj`z|RZvmK(`~{%wL}?zs zU863Wn%7P(1|Oep-Fp9VkhGt2L(fxw{+yKs>W=?xQZho~fa_-;5kst#Xew@}Y7Zfu znA$&XD1b?Z=&}6p`NP&7LaR{1b2hA*L0+dxo@TlC5_@xB%%y2Tp1i#^SnXoZXz5>iAwhfm4(Qs^t^g}PFQzF*9Mof& zP&h;BUxFkcr8Q$O2b_mMf4ZtLTT8Uj;{9G|E{JL5tka>m?;B|lhYRKA+s^Ut(UxR1 zl^WOR5b9(|`oK)Pt|)u(JU%tbk{n(OQudXGf#}>9T$P`6qy&l{k@DA-#FpSO`XHH%x&R*Z} z!e3a@P`5AhTYyEO2?y#F89}8?sNhU@#ZxWU zDEmB+Lx-O z4L506xS)y^^;kO*_x$j@0W?ikR8N5Fyz9jlZ;)^3`93J=7lD5G{zea4rqS053v_`x zdvNP?MXy;AscIzZcb(${1wv|%rC#-9U5Nd-;@KLxIr?ZZ+Q+b2Vwc(YX9O+}Z;DKd z$4oTEA|$7B!Ae;-G{nT!B@54lHoTUOQ-XMp1<75DsBk!OICO-Hq-UP0z?81WoXB08 zlT3^`(DhsO`xtko;4Z82nW4LtAzuEi$XL$KI10(MfY})rdMO4Qno7x_xRfiB{Gvb$ zaxG$A?Nfh^B`G{GNhB}a>vZ4c$xC750^je-?4{N;iqOAWtZy3ol$R*Lst7w>J?>?t zJnK+BuWgCcW4$LiUT*w0G~8Xtm%Tx6#s_{oPpV$>&y~PblwAGohts~KjBZev@NArc8bgnv+%OePv3zHSN#OATDF$#Xn9zmAlBm*VrF^Fc16&7l@3%-ja$eKe?S33T6jCG}1qynTNEITI z9;1IwHku_vDAxg_89(<6!92|*x*OTr;I;%_?~U(WPvj7bIJ{tm$kgEXpsF=G>AodP zv~oILVgz-qzn?whQZ-y@P{iZq@67b2)M6?uP3;uE%2VNIy>S`$X(5Wn(Ip|5H>qCk zY?Zh-Uy`8@Qss#M-0O9YhK9yPLdC=Bo);ZKPBq=*GZ|LEie=6spCDT=gE;7cEXrd z0)}n;P7XuR6mv%w`D@RlW0%9kq&n*Y(B3&m<^ush9*7-=xy0}(3Lh14iU24*=j-pZ z>_~LERulfUC*64F#N~5%s8@{ZCq%Ujsmj8uPM+44O@9%rsN{=YyAWq(Tc?qE8cFCo z1P=+eu$}vAx1$^$!HS1)UU#ieN)j>5Mq{t;V>UE1H6Ni~pJdwqX{mn*r^HKK z*YL7tb-=N3dn2#dCru7KrcPU^ZxVY&r!QZuw4FK4H?yne>gb3l46?a{8pu+d-mSyNx5o->WPS>8&t4Jf_L)t z2QGzJVCvB5qdQ5=U?;M0Sl3pCMVg1`KX1?8z>o=hY0uoW$#XcO+_n!=Nx&$%$jayW z>vhgScS;^UTeYbI$CI;bIK!dCk4|P3@5D{o;u(~m62!hyGgW|T56lDe-uEi{^u7t- zo@}{0T`@1-Upw16C($kH~Y=18J4zb@vGi96|IA{G4sWJ z;hrS)&KhkRy8r02eLl;Nd5@MsBl7P(-=XML0BOu@rtO)l$yK%L>-=2{ zZZf45vLp8D{Q-@FHHfT-kJ#cdC+J15j)R|JU+hq~tWzD+FW2y*?$K1CR+HKGWKN4D zyAV?4tvthA$+oP^K8P&s1-*S>776;+Y=IP8nUp@OTTG2|@G$G81d~Nx?jl4B?2d)1H`hP?p zus`8m_{a+X&z22@CouU;-a84ooR(*14^PLZhi+so_RA)kgEP*D%P#-^U1pp2E@7#R zZ#-GYKNk`DNm~EQ`@2g)$<1Rd*fnjZd9|`!BQsqy#S#N>FRfy{yO$ptW^z01;*RjHt~n-CkvKMZ>SvlG131?2+8AaQ`_CDek>zU zWU8NOcB&YzjrvY~I`*Jo9sMu;;JNcJ+q(BJGr#lCF<6j$ppl3q1A==>I#9DLq%z^-1Tpgx<%_IUD+pKxN>+AY-ZiwLp*`qX(3T&c3tUnPk%@4p-kI~ z6uGW$T_i;tHn0C0KEB5XzlO`D=dPFyt&Wwr9E zq-a7x<7^U|&sK~q*-Y(U%vR;m|CT;AL?L-GHH#jA^WK=ih`Mq=;aSe2fBsPyR51Ib zn8R>)NkVKxKy}bmG1!2Z0T@1FXH6ePPyTNhpp`j8?-zR5O5(1zXhoSz^VzOjPlRI> zEzrJ3j788KrvNNayxKHRJm`ZbtY zyT4~4o>Uuu%s2OZ65^XAGzY`+Opy? zs+hasyG6;QmI8JrjYn%8Pps4AxFVD7RJ(z$=3gd$kkjVer{GNpB-x1}63>cL)jUVZ zq#3g?);{{(I4Yn_9FeF0l?TkCYj2ol^G)wv=nl z$pujMfN7Ozoeo?QCGGm_WrkQr;1t*B&N_;w>oTviBeYf_qrRHPDAFWg1Ka!3K;@F! zlcFE;;`S>WUkO-xFf}(>B*WdMn)J$D!tMMc6!BP;Ud)_x*68*yhabw|ub|v(pS{zW zXw7;Og)S1M^I;WE5fBJtk=~B0Fm9@LR}qP$Cy|G!bsD8V4=$_8{&T z>~VQ5;O|Q9#;^{6qj}4>zE5m}rX6`$Uv{lzB>nh;d%jRsxxLUy@$Nm1<5myd8Iqh% zbP1D=L+B%wntf!89?-|j3B|PL@rI<3_Lo-|`LUC<4?VU7gaEr+V;Mw=8j7+rR0^Md zf*T2DB!_V`J-Tg#HD3xrmj^2kO;{5=SIyT(x(hwd4rznuc--}0{`Wt=AJ7iJ zbL8BkhPMA{5<;bAbda@c72A4}eloovo?NRtm+>-94*IeM?7wsZze+Gr1?dKCLJqnX zuGZA>*E1q1XjJiGJE&cASLp2XagC}UuNAMUZx3}YDyK_~L=Z7mc7-AKdqb;+*8dx* z^~727d!`a;!(SN}CED7f!M#c?obm}WC;y$(QZI&TtATn!tkC~ulZ#5J7bFWGYJi8rskQdxyZXXOGlZK#eYmDcYnqd3582&vUotLy0-l(+YY(7Gpym`McJ` zRkQK+uD22;YbF-I!U|?u%&ZpYe#8(j(ZANI;*8e+t_vDrLl~m3k{|2hui~FG4Qy5h zOfgu#=QX1UU{^k7A>>McoqZmz8loacEpIhzF>0E3&KKU^?m48hxqth%lS;G%GKHCp z0Vn@lZ2>8EIV&vNJ3nUL7Y7s*!6w3u&mayMR3%o{N(AfTRjnJd9%l^5aGUqrpp%_^;D*6(&r4hRS zhB|N;yY=Epq*tBbY0VT1eu-qiz5HL?8|!dN)`{qe`Vs5<`;vDd%{l#TxEhD}Eu`zg z6c;7213x#CQ3Ai&7XC*Xcz>!o7Wq_}X^_W{x{4XGJ%Vm6s{Yn-UKpSsntG zn(7{DZn)3ca1nIvqmA7AT-_LxG_qdq%1;f&p130X`{3nO9_a0H8Y?;kGNksOW1{)= zRECP~KB-U5j$naP6}s0IADw)?gF91sV<%`u1ovUummJcZ2HMx0q$tYjQ+7vH%#@o+ zG4iZ_7HsWrWaUT7fOq}8-a~2!qh(S%yTv_!jOVyrjtBvZn{SA@0f&>3^YH!T1Ck;| z$*VPO$&ah-<3JFn?0F_w}sk7SCx+E5g?jz^(XpojkiC(%SXnh5kph*KTs>~3(R1XE`L+*TQtq_ zb)$9NJSVbdx8$n(c+JwEM7`oc4RIe)*!#FhY7 zP`|rm{N-nre~1&2_H`q@yY^*Ks-YE$=j8xurp`>)Ix6??rH%tu}d9s*W!d+SAwG54O6BxaBB z=+S%SqJKNp6zi1zS!j)a3CK<;H}2%vbLu#dgRlM;C{@33ri$yW|Fu)9nBGX_5SiJa z&#+NP`0Vsu&vhjsOaU2T57+?y1PF8o5wA>MV(?78!ddQnIn!l}q+d3lVtkeN^jnudW-ZN8KNHf=kVT4iry*6bF0Z~l=T!m0EX>L&q?x34A^JEA2@W=iSq z5AfF2w{4s611J{!aRqV^zV0ksvG{nmqXK(HfMCKjwI|g%p+U@SP>m(8%fz68VcYi| z`wQ^c&n`zv{jwJVB41XAf&$%_vSdqkU_FgskHEDO8i#?gHjE@H(=P6cg0ZBt%S|G- zS_SX%3IGXP>cmvR5MzaPgLCza&2P5qah^3CVsp>k``+SiCxnA4O}wr6jJvwhmB9^T zc};6!VE>t6d$FZ_@^p(!d<2oAn6_iDj&X;iGD(Pr9dL_pi^tQ-k<8Dj zb;Yb^47~eGazGb|p)L;`;-{@o3ee#8m$fvgqGMHTD$F$tSj5P>J3wOyU#tHYa{u#{ zh2V`_qg&H{owe}Z9f0x@Z&f7txI2^)ZCdO&ngTi5yzTf)0h8$l}CM!cbmKl9O3G#%(Uhp{cZJ5%PNVE+_Wtpm%tjyn)#(!`eX@@ zrhe2fh*NhewjI9kQ5rIczpTL@h+yc7x9{F@N>21I8>`1ZAuAKR`BFQ4b>_vjDYeQ* zoVM&#I@mR=n6aOrQ7-nUd8U=|7`2zCYb+J-lo+>uw<`Z5^VzU%azBcX+X?F#1edeE zb_JDeb#+hC5{HVJT@!eZ4gxwasSc3`HKqx$tnL6>!ZOSStD&!^AW8}IP@m9~@VP{f z4|``Q@m@d71Qs@YjKgL&Wvd3=0wek7QK*a`T#xAAa3y`gTPFWLOBKKBdmc&0&K7*j zP*!yi$ zT%l{srrI_=HyWuZ@=o7L&oT@A%T#G10F&I(D>*nT;AR22j*OfSY*o#BJ&@uT-q0Bf z=F9?q)8fzI;!k*7LP5d=xBjPWoYR>5O}6~%;ZD$R`te^u08NrIxGp~9H}(W>gJM)_ zqwSXRC+jO3a7{#{^F+Sjmj9I1NW}$|HXV^oRr}6p(B>rP4C3 zy*C9Z!IbsL&~`$v)!!=eHA~Ut`DQ1Z+H(EMRY8N;#jia|MVZ0{Xs6F+w1s+&WxBr7 z^?qZa?owHpQyrPYpH!7^)Zt4#(-^vR#e5bD7U_bt0PNl3h6agM`}?ft&JXs>lEC@} zR`AFF&dmgAz8~&e8hXR}e6rjlG2F9sP*TNOmiRgR45Q~U@2nq(z~;!j-_89S1A=!# zK3kFXikc~)raa4-Ric+I*x_*B?z*G6sxT9RhD7E(*jQ~Z`?~k*HtZF2~ zp;IPq^OnpG*jp|*=wWq$aciDHase2f%|yx|a6`+zMo+z{a+~wkOhGi2`X4Jd^w;IS zxNgIeQT8iX87aufIchGZ3sY&TBGkSMKJIHU^+&%0vi_yH+ESRZrBvT1)+G3X!w+GyTc;Q(vlY?4qMem^k!mTwGefFPVEqtTugMhVc?=*b zdL>$TD^yem7-qmBqVHA(kr(Sp*JtA!&w`tNuho4qclpii{iF$Z?=NC7Yo3W|PH?9q zatqBp_<_)el=n;$v6oCn@t!CuuUlY)><_?HsZKJafg6u1<{ONd)u zm}yR!>r7wGAGA#Smk`^I-d~cd-W^d$jE6>$B2{kA}>=CpVQLVSgCdrdl>O zS0(bIEzL`0rdh0zML=&2zxHP#_WJ?XKNg{Iu^YOGPXgad`PFo?6STaLB_XdY-fMqh zra&?DgJi#gqkBE>dl}<7 zO1itdLrNOy?yjXvy58mU{k?wym%Hbl=ggd$c_a_!MEaIv*vB$tAU@?DRyiT+pT96b z%{n2uz!S0@b$;GhUzwmoMJ?Q1E@{Pwz8ZDI8@)>8q-#Fm2;5N8cOF^Bbh^wD@0gk` z@C5*y^MR3nlnG8g$#;-*bkY&tytjKWG|hxR+LFwRg2R=ZvHcuUIHdv0n_FQ~(zu`s zy^-p>ra2jE6+&B){rRE~DDgM88IANpC_?W0KbuY6tr$@7zNFFoGjF=2jqAWJDtWRv zhF^tZYb$-XBv5~elT8fGlV8k)T*8)Nmi_j5r;=(^LE7#ZfSMI4l@%BSTW}_RplnBU zfzq(jwzByY5CK8ftA20*eJK0W`>`t+_xJ#QydN@6snY6sE?%$a;%|)ChI||CiZaW> zLt*15;V2?+8GjY(70fB7F!OmThaOMmb~P$VoH_TOIl)g}(Jw`ZyIvvL?rcVXUJJl9 zFpMxx>=*-T>amyEY&)kwB;9i+duq7C0Qyyv9MYA9==lquf{oVlOR2kkyLWsy!JqS{-|SI7S+O*u}ooe1d@iQ|uznW2|Th7zpa+jll3Frf3N}r%dN-EnBeB|x^Kd!Sz zNkJue5c%4QPdMk4Vqh%0nGm9qqx1t`zk(Kma)nI7WrBQCl7btAC`UY23DEuT^=b$? zN>vaQ%2TMtrYlb;`+;tbJ9&3H$G9dfp`aptdc)wt2KrS^K---1bW>+Cp9|Y$p->lp6TiGr@*I%Ej^C4*~2WM&AmPcfsPvcggu<_?&ylTDo@+* zOk~+-^gcFHAE)({0{#WjFQ~QOBEqi8QGW!;h6GD%0vaapGE&Mttp}Y6_Ayo0JRci2 z9p0?#cXepNXo^rN37Z?U3(g4kGdz~4G(!}VvgeFHC-WL4qBA5O0Ms+c21;}Hc)z4w z_6`H?-`zNgN>@R2zW%ht4&K@V$XIXb70nAZk)$s)J0rd@3it8 zkf7v`CE*xHR?rjwqTDTa0NQ-9t55ZES?1B(k(}k|c@bph;W%25T z7V~Nn70zxnK_@Dk9d87aB5He>)m5Zs*~m~EgK*T_a=~)!On80Fp!=KC#=X59dHS}@DzA))yJ&oesn)b# z!wyHwv!WK8G+oL}wtl{TQ2QlHE%i zj`(B_ImU1UpOW2loBOa5N8>8{=t%kDcUK;e&tRI77IZG~kM^5{rJ)JEXG!jh|I?hs z^l?llct%bKz1MW?YG2>~K-Ry@71<;dafR!baNWC1vE4ShJQ`M9%-R8NtA;!=HC#ui z^NzB%(Q1MRvM&FaR}wu3?DiqcgogVB)YthvsYN3`!Yq*fY`{U#Z#?rwk%`IawdQw< zvy3b-Im^&g(^Mz_+}W_+r0CD{Z!eMIcVd_{*0$rn8B%`z+|z9oB19ynSz#t)tlVO} z-djz7**|384%qzO5T+1&F!qw8;%I_7MfzZB1c67CR_gRkh!F~v6uP)ka(gbk73&Yp zkq#BnxtxYuy^^uLs~{*?i* zNv7B$Ulg~koVWz(?M-*GN8MLdX@&5c_V>A&$)J(4&o`5CZ>$UG5~_$P$+)Kf?I(vz z)zP~WspaK389^`eU1g0Hra6!94Trl@ORL~oiptilmcb=fRV2Ok`N#vxxlA?*l=F+80Zr5Ok=8URKzf--qamvvuE+9>03S@;S&1$cB@564u^&e zq=0K%tG8m?l4dcHPa>yCci?nyWI$>3#kzn*^J}ux!XS&;X!vN%tGFxp6skEc|D2^8 z$L*# zPrkv`zC$nKuWwJPE>=-T#wCd4q|*9z`+w>j4j~j=AM2T1G_%4+h$!>nr1_l@HDhrQAzKDA>4pu%|oEOnyhQbPtSb3*WQSNCiHZ^ zSx%vSopjc;026TX_8`ruG;+wr#dzGv6PGs5y>)LRNyDGBC;blh^PU8}vmdj-=QiNB z#$=Z?$Crf(Hj#A)`Wg30K1o2A5#K&j+UZk^ujR~Ft#hGfQ$`%yst=f>NROo{*_*?O zE$K71j`O`H-1ObQM0Y5{&#opxkynXoA3v*^_P}_2iwKtaNX*$WbwBL=0>)F;Y$L|d zpjl7`q+MQZ+#2u?AA;CwS zt9J_`o7QNIfK7$aBdI*v{c!9!Pg8A=%lf!WMl$wMzu!7clg)rN2EedNly8FHDK8o?tK z+@}L7!R=bLRfn%kD?$GZ`F%2dWGo_tr6Ut3% zY1#D9PyUhPxWebcaAU~q z^Cw8WVy7!&uO;p)eBk+wg>i{(6m#v_mdn>Dm^f)$bll0McvJlZNOtH}tPzB)r6Q3T^{7^_a zArJFE!{32<4Y|Sa$+D41p>ny}isqXqtOT?FVIeX{%Z4V0+q(SfPCg6aTAh__m5^?@ zmDeM#7vA+MUdsesJhlhBxB%>&8cI*AMIsii2`k&K!WzNr8>b44x1p6`s#s(D95Iv* z6{zj+^vekJ9Yp&c9D=+0n7W;?BG1^1j#!^y&D2K$M#Dedd_t&fPu9y=d3AlBX(P{a zQgLjkeKICMbI$kw$-tf2+3z5g+4`ZfZAw2(YI5X%A9FD=dl0hdVA!32knJxJ?@(pt zhTrtIg!PE1^Rwk!MnwsczO+Li%2%X!Zje8H(Z~drtp4#F>&(scDqK)9P^D}%`EfzND(BYi`{=VN8J=ErlqO}2 z^-0x(=C2k|Kze6N{IsSHlh-dPxhJZIJsl^S<%|$GFrw)Xt;pGIN8mUNir;IKZmF5S zHOh9!wzzo4Dv(8U*}|i4())|%Pr1eJ9X9FPn1>i2l&Tscbc;V0x_ z%Hit1L>&fRh^=*aHLqaek$Wiz>4vP!SGw;(X#)QwezUB%YaFd+&8j03akoJ&w48iY0=>7{o_f@k{TAZKW^)w)%9;0y%mjfPDb64zgv2+n^16SL@9&+ zo!$u|nSeUB5yts~Ff>tcH`)uE_QZ;w)sZUIB_#w54?rQ?Xv6IDA>rSUxGmR2FWbE5QR4k&66<>(p+*^jc^DPDoKRet z?a`aj@o>4(KGQL>-#9%U?_dQe=}FjU^Sca7X#>I

v>`UJ>F@`Z-7dProU^u(y>< zuKNy^&#DP;fNBqqps`G02szYoKH^pcEuUdab#KIzwYO7@;y{L3o&O8`EVmY~wQjJ~ z%*sj6&uNpEc$8_h^^N#Pj?IFiBxg3@7Q#^fnj!Jm47wk$`W{WZ1x`;~xr!p`c~2Ik zCY6pBMLB^qU{%)a>qimA_n(?i)iECN_zZ7T#W;xkvQMgX@-zQr&E+eV4z{IyKc0^a zzDLlm*{V0%ph=>d*0T(^qiLmLw>W9m3`UirwO+pYdVi${D*5I^tzWAM{|2(Yd47nJi58Rf ztHLJmgu_ns<_5!+t1VN&AM&(WEhvtf`S6QO6vh5VDpM1A3_223{W(>+7@;c@!MPDG zmrAYJ>@1Cu*43y$aTs>!A4cyH4=%X?U#j$t6~3NjADcDVu+0dVYdGspqK%*L|Cnz%Kg5Jdox(lktY4RfMH%)0`F_V1v`jt(5gwlTJH;>UeIB07E(G!Fj%}U)xzCxF?op) zc~wxUcCn_xhf^7;QK-*EQWD7~=JRt6Kd|I>ZUcB96F1oo*zPb5hA zdwQ|USCH{YcqOr@OQytj;fr$Yf#TsB9XH)aXO6k+q8@{b<|MG#2#ZVzn5|KBw*C%x zuw~|4guiV^uV~#M{OA2Y1I*`|w`<a+CR93M}7$!++{sqKT%&!)=i_TsZ-r(6`##V zL(L0|z|C_6H0kE+?ZEtTcd@rPP{kX`zNG+$5q9ZUWS&Anq6w+oob^0xLz{z)++g+d zD$1hQy$O*j5dco;b)s4>B;6i@mE2LM65Iyx=6c;}7(l45%|DD5CTWtPaC^$zELjnR4`w|!`A;>@qZyEv~Znp4}+TE^KxRr`{M%i)zaMxTqzHP z!>q=3W%KM~i2?1$ zfGUyXwE^ot9jH7(vE#g*jz|-!zUkPP!#;GP(O)N{4e!e}-Sy9(`!TDGEr9OP+jSqj zK4j!90-X6yTcmS};j=-?SBeJ~Qw|-nrEP*h2@c@VERZKq z8+SYx7-HrDwIZ*Prd$jZLlVr~#GbY;7KlU?J zzA}fBovhlqZn_c05KP)ywvHG&%{jIDDy=KKOG|vYchnyHW@ra-PO>gCZK7I^XE!x0 z>AK;S&HHOhH8P_1%uM1P=aT-GDt=~2+?#@en+C?z;zr{=4@m^+85jEbKA1i4zqvXH zRC)X*4clU;=2EX9U+-ZMcb1Flm5dqxxxWc7!m_U|2STF^)VeMpWpPy6p@cQ_7S z>^*)Lm@(t4OltyuvgNTBBfJ|v*ZrRcR&i{d|Ds(<2-@} zAY33-L}C`f!pgKKhE)H%5s}}+QZvFBUk4feVyOaTObpy^eOdk)Pqem6YtKti&eyTx zSM({^B0B9I*zI^ggJew;(-H3ieN;FCKcNkgPn^lKPXjhQ~;yu@Ly@+(?eNNP%Gm6xwo7$}foLe*7QZg^7R_eLRP%2l_4#_Hwa zG9>DG_hSIV_ptO?27dyvg?tOLP?+(aff7SUUdg-l2s8wDY#?cF2`$_$e1DZ}(;sun(8LeDw*tJMYv+nF~7USTUZ3TVvIVv^AF??X+O< z3}ITNNA&%}%_M!XuE$nBdjHEo+OJQfrCrLTC(nY{E4ExW1nwT1>xKtohRFB*;a>`; z@SilNWM+(ArOmP@d+PY`Bt>cS7KNrU1@@A_V}+q#n2d=jNNI(T}8XR?3(yzjoKz7jQgEge=r&SN{M%U-qt#wIUmm| zeRw+u${*}lx5FkZ?}Y~uN{uWSw|K1tO>teyS~3TcnTF&LB$I$-YmR;&o3W~qaF9Q@ zywS|(EIV@(ojt1PGfh4;Q7eMW6XrrM$AWGSo?FKzS!t%SDX1(zS54L}DtVh^mFxOA z?~~ZNvG;ZbaSo04n0_mU3%<0Vwys~gG$j^T|A_*_9vbpzXo7~(0e#UzPI|~vju`oP zB3@0EE=yZpshy?YjRI`36d5#S(WUzQg`H4Zb%U!Nt%YDs{ zy4MTCgH`)R{8R7#Td}00-aFXPWCzHxr>PV{-o4J%QLn4SNMh%{>c3&NKF#- zQ_g<|MY9@#bU&S2|DwI=SumKtRL|7Pe#3%LKKPRyTM91hT{!)c!Ss`4b0Ex8ds z9(Zbbi9r_7;k%6+c#2a3{S*0pO8!m7`9NjSi_(7Sor59;;dT9$5Z@VGEX?pU+a9llBx_tg;5a*M*)!EyBs}B+`mf+6M*Tj_!g`Tw&5fokJqH{0~ zTd1{a;PilP%dy`V9oRr}XnYZkBS(CE%W2HnG&^%K)PlPUI0=sqcIQmh4Lhy3qaOJF z8OjlF{CVam3$tOq-ynrcw`*nOv8?*Ug5P`A_TDI)*TZgsbK%<|Kc;F49%mA29bOi| zqOsznm$L{y_EP$2#iNx5m;ip3y(AOU;AQFkXujR8`dXf3q zOEdS*f3k@SirqGaqvMD<_eXiS1UuGO)Ql){bkydotF>7*j}lNHZdL_9*C_^@5K=M0 zbDTq&aew*{Jlb8k`s4a$ak9+@SJBN9IKr76*xUBAllR9YfyVtedUna$JhK8TS~Erk z78xmbMasBXB3gDSDV$|$ICk{_-{hcbW+{WZi(dBo>}&u^s50((*(T!nfsK~spSv6u*9=pG)r{A4Zp#8zm=1r zb=xIl&yzgs6K+2DMup3?>nHnF%MT5(HCZ=r`8uRU`A?OIB;!RD})}T?=vZfG2{D4f%g1V~@5FhijY~olm4#)8W9qt+KyucUJF80O7D0SH~JEpS8d8K5YWe1<#z3 z9P9?}AkGjyH#n_P^(Za9pUYeWd;jeT+7)@|i2>89A`R78R=rg$6!Ca)@R1vmNYueN zJ+x;D#V3zvCcNbkBh{r zoebqh;tJxgCDlNOpOiG9Twu|3o$#pq1M;=@fpwjijR!Y;}`0$Bz5j#Yg4{F4!E4iV$v($O4n0xjW>VR#eFITZNv#AKeZxh=_2l#7%5mTOTkB zWfoJOAm&SJ^bl`zj*N&Cqma4;iZ~p^I7~eX&^o{+S`-6gMFmR4E+mPmyCOf0T9FSY z&D-@-b#V20i6ba@U@MS^JqQae!@~cqJ)E)O84<rBJq4`_KRKz1Y+`e2>x6S9%FZa8XA<=`<%4|4$hMP zTTKxYL~*YItaU%dCJVg8lIWjm3kD5!yzbtyc1>#hUfA$88ER910RF4|r?32O>>Rb0 zzFUdVGIqwYbV^uSr^c@O?RrQJd1&|>@am%E=q9Dv))>@dt-jbK$$CW(#- z4JcDu`Ot;_EZk}Ik4-f~LEEm%cD}c0`@Oo*jv|d{EdTcf7l%!SwVc0R@`Ne(?NE2l){NS%jJaYYarVm#&&jDe16w zE~ZZlS+@~L?nyb3nWRw)#zqcu=ai16CMA`nrU*2he?c42G#*E$vgT7*|Md}w4PGXP zHh~5LGYLKSD{*}*mi}^SSMoc4-A50aFZDMQY->Y!q8VqkyB)jyaYoi?v8jpao9C-= zl-P_XyQyO4&>Sy!TCPyv=3tvEnvI%t=*=641xS+4clbA`Bfy8cdK*^xBj-2wWi-0u z;V`Azotaki&gRPIPCRMFZ@kQLxm`;q3>`|H5&>hk)x?N_fj8IbYBVkd@mvsKbx)63kvL& zo(iL+k_uwI4gV@)^TwcO+IFq-bp=6Xd59!e_qZqONZLi#HLTG@tLsn^de%xTc>p>$ z_PMA7tlZ1H$M$g)Xp{0;W@QL3MX6uRwa3hWR*;pE^(d}K)39t$%NS0t*gjFLWw|r` zFSGN{(({S$RO?vTGCRLbUqZ-{vY)*A6&ug)`d$e7tqvIjbEkcQXKcKumM%^eAE5zw z#x`))Xghx8XN381->2p?B|_n$xv%13(%LvCfhPRy3Ksu)9&OzIHm9>sH2U0)R2)HO|swo|9--liyfbUp1PvzfQrZs3~tC z&QwF|#eu0+^Z}2je;GBh2L?txqo& zxmMYDECxBvXxU;yC7(MEG~b4GZPA;k5ox=X8DVci)v`T|x}@k&K!NV(%3Nge&HXCR z0<_!^pDW~l_aiulgt!ex`IMQ*_|4m%RjJ&K|IMzJ(V?lDL?tk5aA2@OGqnVwYvZE*R^nb%U#3Cv8(WW!CiKXb z$!>(6C-}t;5+@u}|78vY*q-vxPhE=~oE*h2%^#k!M{isUmm|(4=CkSIg1(1FXWb;V zI6b;S&&7k`t2chSKu|8E32_UfBa0GShq2oP)?2~aTe)X0z$0!X&>KatP7LvU*aI$u zQye-}2}GDGAGbIV%eAqq4nOckd~g17&|B8QHYdn@$DVe{6yP?zjqAZ69B}fI+KM$%au$HR^=gtij72kgce`pj*9`U4?TTfsG=3$fky$golEg{%5;Svd@rqYdB z|7n?F;@Kqe-_VTnM8~&}|GQt|WHJ4&QM^t|!sU?%uTl0hn&)RNhB&TF(=dD$*4J>J zb{7Y`_fZh(hQTf%t@?8cJa|XrH|o7Nyi`-q&+0{r4Ar!3H2gNjY(vRIMTkhV>gtz) zL{$~p@XTh1zt!dXp+@y2lrj0M*wi-;ZA2<1N{P%W$!4N+t2S#B20lC#wen^Dwy?hv zd)3+++eq4t-Sz*_57MKDLg`AYP8j6+7JFry&G=5)WNCeWF{mEYF1do0XF?Hcg>)(c z=r6ovvCeqSUIJmPCUmc*DzBFO<`&W-7H+C2?w>M3Q^Z~o3POKi0@v%}-jVD|`bZ## zOOdC?fkwpri^=p2MtZ&xCZ_+>m!J&wl7@40ipv?g!ED>|{watA>J=K;c;RYa-6yYYS^Gbrl4 zKkYK4D(t{Ko$o;pLX*Za_6czFAMcc6mCk$_SJ_V7DWIM2-^uOdv|W75bJGU`)^O4s z$_+gh0Xz$n!S!m&*{3x2vP#e}S$_)>N)ij;F~^!F-`zPX&Mpzv>0Ylu>Dar1?(Y}g z(dZ!Un>+?WZ!V(D`4EaRSBXrYnw2T2pT$A#vK!RV37BFzc%YrjdfOo;>NXyMS3QbV z5!6z^l@x6gUUgpD7fd0z++C;ms$x>*Wkw}{v&xs>=Dhz2G&UcucNZpYMfJpndaV9U zwQiq@yF0qH;YadGGVt+pXio_mb=+*JEMTL`#wn1W^RAJ<-a@njHx0YVOEOqe>Le$JzS|FpwA@Kj$b7r zu3JW`-LK3)O2A9Cp6wz187h5+*xmYj+h{{2Ylg>tIu@FCWxyFv{V;rsE*9S;3pJ?_ zRnv@=H}B@P|JbcN+IIts!OK@QU{ZK*)MkdaFP9c^I_Kfvl!s22fvJ!Ttu&|=^cgpe zsbNogyQ!UhQ=z<9dHX(5fMnyt<~02&VJ_}{pgq(47mY9l6OGBiCw{3$&z;bW|YHhW^fO<)n%~{2lFQM z^}mw!JoXtQ4zp6oXDjPTA#v93^dykb$16fuAX!Jj>EC!+M0Jb@&VZ)wC-*!#B8&m| zEDD4l?Ju2`BilKJNB_R5veok9Deo_L1}1BPyT@Hp~%>5J-Q_8kcN3{!MCMouEZ#x$ct*9O5KsH359a7C=#HPoS+!m5m^&+>AJ3>?1)_#6 z8V|?odscbSt}tMwvDqzccJ)T!;<_IDvoEpcQCj+=OSiVtLhjGCdxz!J;eYJMn$2yJ z4RR{hzXuktBw(PDDW?~pKn}hjLBy;H1NZhZPtACC+cTG@!nz)jzeH~bAmaOS=H9sJ z$3m-)N#wGmqeroG=~EF&b^Cnq^ovjqi9ln8ca`h}p$Fs=tHC8 z{o9vY$nh!ze=bzXq-%7{{`Nj7L3y?*)240pnO$EwklJDC(n8L)l?TBrx~3Z;1)kF2 z>Fxkm3TnXKfr+_`!9>swQqx=jrbze=`~Yw!)AZNqr3&(T6>jcqTM=lX#M0t{s(WXO zAuts$~n7|=yci$hw&#%ZA6xGK8xR%lv&4x+bO8R9F1-{sKhcMr=Rt*e|tmv`7O z=rC8AWRuzWnecW6jfupRv6$3 z-@K?1=kIzPE~oU(dWcY5U5nmq(OJeyf#f;K!|n@MMw2Gl820da}n@73SQnU&6j4fZ~}_D=Ex&)a%2bsiR%V zmzd}JKdQIvG?)NE0>j#n4D`*=AkA!c~VT9|0nG;fVD*|3V^dUlqauKMEP<%9vs@ioJPJ_+;l;c_l{aB2#2T5VI zH*3#b%QGtEWc_LARpp7!3i83|Y1_&2RVMrrhS2EiCsji<3h~qx_;w^zepM~_nl?v+ z5fKy6GC5eyl2w|Gl$xWbq6Y4=^6PBvtd^1j7|)z!rIdwTq@j(BRxhXi z3~*EwvwU2LpM_%)R_O6SwgaBj2U65V<*5)5DhwG<8rPnk8`c(?Kb%F=+$`aX! zKV)8|(4F6gO6PC=T?j3exJYUF7eiCmJ^%M@@UQazQ^Vrpbv0inHH%R>vH(0Wg0D*5 zMyz`T-+Yyq+0;WBA<4kZl7t{+%@5a6|3;B-3r^o6krx0$YLL!ch$g+60PoOW zZYsm(`L}N?;TD-5_7q1BHWymVN%MiNCr4|7!QBmndlv^HmX%=A;-$}}pZ_|sG$2uY zc)MuO>NrDEz(FuGca1f=nyo%iLu=>|lD4g5uv?^U0nMZwrYHaVa8EvS!ugVl>!1F7 z8S}Mum(8FxCFeM_+ZfrQr22?g6jZzckE9@qpN0_9F(j7o z>u#PE75CxXu}U*EjGR0*0lx99>t6khr7b7Qq3j6sCJq zt?utSXA9vkULX9$o)cq`CH3}a?R$^7hHr`#>jCP@?HhEU_20a*Ma!kv6?8SWRT&fa z8`NnKFyQBwkwN-ftwt;77jx13O#gpgH>(|Pi%&8|k#KZsKUG+PEcb+%%#7PKHKTy9 z@HNuH^xtp=&NoPZ@EfA}@pFSMBHh0q#Lg5q(>CeFIi}=YqDQD-cdue@fsIUc%qP)4 zITJQ`Ex`K{p>MlmpYBy_=DARfHC!(8@I32uG*?tQcvRM&BX2>Xs`qf~yirtyZu6-S z()612a8kZp?34$oG_t_fMttWYm~D))7)WPu(BJ*cp5C09qZY|FmqU#~#fgONM_UO) z6j`d^r2e<LFuK@zH#ag;= zed9e#KuW@goowvGZ^-zVi0t>wfJ5}bG9d1;LP-m}F3Vo&-U^BOT??&pQM6dT+b4m? zTr5nix#O8zkUu@Jqa@2wCG1AZ+^DhKspYJJ?N_=eSS4;YO^~%TBiST;Q%eHqr$s!r zJbc=9LP2!|YXclCp&aBp^M0HCiRRd!FiLjCd9|UHYGLEWeTglHo8K^ga0S~{ck{~b z8h;&+PjwCs7x#~uB!Aj+@F`#cT`2~ql0%AsYa!vlq&L7VKmUgR-3@Wr|hNyooy>wH(L5W+y#6>_as5n~VAIHaZ@(&n zCNPnszmXhCN5D{jE`;T+B3`=g3&IZC^H8OSF%APE>e#A#GeJaynNjWzd*lJ1V`icY zI&?M9V=4&H=6Anh^pAt&;;C|By%b<`y|K_UiBXrtE#xmcKmT)$i+;XTNdI}Z`-~+; z!rjZz*}!ODsS!rMawnBft5MxXbcmvN^%D_O`D8>k8O;*rA^)LTve%NTz&I zQtar5F8TEGd<{Uo0%)9CZ9`5d%<7_gMHr-dx)zKeH^q*m;y~8c2Ie6;)^6ZxiN3#` z(fCi}fRuTipB7@TT7oV|dx2hlfkIcGy6(MTiFeZ4@V&<7sg2Mig-~ky3)j5}cN^vI z@mt8J&JsX89!PuYqPrr4%b?os_Jh*B#bu5@`kZ zkLRIkr3Rk?#az#^AWBnx^eeMBdbd-h%3C5oak0&th|gYl4LF-a zI#U)}+TL7ssPRh?T>5SE@Tqd{W6DMP0_=veIi zSN0!7aD==vV{Ve`n~I4vc^M}-R#cxfyNRCoR%$fGJrCbc-7UP!ea<-PzK{Z~uodf9 z18Fxitpo$VW7v9jbf&m-#+oaf&pW4iCvh*A6VZ)0H>yhH5dA55j0<_jvw&e!mHHFo zM8HlD(UY@sIv>vv$qIHwfJs+3hebc2XNbM!ef=bqBMY=`=HBkXYcVy55jZ3;xq*dX zkmRq0O3uUKUQ_w8Ya^cm6JgR%=K6`374ZMLxldWj4n6}jH$LBce_8UTfTym#SZ{Gy z@x{z-;1D!%0sVA}IDjLEI)+*{vzTK^If7AmYpvZGc~yU6T)u~N-nCdf7$Q5 zHK;qM!{6*O?2GlLStE&0IUXI~t8BjMR%u7S-UjSN~q4L9@PWOhGp{r!ARx zzhIY|pS_eO!&*~WTHUrj!-`bN&**d(t)jJFKX!VZ+m#tJcQ3n@zLO8-6Q^Upq$Wf2 zem`bCBYAv^WtO7d{&`O1Ji5^qu^zrpBXlGXdPZ(<`u0{e1yqf z{a!cnRnBBlfve2!WBgYT$9f$Cg8+>s+fup=oG8yy&j*LQ^8 zb2Ff?gwV`73Emx%T7x!ka!9ND1)C*Wav&xHT~nEI-ywz{rs+?}GuwG;`i!L4`;#ogUXa4+s!+`Yx! zf@>SxU5mRr{OR+47e5ynTxFc>v-VnZ*_`E=MCdF);Q-Ql=DR7!)eP~PF|!iRbMp3Gu%CEL0Y_3(80fNzk2 zkh^TO!!MnWEe&R_VJ?5qReF>Pt?ZnJzt}l+p zJ;+t88qQF;su%~IZ|g_6t98+&2SyXty3WD$(5#~ET-8pjFsQ0t1Ym6PGosufo<-5g zx;9!`Q-Kg0da#j|c0@zj{i-d~f0m(kGKE2Yqq1@RD@-<&^&S`SxriRoghOoD^b2Tm zLJIk3VqHNb*eLnN$Sgi*=t?U%n;=oJg2g1GNVMfQ8X(D>=O>(-U@-BYe!%3w+n7Hp znMlp=FYAlPS&wBz#(rB})S%V=fi*UX$h(thfA>B{=-#`>fYl}EMR`}duzqR!QL=-~ ztG_Ly{^RoxBD63^LL1))QqH#>@AHFJn6o9ze3E`BcU_Hj*hyX7P2juJ{CfK~=eiF8 z{i$q!5evX7$)1Io70gG}GVe?QCH2KOm7(B4LA6m7WN*wYLriEJj8MK2hj%&JazVBz zf81tB`2aV?mmJyL9H#E;B?keq$6i?uzqRu+#j49c&9R|mLF#K*d@o_C1lLo__(lj6 z25%vaiN4ZHZ8P56KAw{0G)qsTS(DdBN31zre2KiJ8>nAGJ71)yRN$e6C{d^g-qWI; z=IUKLxvdTj=macUgyU7e+<*eQ^e~;RSNqiF?G(pawRm~{GF7lidxyi3h2koI(K9C? znU8R?{xjh6Uc`cUn>*yer17P((5ZvO)R95r=KwDRirMPoRFplTl)nKd^;y0mwYa@z z#|htKODZ-O?Efu)xuf!>kDH{!>0xY~d!)0eBg{p^*Z`_hXw%-U33mueoI(j_Vx6aZ zVd6&BhMAH{1TUm)4t|+`9Zc6I@)yL=TO}!(U9R&?Wll^yt@)$r8!(UxL2IN}3t!4Q zcH~%IPM?;ivtqTi8flM@I6m7VoHMh@_j!^j!SBd~i)aJC;PpIje^O3^OE=dJaeYE( zpb?egnwoU%*SZLV)c`C}uhs_c$OqsTGO<*)o=&3_#KoKFZx-FS%_adHX$W}mT9m(VeQ|v zTze?11gmwJ#`Ms~kXpV{;}nTwYoyr8$vN#t3Hp&846=Qm35b1}kr7~AO(%TOB)ofy zz5hy4UY6R{q$bbQZE(;U&0AX)|5yxZe3h$UyE{B2rY;OqGmC>=Nme+y-srq%1Mvjg zYAPj=jQo0}EyzYObq$hsOsg;i)W!TTA$TfkmcJf-KEJ1%Rvvn2UI3l5JAx_}&so+! zw~I=J*_gKM-F7IlOo?NE9ya{QvpFFr3r2_hE%`PEy~~Yx*2HgfhB97uKL1;^NFmzv zIoE_PG!yeQFCB{+AgO8zt>=MFtfIe|M~lEOpv3p5zH5#?jK7Oq#?@SBO(6NL+W)0Pch+~MuEjaoS~*A8+v zoB5RRWLn|<<7N0$Od-$!OFPUvVzp0j&+|10q?@K9)Xo2t;vU6m86FK|ebM`2e4f}A z%4JV%n|4#&B{0~MdpFMZ&xavic0DY!Q~1H?R8_0hkwijbH9r6O$Nul{`GkK7rCV(O zu{v?Et%lADhmR=l?pHpd%En-ez9-Uo*uB58{mjhHjpT~)H-Qk$Lgx;_Ih7N)_Gw*2NPv0Qr zRDR(<__|}7k_1{B&_KsmqC8>+KJF$-1X81yx1;*p>1bIC5L&bPfSBY z)6x;M8sniS8)h95OnifSwV_A!?^Ck_PIdZRDL>WgXj+=Jtk!fb3@u?*#{MyK| z9ZH;c(gil?nP#PY6ZL+Bb(C{UaW7N}As=633={DcbXQRsk7lrz(Uy=vSwxBUwKsjL z59tZV=uD#v?EXZvf^FB0WVd1I-&D(nrS~4=L*5s%(d*zbf$XfSv7g^o_bPpFuLVj~ zhSCXFxsDEgR{c^hUsLxUAMqFVo<-}#PPR3F=d*n{Di8gu#nM(hn2~P=!_pro|kuCS}!BRaJGSeNW3W^9&3^U!S^=OD(8n&R5CRmh^yB z`AhVtpLH8;nV6VXzjA|M#bc^?S48Xg>|t%T{}2>)ojqOL$OO1EG?u+5cTcdbYzGa1 z+PwR}T7bL6&YHHp%$|@D%b)bP03P@Cz+DIk_rw z<(nmSSF?II4CDfX+`=e;q$VnDGjLH$|ni9;PV-N(G%6mwM z%Iy0qByxE_ag@FP!*o$s`ww~^7wvW zVye_U!S*@?BUpn>5ZA(EAXngc_{7#)Xq5nS6B84s=4>ub4c2)hV;}9T z^g1PEUsn0$7DSW<6e!b=HpDdG>OYJg?#m-nQTMpMtEkQ>GC{0W$QV)Go z%8)8*mB`==O+TP=Ow^)@V4N?Oh<~D|5Fk@WOUx;8Nll2WT==GIo8;ojo687y=XD() zULyA6=%2x}&c60h_Ih9EKb0Mrp6!;z)l+{fKY!bQ3$)aBZ^W(>)9O(8s;c@J8aX!c zT$?LFymxkMX2W{1HkXojJ6J{aRxcWyiY}6K$o+-!i;$HgD3Lc(thR`5;{Rwb*`oKO1e=u4|{v$bg zZU6AO-?4Y)3Fi9t$z0?xCeOfDz~R^E4C)e0YerBt5l~Uiy9R&5<9A=;j`6^S^S6PX zJ(LwoLOs1IFJfw{J%mK?t=(APyoY_m;nv@}n1k<8=903B+^M@!%5eNf&*(QW=o%=Q z5wrqCmNpc-T4Nz?QC|)|)|FT@O68Ii4BPHO*mAV5&uOl5OV6en+;z@!1%6Md0+ zJquHC1w}kni4)Ws)5uPHk>CDX(i*{)mh_0|YKb5ipV5B)C*{i}TSV4*@WYJT&QF26| zj|8{9#340(8!9OX5K8L%z5Nz?(@3bUHLoLb*`?>!?_H|2u8d}j-Du;2ujkGV~dgd$5-Ef=7`|hv{(87F%)YeP@Ge< z?9jn-ziMpH&G~)YPB>HL58HYvI{16L+cXXiUBGD1Ew2HkWm@bMr$$tvR0tbnZ%SO- z4r|2Yp%`EC6wu2#u{drPC1#;R)C2wu--h39lfZ+(T!%O)IILBl3L7`h-v3Mw=cdaC zNu%xe9RC4#ejm8o7n8Mjw}g1SRUeE{r)g^tgI1OiA+a)yp-;*=4{R8_ z-vc`}zjRtk!1OrGI0n4oy_!*?brk3~y2401norDpr(Xya1#+xE)`wiU-C|kfGcIC< zHukm=f8)eE++$1!HADDVeY13gi&m1n-b>w&9MwJJG_~V7MyDa9=m zWHcbpj*+e z{CHZ@_qy_i#v#0{r=|JN40ReeYAEoU6W!{cSvyi5_w*=5_>bu5WzVNw1K)R~gFeqK+litWCsja~uT_zL4u9Mt2Mv}}6x>r!n<>ZN^I&&g zB*5=x?sZ5uY|ii6$!|g6kSKE1Of1ecSg+>U9xR~2%|X{p;%VIHotFw>hI3c6B6!z) zD}AEF2etL=#UZOPAPirJ8#M{0aVu=X14uc4fg_%I*K75PnI0xdD`x)9-mBXD>qixjqOP zo&->eNr!vRFB@94PMWJQ<9rfys1iZ!eyxOg!wmx8o&KPqsFloSO7`zCq)S;(p*lxf z6Fu)f5Ue?JzQNTiq6TXxzGILah)ejp4Vw2{<5HOTb6xc9r2t0Xbf?>gCQeoa0^riO z_#>o~+PMJB7R9w!A9Hz+I7ri#V1h^;M9SwwMGt*b^MSlhN-fNywfN6JYf{vWG*0JJx}i?|uMw@yIb~ax+ElZzG?2Ayv4%W*?CFy9Ca$VYNB8}O z0b)Q+qLDA2lIO2~e%L>1M7Px9{mlwDTGu^(*(~XoxJXTw6I-Sfg%p~ttC%e=mHJ*Q z3-4840cIv9UrD;C6wvEZS6FQ!2{5bKT;_G(PLViRcE+~ z-Zmt*MNRUJJodicmf9k+yUL!FEiTyOuQ;YLxfQ!XXqAs}STR;Yal1|wMoTdcqh?_9 z@%sJudivjvng=t}D^~)nf~Vpl8 zkj{*7V1Stfoe4i7wPTcm&cENgk*k2<2goLe{eL<4X5sLl$OM|ZcZJG`r>spC%<5l@ zNrv9Z1LE?9NrCAJzh^#w3|kN?+gw3hgG8Qm2Za87z&x_QX6OL4-jF%gOxUYv>$D$U4<3r0#K{b>rPaCpNCnE~tahekZ0Lab@J%er3DgcN@1%3@dLvXe z&r#7Gy=f5RV)=gLUAe+>Td@8`o?ghNv&zQd&}LAOX${{d*c&nBD!J0>hAu!%g8I1U zC#inaoEig(jKku;XvFRiv=AFR)>!2|5P~$8i~)Z7 z4D?DTel(sLP{}auFNUghpewNTz7m5G-<$XK2a2oRy!)c6x>S}e3eVaUqV1o`+-n?Vlg`Lm=`OVk6tdsO zB=>k_s3xAg@xry)WKvOAJi;AF8WNi~_mPlt|8%ox71_cK)J454oAUdONpRY3OymVZ z_Uww>(PuT`1m=^Ct&?#qSo`}`S5 zajp2@kd=I+92UQ1#HW}c1sz`dbNBHVH-@Cz&=aiXx@rcD2Dct(bK*S73hR&Vf z$H{K;AdwAUh4+k1S_2Vv`-U&@LP^wQIRCVTg?tWU>cf~fOB39ph8WhcYo34E-98k+ z=8@3n_G=5RZNm=R+lwRMP@CD{i-Ml7st%*^pn8*@VRNEoheNV|xovZ1*H0c0}u)ZjVuP<6(3WIc&+694Xp|^b)ygl&e2H zsoutpEJ)ExLJM77JzCM@0eF&gpSgE-@{TdPEp@2RWTQ4q>KO#56}KBdqgiAe7g$=0 z$Z%69T|YHdSOiA29LiP}6*b7U`}D^u3LI8*+efxs9B!=^~Fxv3%+O8*Mwe|{R z`%`*d#yuyaDH zbya8+2F&9))`8CkN6A6IShlpL4VriobTP+>i&bubfn-OsAhGdiH5^YEb`7r)gAvn|t!aG` zn>BrycsS>oeVl$5qii=Y|N8T$i_95XQTgv9)%m$PsE=%>`Y!m8EsBR3HMez{`Mfqi zmzP5+^MdV%Fhm3gT&p(i#5x|c`S2Cq!yKDg*qMy(rwj1RCTR|@mm_{MFql?6!#G3l zXlx$^3`1Z@h@BVY7H9zHHH(H~HCTzj-;X}cOtBIis&B2GqtJJo9I9>rNrPq6LdrR` z>Ak(Ii;`OEKjqc7#64fZQ7?gfN317gAGRdbzKmZXlx)X0sqML69fT?YtE;MdnKC_e zyrB(Fz)vJZ4LoYR@l8#;5*M#957&{MeAcbu9J!~|Ggetwx;C^jOLQ$@W~xe?JV9Qtzg3?x=>-F-MINXA3c7 z|A*Hyc>{DZm=0YjnL*{Npw?|^%BGU-1UGVe3)bmIizLBY3)gG*O1?@{OLYGL+xbw9YR!ZK+FE5w5Lvb}k+^?-32GFZ{zxsqPkI0}uaG;4rSBN0N4*=LL(7DfcI5mD zG#2ji)dpfP=75ZC_LqL0&5KMuew&P zb?ddpa$ypYOx@IN_xTK%Pk*_Ze4Y?bukvfq=yTSf<0=*meIR=YzO(unOk|_X4RGVE zF_QGdtXQppqWXqnossrGowombP`B3--Ub+S;*dk@wmDx=t0^Y314{@th;B1O4ulh% z9YK8Ck8$h}RTx&)o-{-vBX^NA792?K3@^-q?|VQec#u=hgU;i1_MP8pP=#YG%xn2# zYz#EhpO{zgM>STbNBN{`-N5BIdxdIAc~bF#GrC=|xr)U3Sb2y&*`hW05o7gb3`FSf zT?5}ff&=)WH{*+4XZL^ht0Usic}+z4<@sh^s&Y>3k*(?djNGAa1fWB*Xe{2s**qSK z4Gm(<4kKa>@A*x#vr+0RVF3~;m1M^;Q_`8$tJ&A1|E&3Dg?P*VRgGT+eArLK1{?i*kP;SpRbBpe*b|ut%+(HW{!2)o>CV| zygk4)xDsWR(gBISxrZ@jR)t2P%A}jSH=Ld(*#Ly>72iClT7bh>D zvq|hh@LzKS$PZLSi951<;>%N!-Z}|Q?_f`5Hn{rW`Ye@7oEj|lk>_#T5$j!i^^eg* z3)=`!bknjP>*f-IlJ{{_){G?e1KOS(9|-zr&RTmf*)B=YAkj)z=wN{IlrvZ7-bE|1 z%Ma)t5Z=GoHih*2?B?jeYyQ7d+B~OkAup6*0cDKwAV2}3L|j5B*evCaAEBOGcKDm_ zUnU8ODeCC@5nA*(i2RmkFPBj#H+eP7Issk=rx|d(1LNe>RGL>jO=M;ov`Tu_7VMfn z-_8YEUU5HP6WlzlC(Dr`=z)PXOyP#p+VkabfEN8$EIm7V_)k_!v?Hv*{zA;uo-+SD zcuGar8!~9du`RZcL9xk>{Re1r*VbKBILUKksRX;S4QBhiy2qLurz*mPkbP5h@2I9*4&dLh^ z%x_kTTWQ450ND-Zo3jZBw#|L)*!muIMUQy~G?PGLmgho|lp#&_;s?dSUZ%`VPpL*}?WjlP=k2MkF)Uyv+@SXiDI67@;z z)Ck9#7|(CT1-FoDIFF2Lv*&kxK$v(DsF*LS3?{K*3DTGjKq)@h$Sm}@4Kk09i1iy{ zw3|`%*iR@^_M@lTk83eK*1U44dlIi*%?-v~R`|RQKu6ZQ;&k3t zIh_58G=}RD<=S23%VT9qMC9`>QiM&>DV1DD)W=Mdg(EXz`GFK}OZ2sW` zEU9`PBcYMm!XQ!H*jwg`nM}T|9wFdL^qd|GydBLBl`g z_G8Ce7TRg&$aY!paQ}SivNpIPUXJHX*Px0nmZQt&nMjyQE4`WE1ZRy?-CZjBxOZO` z3FX*xg$!N1uRtm+D!Ly0YBvs84^yJlT=hpd%u5NnnJPVqwkLMKFwX@zb&cM`dBWDK zme)ZopiTJTlRqR*@(Lw)W=ZAATTl1#qt!&3v-%hNyqWch#iFA)FxmL!G0K${|Utfx5W;fTIwD!>kdW6h$qf zvLV8Szr}eK!(qRM6uBbWj^fie%w3Y3hRvaH4yeaV>KvasYo(n8&F-PTAr96PdxPXs zHfZ9rafp6VGoY-bSjd@ccN7ob@1e1{@b)W=_$Tf_fHP448_mx5FtQrW>Jss3Apc*K z{Bp-f3MNaDmBR@HrbPeq!sk#F!;zGR265ga>KIP&x+^^MMd{dTbUK{UmzY3 z^e$Mp4%;`0^NsEOKHnrp%gsh}NwgG=1f4)HrrpaY_Z{9GeqQB2Fp+wQaK3rwl&i$$W6oA~@^BGBo5*K4` zcKWWrz1fYm&tgc}=8_dG_xlHYg$}M$DXTSb!F8OTxe-6;0d0f}6dz*)*QCM^bi!Yg zLw_U=eODS8Rk-Wo9|Y(+aUtE_9%(^UDvps-_I%BfD8>l4Je~p>(#G{`Cg$*m9Cnkc z0+lNlQ`=NbQYA2IRa<@LBpz$u33}MGoylIz1`t90`pq|QQRVGsB4ghAly8pEDLX+$tn`og#8_6 zI`KYFZ8=0t{*Hr7!Cfh=y+c-(x4f{PFH}TuGolbPEA6^QVvp4&s2>B~(YL?951TPw z*D0YIUPzEzuYi{FIEHpCvZqeM0d{FA+KP2xa#H5&D-49}*vLoZX1F4(!gwqDOm3nP z`F`F1%g{S=QAcRlQ(Y<|B(AzVJ-m2S0<|9$%nSM!W++jfvY05m;W0xnuWt@?&z~D7 zKW4kSm-jFF z%#Qg+mmlf0`g_UW3!w%iRSIDWno9LPvWcFeMau>%ihXeUm!IX=o)Mis9sT7wc?V~8 zLJad_loGfqr| zz_@gVk|VgRx*F|s{3Rzso^s9tXhX*kkQ2SQ%gn={)FF~G?#@ecTe^Z7Mj|?PMwWts z6ly`zRGL{ZJo-a(7x=o%n%<+}x7H#RLo(i+|rlgNN2;_O)7NyYO0{A6bIChRG;h3S4&=6(s6Lutq; zDbk}F((M2y2A!f^Y5hsDx5HO*ew_fIDZ0n<2!E$7RnXlX9qn*4$6y~SIV}hhHYImm zWAdR@Wb?Q%YJSNTpScO0mawl!5yP940m);PR#rJFRy0!u4W&M?T#)fce%V(agbM9Bd%oONb{rx4)7dotG2jM8_ z;lGzckULkSL~audynf+?l6~>o6tc%Q0p7?Aa;`6C*vJ_~)5KKCJUVdlX4p=8fjgeb zYpReA&XXUuD+-rO)OO7M9`yaM6quGJlE6mYI2l}C;%Z;}clj$stK&jIZJ^QU-trF^ zz>fgOrF_21E`)cL5W{MCgGf(wpappKWHhZ>v{3{MeUcM7$rInMXsTLx;L(XXeiOWO7=`-&4&t@j$ zzXSzWD186(@h=Hue`K}sJWh|EC6%OX!V_|vD_Jgv5ndIxt1YoM7Kd0k5ec9&gx~Vu z{&rPq{^dC>&evB^5T6vybr2Hfd_f8xY8g zNdOs&8ga2<>-WjammdOlyeoiUvMb8I9uu2ZV%AnC!)IyTa%jL04*6oJKq}m+m-cIh zP~6kiilFI6E0g39k$iOhA0N>C&H&6bShGY?SoCn)JpD5H$IxX#96=aC*!O?^jFdKi z{~r!cGXSOFghIAu?r!IpSV5_j+)<83t5=NiV%=ROAH_19tvr0}a1dLNu8=JC- zCm8nw?)Zeu$!nx%nWRhSdcr8Qv!Mk#s-!h(VW)=u*$MNy3X|eb^#Z>d*SV6nVw?4W z$fvt8khWD`54`hnYgSZ^P$<=2o>aMS{7zY<7_E5?P(d<#tpS#<-F!mF-FMaV@+7INTN(C?$&iy45K^!EJO-=_Lz0+ zS6AoDZtKl?2HxqPYO1V5j6`@GgJeiEx|2kSQ?4kek2Y6;_ALG~Akg~{c*W!-@W?0! z-HjJJ3WOz)25>{WNM-eGo@p#%3-=ul<_N zcUUz-O}(W2qwuSxciA_J1F}E%I8FzYgnXKQbD(oI+;HX&*VdaWrZf&a|Cd$Fs8_AT zwJlv=%o=v5%Mrz$au+gSLK6xLOlr?h{Ib5;@FcCDE;0ht4cPy*RzPU_TIeaY(PHj? z-_hI+D~PTkCGonOP4)!7@?UWS{=gx8)316MF_VJMQxm@In-?g$Ed8)DFy&35_y4>0 z0a!cs3?xKmRRUbraO4bgLDL$hMl=6uz)Idhp0YOWFkOZ845&#`>)7bBJ@@@4G6)N=&EMijSPhNR>++ zQi|FqPrOA>0fRg=8YF;2X|f(<>~9|-K7n`r zY0as%<^q7Vf0IoOAsJcil+s|-rK&Afl>mb+2HNKm#1h4%11^mg%2dAz;2PUWH@H5N z%@dn1D8Og2xNygStl5Wr%3)=6+eJBG9JgXCKzalZRySoK!{`ykv?R@)CX)4Ja`cZx z9i*GU`EM%1v0}Uh$xm$hSGud^k%dvk#65pLI1Mt;AyDHVcKbm6r_!#Va>^WkRbI#T ze;<{ioxh-c{+%xf{%^Fz^?JwcQShoIsPoi=d!l^FEiMNg2@EX*JO7MzwRRZyr0WH? zm+vGfr^RshapBUO=H=4x6*tlqnY>}-Xz@db08mgnJ!o5*u0zsHxr960aS?V~fo7X? zO#?t!3~f+AI;2z93EKU3a5399uMp|l$9*F+{}qUsY0572MI$FZiP~9(% zRn2_bq@eAOM!&|dD_0J0N()6?d;1&~d+{ARZqHklR0DN^TrQ>--pq4cTX$$#C{s%$ z)snRO#7u-qwQWN^ZB2H6e5u<_WV%={Krt>5%bLPE*rP=aN?8MaiE^T^)wmeKC@-Dq zyCB-K`DaTgGyhV6J)cSH!DgM!3xy z5TZH3n11CiR^Lpa zs<0ySE2LEoqstG*af~f{fq*NOvM!3L`ClHEd6ADR z*fk^eBbgzglOK&Mk38nx;NdO8-?%nSGh(yBh8BzXrmSjZ4zsb)7HJWvJaV`3?dQ7mMB{d+{UcRbT(FKWNv(o^a0DW9`{O94 zJ=kGZmexM+1

CQ@LKdC=S*l)%Tc`9e1j+9VvdLfI}N;WTId8l9kd_eIsz~Xgqum z?-f?RL-%GQ4XLfTdi_ZCu=IQ4RCa`q$6s0$^O`7U(GJC9Eg@a_MuQBot>U=tg|s+E zcM@X#%ZYX9a>l14XaHzx-jwV0rQf5?o-xK<65Kq(r{H{(byvX%DTvaV3O;*s0+O$S z1#N8?mpksd=~rQmreMEOvGuQv2PF4*kFWO41aqLXRs-2B;9YkGNc)Hu5_RF zNL|C#8zX-pf}gRjM^>ZOus5>I3^h$kq0l+6H7ANYiIjDThq^PUpYGb(!)-c~ zCT=$*Y+gk<#Qd}kKuAXC-6dl6ah;LWos)|0et8r?ZzVv-_ry+0;vLha>4JI1CLR zaQ*uWi)~!M<03K2#hgm>o6}}wjMQ2$H~Gy#I*xX)#+GIzmO)ybGd-n=^U`2{O^7Lk zLQqw3O#Sze{KZwGA0l#M40W~e6at? z3es0;-d`pR>)5A>4*P&pLZR=-fIlSaCHr3FzC@}az=1LQsZd{V z@E0rv#ixRGjL#MK!_G&>KIu_{nwRlbahsd^47X?O9G^5rCACCr>M_@!0$FY?CBTZ| zPb}RUtO(E?w3>@&xr{lAJ@b= zQz2cm?7Uk3tt)dJXIK{ z8C(1#*p;gmS9JC_;UH-i>1v%Z&V%HZBiyEAPWboDWX@Nca4h)O_#=EhLlxh;hUIDc z^k~6v7-v7w%;}xtlbHT1Q&p9hSJ=vrHOuRE5!t(fXMN`tbaU4P4RSkYZ_1Z8%2e?{7LL53X-6h>vMtc8VL#YrQpPB4-@gV^X7M|&N}iTWZL?75_BLUnHGY^E zNROmGVV@7xSR}^Lc+Ix&=?=v%FjdxOVGUo>TVe=K9N13WYiH_tBBnAZ6-KfV!hx+q zE?*clA<|{EJxMOdGx`&H&`9*6=k@H!ZmTq9x=V*kd<<`d0Q{X^^&P@;+;a{6Z%MI& z85RwH^FLoXSx5M#Tr3GGeLXN(URFH*oq?i88l)@~V1Kk9@zBCK+f4-U zM44LxlByFyT;EHJjO;s+^hFuWFq}>$C@5w>rI?*7k-=JM*o|ny&$z-{%tkTT*LA?u zs}ZK$`)nPnWC2XtR_xmU!*XSWnsaB>scwa~1*6t{Z466U|NfnO{k30ah!yk6;=6Ko$*niN$`?Hj=tN;|+R|wB;!t3~`7ge-|PJ&`0KwcMe#FhKbmi_9! zE~djb!h*tKDw*2+0L&0Idx)2c%E=fGf>*4ki6^l9o6&E@Yb~YH*EN61C7mNF21r#} zvi@BR4d64Xhe4DflZ;A~cM5l4TLXqpG8+ww$6?AoSG)cqNWa%qf-*M$v=O?zu1z`xJcTVOf`M|6F>k%<>A2#eYs`40yL`$(LOh3vd7E2sxY1~z zW`p`twzb0N_dg)RWKWhJx4`o;NwIub46{HRf*Qhr9qNsqcb}_Z78@BTtkv9rr>N_6 z%s)A={(P+tT=t1b+i|Zugs~qH(uYH84^&9%*2RKTxQd2&2rUJoot^DC{~XbY!^t&U zME1_W<`WpLduYvF>OnRqi@RXSY35r0g$NoT$@EmzNWP%Cx_qSd(fIl9UtwJ5m+(dV zxhXd7$Xaf!SCFQ?sX0Cllx+Hps}xn9LK9_gj}6X+xtk{NsTX&{KNkM&sJwfTY2#bY zL(2{tc#oLRpnc@J4-lNjaKDG57POI4lAzqXULS}b{H7Qy#+U`T$RR>K_x%3K4=Xar zuL_7L8f|5yuD^gCs;U{;3#cZyv`fLQOVrXv>=cpwbk$OgUv}lGd-o9v8}w&iltOJu z2`;0$2SRXkFw!<^rBU3jJVFA<4R-;^-(4&L`Yfw1fWWTYs)ZtEzXQ&|KOHSLK@K`? z$2XVvBXYf&-D7go2n18k5Z~-*4iPdFkBCNOV-b0$()|i}I3XhgYqp45y@MN%K%<=E z=@iedWlyDU(Lu{v)SFlR3L~vG|2K*7`mGs)pZ{9+mNvY;Uu7rUo@t3QU2Fa0Biw%; z7)4h3$c~RLXQ6|pDk$TX@q2B8p-tnd^2E@?%r}@oXBguq9?mahtB)P~8dIGd56xDE zeoPn&%ubz`Q015vD~E%dJS?C#Z@B>XGjySXbpD`Wn&ERkbL#z+w-k~Op?fF9%?6!R)c>YN8a^PRS*g0H{{ese|Inr5w_P1;%Fxqj3Dl>D)h9${ zmE+w-+3Dps96kQ?4Hkm{MWySxCn&bBB$h8g)g!KF-)S=l0z%@|wOY|GLrh7#cAy6O zwe%5XX(RlQ zxBytJjEAJ(8q}o^q*`^WWgDg2L#Ma2u}7O5>SXJ-E1{t+&` z=vdM1n@K`iu9-z%pNjKz2iodLn?HsCQQq1;dkS6%Y(<$lV2pO?1&&aM8)e+~63Oek z(4QNMHZ1S4cfg~AVi;d0kvz%?j)dxjJ-czq!o8S(c z$8cAA>%##w(bbqlooW0e32*B{KTi91Dx&Y|RZqN@e`rF!+owlD0xB*wJpSirQiLDO zmRMMctUPv+qWvOpaQu|tuJTe=t}F6(FVZ-En`EkJS~IYm?26+c`znMe&(v|=jUO?E zZ$=rA53bnDC zE8WdOggTxAV%_vfof$r719P(2u#_hM;qMoRtN5me_i-BX6-zFk*fj0UvjL%jRUhbkTpGKK><|S#3p*zW~qb&gDEW#L0 z^wWj!D8OEuK!R;YS;`lAm;QLpRt%cj>e5J)=3P|^h#{Px!5`io`PNwJhN8Yv3+#Lm+ zWOJ4Qi+GD$(xOD`{t3Ip^TFS-+i%bl6Pk3wA<9F+{H^4#%sKYT?MNx(R#mJnf(shX zo)?(XcnjG$w*OhP&!lm3_L29J)a}FG*8Kouw(pq2|1e88{p0j0d6pZNVZHbN+!*?S zwQ!w+UzGKA5Wm8i;9Ro9>gCq}G|81wh7k?<;;67tR(;fz(Yvni@OG%2F2ydrQXTjO zceLU3DCKdjTaB#Vtp&(CeqmM8-ML0N*{@89;yN5xs1fw9m1#n;ih>MR7u^YdbK1Ff z&zdTLoX<OBMBIFT5)?_xgiQ3_w-RM%m={*o86WVVHt{IMyFn+P^ zi#Lb*S-)D*KVF?ul8Lwh`K&DSPIRCwiSrs`XWElcL;X9)GVeRv0X((K}Pac+6>kkjs6_yqkFU)1DP4&9!w!SxY! zs#*m0n2-Ck+L?9*?rqx|+9)z#{CB;sBv$2AQ>S z!+2H^cUMWk?SV=5JAn`GM%XxF#@!hle8h@9+W66mpKQ)l@3KM>`q4!*QOPMU3hWoxSkvFum%|^R1UwKp z9Av~wV-lydPIt5rNId$*3~K=B7IEBmZM#xfb;)k4#zB;w-lw*RGW&anT!va?A?BP7 z2p&K&Vn%*sW5kD*q=3ea4(e#scLbg&|ML0)`TS^twG#aWo80m2r8;SHIwY5h#H$aV z9ZfdmSa>SEi~MhD{qHpiIP!wr>4r!Qh?e;mj484Nb~ejSWpfkycS8Vzg;KQ8LJ$-m z_e?0(`q=jFnoCBTCS@#UTvEVKj#bXs9y)^G^99nOa3QScA91xS5cZ! zdBtJYu|FG^VEc>;FC%R+FX?-(p_u2`7}@XCaY$$PKA$M){ud|g@d60eF8XMR>r(FIekN6S`nYUoO|lA&Cb=&maitfJd}- zGPHu5n{1RmUq(3D*irYekaFmEgX^wPv z^wn=HNgQszf88Be!v}P%ZBU6thieQpa6wtbt;2#^#Ee?xB8Bx#THb*jGhnh4#Tcx| zIJyrrLwYbAO4l}h$KJWQgjZ>n*A_8ernc#!$E`Wuu(@Pb{U21(pMBha)+)4fK6aT( zZ|d{fL1~vm_cW5pwjbUH$Sr+3*u&;2xJnVhhPx**QXev+a5{0$2pAzHO0j9N_rw*C+CHR8S zhD1%Iji{Oy#ifS1je3a020t6;-Acv@@ud!yze5}+dDt2 zmL#b=-BH%Md zdLu$j{`MSHhdz56rL-7P4G_*v6j7~vK}!@B^HxN!|u167@j^dD^foSHY*Hx>Z8{tSzS zNmPW!8rz--c?=p3F+5aZX1z5IwwD5`D51eG6zIdz{~T-Vvd_iWw!L#LJfY6(Ql4NQ#0Xbhl~6{qk;3b(;JKy55KnA@Wz_ zVwg@TGDNTu_b^hSF|bg=58dgicl@87!8vqjf*jd}%O#DWpe?jb*mpjjj#G@c--faS z01!*5d!gO*$Ju*nM8lxl?sLO~LPF*T5s5?meZcdp1yn<|zxgmT_jy;&+de#egKc{% zKm;Ru;6p{D`Az_o)00q8-mgN;*{uGJo{HR`HRBV%+b5--wEld@7|&{T_u3z!C}WV4 z|90u}hMYdh-iGcHtB& zC5yx$i5mh@==WObjyWL|=XXp#kJom3#hP9IvO+oNpNFt<$9 zzkQEb8|2@5jJU%yPLQk~nK#+so9cYaIaPk5-2 z!H6-8O;CAeN?PpZH~a`1R1Q2$)Ia#Ng5pjq!P}uS<0JSa_#yj!N+S(~30!!G+Oe{yW;iBFx)=`~rY?MgVRC*895bQ=+$0HR0~jS*<)IJgk@+TZy=3_KG6>930g=mw zdhzJ|KAZXhPd$>}ANyb_{i|6G!yR?-w&p%^KEs8<{j+P@jaL6EBi?IYQ}#wXxau>6 zC>v}4?o33wlqzz(L7L7~OJp*3p*mVCJuUQw;gw7}!ddWNR*c@x;9;{LuztcKCq>3s zpcRraMA!-L+8g>=k(UZ_SNKXplV~Zl=0--F0rR-mG6vRdX{35LcCBnqkr_g^owUyM zqkYLZl;5O|MWwsSetPEYgc0DKI2py%FC1&4?AK_tFazd!df)s-K{B6IyI0z{lhN*UunHeuWpZSbm;P(h71 zKhMa1d+e>ZZQRRrbX6wzuI^cM-2@xG^ZW`m^6a_K_I~XSwIiPE`(YY059?D-fQnCc zF#0QQ{G~E~xZOx6=O0@p&@LR3_9fPRWEOwr2 zt47^V2$N2!K=CAnc43Tzz~`u0NIDzm6Et|pi)BV|Yd@BxP4u6Hx`lDA;re{Q6~3jg zXcF_iPF{>GCLs%$RjbPFh)Qx%gduX{Z_kwVsa*CI1CO zFuNij^AdTFPC=MyHioX47W6W0vVvj*pFg!TFKSa-kJ<{>$99P>s!r2mp$m7==)eF$^@Ui#geT+jyI$_h9} z&6c$E8#V&i;5WHYF$)t^YJzG1FyE0K2>*=UFJ{Yz&S;~uz*=G+V!K20X3Xi_J=dHX z0CF^`r+w)#pIST1;v7o`#nNg^^9)&zes~jkOefyT@*j9_6rXMvVp@K(_>|y21r^fl zc30~7T?UKz%Rxw*BkfL?kXKw;{_Ey&Q99xPAIQivVK#2}{0R1qOXa4dzd>dCF0^dz zXY`~3Mw^30@%B1Ri;0NpYKE!xDod$7F(1}sM0TmYnE=%-`H3f9MxYUd%u2u;?lCB% zQy}Eui_-MlHa7d$KmIeEVRvtCHA56M7??CUkzh(RoF2tvobKWG3zXB`dT$?0L}H8w zZ|5^-Q;xG4-mD)vVlF=kA~4%;4K+f>*ZH1fJT`h5YxMTH8e3Jqn#X-P(t=Kz_$XGa zlV|t2$zrNb*}z>c2dp~aE$G(K(UpVoWhOR9GIJx8QyG2Pgz$ zj^*dQG7ulZgsw`h*dp?QlhIhK06QBs3zagDSR2vwEO{BLN`Dd?63q(TrN|K0xmA`T zgwP=9=L~v|ak_s4boBAaN9YMeIbSJG4Zpb?+W8h zk6E>%1^+_8GXWr{Xx~ATm#J^eE=N&ZbT`TJ(ByPcrU^J`zl(YF)EvOlB zOHEeGv&RC+)qNI`yF&ej11Hxy#whSs-P2>51{1Ys{|xxzN(21)k>6LAiEQtXXNAQ# zmc66kh1W$)9cZ*zu2Y}XTG$|V74~O@akk;yF1xvS z<1GEL7{^Ugdux@wtiY4vHM$&n3NDxDLo_I_m2w;TYkQ#8Xt>sEcG;|&B~xqPJ$&d{ z1Y+3dB5l71z5owW#mzjoeHx&Wh$%VCkj}ML@fI9!0<#(>tvxw>qB^`uv#hwo*b+1~ zS6(yiU|MAq$zr#z17+Nv%Ve{|D)5~Z#ST#sWUS*34;U+CXkDi1;#(uHq-T5aRX}9S3UBYrRDu2lmcR!I%3b3+M8P}1 zD#J9jH!{X;Lm%!MetdD(bqrS-nQ`_|ZBomxF{xHN>J(eg>~Ox;S`_4UWW0Ji$jaDp z+ZLolwl^4}(R8%dL&$Q&+7T66p@h2>WRXj1bX6Q`jL-Unhna=_YXf`EHjH1)d+W0w z8{9MvXHjfJrMed{l0_)W!FRRTG`i|@XtqZ+9JyQbUUh}wt`S9|&um8zB|~tNf9qlz z@ZlmXcEvW$RE;~CrGeLjw_`&Gs|=sYzJk${*NdX3vz69EBW}1fgz%{SUWXETy4x`l z?K~nE+CpV_1IS<|nc5gTy0QXoS#4jRz*@f;=ghpA{zpSX^=2Gfi;q}IW;#)d_pJA* zhE`pn+qjb;ld>{H7hzHMJi~qr0E&1^k$V3L86f0z^NJqR6n}EQfTHh|GH!Ze>YdI* zNni!(bsXJzUcFhWvtj@BO`qI*P-{I5>e~AAyplcBm?)$zUM@Yyo+#CV1S<;|0BD+5-5k_I!zhObw zpEz!M(=RVTG(JUlO8TghRH(IPfO~ZK2=_tx zp2jr7br0iN_-ap8KqMXiMvAT5Tv9}?fxwwH7h}EaMjT<~lKn(d*u-(jII?~u7Fc-r zc#v0(2`Mw2w{!m&KH;8W2oZ1&e@ew3kRQd1pSr6|8y@!+z0K0d?r%qRk+HdW_m5@C znM#Kl+0D%3k@?Hf4}bQ)6taG9@MP?F@&0ZyBS_VgL0~j6?a-zyA+3&|*&;l2hiU%J z>P8T{x1%5CFe!K&a6OJZT7_TI>Y8A#Y?_&iXSu|K5Hb*l>Qsu8&hJ%GF${@uTE@!0 zqfCCn2N9U7f4Tc4_HrLg42@#L$!#^z*09jHyZ8zSwHDq%x!shLXbLgg-Bnc-acmRM~hY7 zBGd^hDX8w8$9?EBB1ttp9r>o=eN>(V+k2P!H>JuVHqDhsdWb^OrtO18>!J}>Qls-q zXnqkL^6wYl(kYr8M2Z|XkDuOfa}>rtf!NA>QQhi!r9xtZuG+I21nwFgdswx z_exh66Z#UCEx!Z&|HKG600@{I&MoudFr8V#>g!X&(EwMU>tdS7x^L=v>uZiMbh|A^ z#OxTO?Uu88UE&{DcK>J?@Ife81*qdbm}ZUV1#sb1uN!o{>`-7hEwxP6tSPdK zg%Qpy^cl^JGf5H(89lggg1oM>speL^W4XPtT>e~&j@=x-MZ{qQ(%jwnb`69DdOsSI z&~p7y+qV~xchbxJbX~tS%+;w0qB{-fPzG|7GRlfxg%6ixH^(vSPbef>&HnHb z{AT##HmjX;sKV?HnC9pEWi2^LtT_$;)g`!=rT5~5IW`v1&-7*hl z{OW4}!F@*Tx*-W2%R3Fh-nk*+hh|P4{;%EZvY5iTfE~~@9M>nPoM!*J2tDTE&H)(e z5}cxcfqfdDXNI(QD>1!*MvddAN-Kjhma{6FX@7j0083A7GuC}%0uvk4R3k0Xg@TpX zWdRt}JYnFjJT*XL;9w@3!;|=TlWV?LO9x`y<~;Hm2-|_5@AHi0%Sp#r*zHa_fP!Qz z4XvBC6n41y?RT(I+)Ts)EB)0&f@Ct5y!X|;3zI!mnEf3lZkFK==|ORE11FQ2XI~)9y%8RTb@z*TXP@B*UVZ%1fFvt*HjBM zEe^)Fm^mcLCQ~b^EQ+PNAK)3%OsuCVzD||x@x4qw+b9WzIPrLT5B#c|%q&+yRhHSG zGbAUTX>BXZSIx)|hmY}aWL25ewqzfCO#vABrpIu+11fQ}Z3d|bxY8Ma(Id*CG#xy~ z&-`HpQr!2T>QIsE=5e;1_^K1{>TNTxPPC{rZa51+eFx)t>V#bI_DQSVh`dU^lqTO7 zSBxsp<`0_H-BsN$EX&<8k9PX(FfRC9NC;XN+Myo|s!$7GC-CYapMMUG8H_+upS{oZ zbNbico9_=fzm&z5%3K>6{L2uHwa?qluH^eQbnMEMCCw4#LX~<6#uUs)P4&&jDCs=n zP1A+me<{4d2Ed$X)u(^AXQuUUTPQ7-AuHUDDrI|<)U3CP&NxSccCAL7aRzxiB?QE% zu1AQ`<42$dFSum>2I-3LynH00-|+PU`J=fH(YgfYQi7a=JAt{O_s$1);G?!RJp`Iy zoYd2zOq;>PugyPAh~isCO0MMUnfqn-3wd+UHbO2F%L{aZ5T_t;Y*$F9BPxsNlq@zoTL+A;}wTMJ>R&S~e*AZHOXQ!^=9nleU}7M`DU^9$xqSw96R|YPH!8hX$ru#F*G4@xI+B#)E5C_NQKYE z8L8E&?Qy+7QLW|5=JO4b&Kz%jE71hqFd#X-Chsk2EVBeo$J-S@0?!lPr*pjg)k}~y z{j6mkMq#1B>p{iFS#>nzn${4vn26tC5!FDIX^Y+RhzmAyH{60#7MD^mdIu?-ZR93> zR$R+6d6cK~v~biW?bLV@D$pD02yTfiB5eQaI>G9EHT6SaIK7Z5d5(=qbd+bjeWu+u z$rQ60ab*QBFCCu%5YQ(Sp=&&Ed7|FqBMn}XjMIDYmv`I|`mevdA#%-<16V-UZN6>V zO-lAnc9tcbZv*PdHIfz6(HXw}W!ZG1=%6}AG?Q~4Msjc}bT;|RMals+^pJEOhia6& zX~^_o?Y4>JiBZ{m;?l~mb|(?*a`>-EmmoSLrP<~biJ(taW5?1cSLzi0_jOi`9Au?mQ!b?L8&S>?i1J!UTbYbRyOFx*T@G|s*Py%x>&b&tZ z7m+6Srr|S~n%zpNa`paeWhp-u`Qj`{c%H~AtE<1F2INHDyku+0H9{4lTpkq8}}4REcW>Mk%wA?oJ2c>+ieV2Pz-UFHcEu_C1Q!NdeEQ# zRU>ILvY85yee#dwKLu)+83nNc`brxpF0sa|P43gsc4Ws}mDE7Pe6XYeBAvTGX%pE2 zcV8#OdJS~Iw@)@io3&??^>3|~D-RsNb&Vmbkc9LzLqnpgL$Snz`v9Kpq!Wd}3*PbU z=yt_kLsFZI0<~q_mKRet>HD?*KNvT)y?L85*3kZ^nxN&ELCqkxF%GBY3!G{^?GMXM zVEv8Hw~h(IE7iU<^>2-{v<=`Rkp<=vIq*bCT371pRcl+2-dI%%MLcbe>Cdcl3520B`<&uii%v%z{_JBlRevzl_4xlDxziUWS^8_w<`3zD?R9BVIEEnu)=)B76a5@{ zlf~hbb;X4~F!6ZXVbF|pg=626x8cUfr5fkgXsrZDd}C)wE^_}ECw*+L_td)(hrF*V z-m4z35HMVP;{L{1{dn-5*HBrv~{Rl%T z1nOT^$)B-QBltHC2PKCOqIP5L9#Y_^W+Req4h;{v@rgg%2z3N#>~S&G97U1+>Zs-D zg~`@OwhuBTH71%=_f!o@ncZtZIb-evukXG=k_sMOW8oo$3j|ogqM=Mm-a!MjG~xjr zMlGjS!Ie0Kc&Bb52XukPWU-ICaO2}&&+xxdgy05uoKS#2AUsS!)|Z$ zIfs+shsP;h$BC3!U8JN2#_>FdKY~Wt-DrDLs1P{rNzpY=P4MykWrfZ6-~<-9Oj2)> zIZAmoV2cYHppZq+E+S3%X&>c-?|Qaz{g%NV^GFsQR1}4Tv!I}Vr;2RA)<2n*`Tdyi z?Wv`R(YK0xNOsn;Jj?gtW1{bro?)JDUls0rZ-EswdPULvIyxbb&?&2bz>&86>(@4k zhFcS?$90#BNoXt;liOy5wR>{S?DQj?EI;~47=pn>wgs}Y9eA`i2}jyhx)*X;eq(eB z-0Z`GBb#LxE3aQg@St5R858j%bDtoQ<%DsxmyC9)i`SWJWJiLj(00VqD6Dvd%`6C9 zbCy;!xx@evnmhPBKToAbd+jPc6b&aw`lHk@a+(Y0I6B)9}!+z-O$X&vKeV zJ$(B80-GbD0@g@-+|D+}!j|u=kT*Yx*o$0_TvCK+th{4byRFDem^CNbq!1RVd2>p& z{<;Ic@Lv^Sh;y6*q|3bT_2E&I)98>^W6wZ9Te5FZ(cX%nv>z@ZKFTzUmQyi8@s+i( zJYhL2D6Y0;4*5*b|BmJ)*-dX#s5;wxp+??w9v8GaahM@}%=@>cOrRrAVvnLgFQpGkO2>G3*6>vz16M@Hgi1ZW3BjV5oaVl>6X? z@IH36s(0aAS4bo>r!9iplE4Mo%-z2sbGl6&ncu1FG+(aFfdxmpH95V-;Aq>&uOpF< z$K)g$Ar``6gmf!JtIK7~$U4WH%cwx8(#)~h5{q#+g2HsrCyfi_I~*FWNjT9oJH(aN z+XNfx%@;!V<4LH-4*Ag0n$Tt`?}HOyb)nEWZ2J|@F8;m52K05&gP7Lmpn(E4oD&b1 zv!6gU+`Ek>#rsqlDgMvTB9TV_C`}+3=UT;|S>=Fx>AlH@Yg1b^pltf8D7Mc^@+)pYYNR!aLTYR) z@L>w>ps*hCdY+-f^@&w6K0Y=s%y?;NQf=~y{J1?q-ZOreuW$%0-+6^_B)F&5CSD!9 z96y6SgF6w~u6VNBc)ZmL=cy+i_xLO*a>\=kg|wBQmx(i&wX*!>KtF8uifwJMVB z#I;RLl2YEvHD8mDnbM?E*!JfS2HktQ3I08dWpDqn{&s3!iTI$dZ-(ZWzNG2XA__0P z-mqZHXKBh={PlWRH1t}GEzexuGW{i!MrYOt*rBC@6u^hHkbjVg#SWTGFwU@qlX0kw zm(Fp|+K=b1Mx4ebgE)u`9&^TWi0ge(^Wy6QCH7w~4arIRI|@_`Ju3Z1o~f4F*hbz^ zPf`CAPeU8^W3~TEPS7W7mOA!3#w5D^bL)S{BN7Ovu0xS_d$o6($5DH`vGdZ`f_v)L zfyC&cdr3rgu_@Zqp?SmXmi?I=66(f$d%At?bCM7>(aq`5vr)g8a^uUkg5=(6%yhcB zB&Qkc(4Ov&FgyvWh~3>7F3^-P66|cjybx|I>fq#i5$;vzfb_PtBeR^;q_Qjm$#NPN>qOkm|0YJ;XkTMC+i! zj=sDyPJ+-4Ip&EbApD#4Zv`TyR{&wGul?~6KWO?P7=`*}g=MXmyhGcf`W+r;$wXu; z|0|fL+~rIojnKkOUj>yaUjL%Le2{I#cFR_GC%!P=dujAcV?$s`PQaDmG_mEY8bf`p z7(^@1s>LJGePu|YT-LpMu&M{l{z)FiZFzl_ozM)(^B)TPOg-oO#DXo{ike&hZ=-&d zkgM%MC8Jfjdwj(S8ffd0DzF+o*LkWV&`egW6ia^Z@9V#rzlQ77lF(BqN&2POX>T*Y zZ0%UE94BHDBV+SjDxs|23kvUJD$iPW@6V3uu8%^am+z_2u!|H(4-tI);Fo~r@X*p;5$gq`F| zL7#xSA{d>wW@Jg*{^VC#O(tUzG*REjg801jnQ`c}Q>!H*Oc@AL5udV*Cm*%8Bcb znv{#B1tHTcIWkN06T=t7m0OyVS*c*$8`x2Y^qrbmtT}Zz!EUF0PL6)hSzm8V3l(W95Z(K!w#waKwg=6 z=|>N>Stww^ZJA}UT220Rg0&2pl}g>jqC$}`h>3Iiszx$)APA;cDA3Gra#70IZCx75 zpv2KW!L}cY>%l6P6aKCIL#3(3>pLnEbR&ZjpLg3s)^s0VwbF$T^zR&6yxROBf{A{h z1b@!V@LIC-ze~@0J-`>FktpaWgUyn>7}2tAb;B3PE5d9QWyAZklMsZG|MMBu{i=G( z$`<`NByYJommIz@%~|_Tl`nwD6(LJEw0z3SHX!Ha&NKJe_i|zWT2z-L&}ZHJ{Q8^l z%6Tzt8jfz$v#!O z>B}OBm!j{I!o_aN;iNhi!tP_4>R{QA#LgJ#&hhj@90lMIFV!Nhmo2igN;TF0ZSA%T zU+S!pHN3TZ@%2OcSPJcRGQnOQk+vHC#~@;|r4<|Pw-6;OdfqEp1khtH5rjR+>(ABU zexJjJ$ZcebfD0_QVV%bLnT0vpF~YUkz3*k%OwfSto!rfed+ipUdwqN}P+67)$ z4M2m?)xpL8j8h^Al`Q6i_*EKgL=sFlfe`5*_{|`X5!6cYlp$ri=7Y_-B&@vQZ!8EY z%}10a622~oa%{lj0va5ia>uRGq`yYY5p!2*Od4a~SYM2i^Z$@Mc9rVF|2xFD6g-pHHF zCO@45@A?MsHj-5`b;XbxUEAIZu1HZ45iLh+yBPojinK3P;Lq>43HwxUm7Hv+528YA zzQO)=8o~?E-l?s(5{>CkHWObd&4pxR@E7LRNc6WU0&)+fZN1uvDZ%(XI4iYnLQW$D zqaJs|;V?5=>S)sVr3Ksd8^ls>#}}Ns?8iT<@VnUVcqivQ6uznj0F7)S{C2!CPGxcS zm?`uPG>0}zKip~N_f&X|^q%2R_W$SeRSMVn5@&8MCrta7?7j6Mq{M7W+FQ`-p@H>a zecfAqo@rny-cvyxc&r`zl`V{W*UnIVvVRBeXt|sxc|)sNe7d!#nt>3*&d7J5N#4@ycN zqsU^*i<@HtwJpAmp8DN-uhzKGGPDRffJmHE7&`AaSvsmSEEM;z#d-j8v5^AD>5Q?= zk+3rD<^srhcV$=BM}l=Pd0*z)vt=&QVx3jv+ZzXgOdgtVE^H^&C)1bB*NUCLG&^c6 zxWVMVMlu>lt~#WThiBKf)F~Tet%FpCsY-baaB|WIJyeO~mAr9{LT6ZdozaWiaURTf zmGA3)1M~c&>8`2OEDe#k;FHFKac6!`@mJ}is-St-S_gEQQMh~rT*Bqo5`Gz89Q77u z`^e6&`}qeaXJMOIct*hCEho36L2HvwgPc)Z?F@F16(O~HD_lVWsPT=deB`) zBW(=0({-{a(6Hywj`k-2KjjTWx-K{vWu@Bt-M! zV=Ik{p@^IL^{&-Ub^i8Q!s#XYR9B$+%?RUM*%EQ!dvCl&%-kfuAKK!Q6p!c-YSNl1 zK0AborLdJ~8WlkBOQ29pi^t@i(Qaeuw@FJ1iAmWJW|bJgK27mmo9Op_X?m3DNs!0jvk_nLvGxK}X!~Zte4n#{bZ(M1C8W?ohFE9>rK=he2?A z{-A-^k($@`-AJsdU@sd>VzYu`e8BX9R*Ww+hdehpeUU$a#`+})7GuPFCXJ3!=LTB; z%Y{$@w4V=!iFm-U>Mu45mr}K3?w;wd5o&^o=yqK!isL0F@$7Px0{FF+z~E-uT-;R=JY-Kfgs8SP(SLV|vCO16BziL3Eu#931RM5ebBD5(9u zl5))?2Ct9>YdSR5B>=Ow=Wz@yhyVtbr8k&N#ou5aE-Zaup##&;yq1$0@eI9p7-N!8E9ytZRQhh+q`xd%LwVkc|n;Dm+^m^$BCo5*S?CbB`(p!$4}}M zMd9e)O<&worRE7{hYxTVH6ejstF7uLH0O_$1oHR`Ly0~8j)>D9?ejTBX7)1b8BpC)eEmoyRT%$-2 z8;@i8_HHPqIC_(OlC--srnv@(5a0L#eKozX5@<10>DE?VXYT2_3e>&WqwPCdc(Q#p zQw3$}pm|}NLcoyZqabFx8@=F`Ny;xQ&nQynhjT0D1~yzBFM8;zW1mIDG8meWaoO@2 zk!VgJbalnS*D-+S|14fb1@4?8G}41fw$bc&0!_}PiC#@061qc6FAE47FOnW%g|VGJ+_Ee{6|TKQ zFghjX_&1iMQukg1pBx3CTxkq(= zUqsHBO|oA+tRla}ZX02Q5aJ(dA2eI{V0a>EBE7R-@8rBSL=lc*wmg6*8 ztXv0a(LNmDD~>I?xleY6mE~&e`t(U@k94_>bXV=e5bdw_`+hP-Wnp4hqv=7jE;y%0 zEka(42?~jf#u{{1pxU5-e;L;J-PBM!-Em$36B#9-gL2+!J_-Y4VJZ7r?E853jJ;1w zl({3b7pDD?b0Yj^Qu%)qzg*=MMlDzaVQN${>50cIt1X?Y19WJyoil#UhMM;=@Cxlp zxawwPSJ{J?YBRBzb?aMdn%~@QEDCOJwMvn+4KW3qai*u6fzz9`cbJS#TAFlh11T0* z<*h_a2P_tk!aMJUpHjkOxB)0*(gwj6k?_zFsc>+f4;vZ-^KQj-{=}DuXcpT{jQ_Zf z_IN5aFxkcnbH5vda3rQ5B~D08-Zc#GI-tXQ+ugdR~%4>>s@pwIh2A4FpDZ?~9SxuBZj0 zm_3lE*lOotaZGgiK^HG(OX+;j)}n_g-tXNP21fo(>ReZ-ShTiPPGm2?R&JI0MN7b& zdi%dr@J!C>)mQklVYI91k%i)Syd-7i?Jd+I$Lan51V0stpf0+^kuy{jbzovycyIs1`ZsaLUZJ+fb|;g#Ey9lw(@8-Nrc`*;hzDGYs-SYD(P(u&&I`k! z;wR}Dn9KdmJnIYbpTL=Mi&OZHJXB!b_KVkAXxv=2 z38;H^zbD=F`+h?#K39vYzIa-9-p$!{)%LDr<5s5~&wBYnaT@K7bNc&}H=btN?eVrV z5>s@@hHGTs14XbrfZt|1?)6S-*WVt zGt$Q1R;V5`+soIEYz#!i8d1S=W53mV$7F!ujSNZCeQ##E-P!#Vk5@Bv4AoWNw`W}u z1QtF7CQnZVpkZ0WKi~5}H_Q2MijnRlCp6pW@qgYnvHNqWZ1$YHo2}3MdRnk6V)eq1 zt#wg5GzXS2DaRps4|L3rd$1UFbmO?%sBMbBZ@n4VU$?T$GMBrCdXjp-Aha+PkX!kJ z_diz*^A8Y!GPIox=WDCI8d9~wzJKj6zu{&4<6oiwd-J`Byd;kSAM?dMjF3HB%XkL| zfl!aBicZO2UIOr8iWN+E@=yc2u`!q}q10?=ztD8Q3~bUQN_EDX3}K6NtunB-S0^p~ zLW0QefMvUxO4~$t~3=8*!#|R@nA5 z@HI+h{0-v5|B&}^pX%C09fzp%-Bs{c!T6!3dI@}DGLzafX?a-%c`qPR64!GCK;1Ut zxI$_=e^kf*)Gw)<%oia_Px-*HuKZ{zGQEqzPg@kem_71vY6gdwfozU}w*xR>ZlA6= zNz_opcrVcx>b2gE8^fwX*@COO8*3i}pN$`?W@UxeX5w}c`pDUIYf8w`!0Df!MC!sO z0hUj7*kxbziXX=BqU$)+uA<%uJ?oP8$Ix|}`Y9E;sn7;Ke|zh8J(7yxYkoD~3nYgq z+Us-M$|{Pg8OBi0#FOe#E*%6c8`02Fwh>E47=16@Zy+Rdt;G(H2e5`B(QtPbs z(7GDb^G)fp!F?4BnqE&Oy7S$HoO@ZUAIs!+4j3*9XHr zyZIPa_sjXq(#@i{$A7+o6{_%pM3@dMh&Nc=xkbjTWyW^|Z>}DuP-%$Iw7em1V!ylaU&qLP$0(JxtUKBV zlyAbHL_YDS+%}Ywb`vfOwh!MiF#Z{S#P1@e?c%**E9yHm*XaAX)Vn#6)A>kU9!e=E zf1#JX%W~$$gS{#t1D(U|xBZ!F{2!F$_zeP0y{=DJBlVLZR`hcvDdO#S2;D2j+QgliHh@BT5^Q5}dauCmfP4-Skm zIcmC8hI|mIL>eq2+f}(t-8dH`?E6RdX_j+$r|Z)w_D?*rWMn1H)A4Qa=K1z`LHob) zyi}(ye*VT)X(Txjc^PwVk0JGzQpx*alS_mBY5{+aYKr`If);A*%iE|M7GdUTr^5+b&Yv-L*iA6SR17 z2~cQpcT(KlU4s;Nr^Ve}0tAQR?(P&Vy!rl~bKd*~`Rv)9ow;}BzPJumFjm!O!+l|n zfntxD0*ZKIiB{}Vb%eoWME+|KOdG=mhfa0Tl{?ia1IAIFX|I8@@vKpw1LWze&dg`k z`-*I??f@O@pZU4}@W@<|KOOnkDbrdbYsc;>Z@|s$_F-!6a0UP?vJ;OZK{K&-(6C1$ zPia4rap&x|-ZY0uj&5{#&*$GDPTj_y6J_0hOnERlg&Xved?E7KJd7p*@Xao6A;bL8 z^_TV7Oug^^tL~=&AcIcEX_DvDp1BE)H=|VdzmXhW()ukq`_FynGXHpjN^>PHFKLJD znk?*5e{}K%4-EPT6Hf?woJgXws%LO7C!;kS-ndqrf*`@5y!iX+C3(DSxcBiX_&B?8 zGn8BeR!2fYrYIDCaw@W#tOG&sT}Wy7N2$EhEPaNC@R8Nhvg9CNkboi1n48D6Tn z$lspR(b&he(r9Q(o%yccljr*8$90oAbMdJ|wyKG1e7a&tH579g5V~E>W&~HmqLZAl zES%xUDp0}<*>^%~k*=G-JLs`xiB`-5PP|e&?ln3Nj;9ju{h$5_=J>aU_3wn#QyCf8 zh8BX&Y(wZyl#FN`eKhvEX;`F~Bn{EoH_Im66kd{9#VF=HKKs5KKI!}32V zsFi0C`;Ek>`Qk~5y1!rU?Qi#B1ip!3-6Oms3ok1*W>E7p<5@X->?bZFrM{o7synwC zr`Ue)=G}cDu9>2+qB>Q>=rvqyNUdLzI=HHN39yqi-e+geoK%iiYRKrbLP`(Ok=a(P z+`C$8J~pIl1fH0Vocct+)C3Rx$xq2-xY`>m5-x?1$skSs_YBub2k3o{1k)TUOo!M} zq-#BuzXc!Pvg);MPUXMt8B|%AXJGUd10jpb^=V8!-YA#xSy$TTf8+m#5F1sPc7y)S0VGo$A)DUr5Wyc!!Es zWeMu}Cxr^=!UMv(pPqB>^G`1PNv3q;yy(mLR~hWOCg3$u4<@7SErT!&AITJT9DT;{ zT8L364^USvf5WAC+Ff-?uX8JX^-l(I)b49?|HP%y_xKdu=aTdF5kV(vS za`u3n8;T#gz6+OgRwPffl9Tg)Co3_wZQ7a^lRLesn(!Da;=guf!fuos;O4ILwj<2l+C>nb86>`Dh;)UrH&&fQ z;0MnP?!Jp`=dO|-CqWQS-AA&J&d9YOo^4B?mt|UoVzxT1vM-@ewv?l+RV zL(CMXZ}h)MjdDCUq>vjv07y484l5X=z;rCtW^B%vb(RZ{?Qo0Z-u*&(GoMe}kF)6N z!&{wT$uSw#`OMaol!p8^Su+-(io4vH=b|vTsx4Q!o&J}xNw0LiiRmrh7eZcNt6ArW zf6*|PJ8LN=a(8HQ%EQ*OJ#ClIr|Ug=w!YK%I=J^d%J!ucRxy6`EP7|xNWj~iV&w{ z9WQu2+UVQM0wZLT%T6s~&f`?zn!QgJO4xx%-}aj&8;Uk25G(R-A_Wu9@lxyQxuZ`| zU9?ZVqdHsskI@_0%1X9*6G$mq+E4zBw=nd-!7-M9D*j|%DjYcWZqYSlg~OGvDlrDP zHNJBFw+a#Y1_;J7U<0cCef>0j#$$A>+gHS=8QRky5#OI1KP2uE+_`cL4TV#lu2RdM z@)@L<1MH=}|N5SI`P#t!@b^Oe+F}fn03C!f_yy#<&08Q=4S7eo1$4gra>y}^s`%g+ki6nT zxR?SOvfELKLvISRKewINgQOsw?*BXEV$QX=E zD0OyHYD*UR(|NY6tf|YK=0Ps8EjnJKyjnb(55ydC4-fn93>o^7ibP$)Cfipc zNTl*8jFR6KM7^4s;#+&$+Llh+r)O`lQI0JPTgbbB1j#?(ESTZqX8w)R)jK}T??7p< z)jh9>P1134iy>m)PzocH>Rybl>oq?vBl@c_ExLh+h;SB58z9?sdZ`2fv9H$4oqd8U-px=OA2Q_~x652%LZN6Y zJcUVUy+TWdBP9{-(x5l%H%VaW9aBi?J?&Z(!Qvp9MBCa3N)|(#K51_fVatuM(K>u!3N*OF9^s)lj4G1hR9i12kjyy#T78lq@`wNo|!Z$65HMi%R z?(fo@rPg!YuAslTl=-2ZzyN4W`LmE&w(NhM0Of7RW!%%AjZSnf zOy7(TyL1Fw)W1JwTP|yy!PuVshlEv6tW^s;lVMkC5x)NfmBHQ~CxHT|QDDJrcYbTy5f(A>f>7t9 zBJ=%D3@D!Z4SQTFkI$?D_wPWnAyLxMj}@tlGxu%ZZ*g4?9W+J0lUQ{xTK~7Au1*_| z_?H05Df~}!B`U_}C%HrE@2Rm};Vq*87cLPBniN7KIiFuU7QQhaL$L&#nWjPUiY_z1 zFtRyGKYaTHMVAUxQ_8SiL7^Kp&Dp_cU-#jhw(;B~;_PN6*x~ASn5cUCnDC@Mm7Xej zddy#V^nK;N6G0#B$CPb@?*|)2O#jLOq3^R6q5K(b<(QoG{L#1r!_ z6uBnNvB4sVQq5G`s~r^uRNalsDzM0cNq?{=hFNXOpPnM)dM&`%{qmMK^A09EUlpO+ zkMw@$i|d^bz)G~Hrzb*kC$Fg>;aTVfvJcQBCw=5t(V7FANsvh`K+;Ju{h!Z9Q2IQF zbgBlXD6x$Msn`j5ZrvQ|OVQ%O{L+s-3*b7%;$+)^-_cyPMtGXcQI7vzNK9sSN9V1# z9qo`6RkG{gUdpi0sx=I9CHk99Nai1IQCQPtioL(Bi2Lj}?4kxauYiNNU7eaE)0cml zvtVT*BZ#8);-&6KXYcah61O>vwQM}xp7|XNiZ&eDLkhDL9^)>F>Cn5~+uKt=YQsu0 z9EO=mnDs&u%ljv}_-+&zQn#Q)6ASPH1`!F?IB@-g&)qv&BkM*Vk0hUy7gx86T)<8>m-_ zc2t5a+(is!D7?rLLs}XPA$v?g$Uh&FLCOS2%xWs#$cB7N73+)20ZC1Tk@j5hq zBTuI$$lI+Q{5J+4%W@zhhEVORxm#xa4F(v0qjxels!;?lmlhxWNw&}eu>DA7RHRr5 z#2=c@BtnV;VX6`qj#^i9hTV_AGXwg6DG%Khxgh%CD9y5-brm58 z@P*+uusisx;`G`e?nG7)?_a0ei8C2tG!K1@;(!;sw>RRIH@mbI>R`t6)Mc`{Z_ZPG z|L(6=)evADHQBWzL8~n`wT<5zwHdY$Q9l~(P))`^b|Up-=yV+TreVF-qH1cUAuc2N zRIat>wIm7YLQ_vZar@jjJ^BIDRq~k_9gNz_Z;;T(cNc<2d098AcHA{97^;OQ z6Y-k7`_a?V3)^H~^J8bL72)sqz1tL{GbGJAwWB!|5<|D=cUHkfBtqr+S{Pp$-`3}r zLX!Ot5ggX9e5deqga6lZ^qlljJx|0R@v|-?HXKnjLsk_$U#0ig$}O0ZTkiewGgX*o zxxgqgg>FS5Rf`rwo;iqYrPh(>!Z^f}6*_eKgEX`)m?l$Z#ywBgbP>TNgDj5F7CwVI zzNfS0`ygP;z&z`7d6c|Rpzq6L9`2ZB6j4|rz@N9fLk&|lg>nq{NqU2!V(%a%_CR~z zKm%Tci;}y8FFxj135;}HjES~!&1f|v!ePM;BoXfc(ed4!PZkuhG~kYs)9QmYE0d^Yr{*m9qml3aSZq4 zGY8ofTOnTKZ;WNr3Ra~bfPv6toM%*5)x~z|*9oTlNZgX`t254+^740j{SA7pG4*>$ z)>xt)dwuBe4}iO0Cp&6Of%!*=*3rHB(dP;SHDaXUL_C+PPMGL+5mG>z8#&YVu^i&f z7bnzQ{RUOj-^1n<)n5h+>^`7POY>j^S1wfL&^|7%I#MnL#r+O|}-(`_8K0k??gu>-5w?&-2*T z)=wT@&g6H*eMYSj$JBmEt}D`ND+0;60pGmzR*!_lTXS)3zq#^ed3?8R=B8pP+LQ%Xb=olKV|W0#_C;fEAP0i8^2qsRjb{{Q zlkr(7viRbGAWb|7Ca$os&)3$SXvjx8{m)Lo#k~32{?(gO)^pSAx1!gnSYQ1|Vhs za%`OQc#v+eeN~?meee=+>EhT||Mk+G09@fdtP^te?FDMO9*y{yZkDyrbJW=(=uz>- zh?s*Ov_a%FE?wB366uKwOnsg&A=WW=r zF|Fcx8a|kcH@pK$+j-`@^;B$T?ZPKgo%2PhvXU}%aq5m9DFL|+sQMgalem$qEYnfg z2fi`v+G)BKrt40ZcS)`>ExToB#!g*C(`zU?gRkZmh)xf1U69x!*m^2wqRmEa_1 z5akD*FG@m$%gD7D^HNFbGeP(r-TJGX`KN$V7$0vH{@zhSaMF8l+}$zPEKh!j=K=Wq z2r^e)oopv!i3=cvq2pk7{QBr$(rJ0{?XeWD*s#rm{$55CjuGj-&e-q&VleS=PL(H` z{=TpO{Sm!8ma)-zxWa?%m|c61ANj7a6a&$mnEo@d7f{x6IRa_3a+l(#57{3x3NR*V zO-lJgG%9j=rG(uVnB$zcCz0aDqde;zXb*H$J$a|e1cE=JwXvV)QDy2VrY;b8xsx3I zpSGgAJKRJ4@58prX`pGf(VqVsJCTOSt?TrX0kYeSknNv8Txnfgm@uEYHB(%kTu{)n z@>06*i|=sP-AWYrzbrlxnx3Aa_7m?`#)h%KRIlu*%vux z;tS6N^_ROoM}iSdPp}4!y;-e|wxfcKm(<#^$5hP&G<%inh{V?*l{h4m^ust+bxk4! zWR*wN7P)h%UdG{fh%GD7UdP4LRbrR%>LVO7|SaH*pUp+}u!G})|s z{MevOL=BNqRez5us!=OZfk?jeINpr^#8w<*I+3Ad97FUtxJ1TW2Rqj&!!sOwqYj0zh9_AOox ziy8T^e_v&?crY2XXXL{`yWhA}CK5hBw-0BbO-%|j)G_y4L-qsh*VoL{TT%4#jeUah zusr<#Sfm3tnueNefz*aZ?|ZKQ-h>SzwaPL=Ut|hySGwLWUA(|qaiu5zWF7C=d*Z+^ zb}Q3{)A$Yd8ujicNZCB7xO$ws63-|sue0HF=RMIi1F5H^gJ6ecOM4C63C9^Q9DK!itVGojP#m-&7Q~r38E1X z+{+q&ez-P?U|ChJqa&p$3U`L3-fz2qX>OoJ|(Jl)-QT@BA0>?yl19A$AJ$N^<_%DW;?@P3B zV-jr7LheT2x;yXR)v-V)pS#En{HmSbk*y7r7;!r8oU*#Nw#O(srO%si>NCvpNiWwa zrCKQv{9=xj)^ zpJlbV?{0i{d^`$0@pDCfY=2eQioRh+t&~gp!p6o{DZ-GoV=t!f$p4s3yr7ta5zk^j zg;b5R-Y1c6v-d2EDa^0Hn5Y<(!sR$LMsNQZ{K7lZMC3ox@-k+~*u!GBMiN`wa!6I8 zlvL;mI8TBEG93`%v=DuX?>4#tHsHJNR^9 zzm?s$GZl%mmGB3DPr_LH)gi>SLs~?VQbN(dv@vL2o(T|OOY&)~7s5rK$wTZmQYeh- z!cjSV*xW+!>zzGd&zIFw_U6zw@!VS!9FYu$T@5`gnX@oNLfPAGaE+Is!}|Low#|~i}x+Z-HMDHIf7!`)plXfgw_14*BjV-6S(8rin7M&JsZJTHffRoim zBt45YAk#inA}Ns;AM2lFLTW=3ekv>r$JNPrcFqrNV~S;w=<7ivui9D*-z^aLj1gJ@wPIiSr0y`LYY%+tA1L_Y zXvlR`!36Mol`q@)XRPHbQOEYxeO~&0X%dt4#~f%gStB4Qd)qwJqjOzm$#$#h#B|A( zYtrYSRcx~^v!^oDR?n*GN@lI!%&dT_EzPR!BbV= zX~Famp+s9W-;+jJQ!Fo?@3!{yfav@Kl~WQ<*GFnWU0WdM+dF8x#!8Q32je`cmAC76 z!VPntqQ-`X^}rAw%)JCtB&5fCHo`51)jZ?V0V%zpN8v56;7vQ6ilmopT5RWm{SUK( z5$lO?x}R8aX7GOpI6q{}a{ob+GBMuFD`oMN!FyYOA2D6kuhRYdC*Fu+= z5l(WsIxoz22MU4652drummQ-T$gI?C?-X>>vjLMlO)@Xj9?`QJzqRO6T+|fCm8E#dopMn2?EH_?3MTIeu7}W8= zB>J_PJvg4Drs7YVcPZ#e$wOCqf*|Z)#l^*q#o2LBXbLidq@%570};c8WJ`3Nl?N@N za^A51?ett z(bPy`&v~N$xL2ChxFtw+neFaLD#Hrz{t68;?{H7Ti@A|m+#A((3Y9St8^3jm4MBN{ zCzG8zq#PRo;z>a}tz|{d0&?!GD}eN~ln)YYZ*t3PzJPc>Lxh%}9OcFo#);?8R`WxH zC1C7&pH!?R+M4IXFb4Y!*+|KhPskZA(|Aa@e|)>V5E;9=rykQjUR`?B+Z@@@vv(iY z4=FC8_)=re>p*7kq!(spX4+T|QvKnFX6e6{8%mg)lQm~DI$rni%_(iXR$RzR{&Odm~({Gk-uW?7(%L|?G1j%}GrDC)+Xc`1KA z>hNp3%v|o<(W06zI_}>@6+-O2w2EbScRuFAXH=tG&ouqQk-z78UEJWg>NnbnW;)@d zX08>x*^<(vur=w@P?lf9qKSN4QjYLL>O!G0#EeRh!|E8Nvz!I*>O+1{YNC>c&7t`S zNL|Q%!sa&nH>er=gJe^SG1OU!n3Tvb%6#4C@sMtSa$Q@&Gc5-F!KD^qCG~`jVyt9} z4MCu`<9({Tz4qomTF4#^^*l88-K5LjBQw07qn>c)oiq-=6k?9k>0Tj`8l-7rR1 z&+-0&M3qA8dxWidbv_?%Y&6AftC?a?Sh?(6LyNf=r+`^QO1D3uh8s-?(6eTbhb$_MKPVuQwHxzt||e- zYv#hpj$R?$75WT22~Skq{NY_20<(95=g|ml26e%Q@BY!$xpbhj zu>Zw26RK~F|1!$7gE;XS0Zz##3Pel$rz||1(`J20ysKVCCqErA|JDL1!LSR8iiy_J zh&lk(I>k@Wd1?@3sv3}pOuj=FQ3NHqu771D)&f=dzOIWJL+DzJMRcH+j4NWX})c=IX5{!!s-o$F% z4){t$`dZ{CFiC1O3EJXFU@uiku11FZoLyy?w`q}XlKIjT7{m!<8tfZ;%T)>{(0VL~ zqPcvf*58R=vPJn6Qk5A;g`;Q8R5Cu|n~qG8Mgt(m)bSRgq|b+DZ)*vn{lV1Iw_S!( z!fpaf>kKlT^7`!2s8*&Gn))3zKshh2AIcH+)}kr8a2jU56TP{ql0L?LgMSnP$YXJC z8m`He1FG2&AJFdCmM^b{fav&8EDcYF_wb9+4>Tx7AVc5CpZYdjvDNte2l{#yNNKcI z)pNpn){M6DcuIJe^hYG>{C#`8*oIlRSKC408^T;8@KVuloe&4({N}L|TP#?_p-LKd>BG{u~=+kKCPLy<%X5 zb1F5}G|TWJ=I^J_iDp79B7)IX_rDr3Zzm~znv8^YwRl$7A-qm!gjJa#ZvMZ%HA#^O zE--Ynt+lTuko=)!UJ!xj1`es|Gks*}qU(>S!VhXb#J%~>y4Rn!TP0V{)vg56 z!4)-sCY^|FlNmgVLF>VyI94E#FUM#A@2zi*qV}R;x|EMC{bQWPw6dZQALzLK&Rnt0 zU$Oa;ua^$r#eT~QUtXwza#$@~+V z8DHEyv^$dD?vt?hT#o_*$r-{Aa#kbWp{KM+{LguH`BU_r+K6$&sfriuG?VUdc!t(1qTu167&PaexOqC55^X7mr2D7{0W5$f}-*>fD0M2#;vzw7$@ykO_KKkE; zO#2}U6smcwAJP~g+5Di7o?T6us-6_ax_5E()lXl90QtB1)GXpaT+E0Ej*?znZUise zAdl{Q`9*!xk-wdJ>qLbba`*xHrBhl_mzt>_@aL}jp_s3h`tr{ z4J7tSM2o?9=pb}CDQ}&u@^|9woSxt^H=DA~YJW-pkU4ud5*Ox?xq>X87OX zh{urp0+zeCG3K=^kO7nDN0txdLpm8}&9`+^OXUllMBt>LYZCJrpTkbgztv^}e+K9ciRT;TCq@FGzgK}Hbi-*Q$9mmfH$ai*!$M7 zo#%T3h!=b*AO3qL`P6I}a-(@OP#il^q}eWSoa}JMWI!E5oh$${R*(C9L`<#oTHT5L z>i;Z!YF2@5+Dvjs{Jo(g@Hp28t2_?;`n}Y7B!%v1e_Tga@3dd8ErZ`AG|@aSG3K)^ z!x~i5AjaiOx28!%b7FDyBU)~s=<_Scq3OdOq(Zykqs)N$Ry&_~r6FNM1C=t8m;aOY zKp_#}4=Q3K64_|TUg%eC^wzJm^ptxo6>W|Hx^*3qA()SP)$oCg;7+B{HJ|)!2iq#5 zK@0=7z+xl4T&<+>%)p}D5HT+Da^sn`i5Zm?rvt(MYbD8D_yuyKzcUs~_w{qjsg%Gt zdhQK9Mk$OIwl246%#Z|;*I-Mgk(jPaOxMRSFcV5A_9+M}Rr(DWF8{s#E`4$23w2L?Q#RDW?PJ~N4EYRipJ zy%MDe>QW+smU&vkJCtl#c^{4+OpX6TXML-!=m>!YJkZ*oDf<_JG3+}!{%2R#arQ#F z>m1#ytJ5w53wBjN-I z_{f?S;(ht82FvCSxQcpA`H&jxc38MAs@kW((hhNAd_Yz)FNfnwS21@ zYxasDkpdY*7M*DF(PJE3X!R{Tnwq>JnHNN_kiV|*D*D&Z?@KI9%^hfxPPMPSE6iS^bS4KNykG;M!`7rtC1bpy~f?Q1a?sV zc=9KIuEBh5(M-P1N!&aa`!w*hCc#E zx~=zoDew!z!5G63%~9;?Fm_4ERcLGmJ@z0A{ zHXQUTAh^t|TBDOXpI(f(!m)_}@{4_(tQ1=(J(PWKM$EwlQBzT-JdZ3hq0n0aD5BTE z-E>3jKOghME!VLrkqec*%cC548Exy%#Y$hQ$QIGN>v*R0rLuYegNDoI==GZvXnB@IF|G6`~M zIif#TEWRY)T61JoL8Oo(t<2NS!=uVi2OjDjXuarh^vww)Fd|Q9kTf zAm4;%C8|Ea)B)};a*fAgOPo`0v5JJsoXGZpvchHIV~#i4s!17v&p*ybGXNqsP5&asK~_cQSZB`aOF$g5$9R9^Lh zRC#ZvMRH=yuYm?ULqfBU-3h*oL41bfC>FnpvFXp(=1Vu;wHns zuIw_GTUwX{q`1pWLMuHq;z!ZRe3ybmxaI<}`^~+WaG{9pNoLVztTJ@*kwoqW-W)y< zz`?pb0S{KZ3L;$d2wO2{r^NG3tHxJ!>40dTw9N1&7^OH!|1oA8j4};|QuQ?t7V~KT zR1lXbPBe(-ZM%;@;6Iw51#)Y#M~f5BQfSDpt>)-O7@KK3-?^6{%)%6^lTUbg7zK7#}%ww_>z&O-8F!?S6r1%M!+``5ycb%Rj z1{!6uJ!dWHYy_`tyAA$%s#{k})6*0Xw1Al}kx-jF+NC!t4~hHNCTX@sLa(lwr;4mo zmleKSjyvbuZ0@MIi89(@nrHkRcLkwq@0BYGo6+cfwYu!3BA+Gzw*5yi@60UcPzPj{a)YZMrJ zl&Zfmc$ra>dnw88Hzef5*NBM!ec zA*RC43jg;c@XO)Kp61%ysD;DS&y!H(6M8j!s6YzCBqQo+0xJ^OY-;DJ4Czx^Ah>s( zr)o@yr$Aen7jd-%f9ftV+DyM0cRN`JDc84PmiuZnn@AxTx4je-*?)v% zAsU)suoCo>>@ zB(Vy9Joz4pCX*2LIRF7k19jeHq$j}-ArOBjv>um{SbN3n0iEEO(5Odk1i71|yTk~; zsbD^%?`)6+-F9;#9jE!?@{4Eo;=R}KO)cYc;>hu^23mA9TD>*bs4zduDbkH@NuR(M zzxw=5u7C67v_wC$b%2Vn2dgi5K)I|%TN!z|5733XU*^>Qp6n9rN4>JC9ml3V!s1Ck zv%1#%?v~z}9}Nw!0p!JHv>5)T13>Nd4bxDU`J{5%{$B0N_nXbg44pgxMp@Wm)4&`B1PC%(i zay&7kWi-+gT6Rsu*mj?PV%;9rUX~j9$SHT7O_QL~yJFYK-Ty@qwU!5GGYw*l3w2hG zN*c*j-4972Ok8k;31l<30-IhL_HWLLR9G0$o>#=mnkg{8#u8|sm_1Pl(BxPAC;e;t zoNfLGTBAB{x{45K097LA*T0H|=%Ak#_(e+mj7*rj9yx3pp~2vO(W_u@!DGZZl;1Te zuXyfkkmN zO(s2`hwp*a1XNiSk-iM*?AD)-re=P~4r}42tP%{FwQ>9a<;Tiz49Uh7C1r4+nGOF@ zUUD@sP2gRdNkn%xB(fNkO*a?m68mSxOa@Eg{O$L^J~+no;KOb|!kZ+*x%mC+P=nns zGx@_}A`x6rci3FfA*zQ*)bo7emp4p|OWv+QC$s%j!Csvrb=$Sv_%eGb7~?839xz)JlChXzww&|kV_W)@8192!{&(Dv%y#0?mOM?+xD=9 zfK~^hJ&RKGzuQ1c;uO#h>et4ZJ}7xlJf%q?>Gpgq(L@Wh za@B&WdEfM01EH}FY{dhPkt}6*#FNa=)bz6$4VEqYyTVmvXlE=vXywVZ&!>}STkHtz z3cF5WAHTXQTo9*#Z?;(W`1c$92&nGIRt1t?QetmpVK~|d1i!)?YVyIKo@|XkihmYAILBVlF%)O>iFfD3eLTbvYKG8XP)@7U51a|3t zSyVMDLO5{o`Wfd}|48nd`)Furvw1B{J9f8+#*tCAv~3@2_cWjQZlPx?AePfKzu9cZ ziufe6r#D)2Mazw#&cxl^ZDUaiCWI6#^7@=7rp`$eOhp>wJu!kQyH)#R-&ftl4`wtp zE7B%{AW0Q?7kml%uHBz!KlEr@%*(9DCay=J`^5fY53BJ~TDOat^tGj2NTCz)%7NWR zD+-$#^6!`ny@p&S0OgAK{>niO{Mvh*`twg|-YJOfU#1e3$H=e1Ku>ajtSq3Xu3;{d zu8+y{C$N*j+q`Vwyo&cR3FN>=_4rmWR@cQ#nmlUATv&_oJeG?SA*viF(l*t*TNf@V z=_8CCfhi9(_oW9KkVi?A7dJFPU}@xq^6Q~9$_c4dwQmhEugZasSgN z&pV&H{x7917!EmnNRrcraz6d|YK$qGZ#)VW>Sa_4vaRj+`ITDImCn<;6R+-B7xnxB z!#d)zOLN76@TP?b$|4dlAN6dSg8!^{pr+Gm`>D?nSU0p_O**q@8v@@hlS^T+d*?Vw-G3kJtS0u9BnRd(&i248FGu9_ zbn_+>a1GM)R-(Q0MT#pL`HhyJ4aZ#GQws?gakNX~mX!%iX5^elA9LK0L-nwwdgy`( zRphjwwgA0-iV(~;in;c%wy{pAYO3GqiKz&P$@?Yf-#Cvw6C_ea({Xsff4LD0<8@LQ*K3c>*iYJy}JtsWlwoFZK3rTj>C_L6ZFE7;8ryPVNQMK!@!#Iv-F8MylhU5xRI z;)aQZ#{A46I$1j|)@yJ`Uozj2R4-ZCo) zhWDGlZ+S)i$>ITY*HCRb*V9h0N_%8KI+w&aQ*Qq$Q6S5?FCoxc)#c6l?_Eb9l|m4S zoAJ(!ra8dKpcQT|<94h$kY)GhEKPY!_0I#rKp)F6qY@z-XWwJ#P3vzi-7vH6y&r6! z*b~NtKjt2!Xa=NqBfgxZ1`t2AyG3HqmuV8Ei7E{_>FdO?UN1fyz6!w@b#4JVT!ypB zj8154AcQ2uTIdWFzPl2<#-Ua;2w(QvV#ddw#&jFUpESoP=n~5YLDB^Rzl(Q+X@(ht ze+@tH8l*Lv>HyhUgU!hN)@GQ}0##X$*QpZXD-Ru6*E#Y)lI8-6y2Mi;CN;`0Mq{>~alAQQK*7B)Tk2(gJk@ioRm?skdcb0Kp7Z)ko z4R0ak{O6H=1RH;6GFdooiZ8 zyZTJX&R+1u@rshFAt5}S+XtlY=`ulcWkUR6%X;s+<*4)5iCc=+ltJkLqmTdVj6ntt zdx@&;B$*2?Kgg?zjOp|&dXA|~$Sl)K+7AFNI^Rq4u8ZW`r%9d2KW8<1HuY2#dCYU@9 z-BCGsAm}(L2EhMytBqsU_-b1(*!|abM3lEjON2&OLWDtZ<(3$qq@Z~YQm14g6D~j-)g;P}b=ynW zZ!VsAlB`8|#*<%__aIhEg-$eN2EJ)TL*P3f$0%CU$Z%+Re=71gLF2k)A87wEpb~A? zyr!H?&*2ukb=P5-_l#=t`1+gMh7&zX5$5%PG8zpoh3U9S z3sxN~W!<7Mb7)8OG$+lWmSt(&PAT})TA1wQ+rHN_%MoFe5iaB26&Pvo>jkrTumF}QbA%IO+*`1LfaYwM;=ywZX~gmwZN=^yr9o=yeoM&Ge;GuO zbB#HNL>&+Fxy93Q7HJ}ul;6W{!uoek0e&t4=OUl!A4m?ro6r+`MamNCVBHgOJ%6Bv zE@8W#f$Rsfw1Y&I8fO77&J~Au4!-NXwFLFV#F!A$`7K7;SAyV+3fs#s-@0EEKrEFl zMd2ButG|gaFD*6uzHEz=Do{51iiQpvyH#8+T%AAeh+98aHuT)|#ZlIQt(h~U)6g!> zC2;wsic6&jSNlP8Xka?5n8hJGuTUG>5v^JUkKz>qjyU$fif0__&GnRT97CMuBtQ8s`he zp>avue;t_BUAn9#Ox06d4#`{x zDAZ`7zY^5dH(^1|3o*@&O<~pq6Ag}JJ=M=b*W^{ru`yDuA`%GR z)v6kvV7rC-+w}e$`QzIeKc9}aw^L%LClxqR1^J^9S=H(@_nqI(5d(vu2vdd#Eh~E3E;ILRO-mqVI8CITcMqodo&% zT=t8?1(g9MdD~gz>f+sMhj4fDwF40&zq<0&Pn+j!6h(eeq;F}0$N_Gtj~8GhqfIwv z$5RNZ2nZLv`|j;QMB=ru`IqV`^hHk0no!U%6nI&;a=E zd1hy;qmgqRPk$~iXjuv<(TkLB*FJd&vcC-qe#{t50meN@>Cq6OhKGGX{K8DQ__}&o zZRDgvOY5ajy8Bx0iA~0zoNWqBDXVKTGS?02SMkTMO`dfS@tyMfq-9BVw;FQ9r<(7l z@(lvhEo#XE?(t;+mvRq0)f*-<#9rtZTSg$cJGpfl{#~PKxZ~kxO_q_lan`N%)RbOT zpO<|6b_Euu+q^?RcV&jvQ6;$2DM1yg_}kIx5&g6*?zBl7fyO#)zi;PUNSY7Jjr9}M zB9M6z2s4iErrvzU7w>bHs%ZKhQMDiZr&0siJyAa3%y3EbLjNk-?Qx7gxgpMO}@ zJQJ0!oOma)1{i0-kI}j1=Zz|T)VJEu!$R{{?p2{N94f$U#R27V1(Lk0Nt;o3(fIn& z8Nz9p#B2^GX2Tcu3^>Q^?0Phhke~%esoH9zdo@;$SM~xp6$lFq#dS!@Ai5-*L+U>x zf4v~YJacyzbWW20uuc9KLp0$l3FMsvqL;OO+6W4CvLqf79?lve$huA=@J__@Ot{6! zRP7%ls(a7w&+hX>cG_QRgDsBM>Q7#M6#wZ}jq?(3W;`T^%k>0q-tEKNv^EQ%uT6n`PBU=C@=CgX`8jOV3_Wm}dF(N}=Zs-7MAS zn}p@KpJ}wyu#?Ma5BkB=OF%(i;x|2ldunDeC9}eXQ{NG0{u+Fq5@3?W2Szjxw}e}= z4Dk;(XEW_d-WNy{KSpz{=D_1yOT5gKyR6wF=XAoU#o=T!##c;h(HPaJF2v@-1tNLU zU)Y74jk&%5o|@;hDFNJ5TYYVI0_}<%Su9B`Q^Vy!-@uw+lVBF`@ayHj;$BC|mm6Q! z?%&C&s@@qkbd3!5^{kTqBnUckEP~mHW*F_r zAl~%TMW`WY`O7GP`K=1q2RNiH2}lY5$)56&#jRjQS1Ml6BtcNfVr?4~eOaJc(oO{* zYcSx-b+tw>)BJMS)IUo&3lGsgq2_)rYlHrrUP7~w?+Xq~lNEyt=TDhs+?sQQ-jk2- zaco%OfC)dVo`QdVkEzJKUhl)J&3#f4EItMv-k$!w;MxOOA4ZEV-Zt7v}SmVXUL{Df!c}NJ;n)t@*6p@q6Yr0x*i4HW} zIYImIy}lRtJ3bLh5zHlN^lFe6-h`Z?$wUTG8nGL z+W7+F&`0lTfgg>~8C)B$T?vJSCPic>FcTdYLB}xXX20jHCL~N;NqC9Eg+{6y*f>rY zZ)yl?wFV@l^A+BsWwTdC`JQ9@q5|B@jKoByEy*y~)#pa2>UaqLCqh%&p3rjl-RG_v9T-$F($ z)vjqCz3c7p;(!93!vU>NNe-SB0@vkmD>@2p^#>Bp!VdyimNtb-Uwa-|D%?!v3MUOX z+e`hm-_QxH6T=&XZ@Rm0O0HE`jX*sKT!WH=s)+!sb)K$C^M@%Ioxb)xvL5k#458y| z6_gRvoeyu0!@J*46)8B-w@gvQ`oz=*BrA7rg7~Ezbj~Tmm(=p24GU5hVt7W>WNnvo zJO%j+ZH=_HI;{XuzCR2Enj+O_`x@_q*MFwB(hR}B`&v|XA&|PN zC~KlLxlqmvCFEG}V_Uv`m?4w%ERVwL)ta0bD9fv9eu~jd#&stnq4A-k_vn*97h26? z6m{P-P2{!6<@GLSX)!$5wRzH-o9EV_FB9HQUX-qGUBA6nA3{U+YXN&&3~!TH-;FkScLg?!v*gMy=r06F(lV5+Z;{U@x3&4ZBKD8(GE5ZLT=s6c02FV zJXX9J2j0-L`DMTU2d)}DcV;-^Y|9LR*3<*dEFyHW&NXX8D&qvV=K->Jl~{_EerD^_ zJx@Ak*1+r)tVN>{|48&YzcMh`R=xH27dSohOS*Qw)b0Q8(c6MfW01cDlpJg-3tbl*s1FVV~yD#yj>PC5{!A337XUKl)d zf3Y)5+l2kE%0oN{yOlYDini?qKDGNzR+jzMnK!k|c%9=mf8A5V%3qLfGA7{cQbFt* zV#o0DUxH<8X|UE`J8(76jgx^RaIF_VGK>B5D?Eb#$B9`C#3|>1Iu9#Z`Rw#$oHwFh zJ;EdoXZipqtA}vo;Ensj=IhT-u5PsU2-KLrm7n0u21waQtp@JQub*lkMRP{$Iuo~5 zX@~#G56SuZys$#ryeSPLvTq)4#TOA)M&cXGc}}h;0XxF2b#0+KnSu4mkE`%Gea;>$ zN=o#x>5hy8!!sE&&v@@8MPWiWjfS<&uePlPtSuTyg@W+JxN+oS(+K}Ia8eEBgqJ2Ct?PxVHCSO^N)^x7BZ>{tUhVVFf4nWW=B3u!dn& zeB(n$zXJ@YNYPsEa<4@JZfeKxNsu&q_k-q-tG5WQN>zmf@1CpghOQ1kxo5ag;q#PWsZS=Vx0m&|FczGW-#$*(k%zzP7Th zINY12yYfxq68FJHt47VZ#-S~=?S+I{qTOU^!OIa3#MMlh9|F}IHHG-MN?24M{4a$M z<4jke-|N<_NYAONv~i^*^;j8I54dTxnW8be!H*^Rwkvb3-nT?6ffALTOo{9l-mA8$ z&B+X-+hF%tKCK;hQdBk+(UTZ+;Q!bGyPH~6y6JxI2<}Xz`>v32-LS|GJCD(zELAEu zSebA$)=Uf0;2z}-lNgrQB=V4i3uUP-5N@Y5#PJ;9@c26tw~*(!&?7HDj%;*`P5j=z zA(#d0a7g~qps|EC{rDd6k>`N~=_RZDs3i4Bf0#*vBV3B+rs1IP{qmf7?!Ok^H3+;* za_yWQupjHqxDW|k*Z)Fm*0`-mSb=C-Z3d}S5sU>dQtjei9V5MSd<7EAz*qXrE6TOc8_3+hpJ=A&T<(zFNm4>R*)0&uwTZK1l!ezuRjX3rS1dwfe|q_fvL z%5b?r0tVFFU^@$M8e#TYOn!v0GO1!%#KKPQ+~n_bFY3%)EG^4po|UFTH%!_gU6JJn ztG=Rea{d-bAhSy2a%9K#sX)c;)%Fpf8&VZAD_)u}mTAuj)q2_1G8Jkana#y*x+B@b zG8P@yEcT~OBoFvPvtB40`dsyRX1s~kl-P@>16D@#@{8gfZ;Uq;G*VQNp;yKh=@?Gcg4v$ zdf^X~N1kbxxgya3M_ng4L@+7sBi-t2-|Ptit%0xil$jorI^HRyz_7vn|9W!pQ$@lh z+9w9CA|s}kXxYF3At6pCTGq)TVW^ZKAkuC3GZN2$wiL^WX20RbdyQVk%BoRo63tSe zB<^OTM8;8inwT9E*7VTUwC3wIogkbjKc1iBBZq&+iVAz`ovBL{ea^q|o_B^B)uNnQ zq$~breUkWJmEq{Xm!{GgKSK^#@Bib~vgYyB zg4BT9P^aIIbMs1Umv%lp@RTbvG4n9&qhCzE9%zBiH5E~M>=i_m^F=l7k<)S+wK%PM zpoJ2aeo@gJg(?mG_~jR(U_Uc>XaRVAMp|r;kvk-x=VdEOI_l1@Kegb{Yo~;5dxuW? zoyGCFWxSh(#RJ-NMkhI*`3Hk=MRYP~QdAudAn`me+hmJ-AUMXDS7jUr(orxlnB-}3 zyM6Seqp+9hH1 zO6~6~FOs<0&U{HRnKN)nW`XP-gqRfdjtcYN71Ddz-3m}Ti$LP?4299ogk%hHP#`yP zlwMA0FFbN(Alz=FXm3OM`rJ?~s)RF#LU~>vD;tmLoZ#lg{2t@??^Tv`a;Zm$D_V*V zhgr!1_qGjz{1wNF#vU@TMYrIQxPsF%aBOuI#(|1t-%sCBaJYy0R?U_KrF99r3#uOg z;vuCe-=ieaioG3ijm;|Fhr4#a-DYJ(d%5^kh<~Ke=}8=MHAcCD=1*%9A%Kk;!*`)p zXQlW69$2kPw)r#22il5>C_~$UT`QjraT!xoFNnm<3j>bkWGq{?M~Pwr>Nad5|NK*$ zr}MA+y>yrJmHdEJ-GfI@-V`T&bRP$b6zLv1QWw9m$r0i@VvY$>vSgyxoh&Pg@VoBrD0+DflwCXC zEYM!u_#{QYtw?(#6m6nJxAiQ;mxhrg_alRgSeDPifYefx$n!tU@oyFtX(0HN!_qmM z7cvTI@HF)=T(w7>&o%{kpEivwpdV31dYY257DvXLu${x@Lx9oyr_i?{_=WpQA|xYmhP3MR{*XO8 zHiLILvP16MPLoUYzYG7Dy!ryma$bKn!(02cv!))@!4QftuEXXwG4CBN7F)R&4v*{l zo&Mb}96QISIlEVvvRuDy&5~CW z3pq6Y>8)y_Az&w2*sRFhAl7>fUnK0ecnG^(pTin@hv4wRPQd>{{leSaJ4Oiyw6S%G z_|yc8ddyqfXA$j5KH!ocIFlKA6h~uhYX(2AKYE;BP-hEFgAy<&UO<{gG`3^MG&Zn9 zrT2vh(u{Z~U|0joW^ER$UxHrbgqc_ebdC-730Zmvf z_d06#No5734*wF*(h0Zw1`@-gFIQDIx#ggm(fb6W_yJBW{bL*U8xKuZYZuXEmL_fE z77NY3F5~+;#X8dT-lW01)ATzL>iTL2ygiupR6%PQ{!7sS!fdN8M$yHwkE7pX8U9FH zKZ5StpLuU3UT(mP^5PEox@9-4Yig3mjdsiP{+8t-%VR0g8AF5&MFf_L?7(zAlos~P z3gG&LI0r^C+bZxGQH+V4Mi=Q#h2}pt=u$jnU&0b-NX5cFQ5bBz%D#pXf|~iph6r!0 z)=vejxVaGB5FF;Mm$Tv{5wbZ~28fio^PfDfO1aN8=>DV$ZJM@(8-X}?K*kw5c;1?c z(2hK)(V3%6cd1RvM>j?%qOEzB(RGBX8eO5KktPZ=qDq%$jdJmSeWMM)QqNl1OgCP` zo2DX-48f^f>zP$TP%L7=nGJh9!s8jzc}qQU?X`57fQ~|gkbr6A0X{EXp~QauVCtBe zB0Z!kcA%=TMEEbGOb302)$(Ks&-+J^Y_9&@VuV&n^OF3!X6)x>ChmgDX;dJ=U-_Vh zF7Ar3VV~-|zYpOfZ!Lm)Rcb3!i?zz`-ly8-DK8nnD3RD(ZJ42naCtTy{IFv$`)PNC z^?R;HV^)h$vNi8!BLQda*uI)+{U{srm;p~_ao+S}Pe^UmgI6GC&IeDgpXw~d%n|M6 zt241t!ZlCQn%Rn}*WvcmBTo45&0SapK`9#%>YploiEEuPBs$hM4tr==BiX@b$JXi^-x!Gk0+&LkYhgxwDp#_7VoMRHGhNvNsOj z+J44P>c|g^q@q12PeD5AFf_&2Bp1 z0j4=YdwuV^IP9#x{P)dPsp6y@wiESX@NRBAW6 z=#RRIdfv{`=5*JDkyzqhTNDaDUbgcP-=SvcGa~zb^`W8E>u+m9PpTeTrlHy>O8^gE za!-feRTf5`7FlY~z%k%*_Z@lakF138irw#rVYzU~`ft!38a?qNaon^VH&ygsaXYCB z-JVj5^TJsLIP>3;vF&;hmPT>0-Hrk#nZNwxrBrHAJHI&25vh25b9v+cy$B5ZFHe%; zdtv|RCzoyi-v#j!e?2CY{#%Udr7B + +// Ackermann function +unsigned long +ackermann(unsigned long m, unsigned long n) +{ + if (m == 0) { + return n + 1; + } + else if (n == 0) { + return ackermann(m - 1, 1); + } + else { + return ackermann(m - 1, ackermann(m, n - 1)); + } +} + +__attribute__((export_name("run"))) int +run(int m, int n) +{ + int result = ackermann(m, n); + printf("ackermann(%d, %d)=%d\n", m, n, result); + return result; +} + +int +main() +{ + unsigned long m, n, result; + + // Example usage: + m = 3; + n = 2; + result = ackermann(m, n); + printf("Ackermann(%lu, %lu) = %lu\n", m, n, result); + + return 0; +} diff --git a/samples/linux-perf/wasm/fib.c b/samples/linux-perf/wasm/fib.c new file mode 100644 index 000000000..cb928f65d --- /dev/null +++ b/samples/linux-perf/wasm/fib.c @@ -0,0 +1,32 @@ +#include +#include + +int +fibonacci(int n) +{ + if (n <= 0) + return 0; + + if (n == 1) + return 1; + + return fibonacci(n - 1) + fibonacci(n - 2); +} + +__attribute__((export_name("run"))) int +run(int n) +{ + int result = fibonacci(n); + printf("fibonacci(%d)=%d\n", n, result); + return result; +} + +int +main(int argc, char **argv) +{ + int n = atoi(argv[1]); + + printf("fibonacci(%d)=%d\n", n, fibonacci(n)); + + return 0; +} diff --git a/test-tools/flame-graph-helper/.gitignore b/test-tools/flame-graph-helper/.gitignore new file mode 100644 index 000000000..b5ddc69ff --- /dev/null +++ b/test-tools/flame-graph-helper/.gitignore @@ -0,0 +1,2 @@ +*.* +!*.py \ No newline at end of file diff --git a/test-tools/flame-graph-helper/process_folded_data.py b/test-tools/flame-graph-helper/process_folded_data.py new file mode 100644 index 000000000..e4650fe25 --- /dev/null +++ b/test-tools/flame-graph-helper/process_folded_data.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +""" +It is used to process *out.folded* file generated by [FlameGraph](https://github.com/brendangregg/FlameGraph). + +- translate jitted function names, which are in a form like `aot_func#N` or `[module name]#aot_func#N`, into corresponding names in a name section in .wasm +- divide the translated functions into different modules if the module name is specified in the symbol + +Usage: + +After +``` bash +# collect profiling data in perf.data + +$ perf script -i perf.data > out.perf + +$ ./FlameGraph/stackcollapse-perf.pl out.perf > out.folded +``` + +Use this script to translate the function names in out.folded + +``` +$ python translate_wasm_function_name.py --wabt_home --folded out.folded <.wasm> +# out.folded -> out.folded.translated +``` + +""" + +import argparse +import os +from pathlib import Path +import re +import shlex +import subprocess +from typing import Dict, List + + +# parse arguments like "foo=bar,fiz=biz" into a dictatory {foo:bar,fiz=biz} +class ParseKVArgs(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, dict()) + for value in values.split(","): + k, v = value.split("=") + getattr(namespace, self.dest)[k] = v + + +def calculate_import_function_count( + wasm_objdump_bin: Path, module_names: Dict[str, Path] +) -> Dict[str, int]: + """ + for every wasm file in , calculate the number of functions in the import section. + + using " -j Import -x " + """ + + assert wasm_objdump_bin.exists() + + import_function_counts = {} + for module_name, wasm_path in module_names.items(): + assert wasm_path.exists() + command = f"{wasm_objdump_bin} -j Import -x {wasm_path}" + p = subprocess.run( + shlex.split(command), + capture_output=True, + check=False, + text=True, + universal_newlines=True, + ) + + if p.stderr: + print("No content in import section") + import_function_counts[module_name] = 0 + continue + + import_function_count = 0 + for line in p.stdout.split(os.linesep): + line = line.strip() + + if not line: + continue + + if not " func" in line: + continue + + m = re.search(r"^-\s+func", line) + assert m + + import_function_count += 1 + + # print(f"! there are {import_function_count} import function in {module_name}") + import_function_counts[module_name] = import_function_count + + return import_function_counts + + +def collect_name_section_content( + wasm_objdump_bin: Path, module_names: Dict[str, Path] +) -> Dict[str, Dict[int, str]]: + """ + for every wasm file in , get the content of name section. + + execute "wasm_objdump_bin -j name -x wasm_file" + """ + assert wasm_objdump_bin.exists() + + name_sections = {} + for module_name, wasm_path in module_names.items(): + assert wasm_path.exists() + command = f"{wasm_objdump_bin} -j name -x {wasm_path}" + p = subprocess.run( + shlex.split(command), + capture_output=True, + check=False, + text=True, + universal_newlines=True, + ) + + if p.stderr: + print("No content in name section") + name_sections[module_name] = {} + continue + + name_section = {} + for line in p.stdout.split(os.linesep): + line = line.strip() + + if not line: + continue + + if not " func" in line: + continue + + # - func[N] <__imported_wasi_snapshot_preview1_fd_close> + m = re.match(r"- func\[(\d+)\] <(.+)>", line) + assert m + + func_index, func_name = m.groups() + name_section.update({int(func_index): func_name}) + + name_sections[module_name] = name_section + + return name_sections + + +def is_stack_check_mode(folded: Path) -> bool: + """ + check if there is a function name looks like "aot_func_internal#N", it means that WAMR adds a stack check function before the original function. + """ + with open(folded, "rt", encoding="utf-8") as f: + for line in f: + line = line.strip() + if "aot_func_internal" in line: + return True + return False + + +def replace_function_name( + import_function_counts: Dict[str, int], + name_sections: Dict[str, Dict[int, str]], + folded_in: Path, + module_names: Dict[str, Path], +) -> None: + """ + read content in . every line contains symbols which are separated by ";". + + Usually, all jitted functions are in the form of "aot_func#N". N is its function index. Use the index to find the corresponding function name in the name section. + + if there is a function name looks like "aot_func_internal#N", it means that WAMR adds a stack check function before the original function. + In this case, "aot_func#N" should be translated with "_precheck" as a suffix and "aot_func_internal#N" should be treated as the original one + """ + + assert folded_in.exists(), f"{folded_in} doesn't exist" + + stack_check_mode = is_stack_check_mode(folded_in) + + # every wasm has a translated out.folded, like out..folded.translated + folded_out_files = {} + for module_name in module_names.keys(): + wasm_folded_out_path = folded_in.with_suffix(f".{module_name}.translated") + print(f"-> write into {wasm_folded_out_path}") + folded_out_files[module_name] = wasm_folded_out_path.open( + "wt", encoding="utf-8" + ) + # Plus a default translated out.folded + default_folded_out_path = folded_in.with_suffix(".translated") + print(f"-> write into {default_folded_out_path}") + default_folded_out = default_folded_out_path.open("wt", encoding="utf-8") + + with folded_in.open("rt", encoding="utf-8") as f_in: + for line in f_in: + line = line.strip() + + m = re.match(r"(.*) (\d+)", line) + assert m + syms, samples = m.groups() + + new_line = [] + last_function_module_name = "" + for sym in syms.split(";"): + if not "aot_func" in sym: + new_line.append(sym) + continue + + # [module_name]#aot_func#N or aot_func#N + splitted = sym.split("#") + module_name = "" if splitted[0] == "aot_func" else splitted[0] + # remove [ and ] + module_name = module_name[1:-1] + + if len(module_name) == 0 and len(module_names) > 1: + raise RuntimeError( + f"❌ {sym} doesn't have a module name, but there are multiple wasm files" + ) + + if not module_name in module_names: + raise RuntimeError( + f"❌ can't find corresponds wasm file for {module_name}" + ) + + last_function_module_name = module_name + + func_idx = int(splitted[-1]) + # adjust index + func_idx = func_idx + import_function_counts[module_name] + + # print(f"🔍 {module_name} {splitted[1]} {func_idx}") + + if func_idx in name_sections[module_name]: + if len(module_name) > 0: + wasm_func_name = f"[Wasm] [{module_name}] {name_sections[module_name][func_idx]}" + else: + wasm_func_name = ( + f"[Wasm] {name_sections[module_name][func_idx]}" + ) + else: + if len(module_name) > 0: + wasm_func_name = f"[Wasm] [{module_name}] func[{func_idx}]" + else: + wasm_func_name = f"[Wasm] func[{func_idx}]" + + if stack_check_mode: + # aot_func_internal -> xxx + # aot_func --> xxx_precheck + if "aot_func" == splitted[1]: + wasm_func_name += "_precheck" + + new_line.append(wasm_func_name) + + line = ";".join(new_line) + line += f" {samples}" + + # always write into the default output + default_folded_out.write(line + os.linesep) + # based on the module name of last function, write into the corresponding output + if len(last_function_module_name) > 0: + folded_out_files[last_function_module_name].write(line + os.linesep) + + default_folded_out.close() + for f in folded_out_files.values(): + f.close() + + +def main(wabt_home: str, folded: str, module_names: Dict[str, Path]) -> None: + wabt_home = Path(wabt_home) + assert wabt_home.exists() + + folded = Path(folded) + assert folded.exists() + + wasm_objdump_bin = wabt_home.joinpath("bin", "wasm-objdump") + import_function_counts = calculate_import_function_count( + wasm_objdump_bin, module_names + ) + + name_sections = collect_name_section_content(wasm_objdump_bin, module_names) + + replace_function_name(import_function_counts, name_sections, folded, module_names) + + +if __name__ == "__main__": + argparse = argparse.ArgumentParser() + argparse.add_argument( + "--wabt_home", required=True, help="wabt home, like /opt/wabt-1.0.33" + ) + argparse.add_argument( + "--wasm", + action="append", + default=[], + help="wasm files for profiling before. like --wasm apple.wasm --wasm banana.wasm", + ) + argparse.add_argument( + "--wasm_names", + action=ParseKVArgs, + default={}, + metavar="module_name=wasm_file, ...", + help="multiple wasm files and their module names, like a=apple.wasm,b=banana.wasm,c=cake.wasm", + ) + argparse.add_argument( + "folded_file", + help="a out.folded generated by flamegraph/stackcollapse-perf.pl", + ) + + args = argparse.parse_args() + + if not args.wasm and not args.wasm_names: + print("Please specify wasm files with either --wasm or --wasm_names") + exit(1) + + # - only one wasm file. And there is no [module name] in out.folded + # - multiple wasm files. via `--wasm X --wasm Y --wasm Z`. And there is [module name] in out.folded. use the basename of wasm as the module name + # - multiple wasm files. via `--wasm_names X=x,Y=y,Z=z`. And there is [module name] in out.folded. use the specified module name + module_names = {} + if args.wasm_names: + for name, wasm_path in args.wasm_names.items(): + module_names[name] = Path(wasm_path) + else: + # use the basename of wasm as the module name + for wasm in args.wasm: + wasm_path = Path(wasm) + module_names[wasm_path.stem] = wasm_path + + main(args.wabt_home, args.folded_file, module_names) diff --git a/test-tools/trans-jitted-func-name/trans_wasm_func_name.py b/test-tools/trans-jitted-func-name/trans_wasm_func_name.py deleted file mode 100644 index 0206fc287..000000000 --- a/test-tools/trans-jitted-func-name/trans_wasm_func_name.py +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2019 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -""" -It is used to translate jitted functions' names(in out.folded) to coorespond name in name section in .wasm - -Usage: - -After -``` -$ perf script -i perf.data > out.perf - -# fold call stacks -$ ./FlameGraph/stackcollapse-perf.pl out.perf > out.folded -``` - -Add a step: -``` -# translate jitted functions' names -$ python translate_wasm_function_name.py --wabt_home --folded out.folded <.wasm> -# out.folded -> out.folded.translated -$ ls out.folded.translated -``` - -Then -``` -# generate flamegraph -$ ./FlameGraph/flamegraph.pl out.folded.translated > perf.wasm.svg -``` - -""" - -import argparse -import os -from pathlib import Path -import re -import shlex -import subprocess - - -def preflight_check(wabt_home: Path) -> Path: - """ - if wasm-objdump exists in wabt_home - """ - wasm_objdump_bin = wabt_home.joinpath("bin", "wasm-objdump") - if not wasm_objdump_bin.exists(): - raise RuntimeError(f"wasm-objdump not found in {wabt_home}") - - return wasm_objdump_bin - - -def collect_import_section_content(wasm_objdump_bin: Path, wasm_file: Path) -> dict: - """ - execute "wasm_objdump_bin -j Import -x " and return a dict like {function: X, global: Y, memory: Z, table: N} - """ - assert wasm_objdump_bin.exists() - assert wasm_file.exists() - - command = f"{wasm_objdump_bin} -j Import -x {wasm_file}" - p = subprocess.run( - shlex.split(command), - capture_output=True, - check=False, - text=True, - universal_newlines=True, - ) - - if p.stderr: - print("No content in import section") - return {} - - import_section = {} - for line in p.stdout.split(os.linesep): - line = line.strip() - - if not line: - continue - - if re.search(r"^-\s+func", line): - import_section.update(function=import_section.get("function", 0) + 1) - else: - pass - - assert len(import_section) > 0, "failed to retrive content of import section" - return import_section - - -def collect_name_section_content(wasm_objdump_bin: Path, wasm_file: Path) -> dict: - """ - execute "wasm_objdump_bin -j name -x wasm_file" and store the output in a dict - {1: xxxx, 2: yyyy, 3: zzzz} - """ - assert wasm_objdump_bin.exists() - assert wasm_file.exists() - - command = f"{wasm_objdump_bin} -j name -x {wasm_file}" - p = subprocess.run( - shlex.split(command), - capture_output=True, - check=False, - text=True, - universal_newlines=True, - ) - - if p.stderr: - raise RuntimeError(f"not found name section in {wasm_file}") - - name_section = {} - for line in p.stdout.split(os.linesep): - line = line.strip() - - if not line: - continue - - # - func[0] <__imported_wasi_snapshot_preview1_fd_close> - if line.startswith("- func"): - m = re.match(r"- func\[(\d+)\] <(.+)>", line) - assert m - - func_index, func_name = m.groups() - name_section.update({int(func_index): func_name}) - - assert name_section - return name_section - - -def replace_function_name( - import_section: dict, name_section: dict, folded_in: str, folded_out: str -) -> None: - """ - read content in . each line will be like: - - quiche::BalsaFrame::ProcessHeaders;non-virtual thunk to Envoy::Http::Http1::BalsaParser::MessageDone;Envoy::Http::Http1::ConnectionImpl::onMessageComplete;Envoy::Http::Http1::ConnectionImpl::onMessageCompleteImpl;Envoy::Http::Http1::ServerConnectionImpl::onMessageCompleteBase;Envoy::Http::ConnectionManagerImpl::ActiveStream::decodeHeaders;Envoy::Http::FilterManager::decodeHeaders;virtual thunk to Envoy::Extensions::Common::Wasm::Context::decodeHeaders;proxy_wasm::ContextBase::onRequestHeaders;proxy_wasm::wamr::Wamr::getModuleFunctionImpl;wasm_func_call;wasm_runtime_call_wasm;wasm_call_function;call_wasm_with_hw_bound_check;wasm_interp_call_wasm;llvm_jit_call_func_bytecode;wasm_runtime_invoke_native;push_args_end;aot_func_internal#3302;aot_func_internal#3308;asm_sysvec_apic_timer_interrupt;sysvec_apic_timer_interrupt;__sysvec_apic_timer_interrupt;hrtimer_interrupt;__hrtimer_run_queues;__remove_hrtimer;rb_next 1110899 - - symbol names are spearated by ";" - - if there is a symbol named like "aot_func#XXX" or "aot_func_internal#XXX", it will be replaced with the function name in name section by index - """ - folded_in = Path(folded_in) - assert folded_in.exists() - folded_out = Path(folded_out) - - import_function_count = import_section.get("function", 0) - with folded_in.open("rt", encoding="utf-8") as f_in, folded_out.open( - "wt", encoding="utf-8" - ) as f_out: - precheck_mode = False - for line in f_in: - line = line.strip() - if "aot_func_internal" in line: - precheck_mode = True - - f_in.seek(0) - for line in f_in: - new_line = [] - line = line.strip() - - m = re.match(r"(.*) (\d+)", line) - syms, samples = m.groups() - for sym in syms.split(";"): - m = re.match(r"aot_func(_internal)?#(\d+)", sym) - if not m: - new_line.append(sym) - continue - - func_idx = int(m.groups()[-1]) + import_function_count - if func_idx in name_section: - wasm_func_name = f"[Wasm] {name_section[func_idx]}" - else: - wasm_func_name = ( - f"[Wasm] function[{func_idx + import_function_count}]" - ) - - if precheck_mode: - # aot_func_internal -> xxx - # aot_func --> xxx_precheck - wasm_func_name += "_precheck" if not m.groups()[0] else "" - else: - # aot_func --> xxx - pass - - new_line.append(wasm_func_name) - - line = ";".join(new_line) - line += f" {samples}" - f_out.write(line + os.linesep) - - print(f"⚙️ {folded_in} -> {folded_out}") - - -def main(wabt_home: str, wasm_file: str, folded: str) -> None: - wabt_home = Path(wabt_home) - wasm_file = Path(wasm_file) - - wasm_objdump_bin = preflight_check(wabt_home) - import_section = collect_import_section_content(wasm_objdump_bin, wasm_file) - name_section = collect_name_section_content(wasm_objdump_bin, wasm_file) - - replace_function_name(import_section, name_section, folded, folded + ".translated") - - -if __name__ == "__main__": - argparse = argparse.ArgumentParser() - argparse.add_argument( - "--folded", help="stackcollapse-perf.pl generated, like out.folded" - ) - argparse.add_argument("wasm_file", help="wasm file") - argparse.add_argument("--wabt_home", help="wabt home, like /opt/wabt-1.0.33") - - args = argparse.parse_args() - main(args.wabt_home, args.wasm_file, args.folded) From b11dbcba0a7d1b1ed5ea8f99b84f97732fb2c1c4 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Mon, 8 Apr 2024 09:06:49 +0800 Subject: [PATCH 17/36] thread mgr: Free aux stack only when it was allocated (#3282) When thread manager is enabled, the aux stack of exec_env may be allocated by wasm_cluster_allocate_aux_stack or disabled by setting aux_stack_bottom as UINTPTR_MAX directly. For the latter, no need to free it. And fix an issue when paring `--gc-heap-size=n` argument for iwasm, and fix a variable shadowed warning in fast-jit. --- core/iwasm/common/wasm_exec_env.h | 3 +++ .../fast-jit/cg/x86-64/jit_codegen_x86_64.cpp | 6 +++--- core/iwasm/libraries/thread-mgr/thread_manager.c | 15 ++++++++++++--- product-mini/platforms/posix/main.c | 2 +- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/core/iwasm/common/wasm_exec_env.h b/core/iwasm/common/wasm_exec_env.h index f96242332..53d248755 100644 --- a/core/iwasm/common/wasm_exec_env.h +++ b/core/iwasm/common/wasm_exec_env.h @@ -117,6 +117,9 @@ typedef struct WASMExecEnv { /* whether current thread is detached */ bool thread_is_detached; + + /* whether the aux stack is allocated */ + bool is_aux_stack_allocated; #endif #if WASM_ENABLE_GC != 0 diff --git a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp index 53761e70a..79c72503e 100644 --- a/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp +++ b/core/iwasm/fast-jit/cg/x86-64/jit_codegen_x86_64.cpp @@ -9293,8 +9293,8 @@ jit_codegen_init() imm.setValue(INT32_MAX); a.jne(imm); - char *stream = (char *)a.code()->sectionById(0)->buffer().data() - + a.code()->sectionById(0)->buffer().size(); + char *stream_old = (char *)a.code()->sectionById(0)->buffer().data() + + a.code()->sectionById(0)->buffer().size(); /* If yes, call jit_set_exception_with_id to throw exception, and then set eax to JIT_INTERP_ACTION_THROWN, and jump to @@ -9319,7 +9319,7 @@ jit_codegen_init() /* Patch the offset of jne instruction */ char *stream_new = (char *)a.code()->sectionById(0)->buffer().data() + a.code()->sectionById(0)->buffer().size(); - *(int32 *)(stream - 4) = (int32)(stream_new - stream); + *(int32 *)(stream_old - 4) = (int32)(stream_new - stream_old); } /* Load compiled func ptr and call it */ diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index ac8957501..ac4deb92c 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -558,6 +558,7 @@ wasm_cluster_spawn_exec_env(WASMExecEnv *exec_env) aux_stack_size)) { goto fail3; } + new_exec_env->is_aux_stack_allocated = true; /* Inherit suspend_flags of parent thread */ new_exec_env->suspend_flags.flags = @@ -603,7 +604,9 @@ wasm_cluster_destroy_spawned_exec_env(WASMExecEnv *exec_env) exec_env_tls = exec_env; } - /* Free aux stack space */ + /* Free aux stack space which was allocated in + wasm_cluster_spawn_exec_env */ + bh_assert(exec_env_tls->is_aux_stack_allocated); wasm_cluster_free_aux_stack(exec_env_tls, (uint64)exec_env->aux_stack_bottom); @@ -655,7 +658,9 @@ thread_manager_start_routine(void *arg) #endif /* Free aux stack space */ - wasm_cluster_free_aux_stack(exec_env, (uint64)exec_env->aux_stack_bottom); + if (exec_env->is_aux_stack_allocated) + wasm_cluster_free_aux_stack(exec_env, + (uint64)exec_env->aux_stack_bottom); os_mutex_lock(&cluster_list_lock); @@ -723,11 +728,13 @@ wasm_cluster_create_thread(WASMExecEnv *exec_env, aux_stack_size)) { goto fail2; } + new_exec_env->is_aux_stack_allocated = true; } else { /* Disable aux stack */ new_exec_env->aux_stack_boundary = 0; new_exec_env->aux_stack_bottom = UINTPTR_MAX; + new_exec_env->is_aux_stack_allocated = false; } /* Inherit suspend_flags of parent thread */ @@ -1049,7 +1056,9 @@ wasm_cluster_exit_thread(WASMExecEnv *exec_env, void *retval) #endif /* Free aux stack space */ - wasm_cluster_free_aux_stack(exec_env, (uint64)exec_env->aux_stack_bottom); + if (exec_env->is_aux_stack_allocated) + wasm_cluster_free_aux_stack(exec_env, + (uint64)exec_env->aux_stack_bottom); /* App exit the thread, free the resources before exit native thread */ diff --git a/product-mini/platforms/posix/main.c b/product-mini/platforms/posix/main.c index 217eb20cb..2acd2190a 100644 --- a/product-mini/platforms/posix/main.c +++ b/product-mini/platforms/posix/main.c @@ -675,7 +675,7 @@ main(int argc, char *argv[]) #endif #if WASM_ENABLE_GC != 0 else if (!strncmp(argv[0], "--gc-heap-size=", 15)) { - if (argv[0][21] == '\0') + if (argv[0][15] == '\0') return print_help(); gc_heap_size = atoi(argv[0] + 15); } From ef3babc658d211379e877ec27254a7c27cc4bf18 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Mon, 8 Apr 2024 09:23:20 +0800 Subject: [PATCH 18/36] interp: Restore context from prev_frame after tail calling a native function (#3283) The current frame was freed before tail calling to an import or native function and the prev_frame was set as exec_env's cur_frame, so after the tail calling, we should recover context from prev_frame but not current frame. Found in https://github.com/bytecodealliance/wasm-micro-runtime/issues/3279. --- core/iwasm/interpreter/wasm_interp_classic.c | 46 +++++++++++++++++--- core/iwasm/interpreter/wasm_interp_fast.c | 27 ++++++++++-- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index ca972fd4b..3aec3f4c4 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -1509,6 +1509,9 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMStringviewIterObjectRef stringview_iter_obj; #endif #endif +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + bool is_return_call = false; +#endif #if WASM_ENABLE_MEMORY64 != 0 /* TODO: multi-memories for now assuming the memory idx type is consistent * across multi-memories */ @@ -6227,6 +6230,9 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, frame_ip = frame->ip; frame_sp = frame->sp; frame_csp = frame->csp; +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + is_return_call = false; +#endif goto call_func_from_entry; } @@ -6320,6 +6326,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } FREE_FRAME(exec_env, frame); wasm_exec_env_set_cur_frame(exec_env, prev_frame); + is_return_call = true; goto call_func_from_entry; } #endif @@ -6333,6 +6340,9 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } SYNC_ALL_TO_FRAME(); prev_frame = frame; +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + is_return_call = false; +#endif } call_func_from_entry: @@ -6342,15 +6352,27 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (cur_func->import_func_inst) { wasm_interp_call_func_import(module, exec_env, cur_func, prev_frame); +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + if (is_return_call) { + /* the frame was freed before tail calling and + the prev_frame was set as exec_env's cur_frame, + so here we recover context from prev_frame */ + RECOVER_CONTEXT(prev_frame); + } + else +#endif + { + prev_frame = frame->prev_frame; + cur_func = frame->function; + UPDATE_ALL_FROM_FRAME(); + } + #if WASM_ENABLE_EXCE_HANDLING != 0 char uncaught_exception[128] = { 0 }; bool has_exception = wasm_copy_exception(module, uncaught_exception); if (has_exception && strstr(uncaught_exception, "uncaught wasm exception")) { - /* fix framesp */ - UPDATE_ALL_FROM_FRAME(); - uint32 import_exception; /* initialize imported exception index to be invalid */ SET_INVALID_TAGINDEX(import_exception); @@ -6392,12 +6414,22 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, { wasm_interp_call_func_native(module, exec_env, cur_func, prev_frame); +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + if (is_return_call) { + /* the frame was freed before tail calling and + the prev_frame was set as exec_env's cur_frame, + so here we recover context from prev_frame */ + RECOVER_CONTEXT(prev_frame); + } + else +#endif + { + prev_frame = frame->prev_frame; + cur_func = frame->function; + UPDATE_ALL_FROM_FRAME(); + } } - prev_frame = frame->prev_frame; - cur_func = frame->function; - UPDATE_ALL_FROM_FRAME(); - /* update memory size, no need to update memory ptr as it isn't changed in wasm_enlarge_memory */ #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 004371163..c7cb70260 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -1501,6 +1501,9 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMStringviewIterObjectRef stringview_iter_obj; #endif #endif +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + bool is_return_call = false; +#endif #if WASM_ENABLE_LABELS_AS_VALUES != 0 #define HANDLE_OPCODE(op) &&HANDLE_##op @@ -5618,6 +5621,9 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, { frame = prev_frame; frame_ip = frame->ip; +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + is_return_call = false; +#endif goto call_func_from_entry; } @@ -5766,6 +5772,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, FREE_FRAME(exec_env, frame); frame_ip += cur_func->param_count * sizeof(int16); wasm_exec_env_set_cur_frame(exec_env, (WASMRuntimeFrame *)prev_frame); + is_return_call = true; goto call_func_from_entry; } #endif /* WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 */ @@ -5838,6 +5845,9 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } SYNC_ALL_TO_FRAME(); prev_frame = frame; +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + is_return_call = false; +#endif } call_func_from_entry: @@ -5855,9 +5865,20 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, prev_frame); } - prev_frame = frame->prev_frame; - cur_func = frame->function; - UPDATE_ALL_FROM_FRAME(); +#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 + if (is_return_call) { + /* the frame was freed before tail calling and + the prev_frame was set as exec_env's cur_frame, + so here we recover context from prev_frame */ + RECOVER_CONTEXT(prev_frame); + } + else +#endif + { + prev_frame = frame->prev_frame; + cur_func = frame->function; + UPDATE_ALL_FROM_FRAME(); + } /* update memory size, no need to update memory ptr as it isn't changed in wasm_enlarge_memory */ From dacb3c4105adcd3d7db90516dc2ddf7118a8d394 Mon Sep 17 00:00:00 2001 From: dongheng <930490596@qq.com> Date: Mon, 8 Apr 2024 12:34:08 +0800 Subject: [PATCH 19/36] Add wamr to esp-idf components registry (#3287) This PR is for the main branch, but only the released branch will be pushed into Espressif component registry. See also similar fixes in branch release/1.3.x: https://github.com/bytecodealliance/wasm-micro-runtime/pull/3264 https://github.com/bytecodealliance/wasm-micro-runtime/pull/3288 --- CMakeLists.txt | 5 +++++ core/shared/platform/esp-idf/espidf_memmap.c | 19 ++++++++++++++++++- idf_component.yml | 8 ++++++++ product-mini/platforms/esp-idf/CMakeLists.txt | 3 --- .../platforms/esp-idf/main/CMakeLists.txt | 3 +-- .../platforms/esp-idf/main/idf_component.yml | 7 +++++++ product-mini/platforms/esp-idf/main/main.c | 4 ---- 7 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 idf_component.yml create mode 100644 product-mini/platforms/esp-idf/main/idf_component.yml diff --git a/CMakeLists.txt b/CMakeLists.txt index 8df86ddd0..0ffba05a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,11 @@ cmake_minimum_required (VERSION 3.0) +if(ESP_PLATFORM) + include (${COMPONENT_DIR}/build-scripts/esp-idf/wamr/CMakeLists.txt) + return() +endif() + project (iwasm) set (CMAKE_VERBOSE_MAKEFILE OFF) diff --git a/core/shared/platform/esp-idf/espidf_memmap.c b/core/shared/platform/esp-idf/espidf_memmap.c index 6b1b6f045..21f186b90 100644 --- a/core/shared/platform/esp-idf/espidf_memmap.c +++ b/core/shared/platform/esp-idf/espidf_memmap.c @@ -55,7 +55,24 @@ os_mmap(void *hint, size_t size, int prot, int flags, os_file_handle file) #else uint32_t mem_caps = MALLOC_CAP_8BIT; #endif - return heap_caps_malloc(size, mem_caps); + void *buf_origin = + heap_caps_malloc(size + 4 + sizeof(uintptr_t), mem_caps); + if (!buf_origin) { + return NULL; + } + + // Memory allocation with MALLOC_CAP_SPIRAM or MALLOC_CAP_8BIT will + // return 4-byte aligned Reserve extra 4 byte to fixup alignment and + // size for the pointer to the originally allocated address + void *buf_fixed = buf_origin + sizeof(void *); + if ((uintptr_t)buf_fixed & (uintptr_t)0x7) { + buf_fixed = (void *)((uintptr_t)(buf_fixed + 4) & (~(uintptr_t)7)); + } + + uintptr_t *addr_field = buf_fixed - sizeof(uintptr_t); + *addr_field = (uintptr_t)buf_origin; + + return buf_fixed; } } diff --git a/idf_component.yml b/idf_component.yml new file mode 100644 index 000000000..a35cf79b3 --- /dev/null +++ b/idf_component.yml @@ -0,0 +1,8 @@ +version: "2.0.0" +description: WebAssembly Micro Runtime - A lightweight standalone WebAssembly (Wasm) runtime with small footprint, high performance and highly configurable features +url: https://bytecodealliance.org/ +repository: https://github.com/bytecodealliance/wasm-micro-runtime.git +documentation: https://wamr.gitbook.io/ +issues: https://github.com/bytecodealliance/wasm-micro-runtime/issues +dependencies: + idf: ">=4.4" \ No newline at end of file diff --git a/product-mini/platforms/esp-idf/CMakeLists.txt b/product-mini/platforms/esp-idf/CMakeLists.txt index d8a3d2f96..8472df8dd 100644 --- a/product-mini/platforms/esp-idf/CMakeLists.txt +++ b/product-mini/platforms/esp-idf/CMakeLists.txt @@ -6,7 +6,4 @@ cmake_minimum_required(VERSION 3.5) include($ENV{IDF_PATH}/tools/cmake/project.cmake) -set (COMPONENTS ${IDF_TARGET} main freertos esptool_py wamr) -list(APPEND EXTRA_COMPONENT_DIRS "$ENV{WAMR_PATH}/build-scripts/esp-idf") - project(wamr-simple) \ No newline at end of file diff --git a/product-mini/platforms/esp-idf/main/CMakeLists.txt b/product-mini/platforms/esp-idf/main/CMakeLists.txt index 55e725670..1bb61bad9 100644 --- a/product-mini/platforms/esp-idf/main/CMakeLists.txt +++ b/product-mini/platforms/esp-idf/main/CMakeLists.txt @@ -2,5 +2,4 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception idf_component_register(SRCS "main.c" - INCLUDE_DIRS "." - REQUIRES wamr) + INCLUDE_DIRS ".") diff --git a/product-mini/platforms/esp-idf/main/idf_component.yml b/product-mini/platforms/esp-idf/main/idf_component.yml new file mode 100644 index 000000000..da64aa37d --- /dev/null +++ b/product-mini/platforms/esp-idf/main/idf_component.yml @@ -0,0 +1,7 @@ +## IDF Component Manager Manifest File +dependencies: + wasm-micro-runtime: + version: ">=2.0" + override_path: "../../../.." + idf: + version: ">=4.4" \ No newline at end of file diff --git a/product-mini/platforms/esp-idf/main/main.c b/product-mini/platforms/esp-idf/main/main.c index fbfb04c21..1a34096d7 100644 --- a/product-mini/platforms/esp-idf/main/main.c +++ b/product-mini/platforms/esp-idf/main/main.c @@ -12,11 +12,7 @@ #include "esp_log.h" -#ifdef CONFIG_IDF_TARGET_ESP32S3 #define IWASM_MAIN_STACK_SIZE 5120 -#else -#define IWASM_MAIN_STACK_SIZE 4096 -#endif #define LOG_TAG "wamr" From 4a29794a1bf6751d9db726cb830c4b6de3a78127 Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Tue, 9 Apr 2024 16:51:36 +0800 Subject: [PATCH 20/36] Update release CI (#3295) In the release CI and related scripts, when comparing and printing to the CI, use the most recent **ancestor** tag(s) for the release branch rather than the most recent one(s). And fix the build_wamr_sdk.yml and build_wamr_lldb.yml CIs. --- .github/scripts/fetch_and_compare_version.py | 9 ++++++--- .github/workflows/build_wamr_lldb.yml | 4 +--- .github/workflows/build_wamr_sdk.yml | 6 ++++++ .github/workflows/create_tag.yml | 18 ++++++++++++++++-- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/.github/scripts/fetch_and_compare_version.py b/.github/scripts/fetch_and_compare_version.py index ac206cade..ad9e53a0a 100644 --- a/.github/scripts/fetch_and_compare_version.py +++ b/.github/scripts/fetch_and_compare_version.py @@ -42,9 +42,12 @@ def fetch_version_from_code(): def fetch_latest_git_tag(): - list_tag_cmd = ( - 'git tag --list WAMR-*.*.* --sort=committerdate --format="%(refname:short)"' - ) + """ + Get the most recent tag from the HEAD, + if it's main branch, it should be the latest release tag. + if it's release/x.x.x branch, it should be the latest release tag of the branch. + """ + list_tag_cmd = "git describe --tags --abbrev=0 HEAD" p = subprocess.run(shlex.split(list_tag_cmd), capture_output=True, check=True) all_tags = p.stdout.decode().strip() diff --git a/.github/workflows/build_wamr_lldb.yml b/.github/workflows/build_wamr_lldb.yml index 3e1e10ffd..03474c53e 100644 --- a/.github/workflows/build_wamr_lldb.yml +++ b/.github/workflows/build_wamr_lldb.yml @@ -82,9 +82,7 @@ jobs: - name: install utils macos if: steps.lldb_build_cache.outputs.cache-hit != 'true' && contains(inputs.runner, 'macos') run: | - brew remove swig - brew install swig@4.1 cmake ninja libedit - brew link --overwrite swig@4.1 + brew install swig cmake ninja libedit sudo rm -rf /Library/Developer/CommandLineTools - name: install utils ubuntu diff --git a/.github/workflows/build_wamr_sdk.yml b/.github/workflows/build_wamr_sdk.yml index 69dbd7232..519bf9636 100644 --- a/.github/workflows/build_wamr_sdk.yml +++ b/.github/workflows/build_wamr_sdk.yml @@ -58,6 +58,12 @@ jobs: sudo rm ${basename} sudo mv wasi-sdk-* wasi-sdk + - name: download dependencies + run: | + cd ./wamr-app-framework/deps + ./download.sh + working-directory: wamr-sdk + - name: generate wamr-sdk release run: | cd ./wamr-app-framework/wamr-sdk diff --git a/.github/workflows/create_tag.yml b/.github/workflows/create_tag.yml index 27eee2acf..5480592a9 100644 --- a/.github/workflows/create_tag.yml +++ b/.github/workflows/create_tag.yml @@ -32,8 +32,22 @@ jobs: - name: prepare id: preparation run: | - # show latest 3 versions - git tag --list WAMR-*.*.* --sort=committerdate --format="%(refname:short)" | tail -n 3 + # show latest 3 versions on the branch that create release + # Set the initial commit to the head of the branch + commit="HEAD" + # + # Loop to get the three most recent tags + for i in {1..3} + do + # Get the most recent tag reachable from the current commit + tag=$(git describe --tags --abbrev=0 $commit) + + # Print the tag + echo "$tag" + + # Move to the commit before the found tag to find the next tag in the next iteration + commit=$(git rev-list -n 1 $tag^) + done # compare latest git tag and semantic version definition result=$(python3 ./.github/scripts/fetch_and_compare_version.py) echo "script result is ${result}" From bcc2a2d2e1465a0585ae5b59e096a6a81b16c3d4 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Tue, 9 Apr 2024 16:54:42 +0800 Subject: [PATCH 21/36] Sync simd opcode definitions spec (#3290) Remove undefined simd opcodes. --- core/iwasm/compilation/aot_compiler.c | 51 ------------------- .../iwasm/compilation/simd/simd_conversions.c | 9 ---- .../iwasm/compilation/simd/simd_conversions.h | 4 -- .../compilation/simd/simd_floating_point.c | 14 ----- .../compilation/simd/simd_floating_point.h | 8 --- core/iwasm/compilation/simd/simd_int_arith.c | 11 +--- core/iwasm/compilation/simd/simd_int_arith.h | 4 -- .../compilation/simd/simd_sat_int_arith.c | 15 ------ .../compilation/simd/simd_sat_int_arith.h | 4 -- core/iwasm/interpreter/wasm_loader.c | 10 ---- core/iwasm/interpreter/wasm_opcode.h | 20 ++++---- 11 files changed, 11 insertions(+), 139 deletions(-) diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index ef3931b34..5c257742a 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -3452,16 +3452,6 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) break; } - case SIMD_i32x4_narrow_i64x2_s: - case SIMD_i32x4_narrow_i64x2_u: - { - if (!aot_compile_simd_i32x4_narrow_i64x2( - comp_ctx, func_ctx, - SIMD_i32x4_narrow_i64x2_s == opcode)) - return false; - break; - } - case SIMD_i32x4_extend_low_i16x8_s: case SIMD_i32x4_extend_high_i16x8_s: { @@ -3501,16 +3491,6 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) break; } - case SIMD_i32x4_add_sat_s: - case SIMD_i32x4_add_sat_u: - { - if (!aot_compile_simd_i32x4_saturate( - comp_ctx, func_ctx, V128_ADD, - opcode == SIMD_i32x4_add_sat_s)) - return false; - break; - } - case SIMD_i32x4_sub: { if (!aot_compile_simd_i32x4_arith(comp_ctx, func_ctx, @@ -3519,16 +3499,6 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) break; } - case SIMD_i32x4_sub_sat_s: - case SIMD_i32x4_sub_sat_u: - { - if (!aot_compile_simd_i32x4_saturate( - comp_ctx, func_ctx, V128_SUB, - opcode == SIMD_i32x4_add_sat_s)) - return false; - break; - } - case SIMD_i32x4_mul: { if (!aot_compile_simd_i32x4_arith(comp_ctx, func_ctx, @@ -3565,13 +3535,6 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) break; } - case SIMD_i32x4_avgr_u: - { - if (!aot_compile_simd_i32x4_avgr_u(comp_ctx, func_ctx)) - return false; - break; - } - case SIMD_i32x4_extmul_low_i16x8_s: case SIMD_i32x4_extmul_high_i16x8_s: { @@ -3728,13 +3691,6 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) break; } - case SIMD_f32x4_round: - { - if (!aot_compile_simd_f32x4_round(comp_ctx, func_ctx)) - return false; - break; - } - case SIMD_f32x4_sqrt: { if (!aot_compile_simd_f32x4_sqrt(comp_ctx, func_ctx)) @@ -3788,13 +3744,6 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) break; } - case SIMD_f64x2_round: - { - if (!aot_compile_simd_f64x2_round(comp_ctx, func_ctx)) - return false; - break; - } - case SIMD_f64x2_sqrt: { if (!aot_compile_simd_f64x2_sqrt(comp_ctx, func_ctx)) diff --git a/core/iwasm/compilation/simd/simd_conversions.c b/core/iwasm/compilation/simd/simd_conversions.c index 8e4c17ed3..042e28089 100644 --- a/core/iwasm/compilation/simd/simd_conversions.c +++ b/core/iwasm/compilation/simd/simd_conversions.c @@ -226,15 +226,6 @@ aot_compile_simd_i16x8_narrow_i32x4(AOTCompContext *comp_ctx, } } -bool -aot_compile_simd_i32x4_narrow_i64x2(AOTCompContext *comp_ctx, - AOTFuncContext *func_ctx, bool is_signed) -{ - /* TODO: x86 intrinsics */ - return simd_integer_narrow_common(comp_ctx, func_ctx, e_sat_i64x2, - is_signed); -} - enum integer_extend_type { e_ext_i8x16, e_ext_i16x8, diff --git a/core/iwasm/compilation/simd/simd_conversions.h b/core/iwasm/compilation/simd/simd_conversions.h index 87b8bd684..e3a1a3521 100644 --- a/core/iwasm/compilation/simd/simd_conversions.h +++ b/core/iwasm/compilation/simd/simd_conversions.h @@ -20,10 +20,6 @@ bool aot_compile_simd_i16x8_narrow_i32x4(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, bool is_signed); -bool -aot_compile_simd_i32x4_narrow_i64x2(AOTCompContext *comp_ctx, - AOTFuncContext *func_ctx, bool is_signed); - bool aot_compile_simd_i16x8_extend_i8x16(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, bool is_low, diff --git a/core/iwasm/compilation/simd/simd_floating_point.c b/core/iwasm/compilation/simd/simd_floating_point.c index 7fcc1ab65..536ef5b28 100644 --- a/core/iwasm/compilation/simd/simd_floating_point.c +++ b/core/iwasm/compilation/simd/simd_floating_point.c @@ -129,20 +129,6 @@ aot_compile_simd_f64x2_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) "llvm.fabs.v2f64"); } -bool -aot_compile_simd_f32x4_round(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) -{ - return simd_float_intrinsic(comp_ctx, func_ctx, V128_f32x4_TYPE, - "llvm.round.v4f32"); -} - -bool -aot_compile_simd_f64x2_round(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) -{ - return simd_float_intrinsic(comp_ctx, func_ctx, V128_f64x2_TYPE, - "llvm.round.v2f64"); -} - bool aot_compile_simd_f32x4_sqrt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { diff --git a/core/iwasm/compilation/simd/simd_floating_point.h b/core/iwasm/compilation/simd/simd_floating_point.h index 213b4391f..39e37c872 100644 --- a/core/iwasm/compilation/simd/simd_floating_point.h +++ b/core/iwasm/compilation/simd/simd_floating_point.h @@ -32,14 +32,6 @@ aot_compile_simd_f32x4_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); bool aot_compile_simd_f64x2_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); -bool -aot_compile_simd_f32x4_round(AOTCompContext *comp_ctx, - AOTFuncContext *func_ctx); - -bool -aot_compile_simd_f64x2_round(AOTCompContext *comp_ctx, - AOTFuncContext *func_ctx); - bool aot_compile_simd_f32x4_sqrt(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); diff --git a/core/iwasm/compilation/simd/simd_int_arith.c b/core/iwasm/compilation/simd/simd_int_arith.c index 1d0e6967b..6a1902d1f 100644 --- a/core/iwasm/compilation/simd/simd_int_arith.c +++ b/core/iwasm/compilation/simd/simd_int_arith.c @@ -243,7 +243,6 @@ aot_compile_simd_i64x2_abs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) enum integer_avgr_u { e_avgr_u_i8x16, e_avgr_u_i16x8, - e_avgr_u_i32x4, }; /* TODO: try int_x86_mmx_pavg_b and int_x86_mmx_pavg_w */ @@ -257,9 +256,8 @@ simd_v128_avg(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMTypeRef vector_type[] = { V128_i8x16_TYPE, V128_i16x8_TYPE, - V128_i32x4_TYPE, }; - unsigned lanes[] = { 16, 8, 4 }; + unsigned lanes[] = { 16, 8 }; if (!(rhs = simd_pop_v128_and_bitcast(comp_ctx, func_ctx, vector_type[itype], "rhs")) @@ -325,13 +323,6 @@ aot_compile_simd_i16x8_avgr_u(AOTCompContext *comp_ctx, return simd_v128_avg(comp_ctx, func_ctx, e_avgr_u_i16x8); } -bool -aot_compile_simd_i32x4_avgr_u(AOTCompContext *comp_ctx, - AOTFuncContext *func_ctx) -{ - return simd_v128_avg(comp_ctx, func_ctx, e_avgr_u_i32x4); -} - bool aot_compile_simd_i32x4_dot_i16x8(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) diff --git a/core/iwasm/compilation/simd/simd_int_arith.h b/core/iwasm/compilation/simd/simd_int_arith.h index a7a21170a..49827d51d 100644 --- a/core/iwasm/compilation/simd/simd_int_arith.h +++ b/core/iwasm/compilation/simd/simd_int_arith.h @@ -76,10 +76,6 @@ bool aot_compile_simd_i16x8_avgr_u(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); -bool -aot_compile_simd_i32x4_avgr_u(AOTCompContext *comp_ctx, - AOTFuncContext *func_ctx); - bool aot_compile_simd_i32x4_dot_i16x8(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); diff --git a/core/iwasm/compilation/simd/simd_sat_int_arith.c b/core/iwasm/compilation/simd/simd_sat_int_arith.c index 1de4520a7..ea250b7e0 100644 --- a/core/iwasm/compilation/simd/simd_sat_int_arith.c +++ b/core/iwasm/compilation/simd/simd_sat_int_arith.c @@ -64,18 +64,3 @@ aot_compile_simd_i16x8_saturate(AOTCompContext *comp_ctx, is_signed ? intrinsics[arith_op][0] : intrinsics[arith_op][1]); } - -bool -aot_compile_simd_i32x4_saturate(AOTCompContext *comp_ctx, - AOTFuncContext *func_ctx, - V128Arithmetic arith_op, bool is_signed) -{ - char *intrinsics[][2] = { - { "llvm.sadd.sat.v4i32", "llvm.uadd.sat.v4i32" }, - { "llvm.ssub.sat.v4i32", "llvm.usub.sat.v4i32" }, - }; - - return simd_sat_int_arith(comp_ctx, func_ctx, V128_i16x8_TYPE, - is_signed ? intrinsics[arith_op][0] - : intrinsics[arith_op][1]); -} diff --git a/core/iwasm/compilation/simd/simd_sat_int_arith.h b/core/iwasm/compilation/simd/simd_sat_int_arith.h index e30acaaf4..67c602fc5 100644 --- a/core/iwasm/compilation/simd/simd_sat_int_arith.h +++ b/core/iwasm/compilation/simd/simd_sat_int_arith.h @@ -22,10 +22,6 @@ aot_compile_simd_i16x8_saturate(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, V128Arithmetic arith_op, bool is_signed); -bool -aot_compile_simd_i32x4_saturate(AOTCompContext *comp_ctx, - AOTFuncContext *func_ctx, - V128Arithmetic arith_op, bool is_signed); #ifdef __cplusplus } /* end of extern "C" */ #endif diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index a07ce5866..a292663df 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -15129,13 +15129,6 @@ re_scan: break; } - case SIMD_i32x4_narrow_i64x2_s: - case SIMD_i32x4_narrow_i64x2_u: - { - POP2_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); - break; - } - case SIMD_i32x4_extend_low_i16x8_s: case SIMD_i32x4_extend_high_i16x8_s: case SIMD_i32x4_extend_low_i16x8_u: @@ -15162,7 +15155,6 @@ re_scan: case SIMD_i32x4_max_s: case SIMD_i32x4_max_u: case SIMD_i32x4_dot_i16x8_s: - case SIMD_i32x4_avgr_u: case SIMD_i32x4_extmul_low_i16x8_s: case SIMD_i32x4_extmul_high_i16x8_s: case SIMD_i32x4_extmul_low_i16x8_u: @@ -15226,7 +15218,6 @@ re_scan: /* f32x4 operation */ case SIMD_f32x4_abs: case SIMD_f32x4_neg: - case SIMD_f32x4_round: case SIMD_f32x4_sqrt: { POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); @@ -15249,7 +15240,6 @@ re_scan: /* f64x2 operation */ case SIMD_f64x2_abs: case SIMD_f64x2_neg: - case SIMD_f64x2_round: case SIMD_f64x2_sqrt: { POP_AND_PUSH(VALUE_TYPE_V128, VALUE_TYPE_V128); diff --git a/core/iwasm/interpreter/wasm_opcode.h b/core/iwasm/interpreter/wasm_opcode.h index 98e5b1325..db5e5e40b 100644 --- a/core/iwasm/interpreter/wasm_opcode.h +++ b/core/iwasm/interpreter/wasm_opcode.h @@ -593,8 +593,8 @@ typedef enum WASMSimdEXTOpcode { /* placeholder = 0xa2 */ SIMD_i32x4_all_true = 0xa3, SIMD_i32x4_bitmask = 0xa4, - SIMD_i32x4_narrow_i64x2_s = 0xa5, - SIMD_i32x4_narrow_i64x2_u = 0xa6, + /* placeholder = 0xa5 */ + /* placeholder = 0xa6 */ SIMD_i32x4_extend_low_i16x8_s = 0xa7, SIMD_i32x4_extend_high_i16x8_s = 0xa8, SIMD_i32x4_extend_low_i16x8_u = 0xa9, @@ -603,19 +603,19 @@ typedef enum WASMSimdEXTOpcode { SIMD_i32x4_shr_s = 0xac, SIMD_i32x4_shr_u = 0xad, SIMD_i32x4_add = 0xae, - SIMD_i32x4_add_sat_s = 0xaf, - SIMD_i32x4_add_sat_u = 0xb0, + /* placeholder = 0xaf */ + /* placeholder = 0xb0 */ SIMD_i32x4_sub = 0xb1, - SIMD_i32x4_sub_sat_s = 0xb2, - SIMD_i32x4_sub_sat_u = 0xb3, - /* placeholder = 0xb4 */ + /* placeholder = 0xb2 */ + /* placeholder = 0xb3 */ + /* placeholder = 0xb4 */ SIMD_i32x4_mul = 0xb5, SIMD_i32x4_min_s = 0xb6, SIMD_i32x4_min_u = 0xb7, SIMD_i32x4_max_s = 0xb8, SIMD_i32x4_max_u = 0xb9, SIMD_i32x4_dot_i16x8_s = 0xba, - SIMD_i32x4_avgr_u = 0xbb, + /* placeholder = 0xbb */ SIMD_i32x4_extmul_low_i16x8_s = 0xbc, SIMD_i32x4_extmul_high_i16x8_s = 0xbd, SIMD_i32x4_extmul_low_i16x8_u = 0xbe, @@ -658,7 +658,7 @@ typedef enum WASMSimdEXTOpcode { /* f32x4 operation */ SIMD_f32x4_abs = 0xe0, SIMD_f32x4_neg = 0xe1, - SIMD_f32x4_round = 0xe2, + /* placeholder = 0xe2 */ SIMD_f32x4_sqrt = 0xe3, SIMD_f32x4_add = 0xe4, SIMD_f32x4_sub = 0xe5, @@ -672,7 +672,7 @@ typedef enum WASMSimdEXTOpcode { /* f64x2 operation */ SIMD_f64x2_abs = 0xec, SIMD_f64x2_neg = 0xed, - SIMD_f64x2_round = 0xee, + /* placeholder = 0xee */ SIMD_f64x2_sqrt = 0xef, SIMD_f64x2_add = 0xf0, SIMD_f64x2_sub = 0xf1, From 8756d29e190e377972013a7a9d43551a2218bea4 Mon Sep 17 00:00:00 2001 From: mkolchurin <33422855+mkolchurin@users.noreply.github.com> Date: Wed, 10 Apr 2024 03:19:46 +0300 Subject: [PATCH 22/36] zephyr: Add missing pthread library functions (#3291) For use with WAMR_BUILD_LIB_PTHREAD, add os_thread_detach, os_thread_exit, os_cond_broadcast. Signed-off-by: Maxim Kolchurin --- core/shared/platform/zephyr/zephyr_thread.c | 32 ++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/core/shared/platform/zephyr/zephyr_thread.c b/core/shared/platform/zephyr/zephyr_thread.c index 105d53993..53ca71f62 100644 --- a/core/shared/platform/zephyr/zephyr_thread.c +++ b/core/shared/platform/zephyr/zephyr_thread.c @@ -577,4 +577,34 @@ os_thread_get_stack_boundary() void os_thread_jit_write_protect_np(bool enabled) -{} \ No newline at end of file +{} + +int +os_thread_detach(korp_tid thread) +{ + (void)thread; + return BHT_OK; +} + +void +os_thread_exit(void *retval) +{ + (void)retval; + os_thread_cleanup(); + k_thread_abort(k_current_get()); +} + +int +os_cond_broadcast(korp_cond *cond) +{ + os_thread_wait_node *node; + k_mutex_lock(&cond->wait_list_lock, K_FOREVER); + node = cond->thread_wait_list; + while (node) { + os_thread_wait_node *next = node->next; + k_sem_give(&node->sem); + node = next; + } + k_mutex_unlock(&cond->wait_list_lock); + return BHT_OK; +} From 4e634bed3f979f8cea6444cf771701a660ca1112 Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Thu, 11 Apr 2024 07:50:08 +0800 Subject: [PATCH 23/36] Add necessary comments for doxygen (#3299) - Add necessary comments for doxygen to generate API documentation - Update README.md to add a link for End-user APIs documentation --- README.md | 1 + core/iwasm/include/aot_export.h | 6 ++++++ core/iwasm/include/gc_export.h | 6 ++++++ core/iwasm/include/lib_export.h | 5 +++++ core/iwasm/include/wasm_c_api.h | 6 ++++++ core/iwasm/include/wasm_export.h | 6 ++++++ 6 files changed, 30 insertions(+) diff --git a/README.md b/README.md index cb91c22ce..5a53536a5 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ The following platforms are supported, click each link below for how to build iw - [Port WAMR to a new platform](./doc/port_wamr.md) - [VS Code development container](./doc/devcontainer.md) - [Samples](./samples) and [Benchmarks](./tests/benchmarks) +- [End-user APIs documentation](https://bytecodealliance.github.io/wamr.dev/apis/) diff --git a/core/iwasm/include/aot_export.h b/core/iwasm/include/aot_export.h index c1a03d86c..d06fef1dd 100644 --- a/core/iwasm/include/aot_export.h +++ b/core/iwasm/include/aot_export.h @@ -3,6 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ +/** + * @file aot_export.h + * + * @brief This file defines the exported AOT compilation APIs + */ + #ifndef _AOT_EXPORT_H #define _AOT_EXPORT_H diff --git a/core/iwasm/include/gc_export.h b/core/iwasm/include/gc_export.h index 3eb88dbab..777551edc 100644 --- a/core/iwasm/include/gc_export.h +++ b/core/iwasm/include/gc_export.h @@ -3,6 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ +/** + * @file gc_export.h + * + * @brief This file defines the exported GC APIs + */ + #ifndef _GC_EXPORT_H #define _GC_EXPORT_H diff --git a/core/iwasm/include/lib_export.h b/core/iwasm/include/lib_export.h index e4829e4fe..0ca668f52 100644 --- a/core/iwasm/include/lib_export.h +++ b/core/iwasm/include/lib_export.h @@ -3,6 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ +/** + * @file lib_export.h + * + */ + #ifndef _LIB_EXPORT_H_ #define _LIB_EXPORT_H_ diff --git a/core/iwasm/include/wasm_c_api.h b/core/iwasm/include/wasm_c_api.h index 0d62c2751..63d18f3ae 100644 --- a/core/iwasm/include/wasm_c_api.h +++ b/core/iwasm/include/wasm_c_api.h @@ -1,5 +1,11 @@ // WebAssembly C API +/** + * @file wasm_c_api.h + * + * @brief This file defines the WebAssembly C APIs + */ + #ifndef _WASM_C_API_H_ #define _WASM_C_API_H_ diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index e40e94885..bc43ea0b9 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -3,6 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ +/** + * @file wasm_export.h + * + * @brief This file defines the exported common runtime APIs + */ + #ifndef _WASM_EXPORT_H #define _WASM_EXPORT_H From 9b28a8a80ebd42295164fc9ef083092f4af1381b Mon Sep 17 00:00:00 2001 From: dongheng <930490596@qq.com> Date: Thu, 11 Apr 2024 19:46:34 +0800 Subject: [PATCH 24/36] Update esp-idf platform support in main (#3304) 1. Fix API "futimens" and "utimensat" compiling error in different esp-idf version 2. Update component registry description file ps. refer to PR #3296 on branch release/1.3x --- .../shared/platform/esp-idf/espidf_platform.c | 22 +++++++++++++++++-- idf_component.yml | 9 +++++++- .../platforms/esp-idf/main/idf_component.yml | 2 +- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/core/shared/platform/esp-idf/espidf_platform.c b/core/shared/platform/esp-idf/espidf_platform.c index 9c0d02e62..8fea32546 100644 --- a/core/shared/platform/esp-idf/espidf_platform.c +++ b/core/shared/platform/esp-idf/espidf_platform.c @@ -6,6 +6,12 @@ #include "platform_api_vmcore.h" #include "platform_api_extension.h" +#if (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0)) \ + && (ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(5, 2, 0)) +#define UTIMENSAT_TIMESPEC_POINTER 1 +#define FUTIMENS_TIMESPEC_POINTER 1 +#endif + int bh_platform_init() { @@ -234,7 +240,13 @@ unlinkat(int fd, const char *path, int flag) } int -utimensat(int fd, const char *path, const struct timespec ts[2], int flag) +utimensat(int fd, const char *path, +#if UTIMENSAT_TIMESPEC_POINTER + const struct timespec *ts, +#else + const struct timespec ts[2], +#endif + int flag) { errno = ENOSYS; return -1; @@ -257,7 +269,13 @@ ftruncate(int fd, off_t length) #endif int -futimens(int fd, const struct timespec times[2]) +futimens(int fd, +#if FUTIMENS_TIMESPEC_POINTER + const struct timespec *times +#else + const struct timespec times[2] +#endif +) { errno = ENOSYS; return -1; diff --git a/idf_component.yml b/idf_component.yml index a35cf79b3..ff25b32cb 100644 --- a/idf_component.yml +++ b/idf_component.yml @@ -5,4 +5,11 @@ repository: https://github.com/bytecodealliance/wasm-micro-runtime.git documentation: https://wamr.gitbook.io/ issues: https://github.com/bytecodealliance/wasm-micro-runtime/issues dependencies: - idf: ">=4.4" \ No newline at end of file + idf: ">=4.4" +targets: + - esp32 + - esp32s3 + - esp32c3 + - esp32c6 +examples: + - path: product-mini/platforms/esp-idf \ No newline at end of file diff --git a/product-mini/platforms/esp-idf/main/idf_component.yml b/product-mini/platforms/esp-idf/main/idf_component.yml index da64aa37d..1c05f476e 100644 --- a/product-mini/platforms/esp-idf/main/idf_component.yml +++ b/product-mini/platforms/esp-idf/main/idf_component.yml @@ -1,7 +1,7 @@ ## IDF Component Manager Manifest File dependencies: wasm-micro-runtime: - version: ">=2.0" + version: "^2" override_path: "../../../.." idf: version: ">=4.4" \ No newline at end of file From 19a6eb98b08ddf9e8a6ab46c700e75cb7bd3a1ee Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Thu, 11 Apr 2024 19:56:02 +0800 Subject: [PATCH 25/36] Add dependabot (#3303) --- .github/dependabot.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..0676cf741 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,35 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +version: 2 +updates: + +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + +- package-ecosystem: "docker" + directory: "/.devcontainer" + schedule: + interval: "weekly" + +- package-ecosystem: "devcontainers" + directory: "/" + schedule: + interval: "weekly" + +- package-ecosystem: "pip" + directory: "/build-scripts" + schedule: + interval: "weekly" + +- package-ecosystem: "pip" + directory: "/language-bindings/python/wasm-c-api" + schedule: + interval: "weekly" + +- package-ecosystem: "pip" + directory: "/language-bindings/python/wamr-api" + schedule: + interval: "weekly" From 1c690b7561a99b8a7d3d91b7ec19a47c7e96074d Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Thu, 11 Apr 2024 19:56:48 +0800 Subject: [PATCH 26/36] Add more checks in wasm loader (#3300) In opcode f32.const, f64.const and memory.copy, check whether the buffer to read is out of the range of wasm file before reading it. --- core/iwasm/interpreter/wasm_loader.c | 3 +++ core/iwasm/interpreter/wasm_mini_loader.c | 3 +++ 2 files changed, 6 insertions(+) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index a292663df..a7eb6c02d 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -13008,6 +13008,7 @@ re_scan: break; case WASM_OP_F32_CONST: + CHECK_BUF(p, p_end, sizeof(float32)); p += sizeof(float32); #if WASM_ENABLE_FAST_INTERP != 0 skip_label(); @@ -13026,6 +13027,7 @@ re_scan: break; case WASM_OP_F64_CONST: + CHECK_BUF(p, p_end, sizeof(float64)); p += sizeof(float64); #if WASM_ENABLE_FAST_INTERP != 0 skip_label(); @@ -14356,6 +14358,7 @@ re_scan: } case WASM_OP_MEMORY_COPY: { + CHECK_BUF(p, p_end, sizeof(int16)); /* both src and dst memory index should be 0 */ if (*(int16 *)p != 0x0000) goto fail_zero_byte_expected; diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index dc96a194d..3b452af92 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -7351,6 +7351,7 @@ re_scan: break; case WASM_OP_F32_CONST: + CHECK_BUF(p, p_end, sizeof(float32)); p += sizeof(float32); #if WASM_ENABLE_FAST_INTERP != 0 skip_label(); @@ -7369,6 +7370,7 @@ re_scan: break; case WASM_OP_F64_CONST: + CHECK_BUF(p, p_end, sizeof(float64)); p += sizeof(float64); #if WASM_ENABLE_FAST_INTERP != 0 skip_label(); @@ -7676,6 +7678,7 @@ re_scan: } case WASM_OP_MEMORY_COPY: { + CHECK_BUF(p, p_end, sizeof(int16)); /* both src and dst memory index should be 0 */ bh_assert(*(int16 *)p == 0x0000); p += 2; From b54551598ad72c563768b9b6d30750febfb6b55d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:31:05 +0800 Subject: [PATCH 27/36] Bump github/codeql-action from 2 to 3 (#3306) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8656b326c..efeb8a796 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -49,7 +49,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} @@ -66,7 +66,7 @@ jobs: - run: | ./.github/workflows/codeql_buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:${{matrix.language}}" upload: false @@ -95,7 +95,7 @@ jobs: output: ${{ steps.step1.outputs.sarif-output }}/cpp.sarif - name: Upload CodeQL results to code scanning - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: ${{ steps.step1.outputs.sarif-output }} category: "/language:${{matrix.language}}" From ff7bf7ad413c5c0d42b898af6370bdaa930fde3b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:33:30 +0800 Subject: [PATCH 28/36] Bump actions/setup-node from 3 to 4 (#3307) Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3 to 4. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/setup-node dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build_wamr_vscode_ext.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_wamr_vscode_ext.yml b/.github/workflows/build_wamr_vscode_ext.yml index b91f054cf..322ba1c06 100644 --- a/.github/workflows/build_wamr_vscode_ext.yml +++ b/.github/workflows/build_wamr_vscode_ext.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 - name: Use Node.js 16.x - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: 16.x From 1a043b6eb5e3f917a71ee229efd263a4b728bd1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:34:06 +0800 Subject: [PATCH 29/36] Bump actions/upload-artifact from 3 to 4 (#3308) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3 to 4. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index efeb8a796..5126153d1 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -102,7 +102,7 @@ jobs: - name: Upload CodeQL results as an artifact if: success() || failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: codeql-results path: ${{ steps.step1.outputs.sarif-output }} From fef26ead3e268768bce3aa658b2a05dbc3b3ee4e Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Fri, 12 Apr 2024 11:43:40 +0800 Subject: [PATCH 30/36] addr2line.py: Support sourceMappingURL section produced by emcc (#3302) And update the debug-tools sample. --- .../compilation_on_android_ubuntu.yml | 4 +- .github/workflows/compilation_on_macos.yml | 8 +- .github/workflows/nightly_run.yml | 12 +- samples/debug-tools/CMakeLists.txt | 33 ++++- samples/debug-tools/README.md | 35 ++++- .../debug-tools/cmake/FindEMSCRIPTEN.cmake | 45 +++++++ samples/debug-tools/cmake/FindWAMRC.cmake | 27 ++++ samples/debug-tools/cmake/FindWASISDK.cmake | 24 ++++ samples/debug-tools/wasm-apps/CMakeLists.txt | 121 ++++++----------- test-tools/addr2line/addr2line.py | 125 +++++++++++++++--- 10 files changed, 324 insertions(+), 110 deletions(-) create mode 100644 samples/debug-tools/cmake/FindEMSCRIPTEN.cmake create mode 100644 samples/debug-tools/cmake/FindWAMRC.cmake create mode 100644 samples/debug-tools/cmake/FindWASISDK.cmake diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index f1e437774..6b2a1a114 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -389,14 +389,14 @@ jobs: cd /opt sudo wget ${{ matrix.wasi_sdk_release }} sudo tar -xzf wasi-sdk-*.tar.gz - sudo mv wasi-sdk-20.0 wasi-sdk + sudo ln -sf wasi-sdk-20.0 wasi-sdk - name: download and install wabt run: | cd /opt sudo wget ${{ matrix.wabt_release }} sudo tar -xzf wabt-1.0.31-*.tar.gz - sudo mv wabt-1.0.31 wabt + sudo ln -sf wabt-1.0.31 wabt - name: Get LLVM libraries id: retrieve_llvm_libs uses: actions/cache@v4 diff --git a/.github/workflows/compilation_on_macos.yml b/.github/workflows/compilation_on_macos.yml index 4f59f2386..ec0943234 100644 --- a/.github/workflows/compilation_on_macos.yml +++ b/.github/workflows/compilation_on_macos.yml @@ -273,14 +273,14 @@ jobs: cd /opt sudo wget ${{ matrix.wasi_sdk_release }} sudo tar -xzf wasi-sdk-*.tar.gz - sudo mv wasi-sdk-20.0 wasi-sdk + sudo ln -sf wasi-sdk-20.0 wasi-sdk - name: download and install wabt run: | cd /opt sudo wget ${{ matrix.wabt_release }} sudo tar -xzf wabt-1.0.31-*.tar.gz - sudo mv wabt-1.0.31 wabt + sudo ln -sf wabt-1.0.31 wabt - name: Build Sample [basic] run: | @@ -346,7 +346,7 @@ jobs: cmake .. cmake --build . --config Release --parallel 4 working-directory: wamr-compiler - + - name: Build Sample [wasi-threads] run: | cd samples/wasi-threads @@ -378,4 +378,4 @@ jobs: cmake --build . --config Debug --parallel 4 ./iwasm wasm-apps/trap.wasm | grep "#" > call_stack.txt ./iwasm wasm-apps/trap.aot | grep "#" > call_stack_aot.txt - bash -x ../symbolicate.sh + bash -x ../symbolicate.sh diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index 341194df8..4b62d110a 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -8,7 +8,7 @@ on: types: - opened - synchronize - # running nightly pipeline if you're changing it + # running nightly pipeline if you're changing it # stress tests are run only in nightly at the moment, so running them in they are changed paths: - ".github/workflows/nightly_run.yml" @@ -54,7 +54,7 @@ jobs: with: os: "ubuntu-22.04" arch: "X86" - + build_wamrc: needs: [ @@ -65,7 +65,7 @@ jobs: matrix: include: - os: ubuntu-20.04 - llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} steps: - name: checkout uses: actions/checkout@v4 @@ -459,13 +459,13 @@ jobs: cd /opt sudo wget ${{ matrix.wasi_sdk_release }} sudo tar -xzf wasi-sdk-*.tar.gz - sudo mv wasi-sdk-20.0 wasi-sdk + sudo ln -sf wasi-sdk-20.0 wasi-sdk - name: download and install wabt run: | cd /opt sudo wget ${{ matrix.wabt_release }} sudo tar -xzf wabt-1.0.31-*.tar.gz - sudo mv wabt-1.0.31 wabt + sudo ln -sf wabt-1.0.31 wabt - name: Get LLVM libraries id: retrieve_llvm_libs @@ -643,7 +643,7 @@ jobs: sudo tar -xzf wasi-sdk-*.tar.gz sudo mv wasi-sdk-20.0 wasi-sdk - # It is a temporary solution until new wasi-sdk that includes bug fixes is released + # It is a temporary solution until new wasi-sdk that includes bug fixes is released - name: build wasi-libc from source if: matrix.test_option == '$WASI_TEST_OPTIONS' run: | diff --git a/samples/debug-tools/CMakeLists.txt b/samples/debug-tools/CMakeLists.txt index 5143462a3..ce06029a5 100644 --- a/samples/debug-tools/CMakeLists.txt +++ b/samples/debug-tools/CMakeLists.txt @@ -7,6 +7,14 @@ include(CheckPIESupported) project(debug_tools_sample) +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) +find_package(WASISDK REQUIRED) + +option(SOURCE_MAP_DEMO "Enable source map demo" OFF) +if (SOURCE_MAP_DEMO) + find_package(EMSCRIPTEN 3.1.50 REQUIRED) +endif () + ################ runtime settings ################ string (TOLOWER ${CMAKE_HOST_SYSTEM_NAME} WAMR_BUILD_PLATFORM) if (APPLE) @@ -61,7 +69,30 @@ include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) add_library(vmlib ${WAMR_RUNTIME_LIB_SOURCE}) ################ wasm application ################ -add_subdirectory(wasm-apps) +include(ExternalProject) + +# wasm32-wasi +ExternalProject_Add(wasm33-wasi + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps" + CONFIGURE_COMMAND ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps -B build + -DWASI_SDK_PREFIX=${WASISDK_HOME} + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + BUILD_COMMAND ${CMAKE_COMMAND} --build build + INSTALL_COMMAND ${CMAKE_COMMAND} --install build --prefix ${CMAKE_CURRENT_BINARY_DIR} +) + +if (EMSCRIPTEN_FOUND) + # wasm32-emscripten + ExternalProject_Add(wasm32-emscripten + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps" + CONFIGURE_COMMAND ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/wasm-apps -B build + -DCMAKE_TOOLCHAIN_FILE=${EMSCRIPTEN_TOOLCHAIN} + -DCMAKE_VERBOSE_MAKEFILE=On + -DSOURCE_MAP_DEMO=On + BUILD_COMMAND ${CMAKE_COMMAND} --build build + INSTALL_COMMAND ${CMAKE_COMMAND} --install build --prefix ${CMAKE_CURRENT_BINARY_DIR}/emscripten + ) +endif () ################ wamr runtime ################ include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) diff --git a/samples/debug-tools/README.md b/samples/debug-tools/README.md index 634d31197..b0358b9e4 100644 --- a/samples/debug-tools/README.md +++ b/samples/debug-tools/README.md @@ -80,6 +80,39 @@ $ python3 ../../../test-tools/addr2line/addr2line.py \ call_stack.txt --no-addr ``` +#### sourcemap + +This script also supports _sourcemap_ which is produced by [_emscripten_](https://emscripten.org/docs/tools_reference/emcc.html). The _sourcemap_ is used to map the wasm function to the original source file. To use it, add `-gsource-map` option to _emcc_ command line. The output should be a section named "sourceMappingURL" and a separated file named "_.map_. + +If the wasm file is with _sourcemap_, the script will use it to get the source file and line info. It needs an extra command line option `--emsdk` to specify the path of _emsdk_. The script will use _emsymbolizer_ to query the source file and line info. + +````bash +$ python3 ../../../test-tools/addr2line/addr2line.py \ + --wasi-sdk /opt/wasi-sdk \ + --wabt /opt/wabt \ + --wasm-file emscripten/wasm-apps/trap.wasm \ + --emsdk /opt/emsdk \ + call_stack.from_wasm_w_sourcemap.txt + +The output should be something like: + +```text +1: c + at ../../../../../wasm-apps/trap.c:5:1 +2: b + at ../../../../../wasm-apps/trap.c:11:12 +3: a + at ../../../../../wasm-apps/trap.c:17:12 +4: main + at ../../../../../wasm-apps/trap.c:24:5 +5: __main_void + at ../../../../../../../../../emsdk/emscripten/system/lib/standalone/__main_void.c:53:10 +6: _start + at ../../../../../../../../../emsdk/emscripten/system/lib/libc/crt1.c:27:3 +```` + +> The script assume the separated map file _.map_ is in the same directory as the wasm file. + ### Another approach If the wasm file is with "name" section, it is able to output function name in the stack trace. To achieve that, need to enable `WAMR_BUILD_LOAD_CUSTOM_SECTION` and `WAMR_BUILD_CUSTOM_NAME_SECTION`. If using .aot file, need to add `--emit-custom-sections=name` into wamrc command line options. @@ -97,4 +130,4 @@ Then the output should be something like Exception: unreachable ``` -Also, it is able to use *addr2line.py* to add file and line info to the stack trace. +Also, it is able to use _addr2line.py_ to add file and line info to the stack trace. diff --git a/samples/debug-tools/cmake/FindEMSCRIPTEN.cmake b/samples/debug-tools/cmake/FindEMSCRIPTEN.cmake new file mode 100644 index 000000000..8f63ec545 --- /dev/null +++ b/samples/debug-tools/cmake/FindEMSCRIPTEN.cmake @@ -0,0 +1,45 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FindPackageHandleStandardArgs) + +find_path(EMSCRIPTEN_HOME + NAMES upstream/emscripten + PATHS /opt/emsdk + NO_DEFAULT_PATH + NO_CMAKE_PATH + NO_CMAKE_SYSTEM_PATH + NO_CMAKE_FIND_ROOT_PATH + REQUIRED +) + +find_file(EMSCRIPTEN_VERSION_FILE + NAMES emscripten-version.txt + PATHS ${EMSCRIPTEN_HOME}/upstream/emscripten + NO_DEFAULT_PATH + NO_CMAKE_PATH + NO_CMAKE_SYSTEM_PATH + NO_CMAKE_FIND_ROOT_PATH + REQUIRED +) + +file(READ ${EMSCRIPTEN_VERSION_FILE} EMSCRIPTEN_VERSION_FILE_CONTENT) + +string(REGEX + MATCH + "[0-9]+\.[0-9]+(\.[0-9]+)*" + EMSCRIPTEN_VERSION + ${EMSCRIPTEN_VERSION_FILE_CONTENT} +) + +find_package_handle_standard_args(EMSCRIPTEN + REQUIRED_VARS EMSCRIPTEN_HOME + VERSION_VAR EMSCRIPTEN_VERSION + HANDLE_VERSION_RANGE +) + +if(EMSCRIPTEN_FOUND) + set(EMSCRIPTEN_TOOLCHAIN ${EMSCRIPTEN_HOME}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake) + set(EMCC ${EMSCRIPTEN_HOME}/upstream/emscripten/emcc) +endif() +mark_as_advanced(EMSCRIPTEN_TOOLCHAIN EMCC) diff --git a/samples/debug-tools/cmake/FindWAMRC.cmake b/samples/debug-tools/cmake/FindWAMRC.cmake new file mode 100644 index 000000000..20f9416f7 --- /dev/null +++ b/samples/debug-tools/cmake/FindWAMRC.cmake @@ -0,0 +1,27 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FindPackageHandleStandardArgs) + +find_path(WAMRC_HOME + wamr-compiler + PATHS ${CMAKE_CURRENT_SOURCE_DIR}/../../.. + NO_DEFAULT_PATH + NO_CMAKE_PATH + NO_CMAKE_SYSTEM_PATH + NO_CMAKE_FIND_ROOT_PATH + REQUIRED +) + +find_file(WAMRC_BIN + wamrc + HINTS ${WAMRC_HOME}/wamr-compiler/build + NO_DEFAULT_PATH + NO_CMAKE_PATH + NO_CMAKE_SYSTEM_PATH + NO_CMAKE_FIND_ROOT_PATH + REQUIRED +) + +find_package_handle_standard_args(WAMRC REQUIRED_VARS WAMRC_BIN) +mark_as_advanced(WAMRC_BIN) diff --git a/samples/debug-tools/cmake/FindWASISDK.cmake b/samples/debug-tools/cmake/FindWASISDK.cmake new file mode 100644 index 000000000..0caf374df --- /dev/null +++ b/samples/debug-tools/cmake/FindWASISDK.cmake @@ -0,0 +1,24 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +include(FindPackageHandleStandardArgs) + +file(GLOB WASISDK_SEARCH_PATH "/opt/wasi-sdk-*") +find_path(WASISDK_HOME + NAMES share/wasi-sysroot + PATHS ${WASISDK_SEARCH_PATH} + NO_DEFAULT_PATH + REQUIRED +) + +string(REGEX MATCH [0-9]+\.[0-9]+\.*[0-9]* WASISDK_VERSION ${WASISDK_HOME}) + +find_package_handle_standard_args(WASISDK REQUIRED_VARS WASISDK_HOME VERSION_VAR WASISDK_VERSION) + +if(WASISDK_FOUND) + set(WASISDK_CC_COMMAND ${WASISDK_HOME}/bin/clang) + set(WASISDK_CXX_COMMAND ${WASISDK_HOME}/bin/clang++) + set(WASISDK_TOOLCHAIN ${WASISDK_HOME}/share/cmake/wasi-sdk.cmake) + set(WASISDK_SYSROOT ${WASISDK_HOME}/share/wasi-sysroot) +endif() +mark_as_advanced(WASISDK_CC_COMMAND WASISDK_CXX_COMMAND WASISDK_TOOLCHAIN WASISDK_SYSROOT WASISDK_HOME) diff --git a/samples/debug-tools/wasm-apps/CMakeLists.txt b/samples/debug-tools/wasm-apps/CMakeLists.txt index 3ca8aff2a..527b5f37a 100644 --- a/samples/debug-tools/wasm-apps/CMakeLists.txt +++ b/samples/debug-tools/wasm-apps/CMakeLists.txt @@ -1,91 +1,58 @@ # Copyright (C) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -if (APPLE) - set (HAVE_FLAG_SEARCH_PATHS_FIRST 0) - set (CMAKE_C_LINK_FLAGS "") - set (CMAKE_CXX_LINK_FLAGS "") +cmake_minimum_required (VERSION 3.14) + +project (debut_tools_wasm) + +set (CMAKE_BUILD_TYPE Debug) # Otherwise no debug symbols (addr2line) + +list (APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../cmake) +find_package (WAMRC REQUIRED) + +option(SOURCE_MAP_DEMO "Enable source map demo" OFF) +if (SOURCE_MAP_DEMO) + find_package(EMSCRIPTEN 3.1.50 REQUIRED) endif () -if (NOT DEFINED WASI_SDK_DIR) - set (WASI_SDK_DIR "/opt/wasi-sdk") -endif () - -if (DEFINED WASI_SYSROOT) - set (CMAKE_SYSROOT "${WASI_SYSROOT}") -endif () - -set (CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") -set (CMAKE_ASM_COMPILER "${WASI_SDK_DIR}/bin/clang") -set (CMAKE_EXE_LINKER_FLAGS "-target wasm32-wasi") - -################ wabt and wamrc dependencies ################ -message(CHECK_START "Detecting WABT") -if(NOT (DEFINED WABT_DIR OR DEFINED CACHE{WABT_DIR})) - find_path(WABT_DIR - wabt - PATHS /opt - NO_DEFAULT_PATH - NO_CMAKE_FIND_ROOT_PATH - ) - if(DEFINED WABT_DIR) - set(WABT_DIR ${WABT_DIR}/wabt) - endif() -endif() -if(WABT_DIR) - message(CHECK_PASS "found") -else() - message(CHECK_FAIL "not found") -endif() - -message(CHECK_START "Detecting WASM_OBJDUMP at ${WABT_DIR}") -find_program(WASM_OBJDUMP - wasm-objdump - PATHS "${WABT_DIR}/bin" - NO_DEFAULT_PATH - NO_CMAKE_FIND_ROOT_PATH -) -if(WASM_OBJDUMP) - message(CHECK_PASS "found") -else() - message(CHECK_FAIL "not found") -endif() -if((NOT EXISTS ${WASM_OBJDUMP}) ) - message(FATAL_ERROR "Please make sure to have wasm-objdump under the path=${WABT_DIR}/bin ") -endif() - -set(WAMR_COMPILER_DIR ${CMAKE_CURRENT_LIST_DIR}/../../wamr-compiler/build) -message(CHECK_START "Detecting WAMR_COMPILER at ${WAMR_COMPILER_DIR}") -find_file(WAMR_COMPILER - wamrc - PATHS "${CMAKE_CURRENT_LIST_DIR}/../../../wamr-compiler/build" - NO_DEFAULT_PATH - NO_CMAKE_FIND_ROOT_PATH -) -if(WAMR_COMPILER) - message(CHECK_PASS "found") -else() - message(CHECK_FAIL "not found") -endif() -if((NOT EXISTS ${WAMR_COMPILER}) ) - message(FATAL_ERROR "Please build wamrc under the path=${WAMR_ROOT_DIR}/wamr-compiler/") -endif() - ################ wasm and aot compilation ################ function (compile_sample SOURCE_FILE) get_filename_component (FILE_NAME ${SOURCE_FILE} NAME_WLE) - set (WASM_MODULE ${FILE_NAME}.wasm) - add_executable (${WASM_MODULE} ${SOURCE_FILE}) - add_custom_target( - wasm_to_aot + ## wasm + set (WASM_FILE ${FILE_NAME}.wasm) + add_executable (${FILE_NAME} ${SOURCE_FILE}) + set_target_properties (${FILE_NAME} PROPERTIES SUFFIX .wasm) + + ## aot + set (AOT_FILE ${FILE_NAME}.aot) + add_custom_target ( + ${FILE_NAME}_aot ALL - DEPENDS ${WAMR_COMPILER} ${WASM_MODULE} + DEPENDS ${WAMRC_BIN} ${WASM_FILE} # Use --enable-dump-call-stack to generate stack trace (addr2line) - COMMAND ${WAMR_COMPILER} --size-level=0 --enable-dump-call-stack -o wasm-apps/trap.aot wasm-apps/trap.wasm - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMAND ${WAMRC_BIN} --size-level=0 --enable-dump-call-stack -o ${AOT_FILE} ${WASM_FILE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) + + ## wasm + sourcemap + if (DEFINED EMSCRIPTEN) + add_custom_target( + ${FILE_NAME}_w_sourcemap + ALL + DEPENDS ${SOURCE_FILE} + COMMAND ${EMCC} -O0 -gsource-map -o ${FILE_NAME}.sourcemap.wasm ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_FILE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + endif () + + ## install both + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${WASM_FILE} DESTINATION wasm-apps) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${AOT_FILE} DESTINATION wasm-apps) + if (DEFINED EMSCRIPTEN) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}.sourcemap.wasm DESTINATION wasm-apps) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${FILE_NAME}.sourcemap.wasm.map DESTINATION wasm-apps) + endif () endfunction () -set(CMAKE_BUILD_TYPE Debug) # Otherwise no debug symbols (addr2line) -compile_sample(trap.c) \ No newline at end of file +compile_sample(trap.c) diff --git a/test-tools/addr2line/addr2line.py b/test-tools/addr2line/addr2line.py index 594f8e19f..421b0bdb2 100644 --- a/test-tools/addr2line/addr2line.py +++ b/test-tools/addr2line/addr2line.py @@ -43,6 +43,28 @@ For example, there is a call-stack dump: """ +def locate_sourceMappingURL_section(wasm_objdump: Path, wasm_file: Path) -> bool: + """ + Figure out if the wasm file has a sourceMappingURL section. + """ + cmd = f"{wasm_objdump} -h {wasm_file}" + p = subprocess.run( + shlex.split(cmd), + check=True, + capture_output=True, + text=True, + universal_newlines=True, + ) + outputs = p.stdout.split(os.linesep) + + for line in outputs: + line = line.strip() + if "sourceMappingURL" in line: + return True + + return False + + def get_code_section_start(wasm_objdump: Path, wasm_file: Path) -> int: """ Find the start offset of Code section in a wasm file. @@ -62,15 +84,6 @@ def get_code_section_start(wasm_objdump: Path, wasm_file: Path) -> int: ) outputs = p.stdout.split(os.linesep) - # if there is no .debug section, return -1 - for line in outputs: - line = line.strip() - if ".debug_info" in line: - break - else: - print(f"No .debug_info section found {wasm_file}") - return -1 - for line in outputs: line = line.strip() if "Code" in line: @@ -79,7 +92,7 @@ def get_code_section_start(wasm_objdump: Path, wasm_file: Path) -> int: return -1 -def get_line_info_from_function_addr( +def get_line_info_from_function_addr_dwarf( dwarf_dump: Path, wasm_file: Path, offset: int ) -> tuple[str, str, str, str]: """ @@ -126,7 +139,7 @@ def get_dwarf_tag_value(tag: str, line: str) -> str: return m.groups()[0] -def get_line_info_from_function_name( +def get_line_info_from_function_name_dwarf( dwarf_dump: Path, wasm_file: Path, function_name: str ) -> tuple[str, str, str]: """ @@ -160,6 +173,51 @@ def get_line_info_from_function_name( return (function_name, function_file, function_line) +def get_line_info_from_function_addr_sourcemapping( + emsymbolizer: Path, wasm_file: Path, offset: int +) -> tuple[str, str, str, str]: + """ + Find the location info of a given offset in a wasm file which is compiled with emcc. + + {emsymbolizer} {wasm_file} {offset of file} + + there usually are two lines: + ?? + relative path to source file:line:column + """ + debug_info_source = wasm_file.with_name(f"{wasm_file.name}.map") + cmd = f"{emsymbolizer} -t code -f {debug_info_source} {wasm_file} {offset}" + p = subprocess.run( + shlex.split(cmd), + check=False, + capture_output=True, + text=True, + universal_newlines=True, + cwd=Path.cwd(), + ) + outputs = p.stdout.split(os.linesep) + + function_name, function_file = "", "unknown" + function_line, function_column = "?", "?" + + for line in outputs: + line = line.strip() + + if not line: + continue + + m = re.match("(.*):(\d+):(\d+)", line) + if m: + function_file, function_line, function_column = m.groups() + continue + else: + # it's always ??, not sure about that + if "??" != line: + function_name = line + + return (function_name, function_file, function_line, function_column) + + def parse_line_info(line_info: str) -> tuple[str, str, str]: """ line_info -> [file, line, column] @@ -250,6 +308,7 @@ def main(): action="store_true", help="use call stack without addresses or from fast interpreter mode", ) + parser.add_argument("--emsdk", type=Path, help="path to emsdk") args = parser.parse_args() wasm_objdump = args.wabt.joinpath("bin/wasm-objdump") @@ -261,6 +320,15 @@ def main(): llvm_cxxfilt = args.wasi_sdk.joinpath("bin/llvm-cxxfilt") assert llvm_cxxfilt.exists() + emcc_production = locate_sourceMappingURL_section(wasm_objdump, args.wasm_file) + if emcc_production: + if args.emsdk is None: + print("Please provide the path to emsdk via --emsdk") + return -1 + + emsymbolizer = args.emsdk.joinpath("upstream/emscripten/emsymbolizer") + assert emsymbolizer.exists() + code_section_start = get_code_section_start(wasm_objdump, args.wasm_file) if code_section_start == -1: return -1 @@ -281,6 +349,7 @@ def main(): _, offset, index = splitted if args.no_addr: + # FIXME: w/ emcc production if not index.startswith("$f"): # E.g. _start or Text format print(f"{i}: {index}") continue @@ -290,22 +359,40 @@ def main(): print(f"{i}: {line}") continue - line_info = get_line_info_from_function_name( - llvm_dwarf_dump, args.wasm_file, function_index_to_name[index] - ) + if not emcc_production: + _, function_file, function_line = ( + get_line_info_from_function_name_dwarf( + llvm_dwarf_dump, + args.wasm_file, + function_index_to_name[index], + ) + ) + else: + _, function_file, function_line = _, "unknown", "?" - _, function_file, function_line = line_info function_name = demangle(llvm_cxxfilt, function_index_to_name[index]) print(f"{i}: {function_name}") print(f"\tat {function_file}:{function_line}") else: offset = int(offset, 16) + # match the algorithm in wasm_interp_create_call_stack() + # either a *offset* to *code* section start + # or a *offset* in a file + assert offset > code_section_start offset = offset - code_section_start - function_name, function_file, function_line, function_column = ( - get_line_info_from_function_addr( - llvm_dwarf_dump, args.wasm_file, offset + + if emcc_production: + function_name, function_file, function_line, function_column = ( + get_line_info_from_function_addr_sourcemapping( + emsymbolizer, args.wasm_file, offset + ) + ) + else: + function_name, function_file, function_line, function_column = ( + get_line_info_from_function_addr_dwarf( + llvm_dwarf_dump, args.wasm_file, offset + ) ) - ) # if can't parse function_name, use name section or if function_name == "": From 42199f163ec02cc899bad828360cb6128b1c4fb3 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Mon, 15 Apr 2024 18:29:54 +0800 Subject: [PATCH 31/36] Log warning if growing table failed (#3310) --- core/iwasm/aot/aot_runtime.c | 34 +++++++++++++++++---------- core/iwasm/interpreter/wasm_runtime.c | 10 ++++++++ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index f8757fcc6..02508d5cd 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -3366,39 +3366,49 @@ aot_table_fill(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 length, } uint32 -aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx, - uint32 inc_entries, table_elem_type_t init_val) +aot_table_grow(AOTModuleInstance *module_inst, uint32 tbl_idx, uint32 inc_size, + table_elem_type_t init_val) { - uint32 entry_count, i, orig_tbl_sz; AOTTableInstance *tbl_inst; + uint32 i, orig_size, total_size; tbl_inst = module_inst->tables[tbl_idx]; if (!tbl_inst) { return (uint32)-1; } - orig_tbl_sz = tbl_inst->cur_size; + orig_size = tbl_inst->cur_size; - if (!inc_entries) { - return orig_tbl_sz; + if (!inc_size) { + return orig_size; } - if (tbl_inst->cur_size > UINT32_MAX - inc_entries) { + if (tbl_inst->cur_size > UINT32_MAX - inc_size) { +#if WASM_ENABLE_SPEC_TEST == 0 + LOG_WARNING("table grow (%" PRIu32 "-> %" PRIu32 + ") failed because of integer overflow", + tbl_inst->cur_size, inc_size); +#endif return (uint32)-1; } - entry_count = tbl_inst->cur_size + inc_entries; - if (entry_count > tbl_inst->max_size) { + total_size = tbl_inst->cur_size + inc_size; + if (total_size > tbl_inst->max_size) { +#if WASM_ENABLE_SPEC_TEST == 0 + LOG_WARNING("table grow (%" PRIu32 "-> %" PRIu32 + ") failed because of over max size", + tbl_inst->cur_size, inc_size); +#endif return (uint32)-1; } /* fill in */ - for (i = 0; i < inc_entries; ++i) { + for (i = 0; i < inc_size; ++i) { tbl_inst->elems[tbl_inst->cur_size + i] = init_val; } - tbl_inst->cur_size = entry_count; - return orig_tbl_sz; + tbl_inst->cur_size = total_size; + return orig_size; } #endif /* WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */ diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 688f1c2c1..5b0f07936 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -4353,11 +4353,21 @@ llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx, } if (tbl_inst->cur_size > UINT32_MAX - inc_size) { /* integer overflow */ +#if WASM_ENABLE_SPEC_TEST == 0 + LOG_WARNING("table grow (%" PRIu32 "-> %" PRIu32 + ") failed because of integer overflow", + tbl_inst->cur_size, inc_size); +#endif return (uint32)-1; } total_size = tbl_inst->cur_size + inc_size; if (total_size > tbl_inst->max_size) { +#if WASM_ENABLE_SPEC_TEST == 0 + LOG_WARNING("table grow (%" PRIu32 "-> %" PRIu32 + ") failed because of over max size", + tbl_inst->cur_size, inc_size); +#endif return (uint32)-1; } From 30426be82c806391c9709c5c11c891cdf50b528f Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Tue, 16 Apr 2024 16:32:47 +0800 Subject: [PATCH 32/36] Refactor CodeQL CI (#3319) - Move CodeQL scripts to the scripts directory - Only report error in CI if it's a CodeQL reported issue and was not dismissed before and is likely to be an actual error --- .../codeql_buildscript.sh | 5 + .github/scripts/codeql_fail_on_error.py | 124 ++++++++++++++++++ .github/workflows/codeql.yml | 7 +- .github/workflows/codeql_fail_on_error.py | 34 ----- 4 files changed, 134 insertions(+), 36 deletions(-) rename .github/{workflows => scripts}/codeql_buildscript.sh (98%) create mode 100755 .github/scripts/codeql_fail_on_error.py delete mode 100755 .github/workflows/codeql_fail_on_error.py diff --git a/.github/workflows/codeql_buildscript.sh b/.github/scripts/codeql_buildscript.sh similarity index 98% rename from .github/workflows/codeql_buildscript.sh rename to .github/scripts/codeql_buildscript.sh index ed717734e..4bcabfe25 100755 --- a/.github/workflows/codeql_buildscript.sh +++ b/.github/scripts/codeql_buildscript.sh @@ -1,5 +1,10 @@ #!/usr/bin/env bash +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + sudo apt update sudo apt install -y build-essential cmake g++-multilib libgcc-11-dev lib32gcc-11-dev ccache ninja-build ccache diff --git a/.github/scripts/codeql_fail_on_error.py b/.github/scripts/codeql_fail_on_error.py new file mode 100755 index 000000000..f150c38a2 --- /dev/null +++ b/.github/scripts/codeql_fail_on_error.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +import json +import sys +import os +import requests + + +def fetch_dismissed_alerts(repo_name, github_token): + headers = { + "Authorization": f"token {github_token}", + "Accept": "application/vnd.github.v3+json", + } + url = ( + f"https://api.github.com/repos/{repo_name}/code-scanning/alerts?state=dismissed" + ) + response = requests.get(url, headers=headers) + return response.json() # This assumes a successful API call + + +def parse_location(location): + path = location.get("physicalLocation", {}).get("artifactLocation", {}).get("uri") + start_line = location.get("physicalLocation", {}).get("region", {}).get("startLine") + column_range = ( + location.get("physicalLocation", {}).get("region", {}).get("startColumn"), + location.get("physicalLocation", {}).get("region", {}).get("endColumn"), + ) + return (path, start_line, column_range) + + +def is_dismissed(rule_id, path, start_line, column_range, dismissed_alerts): + for alert in dismissed_alerts: + alert_rule_id = alert.get("rule", {}).get("id") + alert_path = alert.get("location", {}).get("path") + alert_start_line = alert.get("location", {}).get("start_line") + alert_column_range = ( + alert.get("location", {}).get("start_column"), + alert.get("location", {}).get("end_column"), + ) + + if ( + rule_id == alert_rule_id + and path == alert_path + and start_line == alert_start_line + and column_range == alert_column_range + ): + return True + return False + + +# Return whether SARIF file contains error-level results +def codeql_sarif_contain_error(filename, dismissed_alerts): + has_error = False + + with open(filename, "r") as f: + s = json.load(f) + + for run in s.get("runs", []): + rules_metadata = run["tool"]["driver"]["rules"] + if not rules_metadata: + rules_metadata = run["tool"]["extensions"][0]["rules"] + + for res in run.get("results", []): + if "ruleIndex" in res: + rule_index = res["ruleIndex"] + elif "rule" in res and "index" in res["rule"]: + rule_index = res["rule"]["index"] + else: + continue + + # check whether it's dismissed before + rule_id = res["ruleId"] + path, start_line, column_range = parse_location(res["locations"][0]) + # the source code is from dependencies + if "_deps" in path: + continue + if is_dismissed(rule_id, path, start_line, column_range, dismissed_alerts): + print( + f"====== Finding a dismissed entry: {rule_id} at {path}:{start_line} is dismissed.======" + ) + print(res) + continue + + try: + rule_level = rules_metadata[rule_index]["defaultConfiguration"]["level"] + except IndexError as e: + print(e, rule_index, len(rules_metadata)) + else: + if rule_level == "error": + # very likely to be an actual error + if rules_metadata[rule_index]["properties"].get("precision") in [ + "high", + "very-high", + ]: + # the security severity is above medium(Common Vulnerability Scoring System (CVSS) >= 4.0) + if "security-severity" in rules_metadata[rule_index][ + "properties" + ] and ( + float( + rules_metadata[rule_index]["properties"][ + "security-severity" + ] + ) + > 4.0 + ): + print("====== Finding a likely error. ======") + print(res) + has_error = True + + return has_error + + +if __name__ == "__main__": + GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") + GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY") + dismissed_alerts = fetch_dismissed_alerts(GITHUB_REPOSITORY, GITHUB_TOKEN) + + if codeql_sarif_contain_error(sys.argv[1], dismissed_alerts): + sys.exit(1) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 5126153d1..d4e7d05f2 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -64,7 +64,7 @@ jobs: # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - run: | - ./.github/workflows/codeql_buildscript.sh + ./.github/scripts/codeql_buildscript.sh - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 with: @@ -110,5 +110,8 @@ jobs: - name: Fail if an error is found run: | - ./.github/workflows/codeql_fail_on_error.py \ + ./.github/scripts/codeql_fail_on_error.py \ ${{ steps.step1.outputs.sarif-output }}/cpp.sarif + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/codeql_fail_on_error.py b/.github/workflows/codeql_fail_on_error.py deleted file mode 100755 index 29791742b..000000000 --- a/.github/workflows/codeql_fail_on_error.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 - -import json -import sys - -# Return whether SARIF file contains error-level results -def codeql_sarif_contain_error(filename): - with open(filename, 'r') as f: - s = json.load(f) - - for run in s.get('runs', []): - rules_metadata = run['tool']['driver']['rules'] - if not rules_metadata: - rules_metadata = run['tool']['extensions'][0]['rules'] - - for res in run.get('results', []): - if 'ruleIndex' in res: - rule_index = res['ruleIndex'] - elif 'rule' in res and 'index' in res['rule']: - rule_index = res['rule']['index'] - else: - continue - try: - rule_level = rules_metadata[rule_index]['defaultConfiguration']['level'] - except IndexError as e: - print(e, rule_index, len(rules_metadata)) - else: - if rule_level == 'error': - return True - return False - -if __name__ == "__main__": - if codeql_sarif_contain_error(sys.argv[1]): - sys.exit(1) From d39d2ba3ca0bdc47ba81e1ee7723e6181fda3b1b Mon Sep 17 00:00:00 2001 From: Yage Hu Date: Wed, 17 Apr 2024 01:06:25 -0700 Subject: [PATCH 33/36] Fix posix_fadvise error handling (#3323) `posix_fadvise()` returns 0 on success and the errno on error. This commit fixes the handling of the return value such that it does not always succeeds. Fixes #3322. --- core/shared/platform/common/posix/posix_file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/shared/platform/common/posix/posix_file.c b/core/shared/platform/common/posix/posix_file.c index 20f94fba3..e7c8383c3 100644 --- a/core/shared/platform/common/posix/posix_file.c +++ b/core/shared/platform/common/posix/posix_file.c @@ -823,7 +823,7 @@ os_fadvise(os_file_handle handle, __wasi_filesize_t offset, int ret = posix_fadvise(handle, (off_t)offset, (off_t)length, nadvice); - if (ret < 0) + if (ret != 0) return convert_errno(ret); return __WASI_ESUCCESS; From 68bd30c6f9ae8830e5c44d160a45be64187e11ef Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Thu, 18 Apr 2024 12:32:01 +0800 Subject: [PATCH 34/36] Enhance GC subtyping checks (#3317) Enhance the GC subtyping checks: - Fix issues in the type equivalence check - Enable the recursive type subtyping check - Add a equivalence type flag in defined types of aot file, if there is an equivalence type before, just set it true and re-use the previous type - Normalize the defined types for interpreter and AOT - Enable spec test case type-equivalence.wast and type-subtyping.wast, and enable some commented cases - Enable set WAMR_BUILD_SANITIZER from cmake variable --- build-scripts/config_common.cmake | 7 +- core/iwasm/aot/aot_loader.c | 94 ++++-- core/iwasm/aot/aot_reloc.h | 1 + core/iwasm/aot/aot_runtime.c | 22 +- core/iwasm/aot/aot_runtime.h | 5 + core/iwasm/common/gc/gc_common.c | 7 +- core/iwasm/common/gc/gc_type.c | 162 ++++++++-- core/iwasm/common/gc/gc_type.h | 6 + core/iwasm/common/wasm_application.c | 3 +- core/iwasm/common/wasm_native.c | 4 +- core/iwasm/compilation/aot_emit_aot_file.c | 64 +++- core/iwasm/compilation/aot_emit_function.c | 94 +++++- core/iwasm/interpreter/wasm.h | 19 +- core/iwasm/interpreter/wasm_interp_classic.c | 4 +- core/iwasm/interpreter/wasm_interp_fast.c | 3 +- core/iwasm/interpreter/wasm_loader.c | 251 ++++++++++----- core/iwasm/interpreter/wasm_mini_loader.c | 2 +- core/iwasm/interpreter/wasm_runtime.c | 16 + core/iwasm/interpreter/wasm_runtime.h | 5 + .../wamr-test-suites/spec-test-script/all.py | 2 +- .../spec-test-script/gc_ignore_cases.patch | 291 +----------------- 21 files changed, 585 insertions(+), 477 deletions(-) diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 8422b060b..0a992754a 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -134,7 +134,9 @@ endif () # Sanitizers -set(WAMR_BUILD_SANITIZER $ENV{WAMR_BUILD_SANITIZER}) +if (NOT DEFINED WAMR_BUILD_SANITIZER) + set(WAMR_BUILD_SANITIZER $ENV{WAMR_BUILD_SANITIZER}) +endif () if (NOT DEFINED WAMR_BUILD_SANITIZER) set(WAMR_BUILD_SANITIZER "") @@ -554,3 +556,6 @@ else () # Disable aot intrinsics for interp, fast-jit and llvm-jit add_definitions (-DWASM_ENABLE_AOT_INTRINSICS=0) endif () +if (NOT WAMR_BUILD_SANITIZER STREQUAL "") + message (" Sanitizer ${WAMR_BUILD_SANITIZER} enabled") +endif () diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index df487039c..bdc80357f 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -1525,30 +1525,42 @@ fail: return false; } +static void +destroy_type(AOTType *type) +{ +#if WASM_ENABLE_GC != 0 + if (type->ref_count > 1) { + /* The type is referenced by other types + of current aot module */ + type->ref_count--; + return; + } + + if (type->type_flag == WASM_TYPE_FUNC) { + AOTFuncType *func_type = (AOTFuncType *)type; + if (func_type->ref_type_maps != NULL) { + bh_assert(func_type->ref_type_map_count > 0); + wasm_runtime_free(func_type->ref_type_maps); + } + } + else if (type->type_flag == WASM_TYPE_STRUCT) { + AOTStructType *struct_type = (AOTStructType *)type; + if (struct_type->ref_type_maps != NULL) { + bh_assert(struct_type->ref_type_map_count > 0); + wasm_runtime_free(struct_type->ref_type_maps); + } + } +#endif + wasm_runtime_free(type); +} + static void destroy_types(AOTType **types, uint32 count) { uint32 i; for (i = 0; i < count; i++) { - if (types[i]) { -#if WASM_ENABLE_GC != 0 - if (types[i]->type_flag == WASM_TYPE_FUNC) { - AOTFuncType *func_type = (AOTFuncType *)types[i]; - if (func_type->ref_type_maps != NULL) { - bh_assert(func_type->ref_type_map_count > 0); - wasm_runtime_free(func_type->ref_type_maps); - } - } - else if (types[i]->type_flag == WASM_TYPE_STRUCT) { - AOTStructType *struct_type = (AOTStructType *)types[i]; - if (struct_type->ref_type_maps != NULL) { - bh_assert(struct_type->ref_type_map_count > 0); - wasm_runtime_free(struct_type->ref_type_maps); - } - } -#endif - wasm_runtime_free(types[i]); + destroy_type(types[i]); } } wasm_runtime_free(types); @@ -1556,14 +1568,17 @@ destroy_types(AOTType **types, uint32 count) #if WASM_ENABLE_GC != 0 static void -init_base_type(AOTType *base_type, uint16 type_flag, bool is_sub_final, - uint32 parent_type_idx, uint16 rec_count, uint16 rec_idx) +init_base_type(AOTType *base_type, uint32 type_idx, uint16 type_flag, + bool is_sub_final, uint32 parent_type_idx, uint16 rec_count, + uint16 rec_idx) { base_type->type_flag = type_flag; + base_type->ref_count = 1; base_type->is_sub_final = is_sub_final; base_type->parent_type_idx = parent_type_idx; base_type->rec_count = rec_count; base_type->rec_idx = rec_idx; + base_type->rec_begin_type_idx = type_idx - rec_idx; } static bool @@ -1576,7 +1591,7 @@ load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, uint32 i, j; uint32 type_flag, param_cell_num, ret_cell_num; uint16 param_count, result_count, ref_type_map_count, rec_count, rec_idx; - bool is_sub_final; + bool is_equivalence_type, is_sub_final; uint32 parent_type_idx; WASMRefType ref_type; @@ -1590,12 +1605,31 @@ load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, /* Create each type */ for (i = 0; i < module->type_count; i++) { - buf = align_ptr(buf, 4); /* Read base type info */ read_uint16(buf, buf_end, type_flag); - read_uint16(buf, buf_end, is_sub_final); + + read_uint8(buf, buf_end, is_equivalence_type); + /* If there is an equivalence type, re-use it */ + if (is_equivalence_type) { + uint8 u8; + /* padding */ + read_uint8(buf, buf_end, u8); + (void)u8; + + read_uint32(buf, buf_end, j); + if (module->types[j]->ref_count == UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "wasm type's ref count too large"); + goto fail; + } + module->types[j]->ref_count++; + module->types[i] = module->types[j]; + continue; + } + + read_uint8(buf, buf_end, is_sub_final); read_uint32(buf, buf_end, parent_type_idx); read_uint16(buf, buf_end, rec_count); read_uint16(buf, buf_end, rec_idx); @@ -1620,7 +1654,7 @@ load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, types[i] = (AOTType *)func_type; - init_base_type((AOTType *)func_type, type_flag, is_sub_final, + init_base_type((AOTType *)func_type, i, type_flag, is_sub_final, parent_type_idx, rec_count, rec_idx); func_type->param_count = param_count; func_type->result_count = result_count; @@ -1726,7 +1760,7 @@ load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, offset = (uint32)offsetof(WASMStructObject, field_data); types[i] = (AOTType *)struct_type; - init_base_type((AOTType *)struct_type, type_flag, is_sub_final, + init_base_type((AOTType *)struct_type, i, type_flag, is_sub_final, parent_type_idx, rec_count, rec_idx); struct_type->field_count = field_count; struct_type->ref_type_map_count = ref_type_map_count; @@ -1812,7 +1846,7 @@ load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, types[i] = (AOTType *)array_type; - init_base_type((AOTType *)array_type, type_flag, is_sub_final, + init_base_type((AOTType *)array_type, i, type_flag, is_sub_final, parent_type_idx, rec_count, rec_idx); read_uint16(buf, buf_end, array_type->elem_flags); read_uint8(buf, buf_end, array_type->elem_type); @@ -1841,7 +1875,6 @@ load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, if (rec_count == 0) { bh_assert(rec_idx == 0); } - for (j = i - rec_idx; j <= i; j++) { AOTType *cur_type = module->types[j]; parent_type_idx = cur_type->parent_type_idx; @@ -1850,6 +1883,11 @@ load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, module->types[j]->parent_type = parent_type; module->types[j]->root_type = parent_type->root_type; + if (parent_type->inherit_depth == UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "parent type's inherit depth too large"); + goto fail; + } module->types[j]->inherit_depth = parent_type->inherit_depth + 1; } @@ -1867,7 +1905,7 @@ load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, AOTType *parent_type = module->types[parent_type_idx]; /* subtyping has been checked during compilation */ bh_assert(wasm_type_is_subtype_of( - module->types[j], parent_type, module->types, i)); + module->types[j], parent_type, module->types, i + 1)); (void)parent_type; } } diff --git a/core/iwasm/aot/aot_reloc.h b/core/iwasm/aot/aot_reloc.h index 293e2fc79..8ead3cd93 100644 --- a/core/iwasm/aot/aot_reloc.h +++ b/core/iwasm/aot/aot_reloc.h @@ -143,6 +143,7 @@ typedef struct { REG_SYM(aot_array_init_with_data), \ REG_SYM(aot_create_func_obj), \ REG_SYM(aot_obj_is_instance_of), \ + REG_SYM(aot_func_type_is_super_of), \ REG_SYM(aot_rtt_type_new), \ REG_SYM(wasm_array_obj_copy), \ REG_SYM(wasm_array_obj_new), \ diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 02508d5cd..2ec001713 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1721,6 +1721,7 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, bh_assert(table_init_data); + bh_assert(table_init_data->table_index < module_inst->table_count); table = module_inst->tables[table_init_data->table_index]; bh_assert(table); @@ -1728,8 +1729,9 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, bh_assert(table_data); wasm_runtime_get_table_inst_elem_type( - (WASMModuleInstanceCommon *)module_inst, i, &tbl_elem_type, - &tbl_elem_ref_type, &tbl_init_size, &tbl_max_size); + (WASMModuleInstanceCommon *)module_inst, + table_init_data->table_index, &tbl_elem_type, &tbl_elem_ref_type, + &tbl_init_size, &tbl_max_size); if (!wasm_elem_is_declarative(table_init_data->mode) && !wasm_reftype_is_subtype_of( @@ -4487,6 +4489,22 @@ aot_obj_is_instance_of(AOTModuleInstance *module_inst, WASMObjectRef gc_obj, return wasm_obj_is_instance_of(gc_obj, type_index, types, type_count); } +bool +aot_func_type_is_super_of(AOTModuleInstance *module_inst, uint32 type_idx1, + uint32 type_idx2) +{ + AOTModule *aot_module = (AOTModule *)module_inst->module; + AOTType **types = aot_module->types; + + if (type_idx1 == type_idx2) + return true; + + bh_assert(types[type_idx1]->type_flag == WASM_TYPE_FUNC); + bh_assert(types[type_idx2]->type_flag == WASM_TYPE_FUNC); + return wasm_func_type_is_super_of((WASMFuncType *)types[type_idx1], + (WASMFuncType *)types[type_idx2]); +} + WASMRttTypeRef aot_rtt_type_new(AOTModuleInstance *module_inst, uint32 type_index) { diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 2d3013467..63f8c872a 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -752,6 +752,11 @@ bool aot_obj_is_instance_of(AOTModuleInstance *module_inst, WASMObjectRef gc_obj, uint32 type_index); +/* Whether func type1 is one of super types of func type2 */ +bool +aot_func_type_is_super_of(AOTModuleInstance *module_inst, uint32 type_idx1, + uint32 type_idx2); + WASMRttTypeRef aot_rtt_type_new(AOTModuleInstance *module_inst, uint32 type_index); diff --git a/core/iwasm/common/gc/gc_common.c b/core/iwasm/common/gc/gc_common.c index 80936f34a..99fca86d4 100644 --- a/core/iwasm/common/gc/gc_common.c +++ b/core/iwasm/common/gc/gc_common.c @@ -304,7 +304,12 @@ wasm_defined_type_equal(WASMType *const def_type1, WASMType *const def_type2, } #endif #if WASM_ENABLE_AOT != 0 - /* TODO */ + if (module->module_type == Wasm_Module_AoT) { + AOTModule *aot_module = (AOTModule *)module; + + types = aot_module->types; + type_count = aot_module->type_count; + } #endif bh_assert(types); diff --git a/core/iwasm/common/gc/gc_type.c b/core/iwasm/common/gc/gc_type.c index 60f0e7e7a..5ade1cb27 100644 --- a/core/iwasm/common/gc/gc_type.c +++ b/core/iwasm/common/gc/gc_type.c @@ -250,6 +250,51 @@ wasm_value_types_is_subtype_of(const uint8 *types1, return true; } +static bool +rec_ref_type_equal(const WASMRefType *ref_type1, const WASMRefType *ref_type2, + uint32 rec_begin_type_idx1, uint32 rec_begin_type_idx2, + uint32 rec_count, const WASMTypePtr *types, + uint32 type_count) +{ + uint32 type_idx1, type_idx2; + + if (!wasm_is_refheaptype_typeidx(&ref_type1->ref_ht_common) + || !wasm_is_refheaptype_typeidx(&ref_type2->ref_ht_common)) + return ref_type1->ref_ht_common.heap_type + == ref_type2->ref_ht_common.heap_type + ? true + : false; + + /* Now both ref types are type of (ref type_idx) */ + type_idx1 = ref_type1->ref_ht_typeidx.type_idx; + type_idx2 = ref_type2->ref_ht_typeidx.type_idx; + + if (type_idx1 >= rec_begin_type_idx1 + && type_idx1 < rec_begin_type_idx1 + rec_count) { + /* The converted iso-recursive types should be the same */ + bool ret = (type_idx2 >= rec_begin_type_idx2 + && type_idx2 < rec_begin_type_idx2 + rec_count + && type_idx1 - rec_begin_type_idx1 + == type_idx2 - rec_begin_type_idx2) + ? true + : false; + return ret; + } + else if (type_idx2 >= rec_begin_type_idx2 + && type_idx2 < rec_begin_type_idx2 + rec_count) { + /* The converted iso-recursive types should be the same */ + bool ret = (type_idx1 >= rec_begin_type_idx1 + && type_idx1 < rec_begin_type_idx1 + rec_count + && type_idx1 - rec_begin_type_idx1 + == type_idx2 - rec_begin_type_idx2) + ? true + : false; + return ret; + } + + return types[type_idx1] == types[type_idx2] ? true : false; +} + bool wasm_func_type_equal(const WASMFuncType *type1, const WASMFuncType *type2, const WASMTypePtr *types, uint32 type_count) @@ -277,9 +322,11 @@ wasm_func_type_equal(const WASMFuncType *type1, const WASMFuncType *type2, ref_type1 = type1->ref_type_maps[j].ref_type; ref_type2 = type2->ref_type_maps[j].ref_type; - if (!wasm_reftype_equal(ref_type1->ref_type, ref_type1, - ref_type2->ref_type, ref_type2, types, - type_count)) + + if (!rec_ref_type_equal( + ref_type1, ref_type2, type1->base_type.rec_begin_type_idx, + type2->base_type.rec_begin_type_idx, + type1->base_type.rec_count, types, type_count)) return false; j++; @@ -316,9 +363,11 @@ wasm_struct_type_equal(const WASMStructType *type1, const WASMStructType *type2, ref_type1 = type1->ref_type_maps[j].ref_type; ref_type2 = type2->ref_type_maps[j].ref_type; - if (!wasm_reftype_equal(ref_type1->ref_type, ref_type1, - ref_type2->ref_type, ref_type2, types, - type_count)) + + if (!rec_ref_type_equal( + ref_type1, ref_type2, type1->base_type.rec_begin_type_idx, + type2->base_type.rec_begin_type_idx, + type1->base_type.rec_count, types, type_count)) return false; j++; @@ -338,21 +387,67 @@ wasm_array_type_equal(const WASMArrayType *type1, const WASMArrayType *type2, if (type1->elem_flags != type2->elem_flags) return false; - return wasm_reftype_equal(type1->elem_type, type1->elem_ref_type, - type2->elem_type, type2->elem_ref_type, types, - type_count); + if (type1->elem_type != type2->elem_type) + return false; + + if (!wasm_is_type_multi_byte_type(type1->elem_type)) + return true; + + return rec_ref_type_equal(type1->elem_ref_type, type2->elem_ref_type, + type1->base_type.rec_begin_type_idx, + type2->base_type.rec_begin_type_idx, + type1->base_type.rec_count, types, type_count); } bool wasm_type_equal(const WASMType *type1, const WASMType *type2, const WASMTypePtr *types, uint32 type_count) { + uint32 rec_begin_type_idx1 = type1->rec_begin_type_idx; + uint32 rec_begin_type_idx2 = type2->rec_begin_type_idx; + uint32 parent_type_idx1, parent_type_idx2, rec_count; + if (type1 == type2) return true; - if (type1->type_flag != type2->type_flag) + if (!(type1->type_flag == type2->type_flag + && type1->is_sub_final == type2->is_sub_final + && type1->rec_count == type2->rec_count + && type1->rec_idx == type2->rec_idx)) return false; + rec_count = type1->rec_count; + + parent_type_idx1 = type1->parent_type_idx; + parent_type_idx2 = type2->parent_type_idx; + + if (parent_type_idx1 >= rec_begin_type_idx1 + && parent_type_idx1 < rec_begin_type_idx1 + rec_count) { + /* The converted iso-recursive types should be the same */ + if (!(parent_type_idx2 >= rec_begin_type_idx2 + && parent_type_idx2 < rec_begin_type_idx2 + rec_count + && parent_type_idx1 - rec_begin_type_idx1 + == parent_type_idx2 - rec_begin_type_idx2)) { + return false; + } + } + else if (parent_type_idx2 >= rec_begin_type_idx2 + && parent_type_idx2 < rec_begin_type_idx2 + rec_count) { + /* The converted iso-recursive types should be the same */ + if (!(parent_type_idx1 >= rec_begin_type_idx1 + && parent_type_idx1 < rec_begin_type_idx1 + rec_count + && parent_type_idx1 - rec_begin_type_idx1 + == parent_type_idx2 - rec_begin_type_idx2)) { + return false; + } + } + else if (type1->parent_type != type2->parent_type) { + /* The parent types should be same since they have been + normalized and equivalence types with different type + indexes are referring to a same WASMType */ + return false; + } + if (wasm_type_is_func_type(type1)) return wasm_func_type_equal((WASMFuncType *)type1, (WASMFuncType *)type2, types, type_count); @@ -653,12 +748,6 @@ wasm_reftype_struct_size(const WASMRefType *ref_type) return (uint32)sizeof(RefHeapType_Common); } -static bool -type_idx_equal(uint32 type_idx1, uint32 type_idx2) -{ - return (type_idx1 == type_idx2) ? true : false; -} - bool wasm_refheaptype_equal(const RefHeapType_Common *ref_heap_type1, const RefHeapType_Common *ref_heap_type2, @@ -673,8 +762,16 @@ wasm_refheaptype_equal(const RefHeapType_Common *ref_heap_type1, if (ref_heap_type1->heap_type != ref_heap_type2->heap_type) { if (wasm_is_refheaptype_typeidx(ref_heap_type1) && wasm_is_refheaptype_typeidx(ref_heap_type2)) { - return type_idx_equal(ref_heap_type1->heap_type, - ref_heap_type2->heap_type); + if (ref_heap_type1->heap_type == ref_heap_type2->heap_type) + return true; + else + /* the type_count may be 0 when called from reftype_equal */ + return ((uint32)ref_heap_type1->heap_type < type_count + && (uint32)ref_heap_type2->heap_type < type_count + && types[ref_heap_type1->heap_type] + == types[ref_heap_type2->heap_type]) + ? true + : false; } return false; } @@ -835,6 +932,13 @@ wasm_type_is_supers_of(const WASMType *type1, const WASMType *type2) return false; } +bool +wasm_func_type_is_super_of(const WASMFuncType *type1, const WASMFuncType *type2) +{ + return wasm_type_is_supers_of((const WASMType *)type1, + (const WASMType *)type2); +} + bool wasm_reftype_is_subtype_of(uint8 type1, const WASMRefType *ref_type1, uint8 type2, const WASMRefType *ref_type2, @@ -914,14 +1018,15 @@ wasm_reftype_is_subtype_of(uint8 type1, const WASMRefType *ref_type1, #endif else if (type1 == REF_TYPE_HT_NULLABLE) { if (wasm_is_refheaptype_typeidx(&ref_type1->ref_ht_common)) { + bh_assert((uint32)ref_type1->ref_ht_typeidx.type_idx < type_count); /* reftype1 is (ref null $t) */ if (type2 == REF_TYPE_HT_NULLABLE && ref_type2 != NULL && wasm_is_refheaptype_typeidx(&ref_type2->ref_ht_common)) { - return type_idx_equal(ref_type1->ref_ht_typeidx.type_idx, - ref_type2->ref_ht_typeidx.type_idx) - || wasm_type_is_supers_of( - types[ref_type2->ref_ht_typeidx.type_idx], - types[ref_type1->ref_ht_typeidx.type_idx]); + bh_assert((uint32)ref_type2->ref_ht_typeidx.type_idx + < type_count); + return wasm_type_is_supers_of( + types[ref_type2->ref_ht_typeidx.type_idx], + types[ref_type1->ref_ht_typeidx.type_idx]); } else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag == WASM_TYPE_STRUCT) @@ -963,16 +1068,17 @@ wasm_reftype_is_subtype_of(uint8 type1, const WASMRefType *ref_type1, else if (type1 == REF_TYPE_HT_NON_NULLABLE) { bh_assert(ref_type1); if (wasm_is_refheaptype_typeidx(&ref_type1->ref_ht_common)) { + bh_assert((uint32)ref_type1->ref_ht_typeidx.type_idx < type_count); /* reftype1 is (ref $t) */ if ((type2 == REF_TYPE_HT_NULLABLE || type2 == REF_TYPE_HT_NON_NULLABLE) && ref_type2 != NULL && wasm_is_refheaptype_typeidx(&ref_type2->ref_ht_common)) { - return type_idx_equal(ref_type1->ref_ht_typeidx.type_idx, - ref_type2->ref_ht_typeidx.type_idx) - || wasm_type_is_supers_of( - types[ref_type2->ref_ht_typeidx.type_idx], - types[ref_type1->ref_ht_typeidx.type_idx]); + bh_assert((uint32)ref_type2->ref_ht_typeidx.type_idx + < type_count); + return wasm_type_is_supers_of( + types[ref_type2->ref_ht_typeidx.type_idx], + types[ref_type1->ref_ht_typeidx.type_idx]); } else if (types[ref_type1->ref_ht_typeidx.type_idx]->type_flag == WASM_TYPE_STRUCT) { diff --git a/core/iwasm/common/gc/gc_type.h b/core/iwasm/common/gc/gc_type.h index 5b3840e45..919c8e501 100644 --- a/core/iwasm/common/gc/gc_type.h +++ b/core/iwasm/common/gc/gc_type.h @@ -47,6 +47,12 @@ wasm_func_type_is_subtype_of(const WASMFuncType *type1, const WASMFuncType *type2, const WASMTypePtr *types, uint32 type_count); +/* Whether func type1 is one of super types of func type2, + used for the func type check in call_indirect/call_ref opcodes */ +bool +wasm_func_type_is_super_of(const WASMFuncType *type1, + const WASMFuncType *type2); + /* Whether func type1's result types are subtype of func type2's result types */ bool diff --git a/core/iwasm/common/wasm_application.c b/core/iwasm/common/wasm_application.c index 13ad2b1a6..f19cb00e4 100644 --- a/core/iwasm/common/wasm_application.c +++ b/core/iwasm/common/wasm_application.c @@ -578,8 +578,7 @@ execute_func(WASMModuleInstanceCommon *module_inst, const char *name, is_anyref = true; } - if (wasm_is_type_multi_byte_type( - type->types[type->param_count + i])) { + if (wasm_is_type_multi_byte_type(type->types[i])) { WASMRefType *ref_type = ref_type_map->ref_type; if (wasm_is_refheaptype_common( &ref_type->ref_ht_common)) { diff --git a/core/iwasm/common/wasm_native.c b/core/iwasm/common/wasm_native.c index 14b295ee7..394dfd2b5 100644 --- a/core/iwasm/common/wasm_native.c +++ b/core/iwasm/common/wasm_native.c @@ -84,9 +84,9 @@ compare_type_with_signautre(uint8 type, const char signature) if ('r' == signature #if WASM_ENABLE_GC != 0 #if WASM_ENABLE_STRINGREF != 0 - && (type >= REF_TYPE_STRINGVIEWITER && type <= REF_TYPE_FUNCREF) + && (type >= REF_TYPE_STRINGVIEWITER && type <= REF_TYPE_NULLFUNCREF) #else - && (type >= REF_TYPE_NULLREF && type <= REF_TYPE_FUNCREF) + && (type >= REF_TYPE_HT_NULLABLE && type <= REF_TYPE_NULLFUNCREF) #endif #else && type == VALUE_TYPE_EXTERNREF diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index 52637686f..426171984 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -484,15 +484,15 @@ static uint32 get_func_type_size(AOTCompContext *comp_ctx, AOTFuncType *func_type) { #if WASM_ENABLE_GC != 0 - /* type flag + is_sub_final + parent_type_idx + rec_count + rec_idx + param - * count + result count - * + ref_type_map_count + types + context of ref_type_map */ + /* type flag + equivalence type flag + is_sub_final + parent_type_idx + + rec_count + rec_idx + param count + result count + + ref_type_map_count + types + context of ref_type_map */ if (comp_ctx->enable_gc) { uint32 size = 0; /* type flag */ size += sizeof(func_type->base_type.type_flag); - /* is_sub_final */ + /* equivalence type flag + is_sub_final */ size += sizeof(uint16); /* parent_type_idx */ size += sizeof(func_type->base_type.parent_type_idx); @@ -529,12 +529,12 @@ static uint32 get_struct_type_size(AOTCompContext *comp_ctx, AOTStructType *struct_type) { uint32 size = 0; - /* type flag + is_sub_final + parent_type_idx + rec_count + rec_idx + field - * count + fields */ + /* type flag + equivalence type flag + is_sub_final + parent_type_idx + + rec_count + rec_idx + field count + fields */ /* type flag */ size += sizeof(struct_type->base_type.type_flag); - /* is_sub_final */ + /* equivalence type flag + is_sub_final */ size += sizeof(uint16); /* parent_type_idx */ size += sizeof(struct_type->base_type.parent_type_idx); @@ -558,12 +558,12 @@ static uint32 get_array_type_size(AOTCompContext *comp_ctx, AOTArrayType *array_type) { uint32 size = 0; - /* type flag + is_sub_final + parent_type_idx + rec_count + rec_idx + - elem_flags + elem_type + elem_ref_type */ + /* type flag + equivalence type flag + is_sub_final + parent_type_idx + + rec_count + rec_idx + elem_flags + elem_type + elem_ref_type */ /* type flag */ size += sizeof(array_type->base_type.type_flag); - /* is_sub_final */ + /* equivalence type flag + is_sub_final */ size += sizeof(uint16); /* parent_type_idx (u32) */ size += sizeof(array_type->base_type.parent_type_idx); @@ -597,7 +597,22 @@ get_type_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data) #if WASM_ENABLE_GC != 0 if (comp_ctx->enable_gc) { for (i = 0; i < comp_data->type_count; i++) { + uint32 j; + size = align_uint(size, 4); + + /* Emit simple info if there is an equivalence type */ + for (j = 0; j < i; j++) { + if (comp_data->types[j] == comp_data->types[i]) { + /* type_flag (2 bytes) + equivalence type flag (1 byte) + + padding (1 byte) + equivalence type index */ + size += 8; + break; + } + } + if (j < i) + continue; + if (comp_data->types[i]->type_flag == WASM_TYPE_FUNC) size += get_func_type_size(comp_ctx, (AOTFuncType *)comp_data->types[i]); @@ -2093,13 +2108,32 @@ aot_emit_type_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, #if WASM_ENABLE_GC != 0 if (comp_ctx->enable_gc) { - int32 idx; AOTType **types = comp_data->types; + int32 idx; + uint32 j; for (i = 0; i < comp_data->type_count; i++) { offset = align_uint(offset, 4); + + /* Emit simple info if there is an equivalence type */ + for (j = 0; j < i; j++) { + if (types[j] == types[i]) { + EMIT_U16(types[i]->type_flag); + /* equivalence type flag is true */ + EMIT_U8(1); + EMIT_U8(0); + /* equivalence type index */ + EMIT_U32(j); + break; + } + } + if (j < i) + continue; + EMIT_U16(types[i]->type_flag); - EMIT_U16(types[i]->is_sub_final); + /* equivalence type flag is false */ + EMIT_U8(0); + EMIT_U8(types[i]->is_sub_final); EMIT_U32(types[i]->parent_type_idx); EMIT_U16(types[i]->rec_count); @@ -2593,7 +2627,7 @@ aot_emit_func_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, if (comp_ctx->enable_gc) { /* emit func_local_ref_flag arrays for both import and AOTed funcs */ AOTFuncType *func_type; - uint32 j, local_ref_flags_cell_num; + uint32 j, local_ref_flags_cell_num, paddings; for (i = 0; i < comp_data->import_func_count; i++) { func_type = comp_data->import_funcs[i].func_type; @@ -2603,6 +2637,8 @@ aot_emit_func_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, local_ref_flags_cell_num += wasm_value_type_cell_num_internal( func_type->types[j], comp_ctx->pointer_size); } + paddings = + local_ref_flags_cell_num < 2 ? 2 - local_ref_flags_cell_num : 0; local_ref_flags_cell_num = local_ref_flags_cell_num > 2 ? local_ref_flags_cell_num : 2; @@ -2614,7 +2650,7 @@ aot_emit_func_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, func_type->types[j])) return false; } - for (; j < 2; j++) + for (j = 0; j < paddings; j++) EMIT_U8(0); } diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index 224173163..cf3824e9a 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -1826,6 +1826,52 @@ fail: return ret; } +#if WASM_ENABLE_GC != 0 +static LLVMValueRef +call_aot_func_type_is_super_of_func(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef type_idx1, + LLVMValueRef type_idx2) +{ + LLVMValueRef param_values[3], ret_value, value, func; + LLVMTypeRef param_types[3], ret_type, func_type, func_ptr_type; + + param_types[0] = comp_ctx->aot_inst_type; + param_types[1] = I32_TYPE; + param_types[2] = I32_TYPE; + ret_type = INT8_TYPE; + +#if WASM_ENABLE_JIT != 0 + if (comp_ctx->is_jit_mode) + GET_AOT_FUNCTION(llvm_jit_func_type_is_super_of, 3); + else +#endif + GET_AOT_FUNCTION(aot_func_type_is_super_of, 3); + + param_values[0] = func_ctx->aot_inst; + param_values[1] = type_idx1; + param_values[2] = type_idx2; + + if (!(ret_value = + LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values, + 3, "call_aot_func_type_is_super_of"))) { + aot_set_last_error("llvm build call failed."); + return NULL; + } + + if (!(ret_value = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, ret_value, + I8_ZERO, "check_fail"))) { + aot_set_last_error("llvm build icmp failed."); + return NULL; + } + + return ret_value; + +fail: + return NULL; +} +#endif + static bool call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, AOTFuncType *aot_func_type, @@ -2018,15 +2064,23 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return false; } - /* Find the equivalent function type whose type index is the smallest: - the callee function's type index is also converted to the smallest - one in wasm loader, so we can just check whether the two type indexes - are equal (the type index of call_indirect opcode and callee func), - we don't need to check whether the whole function types are equal, - including param types and result types. */ - type_idx = - wasm_get_smallest_type_idx((WASMTypePtr *)comp_ctx->comp_data->types, - comp_ctx->comp_data->type_count, type_idx); + if (!comp_ctx->enable_gc) { + /* Find the equivalent function type whose type index is the smallest: + the callee function's type index is also converted to the smallest + one in wasm loader, so we can just check whether the two type indexes + are equal (the type index of call_indirect opcode and callee func), + we don't need to check whether the whole function types are equal, + including param types and result types. */ + type_idx = wasm_get_smallest_type_idx( + (WASMTypePtr *)comp_ctx->comp_data->types, + comp_ctx->comp_data->type_count, type_idx); + } + else { + /* Call aot_func_type_is_super_of to check whether the func type + provided in the bytecode is a super type of the func type of + the function to call */ + } + ftype_idx_const = I32_CONST(type_idx); CHECK_LLVM_CONST(ftype_idx_const); @@ -2254,11 +2308,23 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - /* Check if function type index not equal */ - if (!(cmp_ftype_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, ftype_idx, - ftype_idx_const, "cmp_ftype_idx"))) { - aot_set_last_error("llvm build icmp failed."); - goto fail; +#if WASM_ENABLE_GC != 0 + if (comp_ctx->enable_gc) { + if (!(cmp_ftype_idx = call_aot_func_type_is_super_of_func( + comp_ctx, func_ctx, ftype_idx_const, ftype_idx))) { + goto fail; + } + } + else +#endif + { + /* Check if function type index not equal */ + if (!(cmp_ftype_idx = + LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, ftype_idx, + ftype_idx_const, "cmp_ftype_idx"))) { + aot_set_last_error("llvm build icmp failed."); + goto fail; + } } /* Throw exception if ftype_idx != ftype_idx_const */ diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index 80ce67b8e..5c436cbfa 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -274,7 +274,7 @@ typedef struct InitializerExpression { */ typedef struct RefHeapType_TypeIdx { /* ref_type is REF_TYPE_HT_NULLABLE or - REF_TYPE_HT_NON_NULLABLE, (0x6C or 0x6B) */ + REF_TYPE_HT_NON_NULLABLE, (0x63 or 0x64) */ uint8 ref_type; /* true if ref_type is REF_TYPE_HT_NULLABLE */ bool nullable; @@ -288,7 +288,7 @@ typedef struct RefHeapType_TypeIdx { */ typedef struct RefHeapType_Common { /* ref_type is REF_TYPE_HT_NULLABLE or - REF_TYPE_HT_NON_NULLABLE (0x6C or 0x6B) */ + REF_TYPE_HT_NON_NULLABLE (0x63 or 0x64) */ uint8 ref_type; /* true if ref_type is REF_TYPE_HT_NULLABLE */ bool nullable; @@ -338,18 +338,24 @@ typedef struct WASMType { uint16 type_flag; bool is_sub_final; + /* How many types are referring to this type */ + uint16 ref_count; /* The inheritance depth */ - uint32 inherit_depth; + uint16 inherit_depth; /* The root type */ struct WASMType *root_type; /* The parent type */ struct WASMType *parent_type; uint32 parent_type_idx; - /* number of internal types in the current rec group, if the type is not in - * a recursive group, rec_count = 0 */ + /* The number of internal types in the current rec group, and if + the type is not in a recursive group, rec_count is 1 since a + single type definition is reinterpreted as a short-hand for a + recursive group containing just one type */ uint16 rec_count; uint16 rec_idx; + /* The index of the begin type of this group */ + uint32 rec_begin_type_idx; } WASMType, *WASMTypePtr; #endif /* end of WASM_ENABLE_GC */ @@ -375,9 +381,6 @@ typedef struct WASMFuncType { uint16 ref_type_map_count; WASMRefTypeMap *ref_type_maps; WASMRefTypeMap *result_ref_type_maps; - /* minimal type index of the type equal to this type, - used in type equal check in call_indirect opcode */ - uint32 min_type_idx_normalized; #else uint16 ref_count; #endif diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 3aec3f4c4..89e6ac57b 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -2212,6 +2212,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, WASMFuncType *cur_type, *cur_func_type; WASMTableInstance *tbl_inst; uint32 tbl_idx; + #if WASM_ENABLE_TAIL_CALL != 0 opcode = *(frame_ip - 1); #endif @@ -2282,8 +2283,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } #else - if (cur_type->min_type_idx_normalized - != cur_func_type->min_type_idx_normalized) { + if (!wasm_func_type_is_super_of(cur_type, cur_func_type)) { wasm_set_exception(module, "indirect call type mismatch"); goto got_exception; } diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index c7cb70260..417e3b016 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -1736,8 +1736,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, goto got_exception; } #else - if (cur_type->min_type_idx_normalized - != cur_func_type->min_type_idx_normalized) { + if (!wasm_func_type_is_super_of(cur_type, cur_func_type)) { wasm_set_exception(module, "indirect call type mismatch"); goto got_exception; } diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index a7eb6c02d..513484fce 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -394,10 +394,10 @@ memory_realloc(void *mem_old, uint32 size_old, uint32 size_new, char *error_buf, #if WASM_ENABLE_GC != 0 static bool -check_type_index(const WASMModule *module, uint32 type_index, char *error_buf, - uint32 error_buf_size) +check_type_index(const WASMModule *module, uint32 type_count, uint32 type_index, + char *error_buf, uint32 error_buf_size) { - if (type_index >= module->type_count) { + if (type_index >= type_count) { set_error_buf_v(error_buf, error_buf_size, "unknown type %d", type_index); return false; @@ -409,7 +409,8 @@ static bool check_array_type(const WASMModule *module, uint32 type_index, char *error_buf, uint32 error_buf_size) { - if (!check_type_index(module, type_index, error_buf, error_buf_size)) { + if (!check_type_index(module, module->type_count, type_index, error_buf, + error_buf_size)) { return false; } if (module->types[type_index]->type_flag != WASM_TYPE_ARRAY) { @@ -775,8 +776,8 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, if (!is_byte_a_type(type1)) { p--; read_leb_uint32(p, p_end, type_idx); - if (!check_type_index(module, type_idx, error_buf, - error_buf_size)) + if (!check_type_index(module, module->type_count, type_idx, + error_buf, error_buf_size)) goto fail; wasm_set_refheaptype_typeidx(&cur_ref_type.ref_ht_typeidx, @@ -902,7 +903,8 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, uint32 field_count; read_leb_uint32(p, p_end, type_idx); - if (!check_type_index(module, type_idx, error_buf, + if (!check_type_index(module, module->type_count, + type_idx, error_buf, error_buf_size)) { goto fail; } @@ -966,7 +968,8 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, read_leb_uint32(p, p_end, cur_value.type_index); type_idx = cur_value.type_index; - if (!check_type_index(module, type_idx, error_buf, + if (!check_type_index(module, module->type_count, + type_idx, error_buf, error_buf_size)) { goto fail; } @@ -1001,7 +1004,8 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, read_leb_uint32(p, p_end, cur_value.type_index); type_idx = cur_value.type_index; - if (!check_type_index(module, type_idx, error_buf, + if (!check_type_index(module, module->type_count, + type_idx, error_buf, error_buf_size)) { goto fail; } @@ -1275,6 +1279,13 @@ destroy_array_type(WASMArrayType *type) static void destroy_wasm_type(WASMType *type) { + if (type->ref_count > 1) { + /* The type is referenced by other types + of current wasm module */ + type->ref_count--; + return; + } + if (type->type_flag == WASM_TYPE_FUNC) destroy_func_type((WASMFuncType *)type); else if (type->type_flag == WASM_TYPE_STRUCT) @@ -1289,8 +1300,9 @@ destroy_wasm_type(WASMType *type) /* Resolve (ref null ht) or (ref ht) */ static bool resolve_reftype_htref(const uint8 **p_buf, const uint8 *buf_end, - WASMModule *module, bool nullable, WASMRefType *ref_type, - char *error_buf, uint32 error_buf_size) + WASMModule *module, uint32 type_count, bool nullable, + WASMRefType *ref_type, char *error_buf, + uint32 error_buf_size) { const uint8 *p = *p_buf, *p_end = buf_end; @@ -1301,8 +1313,9 @@ resolve_reftype_htref(const uint8 **p_buf, const uint8 *buf_end, if (wasm_is_refheaptype_typeidx(&ref_type->ref_ht_common)) { /* heap type is (type i), i : typeidx, >= 0 */ - if (!check_type_index(module, ref_type->ref_ht_typeidx.type_idx, - error_buf, error_buf_size)) { + if (!check_type_index(module, type_count, + ref_type->ref_ht_typeidx.type_idx, error_buf, + error_buf_size)) { return false; } } @@ -1320,9 +1333,10 @@ fail: static bool resolve_value_type(const uint8 **p_buf, const uint8 *buf_end, - WASMModule *module, bool *p_need_ref_type_map, - WASMRefType *ref_type, bool allow_packed_type, - char *error_buf, uint32 error_buf_size) + WASMModule *module, uint32 type_count, + bool *p_need_ref_type_map, WASMRefType *ref_type, + bool allow_packed_type, char *error_buf, + uint32 error_buf_size) { const uint8 *p = *p_buf, *p_end = buf_end; uint8 type; @@ -1334,8 +1348,8 @@ resolve_value_type(const uint8 **p_buf, const uint8 *buf_end, if (wasm_is_reftype_htref_nullable(type)) { /* (ref null ht) */ - if (!resolve_reftype_htref(&p, p_end, module, true, ref_type, error_buf, - error_buf_size)) + if (!resolve_reftype_htref(&p, p_end, module, type_count, true, + ref_type, error_buf, error_buf_size)) return false; if (!wasm_is_refheaptype_common(&ref_type->ref_ht_common)) *p_need_ref_type_map = true; @@ -1351,8 +1365,8 @@ resolve_value_type(const uint8 **p_buf, const uint8 *buf_end, } else if (wasm_is_reftype_htref_non_nullable(type)) { /* (ref ht) */ - if (!resolve_reftype_htref(&p, p_end, module, false, ref_type, - error_buf, error_buf_size)) + if (!resolve_reftype_htref(&p, p_end, module, type_count, false, + ref_type, error_buf, error_buf_size)) return false; *p_need_ref_type_map = true; #if WASM_ENABLE_STRINGREF != 0 @@ -1401,7 +1415,8 @@ reftype_set_insert(HashMap *ref_type_set, const WASMRefType *ref_type, static bool resolve_func_type(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, - uint32 type_idx, char *error_buf, uint32 error_buf_size) + uint32 type_count, uint32 type_idx, char *error_buf, + uint32 error_buf_size) { const uint8 *p = *p_buf, *p_end = buf_end, *p_org; uint32 param_count, result_count, i, j = 0; @@ -1417,8 +1432,9 @@ resolve_func_type(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, read_leb_uint32(p, p_end, param_count); p_org = p; for (i = 0; i < param_count; i++) { - if (!resolve_value_type(&p, p_end, module, &need_ref_type_map, - &ref_type, false, error_buf, error_buf_size)) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { return false; } if (need_ref_type_map) @@ -1427,8 +1443,9 @@ resolve_func_type(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, read_leb_uint32(p, p_end, result_count); for (i = 0; i < result_count; i++) { - if (!resolve_value_type(&p, p_end, module, &need_ref_type_map, - &ref_type, false, error_buf, error_buf_size)) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { return false; } if (need_ref_type_map) { @@ -1468,8 +1485,9 @@ resolve_func_type(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, } for (i = 0; i < param_count; i++) { - if (!resolve_value_type(&p, p_end, module, &need_ref_type_map, - &ref_type, false, error_buf, error_buf_size)) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { goto fail; } type->types[i] = ref_type.ref_type; @@ -1485,8 +1503,9 @@ resolve_func_type(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, read_leb_uint32(p, p_end, result_count); for (i = 0; i < result_count; i++) { - if (!resolve_value_type(&p, p_end, module, &need_ref_type_map, - &ref_type, false, error_buf, error_buf_size)) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { goto fail; } type->types[param_count + i] = ref_type.ref_type; @@ -1527,18 +1546,6 @@ resolve_func_type(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, } #endif - /* Calculate the minimal type index of the type equal to this type */ - type->min_type_idx_normalized = type_idx; - for (i = 0; i < type_idx; i++) { - WASMFuncType *func_type = (WASMFuncType *)module->types[i]; - if (func_type->base_type.type_flag == WASM_TYPE_FUNC - && wasm_func_type_equal(type, func_type, module->types, - type_idx + 1)) { - type->min_type_idx_normalized = i; - break; - } - } - *p_buf = p; module->types[type_idx] = (WASMType *)type; @@ -1552,8 +1559,8 @@ fail: static bool resolve_struct_type(const uint8 **p_buf, const uint8 *buf_end, - WASMModule *module, uint32 type_idx, char *error_buf, - uint32 error_buf_size) + WASMModule *module, uint32 type_count, uint32 type_idx, + char *error_buf, uint32 error_buf_size) { const uint8 *p = *p_buf, *p_end = buf_end, *p_org; uint32 field_count, ref_type_map_count = 0, ref_field_count = 0; @@ -1569,8 +1576,9 @@ resolve_struct_type(const uint8 **p_buf, const uint8 *buf_end, read_leb_uint32(p, p_end, field_count); p_org = p; for (i = 0; i < field_count; i++) { - if (!resolve_value_type(&p, p_end, module, &need_ref_type_map, - &ref_type, true, error_buf, error_buf_size)) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, true, error_buf, + error_buf_size)) { return false; } if (need_ref_type_map) @@ -1617,8 +1625,9 @@ resolve_struct_type(const uint8 **p_buf, const uint8 *buf_end, offset = (uint32)offsetof(WASMStructObject, field_data); for (i = 0; i < field_count; i++) { - if (!resolve_value_type(&p, p_end, module, &need_ref_type_map, - &ref_type, true, error_buf, error_buf_size)) { + if (!resolve_value_type(&p, p_end, module, type_count, + &need_ref_type_map, &ref_type, true, error_buf, + error_buf_size)) { goto fail; } type->fields[i].field_type = ref_type.ref_type; @@ -1671,8 +1680,8 @@ fail: static bool resolve_array_type(const uint8 **p_buf, const uint8 *buf_end, - WASMModule *module, uint32 type_idx, char *error_buf, - uint32 error_buf_size) + WASMModule *module, uint32 type_count, uint32 type_idx, + char *error_buf, uint32 error_buf_size) { const uint8 *p = *p_buf, *p_end = buf_end; uint8 mutable; @@ -1680,8 +1689,8 @@ resolve_array_type(const uint8 **p_buf, const uint8 *buf_end, WASMRefType ref_type; WASMArrayType *type = NULL; - if (!resolve_value_type(&p, p_end, module, &need_ref_type_map, &ref_type, - true, error_buf, error_buf_size)) { + if (!resolve_value_type(&p, p_end, module, type_count, &need_ref_type_map, + &ref_type, true, error_buf, error_buf_size)) { return false; } @@ -1730,7 +1739,8 @@ init_ref_type(WASMModule *module, WASMRefType *ref_type, bool nullable, int32 heap_type, char *error_buf, uint32 error_buf_size) { if (heap_type >= 0) { - if (!check_type_index(module, heap_type, error_buf, error_buf_size)) { + if (!check_type_index(module, module->type_count, heap_type, error_buf, + error_buf_size)) { return false; } wasm_set_refheaptype_typeidx(&ref_type->ref_ht_typeidx, nullable, @@ -2010,6 +2020,7 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, if (flag == DEFINED_TYPE_FUNC) { if (!resolve_func_type(&p, buf_end, module, + processed_type_count + rec_count, processed_type_count + j, error_buf, error_buf_size)) { return false; @@ -2017,6 +2028,7 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, } else if (flag == DEFINED_TYPE_STRUCT) { if (!resolve_struct_type(&p, buf_end, module, + processed_type_count + rec_count, processed_type_count + j, error_buf, error_buf_size)) { return false; @@ -2024,6 +2036,7 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, } else if (flag == DEFINED_TYPE_ARRAY) { if (!resolve_array_type(&p, buf_end, module, + processed_type_count + rec_count, processed_type_count + j, error_buf, error_buf_size)) { return false; @@ -2037,13 +2050,13 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, cur_type = module->types[processed_type_count + j]; + cur_type->ref_count = 1; cur_type->parent_type_idx = parent_type_idx; cur_type->is_sub_final = is_sub_final; - if (rec_count > 1) { - cur_type->rec_count = rec_count; - cur_type->rec_idx = j; - } + cur_type->rec_count = rec_count; + cur_type->rec_idx = j; + cur_type->rec_begin_type_idx = processed_type_count; } /* resolve subtyping relationship in current rec group */ @@ -2055,6 +2068,11 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, module->types[cur_type->parent_type_idx]; cur_type->parent_type = parent_type; cur_type->root_type = parent_type->root_type; + if (parent_type->inherit_depth == UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "parent type's inherit depth too large"); + return false; + } cur_type->inherit_depth = parent_type->inherit_depth + 1; } else { @@ -2080,6 +2098,49 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, } } + /* If there is already an equivalence type or a group of equivalence + recursive types created, use it or them instead */ + for (j = 0; j < processed_type_count;) { + WASMType *src_type = module->types[j]; + WASMType *cur_type = module->types[processed_type_count]; + uint32 k, src_rec_count; + + src_rec_count = src_type->rec_count; + if (src_rec_count != rec_count) { + /* no type equivalence */ + j += src_rec_count; + continue; + } + + for (k = 0; k < rec_count; k++) { + src_type = module->types[j + k]; + cur_type = module->types[processed_type_count + k]; + if (!wasm_type_equal(src_type, cur_type, module->types, + module->type_count)) { + break; + } + } + if (k < rec_count) { + /* no type equivalence */ + j += src_rec_count; + continue; + } + + /* type equivalence */ + for (k = 0; k < rec_count; k++) { + if (module->types[j + k]->ref_count == UINT16_MAX) { + set_error_buf(error_buf, error_buf_size, + "wasm type's ref count too large"); + return false; + } + destroy_wasm_type(module->types[processed_type_count + k]); + module->types[processed_type_count + k] = + module->types[j + k]; + module->types[j + k]->ref_count++; + } + break; + } + if (rec_count > 1) { LOG_VERBOSE("Finished processing rec group [%d-%d]", processed_type_count, @@ -2511,8 +2572,9 @@ load_table_import(const uint8 **p_buf, const uint8 *buf_end, return false; } #else /* else of WASM_ENABLE_GC == 0 */ - if (!resolve_value_type(&p, p_end, parent_module, &need_ref_type_map, - &ref_type, false, error_buf, error_buf_size)) { + if (!resolve_value_type(&p, p_end, parent_module, parent_module->type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { return false; } if (wasm_is_reftype_htref_non_nullable(ref_type.ref_type)) { @@ -2947,8 +3009,9 @@ load_global_import(const uint8 **p_buf, const uint8 *buf_end, declare_type = read_uint8(p); declare_mutable = read_uint8(p); #else - if (!resolve_value_type(&p, p_end, parent_module, &need_ref_type_map, - &ref_type, false, error_buf, error_buf_size)) { + if (!resolve_value_type(&p, p_end, parent_module, parent_module->type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { return false; } declare_type = ref_type.ref_type; @@ -3050,8 +3113,9 @@ load_table(const uint8 **p_buf, const uint8 *buf_end, WASMModule *module, return false; } #else /* else of WASM_ENABLE_GC == 0 */ - if (!resolve_value_type(&p, p_end, module, &need_ref_type_map, &ref_type, - false, error_buf, error_buf_size)) { + if (!resolve_value_type(&p, p_end, module, module->type_count, + &need_ref_type_map, &ref_type, false, error_buf, + error_buf_size)) { return false; } table->elem_type = ref_type.ref_type; @@ -3536,7 +3600,8 @@ load_function_section(const uint8 *buf, const uint8 *buf_end, type_index_org = type_index; #endif -#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) +#if (WASM_ENABLE_WAMR_COMPILER != 0 || WASM_ENABLE_JIT != 0) \ + && WASM_ENABLE_GC == 0 type_index = wasm_get_smallest_type_idx( module->types, module->type_count, type_index); #endif @@ -3577,8 +3642,9 @@ load_function_section(const uint8 *buf, const uint8 *buf_end, #endif #else if (!resolve_value_type(&p_code, buf_code_end, module, - &need_ref_type_map, &ref_type, false, - error_buf, error_buf_size)) { + module->type_count, &need_ref_type_map, + &ref_type, false, error_buf, + error_buf_size)) { return false; } local_count += sub_local_count; @@ -3664,8 +3730,9 @@ load_function_section(const uint8 *buf, const uint8 *buf_end, } #else if (!resolve_value_type(&p_code, buf_code_end, module, - &need_ref_type_map, &ref_type, false, - error_buf, error_buf_size)) { + module->type_count, &need_ref_type_map, + &ref_type, false, error_buf, + error_buf_size)) { return false; } if (need_ref_type_map) { @@ -3923,9 +3990,9 @@ load_global_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, global->type = read_uint8(p); mutable = read_uint8(p); #else - if (!resolve_value_type(&p, p_end, module, &need_ref_type_map, - &ref_type, false, error_buf, - error_buf_size)) { + if (!resolve_value_type(&p, p_end, module, module->type_count, + &need_ref_type_map, &ref_type, false, + error_buf, error_buf_size)) { return false; } global->type = ref_type.ref_type; @@ -4231,8 +4298,8 @@ load_elem_type(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, #else p--; if (!resolve_value_type((const uint8 **)&p, p_end, module, - &need_ref_type_map, &elem_ref_type, false, - error_buf, error_buf_size)) { + module->type_count, &need_ref_type_map, + &elem_ref_type, false, error_buf, error_buf_size)) { return false; } if (!wasm_is_type_reftype(elem_ref_type.ref_type)) { @@ -10827,7 +10894,8 @@ re_scan: p_org = p; p--; if (!resolve_value_type((const uint8 **)&p, p_end, - module, &need_ref_type_map, + module, module->type_count, + &need_ref_type_map, &wasm_ref_type, false, error_buf, error_buf_size)) { goto fail; @@ -11303,8 +11371,8 @@ re_scan: bh_memcpy_s(loader_ctx->frame_offset, size, block->param_frame_offsets, size); loader_ctx->frame_offset += (size / sizeof(int16)); - loader_ctx->dynamic_offset = block->start_dynamic_offset; } + loader_ctx->dynamic_offset = block->start_dynamic_offset; #endif break; @@ -11598,6 +11666,15 @@ re_scan: if (opcode == WASM_OP_CALL_REF || opcode == WASM_OP_RETURN_CALL_REF) { read_leb_uint32(p, p_end, type_idx1); + if (!check_type_index(module, module->type_count, type_idx1, + error_buf, error_buf_size)) { + goto fail; + } + if (module->types[type_idx1]->type_flag != WASM_TYPE_FUNC) { + set_error_buf(error_buf, error_buf_size, + "unkown function type"); + goto fail; + } if (!wasm_loader_pop_nullable_typeidx(loader_ctx, &type, &type_idx, error_buf, error_buf_size)) { @@ -11606,8 +11683,8 @@ re_scan: if (type == VALUE_TYPE_ANY) { type_idx = type_idx1; } - if (!check_type_index(module, type_idx, error_buf, - error_buf_size)) { + if (!check_type_index(module, module->type_count, type_idx, + error_buf, error_buf_size)) { goto fail; } if (module->types[type_idx]->type_flag != WASM_TYPE_FUNC) { @@ -11615,7 +11692,9 @@ re_scan: "unkown function type"); goto fail; } - if (type_idx != type_idx1) { + if (!wasm_func_type_is_super_of( + (WASMFuncType *)module->types[type_idx1], + (WASMFuncType *)module->types[type_idx])) { set_error_buf(error_buf, error_buf_size, "function type mismatch"); goto fail; @@ -12055,8 +12134,9 @@ re_scan: #else p_org = p + 1; if (!resolve_value_type((const uint8 **)&p, p_end, module, - &need_ref_type_map, &wasm_ref_type, - false, error_buf, error_buf_size)) { + module->type_count, &need_ref_type_map, + &wasm_ref_type, false, error_buf, + error_buf_size)) { goto fail; } type = wasm_ref_type.ref_type; @@ -12223,8 +12303,8 @@ re_scan: #else read_leb_int32(p, p_end, heap_type); if (heap_type >= 0) { - if (!check_type_index(module, heap_type, error_buf, - error_buf_size)) { + if (!check_type_index(module, module->type_count, heap_type, + error_buf, error_buf_size)) { goto fail; } wasm_set_refheaptype_typeidx(&wasm_ref_type.ref_ht_typeidx, @@ -13288,7 +13368,8 @@ re_scan: #if WASM_ENABLE_FAST_INTERP != 0 emit_uint32(loader_ctx, type_idx); #endif - if (!check_type_index(module, type_idx, error_buf, + if (!check_type_index(module, module->type_count, + type_idx, error_buf, error_buf_size)) { goto fail; } @@ -13374,7 +13455,8 @@ re_scan: #if WASM_ENABLE_FAST_INTERP != 0 emit_uint32(loader_ctx, type_idx); #endif - if (!check_type_index(module, type_idx, error_buf, + if (!check_type_index(module, module->type_count, + type_idx, error_buf, error_buf_size)) { goto fail; } @@ -13787,7 +13869,8 @@ re_scan: emit_uint32(loader_ctx, (uint32)heap_type); #endif if (heap_type >= 0) { - if (!check_type_index(module, heap_type, error_buf, + if (!check_type_index(module, module->type_count, + heap_type, error_buf, error_buf_size)) { goto fail; } diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 3b452af92..3deec511f 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -6226,8 +6226,8 @@ re_scan: bh_memcpy_s(loader_ctx->frame_offset, size, block->param_frame_offsets, size); loader_ctx->frame_offset += (size / sizeof(int16)); - loader_ctx->dynamic_offset = block->start_dynamic_offset; } + loader_ctx->dynamic_offset = block->start_dynamic_offset; #endif break; diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 5b0f07936..cb5e384fa 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -4404,6 +4404,22 @@ llvm_jit_obj_is_instance_of(WASMModuleInstance *module_inst, return wasm_obj_is_instance_of(gc_obj, type_index, types, type_count); } +bool +llvm_jit_func_type_is_super_of(WASMModuleInstance *module_inst, + uint32 type_idx1, uint32 type_idx2) +{ + WASMModule *module = module_inst->module; + WASMType **types = module->types; + + if (type_idx1 == type_idx2) + return true; + + bh_assert(types[type_idx1]->type_flag == WASM_TYPE_FUNC); + bh_assert(types[type_idx2]->type_flag == WASM_TYPE_FUNC); + return wasm_func_type_is_super_of((WASMFuncType *)types[type_idx1], + (WASMFuncType *)types[type_idx2]); +} + WASMRttTypeRef llvm_jit_rtt_type_new(WASMModuleInstance *module_inst, uint32 type_index) { diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 13b738f9e..4249eb5c1 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -811,6 +811,11 @@ bool llvm_jit_obj_is_instance_of(WASMModuleInstance *module_inst, WASMObjectRef gc_obj, uint32 type_index); +/* Whether func type1 is one of super types of func type2 */ +bool +llvm_jit_func_type_is_super_of(WASMModuleInstance *module_inst, + uint32 type_idx1, uint32 type_idx2); + WASMRttTypeRef llvm_jit_rtt_type_new(WASMModuleInstance *module_inst, uint32 type_index); diff --git a/tests/wamr-test-suites/spec-test-script/all.py b/tests/wamr-test-suites/spec-test-script/all.py index 98f5c1e63..13db47203 100644 --- a/tests/wamr-test-suites/spec-test-script/all.py +++ b/tests/wamr-test-suites/spec-test-script/all.py @@ -95,7 +95,7 @@ def ignore_the_case( return True if gc_flag: - if case_name in ["type-equivalence", "type-rec", "array_init_elem", "array_init_data"]: + if case_name in ["array_init_elem", "array_init_data"]: return True if sgx_flag: diff --git a/tests/wamr-test-suites/spec-test-script/gc_ignore_cases.patch b/tests/wamr-test-suites/spec-test-script/gc_ignore_cases.patch index a627a38f6..bc91d6b06 100644 --- a/tests/wamr-test-suites/spec-test-script/gc_ignore_cases.patch +++ b/tests/wamr-test-suites/spec-test-script/gc_ignore_cases.patch @@ -9,78 +9,6 @@ index 335496f0..5b975028 100644 - "integer representation too long" + "invalid type flag" ;; In GC extension, the first byte in rectype define is just one byte, not LEB128 encoded. ) -diff --git a/test/core/binary.wast b/test/core/binary.wast -index 1661a1c6..84c716b9 100644 ---- a/test/core/binary.wast -+++ b/test/core/binary.wast -@@ -1082,7 +1082,7 @@ - ) - - ;; 1 br_table target declared, 2 given --(assert_malformed -+(;assert_malformed - (module binary - "\00asm" "\01\00\00\00" - "\01\04\01" ;; type section -@@ -1132,3 +1132,4 @@ - ) - "unexpected content after last section" - ) -+;) -diff --git a/test/core/data.wast b/test/core/data.wast -index a5c87fbb..6f948bae 100644 ---- a/test/core/data.wast -+++ b/test/core/data.wast -@@ -306,9 +306,10 @@ - "\02\01\41\00\0b" ;; active data segment 0 for memory 1 - "\00" ;; empty vec(byte) - ) -- "unknown memory 1" -+ "unknown memory" - ) - -+(; not supported by wat2wasm - ;; Data segment with memory index 0 (no memory section) - (assert_invalid - (module binary -@@ -317,7 +318,7 @@ - "\00\41\00\0b" ;; active data segment 0 for memory 0 - "\00" ;; empty vec(byte) - ) -- "unknown memory 0" -+ "unknown memory" - ) - - ;; Data segment with memory index 1 (no memory section) -@@ -328,7 +329,7 @@ - "\02\01\41\00\0b" ;; active data segment 0 for memory 1 - "\00" ;; empty vec(byte) - ) -- "unknown memory 1" -+ "unknown memory" - ) - - ;; Data segment with memory index 1 and vec(byte) as above, -@@ -348,7 +349,7 @@ - "\20\21\22\23\24\25\26\27\28\29\2a\2b\2c\2d\2e\2f" - "\30\31\32\33\34\35\36\37\38\39\3a\3b\3c\3d" - ) -- "unknown memory 1" -+ "unknown memory" - ) - - ;; Data segment with memory index 1 and specially crafted vec(byte) after. -@@ -368,8 +369,9 @@ - "\20\21\22\23\24\25\26\27\28\29\2a\2b\2c\2d\2e\2f" - "\30\31\32\33\34\35\36\37\38\39\3a\3b\3c\3d" - ) -- "unknown memory 1" -+ "unknown memory" - ) -+;) - - - ;; Invalid offsets diff --git a/test/core/elem.wast b/test/core/elem.wast index df1610f6..32c1d8b3 100644 --- a/test/core/elem.wast @@ -268,196 +196,11 @@ index 00000000..32650644 + ) + "unsupported initializer expression for table" +) -diff --git a/test/core/gc/ref_test.wast b/test/core/gc/ref_test.wast -index 590b81b8..e0aa49ed 100644 ---- a/test/core/gc/ref_test.wast -+++ b/test/core/gc/ref_test.wast -@@ -310,15 +310,16 @@ - (br_if $l (i32.eqz (ref.test (ref $t0) (table.get (i32.const 11))))) - (br_if $l (i32.eqz (ref.test (ref $t0) (table.get (i32.const 12))))) - -- (br_if $l (i32.eqz (ref.test (ref $t1') (table.get (i32.const 1))))) -- (br_if $l (i32.eqz (ref.test (ref $t1') (table.get (i32.const 2))))) -+ ;; Must have explicit sub relationship -+ ;; (br_if $l (i32.eqz (ref.test (ref $t1') (table.get (i32.const 1))))) -+ ;; (br_if $l (i32.eqz (ref.test (ref $t1') (table.get (i32.const 2))))) - -- (br_if $l (i32.eqz (ref.test (ref $t1) (table.get (i32.const 11))))) -- (br_if $l (i32.eqz (ref.test (ref $t1) (table.get (i32.const 12))))) -+ ;; (br_if $l (i32.eqz (ref.test (ref $t1) (table.get (i32.const 11))))) -+ ;; (br_if $l (i32.eqz (ref.test (ref $t1) (table.get (i32.const 12))))) - -- (br_if $l (i32.eqz (ref.test (ref $t2') (table.get (i32.const 2))))) -+ ;; (br_if $l (i32.eqz (ref.test (ref $t2') (table.get (i32.const 2))))) - -- (br_if $l (i32.eqz (ref.test (ref $t2) (table.get (i32.const 12))))) -+ ;; (br_if $l (i32.eqz (ref.test (ref $t2) (table.get (i32.const 12))))) - - (return) - ) diff --git a/test/core/gc/type-subtyping.wast b/test/core/gc/type-subtyping.wast -index a9022fc3..4e22e91b 100644 +index a9022fc3..4aa36e2a 100644 --- a/test/core/gc/type-subtyping.wast +++ b/test/core/gc/type-subtyping.wast -@@ -112,6 +112,8 @@ - ) - ) - -+;; don't support recursive type equality and subtype check -+(; - (module - (rec (type $f1 (sub (func))) (type (struct (field (ref $f1))))) - (rec (type $f2 (sub (func))) (type (struct (field (ref $f2))))) -@@ -135,6 +137,7 @@ - (func $g (type $g2)) - (global (ref $g1) (ref.func $g)) - ) -+;) - - (assert_invalid - (module -@@ -156,6 +159,8 @@ - (global (ref $f1) (ref.func $g)) - ) - -+;; don't support recursive type equality and subtype check -+(; - (module - (rec (type $f1 (sub (func))) (type $s1 (sub (struct (field (ref $f1)))))) - (rec (type $f2 (sub (func))) (type $s2 (sub (struct (field (ref $f2)))))) -@@ -201,6 +206,7 @@ - (global (ref $g12) (ref.func $g12)) - (global (ref $g22) (ref.func $g12)) - ) -+;) - - (assert_invalid - (module -@@ -226,6 +232,8 @@ - - ;; Runtime types - -+;; don't support recursive type equality and subtype check -+(; - (module - (type $t0 (sub (func (result (ref null func))))) - (rec (type $t1 (sub $t0 (func (result (ref null $t1)))))) -@@ -286,6 +294,7 @@ - (assert_trap (invoke "fail4") "cast") - (assert_trap (invoke "fail5") "cast") - (assert_trap (invoke "fail6") "cast") -+;) - - (module - (type $t1 (sub (func))) -@@ -316,7 +325,8 @@ - (assert_trap (invoke "fail3") "cast") - (assert_trap (invoke "fail4") "cast") - -- -+;; don't support recursive type equality and subtype check -+(; - (module - (rec (type $f1 (sub (func))) (type (struct (field (ref $f1))))) - (rec (type $f2 (sub (func))) (type (struct (field (ref $f2))))) -@@ -346,6 +356,7 @@ - ) - ) - (assert_return (invoke "run") (i32.const 1)) -+;) - - (module - (rec (type $f1 (sub (func))) (type (struct (field (ref $f1))))) -@@ -370,6 +381,8 @@ - ) - (assert_return (invoke "run") (i32.const 1)) - -+;; don't support recursive type equality and subtype check -+(; - (module - (rec (type $f1 (sub (func))) (type $s1 (sub (struct (field (ref $f1)))))) - (rec (type $f2 (sub (func))) (type $s2 (sub (struct (field (ref $f2)))))) -@@ -390,7 +403,6 @@ - ) - (assert_return (invoke "run") (i32.const 1) (i32.const 1)) - -- - (module - (rec (type $f11 (sub (func (result (ref func))))) (type $f12 (sub $f11 (func (result (ref $f11)))))) - (rec (type $f21 (sub (func (result (ref func))))) (type $f22 (sub $f21 (func (result (ref $f21)))))) -@@ -429,7 +441,9 @@ - (i32.const 1) (i32.const 1) (i32.const 1) (i32.const 1) - (i32.const 1) (i32.const 1) (i32.const 1) (i32.const 1) - ) -+;) - -+(; we use normalized function type index - (module - (rec (type $f11 (sub (func))) (type $f12 (sub $f11 (func)))) - (rec (type $f21 (sub (func))) (type $f22 (sub $f11 (func)))) -@@ -439,6 +453,7 @@ - ) - ) - (assert_return (invoke "run") (i32.const 0)) -+;) - - (module - (rec (type $f01 (sub (func))) (type $f02 (sub $f01 (func)))) -@@ -547,15 +562,15 @@ - (func (import "M3" "g") (type $g1)) - ) - --(module -- (rec (type $f1 (sub (func))) (type $s1 (sub (struct (field (ref $f1)))))) -- (rec (type $f2 (sub (func))) (type $s2 (sub (struct (field (ref $f2)))))) -- (rec -- (type $g2 (sub $f2 (func))) -- (type (sub $s2 (struct (field (ref $f1) (ref $f2) (ref $f1) (ref $f2) (ref $g2))))) -- ) -- (func (export "g") (type $g2)) --) -+;; (module -+;; (rec (type $f1 (sub (func))) (type $s1 (sub (struct (field (ref $f1)))))) -+;; (rec (type $f2 (sub (func))) (type $s2 (sub (struct (field (ref $f2)))))) -+;; (rec -+;; (type $g2 (sub $f2 (func))) -+;; (type (sub $s2 (struct (field (ref $f1) (ref $f2) (ref $f1) (ref $f2) (ref $g2))))) -+;; ) -+;; (func (export "g") (type $g2)) -+;; ) - (register "M4") - (module - (rec (type $f1 (sub (func))) (type $s1 (sub (struct (field (ref $f1)))))) -@@ -597,17 +612,17 @@ - (func (import "M6" "g") (type $f1)) - ) - --(module -- (rec (type $f1 (sub (func))) (type $s1 (sub (struct (field (ref $f1)))))) -- (rec (type $f2 (sub (func))) (type $s2 (sub (struct (field (ref $f2)))))) -- (rec -- (type $g2 (sub $f2 (func))) -- (type (sub $s2 (struct (field (ref $f1) (ref $f2) (ref $f1) (ref $f2) (ref $g2))))) -- ) -- (rec (type $h (sub $g2 (func))) (type (struct))) -- (func (export "h") (type $h)) --) --(register "M7") -+;; (module -+;; (rec (type $f1 (sub (func))) (type $s1 (sub (struct (field (ref $f1)))))) -+;; (rec (type $f2 (sub (func))) (type $s2 (sub (struct (field (ref $f2)))))) -+;; (rec -+;; (type $g2 (sub $f2 (func))) -+;; (type (sub $s2 (struct (field (ref $f1) (ref $f2) (ref $f1) (ref $f2) (ref $g2))))) -+;; ) -+;; (rec (type $h (sub $g2 (func))) (type (struct))) -+;; (func (export "h") (type $h)) -+;; ) -+;; (register "M7") - (module - (rec (type $f1 (sub (func))) (type $s1 (sub (struct (field (ref $f1)))))) - (rec (type $f2 (sub (func))) (type $s2 (sub (struct (field (ref $f2)))))) -@@ -740,7 +755,7 @@ +@@ -740,7 +740,7 @@ "sub type" ) @@ -466,7 +209,7 @@ index a9022fc3..4e22e91b 100644 (module (type $f0 (sub (func (param i32) (result i32)))) (type $s0 (sub $f0 (struct))) -@@ -764,7 +779,7 @@ +@@ -764,7 +764,7 @@ "sub type" ) @@ -475,7 +218,7 @@ index a9022fc3..4e22e91b 100644 (module (type $s0 (sub (struct))) (type $f0 (sub $s0 (func (param i32) (result i32)))) -@@ -772,7 +787,7 @@ +@@ -772,7 +772,7 @@ "sub type" ) @@ -1262,29 +1005,3 @@ index 0b2d26f7..bdab6a01 100644 (table $t0 30 30 funcref) (table $t1 30 30 funcref) (elem (table $t1) (i32.const 2) func 3 1 4 1) -diff --git a/test/core/unreached-valid.wast b/test/core/unreached-valid.wast -index f3feb0f3..d8ef8743 100644 ---- a/test/core/unreached-valid.wast -+++ b/test/core/unreached-valid.wast -@@ -60,7 +60,7 @@ - - ;; Validation after unreachable - --(module -+(;module - (func (export "meet-bottom") - (block (result f64) - (block (result f32) -@@ -76,7 +76,6 @@ - - (assert_trap (invoke "meet-bottom") "unreachable") - -- - ;; Bottom heap type - - (module -@@ -106,3 +105,4 @@ - (unreachable) - ) - ) -+;) From ba59e56e19ae812c1c8d91a77955519c453295f1 Mon Sep 17 00:00:00 2001 From: dongsheng28849455 <68947925+dongsheng28849455@users.noreply.github.com> Date: Thu, 18 Apr 2024 19:40:57 +0800 Subject: [PATCH 35/36] User defined memory allocator for different purposes (#3316) Some issues are related with memory fragmentation, which may cause the linear memory cannot be allocated. In WAMR, the memory managed by the system is often trivial, but linear memory usually directly allocates a large block and often remains unchanged for a long time. Their sensitivity and contribution to fragmentation are different, which is suitable for different allocation strategies. If we can control the linear memory's allocation, do not make it from system heap, the overhead of heap management might be avoided. Add `mem_alloc_usage_t usage` as the first argument for user defined malloc/realloc/free functions when `WAMR_BUILD_ALLOC_WITH_USAGE` cmake variable is set as 1, and make passing `Alloc_For_LinearMemory` to the argument when allocating the linear memory. --- build-scripts/config_common.cmake | 3 + core/config.h | 4 + core/iwasm/common/wasm_memory.c | 114 ++++++++++++++++---- core/iwasm/include/wasm_export.h | 8 ++ doc/build_wamr.md | 4 + product-mini/platforms/nuttx/CMakeLists.txt | 4 + product-mini/platforms/nuttx/wamr.mk | 5 + product-mini/platforms/posix/main.c | 9 ++ 8 files changed, 131 insertions(+), 20 deletions(-) diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 0a992754a..c3a957d33 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -556,6 +556,9 @@ else () # Disable aot intrinsics for interp, fast-jit and llvm-jit add_definitions (-DWASM_ENABLE_AOT_INTRINSICS=0) endif () +if (WAMR_BUILD_ALLOC_WITH_USAGE EQUAL 1) + add_definitions(-DWASM_MEM_ALLOC_WITH_USAGE=1) +endif() if (NOT WAMR_BUILD_SANITIZER STREQUAL "") message (" Sanitizer ${WAMR_BUILD_SANITIZER} enabled") endif () diff --git a/core/config.h b/core/config.h index d84ed3f36..c8ccc07a2 100644 --- a/core/config.h +++ b/core/config.h @@ -587,4 +587,8 @@ #define WASM_TABLE_MAX_SIZE 1024 #endif +#ifndef WASM_MEM_ALLOC_WITH_USAGE +#define WASM_MEM_ALLOC_WITH_USAGE 0 +#endif + #endif /* end of _CONFIG_H_ */ diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 50ee917ed..c49c1825a 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -29,16 +29,35 @@ static void *enlarge_memory_error_user_data; #if WASM_MEM_ALLOC_WITH_USER_DATA != 0 static void *allocator_user_data = NULL; -static void *(*malloc_func)(void *user_data, unsigned int size) = NULL; -static void *(*realloc_func)(void *user_data, void *ptr, - unsigned int size) = NULL; -static void (*free_func)(void *user_data, void *ptr) = NULL; -#else -static void *(*malloc_func)(unsigned int size) = NULL; -static void *(*realloc_func)(void *ptr, unsigned int size) = NULL; -static void (*free_func)(void *ptr) = NULL; #endif +static void *(*malloc_func)( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + unsigned int size) = NULL; + +static void *(*realloc_func)( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, bool full_size_mmaped, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + void *ptr, unsigned int size) = NULL; + +static void (*free_func)( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, +#endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + void *user_data, +#endif + void *ptr) = NULL; + static unsigned int global_pool_size; static uint64 @@ -177,11 +196,14 @@ wasm_runtime_malloc_internal(unsigned int size) return mem_allocator_malloc(pool_allocator, size); } else if (memory_mode == MEMORY_MODE_ALLOCATOR) { -#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 - return malloc_func(allocator_user_data, size); -#else - return malloc_func(size); + return malloc_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + Alloc_For_Runtime, #endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + allocator_user_data, +#endif + size); } else { return os_malloc(size); @@ -201,11 +223,14 @@ wasm_runtime_realloc_internal(void *ptr, unsigned int size) } else if (memory_mode == MEMORY_MODE_ALLOCATOR) { if (realloc_func) -#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 - return realloc_func(allocator_user_data, ptr, size); -#else - return realloc_func(ptr, size); + return realloc_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + Alloc_For_Runtime, false, #endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + allocator_user_data, +#endif + ptr, size); else return NULL; } @@ -233,11 +258,14 @@ wasm_runtime_free_internal(void *ptr) mem_allocator_free(pool_allocator, ptr); } else if (memory_mode == MEMORY_MODE_ALLOCATOR) { -#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 - free_func(allocator_user_data, ptr); -#else - free_func(ptr); + free_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + Alloc_For_Runtime, #endif +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + allocator_user_data, +#endif + ptr); } else { os_free(ptr); @@ -765,6 +793,29 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count) bh_assert(total_size_new <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64)); +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + if (!(memory_data_new = + realloc_func(Alloc_For_LinearMemory, full_size_mmaped, +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + NULL, +#endif + memory_data_old, total_size_new))) { + ret = false; + goto return_func; + } + if (heap_size > 0) { + if (mem_allocator_migrate(memory->heap_handle, + (char *)heap_data_old + + (memory_data_new - memory_data_old), + heap_size) + != 0) { + ret = false; + } + } + memory->heap_data = memory_data_new + (heap_data_old - memory_data_old); + memory->heap_data_end = memory->heap_data + heap_size; + memory->memory_data = memory_data_new; +#else if (full_size_mmaped) { #ifdef BH_PLATFORM_WINDOWS if (!os_mem_commit(memory->memory_data_end, @@ -823,6 +874,7 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count) os_writegsbase(memory_data_new); #endif } +#endif /* end of WASM_MEM_ALLOC_WITH_USAGE */ memory->num_bytes_per_page = num_bytes_per_page; memory->cur_page_count = total_page_count; @@ -903,8 +955,19 @@ wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst) #else map_size = 8 * (uint64)BH_GB; #endif + +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + (void)map_size; + free_func(Alloc_For_LinearMemory, +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + NULL, +#endif + memory_inst->memory_data); +#else wasm_munmap_linear_memory(memory_inst->memory_data, memory_inst->memory_data_size, map_size); +#endif + memory_inst->memory_data = NULL; } @@ -954,9 +1017,20 @@ wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory, *memory_data_size = align_as_and_cast(*memory_data_size, page_size); if (map_size > 0) { +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + (void)wasm_mmap_linear_memory; + if (!(*data = malloc_func(Alloc_For_LinearMemory, +#if WASM_MEM_ALLOC_WITH_USER_DATA != 0 + NULL, +#endif + *memory_data_size))) { + return BHT_ERROR; + } +#else if (!(*data = wasm_mmap_linear_memory(map_size, *memory_data_size))) { return BHT_ERROR; } +#endif } return BHT_OK; diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index bc43ea0b9..7cd266e12 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -113,6 +113,11 @@ typedef enum { Alloc_With_System_Allocator, } mem_alloc_type_t; +typedef enum { + Alloc_For_Runtime, + Alloc_For_LinearMemory +} mem_alloc_usage_t; + /* Memory allocator option */ typedef union MemAllocOption { struct { @@ -120,6 +125,9 @@ typedef union MemAllocOption { uint32_t heap_size; } pool; struct { + /* the function signature is varied when + WASM_MEM_ALLOC_WITH_USER_DATA and + WASM_MEM_ALLOC_WITH_USAGE are defined */ void *malloc_func; void *realloc_func; void *free_func; diff --git a/doc/build_wamr.md b/doc/build_wamr.md index b0a8ea35f..5598ea364 100644 --- a/doc/build_wamr.md +++ b/doc/build_wamr.md @@ -254,6 +254,10 @@ Currently we only profile the memory consumption of module, module_instance and > See [Enable segue optimization for wamrc when generating the aot file](./perf_tune.md#3-enable-segue-optimization-for-wamrc-when-generating-the-aot-file) for more details. +#### **User defined linear memory allocator** +- **WAMR_BUILD_ALLOC_WITH_USAGE**=1/0, default to disable if not set +> Notes: by default, the linear memory is allocated by system. when it's set to 1 and Alloc_With_Allocator is selected, it will be allocated by customer. + #### **Enable running PGO(Profile-Guided Optimization) instrumented AOT file** - **WAMR_BUILD_STATIC_PGO**=1/0, default to disable if not set > Note: See [Use the AOT static PGO method](./perf_tune.md#5-use-the-aot-static-pgo-method) for more details. diff --git a/product-mini/platforms/nuttx/CMakeLists.txt b/product-mini/platforms/nuttx/CMakeLists.txt index f83e79916..e9fe5a9e3 100644 --- a/product-mini/platforms/nuttx/CMakeLists.txt +++ b/product-mini/platforms/nuttx/CMakeLists.txt @@ -129,6 +129,10 @@ if(CONFIG_INTERPRETERS_WAMR_GLOBAL_HEAP_POOL) set(WAMR_BUILD_GLOBAL_HEAP_SIZE ${_HEAP_SIZE_}) endif() +if (CONFIG_INTERPRETERS_WAMR_MEM_ALLOC_WITH_USAGE) + set(WAMR_BUILD_MEM_ALLOC_WITH_USAGE 1) +endif() + if(CONFIG_INTERPRETERS_WAMR_ENABLE_SPEC_TEST) set(WAMR_BUILD_SPEC_TEST 1) endif() diff --git a/product-mini/platforms/nuttx/wamr.mk b/product-mini/platforms/nuttx/wamr.mk index e414a7cda..7aac0e358 100644 --- a/product-mini/platforms/nuttx/wamr.mk +++ b/product-mini/platforms/nuttx/wamr.mk @@ -373,6 +373,11 @@ CFLAGS += -DWASM_ENABLE_GLOBAL_HEAP_POOL=1 CFLAGS += -DWASM_GLOBAL_HEAP_SIZE="$(CONFIG_INTERPRETERS_WAMR_GLOBAL_HEAP_POOL_SIZE) * 1024" else CFLAGS += -DWASM_ENABLE_GLOBAL_HEAP_POOL=0 +ifeq ($(CONFIG_INTERPRETERS_WAMR_MEM_ALLOC_WITH_USAGE),y) +CFLAGS += -DWASM_MEM_ALLOC_WITH_USAGE=1 +else +CFLAGS += -DWASM_MEM_ALLOC_WITH_USAGE=0 +endif endif ifeq ($(CONFIG_INTERPRETERS_WAMR_ENABLE_SPEC_TEST),y) diff --git a/product-mini/platforms/posix/main.c b/product-mini/platforms/posix/main.c index 2acd2190a..cb0581ce2 100644 --- a/product-mini/platforms/posix/main.c +++ b/product-mini/platforms/posix/main.c @@ -445,6 +445,9 @@ static char global_heap_buf[WASM_GLOBAL_HEAP_SIZE] = { 0 }; #else static void * malloc_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, +#endif #if WASM_MEM_ALLOC_WITH_USER_DATA != 0 void *user_data, #endif @@ -455,6 +458,9 @@ malloc_func( static void * realloc_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, bool full_size_mmaped, +#endif #if WASM_MEM_ALLOC_WITH_USER_DATA != 0 void *user_data, #endif @@ -465,6 +471,9 @@ realloc_func( static void free_func( +#if WASM_MEM_ALLOC_WITH_USAGE != 0 + mem_alloc_usage_t usage, +#endif #if WASM_MEM_ALLOC_WITH_USER_DATA != 0 void *user_data, #endif From f6481cedec5c1147f2de8d5a38e0acc8d12dfc73 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Fri, 19 Apr 2024 11:19:59 +0900 Subject: [PATCH 36/36] Add a comment on WASM_STACK_GUARD_SIZE (#3332) --- core/config.h | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/core/config.h b/core/config.h index c8ccc07a2..5758132d7 100644 --- a/core/config.h +++ b/core/config.h @@ -445,7 +445,29 @@ #endif /* Reserved bytes to the native thread stack boundary, throw native - stack overflow exception if the guard boudary is reached */ + * stack overflow exception if the guard boudary is reached + * + * WASM_STACK_GUARD_SIZE needs to be large enough for: + * + * - native functions + * w/o hw bound check, the overhead (aot_call_function etc) + the native + * function itself. as of writing this, the former is about 1000 bytes + * on macOS amd64. + * with hw bound check, theoretically, only needs to cover the logic to + * set up the jmp_buf stack. + * + * - aot runtime functions + * eg. aot_enlarge_memory. + * + * - w/o hw bound check, the intepreter loop + * + * - stack check wrapper functions generated by the aot compiler + * (--stack-bounds-checks=1) + * + * Note: on platforms with lazy function binding, don't forget to consider + * the symbol resolution overhead on the first call. For example, + * on Ubuntu amd64 20.04, it seems to consume about 1500 bytes. + */ #ifndef WASM_STACK_GUARD_SIZE #if WASM_ENABLE_UVWASI != 0 /* UVWASI requires larger native stack */