From 5cead96efcc8e0e8399d0423d2dffc823eda68b9 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Wed, 6 Aug 2025 19:11:08 +0800 Subject: [PATCH] Merge branch main into dev/zephyr_file_socket (#4531) --- .github/scripts/run_qemu_arc.sh | 46 + .github/workflows/build_iwasm_release.yml | 132 ++- .../workflows/build_wamr_wasi_extensions.yml | 57 + .github/workflows/build_wamrc.yml | 26 +- .github/workflows/codeql.yml | 6 +- .../compilation_on_android_ubuntu.yml | 44 + .github/workflows/compilation_on_macos.yml | 1 + .github/workflows/compilation_on_sgx.yml | 1 + .github/workflows/compilation_on_zephyr.yml | 129 +++ .github/workflows/nightly_run.yml | 3 + .github/workflows/release_process.yml | 9 + .github/workflows/supply_chain.yml | 2 +- .github/workflows/wamr_wasi_extensions.yml | 57 + CMakeLists.txt | 4 +- README.md | 1 + RELEASE_NOTES.md | 154 +++ build-scripts/config_common.cmake | 26 +- build-scripts/runtime_lib.cmake | 1 + build-scripts/version.cmake | 2 +- ci/coding_guidelines_check.py | 3 +- core/config.h | 10 +- core/iwasm/aot/aot_loader.c | 159 ++- core/iwasm/aot/aot_reloc.h | 8 + core/iwasm/aot/aot_runtime.c | 342 ++++-- core/iwasm/aot/aot_runtime.h | 11 +- core/iwasm/common/arch/fneh.txt | 3 + core/iwasm/common/arch/invokeNative_aarch64.s | 3 + .../common/arch/invokeNative_aarch64_simd.s | 3 + core/iwasm/common/arch/invokeNative_arc.s | 3 + core/iwasm/common/arch/invokeNative_arm.s | 3 + core/iwasm/common/arch/invokeNative_arm_vfp.s | 3 + core/iwasm/common/arch/invokeNative_em64.s | 3 + .../common/arch/invokeNative_em64_simd.s | 3 + core/iwasm/common/arch/invokeNative_ia32.s | 3 + core/iwasm/common/arch/invokeNative_mips.s | 3 + .../common/arch/invokeNative_osx_universal.s | 2 +- core/iwasm/common/arch/invokeNative_thumb.s | 3 + .../common/arch/invokeNative_thumb_vfp.s | 3 + core/iwasm/common/arch/invokeNative_xtensa.s | 3 + core/iwasm/common/wasm_loader_common.c | 15 + core/iwasm/common/wasm_loader_common.h | 5 + core/iwasm/common/wasm_memory.c | 557 +++++++--- core/iwasm/common/wasm_memory.h | 52 +- core/iwasm/common/wasm_runtime_common.c | 110 +- core/iwasm/common/wasm_runtime_common.h | 52 +- core/iwasm/compilation/aot.c | 2 +- core/iwasm/compilation/aot_compiler.c | 7 - core/iwasm/compilation/aot_emit_aot_file.c | 93 +- core/iwasm/compilation/aot_emit_function.c | 3 +- core/iwasm/compilation/aot_emit_memory.c | 752 +++++++++---- core/iwasm/compilation/aot_llvm.c | 181 +++- core/iwasm/compilation/aot_llvm.h | 8 + core/iwasm/fast-jit/fe/jit_emit_table.c | 3 +- core/iwasm/include/aot_comp_option.h | 2 + core/iwasm/include/aot_export.h | 4 +- core/iwasm/include/wasm_export.h | 85 +- core/iwasm/interpreter/wasm.h | 40 +- core/iwasm/interpreter/wasm_interp_classic.c | 50 +- core/iwasm/interpreter/wasm_interp_fast.c | 50 +- core/iwasm/interpreter/wasm_loader.c | 459 ++++++-- core/iwasm/interpreter/wasm_mini_loader.c | 250 ++++- core/iwasm/interpreter/wasm_opcode.h | 5 +- core/iwasm/interpreter/wasm_runtime.c | 298 +++--- core/iwasm/interpreter/wasm_runtime.h | 26 +- .../lib-socket/inc/wasi_socket_ext.h | 25 + .../lib-socket/src/wasi/wasi_socket_ext.c | 184 ++-- .../libraries/libc-wasi/libc_wasi_wrapper.c | 12 + .../libraries/thread-mgr/thread_manager.c | 3 +- .../iwasm/libraries/wasi-nn/include/wasi_nn.h | 8 +- .../libraries/wasi-nn/include/wasi_nn_types.h | 7 + .../wasi-nn/src/utils/wasi_nn_app_native.c | 17 +- core/iwasm/libraries/wasi-nn/src/wasi_nn.c | 341 +++--- .../{wasi_nn_openvino.h => wasi_nn_backend.h} | 23 +- .../libraries/wasi-nn/src/wasi_nn_llamacpp.c | 77 +- .../libraries/wasi-nn/src/wasi_nn_openvino.c | 198 +++- .../libraries/wasi-nn/src/wasi_nn_private.h | 6 +- .../wasi-nn/src/wasi_nn_tensorflowlite.cpp | 107 +- .../wasi-nn/src/wasi_nn_tensorflowlite.hpp | 47 - .../shared/platform/common/posix/posix_file.c | 12 +- .../platform/common/posix/posix_socket.c | 24 +- core/shared/platform/esp-idf/espidf_file.c | 12 +- core/shared/platform/esp-idf/espidf_thread.c | 7 + core/shared/platform/windows/win_file.c | 12 +- core/shared/utils/bh_common.c | 2 +- core/shared/utils/bh_queue.c | 2 +- core/shared/utils/bh_vector.c | 8 +- core/shared/utils/runtime_timer.c | 18 +- core/version.h | 2 +- doc/build_wamr.md | 9 +- doc/build_wasm_app.md | 29 +- doc/security_issue_runbook.md | 46 + doc/security_need_to_know.md | 2 +- doc/socket_api.md | 38 +- product-mini/README.md | 2 +- product-mini/platforms/common/libc_wasi.c | 2 +- product-mini/platforms/posix/main.c | 54 +- product-mini/platforms/windows/main.c | 31 + .../platforms/zephyr/simple/Dockerfile | 59 +- .../platforms/zephyr/simple/west_lite.yml | 4 +- samples/shared-heap/CMakeLists.txt | 30 +- samples/shared-heap/README.md | 50 + .../shared-heap/images/shared_heap_chain.png | Bin 0 -> 111396 bytes samples/shared-heap/src/main.c | 10 +- samples/shared-heap/src/shared_heap_chain.c | 321 ++++++ samples/shared-heap/wasm-apps/CMakeLists.txt | 4 +- samples/shared-heap/wasm-apps/test1.c | 11 + samples/shared-heap/wasm-apps/test2.c | 14 +- samples/socket-api/CMakeLists.txt | 1 + samples/socket-api/README.md | 2 +- samples/socket-api/wasm-src/send_recv.c | 10 +- samples/wasm-c-api-imports/wasm/send_recv.c | 10 +- test-tools/aot-analyzer/include/config.h | 2 +- .../aot-compiler/aot_compiler_fuzz.cc | 1 + .../fuzz/wasm-mutator-fuzz/server/app/main.py | 26 +- tests/regression/ba-issues/README.md | 41 +- tests/regression/ba-issues/run.py | 141 ++- .../regression/ba-issues/running_config.json | 4 +- .../gc-aot/build_spec_interpreter.sh | 4 +- .../test_c_embed_api_thoroughly.py | 14 +- .../test_iwasm_thoroughly.py | 2 +- tests/unit/CMakeLists.txt | 28 +- tests/unit/memory64/memory64_atomic_test.cc | 4 +- tests/unit/memory64/memory64_test.cc | 10 +- .../running-modes/wasm_running_modes_test.cc | 24 +- tests/unit/shared-heap/CMakeLists.txt | 12 +- tests/unit/shared-heap/shared_heap_test.cc | 985 ++++++++++++++++-- .../unit/shared-heap/wasm-apps/CMakeLists.txt | 107 +- .../bulk-memory/test_bulk_memory.wasm | Bin 0 -> 63 bytes .../bulk-memory/test_bulk_memory.wat | 12 + .../wasm-apps/memory64/CMakeLists.txt | 68 ++ tests/unit/shared-heap/wasm-apps/test.c | 30 +- .../shared-heap/wasm-apps/test_addr_conv.c | 14 + .../wamr-test-suites/spec-test-script/all.py | 16 + .../spec-test-script/extended_const.patch | 506 +++++++++ .../spec-test-script/runtest.py | 8 + tests/wamr-test-suites/test_wamr.sh | 30 +- wamr-compiler/CMakeLists.txt | 3 + wamr-compiler/build_llvm.sh | 14 +- wamr-compiler/build_llvm_arc.sh | 4 +- wamr-compiler/build_llvm_xtensa.sh | 4 +- wamr-compiler/main.c | 18 +- wamr-wasi-extensions/README.md | 49 + wamr-wasi-extensions/build_libs.sh | 15 + wamr-wasi-extensions/build_samples.sh | 33 + wamr-wasi-extensions/samples/nn-cli/main.c | 42 +- wamr-wasi-extensions/samples/nn/app.c | 3 +- wamr-wasi-extensions/socket/CMakeLists.txt | 6 + wamr-wasi-extensions/test.sh | 34 +- 148 files changed, 6804 insertions(+), 1787 deletions(-) create mode 100755 .github/scripts/run_qemu_arc.sh create mode 100644 .github/workflows/build_wamr_wasi_extensions.yml create mode 100644 .github/workflows/compilation_on_zephyr.yml create mode 100644 .github/workflows/wamr_wasi_extensions.yml create mode 100644 core/iwasm/common/arch/fneh.txt rename core/iwasm/libraries/wasi-nn/src/{wasi_nn_openvino.h => wasi_nn_backend.h} (64%) delete mode 100644 core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.hpp create mode 100644 doc/security_issue_runbook.md create mode 100644 samples/shared-heap/README.md create mode 100644 samples/shared-heap/images/shared_heap_chain.png create mode 100644 samples/shared-heap/src/shared_heap_chain.c create mode 100644 tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wasm create mode 100644 tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat create mode 100644 tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt create mode 100644 tests/wamr-test-suites/spec-test-script/extended_const.patch create mode 100644 wamr-wasi-extensions/README.md create mode 100755 wamr-wasi-extensions/build_libs.sh create mode 100755 wamr-wasi-extensions/build_samples.sh diff --git a/.github/scripts/run_qemu_arc.sh b/.github/scripts/run_qemu_arc.sh new file mode 100755 index 000000000..ab226521b --- /dev/null +++ b/.github/scripts/run_qemu_arc.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# THIS SCRIPT IS USED TO RUN QEMU ARC EMULATOR DIRECTLY ON CI ONLY. +# USUALLY, you SHOULD NOT RUN IT ON YOUR LOCAL MACHINE. +# INSTEAD, YOU SHOULD USE `west build -t run` COMMAND. + +# Two arguments. first is the path to the zephyr-sdk, second is the path to the zephyr elf. +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +ZEPHYR_SDK_PATH=$1 +ZEPHYR_ELF_PATH=$2 + +if [ ! -d "$ZEPHYR_SDK_PATH" ]; then + echo "Error: Zephyr SDK path does not exist: $ZEPHYR_SDK_PATH" + exit 1 +fi +if [ ! -f "$ZEPHYR_ELF_PATH" ]; then + echo "Error: Zephyr ELF file does not exist: $ZEPHYR_ELF_PATH" + exit 1 +fi + +# this command is copied from the content of build.ninja which is generated by west build. +# please do not modify it unless synchronizing with the build.ninja file. +$ZEPHYR_SDK_PATH/sysroots/x86_64-pokysdk-linux/usr/bin/qemu-system-arc \ + -cpu arcem -m 8M \ + -nographic -no-reboot -monitor none \ + -global cpu.firq=false \ + -global cpu.num-irqlevels=15 \ + -global cpu.num-irq=25 \ + -global cpu.ext-irq=20 \ + -global cpu.freq_hz=10000000 \ + -global cpu.timer0=true \ + -global cpu.timer1=true \ + -global cpu.has-mpu=true \ + -global cpu.mpu-numreg=16 \ + -net none \ + -pidfile qemu.pid \ + -chardev stdio,id=con,mux=on \ + -serial chardev:con \ + -mon chardev=con,mode=readline \ + -icount shift=6,align=off,sleep=off \ + -rtc clock=vm \ + -kernel $ZEPHYR_ELF_PATH diff --git a/.github/workflows/build_iwasm_release.yml b/.github/workflows/build_iwasm_release.yml index a975d5807..d1bdd685d 100644 --- a/.github/workflows/build_iwasm_release.yml +++ b/.github/workflows/build_iwasm_release.yml @@ -23,7 +23,7 @@ on: type: string required: true upload_url: - description: a semantic version number. it is required when `release` is true. + description: upload binary assets to the URL of release type: string required: false ver_num: @@ -32,60 +32,60 @@ on: required: false env: - DEFAULT_BUILD_OPTIONS: + DEFAULT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 \ - -DWAMR_BUILD_CUSTOM_NAME_SECTION=0 \ - -DWAMR_BUILD_DEBUG_INTERP=0 \ - -DWAMR_BUILD_DEBUG_AOT=0 \ - -DWAMR_BUILD_DUMP_CALL_STACK=0 \ - -DWAMR_BUILD_LIBC_UVWASI=0 \ - -DWAMR_BUILD_LIBC_EMCC=0 \ - -DWAMR_BUILD_LIB_RATS=0 \ - -DWAMR_BUILD_LOAD_CUSTOM_SECTION=0 \ - -DWAMR_BUILD_MEMORY_PROFILING=0 \ - -DWAMR_BUILD_MINI_LOADER=0 \ - -DWAMR_BUILD_MULTI_MODULE=0 \ - -DWAMR_BUILD_PERF_PROFILING=0 \ - -DWAMR_BUILD_SPEC_TEST=0 \ - -DWAMR_BUILD_BULK_MEMORY=1 \ - -DWAMR_BUILD_LIB_PTHREAD=1 \ - -DWAMR_BUILD_LIB_PTHREAD_SEMAPHORE=1 \ - -DWAMR_BUILD_LIB_WASI_THREADS=1 \ - -DWAMR_BUILD_LIBC_BUILTIN=1 \ - -DWAMR_BUILD_LIBC_WASI=1 \ - -DWAMR_BUILD_REF_TYPES=1 \ - -DWAMR_BUILD_SIMD=1 \ - -DWAMR_BUILD_SHARED_MEMORY=1 \ - -DWAMR_BUILD_TAIL_CALL=1 \ - -DWAMR_BUILD_THREAD_MGR=1" + -DWAMR_BUILD_CUSTOM_NAME_SECTION=0 \ + -DWAMR_BUILD_DEBUG_INTERP=0 \ + -DWAMR_BUILD_DEBUG_AOT=0 \ + -DWAMR_BUILD_DUMP_CALL_STACK=0 \ + -DWAMR_BUILD_LIBC_UVWASI=0 \ + -DWAMR_BUILD_LIBC_EMCC=0 \ + -DWAMR_BUILD_LIB_RATS=0 \ + -DWAMR_BUILD_LOAD_CUSTOM_SECTION=0 \ + -DWAMR_BUILD_MEMORY_PROFILING=0 \ + -DWAMR_BUILD_MINI_LOADER=0 \ + -DWAMR_BUILD_MULTI_MODULE=0 \ + -DWAMR_BUILD_PERF_PROFILING=0 \ + -DWAMR_BUILD_SPEC_TEST=0 \ + -DWAMR_BUILD_BULK_MEMORY=1 \ + -DWAMR_BUILD_LIB_PTHREAD=1 \ + -DWAMR_BUILD_LIB_PTHREAD_SEMAPHORE=1 \ + -DWAMR_BUILD_LIB_WASI_THREADS=1 \ + -DWAMR_BUILD_LIBC_BUILTIN=1 \ + -DWAMR_BUILD_LIBC_WASI=1 \ + -DWAMR_BUILD_REF_TYPES=1 \ + -DWAMR_BUILD_SIMD=1 \ + -DWAMR_BUILD_SHARED_MEMORY=1 \ + -DWAMR_BUILD_TAIL_CALL=1 \ + -DWAMR_BUILD_THREAD_MGR=1" GC_EH_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_FAST_JIT=0 -DWAMR_BUILD_JIT=0 \ - -DWAMR_BUILD_CUSTOM_NAME_SECTION=0 \ - -DWAMR_BUILD_DEBUG_INTERP=0 \ - -DWAMR_BUILD_DEBUG_AOT=0 \ - -DWAMR_BUILD_DUMP_CALL_STACK=0 \ - -DWAMR_BUILD_LIBC_UVWASI=0 \ - -DWAMR_BUILD_LIBC_EMCC=0 \ - -DWAMR_BUILD_LIB_RATS=0 \ - -DWAMR_BUILD_LOAD_CUSTOM_SECTION=0 \ - -DWAMR_BUILD_MEMORY_PROFILING=0 \ - -DWAMR_BUILD_MINI_LOADER=0 \ - -DWAMR_BUILD_MULTI_MODULE=0 \ - -DWAMR_BUILD_PERF_PROFILING=0 \ - -DWAMR_BUILD_SPEC_TEST=0 \ - -DWAMR_BUILD_BULK_MEMORY=1 \ - -DWAMR_BUILD_LIB_PTHREAD=1 \ - -DWAMR_BUILD_LIB_PTHREAD_SEMAPHORE=1 \ - -DWAMR_BUILD_LIB_WASI_THREADS=1 \ - -DWAMR_BUILD_LIBC_BUILTIN=1 \ - -DWAMR_BUILD_LIBC_WASI=1 \ - -DWAMR_BUILD_REF_TYPES=1 \ - -DWAMR_BUILD_SIMD=1 \ - -DWAMR_BUILD_SHARED_MEMORY=1 \ - -DWAMR_BUILD_TAIL_CALL=1 \ - -DWAMR_BUILD_THREAD_MGR=1 \ - -DWAMR_BUILD_EXCE_HANDLING=1 \ - -DWAMR_BUILD_GC=1" + -DWAMR_BUILD_CUSTOM_NAME_SECTION=0 \ + -DWAMR_BUILD_DEBUG_INTERP=0 \ + -DWAMR_BUILD_DEBUG_AOT=0 \ + -DWAMR_BUILD_DUMP_CALL_STACK=0 \ + -DWAMR_BUILD_LIBC_UVWASI=0 \ + -DWAMR_BUILD_LIBC_EMCC=0 \ + -DWAMR_BUILD_LIB_RATS=0 \ + -DWAMR_BUILD_LOAD_CUSTOM_SECTION=0 \ + -DWAMR_BUILD_MEMORY_PROFILING=0 \ + -DWAMR_BUILD_MINI_LOADER=0 \ + -DWAMR_BUILD_MULTI_MODULE=0 \ + -DWAMR_BUILD_PERF_PROFILING=0 \ + -DWAMR_BUILD_SPEC_TEST=0 \ + -DWAMR_BUILD_BULK_MEMORY=1 \ + -DWAMR_BUILD_LIB_PTHREAD=1 \ + -DWAMR_BUILD_LIB_PTHREAD_SEMAPHORE=1 \ + -DWAMR_BUILD_LIB_WASI_THREADS=1 \ + -DWAMR_BUILD_LIBC_BUILTIN=1 \ + -DWAMR_BUILD_LIBC_WASI=1 \ + -DWAMR_BUILD_REF_TYPES=1 \ + -DWAMR_BUILD_SIMD=1 \ + -DWAMR_BUILD_SHARED_MEMORY=1 \ + -DWAMR_BUILD_TAIL_CALL=1 \ + -DWAMR_BUILD_THREAD_MGR=1 \ + -DWAMR_BUILD_EXCE_HANDLING=1 \ + -DWAMR_BUILD_GC=1" permissions: contents: read @@ -97,9 +97,9 @@ jobs: matrix: include: - build_options: $DEFAULT_BUILD_OPTIONS - suffix: '' + suffix: "" - build_options: $GC_EH_BUILD_OPTIONS - suffix: '-gc-eh' + suffix: "-gc-eh" permissions: contents: write # for uploading release artifacts @@ -126,6 +126,30 @@ jobs: cmake --build build --config Release --parallel 4 working-directory: ${{ inputs.cwd }} + - name: smoke test on non-Windows + if: ${{ !startsWith(inputs.runner, 'windows') }} + shell: bash + run: | + if [[ ! -f build/iwasm ]]; then + echo "iwasm binary is not found in the expected location." + exit 1 + fi + + build/iwasm --version + working-directory: ${{ inputs.cwd }} + + - name: smoke test on Windows + if: ${{ startsWith(inputs.runner, 'windows') }} + shell: bash + run: | + if [[ ! -f build/Release/iwasm ]]; then + echo "iwasm binary is not found in the expected location." + exit 1 + fi + + build/Release/iwasm --version + working-directory: ${{ inputs.cwd }} + - name: Compress the binary on Windows if: inputs.runner == 'windows-latest' run: | diff --git a/.github/workflows/build_wamr_wasi_extensions.yml b/.github/workflows/build_wamr_wasi_extensions.yml new file mode 100644 index 000000000..5412d794d --- /dev/null +++ b/.github/workflows/build_wamr_wasi_extensions.yml @@ -0,0 +1,57 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: build wamr_wasi_extensions release + +on: + workflow_call: + inputs: + upload_url: + description: upload binary assets to the URL of release + type: string + required: false + ver_num: + description: a semantic version number. it is required when `release` is true. + type: string + required: false + +permissions: + contents: read + +jobs: + build_wamr_wasi_extensions: + runs-on: ${{ matrix.os }} + permissions: + contents: write # for uploading release artifacts + strategy: + matrix: + os: [ubuntu-22.04] + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: install-wasi-sdk-wabt + uses: ./.github/actions/install-wasi-sdk-wabt + with: + os: ${{ matrix.os }} + + - name: Build wamr-wasi-extensions + run: | + mkdir dist + ./build_libs.sh $(pwd)/dist/wamr-wasi-extensions + working-directory: wamr-wasi-extensions + + - name: Compress the binary + run: | + zip -r wamr-wasi-extensions-${{ inputs.ver_num }}.zip wamr-wasi-extensions + working-directory: wamr-wasi-extensions/dist + + - name: Upload release zip + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ inputs.upload_url }} + asset_path: wamr-wasi-extensions/dist/wamr-wasi-extensions-${{ inputs.ver_num }}.zip + asset_name: wamr-wasi-extensions-${{ inputs.ver_num }}.zip + asset_content_type: application/zip diff --git a/.github/workflows/build_wamrc.yml b/.github/workflows/build_wamrc.yml index 55d63f13b..df257723f 100644 --- a/.github/workflows/build_wamrc.yml +++ b/.github/workflows/build_wamrc.yml @@ -23,7 +23,7 @@ on: type: string required: true upload_url: - description: a semantic version number. it is required when `release` is true. + description: upload binary assets to the URL of release type: string required: false ver_num: @@ -62,6 +62,30 @@ jobs: cmake --build build --config Release --parallel 4 working-directory: wamr-compiler + - name: smoke test on non-windows + if: ${{ !startsWith(inputs.runner, 'windows') }} + shell: bash + run: | + if [[ ! -f build/wamrc ]]; then + echo "wamrc binary is not found in the expected location." + exit 1 + fi + + build/wamrc --version + working-directory: wamr-compiler + + - name: smoke test on Windows + if: ${{ startsWith(inputs.runner, 'windows') }} + shell: bash + run: | + if [[ ! -f build/Release/wamrc ]]; then + echo "wamrc binary is not found in the expected location." + exit 1 + fi + + build/Release/wamrc --version + working-directory: wamr-compiler + - name: Compress the binary on Windows if: inputs.runner == 'windows-latest' && inputs.release run: | diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 9c3309721..fea92c9f6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -53,7 +53,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3.29.0 + uses: github/codeql-action/init@v3.29.4 with: languages: ${{ matrix.language }} @@ -70,7 +70,7 @@ jobs: - run: | ./.github/scripts/codeql_buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3.29.0 + uses: github/codeql-action/analyze@v3.29.4 with: category: "/language:${{matrix.language}}" upload: false @@ -99,7 +99,7 @@ jobs: output: ${{ steps.step1.outputs.sarif-output }}/cpp.sarif - name: Upload CodeQL results to code scanning - uses: github/codeql-action/upload-sarif@v3.29.0 + uses: github/codeql-action/upload-sarif@v3.29.4 with: sarif_file: ${{ steps.step1.outputs.sarif-output }} category: "/language:${{matrix.language}}" diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 0fbb09e7d..358a0d93f 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -69,6 +69,7 @@ env: GC_TEST_OPTIONS: "-s spec -G -b -P" MEMORY64_TEST_OPTIONS: "-s spec -W -b -P" MULTI_MEMORY_TEST_OPTIONS: "-s spec -E -b -P" + EXTENDED_CONST_EXPR_TEST_OPTIONS: "-s spec -N -b -P" permissions: contents: read @@ -164,6 +165,7 @@ jobs: "-DWAMR_BUILD_MEMORY64=1", "-DWAMR_BUILD_MULTI_MEMORY=1", "-DWAMR_BUILD_SHARED=1", + "-DWAMR_BUILD_EXTENDED_CONST_EXPR=1", ] os: [ubuntu-22.04] platform: [android, linux] @@ -369,6 +371,47 @@ jobs: ctest working-directory: tests/unit + build_regression_tests: + needs: + [build_llvm_libraries_on_ubuntu_2204] + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-22.04] + include: + - os: ubuntu-22.04 + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Get LLVM libraries + id: retrieve_llvm_libs + uses: actions/cache@v4 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ matrix.llvm_cache_key }} + + - name: Quit if cache miss + if: (steps.retrieve_llvm_libs.outputs.cache-hit != 'true') + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + + - name: Build wamrc and iwasm + run: | + ./build_wamr.sh + working-directory: tests/regression/ba-issues + + - name: Run regression tests + run: | + python run.py + working-directory: tests/regression/ba-issues + build_samples_wasm_c_api: needs: [ @@ -609,6 +652,7 @@ jobs: $GC_TEST_OPTIONS, $MEMORY64_TEST_OPTIONS, $MULTI_MEMORY_TEST_OPTIONS, + $EXTENDED_CONST_EXPR_TEST_OPTIONS, ] include: - os: ubuntu-22.04 diff --git a/.github/workflows/compilation_on_macos.yml b/.github/workflows/compilation_on_macos.yml index 44c7973f1..462e273bf 100644 --- a/.github/workflows/compilation_on_macos.yml +++ b/.github/workflows/compilation_on_macos.yml @@ -142,6 +142,7 @@ jobs: "-DWAMR_BUILD_SIMD=1", "-DWAMR_BUILD_TAIL_CALL=1", "-DWAMR_DISABLE_HW_BOUND_CHECK=1", + "-DWAMR_BUILD_EXTENDED_CONST_EXPR=1", ] os: [macos-13] platform: [darwin] diff --git a/.github/workflows/compilation_on_sgx.yml b/.github/workflows/compilation_on_sgx.yml index e6cc29108..7577b0bda 100644 --- a/.github/workflows/compilation_on_sgx.yml +++ b/.github/workflows/compilation_on_sgx.yml @@ -100,6 +100,7 @@ jobs: "-DWAMR_BUILD_MULTI_MODULE=1", "-DWAMR_BUILD_PERF_PROFILING=1", "-DWAMR_BUILD_REF_TYPES=1", + "-DWAMR_BUILD_EXTENDED_CONST_EXPR=1", # doesn't support "-DWAMR_BUILD_SIMD=0", "-DWAMR_BUILD_TAIL_CALL=1", diff --git a/.github/workflows/compilation_on_zephyr.yml b/.github/workflows/compilation_on_zephyr.yml new file mode 100644 index 000000000..274fc3740 --- /dev/null +++ b/.github/workflows/compilation_on_zephyr.yml @@ -0,0 +1,129 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: compilation on zephyr + +on: + # will be triggered on PR events + pull_request: + types: + - opened + - synchronize + paths: + - ".github/**" + - "build-scripts/**" + - "core/**" + - "!core/deps/**" + - "product-mini/platforms/common/**" + - "product-mini/platforms/zephyr/**" + - "samples/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + # will be triggered on push events + push: + branches: + - main + - "dev/**" + paths: + - ".github/**" + - "build-scripts/**" + - "core/**" + - "!core/deps/**" + - "product-mini/platforms/common/**" + - "product-mini/platforms/zephyr/**" + - "samples/**" + - "!samples/workload/**" + - "tests/wamr-test-suites/**" + - "wamr-compiler/**" + # allow to be triggered manually + workflow_dispatch: + +# Cancel any in-flight jobs for the same PR/branch so there's only one active +# at a time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # FOR SETUP + ZEPHYR_SDK_VERSION: "0.16.9" + ZEPHYR_VERSION: "v3.7.0" + # FOR BUILD + +permissions: + contents: read + +jobs: + smoke_test: + runs-on: ubuntu-22.04 + container: + # For Zephyr 3.7 LTS, use the v0.26-branch or the latest v0.26.x release Docker image. + # ci require a larger runner to avoid "no space left on device" + image: ghcr.io/zephyrproject-rtos/ci-base:v0.26-branch + options: --user root + + steps: + # https://docs.zephyrproject.org/latest/develop/application/index.html#zephyr-workspace-application + # zephyrproject/ --> CI ROOT + # ├─── .west/ + # │ └─── config + # ├─── bootloader/ + # ├─── zephyr/ --> Zephyr source code + # ├─── zephyr-sdk/ + # ├─── modules/ + # │ |─── wasm-micro-runtime --> WAMR source code + # ├─── tools/ + # ├─── vendor/ + # └─── application/ --> DUMMY. keep west_lite.yml here + + - name: Checkout code + uses: actions/checkout@v3 + with: + path: modules/wasm-micro-runtime + + - name: Prepare Zephyr environment + shell: bash + run: | + mkdir -p application + cp modules/wasm-micro-runtime/product-mini/platforms/zephyr/simple/west_lite.yml application/west_lite.yml + + - name: Setup Zephyr project + uses: zephyrproject-rtos/action-zephyr-setup@v1 + with: + app-path: application + manifest-file-name: west_lite.yml + sdk-version: ${{ env.ZEPHYR_SDK_VERSION }} + toolchains: arc-zephyr-elf:arc64-zephyr-elf + + - name: Build a sample application(simple) + shell: bash + run: | + pushd product-mini/platforms/zephyr/simple + west build . -b qemu_arc/qemu_arc_hs -p always -- -DWAMR_BUILD_TARGET=ARC + popd + + # west build -t run will fork several processes, which will cause the job to hang. + # run in the background and kill it after 5 seconds + .github/scripts/run_qemu_arc.sh \ + ../../zephyr-sdk \ + product-mini/platforms/zephyr/simple/build/zephyr/zephyr.elf & + sleep 5 + pkill qemu-system-arc + working-directory: modules/wasm-micro-runtime + + - name: Build a sample application(user-mode) + shell: bash + run: | + pushd product-mini/platforms/zephyr/user-mode + west build . -b qemu_arc/qemu_arc_hs -p always -- -DWAMR_BUILD_TARGET=ARC + popd + + # west build -t run will fork several processes, which will cause the job to hang. + # run in the background and kill it after 5 seconds + .github/scripts/run_qemu_arc.sh \ + ../../zephyr-sdk \ + product-mini/platforms/zephyr/user-mode/build/zephyr/zephyr.elf & + sleep 5 + pkill qemu-system-arc + working-directory: modules/wasm-micro-runtime diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index 1ae405f60..b2bd04d1d 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -37,6 +37,7 @@ env: MULTI_TIER_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1" # For Spec Test DEFAULT_TEST_OPTIONS: "-s spec -b -P" + EXTENDED_CONST_EXPR_TEST_OPTIONS: "-s spec -b -P -N" MULTI_MODULES_TEST_OPTIONS: "-s spec -b -P -M" SIMD_TEST_OPTIONS: "-s spec -b -P -S" THREADS_TEST_OPTIONS: "-s spec -b -P -p" @@ -128,6 +129,7 @@ jobs: "-DWAMR_BUILD_MEMORY64=1", "-DWAMR_BUILD_MULTI_MEMORY=1", "-DWAMR_BUILD_SHARED=1", + "-DWAMR_BUILD_EXTENDED_CONST_EXPR=1", ] os: [ubuntu-22.04] platform: [android, linux] @@ -588,6 +590,7 @@ jobs: $DEFAULT_TEST_OPTIONS, $MULTI_MODULES_TEST_OPTIONS, $SIMD_TEST_OPTIONS, + $EXTENDED_CONST_EXPR_TEST_OPTIONS, $THREADS_TEST_OPTIONS, $WASI_TEST_OPTIONS, ] diff --git a/.github/workflows/release_process.yml b/.github/workflows/release_process.yml index 91f081fd2..bb99681dc 100644 --- a/.github/workflows/release_process.yml +++ b/.github/workflows/release_process.yml @@ -239,3 +239,12 @@ jobs: arch: universal upload_url: ${{ needs.create_release.outputs.upload_url }} ver_num: ${{ needs.create_tag.outputs.new_ver}} + + release_wamr_wasi_extensions: + permissions: + contents: write # upload release artifact + needs: [create_tag, create_release] + uses: ./.github/workflows/build_wamr_wasi_extensions.yml + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + ver_num: ${{ needs.create_tag.outputs.new_ver }} diff --git a/.github/workflows/supply_chain.yml b/.github/workflows/supply_chain.yml index 827b300f1..57e59d0dd 100644 --- a/.github/workflows/supply_chain.yml +++ b/.github/workflows/supply_chain.yml @@ -60,6 +60,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@2847b7f7ab9f48fc49eca90a53fff6007285f399 + uses: github/codeql-action/upload-sarif@701df0e49d84a24bd8f0d01f80c0dbf69ab07674 with: sarif_file: results.sarif diff --git a/.github/workflows/wamr_wasi_extensions.yml b/.github/workflows/wamr_wasi_extensions.yml new file mode 100644 index 000000000..e9d10fe93 --- /dev/null +++ b/.github/workflows/wamr_wasi_extensions.yml @@ -0,0 +1,57 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: wamr_wasi_extensions + +on: + pull_request: + types: + - opened + - synchronize + paths: + - ".github/workflows/wamr_wasi_extensions.yml" + - "wamr_wasi_extensios/**" + - "core/iwasm/libraries/wasi-nn/include/**" + - "core/iwasm/libraries/lib-socket/**" + # allow to be triggered manually + workflow_dispatch: + +# Cancel any in-flight jobs for the same PR/branch so there's only one active +# at a time +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_wamr_wasi_extensions: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-22.04, macos-13, macos-14] + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: install-wasi-sdk-wabt + uses: ./.github/actions/install-wasi-sdk-wabt + with: + os: ${{ matrix.os }} + + - name: Build wamr-wasi-extensions + run: | + mkdir dist + ./build_libs.sh $(pwd)/dist/wamr-wasi-extensions + working-directory: wamr-wasi-extensions + + - name: Build wamr-wasi-extensions samples + run: | + ./build_samples.sh $(pwd)/dist/wamr-wasi-extensions + working-directory: wamr-wasi-extensions + + - name: Upload artifacts + if: matrix.os == 'macos-14' + uses: actions/upload-artifact@v4 + with: + name: wamr-wasi-extensions + path: wamr-wasi-extensions/dist + retention-days: 10 diff --git a/CMakeLists.txt b/CMakeLists.txt index 88a1642b8..4b28fa89c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,9 +99,9 @@ if (NOT DEFINED WAMR_BUILD_LIB_WASI_THREADS) set (WAMR_BUILD_LIB_WASI_THREADS 0) endif () -if (NOT DEFINED WAMR_ENABLE_COPY_CALLSTACK) +if (NOT DEFINED WAMR_BUILD_COPY_CALL_STACK) # Disable copy callstack by default - set (WAMR_ENABLE_COPY_CALLSTACK 0) + set (WAMR_BUILD_COPY_CALL_STACK 0) endif() if (NOT DEFINED WAMR_BUILD_MINI_LOADER) diff --git a/README.md b/README.md index 39ce5fd03..859ee3540 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ WebAssembly Micro Runtime (WAMR) is a lightweight standalone WebAssembly (Wasm) - [Reference Types](https://github.com/WebAssembly/reference-types), ref to [document](doc/ref_types.md) and [sample](samples/ref-types) - [Bulk memory operations](https://github.com/WebAssembly/bulk-memory-operations), [Shared memory](https://github.com/WebAssembly/threads/blob/main/proposals/threads/Overview.md#shared-linear-memory), [Memory64](https://github.com/WebAssembly/memory64) - [Tail-call](https://github.com/WebAssembly/tail-call), [Garbage Collection](https://github.com/WebAssembly/gc), [Exception Handling](https://github.com/WebAssembly/exception-handling) +- [Extended Constant Expressions](https://github.com/WebAssembly/extended-const) ### Supported architectures and platforms The WAMR VMcore supports the following architectures: diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 167da4703..4904d3949 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,157 @@ +## WAMR-2.4.1 + +### Breaking Changes + +- wasi_socket_ext.c: fix error reporting (#4476) +- lib-socket: make getaddrinfo return EAI\_ values (#4498) +- bump AOT_CURRENT_VERSION for extended-const (#4511) + +### New Features + +### Bug Fixes + +- modify macro related to simde when WASM_OP_SELECT_128 (#4461) +- posix os_socket_accept: stop assuming socklen_t is unsigned int (#4488) +- wasi_socket_ext.c: fix the number of getaddrinfo results (#4466) +- Fix typos (#4472) +- fix regression running_config.json (#4477) +- posix os_socket_addr_resolve: relax compatibility check (#4469) +- Add validation for recursive type count in loader (#4440) + +### Enhancements + +- Add CI on Zephyr (#4336) +- introduce wasm_runtime_instantiate_ex2 (#4444) +- Add a CLI option to specify shared heap size on Windows platform (#4503) +- wasi-nn: add a comment on load_by_name_with_config (#4492) +- nn-cli: add an option to use load_by_name (#4490) +- wamr-wasi-extensions: document (#4493) +- doc/socket_api.md: some historical notes (#4494) +- lib-socket: implement getsockopt(SOL_SOCKET,SO_TYPE) (#4458) + +### Others + +- build(deps): Bump github/codeql-action from 3.29.2 to 3.29.3 (#4507) + +## WAMR-2.4.0 + +### Breaking Changes + +- Refactor copy callstack feature (#4401) +- Enable WAMR_BUILD_WASI_EPHEMERAL_NN by default (#4381) +- Enable aot memory64 sw bounds checks by default (#4350) + +### New Features + +- Support extended constant expressions (#4432) +- Shared heap enhancements for Interpreter and AOT (#4400) + +### Bug Fixes + +- posix os_socket_addr_resolve: return the consistent max_info_size (#4467) +- fix a wamrc debug mode compile issue (#4470) +- wasi-nn: do not pretend to support legacy abi in openvino and llamacpp (#4468) +- appease a few compiler warnings (-Wstrict-prototypes) (#4465) +- enable aux stack frame for aot compiler fuzz test (#4462) +- improve logic of `heap_type` validation when `ref.null` (#4372) +- wasi_nn_llamacpp.c: explicitly reject unimplemented input index (#4446) +- wasi: avoid user-triggerable 0-sized allocations (#4452) +- Fix socket shutdown (#12) (#4449) +- wasi_nn_llamacpp.c: validate input tensor type/dimensions (#4442) +- wasi_nn_llamacpp.c: reject invalid graph and execution context (#4422) +- wasi_nn_openvino.c: avoid self-assignment warning (#4434) +- Fix potential integer overflow issues (#4429) +- Improve run.py of regression (#4417) +- wasi-nn: reduce code duplication a bit (#4433) +- Refactor AOTObjectData definition to use a forward declaration (#4428) +- CI: revert SGX retry attempts (#4421) +- loader: fix a potential overflow issue (#4427) +- wasi_nn_openvino.c: fix a debug build (#4416) +- Fix few shadow warnings (#4409) +- wasi_nn_llamacpp.c: remove an unused variable (#4415) +- wasi_nn_llamacpp.c: fix buffer overruns in set_input (#4420) +- wasi-nn: make the host use the wasi_ephemeral_nn version of tensor_data (#4411) +- Collective fix (#4413) +- fix bug in bh_vector when extending (#4414) +- wasi_nn_llamacpp.c: make this compilable (#4403) +- Fix handling of non-nullable global_type during global import (#4408) +- loader: add type index checking (#4402) +- wasi_nn_tensorflowlite.cpp: fix get_output return size (#4390) +- wasi-nn: fix context lifetime issues (#4396) +- CI: fix the description of upload_url (#4407) +- wamr-wasi-extensions/socket: disable reference-types (#4392) +- wasi_nn_openvino.c: implement multiple models per instance (#4380) +- Improve spec test execution by adding retry logic for transient errors (#4393) +- wasi-nn: add minimum serialization on WASINNContext (#4387) +- deprecate legacy WAMR-specific "wasi_nn" module (#4382) +- wasi-nn: fix tensor_data abi for wasi_ephemeral_nn (#4379) +- core/iwasm/libraries/wasi-nn/test: use the correct version of keras (#4383) +- Fix several issues related to night-run CI and test scripts. (#4385) +- wasi_nn_tensorflowlite.cpp: reject non-fp32 input earlier (#4388) +- core/iwasm/libraries/wasi-nn/test/build.sh: add a tip for intel mac (#4389) +- wasi-nn: don't try to deinit uninitialized backend (#4375) +- wasi-nn: apply the shared library hack to darwin as well (#4374) +- add nn-cli example (#4373) +- wasi_nn_openvino.c: remove pre/postprocessing and layout assumptions (#4361) +- send an empty/error reply from server (#4362) +- wasi_nn_openvino.c: add a missing buffer overflow check in get_output (#4353) +- wasi_ephemeral_nn.h: prefix identfiers to avoid too generic names (#4358) +- wamr-wasi-extensions: add lib-socket things (#4360) +- wasi_nn_openvino.c: remove broken xml check (#4365) +- add validation for array type in load_init_expr(GC only) (#4370) +- wasi-nn: fix backend leak on multiple loads (#4366) +- Collective fix for typos and minor bugs (#4369) +- Modify AOT static PGO to conform to llvm-18 and add a CI job to test static PGO on the coremark benchmark (#4345) +- Update WABT downloads URL (#4357) +- clean up incompatible running mode checks in test script and ci (#4342) +- Follow #4268 to deprecate wamr_ide-related components (#4341) +- Update type validation in load_table_import() and load_table() (#4296) +- wasi_nn_openvino.c: remove the tensor layout adjustment logic (#4308) +- add heap-type check for GC when ref.null (#4300) +- wasi_nn_types.h: remove a seemingly stale comment (#4348) +- wasi_socket_ext.c: avoid tls to make this library-friendly (#4338) +- wasi-nn: do not assign wasi_nn_ctx->backend multiple times (#4329) +- wasi_nn.h: make this compatible with wasi_ephemeral_nn (#4330) +- remove temporary wasi-libc build steps from CI workflows (#4343) +- wasi-nn: fix the size of tensor->type (#4333) +- wasi-nn: move some host-only things out of wasi_nn_types.h (#4334) +- Collective fix: fix some typos (#4337) +- Update binary compression steps to follow symlinks for actual files (#4321) +- Add wamrc compilation into Windows CI workflow (#4327) +- wasi-nn: remove unused wasi_nn_dump_tensor_dimension prototype (#4325) +- wasi_nn.h: add import_name attribute (#4328) +- wasi-nn: protect the backend lookup table with a lock (#4319) +- handle nullable heap reference types in import section (#4302) +- wasi_nn_openvino.c: make this buildable (#4305) +- wasi-nn: fix shared library filenames for macOS (#4306) +- fix wasi-nn abi definitions (#4307) +- wasi-nn: remove "backends" argument from detect_and_load_backend() (#4309) +- wasi_nn_openvino.c: fix a few printf formats (#4310) +- Bump uvwasi to latest commit #392e1f1 (#4312) + +### Enhancements + +- Add readme for extended const (#4471) +- Add security issue runbook (#4450) +- docs: fix cmake variable typo (#4441) +- CI: add wamr_wasi_extensions to the release assets (#4425) +- CI: build wamr-wasi-extensions (#4394) +- improve installation steps for wasi-sdk and wabt on Windows (#4359) +- wamr-wasi-extensions: add a cmake package to provide our wasi extension (#4344) +- Update Dockerfile for Zephyr SDK and Zephyr-project versioning (#4335) +- add load_by_name in wasi-nn (#4298) + +### Others + +- build(deps): Bump ossf/scorecard-action from 2.4.1 to 2.4.2 (#4315) +- build(deps): Bump github/codeql-action from 3.29.1 to 3.29.2 (#4459) +- build(deps): Bump github/codeql-action from 3.29.0 to 3.29.1 (#4436) +- build(deps): Bump github/codeql-action from 3.28.19 to 3.29.0 (#4371) +- build(deps): Bump github/codeql-action from 3.28.18 to 3.28.19 (#4346) +- build(deps): Bump requests from 2.32.3 to 2.32.4 in /build-scripts (#4349) + +--- + ## WAMR-2.3.1 ### Breaking Changes diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 1a77d4cac..8cdce4a01 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -211,6 +211,10 @@ if (NOT DEFINED WAMR_BUILD_TAIL_CALL) set (WAMR_BUILD_TAIL_CALL 0) endif () +if (NOT DEFINED WAMR_BUILD_EXTENDED_CONST_EXPR) + set (WAMR_BUILD_EXTENDED_CONST_EXPR 0) +endif () + ######################################## # Compilation options to marco ######################################## @@ -334,15 +338,10 @@ if (WAMR_BUILD_SHARED_HEAP EQUAL 1) add_definitions (-DWASM_ENABLE_SHARED_HEAP=1) message (" Shared heap enabled") endif() - -if (WAMR_ENABLE_COPY_CALLSTACK EQUAL 1) - add_definitions (-DWAMR_ENABLE_COPY_CALLSTACK=1) +if (WAMR_BUILD_COPY_CALL_STACK EQUAL 1) + add_definitions (-DWASM_ENABLE_COPY_CALL_STACK=1) message(" Copy callstack enabled") -else () - add_definitions (-DWAMR_ENABLE_COPY_CALLSTACK=0) - message(" Copy callstack disabled") endif() - if (WAMR_BUILD_MEMORY64 EQUAL 1) # if native is 32-bit or cross-compiled to 32-bit if (NOT WAMR_BUILD_TARGET MATCHES ".*64.*") @@ -539,6 +538,9 @@ if (WAMR_BUILD_WASI_NN EQUAL 1) if (DEFINED WAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH) add_definitions (-DWASM_WASI_NN_EXTERNAL_DELEGATE_PATH="${WAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH}") endif () + if (NOT DEFINED WAMR_BUILD_WASI_EPHEMERAL_NN) + set(WAMR_BUILD_WASI_EPHEMERAL_NN 1) + endif() if (WAMR_BUILD_WASI_EPHEMERAL_NN EQUAL 1) message (" WASI-NN: use 'wasi_ephemeral_nn' instead of 'wasi-nn'") add_definitions (-DWASM_ENABLE_WASI_EPHEMERAL_NN=1) @@ -675,7 +677,13 @@ if (WAMR_BUILD_INSTRUCTION_METERING EQUAL 1) message (" Instruction metering enabled") add_definitions (-DWASM_ENABLE_INSTRUCTION_METERING=1) endif () - +if (WAMR_BUILD_EXTENDED_CONST_EXPR EQUAL 1) + message (" Extended constant expression enabled") + add_definitions(-DWASM_ENABLE_EXTENDED_CONST_EXPR=1) +else() + message (" Extended constant expression disabled") + add_definitions(-DWASM_ENABLE_EXTENDED_CONST_EXPR=0) +endif () ######################################## # Show Phase4 Wasm proposals status. ######################################## @@ -689,6 +697,7 @@ message ( " \"WebAssembly C and C++ API\"\n" " Configurable. 0 is OFF. 1 is ON:\n" " \"Bulk Memory Operation\" via WAMR_BUILD_BULK_MEMORY: ${WAMR_BUILD_BULK_MEMORY}\n" +" \"Extended Constant Expressions\" via WAMR_BUILD_EXTENDED_CONST_EXPR: ${WAMR_BUILD_EXTENDED_CONST_EXPR}\n" " \"Fixed-width SIMD\" via WAMR_BUILD_SIMD: ${WAMR_BUILD_SIMD}\n" " \"Garbage collection\" via WAMR_BUILD_GC: ${WAMR_BUILD_GC}\n" " \"Legacy Exception handling\" via WAMR_BUILD_EXCE_HANDLING: ${WAMR_BUILD_EXCE_HANDLING}\n" @@ -703,7 +712,6 @@ message ( " \"Branch Hinting\"\n" " \"Custom Annotation Syntax in the Text Format\"\n" " \"Exception handling\"\n" -" \"Extended Constant Expressions\"\n" " \"Import/Export of Mutable Globals\"\n" " \"JS String Builtins\"\n" " \"Relaxed SIMD\"\n" diff --git a/build-scripts/runtime_lib.cmake b/build-scripts/runtime_lib.cmake index e538b5d91..f7639f6a6 100644 --- a/build-scripts/runtime_lib.cmake +++ b/build-scripts/runtime_lib.cmake @@ -106,6 +106,7 @@ endif () if (WAMR_BUILD_WASI_NN EQUAL 1) include (${IWASM_DIR}/libraries/wasi-nn/cmake/wasi_nn.cmake) + set (WAMR_BUILD_MODULE_INST_CONTEXT 1) endif () if (WAMR_BUILD_LIB_PTHREAD EQUAL 1) diff --git a/build-scripts/version.cmake b/build-scripts/version.cmake index c98b10f6f..637b3ba7b 100644 --- a/build-scripts/version.cmake +++ b/build-scripts/version.cmake @@ -7,7 +7,7 @@ if(NOT WAMR_ROOT_DIR) endif() set(WAMR_VERSION_MAJOR 2) -set(WAMR_VERSION_MINOR 3) +set(WAMR_VERSION_MINOR 4) set(WAMR_VERSION_PATCH 1) message("-- WAMR version: ${WAMR_VERSION_MAJOR}.${WAMR_VERSION_MINOR}.${WAMR_VERSION_PATCH}") diff --git a/ci/coding_guidelines_check.py b/ci/coding_guidelines_check.py index 43c366259..131bca5b6 100644 --- a/ci/coding_guidelines_check.py +++ b/ci/coding_guidelines_check.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # import argparse -import re from pathlib import Path import re import shlex @@ -39,7 +38,7 @@ INVALID_FILE_NAME_SEGMENT = r"([a-zA-Z0-9]+\-[a-zA-Z0-9]+)" def locate_command(command: str) -> bool: if not shutil.which(command): - print(f"Command '{command}'' not found") + print(f"Command '{command}' not found") return False return True diff --git a/core/config.h b/core/config.h index a4e1499e3..3ee0b2cd9 100644 --- a/core/config.h +++ b/core/config.h @@ -84,7 +84,7 @@ #endif #define AOT_MAGIC_NUMBER 0x746f6100 -#define AOT_CURRENT_VERSION 4 +#define AOT_CURRENT_VERSION 5 #ifndef WASM_ENABLE_JIT #define WASM_ENABLE_JIT 0 @@ -193,8 +193,8 @@ #error "Heap aux stack allocation must be enabled for WASI threads" #endif -#ifndef WAMR_ENABLE_COPY_CALLSTACK -#define WAMR_ENABLE_COPY_CALLSTACK 0 +#ifndef WASM_ENABLE_COPY_CALL_STACK +#define WASM_ENABLE_COPY_CALL_STACK 0 #endif #ifndef WASM_ENABLE_BASE_LIB @@ -720,4 +720,8 @@ unless used elsewhere */ #define WASM_ENABLE_INSTRUCTION_METERING 0 #endif +#ifndef WASM_ENABLE_EXTENDED_CONST_EXPR +#define WASM_ENABLE_EXTENDED_CONST_EXPR 0 +#endif + #endif /* end of _CONFIG_H_ */ diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index f274471f3..b0f155665 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -968,6 +968,35 @@ fail: return false; } +#if WASM_ENABLE_GC != 0 || WASM_ENABLE_EXTENDED_CONST_EXPR != 0 +static void +destroy_init_expr(InitializerExpression *expr) +{ +#if WASM_ENABLE_GC != 0 + if (expr->init_expr_type == INIT_EXPR_TYPE_STRUCT_NEW + || expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW + || expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { + wasm_runtime_free(expr->u.unary.v.data); + } +#endif + +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + // free left expr and right expr for binary oprand + if (!is_expr_binary_op(expr->init_expr_type)) { + return; + } + if (expr->u.binary.l_expr) { + destroy_init_expr_recursive(expr->u.binary.l_expr); + } + if (expr->u.binary.r_expr) { + destroy_init_expr_recursive(expr->u.binary.r_expr); + } + expr->u.binary.l_expr = expr->u.binary.r_expr = NULL; +#endif +} +#endif /* end of WASM_ENABLE_GC != 0 || WASM_ENABLE_EXTENDED_CONST_EXPR != 0 \ + */ + static void destroy_import_memories(AOTImportMemory *import_memories) { @@ -993,6 +1022,10 @@ destroy_mem_init_data_list(AOTModule *module, AOTMemInitData **data_list, /* If the module owns the binary data, free the bytes buffer */ if (module->is_binary_freeable && data_list[i]->bytes) wasm_runtime_free(data_list[i]->bytes); + +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + destroy_init_expr(&data_list[i]->offset); +#endif /* Free the data segment structure itself */ wasm_runtime_free(data_list[i]); } @@ -1043,11 +1076,11 @@ load_mem_init_data_list(const uint8 **p_buf, const uint8 *buf_end, uint32 byte_count; uint32 is_passive; uint32 memory_index; - InitializerExpression init_value; + InitializerExpression offset_expr; read_uint32(buf, buf_end, is_passive); read_uint32(buf, buf_end, memory_index); - if (!load_init_expr(&buf, buf_end, module, &init_value, error_buf, + if (!load_init_expr(&buf, buf_end, module, &offset_expr, error_buf, error_buf_size)) { return false; } @@ -1062,8 +1095,7 @@ load_mem_init_data_list(const uint8 **p_buf, const uint8 *buf_end, data_list[i]->is_passive = (bool)is_passive; data_list[i]->memory_index = memory_index; #endif - data_list[i]->offset.init_expr_type = init_value.init_expr_type; - data_list[i]->offset.u = init_value.u; + data_list[i]->offset = offset_expr; data_list[i]->byte_count = byte_count; data_list[i]->bytes = NULL; /* If the module owns the binary data, clone the bytes buffer */ @@ -1148,18 +1180,6 @@ fail: return false; } -#if WASM_ENABLE_GC != 0 -static void -destroy_init_expr(InitializerExpression *expr) -{ - if (expr->init_expr_type == INIT_EXPR_TYPE_STRUCT_NEW - || expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW - || expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { - wasm_runtime_free(expr->u.data); - } -} -#endif /* end of WASM_ENABLE_GC != 0 */ - static void destroy_import_tables(AOTImportTable *import_tables) { @@ -1183,6 +1203,9 @@ destroy_table_init_data_list(AOTTableInitData **data_list, uint32 count) for (j = 0; j < data_list[i]->value_count; j++) { destroy_init_expr(&data_list[i]->init_values[j]); } +#endif +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + destroy_init_expr(&data_list[i]->offset); #endif wasm_runtime_free(data_list[i]); } @@ -1208,34 +1231,34 @@ load_init_expr(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, break; case INIT_EXPR_TYPE_I32_CONST: case INIT_EXPR_TYPE_F32_CONST: - read_uint32(buf, buf_end, expr->u.i32); + read_uint32(buf, buf_end, expr->u.unary.v.i32); break; case INIT_EXPR_TYPE_I64_CONST: case INIT_EXPR_TYPE_F64_CONST: - read_uint64(buf, buf_end, expr->u.i64); + read_uint64(buf, buf_end, expr->u.unary.v.i64); break; case INIT_EXPR_TYPE_V128_CONST: - i64x2 = (uint64 *)expr->u.v128.i64x2; + i64x2 = (uint64 *)expr->u.unary.v.v128.i64x2; CHECK_BUF(buf, buf_end, sizeof(uint64) * 2); wasm_runtime_read_v128(buf, &i64x2[0], &i64x2[1]); buf += sizeof(uint64) * 2; break; case INIT_EXPR_TYPE_GET_GLOBAL: - read_uint32(buf, buf_end, expr->u.global_index); + read_uint32(buf, buf_end, expr->u.unary.v.global_index); break; /* INIT_EXPR_TYPE_FUNCREF_CONST can be used when both reference types and GC are disabled */ case INIT_EXPR_TYPE_FUNCREF_CONST: - read_uint32(buf, buf_end, expr->u.ref_index); + read_uint32(buf, buf_end, expr->u.unary.v.ref_index); break; #if WASM_ENABLE_GC != 0 || WASM_ENABLE_REF_TYPES != 0 case INIT_EXPR_TYPE_REFNULL_CONST: - read_uint32(buf, buf_end, expr->u.ref_index); + read_uint32(buf, buf_end, expr->u.unary.v.ref_index); break; #endif /* end of WASM_ENABLE_GC != 0 || WASM_ENABLE_REF_TYPES != 0 */ #if WASM_ENABLE_GC != 0 case INIT_EXPR_TYPE_I31_NEW: - read_uint32(buf, buf_end, expr->u.i32); + read_uint32(buf, buf_end, expr->u.unary.v.i32); break; case INIT_EXPR_TYPE_STRUCT_NEW: { @@ -1256,7 +1279,7 @@ load_init_expr(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, free_if_fail = true; init_values->count = field_count; init_values->type_idx = type_idx; - expr->u.data = init_values; + expr->u.unary.v.data = init_values; if (type_idx >= module->type_count) { set_error_buf(error_buf, error_buf_size, @@ -1294,7 +1317,7 @@ load_init_expr(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, break; } case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT: - read_uint32(buf, buf_end, expr->u.type_index); + read_uint32(buf, buf_end, expr->u.unary.v.type_index); break; case INIT_EXPR_TYPE_ARRAY_NEW: case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT: @@ -1317,8 +1340,8 @@ load_init_expr(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, } if (init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) { - expr->u.array_new_default.type_index = type_idx; - expr->u.array_new_default.length = length; + expr->u.unary.v.array_new_default.type_index = type_idx; + expr->u.unary.v.array_new_default.length = length; } else { uint32 i, elem_size, elem_data_count; @@ -1329,7 +1352,7 @@ load_init_expr(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, return false; } free_if_fail = true; - expr->u.data = init_values; + expr->u.unary.v.data = init_values; init_values->type_idx = type_idx; init_values->length = length; @@ -1357,6 +1380,34 @@ load_init_expr(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, break; } #endif /* end of WASM_ENABLE_GC != 0 */ +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + case INIT_EXPR_TYPE_I32_ADD: + case INIT_EXPR_TYPE_I32_SUB: + case INIT_EXPR_TYPE_I32_MUL: + case INIT_EXPR_TYPE_I64_ADD: + case INIT_EXPR_TYPE_I64_SUB: + case INIT_EXPR_TYPE_I64_MUL: + { + expr->u.binary.l_expr = expr->u.binary.r_expr = NULL; + if (!(expr->u.binary.l_expr = + loader_malloc(sizeof(InitializerExpression), error_buf, + error_buf_size))) { + goto fail; + } + if (!load_init_expr(&buf, buf_end, module, expr->u.binary.l_expr, + error_buf, error_buf_size)) + goto fail; + if (!(expr->u.binary.r_expr = + loader_malloc(sizeof(InitializerExpression), error_buf, + error_buf_size))) { + goto fail; + } + if (!load_init_expr(&buf, buf_end, module, expr->u.binary.r_expr, + error_buf, error_buf_size)) + goto fail; + break; + } +#endif /* end of WASM_ENABLE_EXTENDED_CONST_EXPR != 0 */ default: set_error_buf(error_buf, error_buf_size, "invalid init expr type."); return false; @@ -1369,10 +1420,13 @@ load_init_expr(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, fail: #if WASM_ENABLE_GC != 0 if (free_if_fail) { - wasm_runtime_free(expr->u.data); + wasm_runtime_free(expr->u.unary.v.data); } #else (void)free_if_fail; +#endif +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + destroy_init_expr(expr); #endif return false; } @@ -1535,14 +1589,16 @@ load_table_init_data_list(const uint8 **p_buf, const uint8 *buf_end, /* Create each table data segment */ for (i = 0; i < module->table_init_data_count; i++) { uint32 mode, elem_type; - uint32 table_index, init_expr_type, value_count; - uint64 init_expr_value, size1; + uint32 table_index, value_count; + uint64 size1; + InitializerExpression offset_expr; read_uint32(buf, buf_end, mode); read_uint32(buf, buf_end, elem_type); read_uint32(buf, buf_end, table_index); - read_uint32(buf, buf_end, init_expr_type); - read_uint64(buf, buf_end, init_expr_value); + if (!load_init_expr(&buf, buf_end, module, &offset_expr, error_buf, + error_buf_size)) + return false; #if WASM_ENABLE_GC != 0 if (wasm_is_type_multi_byte_type(elem_type)) { uint16 ref_type, nullable; @@ -1588,8 +1644,7 @@ load_table_init_data_list(const uint8 **p_buf, const uint8 *buf_end, } } #endif - data_list[i]->offset.init_expr_type = (uint8)init_expr_type; - data_list[i]->offset.u.i64 = (int64)init_expr_value; + data_list[i]->offset = offset_expr; data_list[i]->value_count = value_count; for (j = 0; j < data_list[i]->value_count; j++) { if (!load_init_expr(&buf, buf_end, module, @@ -1730,6 +1785,13 @@ load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, (void)u8; read_uint32(buf, buf_end, j); +#if WASM_ENABLE_AOT_VALIDATOR != 0 + /* an equivalence type should be before the type it refers to */ + if (j > i) { + set_error_buf(error_buf, error_buf_size, "invalid type index"); + goto fail; + } +#endif if (module->types[j]->ref_count == UINT16_MAX) { set_error_buf(error_buf, error_buf_size, "wasm type's ref count too large"); @@ -1744,7 +1806,12 @@ load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, read_uint32(buf, buf_end, parent_type_idx); read_uint16(buf, buf_end, rec_count); read_uint16(buf, buf_end, rec_idx); - +#if WASM_ENABLE_AOT_VALIDATOR != 0 + if (rec_idx > i) { + set_error_buf(error_buf, error_buf_size, "invalid rec_idx"); + goto fail; + } +#endif if (type_flag == WASM_TYPE_FUNC) { AOTFuncType *func_type; @@ -1993,6 +2060,13 @@ load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, AOTType *cur_type = module->types[j]; parent_type_idx = cur_type->parent_type_idx; if (parent_type_idx != (uint32)-1) { /* has parent */ +#if WASM_ENABLE_AOT_VALIDATOR != 0 + if (parent_type_idx >= module->type_count) { + set_error_buf(error_buf, error_buf_size, + "invalid parent type index"); + goto fail; + } +#endif AOTType *parent_type = module->types[parent_type_idx]; module->types[j]->parent_type = parent_type; @@ -2016,6 +2090,13 @@ load_types(const uint8 **p_buf, const uint8 *buf_end, AOTModule *module, AOTType *cur_type = module->types[j]; parent_type_idx = cur_type->parent_type_idx; if (parent_type_idx != (uint32)-1) { /* has parent */ +#if WASM_ENABLE_AOT_VALIDATOR != 0 + if (parent_type_idx >= module->type_count) { + set_error_buf(error_buf, error_buf_size, + "invalid parent type index"); + goto fail; + } +#endif AOTType *parent_type = module->types[parent_type_idx]; /* subtyping has been checked during compilation */ bh_assert(wasm_type_is_subtype_of( @@ -4340,7 +4421,7 @@ aot_compatible_version(uint32 version) * refer to "AoT-compiled module compatibility among WAMR versions" in * ./doc/biuld_wasm_app.md */ - return version == 4 || version == 3; + return version == AOT_CURRENT_VERSION; } static bool @@ -4480,7 +4561,7 @@ aot_unload(AOTModule *module) destroy_import_globals(module->import_globals); if (module->globals) { -#if WASM_ENABLE_GC != 0 +#if WASM_ENABLE_GC != 0 || WASM_ENABLE_EXTENDED_CONST_EXPR != 0 uint32 i; for (i = 0; i < module->global_count; i++) { destroy_init_expr(&module->globals[i].init_expr); @@ -4636,7 +4717,7 @@ aot_unload(AOTModule *module) } uint32 -aot_get_plt_table_size() +aot_get_plt_table_size(void) { return get_plt_table_size(); } diff --git a/core/iwasm/aot/aot_reloc.h b/core/iwasm/aot/aot_reloc.h index ca9ba17f1..ed225b5e0 100644 --- a/core/iwasm/aot/aot_reloc.h +++ b/core/iwasm/aot/aot_reloc.h @@ -185,6 +185,13 @@ typedef struct { #define REG_STRINGREF_SYM() #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +#define REG_SHARED_HEAP_SYM() \ + REG_SYM(wasm_runtime_check_and_update_last_used_shared_heap), +#else +#define REG_SHARED_HEAP_SYM() +#endif + #define REG_COMMON_SYMBOLS \ REG_SYM(aot_set_exception_with_id), \ REG_SYM(aot_invoke_native), \ @@ -218,6 +225,7 @@ typedef struct { REG_LLVM_PGO_SYM() \ REG_GC_SYM() \ REG_STRINGREF_SYM() \ + REG_SHARED_HEAP_SYM() \ #define CHECK_RELOC_OFFSET(data_size) do { \ if (!check_reloc_offset(target_section_size, \ diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index c5e143189..d2621fb2f 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -60,6 +60,16 @@ bh_static_assert(offsetof(AOTModuleInstanceExtra, stack_sizes) == 0); bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj) == 8); bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_start_off) == 16); +bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_end_off) == 24); +bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap) == 32); + +bh_static_assert(offsetof(WASMSharedHeap, next) == 0); +bh_static_assert(offsetof(WASMSharedHeap, chain_next) == 8); +bh_static_assert(offsetof(WASMSharedHeap, heap_handle) == 16); +bh_static_assert(offsetof(WASMSharedHeap, base_addr) == 24); +bh_static_assert(offsetof(WASMSharedHeap, size) == 32); +bh_static_assert(offsetof(WASMSharedHeap, start_off_mem64) == 40); +bh_static_assert(offsetof(WASMSharedHeap, start_off_mem32) == 48); bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3); @@ -279,18 +289,21 @@ assign_table_init_value(AOTModuleInstance *module_inst, AOTModule *module, switch (flag) { case INIT_EXPR_TYPE_GET_GLOBAL: { - if (!check_global_init_expr(module, init_expr->u.global_index, + if (!check_global_init_expr(module, + init_expr->u.unary.v.global_index, error_buf, error_buf_size)) { return false; } - if (init_expr->u.global_index < module->import_global_count) { + if (init_expr->u.unary.v.global_index + < module->import_global_count) { PUT_REF_TO_ADDR( - addr, module->import_globals[init_expr->u.global_index] - .global_data_linked.gc_obj); + addr, + module->import_globals[init_expr->u.unary.v.global_index] + .global_data_linked.gc_obj); } else { - uint32 global_idx = - init_expr->u.global_index - module->import_global_count; + uint32 global_idx = init_expr->u.unary.v.global_index + - module->import_global_count; return assign_table_init_value( module_inst, module, &module->globals[global_idx].init_expr, addr, error_buf, error_buf_size); @@ -306,7 +319,7 @@ assign_table_init_value(AOTModuleInstance *module_inst, AOTModule *module, case INIT_EXPR_TYPE_FUNCREF_CONST: { WASMFuncObjectRef func_obj = NULL; - uint32 func_idx = init_expr->u.u32; + uint32 func_idx = init_expr->u.unary.v.u32; if (func_idx != UINT32_MAX) { if (!(func_obj = @@ -321,7 +334,8 @@ assign_table_init_value(AOTModuleInstance *module_inst, AOTModule *module, } case INIT_EXPR_TYPE_I31_NEW: { - WASMI31ObjectRef i31_obj = wasm_i31_obj_new(init_expr->u.i32); + WASMI31ObjectRef i31_obj = + wasm_i31_obj_new(init_expr->u.unary.v.i32); PUT_REF_TO_ADDR(addr, i31_obj); break; } @@ -335,11 +349,12 @@ assign_table_init_value(AOTModuleInstance *module_inst, AOTModule *module, uint32 type_idx; if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { - init_values = (WASMStructNewInitValues *)init_expr->u.data; + init_values = + (WASMStructNewInitValues *)init_expr->u.unary.v.data; type_idx = init_values->type_idx; } else { - type_idx = init_expr->u.type_index; + type_idx = init_expr->u.unary.v.type_index; } struct_type = (WASMStructType *)module->types[type_idx]; @@ -388,12 +403,13 @@ assign_table_init_value(AOTModuleInstance *module_inst, AOTModule *module, uint32 type_idx, len; if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) { - type_idx = init_expr->u.array_new_default.type_index; - len = init_expr->u.array_new_default.length; + type_idx = init_expr->u.unary.v.array_new_default.type_index; + len = init_expr->u.unary.v.array_new_default.length; arr_init_val = &empty_val; } else { - init_values = (WASMArrayNewInitValues *)init_expr->u.data; + init_values = + (WASMArrayNewInitValues *)init_expr->u.unary.v.data; type_idx = init_values->type_idx; len = init_values->length; @@ -444,6 +460,90 @@ assign_table_init_value(AOTModuleInstance *module_inst, AOTModule *module, } #endif /* end of WASM_ENABLE_GC != 0 */ +static bool +get_init_value_recursive(AOTModuleInstance *module_inst, AOTModule *module, + InitializerExpression *expr, WASMValue *value, + char *error_buf, uint32 error_buf_size) +{ + uint8 flag = expr->init_expr_type; + switch (flag) { + case INIT_EXPR_TYPE_GET_GLOBAL: + { + if (!check_global_init_expr(module, expr->u.unary.v.global_index, + error_buf, error_buf_size)) { + return false; + } +#if WASM_ENABLE_GC == 0 + *value = module->import_globals[expr->u.unary.v.global_index] + .global_data_linked; +#else + if (expr->u.unary.v.global_index < module->import_global_count) { + *value = module->import_globals[expr->u.unary.v.global_index] + .global_data_linked; + } + else { + *value = module + ->globals[expr->u.unary.v.global_index + - module->import_global_count] + .init_expr.u.unary.v; + } +#endif + break; + } + case INIT_EXPR_TYPE_I32_CONST: + case INIT_EXPR_TYPE_I64_CONST: + { + *value = expr->u.unary.v; + break; + } +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + case INIT_EXPR_TYPE_I32_ADD: + case INIT_EXPR_TYPE_I32_SUB: + case INIT_EXPR_TYPE_I32_MUL: + case INIT_EXPR_TYPE_I64_ADD: + case INIT_EXPR_TYPE_I64_SUB: + case INIT_EXPR_TYPE_I64_MUL: + { + WASMValue l_value, r_value; + if (!get_init_value_recursive(module_inst, module, + expr->u.binary.l_expr, &l_value, + error_buf, error_buf_size)) { + return false; + } + if (!get_init_value_recursive(module_inst, module, + expr->u.binary.r_expr, &r_value, + error_buf, error_buf_size)) { + return false; + } + + if (flag == INIT_EXPR_TYPE_I32_ADD) { + value->i32 = l_value.i32 + r_value.i32; + } + else if (flag == INIT_EXPR_TYPE_I32_SUB) { + value->i32 = l_value.i32 - r_value.i32; + } + else if (flag == INIT_EXPR_TYPE_I32_MUL) { + value->i32 = l_value.i32 * r_value.i32; + } + else if (flag == INIT_EXPR_TYPE_I64_ADD) { + value->i64 = l_value.i64 + r_value.i64; + } + else if (flag == INIT_EXPR_TYPE_I64_SUB) { + value->i64 = l_value.i64 - r_value.i64; + } + else if (flag == INIT_EXPR_TYPE_I64_MUL) { + value->i64 = l_value.i64 * r_value.i64; + } + break; + } +#endif + default: + return false; + } + + return true; +} + static bool global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, char *error_buf, uint32 error_buf_size) @@ -472,30 +572,24 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, flag = init_expr->init_expr_type; switch (flag) { case INIT_EXPR_TYPE_GET_GLOBAL: + case INIT_EXPR_TYPE_I32_CONST: + case INIT_EXPR_TYPE_I64_CONST: +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + case INIT_EXPR_TYPE_I32_ADD: + case INIT_EXPR_TYPE_I32_SUB: + case INIT_EXPR_TYPE_I32_MUL: + case INIT_EXPR_TYPE_I64_ADD: + case INIT_EXPR_TYPE_I64_SUB: + case INIT_EXPR_TYPE_I64_MUL: +#endif { - if (!check_global_init_expr(module, init_expr->u.global_index, - error_buf, error_buf_size)) { + WASMValue value; + if (!get_init_value_recursive(module_inst, module, init_expr, + &value, error_buf, + error_buf_size)) { return false; } -#if WASM_ENABLE_GC == 0 - init_global_data( - p, global->type.val_type, - &module->import_globals[init_expr->u.global_index] - .global_data_linked); -#else - if (init_expr->u.global_index < module->import_global_count) { - init_global_data( - p, global->type.val_type, - &module->import_globals[init_expr->u.global_index] - .global_data_linked); - } - else { - uint32 global_idx = - init_expr->u.global_index - module->import_global_count; - init_global_data(p, global->type.val_type, - &module->globals[global_idx].init_expr.u); - } -#endif + init_global_data(p, global->type.val_type, &value); break; } #if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0 @@ -516,7 +610,7 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, case INIT_EXPR_TYPE_FUNCREF_CONST: { WASMFuncObjectRef func_obj = NULL; - uint32 func_idx = init_expr->u.u32; + uint32 func_idx = init_expr->u.unary.v.ref_index; if (func_idx != UINT32_MAX) { if (!(func_obj = @@ -531,7 +625,8 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, } case INIT_EXPR_TYPE_I31_NEW: { - WASMI31ObjectRef i31_obj = wasm_i31_obj_new(init_expr->u.i32); + WASMI31ObjectRef i31_obj = + wasm_i31_obj_new(init_expr->u.unary.v.i32); PUT_REF_TO_ADDR(p, i31_obj); break; } @@ -545,11 +640,12 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, uint32 type_idx; if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { - init_values = (WASMStructNewInitValues *)init_expr->u.data; + init_values = + (WASMStructNewInitValues *)init_expr->u.unary.v.data; type_idx = init_values->type_idx; } else { - type_idx = init_expr->u.type_index; + type_idx = init_expr->u.unary.v.type_index; } struct_type = (WASMStructType *)module->types[type_idx]; @@ -599,12 +695,14 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, uint32 type_idx, len; if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) { - type_idx = init_expr->u.array_new_default.type_index; - len = init_expr->u.array_new_default.length; + type_idx = + init_expr->u.unary.v.array_new_default.type_index; + len = init_expr->u.unary.v.array_new_default.length; arr_init_val = &empty_val; } else { - init_values = (WASMArrayNewInitValues *)init_expr->u.data; + init_values = + (WASMArrayNewInitValues *)init_expr->u.unary.v.data; type_idx = init_values->type_idx; len = init_values->length; @@ -650,7 +748,8 @@ global_instantiate(AOTModuleInstance *module_inst, AOTModule *module, #endif /* end of WASM_ENABLE_GC != 0 */ default: { - init_global_data(p, global->type.val_type, &init_expr->u); + init_global_data(p, global->type.val_type, + &init_expr->u.unary.v); break; } } @@ -671,6 +770,7 @@ tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module, uint64 total_size; AOTTableInitData *table_seg; AOTTableInstance *tbl_inst = first_tbl_inst; + uint8 offset_flag; total_size = (uint64)sizeof(AOTTableInstance *) * module_inst->table_count; if (total_size > 0 @@ -743,28 +843,25 @@ tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module, tbl_inst = module_inst->tables[table_seg->table_index]; bh_assert(tbl_inst); + offset_flag = table_seg->offset.init_expr_type; + #if WASM_ENABLE_REF_TYPES != 0 - bh_assert( - table_seg->offset.init_expr_type - == (tbl_inst->is_table64 ? INIT_EXPR_TYPE_I64_CONST - : INIT_EXPR_TYPE_I32_CONST) - || table_seg->offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL - || table_seg->offset.init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST - || table_seg->offset.init_expr_type - == INIT_EXPR_TYPE_REFNULL_CONST); + bh_assert(offset_flag == INIT_EXPR_TYPE_GET_GLOBAL + || offset_flag == INIT_EXPR_TYPE_FUNCREF_CONST + || offset_flag == INIT_EXPR_TYPE_REFNULL_CONST + || (tbl_inst->is_table64 ? is_valid_i64_offset(offset_flag) + : is_valid_i32_offset(offset_flag))); #else - bh_assert(table_seg->offset.init_expr_type - == (tbl_inst->is_table64 ? INIT_EXPR_TYPE_I64_CONST - : INIT_EXPR_TYPE_I32_CONST) - || table_seg->offset.init_expr_type - == INIT_EXPR_TYPE_GET_GLOBAL); + bh_assert(offset_flag == INIT_EXPR_TYPE_GET_GLOBAL + || (tbl_inst->is_table64 ? is_valid_i64_offset(offset_flag) + : is_valid_i32_offset(offset_flag))); #endif /* Resolve table data base offset */ /* TODO: The table64 current implementation assumes table max size * UINT32_MAX, so the offset conversion here is safe */ - if (table_seg->offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { - global_index = table_seg->offset.u.global_index; + if (offset_flag == INIT_EXPR_TYPE_GET_GLOBAL) { + global_index = table_seg->offset.u.unary.v.global_index; if (!check_global_init_expr(module, global_index, error_buf, error_buf_size)) { @@ -782,8 +879,15 @@ tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module, base_offset = *(uint32 *)(module_inst->global_data + global_data_offset); } - else - base_offset = (uint32)table_seg->offset.u.i32; + else { + WASMValue offset_value; + if (!get_init_value_recursive(module_inst, module, + &table_seg->offset, &offset_value, + error_buf, error_buf_size)) { + return false; + } + base_offset = (uint32)offset_value.i32; + } /* Copy table data */ /* base_offset only since length might negative */ @@ -818,7 +922,7 @@ tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module, #if WASM_ENABLE_GC == 0 for (j = 0; j < length; j++) { tbl_inst->elems[base_offset + j] = - table_seg->init_values[j].u.ref_index; + table_seg->init_values[j].u.unary.v.ref_index; } #endif } @@ -1118,6 +1222,7 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, AOTMemInitData *data_seg; uint64 total_size; mem_offset_t base_offset; + uint8 offset_flag; module_inst->memory_count = memory_count; total_size = sizeof(AOTMemoryInstance *) * (uint64)memory_count; @@ -1156,15 +1261,15 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, initialized */ continue; - bh_assert(data_seg->offset.init_expr_type - == (memory_inst->is_memory64 ? INIT_EXPR_TYPE_I64_CONST - : INIT_EXPR_TYPE_I32_CONST) - || data_seg->offset.init_expr_type - == INIT_EXPR_TYPE_GET_GLOBAL); + offset_flag = data_seg->offset.init_expr_type; + bh_assert(offset_flag == INIT_EXPR_TYPE_GET_GLOBAL + || (memory_inst->is_memory64 + ? is_valid_i64_offset(offset_flag) + : is_valid_i32_offset(offset_flag))); /* Resolve memory data base offset */ - if (data_seg->offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { - global_index = data_seg->offset.u.global_index; + if (offset_flag == INIT_EXPR_TYPE_GET_GLOBAL) { + global_index = data_seg->offset.u.unary.v.global_index; if (!check_global_init_expr(module, global_index, error_buf, error_buf_size)) { @@ -1192,14 +1297,20 @@ memories_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, } } else { + WASMValue offset_value; + if (!get_init_value_recursive(module_inst, module, + &data_seg->offset, &offset_value, + error_buf, error_buf_size)) { + return false; + } #if WASM_ENABLE_MEMORY64 != 0 if (memory_inst->is_memory64) { - base_offset = data_seg->offset.u.i64; + base_offset = offset_value.i64; } else #endif { - base_offset = data_seg->offset.u.u32; + base_offset = offset_value.u32; } } @@ -1989,6 +2100,8 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, #else extra->shared_heap_start_off.u32[0] = UINT32_MAX; #endif + /* After shared heap chain, will early stop if shared heap is NULL */ + extra->shared_heap = NULL; #if WASM_ENABLE_PERF_PROFILING != 0 total_size = sizeof(AOTFuncPerfProfInfo) @@ -2043,6 +2156,7 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, uint8 tbl_elem_type; uint32 tbl_init_size, tbl_max_size, j; WASMRefType *tbl_elem_ref_type; + WASMValue offset_value; bh_assert(table_init_data); @@ -2074,69 +2188,73 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, if (!wasm_elem_is_active(table_init_data->mode)) { continue; } - - bh_assert(table_init_data->offset.init_expr_type - == INIT_EXPR_TYPE_I32_CONST - || table_init_data->offset.init_expr_type - == INIT_EXPR_TYPE_GET_GLOBAL - || table_init_data->offset.init_expr_type - == INIT_EXPR_TYPE_FUNCREF_CONST - || table_init_data->offset.init_expr_type - == INIT_EXPR_TYPE_REFNULL_CONST); + uint8 offset_flag = table_init_data->offset.init_expr_type; + bh_assert(offset_flag == INIT_EXPR_TYPE_GET_GLOBAL + || offset_flag == INIT_EXPR_TYPE_FUNCREF_CONST + || offset_flag == INIT_EXPR_TYPE_REFNULL_CONST + || offset_flag == INIT_EXPR_TYPE_I32_CONST + || offset_flag == INIT_EXPR_TYPE_I32_ADD + || offset_flag == INIT_EXPR_TYPE_I32_SUB + || offset_flag == INIT_EXPR_TYPE_I32_MUL); /* init vec(funcidx) or vec(expr) */ - if (table_init_data->offset.init_expr_type - == INIT_EXPR_TYPE_GET_GLOBAL) { + if (offset_flag == INIT_EXPR_TYPE_GET_GLOBAL) { uint32 data_offset; - if (!check_global_init_expr(module, - table_init_data->offset.u.global_index, - error_buf, error_buf_size)) { + if (!check_global_init_expr( + module, table_init_data->offset.u.unary.v.global_index, + error_buf, error_buf_size)) { goto fail; } - if (table_init_data->offset.u.global_index + if (table_init_data->offset.u.unary.v.global_index < module->import_global_count) { - data_offset = - module - ->import_globals[table_init_data->offset.u.global_index] - .data_offset; + data_offset = module + ->import_globals[table_init_data->offset.u + .unary.v.global_index] + .data_offset; } else { data_offset = module - ->globals[table_init_data->offset.u.global_index + ->globals[table_init_data->offset.u.unary.v.global_index - module->import_global_count] .data_offset; } - - table_init_data->offset.u.i32 = + offset_value.i32 = *(uint32 *)(module_inst->global_data + data_offset); } + else { + if (!get_init_value_recursive( + module_inst, module, &table_init_data->offset, + &offset_value, error_buf, error_buf_size)) { + goto fail; + } + } /* check offset since length might negative */ - if ((uint32)table_init_data->offset.u.i32 > table->cur_size) { - LOG_DEBUG("base_offset(%d) > table->cur_size(%d)", - table_init_data->offset.u.i32, table->cur_size); + if ((uint32)offset_value.i32 > table->cur_size) { + LOG_DEBUG("base_offset(%d) > table->cur_size(%d)", offset_value.i32, + table->cur_size); set_error_buf(error_buf, error_buf_size, "out of bounds table access"); goto fail; } - if ((uint32)table_init_data->offset.u.i32 + table_init_data->value_count + if ((uint32)offset_value.i32 + table_init_data->value_count > table->cur_size) { LOG_DEBUG("base_offset(%d) + length(%d) > table->cur_size(%d)", - table_init_data->offset.u.i32, - table_init_data->value_count, table->cur_size); + offset_value.i32, table_init_data->value_count, + table->cur_size); set_error_buf(error_buf, error_buf_size, "out of bounds table access"); goto fail; } for (j = 0; j < module->table_init_data_list[i]->value_count; j++) { - if (!assign_table_init_value( - module_inst, module, &table_init_data->init_values[j], - table_data + table_init_data->offset.u.i32 + j, error_buf, - error_buf_size)) { + if (!assign_table_init_value(module_inst, module, + &table_init_data->init_values[j], + table_data + offset_value.i32 + j, + error_buf, error_buf_size)) { goto fail; } } @@ -3639,7 +3757,7 @@ aot_get_module_inst_mem_consumption(const AOTModuleInstance *module_inst, for (i = 0; i < module_inst->memory_count; i++) { AOTMemoryInstance *mem_inst = module_inst->memories[i]; mem_conspn->memories_size += - mem_inst->num_bytes_per_page * mem_inst->cur_page_count; + (uint64)mem_inst->num_bytes_per_page * mem_inst->cur_page_count; mem_conspn->app_heap_size = mem_inst->heap_data_end - mem_inst->heap_data; /* size of app heap structure */ @@ -3729,10 +3847,10 @@ aot_table_init(AOTModuleInstance *module_inst, uint32 tbl_idx, for (i = 0; i < length; i++) { #if WASM_ENABLE_GC != 0 /* UINT32_MAX indicates that it is a null ref */ - if (init_values[i].u.ref_index != UINT32_MAX) { - if (!(func_obj = aot_create_func_obj(module_inst, - init_values[i].u.ref_index, - true, NULL, 0))) { + if (init_values[i].u.unary.v.ref_index != UINT32_MAX) { + if (!(func_obj = aot_create_func_obj( + module_inst, init_values[i].u.unary.v.ref_index, true, + NULL, 0))) { aot_set_exception_with_id(module_inst, EXCE_NULL_FUNC_OBJ); return; } @@ -3742,7 +3860,7 @@ aot_table_init(AOTModuleInstance *module_inst, uint32 tbl_idx, table_elems[i] = NULL_REF; } #else - table_elems[i] = init_values[i].u.ref_index; + table_elems[i] = init_values[i].u.unary.v.ref_index; #endif } } @@ -4137,9 +4255,9 @@ aot_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame) } #endif /* end of WASM_ENABLE_AOT_STACK_FRAME != 0 */ -#if WAMR_ENABLE_COPY_CALLSTACK != 0 +#if WASM_ENABLE_COPY_CALL_STACK != 0 uint32 -aot_copy_callstack_tiny_frame(WASMExecEnv *exec_env, wasm_frame_t *buffer, +aot_copy_callstack_tiny_frame(WASMExecEnv *exec_env, WASMCApiFrame *buffer, const uint32 length, const uint32 skip_n, char *error_buf, uint32 error_buf_size) { @@ -4193,7 +4311,7 @@ aot_copy_callstack_tiny_frame(WASMExecEnv *exec_env, wasm_frame_t *buffer, } uint32 -aot_copy_callstack_standard_frame(WASMExecEnv *exec_env, wasm_frame_t *buffer, +aot_copy_callstack_standard_frame(WASMExecEnv *exec_env, WASMCApiFrame *buffer, const uint32 length, const uint32 skip_n, char *error_buf, uint32_t error_buf_size) { @@ -4243,7 +4361,7 @@ aot_copy_callstack_standard_frame(WASMExecEnv *exec_env, wasm_frame_t *buffer, } uint32 -aot_copy_callstack(WASMExecEnv *exec_env, wasm_frame_t *buffer, +aot_copy_callstack(WASMExecEnv *exec_env, WASMCApiFrame *buffer, const uint32 length, const uint32 skip_n, char *error_buf, uint32_t error_buf_size) { @@ -4265,7 +4383,7 @@ aot_copy_callstack(WASMExecEnv *exec_env, wasm_frame_t *buffer, error_buf, error_buf_size); } } -#endif // WAMR_ENABLE_COPY_CALLSTACK +#endif // WASM_ENABLE_COPY_CALL_STACK #if WASM_ENABLE_DUMP_CALL_STACK != 0 bool diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index e5678a8d6..d06cd1081 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -125,6 +125,8 @@ typedef struct AOTModuleInstanceExtra { */ DefPointer(uint8 *, shared_heap_base_addr_adj); MemBound shared_heap_start_off; + MemBound shared_heap_end_off; + DefPointer(WASMSharedHeap *, shared_heap); WASMModuleInstanceExtraCommon common; @@ -142,9 +144,6 @@ typedef struct AOTModuleInstanceExtra { WASMModuleInstanceCommon **import_func_module_insts; #endif -#if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; -#endif } AOTModuleInstanceExtra; #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) @@ -787,12 +786,12 @@ aot_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame); bool aot_create_call_stack(struct WASMExecEnv *exec_env); -#if WAMR_ENABLE_COPY_CALLSTACK != 0 +#if WASM_ENABLE_COPY_CALL_STACK != 0 uint32 -aot_copy_callstack(WASMExecEnv *exec_env, wasm_frame_t *buffer, +aot_copy_callstack(WASMExecEnv *exec_env, WASMCApiFrame *buffer, const uint32 length, const uint32 skip_n, char *error_buf, uint32_t error_buf_size); -#endif // WAMR_ENABLE_COPY_CALLSTACK +#endif // WASM_ENABLE_COPY_CALL_STACK /** * @brief Dump wasm call stack or get the size diff --git a/core/iwasm/common/arch/fneh.txt b/core/iwasm/common/arch/fneh.txt new file mode 100644 index 000000000..965af94f2 --- /dev/null +++ b/core/iwasm/common/arch/fneh.txt @@ -0,0 +1,3 @@ +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/arch/invokeNative_aarch64.s b/core/iwasm/common/arch/invokeNative_aarch64.s index f48a429c5..ea5d9c749 100644 --- a/core/iwasm/common/arch/invokeNative_aarch64.s +++ b/core/iwasm/common/arch/invokeNative_aarch64.s @@ -79,3 +79,6 @@ return: add sp, sp, #0x30 /* restore sp */ ret +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/arch/invokeNative_aarch64_simd.s b/core/iwasm/common/arch/invokeNative_aarch64_simd.s index f940c3403..e1c720762 100644 --- a/core/iwasm/common/arch/invokeNative_aarch64_simd.s +++ b/core/iwasm/common/arch/invokeNative_aarch64_simd.s @@ -77,3 +77,6 @@ return: add sp, sp, #0x30 /* restore sp */ ret +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/arch/invokeNative_arc.s b/core/iwasm/common/arch/invokeNative_arc.s index e448eea65..e277dda03 100644 --- a/core/iwasm/common/arch/invokeNative_arc.s +++ b/core/iwasm/common/arch/invokeNative_arc.s @@ -67,3 +67,6 @@ call_func: j_s [blink] /* ret */ nop_s +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/arch/invokeNative_arm.s b/core/iwasm/common/arch/invokeNative_arm.s index bfe8e3b09..afcd51449 100644 --- a/core/iwasm/common/arch/invokeNative_arm.s +++ b/core/iwasm/common/arch/invokeNative_arm.s @@ -73,3 +73,6 @@ return: add sp, sp, #4 ldmfd sp!, {r4, r5, r6, r7, lr} bx lr +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/arch/invokeNative_arm_vfp.s b/core/iwasm/common/arch/invokeNative_arm_vfp.s index 19bc5d67c..52d2a7258 100644 --- a/core/iwasm/common/arch/invokeNative_arm_vfp.s +++ b/core/iwasm/common/arch/invokeNative_arm_vfp.s @@ -84,3 +84,6 @@ return: ldmfd sp!, {r4, r5, r6, r7, lr} bx lr +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/arch/invokeNative_em64.s b/core/iwasm/common/arch/invokeNative_em64.s index 739e84e4c..fa34c2eee 100644 --- a/core/iwasm/common/arch/invokeNative_em64.s +++ b/core/iwasm/common/arch/invokeNative_em64.s @@ -62,3 +62,6 @@ push_args_end: leave ret +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/arch/invokeNative_em64_simd.s b/core/iwasm/common/arch/invokeNative_em64_simd.s index 0043ac941..6c8d3febb 100644 --- a/core/iwasm/common/arch/invokeNative_em64_simd.s +++ b/core/iwasm/common/arch/invokeNative_em64_simd.s @@ -62,3 +62,6 @@ push_args_end: leave ret +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/arch/invokeNative_ia32.s b/core/iwasm/common/arch/invokeNative_ia32.s index de1c1a5e1..845cd93bf 100644 --- a/core/iwasm/common/arch/invokeNative_ia32.s +++ b/core/iwasm/common/arch/invokeNative_ia32.s @@ -35,3 +35,6 @@ skip_push_args: leave ret +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/arch/invokeNative_mips.s b/core/iwasm/common/arch/invokeNative_mips.s index 645f4f2ec..d6e48114e 100644 --- a/core/iwasm/common/arch/invokeNative_mips.s +++ b/core/iwasm/common/arch/invokeNative_mips.s @@ -72,3 +72,6 @@ done: j $31 .end invokeNative +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/arch/invokeNative_osx_universal.s b/core/iwasm/common/arch/invokeNative_osx_universal.s index e2ca654fd..fadaae564 100644 --- a/core/iwasm/common/arch/invokeNative_osx_universal.s +++ b/core/iwasm/common/arch/invokeNative_osx_universal.s @@ -15,4 +15,4 @@ #else #include "invokeNative_em64_simd.s" #endif -#endif \ No newline at end of file +#endif diff --git a/core/iwasm/common/arch/invokeNative_thumb.s b/core/iwasm/common/arch/invokeNative_thumb.s index 3669fe77e..8087c63b8 100644 --- a/core/iwasm/common/arch/invokeNative_thumb.s +++ b/core/iwasm/common/arch/invokeNative_thumb.s @@ -89,3 +89,6 @@ return: pop {r4, r5, r6, r7} mov lr, r3 bx lr +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/arch/invokeNative_thumb_vfp.s b/core/iwasm/common/arch/invokeNative_thumb_vfp.s index 3f12b91d3..1bad1a6a8 100644 --- a/core/iwasm/common/arch/invokeNative_thumb_vfp.s +++ b/core/iwasm/common/arch/invokeNative_thumb_vfp.s @@ -98,3 +98,6 @@ return: mov lr, r3 bx lr +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/arch/invokeNative_xtensa.s b/core/iwasm/common/arch/invokeNative_xtensa.s index ce03f12c1..9bc094d3c 100644 --- a/core/iwasm/common/arch/invokeNative_xtensa.s +++ b/core/iwasm/common/arch/invokeNative_xtensa.s @@ -72,3 +72,6 @@ call_func: return: retw.n +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/core/iwasm/common/wasm_loader_common.c b/core/iwasm/common/wasm_loader_common.c index dc8009205..4243e9198 100644 --- a/core/iwasm/common/wasm_loader_common.c +++ b/core/iwasm/common/wasm_loader_common.c @@ -225,3 +225,18 @@ read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign, return false; } } + +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 +void +destroy_init_expr_recursive(InitializerExpression *expr) +{ + if (expr == NULL) { + return; + } + if (is_expr_binary_op(expr->init_expr_type)) { + destroy_init_expr_recursive(expr->u.binary.l_expr); + destroy_init_expr_recursive(expr->u.binary.r_expr); + } + wasm_runtime_free(expr); +} +#endif /* end of WASM_ENABLE_EXTENDED_CONST_EXPR != 0 */ diff --git a/core/iwasm/common/wasm_loader_common.h b/core/iwasm/common/wasm_loader_common.h index 81174fb26..d5394bf99 100644 --- a/core/iwasm/common/wasm_loader_common.h +++ b/core/iwasm/common/wasm_loader_common.h @@ -50,6 +50,11 @@ void wasm_loader_set_error_buf(char *error_buf, uint32 error_buf_size, const char *string, bool is_aot); +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 +void +destroy_init_expr_recursive(InitializerExpression *expr); +#endif + #ifdef __cplusplus } #endif diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index 1f942ec6c..04e3b9976 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -143,7 +143,7 @@ is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) #if WASM_ENABLE_SHARED_HEAP != 0 static void * -wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size); +wasm_mmap_linear_memory(uint64 map_size, uint64 commit_size); static void wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size); @@ -177,39 +177,58 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args) goto fail1; } - if (!(heap->heap_handle = - runtime_malloc(mem_allocator_get_heap_struct_size()))) { + size = align_uint(size, os_getpagesize()); + if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { + LOG_WARNING("Invalid size of shared heap"); goto fail2; } - size = align_uint(size, os_getpagesize()); heap->size = size; heap->start_off_mem64 = UINT64_MAX - heap->size + 1; heap->start_off_mem32 = UINT32_MAX - heap->size + 1; + heap->attached_count = 0; - if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) { - LOG_WARNING("Invalid size of shared heap"); - goto fail3; + if (init_args->pre_allocated_addr != NULL) { + /* Create shared heap from a pre allocated buffer, its size need to + * align with system page */ + if (size != init_args->size) { + LOG_WARNING("Pre allocated size need to be aligned with system " + "page size to create shared heap"); + goto fail2; + } + + heap->heap_handle = NULL; + heap->base_addr = init_args->pre_allocated_addr; + LOG_VERBOSE("Create preallocated shared heap %p with size %u", + heap->base_addr, size); } + else { + if (!(heap->heap_handle = + runtime_malloc(mem_allocator_get_heap_struct_size()))) { + goto fail2; + } #ifndef OS_ENABLE_HW_BOUND_CHECK - map_size = size; + map_size = size; #else - /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: - * ea = i + memarg.offset - * both i and memarg.offset are u32 in range 0 to 4G - * so the range of ea is 0 to 8G - */ - map_size = 8 * (uint64)BH_GB; + /* Totally 8G is mapped, the opcode load/store address range is 0 to 8G: + * ea = i + memarg.offset + * both i and memarg.offset are u32 in range 0 to 4G + * so the range of ea is 0 to 8G + */ + map_size = 8 * (uint64)BH_GB; #endif - if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) { - goto fail3; - } - if (!mem_allocator_create_with_struct_and_pool( - heap->heap_handle, heap_struct_size, heap->base_addr, size)) { - LOG_WARNING("init share heap failed"); - goto fail4; + if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) { + goto fail3; + } + if (!mem_allocator_create_with_struct_and_pool( + heap->heap_handle, heap_struct_size, heap->base_addr, size)) { + LOG_WARNING("init share heap failed"); + goto fail4; + } + LOG_VERBOSE("Create pool shared heap %p with size %u", heap->base_addr, + size); } os_mutex_lock(&shared_heap_list_lock); @@ -233,6 +252,219 @@ fail1: return NULL; } +WASMSharedHeap * +wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body) +{ + WASMSharedHeap *cur; + bool heap_handle_exist = false; + + if (!head || !body) { + LOG_WARNING("Invalid shared heap to chain."); + return NULL; + } + heap_handle_exist = head->heap_handle != NULL; + + os_mutex_lock(&shared_heap_list_lock); + if (head->attached_count != 0 || body->attached_count != 0) { + LOG_WARNING("To create shared heap chain, all shared heap need to be " + "detached first."); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + for (cur = shared_heap_list; cur; cur = cur->next) { + if (cur->chain_next == body || cur->chain_next == head) { + LOG_WARNING( + "To create shared heap chain, both the 'head' and 'body' " + "shared heap can't already be the 'body' in another a chain"); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + if (cur == head && cur->chain_next) { + LOG_WARNING( + "To create shared heap chain, the 'head' shared heap can't " + "already be the 'head' in another a chain"); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + } + for (cur = body; cur; cur = cur->chain_next) { + if (cur->heap_handle && heap_handle_exist) { + LOG_WARNING( + "To create shared heap chain, only one of shared heap can " + "dynamically shared_heap_malloc and shared_heap_free, the rest " + "can only be pre-allocated shared heap"); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + if (cur->heap_handle) + heap_handle_exist = true; + } + + head->start_off_mem64 = body->start_off_mem64 - head->size; + head->start_off_mem32 = body->start_off_mem32 - head->size; + head->chain_next = body; + os_mutex_unlock(&shared_heap_list_lock); + return head; +} + +WASMSharedHeap * +wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain) +{ + WASMSharedHeap *cur, *tmp; + + if (!head || !head->chain_next) { + LOG_WARNING("Invalid shared heap chain to disconnect the head from."); + return NULL; + } + + os_mutex_lock(&shared_heap_list_lock); + if (head->attached_count != 0) { + LOG_WARNING("To disconnect the shared heap head from the shared heap " + "chain, the shared heap chain needs to be detached first."); + os_mutex_unlock(&shared_heap_list_lock); + return NULL; + } + + cur = head; + while (cur && cur->chain_next) { + cur->start_off_mem64 = UINT64_MAX - cur->size + 1; + cur->start_off_mem32 = UINT32_MAX - cur->size + 1; + tmp = cur; + cur = cur->chain_next; + tmp->chain_next = NULL; + if (!entire_chain) + break; + } + os_mutex_unlock(&shared_heap_list_lock); + return cur; +} + +static uint8 * +get_last_used_shared_heap_base_addr_adj(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + return e->shared_heap_base_addr_adj; + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + return e->shared_heap_base_addr_adj; + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + return 0; +} + +static uintptr_t +get_last_used_shared_heap_start_offset(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_start_off.u64; +#else + return e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_start_off.u64; +#else + return e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + return 0; +} + +static uintptr_t +get_last_used_shared_heap_end_offset(WASMModuleInstanceCommon *module_inst) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_end_off.u64; +#else + return e->shared_heap_end_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + return e->shared_heap_end_off.u64; +#else + return e->shared_heap_end_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ + return 0; +} + +static void +update_last_used_shared_heap(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap, bool is_memory64) +{ +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + WASMModuleInstanceExtra *e = + (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + if (is_memory64) + e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; + else + e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; + e->shared_heap_end_off.u64 = + e->shared_heap_start_off.u64 - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u64; +#else + e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; + e->shared_heap_end_off.u32[0] = + e->shared_heap_start_off.u32[0] - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_INTERP != 0 */ +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + AOTModuleInstanceExtra *e = + (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; +#if UINTPTR_MAX == UINT64_MAX + if (is_memory64) + e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; + else + e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; + e->shared_heap_end_off.u64 = + e->shared_heap_start_off.u64 - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u64; +#else + e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; + e->shared_heap_end_off.u32[0] = + e->shared_heap_start_off.u32[0] - 1 + shared_heap->size; + e->shared_heap_base_addr_adj = + shared_heap->base_addr - e->shared_heap_start_off.u32[0]; +#endif + } +#endif /* end of WASM_ENABLE_AOT != 0 */ +} + bool wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, WASMSharedHeap *shared_heap) @@ -263,20 +495,6 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, return false; } e->shared_heap = shared_heap; -#if WASM_ENABLE_JIT != 0 -#if UINTPTR_MAX == UINT64_MAX - if (memory->is_memory64) - e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; - else - e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; - e->shared_heap_base_addr_adj = - shared_heap->base_addr - e->shared_heap_start_off.u64; -#else - e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; - e->shared_heap_base_addr_adj = - shared_heap->base_addr - e->shared_heap_start_off.u32[0]; -#endif -#endif /* end of WASM_ENABLE_JIT != 0 */ } #endif /* end of WASM_ENABLE_INTERP != 0 */ #if WASM_ENABLE_AOT != 0 @@ -288,21 +506,15 @@ wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, return false; } e->shared_heap = shared_heap; -#if UINTPTR_MAX == UINT64_MAX - if (memory->is_memory64) - e->shared_heap_start_off.u64 = shared_heap->start_off_mem64; - else - e->shared_heap_start_off.u64 = shared_heap->start_off_mem32; - e->shared_heap_base_addr_adj = - shared_heap->base_addr - e->shared_heap_start_off.u64; -#else - e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32; - e->shared_heap_base_addr_adj = - shared_heap->base_addr - e->shared_heap_start_off.u32[0]; -#endif } #endif /* end of WASM_ENABLE_AOT != 0 */ + update_last_used_shared_heap(module_inst, shared_heap, memory->is_memory64); + os_mutex_lock(&shared_heap_list_lock); + shared_heap->attached_count++; + os_mutex_unlock(&shared_heap_list_lock); + LOG_VERBOSE("Shared heap %p is attached to module instance %p", shared_heap, + module_inst); return true; } @@ -320,34 +532,51 @@ wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, void wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) { + /* Reset shared_heap_end_off = UINT64/32_MAX - 1 to handling a corner case, + app_offset >= shared_heap_start && app_offset <= shared_heap_end-bytes+1 + when bytes=1 and both e->shared_heap_start_off and e->shared_heap_end_off + is 0xffffffff */ #if WASM_ENABLE_INTERP != 0 if (module_inst->module_type == Wasm_Module_Bytecode) { WASMModuleInstanceExtra *e = (WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e; + if (e->shared_heap != NULL) { + os_mutex_lock(&shared_heap_list_lock); + e->shared_heap->attached_count--; + os_mutex_unlock(&shared_heap_list_lock); + } e->shared_heap = NULL; -#if WASM_ENABLE_JIT != 0 #if UINTPTR_MAX == UINT64_MAX e->shared_heap_start_off.u64 = UINT64_MAX; + e->shared_heap_end_off.u64 = UINT64_MAX - 1; #else e->shared_heap_start_off.u32[0] = UINT32_MAX; + e->shared_heap_end_off.u32[0] = UINT32_MAX - 1; #endif e->shared_heap_base_addr_adj = NULL; -#endif } #endif /* end of WASM_ENABLE_INTERP != 0 */ #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { AOTModuleInstanceExtra *e = (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e; + if (e->shared_heap != NULL) { + os_mutex_lock(&shared_heap_list_lock); + e->shared_heap->attached_count--; + os_mutex_unlock(&shared_heap_list_lock); + } e->shared_heap = NULL; #if UINTPTR_MAX == UINT64_MAX e->shared_heap_start_off.u64 = UINT64_MAX; + e->shared_heap_end_off.u64 = UINT64_MAX - 1; #else e->shared_heap_start_off.u32[0] = UINT32_MAX; + e->shared_heap_end_off.u32[0] = UINT32_MAX - 1; #endif e->shared_heap_base_addr_adj = NULL; } #endif /* end of WASM_ENABLE_AOT != 0 */ + LOG_VERBOSE("Shared heap is detached from module instance %p", module_inst); } void @@ -385,71 +614,93 @@ wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) return get_shared_heap(module_inst_comm); } -static bool +bool is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, bool is_memory64, uint64 app_offset, uint32 bytes) { - WASMSharedHeap *heap = get_shared_heap(module_inst); + WASMSharedHeap *heap = get_shared_heap(module_inst), *cur; + uint64 shared_heap_start, shared_heap_end; if (!heap) { - return false; + goto fail; } if (bytes == 0) { bytes = 1; } - if (!is_memory64) { - if (app_offset >= heap->start_off_mem32 - && app_offset <= UINT32_MAX - bytes + 1) { - return true; - } + shared_heap_start = + (uint64)get_last_used_shared_heap_start_offset(module_inst); + shared_heap_end = (uint64)get_last_used_shared_heap_end_offset(module_inst); + if (bytes - 1 <= shared_heap_end && app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { + return true; } - else { - if (app_offset >= heap->start_off_mem64 - && app_offset <= UINT64_MAX - bytes + 1) { + + /* Early stop for app start address not in the shared heap(chain) at all */ + shared_heap_start = + is_memory64 ? heap->start_off_mem64 : heap->start_off_mem32; + shared_heap_end = is_memory64 ? UINT64_MAX : UINT32_MAX; + if (bytes - 1 > shared_heap_end || app_offset < shared_heap_start + || app_offset > shared_heap_end - bytes + 1) { + goto fail; + } + + /* Find the exact shared heap that app addr is in, and update last used + * shared heap info in module inst extra */ + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start - 1 + cur->size; + if (bytes - 1 <= shared_heap_end && app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { + update_last_used_shared_heap(module_inst, cur, is_memory64); return true; } } +fail: return false; } static bool is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, - uint8 *addr, uint32 bytes) + bool is_memory64, uint8 *addr, uint32 bytes) { - WASMSharedHeap *heap = get_shared_heap(module_inst); - uintptr_t base_addr; - uintptr_t addr_int; - uintptr_t end_addr; + WASMSharedHeap *cur, *heap = get_shared_heap(module_inst); + uintptr_t base_addr, addr_int, end_addr; if (!heap) { - return false; + goto fail; } - base_addr = (uintptr_t)heap->base_addr; - addr_int = (uintptr_t)addr; - if (addr_int < base_addr) { - return false; + /* Iterate through shared heap chain to find whether native addr in one of + * shared heap */ + for (cur = heap; cur != NULL; cur = cur->chain_next) { + base_addr = (uintptr_t)cur->base_addr; + addr_int = (uintptr_t)addr; + if (addr_int < base_addr) + continue; + + end_addr = addr_int + bytes; + /* Check for overflow */ + if (end_addr <= addr_int) + continue; + + if (end_addr > base_addr + cur->size) + continue; + + update_last_used_shared_heap(module_inst, cur, is_memory64); + return true; } - end_addr = addr_int + bytes; - /* Check for overflow */ - if (end_addr <= addr_int) { - return false; - } - - if (end_addr > base_addr + heap->size) { - return false; - } - - return true; +fail: + return false; } uint64 wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, - uint64_t size, void **p_native_addr) + uint64 size, void **p_native_addr) { WASMMemoryInstance *memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); @@ -459,6 +710,14 @@ wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, if (!memory || !shared_heap) return 0; + while (shared_heap && !shared_heap->heap_handle) { + shared_heap = shared_heap->chain_next; + } + if (!shared_heap) { + LOG_WARNING("Can't allocate from pre allocated shared heap"); + return 0; + } + native_addr = mem_allocator_malloc(shared_heap->heap_handle, size); if (!native_addr) return 0; @@ -467,12 +726,10 @@ wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, *p_native_addr = native_addr; } - if (memory->is_memory64) - return shared_heap->start_off_mem64 - + ((uint8 *)native_addr - shared_heap->base_addr); - else - return shared_heap->start_off_mem32 - + ((uint8 *)native_addr - shared_heap->base_addr); + return memory->is_memory64 + ? shared_heap->start_off_mem64 + : shared_heap->start_off_mem32 + + ((uint8 *)native_addr - shared_heap->base_addr); } void @@ -487,6 +744,14 @@ wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr) return; } + while (shared_heap && !shared_heap->heap_handle) { + shared_heap = shared_heap->chain_next; + } + if (!shared_heap) { + LOG_WARNING("The address to free is from pre allocated shared heap"); + return; + } + if (memory->is_memory64) { if (ptr < shared_heap->start_off_mem64) { /* ptr can not > UINT64_MAX */ LOG_WARNING("The address to free isn't in shared heap"); @@ -564,14 +829,16 @@ destroy_shared_heaps() while (heap) { cur = heap; heap = heap->next; - mem_allocator_destroy(cur->heap_handle); - wasm_runtime_free(cur->heap_handle); + if (cur->heap_handle) { + mem_allocator_destroy(cur->heap_handle); + wasm_runtime_free(cur->heap_handle); #ifndef OS_ENABLE_HW_BOUND_CHECK - map_size = cur->size; + map_size = cur->size; #else - map_size = 8 * (uint64)BH_GB; + map_size = 8 * (uint64)BH_GB; #endif - wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size); + wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size); + } wasm_runtime_free(cur); } os_mutex_destroy(&shared_heap_list_lock); @@ -798,6 +1065,10 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, WASMMemoryInstance *memory_inst; uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; char *str, *str_end; +#if WASM_ENABLE_SHARED_HEAP != 0 + uintptr_t shared_heap_end_off; + char *shared_heap_base_addr_adj; +#endif bh_assert(module_inst_comm->module_type == Wasm_Module_Bytecode || module_inst_comm->module_type == Wasm_Module_AoT); @@ -814,12 +1085,12 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, app_str_offset, 1)) { - WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); - str = (char *)shared_heap->base_addr - + (memory_inst->is_memory64 - ? (app_str_offset - shared_heap->start_off_mem64) - : (app_str_offset - shared_heap->start_off_mem32)); - str_end = (char *)shared_heap->base_addr + shared_heap->size; + shared_heap_end_off = + get_last_used_shared_heap_end_offset(module_inst_comm); + shared_heap_base_addr_adj = + (char *)get_last_used_shared_heap_base_addr_adj(module_inst_comm); + str = shared_heap_base_addr_adj + app_str_offset; + str_end = shared_heap_base_addr_adj + shared_heap_end_off + 1; } else #endif @@ -884,7 +1155,8 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size)) { + if (is_native_addr_in_shared_heap( + module_inst_comm, memory_inst->is_memory64, native_ptr, size)) { return true; } #endif @@ -926,17 +1198,8 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, app_offset, 1)) { - WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); - uint64 shared_heap_start = 0; - - if (memory_inst && !memory_inst->is_memory64) { - shared_heap_start = shared_heap->start_off_mem32; - } - else if (memory_inst && memory_inst->is_memory64) { - shared_heap_start = shared_heap->start_off_mem64; - } - - return shared_heap->base_addr + app_offset - shared_heap_start; + return get_last_used_shared_heap_base_addr_adj(module_inst_comm) + + app_offset; } #endif @@ -974,29 +1237,17 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, bounds_checks = is_bounds_checks_enabled(module_inst_comm); -#if WASM_ENABLE_SHARED_HEAP != 0 - /* If shared heap is enabled, bounds check is always needed */ - bounds_checks = true; -#endif - memory_inst = wasm_get_default_memory(module_inst); if (!memory_inst) { return 0; } #if WASM_ENABLE_SHARED_HEAP != 0 - if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1)) { - WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); - uint64 shared_heap_start = 0; - - if (memory_inst && !memory_inst->is_memory64) { - shared_heap_start = shared_heap->start_off_mem32; - } - else if (memory_inst && memory_inst->is_memory64) { - shared_heap_start = shared_heap->start_off_mem64; - } - - return shared_heap_start + (addr - shared_heap->base_addr); + if (is_native_addr_in_shared_heap(module_inst_comm, + memory_inst->is_memory64, addr, 1)) { + return (uint64)(uintptr_t)(addr + - get_last_used_shared_heap_base_addr_adj( + module_inst_comm)); } #endif @@ -1098,8 +1349,8 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, uint8 *native_addr; bool bounds_checks; #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; - bool is_in_shared_heap = false; + uint8 *shared_heap_base_addr_adj = NULL; + uintptr_t shared_heap_end_off = 0; #endif bh_assert(app_buf_addr <= UINTPTR_MAX && app_buf_size <= UINTPTR_MAX); @@ -1113,36 +1364,17 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, memory_inst->is_memory64, app_buf_addr, app_buf_size)) { - shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module_inst); - native_addr = shared_heap->base_addr - + (memory_inst->is_memory64 - ? (app_buf_addr - shared_heap->start_off_mem64) - : (app_buf_addr - shared_heap->start_off_mem32)); - is_in_shared_heap = true; - } - else -#endif - { - native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; - } - - bounds_checks = - is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst); - - if (!bounds_checks) { - if (app_buf_addr == 0) { - native_addr = NULL; - } - goto success; - } - -#if WASM_ENABLE_SHARED_HEAP != 0 - if (is_in_shared_heap) { const char *str, *str_end; + shared_heap_base_addr_adj = get_last_used_shared_heap_base_addr_adj( + (WASMModuleInstanceCommon *)module_inst); + shared_heap_end_off = get_last_used_shared_heap_end_offset( + (WASMModuleInstanceCommon *)module_inst); + native_addr = shared_heap_base_addr_adj + (uintptr_t)app_buf_addr; - /* The whole string must be in the linear memory */ + /* The whole string must be in the shared heap */ str = (const char *)native_addr; - str_end = (const char *)shared_heap->base_addr + shared_heap->size; + str_end = + (const char *)shared_heap_base_addr_adj + shared_heap_end_off + 1; while (str < str_end && *str != '\0') str++; if (str == str_end) { @@ -1154,6 +1386,17 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, } #endif + native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; + bounds_checks = + is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst); + + if (!bounds_checks) { + if (app_buf_addr == 0) { + native_addr = NULL; + } + goto success; + } + /* No need to check the app_offset and buf_size if memory access boundary check with hardware trap is enabled */ #ifndef OS_ENABLE_HW_BOUND_CHECK diff --git a/core/iwasm/common/wasm_memory.h b/core/iwasm/common/wasm_memory.h index bceea0ee4..48bd2179f 100644 --- a/core/iwasm/common/wasm_memory.h +++ b/core/iwasm/common/wasm_memory.h @@ -41,10 +41,60 @@ SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size) #define SET_LINEAR_MEMORY_SIZE(memory, size) memory->memory_data_size = size #endif +#if WASM_ENABLE_INTERP != 0 #if WASM_ENABLE_SHARED_HEAP != 0 + +#if WASM_ENABLE_MULTI_MEMORY != 0 +/* Only enable shared heap for the default memory */ +#define is_default_memory (memidx == 0) +#else +#define is_default_memory true +#endif + +#if UINTPTR_MAX == UINT64_MAX +#define get_shared_heap_end_off() module->e->shared_heap_end_off.u64 +#else +#define get_shared_heap_end_off() \ + (uint64)(module->e->shared_heap_end_off.u32[0]) +#endif + +#if WASM_ENABLE_MEMORY64 != 0 +#define shared_heap_is_memory64 is_memory64 +#else +#define shared_heap_is_memory64 false +#endif + +#define app_addr_in_shared_heap(app_addr, bytes) \ + (is_default_memory \ + && is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module, \ + shared_heap_is_memory64, (uint64)app_addr, \ + bytes)) +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = module->e->shared_heap_base_addr_adj + app_addr +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else + +#else /* else of WASM_ENABLE_SHARED_HEAP != 0 */ +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ +#endif /* end of WASM_ENABLE_INTERP != 0 */ + +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, + bool is_memory64, uint64 app_offset, uint32 bytes); + WASMSharedHeap * wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); +WASMSharedHeap * +wasm_runtime_chain_shared_heaps(WASMSharedHeap *head, WASMSharedHeap *body); + +WASMSharedHeap * +wasm_runtime_unchain_shared_heaps(WASMSharedHeap *head, bool entire_chain); + bool wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, WASMSharedHeap *shared_heap); @@ -68,7 +118,7 @@ wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, void wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr); -#endif +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index dcee0aeaf..943b46fc4 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -1654,14 +1654,70 @@ wasm_runtime_instantiate(WASMModuleCommon *module, uint32 stack_size, error_buf_size); } +static void +instantiation_args_set_defaults(struct InstantiationArgs2 *args) +{ + memset(args, 0, sizeof(*args)); +} + WASMModuleInstanceCommon * wasm_runtime_instantiate_ex(WASMModuleCommon *module, const InstantiationArgs *args, char *error_buf, uint32 error_buf_size) +{ + struct InstantiationArgs2 v2; + instantiation_args_set_defaults(&v2); + v2.v1 = *args; + return wasm_runtime_instantiate_ex2(module, &v2, error_buf, error_buf_size); +} + +bool +wasm_runtime_instantiation_args_create(struct InstantiationArgs2 **p) +{ + struct InstantiationArgs2 *args = wasm_runtime_malloc(sizeof(*args)); + if (args == NULL) { + return false; + } + instantiation_args_set_defaults(args); + *p = args; + return true; +} + +void +wasm_runtime_instantiation_args_destroy(struct InstantiationArgs2 *p) +{ + wasm_runtime_free(p); +} + +void +wasm_runtime_instantiation_args_set_default_stack_size( + struct InstantiationArgs2 *p, uint32 v) +{ + p->v1.default_stack_size = v; +} + +void +wasm_runtime_instantiation_args_set_host_managed_heap_size( + struct InstantiationArgs2 *p, uint32 v) +{ + p->v1.host_managed_heap_size = v; +} + +void +wasm_runtime_instantiation_args_set_max_memory_pages( + struct InstantiationArgs2 *p, uint32 v) +{ + p->v1.max_memory_pages = v; +} + +WASMModuleInstanceCommon * +wasm_runtime_instantiate_ex2(WASMModuleCommon *module, + const struct InstantiationArgs2 *args, + char *error_buf, uint32 error_buf_size) { return wasm_runtime_instantiate_internal( - module, NULL, NULL, args->default_stack_size, - args->host_managed_heap_size, args->max_memory_pages, error_buf, + module, NULL, NULL, args->v1.default_stack_size, + args->v1.host_managed_heap_size, args->v1.max_memory_pages, error_buf, error_buf_size); } @@ -1743,9 +1799,9 @@ wasm_runtime_destroy_exec_env(WASMExecEnv *exec_env) wasm_exec_env_destroy(exec_env); } -#if WAMR_ENABLE_COPY_CALLSTACK != 0 +#if WASM_ENABLE_COPY_CALL_STACK != 0 uint32 -wasm_copy_callstack(const wasm_exec_env_t exec_env, wasm_frame_t *buffer, +wasm_copy_callstack(const wasm_exec_env_t exec_env, WASMCApiFrame *buffer, const uint32 length, const uint32 skip_n, char *error_buf, uint32_t error_buf_size) { @@ -1780,7 +1836,7 @@ wasm_copy_callstack(const wasm_exec_env_t exec_env, wasm_frame_t *buffer, strncpy(error_buf, err_msg, error_buf_size); return 0; } -#endif // WAMR_ENABLE_COPY_CALLSTACK +#endif // WASM_ENABLE_COPY_CALL_STACK bool wasm_runtime_init_thread_env(void) @@ -3754,7 +3810,15 @@ wasm_runtime_init_wasi(WASMModuleInstanceCommon *module_inst, address = strtok(cp, "/"); mask = strtok(NULL, "/"); - ret = addr_pool_insert(apool, address, (uint8)(mask ? atoi(mask) : 0)); + if (!mask) { + snprintf(error_buf, error_buf_size, + "Invalid address pool entry: %s, must be in the format of " + "ADDRESS/MASK", + addr_pool[i]); + goto fail; + } + + ret = addr_pool_insert(apool, address, (uint8)atoi(mask)); wasm_runtime_free(cp); if (!ret) { set_error_buf(error_buf, error_buf_size, @@ -7883,3 +7947,37 @@ wasm_runtime_is_underlying_binary_freeable(WASMModuleCommon *const module) return true; } + +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_runtime_check_and_update_last_used_shared_heap( + WASMModuleInstanceCommon *module_inst, uintptr_t app_offset, size_t bytes, + uintptr_t *shared_heap_start_off_p, uintptr_t *shared_heap_end_off_p, + uint8 **shared_heap_base_addr_adj_p, bool is_memory64) +{ + WASMSharedHeap *heap = wasm_runtime_get_shared_heap(module_inst), *cur; + uint64 shared_heap_start, shared_heap_end; + + if (bytes == 0) { + bytes = 1; + } + + /* Find the exact shared heap that app addr is in, and update last used + * shared heap info in func context */ + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start - 1 + cur->size; + if (bytes - 1 <= shared_heap_end && app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { + *shared_heap_start_off_p = (uintptr_t)shared_heap_start; + *shared_heap_end_off_p = (uintptr_t)shared_heap_end; + *shared_heap_base_addr_adj_p = + cur->base_addr - (uintptr_t)shared_heap_start; + return true; + } + } + + return false; +} +#endif diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 64a6cd793..324620bef 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -612,6 +612,10 @@ WASMExecEnv * wasm_runtime_get_exec_env_tls(void); #endif +struct InstantiationArgs2 { + InstantiationArgs v1; +}; + /* See wasm_export.h for description */ WASM_RUNTIME_API_EXTERN bool wasm_runtime_init(void); @@ -700,6 +704,40 @@ wasm_runtime_instantiate_ex(WASMModuleCommon *module, const InstantiationArgs *args, char *error_buf, uint32 error_buf_size); +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN +bool +wasm_runtime_instantiation_args_create(struct InstantiationArgs2 **p); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN +void +wasm_runtime_instantiation_args_destroy(struct InstantiationArgs2 *p); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN +void +wasm_runtime_instantiation_args_set_default_stack_size( + struct InstantiationArgs2 *p, uint32 v); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN +void +wasm_runtime_instantiation_args_set_host_managed_heap_size( + struct InstantiationArgs2 *p, uint32 v); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN +void +wasm_runtime_instantiation_args_set_max_memory_pages( + struct InstantiationArgs2 *p, uint32 v); + +/* See wasm_export.h for description */ +WASM_RUNTIME_API_EXTERN WASMModuleInstanceCommon * +wasm_runtime_instantiate_ex2(WASMModuleCommon *module, + const struct InstantiationArgs2 *args, + char *error_buf, uint32 error_buf_size); + /* See wasm_export.h for description */ WASM_RUNTIME_API_EXTERN bool wasm_runtime_set_running_mode(wasm_module_inst_t module_inst, @@ -758,12 +796,12 @@ wasm_runtime_create_exec_env(WASMModuleInstanceCommon *module_inst, WASM_RUNTIME_API_EXTERN void wasm_runtime_destroy_exec_env(WASMExecEnv *exec_env); -#if WAMR_ENABLE_COPY_CALLSTACK != 0 +#if WASM_ENABLE_COPY_CALL_STACK != 0 WASM_RUNTIME_API_EXTERN uint32_t -wasm_copy_callstack(const wasm_exec_env_t exec_env, wasm_frame_t *buffer, +wasm_copy_callstack(const wasm_exec_env_t exec_env, WASMCApiFrame *buffer, const uint32 length, const uint32 skip_n, char *error_buf, uint32 error_buf_size); -#endif // WAMR_ENABLE_COPY_CALLSTACK +#endif // WASM_ENABLE_COPY_CALL_STACK /* See wasm_export.h for description */ WASM_RUNTIME_API_EXTERN WASMModuleInstanceCommon * @@ -1336,6 +1374,14 @@ void wasm_runtime_set_linux_perf(bool flag); #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_runtime_check_and_update_last_used_shared_heap( + WASMModuleInstanceCommon *module_inst, uintptr_t app_offset, size_t bytes, + uintptr_t *shared_heap_start_off_p, uintptr_t *shared_heap_end_off_p, + uint8 **shared_heap_base_addr_adj_p, bool is_memory64); +#endif + #ifdef __cplusplus } #endif diff --git a/core/iwasm/compilation/aot.c b/core/iwasm/compilation/aot.c index 81fdb1284..5e1e554ac 100644 --- a/core/iwasm/compilation/aot.c +++ b/core/iwasm/compilation/aot.c @@ -8,7 +8,7 @@ static char aot_error[128]; char * -aot_get_last_error() +aot_get_last_error(void) { return aot_error[0] == '\0' ? "" : aot_error; } diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index 5d9664ee8..82f70ca3d 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -1316,13 +1316,6 @@ aot_compile_func(AOTCompContext *comp_ctx, uint32 func_index) return false; break; -#if WASM_ENABLE_SIMD != 0 - case WASM_OP_SELECT_128: - if (!aot_compile_op_select(comp_ctx, func_ctx, true)) - return false; - break; -#endif - #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 case WASM_OP_SELECT_T: { diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index b41399acb..12749305b 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -48,7 +48,7 @@ typedef struct AOTSymbolList { } AOTSymbolList; /* AOT object data */ -typedef struct AOTObjectData { +struct AOTObjectData { AOTCompContext *comp_ctx; LLVMMemoryBufferRef mem_buf; @@ -82,7 +82,7 @@ typedef struct AOTObjectData { const char *stack_sizes_section_name; uint32 stack_sizes_offset; uint32 *stack_sizes; -} AOTObjectData; +}; #if 0 static void dump_buf(uint8 *buf, uint32 size, char *title) @@ -216,7 +216,7 @@ get_init_expr_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data, #if WASM_ENABLE_GC != 0 WASMModule *module = comp_data->wasm_module; #endif - + bh_assert(expr != NULL); /* + init value size */ switch (expr->init_expr_type) { case INIT_EXPR_NONE: @@ -248,7 +248,7 @@ get_init_expr_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data, { uint32 i; WASMStructNewInitValues *struct_new_init_values = - (WASMStructNewInitValues *)expr->u.data; + (WASMStructNewInitValues *)expr->u.unary.v.data; /* type_index + field_count + fields */ size += sizeof(uint32) + sizeof(uint32); @@ -285,7 +285,7 @@ get_init_expr_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data, case INIT_EXPR_TYPE_ARRAY_NEW_FIXED: { WASMArrayNewInitValues *array_new_init_values = - (WASMArrayNewInitValues *)expr->u.data; + (WASMArrayNewInitValues *)expr->u.unary.v.data; WASMArrayType *array_type = NULL; uint32 value_count; @@ -302,12 +302,27 @@ get_init_expr_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data, /* array_elem_type + type_index + len + elems */ size += sizeof(uint32) * 3 - + wasm_value_type_size_internal(array_type->elem_type, - comp_ctx->pointer_size) + + (uint64)wasm_value_type_size_internal( + array_type->elem_type, comp_ctx->pointer_size) * value_count; break; } #endif /* end of WASM_ENABLE_GC != 0 */ +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + case INIT_EXPR_TYPE_I32_ADD: + case INIT_EXPR_TYPE_I32_SUB: + case INIT_EXPR_TYPE_I32_MUL: + case INIT_EXPR_TYPE_I64_ADD: + case INIT_EXPR_TYPE_I64_SUB: + case INIT_EXPR_TYPE_I64_MUL: + { + size += + get_init_expr_size(comp_ctx, comp_data, expr->u.binary.l_expr); + size += + get_init_expr_size(comp_ctx, comp_data, expr->u.binary.r_expr); + break; + } +#endif default: bh_assert(0); } @@ -324,15 +339,16 @@ get_table_init_data_size(AOTCompContext *comp_ctx, /* * mode (4 bytes), elem_type (4 bytes) * - * table_index(4 bytes) + init expr type (4 bytes) + init expr value (8 - * bytes) + * table_index(4 bytes) */ - size = (uint32)(sizeof(uint32) * 2 + sizeof(uint32) + sizeof(uint32) - + sizeof(uint64)) + size = (uint32)(sizeof(uint32) * 2 + sizeof(uint32)) /* Size of WasmRefType - inner padding (ref type + nullable + heap_type) */ + 8; + size += get_init_expr_size(comp_ctx, comp_ctx->comp_data, + &table_init_data->offset); + /* + value count/func index count (4 bytes) + init_values */ size += sizeof(uint32); for (i = 0; i < table_init_data->value_count; i++) { @@ -1811,6 +1827,10 @@ static bool aot_emit_init_expr(uint8 *buf, uint8 *buf_end, uint32 *p_offset, AOTCompContext *comp_ctx, InitializerExpression *expr) { + if (expr == NULL) { + aot_set_last_error("invalid init expr."); + return false; + } uint32 offset = *p_offset; #if WASM_ENABLE_GC != 0 WASMModule *module = comp_ctx->comp_data->wasm_module; @@ -1824,31 +1844,31 @@ aot_emit_init_expr(uint8 *buf, uint8 *buf_end, uint32 *p_offset, break; case INIT_EXPR_TYPE_I32_CONST: case INIT_EXPR_TYPE_F32_CONST: - EMIT_U32(expr->u.i32); + EMIT_U32(expr->u.unary.v.i32); break; case INIT_EXPR_TYPE_I64_CONST: case INIT_EXPR_TYPE_F64_CONST: - EMIT_U64(expr->u.i64); + EMIT_U64(expr->u.unary.v.i64); break; case INIT_EXPR_TYPE_V128_CONST: - EMIT_V128(expr->u.v128); + EMIT_V128(expr->u.unary.v.v128); break; case INIT_EXPR_TYPE_GET_GLOBAL: - EMIT_U32(expr->u.global_index); + EMIT_U32(expr->u.unary.v.global_index); break; case INIT_EXPR_TYPE_FUNCREF_CONST: case INIT_EXPR_TYPE_REFNULL_CONST: - EMIT_U32(expr->u.ref_index); + EMIT_U32(expr->u.unary.v.ref_index); break; #if WASM_ENABLE_GC != 0 case INIT_EXPR_TYPE_I31_NEW: - EMIT_U32(expr->u.i32); + EMIT_U32(expr->u.unary.v.i32); break; case INIT_EXPR_TYPE_STRUCT_NEW: { uint32 i; WASMStructNewInitValues *init_values = - (WASMStructNewInitValues *)expr->u.data; + (WASMStructNewInitValues *)expr->u.unary.v.data; WASMStructType *struct_type = NULL; EMIT_U32(init_values->type_idx); @@ -1879,21 +1899,21 @@ aot_emit_init_expr(uint8 *buf, uint8 *buf_end, uint32 *p_offset, break; } case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT: - EMIT_U32(expr->u.type_index); + EMIT_U32(expr->u.unary.v.type_index); break; case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT: { WASMArrayType *array_type = NULL; - bh_assert(expr->u.array_new_default.type_index + bh_assert(expr->u.unary.v.array_new_default.type_index < module->type_count); array_type = (WASMArrayType *) - module->types[expr->u.array_new_default.type_index]; + module->types[expr->u.unary.v.array_new_default.type_index]; EMIT_U32(array_type->elem_type); - EMIT_U32(expr->u.array_new_default.type_index); - EMIT_U32(expr->u.array_new_default.length); + EMIT_U32(expr->u.unary.v.array_new_default.type_index); + EMIT_U32(expr->u.unary.v.array_new_default.length); break; } case INIT_EXPR_TYPE_ARRAY_NEW: @@ -1901,7 +1921,7 @@ aot_emit_init_expr(uint8 *buf, uint8 *buf_end, uint32 *p_offset, { uint32 value_count, i, field_size; WASMArrayNewInitValues *init_values = - (WASMArrayNewInitValues *)expr->u.data; + (WASMArrayNewInitValues *)expr->u.unary.v.data; WASMArrayType *array_type = NULL; bh_assert(init_values->type_idx < module->type_count); @@ -1933,6 +1953,25 @@ aot_emit_init_expr(uint8 *buf, uint8 *buf_end, uint32 *p_offset, break; } #endif /* end of WASM_ENABLE_GC != 0 */ +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + case INIT_EXPR_TYPE_I32_ADD: + case INIT_EXPR_TYPE_I32_SUB: + case INIT_EXPR_TYPE_I32_MUL: + case INIT_EXPR_TYPE_I64_ADD: + case INIT_EXPR_TYPE_I64_SUB: + case INIT_EXPR_TYPE_I64_MUL: + if (comp_ctx->enable_extended_const) { + if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx, + expr->u.binary.l_expr)) { + return false; + } + if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx, + expr->u.binary.r_expr)) { + return false; + } + } + break; +#endif default: aot_set_last_error("invalid init expr type."); return false; @@ -2034,8 +2073,10 @@ aot_emit_table_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset, EMIT_U32(init_datas[i]->mode); EMIT_U32(init_datas[i]->elem_type); EMIT_U32(init_datas[i]->table_index); - EMIT_U32(init_datas[i]->offset.init_expr_type); - EMIT_U64(init_datas[i]->offset.u.i64); + if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx, + &init_datas[i]->offset)) + return false; + #if WASM_ENABLE_GC != 0 if (comp_ctx->enable_gc && init_datas[i]->elem_ref_type) { EMIT_U16(init_datas[i]->elem_ref_type->ref_ht_common.ref_type); diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index d22c1d5dd..a1bc46332 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -347,7 +347,8 @@ call_aot_invoke_c_api_native(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Get &c_api_func_imports[func_idx], note size of CApiFuncImport is pointer_size * 3 */ - offset = I32_CONST((comp_ctx->pointer_size * 3) * import_func_idx); + offset = I32_CONST((unsigned long long)comp_ctx->pointer_size * 3 + * import_func_idx); CHECK_LLVM_CONST(offset); c_api_func_import = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, c_api_func_imports, diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index e685fccd9..0659c2b40 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -10,6 +10,40 @@ #include "aot_intrinsic.h" #include "aot_emit_control.h" +#define BUILD_IS_NOT_NULL(value, res, name) \ + do { \ + if (!(res = LLVMBuildIsNotNull(comp_ctx->builder, value, name))) { \ + aot_set_last_error("llvm build is not null failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_BR(llvm_block) \ + do { \ + if (!LLVMBuildBr(comp_ctx->builder, llvm_block)) { \ + aot_set_last_error("llvm build br failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_COND_BR(value_if, block_then, block_else) \ + do { \ + if (!LLVMBuildCondBr(comp_ctx->builder, value_if, block_then, \ + block_else)) { \ + aot_set_last_error("llvm build cond br failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_TRUNC(value, data_type) \ + do { \ + if (!(value = LLVMBuildTrunc(comp_ctx->builder, value, data_type, \ + "val_trunc"))) { \ + aot_set_last_error("llvm build trunc failed."); \ + goto fail; \ + } \ + } while (0) + #define BUILD_ICMP(op, left, right, res, name) \ do { \ if (!(res = \ @@ -111,6 +145,418 @@ ffs(int n) static LLVMValueRef get_memory_curr_page_count(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); +#if WASM_ENABLE_SHARED_HEAP != 0 +uint32 +get_module_inst_extra_offset(AOTCompContext *comp_ctx); + +#define BUILD_LOAD_PTR(ptr, data_type, res) \ + do { \ + if (!(res = LLVMBuildLoad2(comp_ctx->builder, data_type, ptr, \ + "load_value"))) { \ + aot_set_last_error("llvm build load failed"); \ + goto fail; \ + } \ + } while (0) + +/* Update last used shared heap info(alloc ptr) in function ctx: + * 1. shared_heap_start_off 2. shared_heap_end_off 3. shared_heap_base_addr_adj + */ +bool +aot_check_shared_heap_chain_and_update(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMBasicBlockRef check_succ, + LLVMValueRef start_offset, + LLVMValueRef bytes, bool is_memory64) +{ + LLVMValueRef param_values[7], ret_value, func, value, cmp; + LLVMTypeRef param_types[7], ret_type, func_type, func_ptr_type; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INTPTR_T_TYPE; + param_types[2] = SIZE_T_TYPE; + param_types[3] = INTPTR_T_PTR_TYPE; + param_types[4] = INTPTR_T_PTR_TYPE; + param_types[5] = INT8_PTR_TYPE; + param_types[6] = INT8_TYPE; + ret_type = INT8_TYPE; + + GET_AOT_FUNCTION(wasm_runtime_check_and_update_last_used_shared_heap, 7); + + /* Call function */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = start_offset; + param_values[2] = bytes; + /* pass alloc ptr */ + param_values[3] = func_ctx->shared_heap_start_off; + param_values[4] = func_ctx->shared_heap_end_off; + param_values[5] = func_ctx->shared_heap_base_addr_adj; + param_values[6] = is_memory64 ? I8_ONE : I8_ZERO; + + if (!(ret_value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 7, "call"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + BUILD_ICMP(LLVMIntEQ, ret_value, I8_ZERO, cmp, "shared_heap_oob"); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, + check_succ)) { + goto fail; + } + + return true; +fail: + return false; +} + +/* + * Setup the basic blocks for shared heap and shared chain memory checks. + * + * Arguments: + * block_curr: The current basic block. + * app_addr_in_cache_shared_heap: Output, block for cache shared heap. + * app_addr_in_linear_mem: Output, block for linear memory. + * app_addr_in_shared_heap_chain: Output, block for shared heap chain + * (only for shared heap chain). + * check_shared_heap_chain: Output, block for checking shared heap chain + * (only for shared heap chain). + * + * Topology: + * If enable_shared_heap: + * block_curr -> app_addr_in_cache_shared_heap + * -> app_addr_in_linear_mem + * If enable_shared_chain: + * block_curr -> app_addr_in_shared_heap_chain + * -> app_addr_in_cache_shared_heap + * -> check_shared_heap_chain + * -> app_addr_in_linear_mem + */ +static bool +setup_shared_heap_blocks(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef block_curr, + LLVMBasicBlockRef *app_addr_in_cache_shared_heap, + LLVMBasicBlockRef *app_addr_in_linear_mem, + LLVMBasicBlockRef *app_addr_in_shared_heap_chain, + LLVMBasicBlockRef *check_shared_heap_chain) +{ + ADD_BASIC_BLOCK(*app_addr_in_cache_shared_heap, + "app_addr_in_cache_shared_heap"); + ADD_BASIC_BLOCK(*app_addr_in_linear_mem, "app_addr_in_linear_mem"); + + if (comp_ctx->enable_shared_heap) { + LLVMMoveBasicBlockAfter(*app_addr_in_cache_shared_heap, block_curr); + LLVMMoveBasicBlockAfter(*app_addr_in_linear_mem, + *app_addr_in_cache_shared_heap); + } + else if (comp_ctx->enable_shared_chain) { + ADD_BASIC_BLOCK(*app_addr_in_shared_heap_chain, + "app_addr_in_shared_heap_chain"); + ADD_BASIC_BLOCK(*check_shared_heap_chain, "check_shared_heap_chain"); + LLVMMoveBasicBlockAfter(*app_addr_in_shared_heap_chain, block_curr); + LLVMMoveBasicBlockAfter(*app_addr_in_cache_shared_heap, + *app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(*check_shared_heap_chain, + *app_addr_in_cache_shared_heap); + LLVMMoveBasicBlockAfter(*app_addr_in_linear_mem, + *app_addr_in_cache_shared_heap); + } + + return true; +fail: + return false; +} + +/* + * Build a branch to check if start_offset is in the shared heap chain region. + * + * Arguments: + * start_offset: The offset to check. + * app_addr_in_shared_heap_chain: Block to branch if in shared heap chain. + * app_addr_in_linear_mem: Block to branch if not in shared heap chain. + */ +static bool +build_check_app_addr_in_shared_heap_chain( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef start_offset, LLVMBasicBlockRef app_addr_in_shared_heap_chain, + LLVMBasicBlockRef app_addr_in_linear_mem) +{ + LLVMValueRef is_in_shared_heap = NULL; + + /* Use start_offset > func_ctx->shared_heap_head_start_off to test + * start_off falls in shared heap chain memory region. The shared heap + * chain oob will be detected in app_addr_in_shared_heap block or + * aot_check_shared_heap_chain_and_update function + */ + BUILD_ICMP(LLVMIntUGT, start_offset, func_ctx->shared_heap_head_start_off, + is_in_shared_heap, "shared_heap_lb_cmp"); + BUILD_COND_BR(is_in_shared_heap, app_addr_in_shared_heap_chain, + app_addr_in_linear_mem); + + SET_BUILD_POS(app_addr_in_shared_heap_chain); + + return true; +fail: + return false; +} + +/* + * Build the conditional branch for cache shared heap or shared heap chain. + * + * Arguments: + * cmp: The condition for being in cache shared heap. + * app_addr_in_cache_shared_heap: Block for cache shared heap. + * app_addr_in_linear_mem: Block for linear memory. + * check_shared_heap_chain: Block for checking shared heap chain. + * bytes: The access size in bytes. + * start_offset: The offset to check. + * is_memory64: Whether memory is 64-bit. + */ +static bool +build_shared_heap_conditional_branching( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef cmp, + LLVMBasicBlockRef app_addr_in_cache_shared_heap, + LLVMBasicBlockRef app_addr_in_linear_mem, + LLVMBasicBlockRef check_shared_heap_chain, LLVMValueRef bytes, + LLVMValueRef start_offset, bool is_memory64) +{ + if (comp_ctx->enable_shared_heap) { + BUILD_COND_BR(cmp, app_addr_in_cache_shared_heap, + app_addr_in_linear_mem); + } + else if (comp_ctx->enable_shared_chain) { + BUILD_COND_BR(cmp, app_addr_in_cache_shared_heap, + check_shared_heap_chain); + SET_BUILD_POS(check_shared_heap_chain); + if (!aot_check_shared_heap_chain_and_update( + comp_ctx, func_ctx, app_addr_in_cache_shared_heap, start_offset, + bytes, is_memory64)) + goto fail; + } + return true; +fail: + return false; +} + +/* + * Get the native address in the cache shared heap. + * + * Arguments: + * start_offset: The offset to use for address calculation. + * maddr: Output, the native address that in the cache shared heap. + */ +static bool +build_get_maddr_in_cache_shared_heap(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef start_offset, + LLVMValueRef *maddr) +{ + LLVMValueRef shared_heap_base_addr_adj; + /* load the local variable */ + BUILD_LOAD_PTR(func_ctx->shared_heap_base_addr_adj, INT8_PTR_TYPE, + shared_heap_base_addr_adj); + if (!(*maddr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, shared_heap_base_addr_adj, + &start_offset, 1, "maddr_cache_shared_heap"))) { + aot_set_last_error("llvm build inbounds gep failed"); + goto fail; + } + + return true; +fail: + return false; +} + +/* + * Check for memory overflow in shared heap for normal memory access. + * + * Arguments: + * block_curr: The current basic block. + * block_maddr_phi: The phi block for memory address. + * maddr_phi: The phi node for memory address. + * start_offset: The first offset to check. + * mem_base_addr: The base address of memory. Only used with segue. + * bytes_u32: The access size in bytes. + * is_memory64: Whether memory is wasm64 memory. + * is_target_64bit: Whether target is 64-bit. + * enable_segue: Whether to use segment register addressing. + */ +static bool +aot_check_shared_heap_memory_overflow( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, + LLVMValueRef maddr_phi, LLVMValueRef start_offset, + LLVMValueRef mem_base_addr, uint32 bytes_u32, bool is_memory64, + bool is_target_64bit, bool enable_segue) +{ + LLVMBasicBlockRef app_addr_in_cache_shared_heap, app_addr_in_linear_mem; + LLVMBasicBlockRef app_addr_in_shared_heap_chain = NULL, + check_shared_heap_chain = NULL; + LLVMValueRef cmp, cmp1, cmp2, shared_heap_start_off, shared_heap_end_off, + shared_heap_check_bound, maddr = NULL; + /* On 64/32-bit target, the offset is 64/32-bit */ + LLVMTypeRef offset_type = is_target_64bit ? I64_TYPE : I32_TYPE; + LLVMValueRef length, bytes; + + if (!setup_shared_heap_blocks( + comp_ctx, func_ctx, block_curr, &app_addr_in_cache_shared_heap, + &app_addr_in_linear_mem, &app_addr_in_shared_heap_chain, + &check_shared_heap_chain)) + goto fail; + LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); + + /* Early branching when it's not in shared heap chain at all */ + if (comp_ctx->enable_shared_chain + && !build_check_app_addr_in_shared_heap_chain( + comp_ctx, func_ctx, start_offset, app_addr_in_shared_heap_chain, + app_addr_in_linear_mem)) + goto fail; + + /* Load the local variable of the function */ + BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, offset_type, + shared_heap_start_off); + BUILD_LOAD_PTR(func_ctx->shared_heap_end_off, offset_type, + shared_heap_end_off); + /* Check if the app address is in the cache shared heap range. + * If yes, branch to the cache branch; if not, check the shared heap chain + */ + BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, + "cmp_cache_shared_heap_start"); + length = + is_target_64bit ? I64_CONST(bytes_u32 - 1) : I32_CONST(bytes_u32 - 1); + CHECK_LLVM_CONST(length); + BUILD_OP(Sub, shared_heap_end_off, length, shared_heap_check_bound, + "cache_shared_heap_end_bound"); + BUILD_ICMP(LLVMIntULE, start_offset, shared_heap_check_bound, cmp1, + "cmp_cache_shared_heap_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); + /* Conditional branching based on whether in cached shared heap */ + bytes = is_target_64bit ? I64_CONST(bytes_u32) : I32_CONST(bytes_u32); + if (!build_shared_heap_conditional_branching( + comp_ctx, func_ctx, cmp2, app_addr_in_cache_shared_heap, + app_addr_in_linear_mem, check_shared_heap_chain, bytes, + start_offset, is_memory64)) + goto fail; + + SET_BUILD_POS(app_addr_in_cache_shared_heap); + if (!build_get_maddr_in_cache_shared_heap(comp_ctx, func_ctx, start_offset, + &maddr)) + goto fail; + + if (enable_segue) { + LLVMValueRef mem_base_addr_u64, maddr_u64, offset_to_mem_base; + if (!(maddr_u64 = LLVMBuildPtrToInt(comp_ctx->builder, maddr, I64_TYPE, + "maddr_u64")) + || !(mem_base_addr_u64 = + LLVMBuildPtrToInt(comp_ctx->builder, mem_base_addr, + I64_TYPE, "mem_base_addr_u64"))) { + aot_set_last_error("llvm build ptr to int failed"); + goto fail; + } + if (!(offset_to_mem_base = + LLVMBuildSub(comp_ctx->builder, maddr_u64, mem_base_addr_u64, + "offset_to_mem_base"))) { + aot_set_last_error("llvm build sub failed"); + goto fail; + } + if (!(maddr = LLVMBuildIntToPtr(comp_ctx->builder, offset_to_mem_base, + INT8_PTR_TYPE_GS, + "maddr_shared_heap_segue"))) { + aot_set_last_error("llvm build int to ptr failed."); + goto fail; + } + } + + LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_cache_shared_heap, 1); + BUILD_BR(block_maddr_phi); + SET_BUILD_POS(app_addr_in_linear_mem); + + return true; +fail: + return false; +} + +/* + * Check for memory overflow in shared heap for bulk memory access. + * + * Arguments: + * block_curr: The current basic block. + * block_maddr_phi: The phi block for memory address. + * check_succ: The block to branch to on success. + * maddr_phi: The phi node for memory address. + * start_offset: The offset to check. + * max_addr: The maximum address to check. + * bytes: The access size in bytes (LLVMValueRef). + * is_memory64: Whether memory is wasm64 memory. + * is_target_64bit: Whether target is 64-bit. + */ +static bool +aot_check_bulk_memory_shared_heap_memory_overflow( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, + LLVMBasicBlockRef check_succ, LLVMValueRef maddr_phi, + LLVMValueRef start_offset, LLVMValueRef max_addr, LLVMValueRef bytes, + bool is_memory64, bool is_target_64bit) +{ + LLVMBasicBlockRef app_addr_in_cache_shared_heap, app_addr_in_linear_mem; + LLVMBasicBlockRef app_addr_in_shared_heap_chain = NULL, + check_shared_heap_chain = NULL; + LLVMValueRef cmp, cmp1, cmp2, shared_heap_start_off, shared_heap_end_off, + maddr = NULL, max_offset; + /* On 64/32-bit target, the offset is 64/32-bit */ + LLVMTypeRef offset_type = is_target_64bit ? I64_TYPE : I32_TYPE; + + if (!setup_shared_heap_blocks( + comp_ctx, func_ctx, block_curr, &app_addr_in_cache_shared_heap, + &app_addr_in_linear_mem, &app_addr_in_shared_heap_chain, + &check_shared_heap_chain)) + goto fail; + LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); + + /* Early branching when it's not in shared heap chain at all */ + if (comp_ctx->enable_shared_chain + && !build_check_app_addr_in_shared_heap_chain( + comp_ctx, func_ctx, start_offset, app_addr_in_shared_heap_chain, + app_addr_in_linear_mem)) + goto fail; + + /* Load the local variable of the function */ + BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, offset_type, + shared_heap_start_off); + BUILD_LOAD_PTR(func_ctx->shared_heap_end_off, offset_type, + shared_heap_end_off); + /* Check if the app address is in the cache shared heap range. + * If yes, branch to the cache branch; if not, check the shared heap chain + */ + BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, + "cmp_cache_shared_heap_start"); + BUILD_OP(Add, max_addr, is_target_64bit ? I64_NEG_ONE : I32_NEG_ONE, + max_offset, "max_offset"); + BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_end_off, cmp1, + "cmp_cache_shared_heap_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); + /* Conditional branching based on whether in cached shared heap */ + if (!build_shared_heap_conditional_branching( + comp_ctx, func_ctx, cmp2, app_addr_in_cache_shared_heap, + app_addr_in_linear_mem, check_shared_heap_chain, bytes, + start_offset, is_memory64)) + goto fail; + + SET_BUILD_POS(app_addr_in_cache_shared_heap); + if (!build_get_maddr_in_cache_shared_heap(comp_ctx, func_ctx, start_offset, + &maddr)) + goto fail; + + LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_cache_shared_heap, 1); + BUILD_BR(block_maddr_phi); + SET_BUILD_POS(app_addr_in_linear_mem); + + return true; +fail: + return false; +} +#endif + LLVMValueRef aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, mem_offset_t offset, uint32 bytes, bool enable_segue, @@ -118,10 +564,10 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { LLVMValueRef offset_const = MEMORY64_COND_VALUE(I64_CONST(offset), I32_CONST(offset)); - LLVMValueRef addr, maddr, maddr_phi = NULL, offset1, cmp1, cmp2, cmp; + LLVMValueRef addr, maddr, offset1, cmp1, cmp; LLVMValueRef mem_base_addr, mem_check_bound; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; + LLVMBasicBlockRef check_succ; AOTValue *aot_value_top; uint32 local_idx_of_aot_value = 0; uint64 const_value; @@ -136,6 +582,10 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, #else bool is_memory64 = IS_MEMORY64; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + LLVMValueRef maddr_phi = NULL; + LLVMBasicBlockRef block_maddr_phi = NULL; +#endif is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; @@ -262,6 +712,13 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, *alignp = 1; } + /* The overflow check needs to be done under following conditions: + * 1. In 64-bit target, offset and addr will be extended to 64-bit + * 1.1 offset + addr can overflow when it's memory64 + * 1.2 no overflow when it's memory32 + * 2. In 32-bit target, offset and addr will be 32-bit + * 2.1 offset + addr can overflow when it's memory32 + */ if (is_target_64bit) { if (!(offset_const = LLVMBuildZExt(comp_ctx->builder, offset_const, I64_TYPE, "offset_i64")) @@ -275,7 +732,9 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* offset1 = offset + addr; */ BUILD_OP(Add, offset_const, addr, offset1, "offset1"); - if (is_memory64 && comp_ctx->enable_bound_check) { + /* 1.1 offset + addr can overflow when it's memory64 + * 2.1 Or when it's on 32-bit platform */ + if (is_memory64 || !is_target_64bit) { /* Check whether integer overflow occurs in offset + addr */ LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, @@ -289,23 +748,14 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } SET_BUILD_POS(check_integer_overflow_end); + block_curr = check_integer_overflow_end; } - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; - LLVMValueRef is_in_shared_heap, shared_heap_check_bound = NULL; - - /* Add basic blocks */ - ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap"); - ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); +#if WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); - - LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr); - LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, - app_addr_in_shared_heap); - LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); - - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + SET_BUILD_POS(block_maddr_phi); if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, enable_segue ? INT8_PTR_TYPE_GS : INT8_PTR_TYPE, @@ -313,110 +763,16 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("llvm build phi failed"); goto fail; } + SET_BUILD_POS(block_curr); - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); - - if (!is_target_64bit) { - /* Check whether integer overflow occurs in addr + offset */ - LLVMBasicBlockRef check_integer_overflow_end; - ADD_BASIC_BLOCK(check_integer_overflow_end, - "check_integer_overflow_end"); - LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); - - BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); - if (!aot_emit_exception(comp_ctx, func_ctx, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, - cmp1, check_integer_overflow_end)) { - goto fail; - } - SET_BUILD_POS(check_integer_overflow_end); - } - - shared_heap_check_bound = - is_memory64 ? I64_CONST(UINT64_MAX - bytes + 1) - : (comp_ctx->pointer_size == sizeof(uint64) - ? I64_CONST(UINT32_MAX - bytes + 1) - : I32_CONST(UINT32_MAX - bytes + 1)); - CHECK_LLVM_CONST(shared_heap_check_bound); - - /* Check whether the bytes to access are in shared heap */ - if (!comp_ctx->enable_bound_check) { - /* Use IntUGT but not IntUGE to compare, since (1) in the ems - memory allocator, the hmu node includes hmu header and hmu - memory, only the latter is returned to the caller as the - allocated memory, the hmu header isn't returned so the - first byte of the shared heap won't be accessed, (2) using - IntUGT gets better performance than IntUGE in some cases */ - BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off, - is_in_shared_heap, "is_in_shared_heap"); - /* We don't check the shared heap's upper boundary if boundary - check isn't enabled, the runtime may also use the guard pages - of shared heap to check the boundary if hardware boundary - check feature is enabled. */ - } - else { - /* Use IntUGT but not IntUGE to compare, same as above */ - BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off, - cmp1, "cmp1"); - /* Check the shared heap's upper boundary if boundary check is - enabled */ - BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp2, - "cmp2"); - BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap"); - } - - if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap, - app_addr_in_shared_heap, app_addr_in_linear_mem)) { - aot_set_last_error("llvm build cond br failed"); + if (!aot_check_shared_heap_memory_overflow( + comp_ctx, func_ctx, block_curr, block_maddr_phi, maddr_phi, + offset1, mem_base_addr, bytes, is_memory64, is_target_64bit, + enable_segue)) { goto fail; } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap); - - /* Get native address inside shared heap */ - if (!(maddr = - LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->shared_heap_base_addr_adj, - &offset1, 1, "maddr_shared_heap"))) { - aot_set_last_error("llvm build inbounds gep failed"); - goto fail; - } - - if (enable_segue) { - LLVMValueRef mem_base_addr_u64, maddr_u64, offset_to_mem_base; - - if (!(maddr_u64 = LLVMBuildPtrToInt(comp_ctx->builder, maddr, - I64_TYPE, "maddr_u64")) - || !(mem_base_addr_u64 = - LLVMBuildPtrToInt(comp_ctx->builder, mem_base_addr, - I64_TYPE, "mem_base_addr_u64"))) { - aot_set_last_error("llvm build ptr to int failed"); - goto fail; - } - if (!(offset_to_mem_base = - LLVMBuildSub(comp_ctx->builder, maddr_u64, - mem_base_addr_u64, "offset_to_mem_base"))) { - aot_set_last_error("llvm build sub failed"); - goto fail; - } - if (!(maddr = LLVMBuildIntToPtr( - comp_ctx->builder, offset_to_mem_base, INT8_PTR_TYPE_GS, - "maddr_shared_heap_segue"))) { - aot_set_last_error("llvm build int to ptr failed."); - goto fail; - } - } - - LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1); - - if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { - aot_set_last_error("llvm build br failed"); - goto fail; - } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); - block_curr = LLVMGetInsertBlock(comp_ctx->builder); } +#endif if (comp_ctx->enable_bound_check && !(is_local_of_aot_value @@ -449,21 +805,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - if (is_target_64bit) { - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); - } - else { - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - /* Check integer overflow has been checked above */ - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); - } - else { - /* Check integer overflow */ - BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2"); - BUILD_OP(Or, cmp1, cmp2, cmp, "cmp"); - } - } + BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); /* Add basic blocks */ ADD_BASIC_BLOCK(check_succ, "check_succ"); @@ -509,17 +851,20 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { +#if WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { aot_set_last_error("llvm build br failed"); goto fail; } - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + SET_BUILD_POS(block_maddr_phi); return maddr_phi; } else +#endif return maddr; fail: return NULL; @@ -544,15 +889,6 @@ fail: LLVMSetAlignment(value, known_align); \ } while (0) -#define BUILD_TRUNC(value, data_type) \ - do { \ - if (!(value = LLVMBuildTrunc(comp_ctx->builder, value, data_type, \ - "val_trunc"))) { \ - aot_set_last_error("llvm build trunc failed."); \ - goto fail; \ - } \ - } while (0) - #define BUILD_STORE() \ do { \ LLVMValueRef res; \ @@ -1150,16 +1486,23 @@ LLVMValueRef check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef offset, LLVMValueRef bytes) { - LLVMValueRef maddr, max_addr, cmp; - LLVMValueRef mem_base_addr, maddr_phi = NULL; + LLVMValueRef maddr, max_addr, cmp, cmp1; + LLVMValueRef mem_base_addr; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; + LLVMBasicBlockRef check_succ; LLVMValueRef mem_size; + bool is_target_64bit; #if WASM_ENABLE_MEMORY64 == 0 bool is_memory64 = false; #else bool is_memory64 = IS_MEMORY64; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + LLVMValueRef maddr_phi = NULL; + LLVMBasicBlockRef block_maddr_phi = NULL; +#endif + + is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; /* Get memory base address and memory data size */ #if WASM_ENABLE_SHARED_MEMORY != 0 @@ -1221,111 +1564,71 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, ADD_BASIC_BLOCK(check_succ, "check_succ"); LLVMMoveBasicBlockAfter(check_succ, block_curr); - offset = - LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset"); - bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len"); - if (!offset || !bytes) { - aot_set_last_error("llvm build zext failed."); - goto fail; + /* Same logic with aot_check_memory_overflow, offset and bytes are 32/64 + * bits on 32/64 bits platform */ + if (is_target_64bit) { + offset = + LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset"); + bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len"); + if (!offset || !bytes) { + aot_set_last_error("llvm build zext failed."); + goto fail; + } } BUILD_OP(Add, offset, bytes, max_addr, "max_addr"); - if (is_memory64 && comp_ctx->enable_bound_check) { - /* Check whether integer overflow occurs in offset + addr */ + /* Check overflow when it's memory64 or it's on 32 bits platform */ + if (is_memory64 || !is_target_64bit) { + /* Check whether integer overflow occurs in offset + bytes */ LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, "check_integer_overflow_end"); LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); + /* offset + bytes can overflow yet is valid(for example, 0xffffffff, 1), + * allow it to be 0(either 0, 0 or overflow and valid) */ BUILD_ICMP(LLVMIntULT, max_addr, offset, cmp, "cmp"); + BUILD_ICMP(LLVMIntNE, max_addr, is_target_64bit ? I64_ZERO : I32_ZERO, + cmp1, "cmp1"); + BUILD_OP(And, cmp, cmp1, cmp, "overflow"); if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, check_integer_overflow_end)) { goto fail; } SET_BUILD_POS(check_integer_overflow_end); + block_curr = check_integer_overflow_end; } - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; - LLVMValueRef shared_heap_start_off, shared_heap_check_bound; - LLVMValueRef max_offset, cmp1, cmp2, is_in_shared_heap; - - /* Add basic blocks */ - ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap"); - ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); +#if WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); - - LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr); - LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, - app_addr_in_shared_heap); - LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); - - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + SET_BUILD_POS(block_maddr_phi); if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE, "maddr_phi"))) { aot_set_last_error("llvm build phi failed"); goto fail; } + SET_BUILD_POS(block_curr); - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); - - shared_heap_start_off = func_ctx->shared_heap_start_off; - if (comp_ctx->pointer_size == sizeof(uint32)) { - if (!(shared_heap_start_off = - LLVMBuildZExt(comp_ctx->builder, shared_heap_start_off, - I64_TYPE, "shared_heap_start_off_u64"))) { - aot_set_last_error("llvm build zext failed"); - goto fail; - } - } - shared_heap_check_bound = - is_memory64 ? I64_CONST(UINT64_MAX) : I64_CONST(UINT32_MAX); - CHECK_LLVM_CONST(shared_heap_check_bound); - - /* Check whether the bytes to access are in shared heap */ - if (!comp_ctx->enable_bound_check) { - /* Use IntUGT but not IntUGE to compare, same as the check - in aot_check_memory_overflow */ - BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off, - is_in_shared_heap, "is_in_shared_heap"); - } - else { - BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off, - cmp1, "cmp1"); - BUILD_OP(Add, max_addr, I64_NEG_ONE, max_offset, "max_offset"); - BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_check_bound, cmp2, - "cmp2"); - BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap"); - } - - if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap, - app_addr_in_shared_heap, app_addr_in_linear_mem)) { - aot_set_last_error("llvm build cond br failed"); + if (!aot_check_bulk_memory_shared_heap_memory_overflow( + comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, + maddr_phi, offset, max_addr, bytes, is_memory64, + is_target_64bit)) { goto fail; } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap); - - /* Get native address inside shared heap */ - if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->shared_heap_base_addr_adj, - &offset, 1, "maddr_shared_heap"))) { - aot_set_last_error("llvm build inbounds gep failed"); - goto fail; - } - LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1); - - if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { - aot_set_last_error("llvm build br failed"); - goto fail; - } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); - block_curr = LLVMGetInsertBlock(comp_ctx->builder); } +#endif + /* mem_size is always 64-bit, extend max_addr on 32 bits platform */ + if (!is_target_64bit + && !(max_addr = LLVMBuildZExt(comp_ctx->builder, max_addr, I64_TYPE, + "extend_max_addr"))) { + aot_set_last_error("llvm build zext failed."); + goto fail; + } BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr"); if (!aot_emit_exception(comp_ctx, func_ctx, @@ -1341,7 +1644,9 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { +#if WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { @@ -1352,6 +1657,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return maddr_phi; } else +#endif return maddr; fail: return NULL; diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 7cbb4a57e..5190a39d4 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1517,73 +1517,154 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } +#define BUILD_IS_NOT_NULL(value, res, name) \ + do { \ + if (!(res = LLVMBuildIsNotNull(comp_ctx->builder, value, name))) { \ + aot_set_last_error("llvm build is not null failed."); \ + goto fail; \ + } \ + } while (0) + +#define get_module_extra_field_offset(field) \ + do { \ + offset_u32 = get_module_inst_extra_offset(comp_ctx); \ + if (comp_ctx->is_jit_mode) \ + offset_u32 += offsetof(WASMModuleInstanceExtra, field); \ + else \ + offset_u32 += offsetof(AOTModuleInstanceExtra, field); \ + } while (0) + +#define LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(field, type) \ + do { \ + get_module_extra_field_offset(field); \ + offset = I32_CONST(offset_u32); \ + CHECK_LLVM_CONST(offset); \ + if (!(field_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, \ + func_ctx->aot_inst, &offset, 1, \ + #field "_p"))) { \ + aot_set_last_error("llvm build inbounds gep failed"); \ + goto fail; \ + } \ + if (!(load_val = \ + LLVMBuildLoad2(comp_ctx->builder, type, field_p, #field))) { \ + aot_set_last_error("llvm build load failed"); \ + goto fail; \ + } \ + if (!(func_ctx->field = \ + LLVMBuildAlloca(comp_ctx->builder, type, #field))) { \ + aot_set_last_error("llvm build alloca failed"); \ + goto fail; \ + } \ + if (!LLVMBuildStore(comp_ctx->builder, load_val, func_ctx->field)) { \ + aot_set_last_error("llvm build store failed"); \ + goto fail; \ + } \ + } while (0) + static bool create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { - LLVMValueRef offset, base_addr_p, start_off_p, cmp; +#if WASM_ENABLE_SHARED_HEAP != 0 + LLVMValueRef offset, field_p, load_val, shared_heap_head_p, + shared_heap_head, cmp, field_p_or_default, shared_heap_head_start_off, + shared_heap_head_start_off_minus_one; + LLVMTypeRef shared_heap_offset_type; uint32 offset_u32; - - /* Load aot_inst->e->shared_heap_base_addr_adj */ - offset_u32 = get_module_inst_extra_offset(comp_ctx); -#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 - if (comp_ctx->is_jit_mode) - offset_u32 += - offsetof(WASMModuleInstanceExtra, shared_heap_base_addr_adj); - else +#if WASM_ENABLE_MEMORY64 == 0 + bool is_memory64 = false; +#else + bool is_memory64 = IS_MEMORY64; #endif - offset_u32 += - offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj); + + shared_heap_offset_type = + comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE; + + /* shared_heap_base_addr_adj, shared_heap_start_off, and + * shared_heap_end_off can be updated later, use local variable to + * represent them */ + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_base_addr_adj, + INT8_PTR_TYPE); + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_start_off, + shared_heap_offset_type); + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_end_off, + shared_heap_offset_type); + + /* Shared Heap head start off won't be updated, no need to alloca */ + get_module_extra_field_offset(shared_heap); offset = I32_CONST(offset_u32); CHECK_LLVM_CONST(offset); - - if (!(base_addr_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->aot_inst, &offset, 1, - "shared_heap_base_addr_adj_p"))) { + if (!(shared_heap_head_p = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, + "shared_heap_head_p"))) { aot_set_last_error("llvm build inbounds gep failed"); - return false; + goto fail; } - if (!(func_ctx->shared_heap_base_addr_adj = - LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, base_addr_p, - "shared_heap_base_addr_adj"))) { + if (!(shared_heap_head = + LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, + shared_heap_head_p, "shared_heap_head"))) { aot_set_last_error("llvm build load failed"); - return false; + goto fail; } + BUILD_IS_NOT_NULL(shared_heap_head, cmp, "has_shared_heap"); - /* Load aot_inst->e->shared_heap_start_off */ - offset_u32 = get_module_inst_extra_offset(comp_ctx); -#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 - if (comp_ctx->is_jit_mode) - offset_u32 += offsetof(WASMModuleInstanceExtra, shared_heap_start_off); - else -#endif - offset_u32 += offsetof(AOTModuleInstanceExtra, shared_heap_start_off); + if (is_memory64) { + offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); + } + else { + offset_u32 = offsetof(WASMSharedHeap, start_off_mem32); + } offset = I32_CONST(offset_u32); CHECK_LLVM_CONST(offset); - - if (!(start_off_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->aot_inst, &offset, 1, - "shared_heap_start_off_p"))) { + if (!(field_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + shared_heap_head, &offset, 1, + "head_start_off_p"))) { aot_set_last_error("llvm build inbounds gep failed"); - return false; + goto fail; } - if (!(func_ctx->shared_heap_start_off = LLVMBuildLoad2( - comp_ctx->builder, - comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE, - start_off_p, "shared_heap_start_off"))) { + + /* Select a valid shared heap head ptr or safe alloca ptr stores + * shared_heap_start_off(UINT32_MAX/UINT64_MAX) */ + if (!(field_p_or_default = LLVMBuildSelect(comp_ctx->builder, cmp, field_p, + func_ctx->shared_heap_start_off, + "ptr_or_default"))) { + aot_set_last_error("llvm build select failed"); + goto fail; + } + + if (!(shared_heap_head_start_off = LLVMBuildLoad2( + comp_ctx->builder, shared_heap_offset_type, field_p_or_default, + "shared_heap_head_start_off"))) { aot_set_last_error("llvm build load failed"); - return false; + goto fail; + } + if (!(shared_heap_head_start_off_minus_one = LLVMBuildAdd( + comp_ctx->builder, shared_heap_head_start_off, + comp_ctx->pointer_size == sizeof(uint64) ? I64_NEG_ONE + : I32_NEG_ONE, + "head_start_off_minus_one"))) { + aot_set_last_error("llvm build load failed"); + goto fail; } - if (!(cmp = LLVMBuildIsNotNull(comp_ctx->builder, - func_ctx->shared_heap_base_addr_adj, - "has_shared_heap"))) { - aot_set_last_error("llvm build is not null failed"); - return false; + /* if there is attached shared heap(s), the value will be valid start_off-1, + * otherwise it will be UINT32_MAX/UINT64_MAX, so during the bounds checks, + * when has attached shared heap: + * offset > start_off - 1 => offset >= start_off + * when no attached shared heap: + * offset > UINT32_MAX/UINT64_MAX is always false + * */ + if (!(func_ctx->shared_heap_head_start_off = LLVMBuildSelect( + comp_ctx->builder, cmp, shared_heap_head_start_off_minus_one, + shared_heap_head_start_off, "head_start_off"))) { + aot_set_last_error("llvm build select failed"); + goto fail; } - return true; fail: return false; +#else /* else of WASM_ENABLE_SHARED_HEAP != 0 */ + return true; +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ } static bool @@ -1877,7 +1958,7 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx, } /* Load shared heap, shared heap start off mem32 or mem64 */ - if (comp_ctx->enable_shared_heap + if ((comp_ctx->enable_shared_heap || comp_ctx->enable_shared_chain) && !create_shared_heap_info(comp_ctx, func_ctx)) { goto fail; } @@ -2703,6 +2784,12 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) if (option->enable_shared_heap) comp_ctx->enable_shared_heap = true; + if (option->enable_shared_chain) + comp_ctx->enable_shared_chain = true; + + if (option->enable_extended_const) + comp_ctx->enable_extended_const = true; + comp_ctx->opt_level = option->opt_level; comp_ctx->size_level = option->size_level; @@ -3999,7 +4086,7 @@ aot_get_func_from_table(const AOTCompContext *comp_ctx, LLVMValueRef base, if (!(func = LLVMBuildBitCast(comp_ctx->builder, func, func_type, "func"))) { - aot_set_last_error("cast function fialed."); + aot_set_last_error("cast function failed."); goto fail; } @@ -4068,7 +4155,7 @@ aot_load_const_from_table(AOTCompContext *comp_ctx, LLVMValueRef base, if (!(const_addr = LLVMBuildBitCast(comp_ctx->builder, const_addr, const_ptr_type, "const_addr"))) { - aot_set_last_error("cast const fialed."); + aot_set_last_error("cast const failed."); return NULL; } diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 6b1233c39..9f62f6616 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -254,8 +254,12 @@ typedef struct AOTFuncContext { bool mem_space_unchanged; AOTCheckedAddrList checked_addr_list; + /* The last accessed shared heap info */ LLVMValueRef shared_heap_base_addr_adj; LLVMValueRef shared_heap_start_off; + LLVMValueRef shared_heap_end_off; + /* The start offset of the head of shared heap chain */ + LLVMValueRef shared_heap_head_start_off; LLVMBasicBlockRef got_exception_block; LLVMBasicBlockRef func_return_block; @@ -457,6 +461,9 @@ typedef struct AOTCompContext { /* Enable LLVM PGO (Profile-Guided Optimization) */ bool enable_llvm_pgo; + /* Enable extended constant expression */ + bool enable_extended_const; + /* Treat unknown import function as wasm-c-api import function and allow to directly invoke it from AOT/JIT code */ bool quick_invoke_c_api_import; @@ -486,6 +493,7 @@ typedef struct AOTCompContext { bool enable_gc; bool enable_shared_heap; + bool enable_shared_chain; uint32 opt_level; uint32 size_level; diff --git a/core/iwasm/fast-jit/fe/jit_emit_table.c b/core/iwasm/fast-jit/fe/jit_emit_table.c index 6da5b820f..efdf5cf14 100644 --- a/core/iwasm/fast-jit/fe/jit_emit_table.c +++ b/core/iwasm/fast-jit/fe/jit_emit_table.c @@ -121,7 +121,8 @@ wasm_init_table(WASMModuleInstance *inst, uint32 tbl_idx, uint32 seg_idx, + dst_offset * sizeof(table_elem_type_t)); init_values = tbl_seg_init_values + src_offset; for (i = 0; i < len; i++) { - addr[i] = (table_elem_type_t)(uintptr_t)init_values[+i].u.ref_index; + addr[i] = + (table_elem_type_t)(uintptr_t)init_values[+i].u.unary.v.ref_index; } return 0; diff --git a/core/iwasm/include/aot_comp_option.h b/core/iwasm/include/aot_comp_option.h index 839176623..069ceab31 100644 --- a/core/iwasm/include/aot_comp_option.h +++ b/core/iwasm/include/aot_comp_option.h @@ -68,6 +68,7 @@ typedef struct AOTCompOption { bool enable_ref_types; bool enable_gc; bool enable_aux_stack_check; + bool enable_extended_const; AOTStackFrameType aux_stack_frame_type; AOTCallStackFeatures call_stack_features; bool enable_perf_profiling; @@ -79,6 +80,7 @@ typedef struct AOTCompOption { bool enable_stack_estimation; bool quick_invoke_c_api_import; bool enable_shared_heap; + bool enable_shared_chain; char *use_prof_file; uint32_t opt_level; uint32_t size_level; diff --git a/core/iwasm/include/aot_export.h b/core/iwasm/include/aot_export.h index 4bdf58e1f..e4072ab72 100644 --- a/core/iwasm/include/aot_export.h +++ b/core/iwasm/include/aot_export.h @@ -98,10 +98,10 @@ void aot_destroy_aot_file(uint8_t *aot_file); char * -aot_get_last_error(); +aot_get_last_error(void); uint32_t -aot_get_plt_table_size(); +aot_get_plt_table_size(void); #ifdef __cplusplus } diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index b4ab34bea..81efb8f6f 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -139,8 +139,6 @@ typedef struct wasm_frame_t { uint32_t *lp; } WASMCApiFrame; -typedef WASMCApiFrame wasm_frame_t; - /* WASM section */ typedef struct wasm_section_t { struct wasm_section_t *next; @@ -291,6 +289,8 @@ typedef struct InstantiationArgs { } InstantiationArgs; #endif /* INSTANTIATION_ARGS_OPTION_DEFINED */ +struct InstantiationArgs2; + #ifndef WASM_VALKIND_T_DEFINED #define WASM_VALKIND_T_DEFINED typedef uint8_t wasm_valkind_t; @@ -351,6 +351,7 @@ typedef enum { typedef struct SharedHeapInitArgs { uint32_t size; + void *pre_allocated_addr; } SharedHeapInitArgs; /** @@ -734,6 +735,46 @@ wasm_runtime_instantiate_ex(const wasm_module_t module, const InstantiationArgs *args, char *error_buf, uint32_t error_buf_size); +/** + * Create an InstantiationArgs2 object with default parameters. + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_runtime_instantiation_args_create(struct InstantiationArgs2 **p); + +/** + * Dispose an InstantiationArgs2 object. + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_instantiation_args_destroy(struct InstantiationArgs2 *p); + +/** + * Setter functions for the InstantiationArgs2 object. + */ +WASM_RUNTIME_API_EXTERN void +wasm_runtime_instantiation_args_set_default_stack_size( + struct InstantiationArgs2 *p, uint32_t v); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_instantiation_args_set_host_managed_heap_size( + struct InstantiationArgs2 *p, uint32_t v); + +WASM_RUNTIME_API_EXTERN void +wasm_runtime_instantiation_args_set_max_memory_pages( + struct InstantiationArgs2 *p, uint32_t v); + +/** + * Instantiate a WASM module, with specified instantiation arguments + * + * Same as wasm_runtime_instantiate_ex, but this version takes + * InstantiationArgs2, which can be extended without breaking the ABI. + */ +WASM_RUNTIME_API_EXTERN wasm_module_inst_t +wasm_runtime_instantiate_ex2(const wasm_module_t module, + const struct InstantiationArgs2 *args, + char *error_buf, uint32_t error_buf_size); + /** * Set the running mode of a WASM module instance, override the * default running mode of the runtime. Note that it only makes sense when @@ -904,7 +945,7 @@ wasm_runtime_destroy_exec_env(wasm_exec_env_t exec_env); * @return number of copied frames */ WASM_RUNTIME_API_EXTERN uint32_t -wasm_copy_callstack(const wasm_exec_env_t exec_env, wasm_frame_t *buffer, +wasm_copy_callstack(const wasm_exec_env_t exec_env, WASMCApiFrame *buffer, const uint32_t length, const uint32_t skip_n, char *error_buf, uint32_t error_buf_size); @@ -2316,7 +2357,37 @@ WASM_RUNTIME_API_EXTERN wasm_shared_heap_t wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args); /** - * Attach a shared heap to a module instance + * This function links two shared heap(lists), `head` and `body` in to a single + * shared heap list, where `head` becomes the new shared heap list head. The + * shared heap list remains one continuous shared heap in wasm app's point of + * view. At most one shared heap in shared heap list can be dynamically + * allocated, the rest have to be the pre-allocated shared heap. * + * + * @param head The head of the shared heap chain. + * @param body The body of the shared heap chain to be appended. + * @return The new head of the shared heap chain. NULL if failed. + */ +WASM_RUNTIME_API_EXTERN wasm_shared_heap_t +wasm_runtime_chain_shared_heaps(wasm_shared_heap_t head, + wasm_shared_heap_t body); + +/** + * This function unchains the shared heaps from the given head. If + * `entire_chain` is true, it will unchain the entire chain of shared heaps. + * Otherwise, it will unchain only the first shared heap in the chain. + * + * @param head The head of the shared heap chain. + * @param entire_chain A boolean flag indicating whether to unchain the entire + * chain. + * @return The new head of the shared heap chain. Or the last shared heap in the + * chain if `entire_chain` is true. + */ +wasm_shared_heap_t +wasm_runtime_unchain_shared_heaps(wasm_shared_heap_t head, bool entire_chain); + +/** + * Attach a shared heap, it can be the head of shared heap chain, in that case, + * attach the shared heap chain, to a module instance * * @param module_inst the module instance * @param shared_heap the shared heap @@ -2335,7 +2406,8 @@ WASM_RUNTIME_API_EXTERN void wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst); /** - * Allocate memory from a shared heap + * Allocate memory from a shared heap, or the non-preallocated shared heap from + * the shared heap chain * * @param module_inst the module instance * @param size required memory size @@ -2352,7 +2424,8 @@ wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64_t size, void **p_native_addr); /** - * Free the memory allocated from shared heap + * Free the memory allocated from shared heap, or the non-preallocated shared + * heap from the shared heap chain * * @param module_inst the module instance * @param ptr the offset in wasm app diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index ddc0b15b4..0dd73958e 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -135,6 +135,12 @@ typedef void *table_elem_type_t; #define INIT_EXPR_TYPE_F64_CONST 0x44 #define INIT_EXPR_TYPE_V128_CONST 0xFD #define INIT_EXPR_TYPE_GET_GLOBAL 0x23 +#define INIT_EXPR_TYPE_I32_ADD 0x6A +#define INIT_EXPR_TYPE_I32_SUB 0x6B +#define INIT_EXPR_TYPE_I32_MUL 0x6C +#define INIT_EXPR_TYPE_I64_ADD 0x7C +#define INIT_EXPR_TYPE_I64_SUB 0x7D +#define INIT_EXPR_TYPE_I64_MUL 0x7E #define INIT_EXPR_TYPE_REFNULL_CONST 0xD0 #define INIT_EXPR_TYPE_FUNCREF_CONST 0xD2 #define INIT_EXPR_TYPE_STRUCT_NEW 0xD3 @@ -277,9 +283,41 @@ typedef struct InitializerExpression { /* type of INIT_EXPR_TYPE_XXX, which is an instruction of constant expression */ uint8 init_expr_type; - WASMValue u; + union { + struct { + WASMValue v; + } unary; + struct { + struct InitializerExpression *l_expr; + struct InitializerExpression *r_expr; + } binary; + } u; } InitializerExpression; +static inline bool +is_expr_binary_op(uint8 flag) +{ + return flag == INIT_EXPR_TYPE_I32_ADD || flag == INIT_EXPR_TYPE_I32_SUB + || flag == INIT_EXPR_TYPE_I32_MUL || flag == INIT_EXPR_TYPE_I64_ADD + || flag == INIT_EXPR_TYPE_I64_SUB || flag == INIT_EXPR_TYPE_I64_MUL; +} + +/* check if table or data offset is valid for i32 offset */ +static inline bool +is_valid_i32_offset(uint8 flag) +{ + return flag == INIT_EXPR_TYPE_I32_CONST || flag == INIT_EXPR_TYPE_I32_ADD + || flag == INIT_EXPR_TYPE_I32_SUB || flag == INIT_EXPR_TYPE_I32_MUL; +} + +/* check if table or data offset is valid for i64 offset */ +static inline bool +is_valid_i64_offset(uint8 flag) +{ + return flag == INIT_EXPR_TYPE_I64_CONST || flag == INIT_EXPR_TYPE_I64_ADD + || flag == INIT_EXPR_TYPE_I64_SUB || flag == INIT_EXPR_TYPE_I64_MUL; +} + #if WASM_ENABLE_GC != 0 /** * Reference type of (ref null ht) or (ref ht), diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 1e98b0fa9..edc473f2c 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -46,28 +46,6 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if WASM_ENABLE_SHARED_HEAP != 0 -#if WASM_ENABLE_MULTI_MEMORY != 0 -/* Only enable shared heap for the default memory */ -#define is_default_memory (memidx == 0) -#else -#define is_default_memory true -#endif -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) - -#define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) - -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ - if (app_addr_in_shared_heap(app_addr, bytes)) \ - shared_heap_addr_app_to_native(app_addr, native_addr); \ - else -#else -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) -#endif - #if WASM_ENABLE_MEMORY64 == 0 #if (!defined(OS_ENABLE_HW_BOUND_CHECK) \ @@ -1670,22 +1648,6 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (memory) is_memory64 = memory->is_memory64; #endif -#if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap = module->e->shared_heap; - uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; -#if WASM_ENABLE_MEMORY64 != 0 - uint64 shared_heap_start_off = - shared_heap ? (is_memory64 ? shared_heap->start_off_mem64 - : shared_heap->start_off_mem32) - : 0; - uint64 shared_heap_end_off = - shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0; -#else - uint64 shared_heap_start_off = - shared_heap ? shared_heap->start_off_mem32 : 0; - uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0; -#endif -#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_MULTI_MEMORY != 0 uint32 memidx = 0; uint32 memidx_cached = (uint32)-1; @@ -4088,7 +4050,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, case WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY: case WASM_OP_STRING_ENCODE_WTF8_ARRAY: { - uint32 start, array_len, count; + uint32 start, array_len; int32 bytes_written; EncodingFlag flag = WTF8; WASMArrayType *array_type; @@ -5996,12 +5958,14 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, || init_values[i].init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST); #if WASM_ENABLE_GC == 0 - table_elems[i] = - (table_elem_type_t)init_values[i].u.ref_index; + table_elems[i] = (table_elem_type_t)init_values[i] + .u.unary.v.ref_index; #else - if (init_values[i].u.ref_index != UINT32_MAX) { + if (init_values[i].u.unary.v.ref_index + != UINT32_MAX) { if (!(func_obj = wasm_create_func_obj( - module, init_values[i].u.ref_index, + module, + init_values[i].u.unary.v.ref_index, true, NULL, 0))) { goto got_exception; } diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 4e5edf41b..36d4538ff 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -41,22 +41,6 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if WASM_ENABLE_SHARED_HEAP != 0 -#define app_addr_in_shared_heap(app_addr, bytes) \ - (shared_heap && (app_addr) >= shared_heap_start_off \ - && (app_addr) <= shared_heap_end_off - bytes + 1) - -#define shared_heap_addr_app_to_native(app_addr, native_addr) \ - native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) - -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ - if (app_addr_in_shared_heap(app_addr, bytes)) \ - shared_heap_addr_app_to_native(app_addr, native_addr); \ - else -#else -#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) -#endif - #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 #define CHECK_MEMORY_OVERFLOW(bytes) \ @@ -1590,21 +1574,13 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, bool is_return_call = false; #endif #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap = module->e ? module->e->shared_heap : NULL; - uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; - /* -#if WASM_ENABLE_MEMORY64 != 0 - uint64 shared_heap_start_off = - shared_heap ? (is_memory64 ? shared_heap->start_off_mem64 - : shared_heap->start_off_mem32) - : 0; - uint64 shared_heap_end_off = - shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0; -#else - */ /* TODO: uncomment the code when memory64 is enabled for fast-interp */ - uint64 shared_heap_start_off = - shared_heap ? shared_heap->start_off_mem32 : 0; - uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0; + /* TODO: currently flowing two variables are only dummy for shared heap + * boundary check, need to be updated when multi-memory or memory64 + * proposals are to be implemented */ + bool is_memory64 = false; + uint32 memidx = 0; + (void)is_memory64; + (void)memidx; /* #endif */ #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ @@ -1911,7 +1887,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, } HANDLE_OP_END(); } -#if WASM_ENABLE_SIMD != 0 +#if WASM_ENABLE_SIMDE != 0 HANDLE_OP(WASM_OP_SELECT_128) { cond = frame_lp[GET_OFFSET()]; @@ -5374,12 +5350,14 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, || init_values[i].init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST); #if WASM_ENABLE_GC == 0 - table_elems[i] = - (table_elem_type_t)init_values[i].u.ref_index; + table_elems[i] = (table_elem_type_t)init_values[i] + .u.unary.v.ref_index; #else - if (init_values[i].u.ref_index != UINT32_MAX) { + if (init_values[i].u.unary.v.ref_index + != UINT32_MAX) { if (!(func_obj = wasm_create_func_obj( - module, init_values[i].u.ref_index, + module, + init_values[i].u.unary.v.ref_index, true, NULL, 0))) { goto got_exception; } diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index c8b4e6b7d..bca5fcbbf 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -453,6 +453,9 @@ typedef struct InitValue { WASMRefType ref_type; #endif WASMValue value; +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + InitializerExpression *expr; +#endif } InitValue; typedef struct ConstExprContext { @@ -477,7 +480,11 @@ push_const_expr_stack(ConstExprContext *ctx, uint8 flag, uint8 type, #if WASM_ENABLE_GC != 0 WASMRefType *ref_type, uint8 gc_opcode, #endif - WASMValue *value, char *error_buf, uint32 error_buf_size) + WASMValue *value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + InitializerExpression *expr, +#endif + char *error_buf, uint32 error_buf_size) { InitValue *cur_value; @@ -503,6 +510,10 @@ push_const_expr_stack(ConstExprContext *ctx, uint8 flag, uint8 type, cur_value->flag = flag; cur_value->value = *value; +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + cur_value->expr = expr; +#endif + #if WASM_ENABLE_GC != 0 cur_value->gc_opcode = gc_opcode; if (wasm_is_type_multi_byte_type(type)) { @@ -587,7 +598,11 @@ pop_const_expr_stack(ConstExprContext *ctx, uint8 *p_flag, uint8 type, #if WASM_ENABLE_GC != 0 WASMRefType *ref_type, uint8 *p_gc_opcode, #endif - WASMValue *p_value, char *error_buf, uint32 error_buf_size) + WASMValue *p_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + InitializerExpression **p_expr, +#endif + char *error_buf, uint32 error_buf_size) { InitValue *cur_value; @@ -623,7 +638,10 @@ pop_const_expr_stack(ConstExprContext *ctx, uint8 *p_flag, uint8 type, if (p_gc_opcode) *p_gc_opcode = cur_value->gc_opcode; #endif - +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + if (p_expr) + *p_expr = cur_value->expr; +#endif return true; #if WASM_ENABLE_GC != 0 @@ -639,7 +657,7 @@ fail: } static void -destroy_const_expr_stack(ConstExprContext *ctx) +destroy_const_expr_stack(ConstExprContext *ctx, bool free_exprs) { #if WASM_ENABLE_GC != 0 uint32 i; @@ -654,24 +672,62 @@ destroy_const_expr_stack(ConstExprContext *ctx) } } #endif +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + if (free_exprs) { + for (uint32 j = 0; j < ctx->sp; j++) { + if (is_expr_binary_op(ctx->stack[j].expr->init_expr_type)) { + destroy_init_expr_recursive(ctx->stack[j].expr); + ctx->stack[j].expr = NULL; + } + } + } +#endif if (ctx->stack != ctx->data) { wasm_runtime_free(ctx->stack); } } -#if WASM_ENABLE_GC != 0 +#if WASM_ENABLE_GC != 0 || WASM_ENABLE_EXTENDED_CONST_EXPR != 0 static void destroy_init_expr(WASMModule *module, InitializerExpression *expr) { +#if WASM_ENABLE_GC != 0 if (expr->init_expr_type == INIT_EXPR_TYPE_STRUCT_NEW || expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW || expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) { - destroy_init_expr_data_recursive(module, expr->u.data); + destroy_init_expr_data_recursive(module, expr->u.unary.v.data); } -} -#endif /* end of WASM_ENABLE_GC != 0 */ +#endif +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + // free left expr and right exprs for binary oprand + if (!is_expr_binary_op(expr->init_expr_type)) { + return; + } + if (expr->u.binary.l_expr) { + destroy_init_expr_recursive(expr->u.binary.l_expr); + } + if (expr->u.binary.r_expr) { + destroy_init_expr_recursive(expr->u.binary.r_expr); + } + expr->u.binary.l_expr = expr->u.binary.r_expr = NULL; +#endif +} +#endif + +/* for init expr + * (data (i32.add (i32.const 0) (i32.sub (i32.const 1) (i32.const 2)))), + * the binary format is + * 0x11: 41 00 ; i32.const 0 + * 0x13: 41 01 ; i32.const 1 + * 0x15: 41 02 ; i32.const 2 + * 0x17: 6b ; i32.sub + * 0x18: 6a ; i32.add + * for traversal: read opcodes and push them onto the stack. When encountering + * a binary opcode, pop two values from the stack which become the left and + * right child nodes of this binary operation node. + */ static bool load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, InitializerExpression *init_expr, uint8 type, void *ref_type, @@ -687,6 +743,9 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, uint8 opcode; WASMRefType cur_ref_type = { 0 }; #endif +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + InitializerExpression *cur_expr = NULL; +#endif init_const_expr_stack(&const_expr_ctx, module); @@ -699,24 +758,32 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, case INIT_EXPR_TYPE_I32_CONST: read_leb_int32(p, p_end, cur_value.i32); - if (!push_const_expr_stack( - &const_expr_ctx, flag, VALUE_TYPE_I32, + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_I32, #if WASM_ENABLE_GC != 0 - NULL, 0, + NULL, 0, #endif - &cur_value, error_buf, error_buf_size)) + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) goto fail; break; /* i64.const */ case INIT_EXPR_TYPE_I64_CONST: read_leb_int64(p, p_end, cur_value.i64); - if (!push_const_expr_stack( - &const_expr_ctx, flag, VALUE_TYPE_I64, + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_I64, #if WASM_ENABLE_GC != 0 - NULL, 0, + NULL, 0, #endif - &cur_value, error_buf, error_buf_size)) + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) goto fail; break; /* f32.const */ @@ -726,12 +793,16 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, for (i = 0; i < sizeof(float32); i++) *p_float++ = *p++; - if (!push_const_expr_stack( - &const_expr_ctx, flag, VALUE_TYPE_F32, + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_F32, #if WASM_ENABLE_GC != 0 - NULL, 0, + NULL, 0, #endif - &cur_value, error_buf, error_buf_size)) + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) goto fail; break; /* f64.const */ @@ -741,12 +812,16 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, for (i = 0; i < sizeof(float64); i++) *p_float++ = *p++; - if (!push_const_expr_stack( - &const_expr_ctx, flag, VALUE_TYPE_F64, + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_F64, #if WASM_ENABLE_GC != 0 - NULL, 0, + NULL, 0, #endif - &cur_value, error_buf, error_buf_size)) + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) goto fail; break; #if WASM_ENABLE_SIMD != 0 @@ -767,12 +842,16 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, cur_value.v128.i64x2[0] = high; cur_value.v128.i64x2[1] = low; - if (!push_const_expr_stack( - &const_expr_ctx, flag, VALUE_TYPE_V128, + if (!push_const_expr_stack(&const_expr_ctx, flag, + VALUE_TYPE_V128, #if WASM_ENABLE_GC != 0 - NULL, 0, + NULL, 0, #endif - &cur_value, error_buf, error_buf_size)) + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) goto fail; #if WASM_ENABLE_WAMR_COMPILER != 0 /* If any init_expr is v128.const, mark SIMD used */ @@ -783,7 +862,92 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, #endif /* end of (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) || \ (WASM_ENABLE_FAST_INTERP != 0) */ #endif /* end of WASM_ENABLE_SIMD */ +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + case INIT_EXPR_TYPE_I32_ADD: + case INIT_EXPR_TYPE_I32_SUB: + case INIT_EXPR_TYPE_I32_MUL: + case INIT_EXPR_TYPE_I64_ADD: + case INIT_EXPR_TYPE_I64_SUB: + case INIT_EXPR_TYPE_I64_MUL: + { + InitializerExpression *l_expr, *r_expr; + WASMValue l_value, r_value; + uint8 l_flag, r_flag; + uint8 value_type; + + if (flag == INIT_EXPR_TYPE_I32_ADD + || flag == INIT_EXPR_TYPE_I32_SUB + || flag == INIT_EXPR_TYPE_I32_MUL) { + value_type = VALUE_TYPE_I32; + } + else { + value_type = VALUE_TYPE_I64; + } + + /* If right flag indicates a binary operation, right expr will + * be popped from stack. Otherwise, allocate a new expr for + * right expr. Same for left expr. + */ + if (!(pop_const_expr_stack(&const_expr_ctx, &r_flag, value_type, +#if WASM_ENABLE_GC != 0 + NULL, NULL, +#endif + &r_value, &r_expr, error_buf, + error_buf_size))) { + goto fail; + } + if (!is_expr_binary_op(r_flag)) { + if (!(r_expr = loader_malloc(sizeof(InitializerExpression), + error_buf, error_buf_size))) { + goto fail; + } + r_expr->init_expr_type = r_flag; + r_expr->u.unary.v = r_value; + } + + if (!(pop_const_expr_stack(&const_expr_ctx, &l_flag, value_type, +#if WASM_ENABLE_GC != 0 + NULL, NULL, +#endif + &l_value, &l_expr, error_buf, + error_buf_size))) { + destroy_init_expr_recursive(r_expr); + goto fail; + } + if (!is_expr_binary_op(l_flag)) { + if (!(l_expr = loader_malloc(sizeof(InitializerExpression), + error_buf, error_buf_size))) { + destroy_init_expr_recursive(r_expr); + goto fail; + } + l_expr->init_expr_type = l_flag; + l_expr->u.unary.v = l_value; + } + + if (!(cur_expr = loader_malloc(sizeof(InitializerExpression), + error_buf, error_buf_size))) { + destroy_init_expr_recursive(l_expr); + destroy_init_expr_recursive(r_expr); + goto fail; + } + cur_expr->init_expr_type = flag; + cur_expr->u.binary.l_expr = l_expr; + cur_expr->u.binary.r_expr = r_expr; + + if (!push_const_expr_stack(&const_expr_ctx, flag, value_type, +#if WASM_ENABLE_GC != 0 + NULL, 0, +#endif + &cur_value, cur_expr, error_buf, + error_buf_size)) { + destroy_init_expr_recursive(cur_expr); + goto fail; + } + + break; + } +#endif /* end of WASM_ENABLE_EXTENDED_CONST_EXPR */ #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 /* ref.func */ case INIT_EXPR_TYPE_FUNCREF_CONST: @@ -799,6 +963,9 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, #if WASM_ENABLE_GC == 0 if (!push_const_expr_stack(&const_expr_ctx, flag, VALUE_TYPE_FUNCREF, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) goto fail; #else @@ -816,8 +983,11 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, false, type_idx); if (!push_const_expr_stack(&const_expr_ctx, flag, cur_ref_type.ref_type, &cur_ref_type, - 0, &cur_value, error_buf, - error_buf_size)) + 0, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) goto fail; #endif #if WASM_ENABLE_WAMR_COMPILER != 0 @@ -829,45 +999,71 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, /* ref.null */ case INIT_EXPR_TYPE_REFNULL_CONST: { - uint8 type1; - #if WASM_ENABLE_GC == 0 + uint8 type1; CHECK_BUF(p, p_end, 1); type1 = read_uint8(p); - cur_value.ref_index = NULL_REF; if (!push_const_expr_stack(&const_expr_ctx, flag, type1, - &cur_value, error_buf, - error_buf_size)) + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) goto fail; #else + /* + * According to the current GC SPEC rules, the heap_type must be + * validated when ref.null is used. It can be an absheaptype, + * or the type C.types[type_idx] must be defined in the context. + */ int32 heap_type; read_leb_int32(p, p_end, heap_type); - type1 = (uint8)((int32)0x80 + heap_type); - cur_value.gc_obj = NULL_REF; - if (!is_byte_a_type(type1) - || !wasm_is_valid_heap_type(heap_type) - || wasm_is_type_multi_byte_type(type1)) { - p--; - read_leb_uint32(p, p_end, type_idx); - if (!check_type_index(module, module->type_count, type_idx, - error_buf, error_buf_size)) - goto fail; + /* + * The current check of heap_type can deterministically infer + * the result of the previous condition + * `(!is_byte_a_type(type1) || + * wasm_is_type_multi_byte_type(type1))`. Therefore, the + * original condition is redundant and has been removed. + * + * This logic is consistent with the implementation of the + * `WASM_OP_REF_NULL` case in the `wasm_loader_prepare_bytecode` + * function. + */ + if (heap_type >= 0) { + if (!check_type_index(module, module->type_count, heap_type, + error_buf, error_buf_size)) { + goto fail; + } wasm_set_refheaptype_typeidx(&cur_ref_type.ref_ht_typeidx, - true, type_idx); + true, heap_type); if (!push_const_expr_stack(&const_expr_ctx, flag, cur_ref_type.ref_type, &cur_ref_type, 0, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) goto fail; } else { - if (!push_const_expr_stack(&const_expr_ctx, flag, type1, - NULL, 0, &cur_value, error_buf, - error_buf_size)) + if (!wasm_is_valid_heap_type(heap_type)) { + set_error_buf_v(error_buf, error_buf_size, + "unknown type %d", heap_type); + goto fail; + } + cur_ref_type.ref_ht_common.ref_type = + (uint8)((int32)0x80 + heap_type); + if (!push_const_expr_stack(&const_expr_ctx, flag, + cur_ref_type.ref_type, NULL, 0, + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) goto fail; } #endif @@ -956,8 +1152,11 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, #if WASM_ENABLE_GC != 0 &cur_ref_type, 0, #endif - &cur_value, error_buf, - error_buf_size)) + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) goto fail; break; @@ -1020,6 +1219,9 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, &const_expr_ctx, NULL, field_type, field_ref_type, NULL, &struct_init_values->fields[field_idx], +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) { destroy_init_expr_data_recursive( module, struct_init_values); @@ -1033,6 +1235,9 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, if (!push_const_expr_stack( &const_expr_ctx, flag, cur_ref_type.ref_type, &cur_ref_type, (uint8)opcode1, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) { destroy_init_expr_data_recursive( module, struct_init_values); @@ -1064,6 +1269,9 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, if (!push_const_expr_stack( &const_expr_ctx, flag, cur_ref_type.ref_type, &cur_ref_type, (uint8)opcode1, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) { goto fail; } @@ -1112,8 +1320,11 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, if (!pop_const_expr_stack( &const_expr_ctx, NULL, VALUE_TYPE_I32, - NULL, NULL, &len_val, error_buf, - error_buf_size)) { + NULL, NULL, &len_val, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { goto fail; } @@ -1132,6 +1343,9 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, &const_expr_ctx, NULL, elem_type, elem_ref_type, NULL, &array_init_values->elem_data[0], +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) { destroy_init_expr_data_recursive( module, array_init_values); @@ -1164,6 +1378,9 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, elem_ref_type, NULL, &array_init_values ->elem_data[i - 1], +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) { destroy_init_expr_data_recursive( module, array_init_values); @@ -1180,10 +1397,13 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, uint32 len; /* POP(i32) */ - if (!pop_const_expr_stack(&const_expr_ctx, NULL, - VALUE_TYPE_I32, NULL, - NULL, &len_val, error_buf, - error_buf_size)) { + if (!pop_const_expr_stack( + &const_expr_ctx, NULL, VALUE_TYPE_I32, NULL, + NULL, &len_val, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { goto fail; } len = len_val.i32; @@ -1197,6 +1417,9 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, if (!push_const_expr_stack( &const_expr_ctx, flag, cur_ref_type.ref_type, &cur_ref_type, (uint8)opcode1, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) { if (array_init_values) { destroy_init_expr_data_recursive( @@ -1223,9 +1446,13 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, case WASM_OP_REF_I31: { /* POP(i32) */ - if (!pop_const_expr_stack( - &const_expr_ctx, NULL, VALUE_TYPE_I32, NULL, - NULL, &cur_value, error_buf, error_buf_size)) { + if (!pop_const_expr_stack(&const_expr_ctx, NULL, + VALUE_TYPE_I32, NULL, NULL, + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { goto fail; } @@ -1234,6 +1461,9 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, if (!push_const_expr_stack( &const_expr_ctx, flag, cur_ref_type.ref_type, &cur_ref_type, (uint8)opcode1, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) { goto fail; } @@ -1268,7 +1498,11 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, #if WASM_ENABLE_GC != 0 ref_type, &opcode, #endif - &cur_value, error_buf, error_buf_size)) { + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + &cur_expr, +#endif + error_buf, error_buf_size)) { goto fail; } @@ -1278,8 +1512,21 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, goto fail; } +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + if (cur_expr != NULL) { + bh_memcpy_s(init_expr, sizeof(InitializerExpression), cur_expr, + sizeof(InitializerExpression)); + wasm_runtime_free(cur_expr); + } + else { + init_expr->init_expr_type = flag; + init_expr->u.unary.v = cur_value; + } + +#else init_expr->init_expr_type = flag; - init_expr->u = cur_value; + init_expr->u.unary.v = cur_value; +#endif /* end of WASM_ENABLE_EXTENDED_CONST_EXPR != 0 */ #if WASM_ENABLE_GC != 0 if (init_expr->init_expr_type == WASM_OP_GC_PREFIX) { @@ -1310,11 +1557,11 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, #endif /* end of WASM_ENABLE_GC != 0 */ *p_buf = p; - destroy_const_expr_stack(&const_expr_ctx); + destroy_const_expr_stack(&const_expr_ctx, false); return true; fail: - destroy_const_expr_stack(&const_expr_ctx); + destroy_const_expr_stack(&const_expr_ctx, true); return false; } @@ -2042,9 +2289,9 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, "recursive type count too large"); return false; } - module->type_count += rec_count - 1; new_total_size = - sizeof(WASMFuncType *) * (uint64)module->type_count; + sizeof(WASMFuncType *) + * (uint64)(module->type_count + rec_count - 1); if (new_total_size > UINT32_MAX) { set_error_buf(error_buf, error_buf_size, "allocate memory failed"); @@ -2052,12 +2299,18 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, } MEM_REALLOC(module->types, (uint32)total_size, (uint32)new_total_size); + module->type_count += rec_count - 1; total_size = new_total_size; } - LOG_VERBOSE("Processing rec group [%d-%d]", - processed_type_count, - processed_type_count + rec_count - 1); + if (rec_count < 1) { + LOG_VERBOSE("Processing 0-entry rec group"); + } + else { + LOG_VERBOSE("Processing rec group [%d-%d]", + processed_type_count, + processed_type_count + rec_count - 1); + } } else { p--; @@ -2243,6 +2496,13 @@ load_type_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, #endif /* end of WASM_ENABLE_GC == 0 */ } + for (i = 0; i < module->type_count; i++) { + if (module->types[i] == NULL) { + set_error_buf_v(error_buf, error_buf_size, "unknown type %d", i); + return false; + } + } + if (p != p_end) { set_error_buf(error_buf, error_buf_size, "section size mismatch"); return false; @@ -3351,7 +3611,8 @@ load_import_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, /* valtype */ CHECK_BUF(p, p_end, 1); global_type = read_uint8(p); - if (wasm_is_reftype_htref_nullable(global_type)) { + if (wasm_is_reftype_htref_nullable(global_type) + || wasm_is_reftype_htref_non_nullable(global_type)) { int32 heap_type; read_leb_int32(p, p_end, heap_type); (void)heap_type; @@ -4070,9 +4331,9 @@ load_global_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, if (global->init_expr.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { uint8 global_type; WASMRefType *global_ref_type; - uint32 global_idx = global->init_expr.u.global_index; + uint32 global_idx = global->init_expr.u.unary.v.global_index; - if (global->init_expr.u.global_index + if (global->init_expr.u.unary.v.global_index >= module->import_global_count + i) { set_error_buf(error_buf, error_buf_size, "unknown global"); return false; @@ -4469,7 +4730,7 @@ load_func_index_vec(const uint8 **p_buf, const uint8 *buf_end, } init_expr->init_expr_type = INIT_EXPR_TYPE_FUNCREF_CONST; - init_expr->u.ref_index = function_index; + init_expr->u.unary.v.ref_index = function_index; } *p_buf = p; @@ -4742,7 +5003,7 @@ load_table_segment_section(const uint8 *buf, const uint8 *buf_end, #if WASM_ENABLE_MEMORY64 != 0 if (table_elem_idx_type == VALUE_TYPE_I64 - && table_segment->base_offset.u.u64 > UINT32_MAX) { + && table_segment->base_offset.u.unary.v.u64 > UINT32_MAX) { set_error_buf(error_buf, error_buf_size, "In table64, table base offset can't be " "larger than UINT32_MAX"); @@ -4902,6 +5163,9 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end, if (!(dataseg = module->data_segments[i] = loader_malloc( sizeof(WASMDataSeg), error_buf, error_buf_size))) { +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + destroy_init_expr(module, &init_expr); +#endif return false; } @@ -6029,7 +6293,8 @@ load_from_sections(WASMModule *module, WASMSection *sections, && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST) { aux_heap_base_global = global; - aux_heap_base = (uint64)(uint32)global->init_expr.u.i32; + aux_heap_base = + (uint64)(uint32)global->init_expr.u.unary.v.i32; aux_heap_base_global_index = export->index; LOG_VERBOSE("Found aux __heap_base global, value: %" PRIu64, aux_heap_base); @@ -6050,7 +6315,8 @@ load_from_sections(WASMModule *module, WASMSection *sections, && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST) { aux_data_end_global = global; - aux_data_end = (uint64)(uint32)global->init_expr.u.i32; + aux_data_end = + (uint64)(uint32)global->init_expr.u.unary.v.i32; aux_data_end_global_index = export->index; LOG_VERBOSE("Found aux __data_end global, value: %" PRIu64, aux_data_end); @@ -6091,10 +6357,11 @@ load_from_sections(WASMModule *module, WASMSection *sections, && global->type.val_type == VALUE_TYPE_I32 && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST - && (uint64)(uint32)global->init_expr.u.i32 + && (uint64)(uint32)global->init_expr.u.unary.v.i32 <= aux_heap_base) { aux_stack_top_global = global; - aux_stack_top = (uint64)(uint32)global->init_expr.u.i32; + aux_stack_top = + (uint64)(uint32)global->init_expr.u.unary.v.i32; module->aux_stack_top_global_index = module->import_global_count + global_index; module->aux_stack_bottom = aux_stack_top; @@ -6945,7 +7212,7 @@ wasm_loader_unload(WASMModule *module) wasm_runtime_free(module->memories); if (module->globals) { -#if WASM_ENABLE_GC != 0 +#if WASM_ENABLE_GC != 0 || WASM_ENABLE_EXTENDED_CONST_EXPR != 0 for (i = 0; i < module->global_count; i++) { destroy_init_expr(module, &module->globals[i].init_expr); } @@ -6978,6 +7245,9 @@ wasm_loader_unload(WASMModule *module) #endif wasm_runtime_free(module->table_segments[i].init_values); } +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + destroy_init_expr(module, &module->table_segments[i].base_offset); +#endif } wasm_runtime_free(module->table_segments); } @@ -6987,6 +7257,10 @@ wasm_loader_unload(WASMModule *module) if (module->data_segments[i]) { if (module->data_segments[i]->is_data_cloned) wasm_runtime_free(module->data_segments[i]->data); +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + destroy_init_expr(module, + &(module->data_segments[i]->base_offset)); +#endif wasm_runtime_free(module->data_segments[i]); } } @@ -7356,7 +7630,7 @@ wasm_loader_find_block_addr(WASMExecEnv *exec_env, BlockAddr *block_addr_cache, case WASM_OP_SELECT: case WASM_OP_DROP_64: case WASM_OP_SELECT_64: -#if WASM_ENABLE_FAST_INTERP != 0 && WASM_ENABLE_SIMD != 0 +#if WASM_ENABLE_SIMDE != 0 case WASM_OP_SELECT_128: #endif break; @@ -12881,9 +13155,7 @@ re_scan: #if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) \ || (WASM_ENABLE_FAST_INTERP != 0) case VALUE_TYPE_V128: -#if WASM_ENABLE_FAST_INTERP == 0 - *(p - 1) = WASM_OP_SELECT_128; -#else +#if WASM_ENABLE_SIMDE != 0 if (loader_ctx->p_code_compiled) { uint8 opcode_tmp = WASM_OP_SELECT_128; #if WASM_ENABLE_LABELS_AS_VALUES != 0 @@ -13011,8 +13283,7 @@ re_scan: uint8 opcode_tmp = WASM_OP_SELECT; if (type == VALUE_TYPE_V128) { -#if WASM_ENABLE_JIT != 0 \ - || WASM_ENABLE_FAST_INTERP != 0 && WASM_ENABLE_SIMD != 0 +#if WASM_ENABLE_SIMDE != 0 opcode_tmp = WASM_OP_SELECT_128; #else set_error_buf(error_buf, error_buf_size, @@ -13258,7 +13529,8 @@ re_scan: == VALUE_TYPE_FUNCREF && module->globals[i].init_expr.init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST - && module->globals[i].init_expr.u.u32 == func_idx) { + && module->globals[i].init_expr.u.unary.v.u32 + == func_idx) { func_declared = true; break; } @@ -13287,7 +13559,8 @@ re_scan: #endif ) { for (j = 0; j < table_seg->value_count; j++) { - if (table_seg->init_values[j].u.ref_index + if (table_seg->init_values[j] + .u.unary.v.ref_index == func_idx) { func_declared = true; break; @@ -15023,8 +15296,6 @@ re_scan: case WASM_OP_STRING_NEW_LOSSY_UTF8: case WASM_OP_STRING_NEW_WTF8: { - uint32 memidx; - #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 func->has_memory_operations = true; #endif @@ -15036,7 +15307,6 @@ re_scan: POP_I32(); POP_I32(); PUSH_REF(REF_TYPE_STRINGREF); - (void)memidx; break; } case WASM_OP_STRING_CONST: @@ -15064,8 +15334,6 @@ re_scan: case WASM_OP_STRING_ENCODE_LOSSY_UTF8: case WASM_OP_STRING_ENCODE_WTF8: { - uint32 memidx; - #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 func->has_memory_operations = true; #endif @@ -15077,7 +15345,6 @@ re_scan: POP_I32(); POP_STRINGREF(); PUSH_I32(); - (void)memidx; break; } case WASM_OP_STRING_CONCAT: @@ -15118,8 +15385,6 @@ re_scan: case WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8: case WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8: { - uint32 memidx; - #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 func->has_memory_operations = true; #endif @@ -15134,7 +15399,6 @@ re_scan: POP_REF(REF_TYPE_STRINGVIEWWTF8); PUSH_I32(); PUSH_I32(); - (void)memidx; break; } case WASM_OP_STRINGVIEW_WTF8_SLICE: @@ -15166,8 +15430,6 @@ re_scan: } case WASM_OP_STRINGVIEW_WTF16_ENCODE: { - uint32 memidx; - #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 func->has_memory_operations = true; #endif @@ -15181,7 +15443,6 @@ re_scan: POP_I32(); POP_REF(REF_TYPE_STRINGVIEWWTF16); PUSH_I32(); - (void)memidx; break; } case WASM_OP_STRINGVIEW_WTF16_SLICE: diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 7ff1078c3..771538a14 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -261,6 +261,9 @@ typedef struct InitValue { uint8 type; uint8 flag; WASMValue value; +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + InitializerExpression *expr; +#endif } InitValue; typedef struct ConstExprContext { @@ -282,7 +285,11 @@ init_const_expr_stack(ConstExprContext *ctx, WASMModule *module) static bool push_const_expr_stack(ConstExprContext *ctx, uint8 flag, uint8 type, - WASMValue *value, char *error_buf, uint32 error_buf_size) + WASMValue *value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + InitializerExpression *expr, +#endif + char *error_buf, uint32 error_buf_size) { InitValue *cur_value; @@ -305,6 +312,9 @@ push_const_expr_stack(ConstExprContext *ctx, uint8 flag, uint8 type, cur_value->type = type; cur_value->flag = flag; cur_value->value = *value; +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + cur_value->expr = expr; +#endif return true; fail: @@ -313,7 +323,11 @@ fail: static bool pop_const_expr_stack(ConstExprContext *ctx, uint8 *p_flag, uint8 type, - WASMValue *p_value, char *error_buf, uint32 error_buf_size) + WASMValue *p_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + InitializerExpression **p_expr, +#endif + char *error_buf, uint32 error_buf_size) { InitValue *cur_value; @@ -331,18 +345,50 @@ pop_const_expr_stack(ConstExprContext *ctx, uint8 *p_flag, uint8 type, *p_flag = cur_value->flag; if (p_value) *p_value = cur_value->value; +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + if (p_expr) + *p_expr = cur_value->expr; +#endif return true; } static void -destroy_const_expr_stack(ConstExprContext *ctx) +destroy_const_expr_stack(ConstExprContext *ctx, bool free_exprs) { +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + if (free_exprs) { + for (uint32 j = 0; j < ctx->sp; j++) { + if (is_expr_binary_op(ctx->stack[j].expr->init_expr_type)) { + destroy_init_expr_recursive(ctx->stack[j].expr); + ctx->stack[j].expr = NULL; + } + } + } +#endif if (ctx->stack != ctx->data) { wasm_runtime_free(ctx->stack); } } +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 +static void +destroy_init_expr(InitializerExpression *expr) +{ + // free left expr and right exprs for binary oprand + if (is_expr_binary_op(expr->init_expr_type)) { + return; + } + if (expr->u.binary.l_expr) { + destroy_init_expr_recursive(expr->u.binary.l_expr); + } + if (expr->u.binary.r_expr) { + destroy_init_expr_recursive(expr->u.binary.r_expr); + } + expr->u.binary.l_expr = expr->u.binary.r_expr = NULL; +} +#endif /* end of WASM_ENABLE_EXTENDED_CONST_EXPR != 0 */ + static bool load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, InitializerExpression *init_expr, uint8 type, char *error_buf, @@ -353,6 +399,9 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, uint32 i; ConstExprContext const_expr_ctx = { 0 }; WASMValue cur_value = { 0 }; +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + InitializerExpression *cur_expr = NULL; +#endif init_const_expr_stack(&const_expr_ctx, module); @@ -367,8 +416,11 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, if (!push_const_expr_stack(&const_expr_ctx, flag, VALUE_TYPE_I32, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) { - bh_assert(0); + goto fail; } break; /* i64.const */ @@ -377,8 +429,11 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, if (!push_const_expr_stack(&const_expr_ctx, flag, VALUE_TYPE_I64, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) { - bh_assert(0); + goto fail; } break; /* f32.const */ @@ -390,8 +445,11 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, if (!push_const_expr_stack(&const_expr_ctx, flag, VALUE_TYPE_F32, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) { - bh_assert(0); + goto fail; } break; /* f64.const */ @@ -403,8 +461,11 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, if (!push_const_expr_stack(&const_expr_ctx, flag, VALUE_TYPE_F64, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) { - bh_assert(0); + goto fail; } break; @@ -417,13 +478,16 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, cur_value.ref_index = func_idx; if (!check_function_index(module, func_idx, error_buf, error_buf_size)) { - bh_assert(0); + goto fail; } if (!push_const_expr_stack(&const_expr_ctx, flag, VALUE_TYPE_FUNCREF, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif error_buf, error_buf_size)) { - bh_assert(0); + goto fail; } break; } @@ -438,9 +502,12 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, cur_value.ref_index = UINT32_MAX; if (!push_const_expr_stack(&const_expr_ctx, flag, type1, - &cur_value, error_buf, - error_buf_size)) { - bh_assert(0); + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) { + goto fail; } break; } @@ -471,15 +538,93 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, } if (!push_const_expr_stack(&const_expr_ctx, flag, global_type, - &cur_value, error_buf, - error_buf_size)) - bh_assert(0); + &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + NULL, +#endif + error_buf, error_buf_size)) + goto fail; break; } +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + case INIT_EXPR_TYPE_I32_ADD: + case INIT_EXPR_TYPE_I64_ADD: + case INIT_EXPR_TYPE_I32_SUB: + case INIT_EXPR_TYPE_I64_SUB: + case INIT_EXPR_TYPE_I32_MUL: + case INIT_EXPR_TYPE_I64_MUL: + { + InitializerExpression *l_expr, *r_expr; + WASMValue l_value, r_value; + uint8 l_flag, r_flag; + uint8 value_type; + + if (flag == INIT_EXPR_TYPE_I32_ADD + || flag == INIT_EXPR_TYPE_I32_SUB + || flag == INIT_EXPR_TYPE_I32_MUL) { + value_type = VALUE_TYPE_I32; + } + else { + value_type = VALUE_TYPE_I64; + } + + /* If right flag indicates a binary operation, right expr will + * be popped from stack. Otherwise, allocate a new expr for + * right expr. Same for left expr. + */ + if (!(pop_const_expr_stack(&const_expr_ctx, &r_flag, value_type, + &r_value, &r_expr, error_buf, + error_buf_size))) { + goto fail; + } + if (!is_expr_binary_op(r_flag)) { + if (!(r_expr = loader_malloc(sizeof(InitializerExpression), + error_buf, error_buf_size))) { + goto fail; + } + r_expr->init_expr_type = r_flag; + r_expr->u.unary.v = r_value; + } + + if (!(pop_const_expr_stack(&const_expr_ctx, &l_flag, value_type, + &l_value, &l_expr, error_buf, + error_buf_size))) { + destroy_init_expr_recursive(r_expr); + goto fail; + } + if (!is_expr_binary_op(l_flag)) { + if (!(l_expr = loader_malloc(sizeof(InitializerExpression), + error_buf, error_buf_size))) { + destroy_init_expr_recursive(r_expr); + goto fail; + } + l_expr->init_expr_type = l_flag; + l_expr->u.unary.v = l_value; + } + + if (!(cur_expr = loader_malloc(sizeof(InitializerExpression), + error_buf, error_buf_size))) { + destroy_init_expr_recursive(l_expr); + destroy_init_expr_recursive(r_expr); + goto fail; + } + cur_expr->init_expr_type = flag; + cur_expr->u.binary.l_expr = l_expr; + cur_expr->u.binary.r_expr = r_expr; + + if (!push_const_expr_stack(&const_expr_ctx, flag, value_type, + &cur_value, cur_expr, error_buf, + error_buf_size)) { + destroy_init_expr_recursive(cur_expr); + goto fail; + } + break; + } +#endif default: { - bh_assert(0); + goto fail; } } @@ -489,18 +634,42 @@ load_init_expr(WASMModule *module, const uint8 **p_buf, const uint8 *buf_end, /* There should be only one value left on the init value stack */ if (!pop_const_expr_stack(&const_expr_ctx, &flag, type, &cur_value, +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + &cur_expr, +#endif error_buf, error_buf_size)) { - bh_assert(0); + goto fail; } - bh_assert(const_expr_ctx.sp == 0); + if (const_expr_ctx.sp != 0) { + set_error_buf(error_buf, error_buf_size, + "type mismatch: illegal constant opcode sequence"); + goto fail; + } +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + if (cur_expr != NULL) { + bh_memcpy_s(init_expr, sizeof(InitializerExpression), cur_expr, + sizeof(InitializerExpression)); + wasm_runtime_free(cur_expr); + } + else { + init_expr->init_expr_type = flag; + init_expr->u.unary.v = cur_value; + } + +#else init_expr->init_expr_type = flag; - init_expr->u = cur_value; + init_expr->u.unary.v = cur_value; +#endif /* end of WASM_ENABLE_EXTENDED_CONST_EXPR != 0 */ *p_buf = p; - destroy_const_expr_stack(&const_expr_ctx); + destroy_const_expr_stack(&const_expr_ctx, false); return true; + +fail: + destroy_const_expr_stack(&const_expr_ctx, true); + return false; } static bool @@ -1385,13 +1554,14 @@ load_global_section(const uint8 *buf, const uint8 *buf_end, WASMModule *module, * global.get instructions are * only allowed to refer to imported globals. */ - uint32 target_global_index = global->init_expr.u.global_index; + uint32 target_global_index = + global->init_expr.u.unary.v.global_index; bh_assert(target_global_index < module->import_global_count); (void)target_global_index; } else if (INIT_EXPR_TYPE_FUNCREF_CONST == global->init_expr.init_expr_type) { - bh_assert(global->init_expr.u.ref_index + bh_assert(global->init_expr.u.unary.v.ref_index < module->import_function_count + module->function_count); } @@ -1575,7 +1745,7 @@ load_func_index_vec(const uint8 **p_buf, const uint8 *buf_end, } init_expr->init_expr_type = INIT_EXPR_TYPE_FUNCREF_CONST; - init_expr->u.ref_index = function_index; + init_expr->u.unary.v.ref_index = function_index; } *p_buf = p; @@ -1890,6 +2060,9 @@ load_data_segment_section(const uint8 *buf, const uint8 *buf_end, if (!(dataseg = module->data_segments[i] = loader_malloc( sizeof(WASMDataSeg), error_buf, error_buf_size))) { +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + destroy_init_expr(&init_expr); +#endif return false; } @@ -2778,7 +2951,8 @@ load_from_sections(WASMModule *module, WASMSection *sections, && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST) { aux_heap_base_global = global; - aux_heap_base = (uint64)(uint32)global->init_expr.u.i32; + aux_heap_base = + (uint64)(uint32)global->init_expr.u.unary.v.i32; aux_heap_base_global_index = export->index; LOG_VERBOSE("Found aux __heap_base global, value: %" PRIu64, aux_heap_base); @@ -2798,7 +2972,8 @@ load_from_sections(WASMModule *module, WASMSection *sections, && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST) { aux_data_end_global = global; - aux_data_end = (uint64)(uint32)global->init_expr.u.i32; + aux_data_end = + (uint64)(uint32)global->init_expr.u.unary.v.i32; aux_data_end_global_index = export->index; LOG_VERBOSE("Found aux __data_end global, value: %" PRIu64, aux_data_end); @@ -2838,10 +3013,11 @@ load_from_sections(WASMModule *module, WASMSection *sections, && global->type.val_type == VALUE_TYPE_I32 && global->init_expr.init_expr_type == INIT_EXPR_TYPE_I32_CONST - && (uint64)(uint32)global->init_expr.u.i32 + && (uint64)(uint32)global->init_expr.u.unary.v.i32 <= aux_heap_base) { aux_stack_top_global = global; - aux_stack_top = (uint64)(uint32)global->init_expr.u.i32; + aux_stack_top = + (uint64)(uint32)global->init_expr.u.unary.v.i32; module->aux_stack_top_global_index = module->import_global_count + global_index; module->aux_stack_bottom = aux_stack_top; @@ -3448,8 +3624,14 @@ wasm_loader_unload(WASMModule *module) if (module->memories) wasm_runtime_free(module->memories); - if (module->globals) + if (module->globals) { +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + for (i = 0; i < module->global_count; i++) { + destroy_init_expr(&module->globals[i].init_expr); + } +#endif wasm_runtime_free(module->globals); + } if (module->exports) wasm_runtime_free(module->exports); @@ -3458,6 +3640,9 @@ wasm_loader_unload(WASMModule *module) for (i = 0; i < module->table_seg_count; i++) { if (module->table_segments[i].init_values) wasm_runtime_free(module->table_segments[i].init_values); +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + destroy_init_expr(&module->table_segments[i].base_offset); +#endif } wasm_runtime_free(module->table_segments); } @@ -3467,6 +3652,9 @@ wasm_loader_unload(WASMModule *module) if (module->data_segments[i]) { if (module->data_segments[i]->is_data_cloned) wasm_runtime_free(module->data_segments[i]->data); +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + destroy_init_expr(&module->data_segments[i]->base_offset); +#endif wasm_runtime_free(module->data_segments[i]); } } @@ -7320,7 +7508,8 @@ re_scan: == VALUE_TYPE_FUNCREF && module->globals[i].init_expr.init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST - && module->globals[i].init_expr.u.u32 == func_idx) { + && module->globals[i].init_expr.u.unary.v.ref_index + == func_idx) { func_declared = true; break; } @@ -7334,7 +7523,8 @@ re_scan: i++, table_seg++) { if (table_seg->elem_type == VALUE_TYPE_FUNCREF) { for (j = 0; j < table_seg->value_count; j++) { - if (table_seg->init_values[j].u.ref_index + if (table_seg->init_values[j] + .u.unary.v.ref_index == func_idx) { func_declared = true; break; diff --git a/core/iwasm/interpreter/wasm_opcode.h b/core/iwasm/interpreter/wasm_opcode.h index 37d32b122..b049e0a3a 100644 --- a/core/iwasm/interpreter/wasm_opcode.h +++ b/core/iwasm/interpreter/wasm_opcode.h @@ -278,8 +278,7 @@ typedef enum WASMOpcode { DEBUG_OP_BREAK = 0xdc, /* debug break point */ #endif -#if WASM_ENABLE_JIT != 0 || WASM_ENABLE_FAST_INTERP != 0 \ - || WASM_ENABLE_WAMR_COMPILER != 0 && WASM_ENABLE_SIMD != 0 +#if WASM_ENABLE_SIMDE != 0 EXT_OP_SET_LOCAL_FAST_V128 = 0xdd, EXT_OP_TEE_LOCAL_FAST_V128 = 0xde, EXT_OP_COPY_STACK_TOP_V128 = 0xdf, @@ -798,7 +797,7 @@ typedef enum WASMAtomicEXTOpcode { #define SET_GOTO_TABLE_SIMD_PREFIX_ELEM() #endif -#if WASM_ENABLE_SIMD != 0 && WASM_ENABLE_SIMDE != 0 +#if WASM_ENABLE_SIMDE != 0 #define DEF_EXT_V128_HANDLE() \ SET_GOTO_TABLE_ELEM(EXT_OP_SET_LOCAL_FAST_V128), /* 0xdd */ \ SET_GOTO_TABLE_ELEM(EXT_OP_TEE_LOCAL_FAST_V128), /* 0xde */ \ diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 3cc2afe04..b4aa483d7 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -1165,6 +1165,81 @@ instantiate_array_global_recursive(WASMModule *module, } #endif +static bool +get_init_value_recursive(WASMModule *module, InitializerExpression *expr, + WASMGlobalInstance *globals, WASMValue *value, + char *error_buf, uint32 error_buf_size) +{ + uint8 flag = expr->init_expr_type; + switch (flag) { + case INIT_EXPR_TYPE_GET_GLOBAL: + { + if (!check_global_init_expr(module, expr->u.unary.v.global_index, + error_buf, error_buf_size)) { + goto fail; + } + + *value = globals[expr->u.unary.v.global_index].initial_value; + break; + } + case INIT_EXPR_TYPE_I32_CONST: + case INIT_EXPR_TYPE_I64_CONST: + { + *value = expr->u.unary.v; + break; + } +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + case INIT_EXPR_TYPE_I32_ADD: + case INIT_EXPR_TYPE_I32_SUB: + case INIT_EXPR_TYPE_I32_MUL: + case INIT_EXPR_TYPE_I64_ADD: + case INIT_EXPR_TYPE_I64_SUB: + case INIT_EXPR_TYPE_I64_MUL: + { + WASMValue l_value, r_value; + if (!expr->u.binary.l_expr || !expr->u.binary.r_expr) { + goto fail; + } + if (!get_init_value_recursive(module, expr->u.binary.l_expr, + globals, &l_value, error_buf, + error_buf_size)) { + goto fail; + } + if (!get_init_value_recursive(module, expr->u.binary.r_expr, + globals, &r_value, error_buf, + error_buf_size)) { + goto fail; + } + + if (flag == INIT_EXPR_TYPE_I32_ADD) { + value->i32 = l_value.i32 + r_value.i32; + } + else if (flag == INIT_EXPR_TYPE_I32_SUB) { + value->i32 = l_value.i32 - r_value.i32; + } + else if (flag == INIT_EXPR_TYPE_I32_MUL) { + value->i32 = l_value.i32 * r_value.i32; + } + else if (flag == INIT_EXPR_TYPE_I64_ADD) { + value->i64 = l_value.i64 + r_value.i64; + } + else if (flag == INIT_EXPR_TYPE_I64_SUB) { + value->i64 = l_value.i64 - r_value.i64; + } + else if (flag == INIT_EXPR_TYPE_I64_MUL) { + value->i64 = l_value.i64 * r_value.i64; + } + break; + } +#endif /* end of WASM_ENABLE_EXTENDED_CONST_EXPR != 0 */ + default: + goto fail; + } + return true; +fail: + return false; +} + /** * Instantiate globals in a module. */ @@ -1209,7 +1284,7 @@ globals_instantiate(WASMModule *module, WASMModuleInstance *module_inst, /* The linked global instance has been initialized, we just need to copy the value. */ global->initial_value = - global_import->import_global_linked->init_expr.u; + global_import->import_global_linked->init_expr.u.unary.v; } else #endif @@ -1245,17 +1320,23 @@ globals_instantiate(WASMModule *module, WASMModuleInstance *module_inst, #endif switch (flag) { + case INIT_EXPR_TYPE_I32_CONST: + case INIT_EXPR_TYPE_I64_CONST: case INIT_EXPR_TYPE_GET_GLOBAL: +#if WASM_ENABLE_EXTENDED_CONST_EXPR != 0 + case INIT_EXPR_TYPE_I32_ADD: + case INIT_EXPR_TYPE_I32_SUB: + case INIT_EXPR_TYPE_I32_MUL: + case INIT_EXPR_TYPE_I64_ADD: + case INIT_EXPR_TYPE_I64_SUB: + case INIT_EXPR_TYPE_I64_MUL: +#endif { - if (!check_global_init_expr(module, init_expr->u.global_index, - error_buf, error_buf_size)) { + if (!get_init_value_recursive(module, init_expr, globals, + &global->initial_value, error_buf, + error_buf_size)) { goto fail; } - - bh_memcpy_s( - &(global->initial_value), sizeof(WASMValue), - &(globals[init_expr->u.global_index].initial_value), - sizeof(globals[init_expr->u.global_index].initial_value)); break; } #if WASM_ENABLE_GC != 0 @@ -1267,11 +1348,12 @@ globals_instantiate(WASMModule *module, WASMModuleInstance *module_inst, uint32 type_idx; if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { - init_values = (WASMStructNewInitValues *)init_expr->u.data; + init_values = + (WASMStructNewInitValues *)init_expr->u.unary.v.data; type_idx = init_values->type_idx; } else { - type_idx = init_expr->u.type_index; + type_idx = init_expr->u.unary.v.type_index; } struct_obj = instantiate_struct_global_recursive( @@ -1294,12 +1376,14 @@ globals_instantiate(WASMModule *module, WASMModuleInstance *module_inst, uint32 type_idx, len; if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) { - type_idx = init_expr->u.array_new_default.type_index; - len = init_expr->u.array_new_default.length; + type_idx = + init_expr->u.unary.v.array_new_default.type_index; + len = init_expr->u.unary.v.array_new_default.length; array_init_value = &empty_value; } else { - init_values = (WASMArrayNewInitValues *)init_expr->u.data; + init_values = + (WASMArrayNewInitValues *)init_expr->u.unary.v.data; type_idx = init_values->type_idx; len = init_values->length; @@ -1318,13 +1402,12 @@ globals_instantiate(WASMModule *module, WASMModuleInstance *module_inst, case INIT_EXPR_TYPE_I31_NEW: { global->initial_value.gc_obj = - (wasm_obj_t)wasm_i31_obj_new(init_expr->u.i32); + (wasm_obj_t)wasm_i31_obj_new(init_expr->u.unary.v.i32); break; } #endif /* end of WASM_ENABLE_GC != 0 */ default: - bh_memcpy_s(&(global->initial_value), sizeof(WASMValue), - &(init_expr->u), sizeof(init_expr->u)); + global->initial_value = init_expr->u.unary.v; break; } @@ -2668,7 +2751,7 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, } STORE_PTR((void **)global_data, func_obj); global_data += sizeof(void *); - /* Also update the inital_value since other globals may + /* Also update the initial_value since other globals may * refer to this */ global->initial_value.gc_obj = (wasm_obj_t)func_obj; break; @@ -2698,6 +2781,7 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, uint8 *memory_data = NULL; uint64 memory_size = 0; WASMDataSeg *data_seg = module->data_segments[i]; + WASMValue offset_value; #if WASM_ENABLE_BULK_MEMORY != 0 if (data_seg->is_passive) @@ -2717,54 +2801,37 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, (uint64)memory->num_bytes_per_page * memory->cur_page_count; bh_assert(memory_data || memory_size == 0); - bh_assert(data_seg->base_offset.init_expr_type - == INIT_EXPR_TYPE_GET_GLOBAL - || data_seg->base_offset.init_expr_type - == (memory->is_memory64 ? INIT_EXPR_TYPE_I64_CONST - : INIT_EXPR_TYPE_I32_CONST)); + uint8 offset_flag = data_seg->base_offset.init_expr_type; + bh_assert(offset_flag == INIT_EXPR_TYPE_GET_GLOBAL + || (memory->is_memory64 ? is_valid_i64_offset(offset_flag) + : is_valid_i32_offset(offset_flag))); - if (data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { - if (!check_global_init_expr(module, - data_seg->base_offset.u.global_index, - error_buf, error_buf_size)) { - goto fail; - } + if (!get_init_value_recursive(module, &data_seg->base_offset, globals, + &offset_value, error_buf, + error_buf_size)) { + goto fail; + } + if (offset_flag == INIT_EXPR_TYPE_GET_GLOBAL) { if (!globals - || globals[data_seg->base_offset.u.global_index].type + || globals[data_seg->base_offset.u.unary.v.global_index].type != (memory->is_memory64 ? VALUE_TYPE_I64 : VALUE_TYPE_I32)) { set_error_buf(error_buf, error_buf_size, "data segment does not fit"); goto fail; } - -#if WASM_ENABLE_MEMORY64 != 0 - if (memory->is_memory64) { - base_offset = - (uint64)globals[data_seg->base_offset.u.global_index] - .initial_value.i64; - } - else -#endif - { - base_offset = - (uint32)globals[data_seg->base_offset.u.global_index] - .initial_value.i32; - } - } - else { -#if WASM_ENABLE_MEMORY64 != 0 - if (memory->is_memory64) { - base_offset = (uint64)data_seg->base_offset.u.i64; - } - else -#endif - { - base_offset = (uint32)data_seg->base_offset.u.i32; - } } +#if WASM_ENABLE_MEMORY64 != 0 + if (memory->is_memory64) { + base_offset = (uint64)offset_value.i64; + } + else +#endif + { + base_offset = (uint32)offset_value.i32; + } /* check offset */ if (base_offset > memory_size) { #if WASM_ENABLE_MEMORY64 != 0 @@ -2818,6 +2885,7 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, #else module_inst->e->shared_heap_start_off.u32[0] = UINT32_MAX; #endif + module_inst->e->shared_heap = NULL; #endif #if WASM_ENABLE_GC != 0 @@ -2841,36 +2909,39 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, || table->init_expr.init_expr_type == INIT_EXPR_TYPE_REFNULL_CONST); if (table->init_expr.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) { - if (!check_global_init_expr(module, table->init_expr.u.global_index, + if (!check_global_init_expr(module, + table->init_expr.u.unary.v.global_index, error_buf, error_buf_size)) { goto fail; } - table->init_expr.u.gc_obj = - globals[table->init_expr.u.global_index].initial_value.gc_obj; + table->init_expr.u.unary.v.gc_obj = + globals[table->init_expr.u.unary.v.global_index] + .initial_value.gc_obj; } else if (table->init_expr.init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST) { - uint32 func_idx = table->init_expr.u.ref_index; + uint32 func_idx = table->init_expr.u.unary.v.ref_index; if (func_idx != UINT32_MAX) { - if (!(table->init_expr.u.gc_obj = + if (!(table->init_expr.u.unary.v.gc_obj = wasm_create_func_obj(module_inst, func_idx, false, error_buf, error_buf_size))) goto fail; } else { - table->init_expr.u.gc_obj = NULL_REF; + table->init_expr.u.unary.v.gc_obj = NULL_REF; } } else if (table->init_expr.init_expr_type == INIT_EXPR_TYPE_REFNULL_CONST) { - table->init_expr.u.gc_obj = NULL_REF; + table->init_expr.u.unary.v.gc_obj = NULL_REF; } LOG_DEBUG("Init table [%d] elements from [%d] to [%d] as: %p", i, 0, - table_inst->cur_size, (void *)table->init_expr.u.gc_obj); + table_inst->cur_size, + (void *)table->init_expr.u.unary.v.gc_obj); for (j = 0; j < table_inst->cur_size; j++) { - *(table_data + j) = table->init_expr.u.gc_obj; + *(table_data + j) = table->init_expr.u.unary.v.gc_obj; } } #endif /* end of WASM_ENABLE_GC != 0 */ @@ -2882,6 +2953,7 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, /* has check it in loader */ WASMTableInstance *table = module_inst->tables[table_seg->table_index]; table_elem_type_t *table_data; + WASMValue offset_value; uint32 j; #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 uint8 tbl_elem_type; @@ -2940,48 +3012,37 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, continue; #endif + uint8 offset_flag = table_seg->base_offset.init_expr_type; #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 - bh_assert(table_seg->base_offset.init_expr_type - == INIT_EXPR_TYPE_I32_CONST - || table_seg->base_offset.init_expr_type - == INIT_EXPR_TYPE_GET_GLOBAL - || table_seg->base_offset.init_expr_type - == INIT_EXPR_TYPE_FUNCREF_CONST - || table_seg->base_offset.init_expr_type - == INIT_EXPR_TYPE_REFNULL_CONST); + bh_assert(offset_flag == INIT_EXPR_TYPE_GET_GLOBAL + || offset_flag == INIT_EXPR_TYPE_FUNCREF_CONST + || offset_flag == INIT_EXPR_TYPE_REFNULL_CONST + || is_valid_i32_offset(offset_flag)); #else - bh_assert(table_seg->base_offset.init_expr_type - == INIT_EXPR_TYPE_I32_CONST - || table_seg->base_offset.init_expr_type - == INIT_EXPR_TYPE_GET_GLOBAL); + bh_assert(offset_flag == INIT_EXPR_TYPE_GET_GLOBAL + || is_valid_i32_offset(offset_flag)); #endif - /* init vec(funcidx) or vec(expr) */ - if (table_seg->base_offset.init_expr_type - == INIT_EXPR_TYPE_GET_GLOBAL) { - if (!check_global_init_expr(module, - table_seg->base_offset.u.global_index, - error_buf, error_buf_size)) { - goto fail; - } + if (!get_init_value_recursive(module, &table_seg->base_offset, globals, + &offset_value, error_buf, + error_buf_size)) { + goto fail; + } + if (offset_flag == INIT_EXPR_TYPE_GET_GLOBAL) { if (!globals - || globals[table_seg->base_offset.u.global_index].type + || globals[table_seg->base_offset.u.unary.v.global_index].type != VALUE_TYPE_I32) { set_error_buf(error_buf, error_buf_size, "type mismatch: elements segment does not fit"); goto fail; } - - table_seg->base_offset.u.i32 = - globals[table_seg->base_offset.u.global_index] - .initial_value.i32; } /* check offset since length might negative */ - if ((uint32)table_seg->base_offset.u.i32 > table->cur_size) { - LOG_DEBUG("base_offset(%d) > table->cur_size(%d)", - table_seg->base_offset.u.i32, table->cur_size); + if ((uint32)offset_value.i32 > table->cur_size) { + LOG_DEBUG("base_offset(%d) > table->cur_size(%d)", offset_value.i32, + table->cur_size); #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 set_error_buf(error_buf, error_buf_size, "out of bounds table access"); @@ -2994,9 +3055,9 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, /* check offset + length(could be zero) */ length = table_seg->value_count; - if ((uint32)table_seg->base_offset.u.i32 + length > table->cur_size) { + if ((uint32)offset_value.i32 + length > table->cur_size) { LOG_DEBUG("base_offset(%d) + length(%d)> table->cur_size(%d)", - table_seg->base_offset.u.i32, length, table->cur_size); + offset_value.i32, length, table->cur_size); #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 set_error_buf(error_buf, error_buf_size, "out of bounds table access"); @@ -3026,10 +3087,10 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, case INIT_EXPR_TYPE_FUNCREF_CONST: { #if WASM_ENABLE_GC == 0 - ref = (void *)(uintptr_t)init_expr->u.ref_index; + ref = (void *)(uintptr_t)init_expr->u.unary.v.ref_index; #else WASMFuncObjectRef func_obj; - uint32 func_idx = init_expr->u.ref_index; + uint32 func_idx = init_expr->u.unary.v.ref_index; /* UINT32_MAX indicates that it is a null reference */ if (func_idx != UINT32_MAX) { if (!(func_obj = wasm_create_func_obj( @@ -3048,14 +3109,14 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, #if WASM_ENABLE_GC != 0 case INIT_EXPR_TYPE_GET_GLOBAL: { - if (!check_global_init_expr(module, - init_expr->u.global_index, - error_buf, error_buf_size)) { + if (!check_global_init_expr( + module, init_expr->u.unary.v.global_index, + error_buf, error_buf_size)) { goto fail; } - ref = - globals[init_expr->u.global_index].initial_value.gc_obj; + ref = globals[init_expr->u.unary.v.global_index] + .initial_value.gc_obj; break; } case INIT_EXPR_TYPE_STRUCT_NEW: @@ -3068,12 +3129,12 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, uint32 type_idx; if (flag == INIT_EXPR_TYPE_STRUCT_NEW) { - init_values = - (WASMStructNewInitValues *)init_expr->u.data; + init_values = (WASMStructNewInitValues *) + init_expr->u.unary.v.data; type_idx = init_values->type_idx; } else { - type_idx = init_expr->u.type_index; + type_idx = init_expr->u.unary.v.type_index; } struct_type = (WASMStructType *)module->types[type_idx]; @@ -3124,13 +3185,14 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, uint32 type_idx, len; if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) { - type_idx = init_expr->u.array_new_default.type_index; - len = init_expr->u.array_new_default.length; + type_idx = + init_expr->u.unary.v.array_new_default.type_index; + len = init_expr->u.unary.v.array_new_default.length; arr_init_val = &empty_val; } else { init_values = - (WASMArrayNewInitValues *)init_expr->u.data; + (WASMArrayNewInitValues *)init_expr->u.unary.v.data; type_idx = init_values->type_idx; len = init_values->length; @@ -3176,14 +3238,14 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, } case INIT_EXPR_TYPE_I31_NEW: { - ref = (wasm_obj_t)wasm_i31_obj_new(init_expr->u.i32); + ref = + (wasm_obj_t)wasm_i31_obj_new(init_expr->u.unary.v.i32); break; } #endif /* end of WASM_ENABLE_GC != 0 */ } - *(table_data + table_seg->base_offset.u.i32 + j) = - (table_elem_type_t)ref; + *(table_data + offset_value.i32 + j) = (table_elem_type_t)ref; } } @@ -4161,7 +4223,7 @@ wasm_get_module_inst_mem_consumption(const WASMModuleInstance *module_inst, sizeof(WASMMemoryInstance *) * module_inst->memory_count; for (i = 0; i < module_inst->memory_count; i++) { WASMMemoryInstance *memory = module_inst->memories[i]; - size = memory->num_bytes_per_page * memory->cur_page_count; + size = (uint64)memory->num_bytes_per_page * memory->cur_page_count; mem_conspn->memories_size += size; mem_conspn->app_heap_size += memory->heap_data_end - memory->heap_data; /* size of app heap structure */ @@ -4195,9 +4257,9 @@ wasm_get_module_inst_mem_consumption(const WASMModuleInstance *module_inst, #endif /* end of (WASM_ENABLE_MEMORY_PROFILING != 0) \ || (WASM_ENABLE_MEMORY_TRACING != 0) */ -#if WAMR_ENABLE_COPY_CALLSTACK != 0 +#if WASM_ENABLE_COPY_CALL_STACK != 0 uint32 -wasm_interp_copy_callstack(WASMExecEnv *exec_env, wasm_frame_t *buffer, +wasm_interp_copy_callstack(WASMExecEnv *exec_env, WASMCApiFrame *buffer, uint32 length, uint32 skip_n, char *error_buf, uint32_t error_buf_size) { @@ -4242,7 +4304,7 @@ wasm_interp_copy_callstack(WASMExecEnv *exec_env, wasm_frame_t *buffer, } return count >= skip_n ? count - skip_n : 0; } -#endif // WAMR_ENABLE_COPY_CALLSTACK +#endif // WASM_ENABLE_COPY_CALL_STACK #if WASM_ENABLE_DUMP_CALL_STACK != 0 bool @@ -4705,10 +4767,10 @@ llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx, for (i = 0; i < length; i++) { #if WASM_ENABLE_GC != 0 /* UINT32_MAX indicates that it is a null ref */ - if (init_values[i].u.ref_index != UINT32_MAX) { - if (!(func_obj = wasm_create_func_obj(module_inst, - init_values[i].u.ref_index, - true, NULL, 0))) { + if (init_values[i].u.unary.v.ref_index != UINT32_MAX) { + if (!(func_obj = wasm_create_func_obj( + module_inst, init_values[i].u.unary.v.ref_index, true, + NULL, 0))) { wasm_set_exception(module_inst, "null function reference"); return; } @@ -4718,7 +4780,7 @@ llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx, table_elems[i] = NULL_REF; } #else - table_elems[i] = init_values[i].u.ref_index; + table_elems[i] = init_values[i].u.unary.v.ref_index; #endif } } diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 8d38c8831..16c670f0f 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -93,12 +93,21 @@ typedef union { } MemBound; typedef struct WASMSharedHeap { - struct WASMSharedHeap *next; - void *heap_handle; - uint8 *base_addr; + /* The global shared heap list maintained in runtime, used for runtime + * destroy */ + DefPointer(struct WASMSharedHeap *, next); + /* The logical shared heap chain the shared heap in */ + DefPointer(struct WASMSharedHeap *, chain_next); + /* Will be null if shared heap is created from pre allocated memory chunk + * and don't need to dynamic malloc and free */ + DefPointer(void *, heap_handle); + DefPointer(uint8 *, base_addr); uint64 size; uint64 start_off_mem64; uint64 start_off_mem32; + /* The number of wasm apps it attached to, for a shared heap chain, only the + * list head need to maintain the valid attached_count */ + uint8 attached_count; } WASMSharedHeap; struct WASMMemoryInstance { @@ -364,8 +373,6 @@ typedef struct WASMModuleInstanceExtra { #endif #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *shared_heap; -#if WASM_ENABLE_JIT != 0 /* * Adjusted shared heap based addr to simple the calculation * in the aot code. The value is: @@ -373,7 +380,8 @@ typedef struct WASMModuleInstanceExtra { */ uint8 *shared_heap_base_addr_adj; MemBound shared_heap_start_off; -#endif + MemBound shared_heap_end_off; + WASMSharedHeap *shared_heap; #endif #if WASM_ENABLE_DEBUG_INTERP != 0 \ @@ -731,12 +739,12 @@ wasm_get_table_inst(const WASMModuleInstance *module_inst, uint32 tbl_idx) #if WASM_ENABLE_DUMP_CALL_STACK != 0 -#if WAMR_ENABLE_COPY_CALLSTACK != 0 +#if WASM_ENABLE_COPY_CALL_STACK != 0 uint32 -wasm_interp_copy_callstack(WASMExecEnv *exec_env, wasm_frame_t *buffer, +wasm_interp_copy_callstack(WASMExecEnv *exec_env, WASMCApiFrame *buffer, uint32 length, uint32 skip_n, char *error_buf, uint32_t error_buf_size); -#endif // WAMR_ENABLE_COPY_CALLSTACK +#endif // WASM_ENABLE_COPY_CALL_STACK bool wasm_interp_create_call_stack(struct WASMExecEnv *exec_env); diff --git a/core/iwasm/libraries/lib-socket/inc/wasi_socket_ext.h b/core/iwasm/libraries/lib-socket/inc/wasi_socket_ext.h index 78eb457f5..2bad1ebed 100644 --- a/core/iwasm/libraries/lib-socket/inc/wasi_socket_ext.h +++ b/core/iwasm/libraries/lib-socket/inc/wasi_socket_ext.h @@ -135,6 +135,28 @@ typedef struct __wasi_addr_info_hints_t { #define IPV6_LEAVE_GROUP 21 #define IPV6_V6ONLY 26 +/* getaddrinfo error codes. + * + * we use values compatible with wasi-libc/musl netdb.h. + * https://github.com/WebAssembly/wasi-libc/blob/4ea6fdfa288e15a57c02fe31dda78e5ddc87c3c7/libc-top-half/musl/include/netdb.h#L43-L53 + * + * for now, non-posix error codes are excluded: + * EAI_PROTOCOL and EAI_BADHINTS (BSDs) + * EAI_ADDRFAMILY, EAI_NODATA + * https://github.com/WebAssembly/wasi-libc/blob/4ea6fdfa288e15a57c02fe31dda78e5ddc87c3c7/libc-top-half/musl/include/netdb.h#L145-L152 + */ + +#define EAI_AGAIN -3 +#define EAI_BADFLAGS -1 +#define EAI_FAIL -4 +#define EAI_FAMILY -6 +#define EAI_MEMORY -10 +#define EAI_NONAME -2 +#define EAI_OVERFLOW -12 +#define EAI_SERVICE -8 +#define EAI_SOCKTYPE -7 +#define EAI_SYSTEM -11 + struct addrinfo { int ai_flags; /* Input flags. */ int ai_family; /* Protocol family for socket. */ @@ -197,6 +219,9 @@ getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, void freeaddrinfo(struct addrinfo *res); + +const char * +gai_strerror(int code); #endif /** diff --git a/core/iwasm/libraries/lib-socket/src/wasi/wasi_socket_ext.c b/core/iwasm/libraries/lib-socket/src/wasi/wasi_socket_ext.c index f573d35b8..8f80e0bbb 100644 --- a/core/iwasm/libraries/lib-socket/src/wasi/wasi_socket_ext.c +++ b/core/iwasm/libraries/lib-socket/src/wasi/wasi_socket_ext.c @@ -38,6 +38,13 @@ __errno_location(void); return -1; \ } +/* REVISIT: in many cases, EAI_SYSTEM may not be an ideal error code */ +#define GAI_HANDLE_ERROR(error) \ + if (error != __WASI_ERRNO_SUCCESS) { \ + errno = error; \ + return EAI_SYSTEM; \ + } + static void ipv4_addr_to_wasi_ip4_addr(uint32_t addr_num, __wasi_addr_ip4_t *out) { @@ -191,7 +198,7 @@ bind(int sockfd, const struct sockaddr *addr, socklen_t addrlen) error = __wasi_sock_bind(sockfd, &wasi_addr); HANDLE_ERROR(error) - return __WASI_ERRNO_SUCCESS; + return 0; } int @@ -212,7 +219,7 @@ connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) error = __wasi_sock_connect(sockfd, &wasi_addr); HANDLE_ERROR(error) - return __WASI_ERRNO_SUCCESS; + return 0; } int @@ -220,7 +227,7 @@ listen(int sockfd, int backlog) { __wasi_errno_t error = __wasi_sock_listen(sockfd, backlog); HANDLE_ERROR(error) - return __WASI_ERRNO_SUCCESS; + return 0; } ssize_t @@ -375,7 +382,7 @@ socket(int domain, int type, int protocol) af = INET6; } else { - return __WASI_ERRNO_NOPROTOOPT; + HANDLE_ERROR(__WASI_ERRNO_NOPROTOOPT) } if (SOCK_DGRAM == type) { @@ -385,7 +392,7 @@ socket(int domain, int type, int protocol) socktype = SOCKET_STREAM; } else { - return __WASI_ERRNO_NOPROTOOPT; + HANDLE_ERROR(__WASI_ERRNO_NOPROTOOPT) } error = __wasi_sock_open(poolfd, af, socktype, &sockfd); @@ -408,7 +415,7 @@ getsockname(int sockfd, struct sockaddr *addr, socklen_t *addrlen) error = wasi_addr_to_sockaddr(&wasi_addr, addr, addrlen); HANDLE_ERROR(error) - return __WASI_ERRNO_SUCCESS; + return 0; } int @@ -425,7 +432,7 @@ getpeername(int sockfd, struct sockaddr *addr, socklen_t *addrlen) error = wasi_addr_to_sockaddr(&wasi_addr, addr, addrlen); HANDLE_ERROR(error) - return __WASI_ERRNO_SUCCESS; + return 0; } struct aibuf { @@ -518,7 +525,7 @@ getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct aibuf *aibuf_res; error = addrinfo_hints_to_wasi_hints(hints, &wasi_hints); - HANDLE_ERROR(error) + GAI_HANDLE_ERROR(error) do { if (addr_info) @@ -529,7 +536,7 @@ getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, * sizeof(__wasi_addr_info_t)); if (!addr_info) { - HANDLE_ERROR(__WASI_ERRNO_NOMEM) + return EAI_MEMORY; } error = __wasi_sock_addr_resolve(node, service == NULL ? "" : service, @@ -537,29 +544,25 @@ getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, &max_info_size); if (error != __WASI_ERRNO_SUCCESS) { free(addr_info); - HANDLE_ERROR(error); + GAI_HANDLE_ERROR(error); } } while (max_info_size > addr_info_size); + addr_info_size = max_info_size; if (addr_info_size == 0) { free(addr_info); - *res = NULL; - return __WASI_ERRNO_SUCCESS; + return EAI_NONAME; } aibuf_res = (struct aibuf *)calloc(1, addr_info_size * sizeof(struct aibuf)); if (!aibuf_res) { free(addr_info); - HANDLE_ERROR(__WASI_ERRNO_NOMEM) + return EAI_MEMORY; } *res = &aibuf_res[0].ai; - if (addr_info_size) { - addr_info_size = max_info_size; - } - for (i = 0; i < addr_info_size; i++) { struct addrinfo *ai = &aibuf_res[i].ai; ai->ai_addr = (struct sockaddr *)&aibuf_res[i].sa; @@ -568,14 +571,14 @@ getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, if (error != __WASI_ERRNO_SUCCESS) { free(addr_info); free(aibuf_res); - HANDLE_ERROR(error) + GAI_HANDLE_ERROR(error) } ai->ai_next = i == addr_info_size - 1 ? NULL : &aibuf_res[i + 1].ai; } free(addr_info); - return __WASI_ERRNO_SUCCESS; + return 0; } void @@ -587,6 +590,28 @@ freeaddrinfo(struct addrinfo *res) free(res); } +const char * +gai_strerror(int code) +{ + switch (code) { +#define ERR(a) \ + case a: \ + return #a + ERR(EAI_AGAIN); + ERR(EAI_BADFLAGS); + ERR(EAI_FAIL); + ERR(EAI_FAMILY); + ERR(EAI_MEMORY); + ERR(EAI_NONAME); + ERR(EAI_OVERFLOW); + ERR(EAI_SERVICE); + ERR(EAI_SOCKTYPE); + ERR(EAI_SYSTEM); +#undef ERR + } + return "Unknown error"; +} + static struct timeval time_us_to_timeval(uint64_t time_us) { @@ -610,6 +635,7 @@ get_sol_socket_option(int sockfd, int optname, void *__restrict optval, uint64_t timeout_us; bool is_linger_enabled; int linger_s; + __wasi_fdstat_t sb; switch (optname) { case SO_RCVTIMEO: @@ -617,38 +643,38 @@ get_sol_socket_option(int sockfd, int optname, void *__restrict optval, error = __wasi_sock_get_recv_timeout(sockfd, &timeout_us); HANDLE_ERROR(error); *(struct timeval *)optval = time_us_to_timeval(timeout_us); - return error; + return 0; case SO_SNDTIMEO: assert(*optlen == sizeof(struct timeval)); error = __wasi_sock_get_send_timeout(sockfd, &timeout_us); HANDLE_ERROR(error); *(struct timeval *)optval = time_us_to_timeval(timeout_us); - return error; + return 0; case SO_SNDBUF: assert(*optlen == sizeof(int)); error = __wasi_sock_get_send_buf_size(sockfd, (size_t *)optval); HANDLE_ERROR(error); - return error; + return 0; case SO_RCVBUF: assert(*optlen == sizeof(int)); error = __wasi_sock_get_recv_buf_size(sockfd, (size_t *)optval); HANDLE_ERROR(error); - return error; + return 0; case SO_KEEPALIVE: assert(*optlen == sizeof(int)); error = __wasi_sock_get_keep_alive(sockfd, (bool *)optval); HANDLE_ERROR(error); - return error; + return 0; case SO_REUSEADDR: assert(*optlen == sizeof(int)); error = __wasi_sock_get_reuse_addr(sockfd, (bool *)optval); HANDLE_ERROR(error); - return error; + return 0; case SO_REUSEPORT: assert(*optlen == sizeof(int)); error = __wasi_sock_get_reuse_port(sockfd, (bool *)optval); HANDLE_ERROR(error); - return error; + return 0; case SO_LINGER: assert(*optlen == sizeof(struct linger)); error = @@ -656,16 +682,32 @@ get_sol_socket_option(int sockfd, int optname, void *__restrict optval, HANDLE_ERROR(error); ((struct linger *)optval)->l_onoff = (int)is_linger_enabled; ((struct linger *)optval)->l_linger = linger_s; - return error; + return 0; case SO_BROADCAST: assert(*optlen == sizeof(int)); error = __wasi_sock_get_broadcast(sockfd, (bool *)optval); HANDLE_ERROR(error); - return error; + return 0; + case SO_TYPE: + assert(*optlen == sizeof(int)); + error = __wasi_fd_fdstat_get(sockfd, &sb); + HANDLE_ERROR(error); + switch (sb.fs_filetype) { + case __WASI_FILETYPE_SOCKET_DGRAM: + *(int *)optval = SOCK_DGRAM; + break; + case __WASI_FILETYPE_SOCKET_STREAM: + *(int *)optval = SOCK_STREAM; + break; + default: + errno = __WASI_ERRNO_NOTSOCK; + return -1; + } + return 0; default: error = __WASI_ERRNO_NOTSUP; HANDLE_ERROR(error); - return error; + return 0; } } @@ -679,32 +721,32 @@ get_ipproto_tcp_option(int sockfd, int optname, void *__restrict optval, assert(*optlen == sizeof(uint32_t)); error = __wasi_sock_get_tcp_keep_idle(sockfd, (uint32_t *)optval); HANDLE_ERROR(error); - return error; + return 0; case TCP_KEEPINTVL: assert(*optlen == sizeof(uint32_t)); error = __wasi_sock_get_tcp_keep_intvl(sockfd, (uint32_t *)optval); HANDLE_ERROR(error); - return error; + return 0; case TCP_FASTOPEN_CONNECT: assert(*optlen == sizeof(int)); error = __wasi_sock_get_tcp_fastopen_connect(sockfd, (bool *)optval); HANDLE_ERROR(error); - return error; + return 0; case TCP_NODELAY: assert(*optlen == sizeof(int)); error = __wasi_sock_get_tcp_no_delay(sockfd, (bool *)optval); HANDLE_ERROR(error); - return error; + return 0; case TCP_QUICKACK: assert(*optlen == sizeof(int)); error = __wasi_sock_get_tcp_quick_ack(sockfd, (bool *)optval); HANDLE_ERROR(error); - return error; + return 0; default: error = __WASI_ERRNO_NOTSUP; HANDLE_ERROR(error); - return error; + return 0; } } @@ -720,21 +762,21 @@ get_ipproto_ip_option(int sockfd, int optname, void *__restrict optval, error = __wasi_sock_get_ip_multicast_loop(sockfd, false, (bool *)optval); HANDLE_ERROR(error); - return error; + return 0; case IP_TTL: assert(*optlen == sizeof(int)); error = __wasi_sock_get_ip_ttl(sockfd, (uint8_t *)optval); HANDLE_ERROR(error); - return error; + return 0; case IP_MULTICAST_TTL: assert(*optlen == sizeof(int)); error = __wasi_sock_get_ip_multicast_ttl(sockfd, (uint8_t *)optval); HANDLE_ERROR(error); - return error; + return 0; default: error = __WASI_ERRNO_NOTSUP; HANDLE_ERROR(error); - return error; + return 0; } } @@ -749,17 +791,17 @@ get_ipproto_ipv6_option(int sockfd, int optname, void *__restrict optval, assert(*optlen == sizeof(int)); error = __wasi_sock_get_ipv6_only(sockfd, (bool *)optval); HANDLE_ERROR(error); - return error; + return 0; case IPV6_MULTICAST_LOOP: assert(*optlen == sizeof(int)); error = __wasi_sock_get_ip_multicast_loop(sockfd, true, (bool *)optval); HANDLE_ERROR(error); - return error; + return 0; default: error = __WASI_ERRNO_NOTSUP; HANDLE_ERROR(error); - return error; + return 0; } } @@ -781,7 +823,7 @@ getsockopt(int sockfd, int level, int optname, void *__restrict optval, default: error = __WASI_ERRNO_NOTSUP; HANDLE_ERROR(error); - return error; + return 0; } } @@ -799,7 +841,7 @@ set_sol_socket_option(int sockfd, int optname, const void *optval, timeout_us = timeval_to_time_us(*(struct timeval *)optval); error = __wasi_sock_set_recv_timeout(sockfd, timeout_us); HANDLE_ERROR(error); - return error; + return 0; } case SO_SNDTIMEO: { @@ -807,42 +849,42 @@ set_sol_socket_option(int sockfd, int optname, const void *optval, timeout_us = timeval_to_time_us(*(struct timeval *)optval); error = __wasi_sock_set_send_timeout(sockfd, timeout_us); HANDLE_ERROR(error); - return error; + return 0; } case SO_SNDBUF: { assert(optlen == sizeof(int)); error = __wasi_sock_set_send_buf_size(sockfd, *(size_t *)optval); HANDLE_ERROR(error); - return error; + return 0; } case SO_RCVBUF: { assert(optlen == sizeof(int)); error = __wasi_sock_set_recv_buf_size(sockfd, *(size_t *)optval); HANDLE_ERROR(error); - return error; + return 0; } case SO_KEEPALIVE: { assert(optlen == sizeof(int)); error = __wasi_sock_set_keep_alive(sockfd, *(bool *)optval); HANDLE_ERROR(error); - return error; + return 0; } case SO_REUSEADDR: { assert(optlen == sizeof(int)); error = __wasi_sock_set_reuse_addr(sockfd, *(bool *)optval); HANDLE_ERROR(error); - return error; + return 0; } case SO_REUSEPORT: { assert(optlen == sizeof(int)); error = __wasi_sock_set_reuse_port(sockfd, *(bool *)optval); HANDLE_ERROR(error); - return error; + return 0; } case SO_LINGER: { @@ -851,20 +893,20 @@ set_sol_socket_option(int sockfd, int optname, const void *optval, error = __wasi_sock_set_linger(sockfd, (bool)linger_opt->l_onoff, linger_opt->l_linger); HANDLE_ERROR(error); - return error; + return 0; } case SO_BROADCAST: { assert(optlen == sizeof(int)); error = __wasi_sock_set_broadcast(sockfd, *(bool *)optval); HANDLE_ERROR(error); - return error; + return 0; } default: { error = __WASI_ERRNO_NOTSUP; HANDLE_ERROR(error); - return error; + return 0; } } } @@ -880,32 +922,32 @@ set_ipproto_tcp_option(int sockfd, int optname, const void *optval, assert(optlen == sizeof(int)); error = __wasi_sock_set_tcp_no_delay(sockfd, *(bool *)optval); HANDLE_ERROR(error); - return error; + return 0; case TCP_KEEPIDLE: assert(optlen == sizeof(uint32_t)); error = __wasi_sock_set_tcp_keep_idle(sockfd, *(uint32_t *)optval); HANDLE_ERROR(error); - return error; + return 0; case TCP_KEEPINTVL: assert(optlen == sizeof(uint32_t)); error = __wasi_sock_set_tcp_keep_intvl(sockfd, *(uint32_t *)optval); HANDLE_ERROR(error); - return error; + return 0; case TCP_FASTOPEN_CONNECT: assert(optlen == sizeof(int)); error = __wasi_sock_set_tcp_fastopen_connect(sockfd, *(bool *)optval); HANDLE_ERROR(error); - return error; + return 0; case TCP_QUICKACK: assert(optlen == sizeof(int)); error = __wasi_sock_set_tcp_quick_ack(sockfd, *(bool *)optval); HANDLE_ERROR(error); - return error; + return 0; default: error = __WASI_ERRNO_NOTSUP; HANDLE_ERROR(error); - return error; + return 0; } } @@ -923,7 +965,7 @@ set_ipproto_ip_option(int sockfd, int optname, const void *optval, error = __wasi_sock_set_ip_multicast_loop(sockfd, false, *(bool *)optval); HANDLE_ERROR(error); - return error; + return 0; case IP_ADD_MEMBERSHIP: assert(optlen == sizeof(struct ip_mreq)); ip_mreq_opt = (struct ip_mreq *)optval; @@ -933,7 +975,7 @@ set_ipproto_ip_option(int sockfd, int optname, const void *optval, error = __wasi_sock_set_ip_add_membership( sockfd, &imr_multiaddr, ip_mreq_opt->imr_interface.s_addr); HANDLE_ERROR(error); - return error; + return 0; case IP_DROP_MEMBERSHIP: assert(optlen == sizeof(struct ip_mreq)); ip_mreq_opt = (struct ip_mreq *)optval; @@ -943,22 +985,22 @@ set_ipproto_ip_option(int sockfd, int optname, const void *optval, error = __wasi_sock_set_ip_drop_membership( sockfd, &imr_multiaddr, ip_mreq_opt->imr_interface.s_addr); HANDLE_ERROR(error); - return error; + return 0; case IP_TTL: assert(optlen == sizeof(int)); error = __wasi_sock_set_ip_ttl(sockfd, *(uint8_t *)optval); HANDLE_ERROR(error); - return error; + return 0; case IP_MULTICAST_TTL: assert(optlen == sizeof(int)); error = __wasi_sock_set_ip_multicast_ttl(sockfd, *(uint8_t *)optval); HANDLE_ERROR(error); - return error; + return 0; default: error = __WASI_ERRNO_NOTSUP; HANDLE_ERROR(error); - return error; + return 0; } } @@ -975,13 +1017,13 @@ set_ipproto_ipv6_option(int sockfd, int optname, const void *optval, assert(optlen == sizeof(int)); error = __wasi_sock_set_ipv6_only(sockfd, *(bool *)optval); HANDLE_ERROR(error); - return error; + return 0; case IPV6_MULTICAST_LOOP: assert(optlen == sizeof(int)); error = __wasi_sock_set_ip_multicast_loop(sockfd, true, *(bool *)optval); HANDLE_ERROR(error); - return error; + return 0; case IPV6_JOIN_GROUP: assert(optlen == sizeof(struct ipv6_mreq)); ipv6_mreq_opt = (struct ipv6_mreq *)optval; @@ -992,7 +1034,7 @@ set_ipproto_ipv6_option(int sockfd, int optname, const void *optval, error = __wasi_sock_set_ip_add_membership( sockfd, &imr_multiaddr, ipv6_mreq_opt->ipv6mr_interface); HANDLE_ERROR(error); - return error; + return 0; case IPV6_LEAVE_GROUP: assert(optlen == sizeof(struct ipv6_mreq)); ipv6_mreq_opt = (struct ipv6_mreq *)optval; @@ -1003,11 +1045,11 @@ set_ipproto_ipv6_option(int sockfd, int optname, const void *optval, error = __wasi_sock_set_ip_drop_membership( sockfd, &imr_multiaddr, ipv6_mreq_opt->ipv6mr_interface); HANDLE_ERROR(error); - return error; + return 0; default: error = __WASI_ERRNO_NOTSUP; HANDLE_ERROR(error); - return error; + return 0; } } @@ -1029,6 +1071,6 @@ setsockopt(int sockfd, int level, int optname, const void *optval, default: error = __WASI_ERRNO_NOTSUP; HANDLE_ERROR(error); - return error; + return 0; } } diff --git a/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c b/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c index 609a0ec3a..acd7c31c9 100644 --- a/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c +++ b/core/iwasm/libraries/libc-wasi/libc_wasi_wrapper.c @@ -375,6 +375,9 @@ wasi_fd_pread(wasm_exec_env_t exec_env, wasi_fd_t fd, iovec_app_t *iovec_app, return (wasi_errno_t)-1; total_size = sizeof(wasi_iovec_t) * (uint64)iovs_len; + if (total_size == 0) { + total_size = 1; /* avoid user-triggered 0-sized allocation */ + } if (total_size >= UINT32_MAX || !(iovec_begin = wasm_runtime_malloc((uint32)total_size))) return (wasi_errno_t)-1; @@ -430,6 +433,9 @@ wasi_fd_pwrite(wasm_exec_env_t exec_env, wasi_fd_t fd, return (wasi_errno_t)-1; total_size = sizeof(wasi_ciovec_t) * (uint64)iovs_len; + if (total_size == 0) { + total_size = 1; /* avoid user-triggered 0-sized allocation */ + } if (total_size >= UINT32_MAX || !(ciovec_begin = wasm_runtime_malloc((uint32)total_size))) return (wasi_errno_t)-1; @@ -484,6 +490,9 @@ wasi_fd_read(wasm_exec_env_t exec_env, wasi_fd_t fd, return (wasi_errno_t)-1; total_size = sizeof(wasi_iovec_t) * (uint64)iovs_len; + if (total_size == 0) { + total_size = 1; /* avoid user-triggered 0-sized allocation */ + } if (total_size >= UINT32_MAX || !(iovec_begin = wasm_runtime_malloc((uint32)total_size))) return (wasi_errno_t)-1; @@ -654,6 +663,9 @@ wasi_fd_write(wasm_exec_env_t exec_env, wasi_fd_t fd, return (wasi_errno_t)-1; total_size = sizeof(wasi_ciovec_t) * (uint64)iovs_len; + if (total_size == 0) { + total_size = 1; /* avoid user-triggered 0-sized allocation */ + } if (total_size >= UINT32_MAX || !(ciovec_begin = wasm_runtime_malloc((uint32)total_size))) return (wasi_errno_t)-1; diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index 55e0526c3..8f3a6317b 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -301,7 +301,8 @@ wasm_cluster_create(WASMExecEnv *exec_env) aux_stack_start -= cluster->stack_size; for (i = 0; i < cluster_max_thread_num; i++) { - cluster->stack_tops[i] = aux_stack_start - cluster->stack_size * i; + cluster->stack_tops[i] = + aux_stack_start - (uint64)cluster->stack_size * i; } } #endif diff --git a/core/iwasm/libraries/wasi-nn/include/wasi_nn.h b/core/iwasm/libraries/wasi-nn/include/wasi_nn.h index 48ffe1263..cda26324e 100644 --- a/core/iwasm/libraries/wasi-nn/include/wasi_nn.h +++ b/core/iwasm/libraries/wasi-nn/include/wasi_nn.h @@ -21,6 +21,7 @@ #else #define WASI_NN_IMPORT(name) \ __attribute__((import_module("wasi_nn"), import_name(name))) +#warning You are using "wasi_nn", which is a legacy WAMR-specific ABI. It's deperecated and will likely be removed in future versions of WAMR. Please use "wasi_ephemeral_nn" instead. (For a WASM module, use the wasi_ephemeral_nn.h header instead. For the runtime configurations, enable WASM_ENABLE_WASI_EPHEMERAL_NN/WAMR_BUILD_WASI_EPHEMERAL_NN.) #endif /** @@ -108,14 +109,13 @@ WASI_NN_NAME(compute) WASI_NN_ERROR_TYPE WASI_NN_NAME(get_output) (WASI_NN_NAME(graph_execution_context) ctx, uint32_t index, - WASI_NN_NAME(tensor_data) output_tensor, uint32_t output_tensor_max_size, + uint8_t *output_tensor, uint32_t output_tensor_max_size, uint32_t *output_tensor_size) WASI_NN_IMPORT("get_output"); #else WASI_NN_ERROR_TYPE WASI_NN_NAME(get_output) -(graph_execution_context ctx, uint32_t index, - WASI_NN_NAME(tensor_data) output_tensor, uint32_t *output_tensor_size) - WASI_NN_IMPORT("get_output"); +(graph_execution_context ctx, uint32_t index, uint8_t *output_tensor, + uint32_t *output_tensor_size) WASI_NN_IMPORT("get_output"); #endif #endif diff --git a/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h b/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h index 7980197b7..952fb65e2 100644 --- a/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h +++ b/core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h @@ -99,7 +99,14 @@ typedef enum { // 4-byte f32 elements would have a data array of length 16). Naturally, this // representation requires some knowledge of how to lay out data in // memory--e.g., using row-major ordering--and could perhaps be improved. +#if !defined(__wasm__) || WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +typedef struct { + uint8_t *buf; + uint32_t size; +} WASI_NN_NAME(tensor_data); +#else typedef uint8_t *WASI_NN_NAME(tensor_data); +#endif // A tensor. typedef struct { diff --git a/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c b/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c index 6e91c949b..4d56fed93 100644 --- a/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c +++ b/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c @@ -99,7 +99,8 @@ graph_builder_array_app_native(wasm_module_inst_t instance, static wasi_nn_error tensor_data_app_native(wasm_module_inst_t instance, uint32_t total_elements, - tensor_wasm *input_tensor_wasm, tensor_data *data) + tensor_wasm *input_tensor_wasm, void **data, + uint32_t *size) { #if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 #define data_size input_tensor_wasm->data_size @@ -113,8 +114,9 @@ tensor_data_app_native(wasm_module_inst_t instance, uint32_t total_elements, NN_ERR_PRINTF("input_tensor_wasm->data_offset is invalid"); return invalid_argument; } - *data = (tensor_data)wasm_runtime_addr_app_to_native( + *data = wasm_runtime_addr_app_to_native( instance, (uint64)input_tensor_wasm->data_offset); + *size = data_size; return success; #undef data_size } @@ -188,16 +190,19 @@ tensor_app_native(wasm_module_inst_t instance, tensor_wasm *input_tensor_wasm, NN_DBG_PRINTF("Tensor type: %d", input_tensor_wasm->type); NN_DBG_PRINTF("Total number of elements: %d", total_elements); - tensor_data data = NULL; + void *data = NULL; + uint32_t datasize; if (success - != (res = tensor_data_app_native(instance, total_elements, - input_tensor_wasm, &data))) { + != (res = + tensor_data_app_native(instance, total_elements, + input_tensor_wasm, &data, &datasize))) { wasm_runtime_free(dimensions); return res; } input_tensor->type = input_tensor_wasm->type; input_tensor->dimensions = dimensions; - input_tensor->data = data; + input_tensor->data.buf = data; + input_tensor->data.size = datasize; return success; } diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn.c b/core/iwasm/libraries/wasi-nn/src/wasi_nn.c index 6098ef465..7921ec953 100644 --- a/core/iwasm/libraries/wasi-nn/src/wasi_nn.c +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn.c @@ -20,6 +20,10 @@ #include "wasi_nn_types.h" #include "wasm_export.h" +#if WASM_ENABLE_WASI_EPHEMERAL_NN == 0 +#warning You are using "wasi_nn", which is a legacy WAMR-specific ABI. It's deperecated and will likely be removed in future versions of WAMR. Please use "wasi_ephemeral_nn" instead. (For a WASM module, use the wasi_ephemeral_nn.h header instead. For the runtime configurations, enable WASM_ENABLE_WASI_EPHEMERAL_NN/WAMR_BUILD_WASI_EPHEMERAL_NN.) +#endif + #define HASHMAP_INITIAL_SIZE 20 #if defined(__APPLE__) #define LIB_EXTENTION ".dylib" @@ -51,53 +55,21 @@ struct backends_api_functions { NN_ERR_PRINTF("Error %s() -> %d", #func, wasi_error); \ } while (0) -/* HashMap utils */ -static HashMap *hashmap; - -static uint32 -hash_func(const void *key) -{ - // fnv1a_hash - const uint32 FNV_PRIME = 16777619; - const uint32 FNV_OFFSET_BASIS = 2166136261U; - - uint32 hash = FNV_OFFSET_BASIS; - const unsigned char *bytes = (const unsigned char *)key; - - for (size_t i = 0; i < sizeof(uintptr_t); ++i) { - hash ^= bytes[i]; - hash *= FNV_PRIME; - } - - return hash; -} - -static bool -key_equal_func(void *key1, void *key2) -{ - return key1 == key2; -} - -static void -key_destroy_func(void *key1) -{ - /* key type is wasm_module_inst_t*. do nothing */ -} +static void *wasi_nn_key; static void wasi_nn_ctx_destroy(WASINNContext *wasi_nn_ctx) { - NN_DBG_PRINTF("[WASI NN] DEINIT..."); - if (wasi_nn_ctx == NULL) { - NN_ERR_PRINTF( - "Error when deallocating memory. WASI-NN context is NULL"); return; } + NN_DBG_PRINTF("[WASI NN] DEINIT..."); NN_DBG_PRINTF("Freeing wasi-nn"); NN_DBG_PRINTF("-> is_model_loaded: %d", wasi_nn_ctx->is_model_loaded); NN_DBG_PRINTF("-> current_encoding: %d", wasi_nn_ctx->backend); + bh_assert(!wasi_nn_ctx->busy); + /* deinit() the backend */ if (wasi_nn_ctx->is_backend_ctx_initialized) { wasi_nn_error res; @@ -105,13 +77,14 @@ wasi_nn_ctx_destroy(WASINNContext *wasi_nn_ctx) wasi_nn_ctx->backend_ctx); } + os_mutex_destroy(&wasi_nn_ctx->lock); wasm_runtime_free(wasi_nn_ctx); } static void -value_destroy_func(void *value) +dtor(wasm_module_inst_t inst, void *ctx) { - wasi_nn_ctx_destroy((WASINNContext *)value); + wasi_nn_ctx_destroy(ctx); } bool @@ -124,12 +97,9 @@ wasi_nn_initialize() return false; } - // hashmap { instance: wasi_nn_ctx } - hashmap = bh_hash_map_create(HASHMAP_INITIAL_SIZE, true, hash_func, - key_equal_func, key_destroy_func, - value_destroy_func); - if (hashmap == NULL) { - NN_ERR_PRINTF("Error while initializing hashmap"); + wasi_nn_key = wasm_runtime_create_context_key(dtor); + if (wasi_nn_key == NULL) { + NN_ERR_PRINTF("Failed to create context key"); os_mutex_destroy(&wasi_nn_lock); return false; } @@ -150,6 +120,11 @@ wasi_nn_initialize_context() } memset(wasi_nn_ctx, 0, sizeof(WASINNContext)); + if (os_mutex_init(&wasi_nn_ctx->lock)) { + NN_ERR_PRINTF("Error when initializing a lock for WASI-NN context"); + wasm_runtime_free(wasi_nn_ctx); + return NULL; + } return wasi_nn_ctx; } @@ -158,29 +133,59 @@ static WASINNContext * wasm_runtime_get_wasi_nn_ctx(wasm_module_inst_t instance) { WASINNContext *wasi_nn_ctx = - (WASINNContext *)bh_hash_map_find(hashmap, (void *)instance); + wasm_runtime_get_context(instance, wasi_nn_key); if (wasi_nn_ctx == NULL) { - wasi_nn_ctx = wasi_nn_initialize_context(); - if (wasi_nn_ctx == NULL) - return NULL; - - bool ok = - bh_hash_map_insert(hashmap, (void *)instance, (void *)wasi_nn_ctx); - if (!ok) { - NN_ERR_PRINTF("Error while storing context"); - wasi_nn_ctx_destroy(wasi_nn_ctx); + WASINNContext *newctx = wasi_nn_initialize_context(); + if (newctx == NULL) return NULL; + os_mutex_lock(&wasi_nn_lock); + wasi_nn_ctx = wasm_runtime_get_context(instance, wasi_nn_key); + if (wasi_nn_ctx == NULL) { + wasm_runtime_set_context_spread(instance, wasi_nn_key, newctx); + wasi_nn_ctx = newctx; + newctx = NULL; + } + os_mutex_unlock(&wasi_nn_lock); + if (newctx != NULL) { + wasi_nn_ctx_destroy(newctx); } } - return wasi_nn_ctx; } +static WASINNContext * +lock_ctx(wasm_module_inst_t instance) +{ + WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); + if (wasi_nn_ctx == NULL) { + return NULL; + } + os_mutex_lock(&wasi_nn_ctx->lock); + if (wasi_nn_ctx->busy) { + os_mutex_unlock(&wasi_nn_ctx->lock); + return NULL; + } + wasi_nn_ctx->busy = true; + os_mutex_unlock(&wasi_nn_ctx->lock); + return wasi_nn_ctx; +} + +static void +unlock_ctx(WASINNContext *wasi_nn_ctx) +{ + if (wasi_nn_ctx == NULL) { + return; + } + os_mutex_lock(&wasi_nn_ctx->lock); + bh_assert(wasi_nn_ctx->busy); + wasi_nn_ctx->busy = false; + os_mutex_unlock(&wasi_nn_ctx->lock); +} + void wasi_nn_destroy() { - // destroy hashmap will destroy keys and values - bh_hash_map_destroy(hashmap); + wasm_runtime_destroy_context_key(wasi_nn_key); // close backends' libraries and registered functions for (unsigned i = 0; i < sizeof(lookup) / sizeof(lookup[0]); i++) { @@ -401,7 +406,7 @@ detect_and_load_backend(graph_encoding backend_hint, static wasi_nn_error ensure_backend(wasm_module_inst_t instance, graph_encoding encoding, - WASINNContext **wasi_nn_ctx_ptr) + WASINNContext *wasi_nn_ctx) { wasi_nn_error res; @@ -412,7 +417,6 @@ ensure_backend(wasm_module_inst_t instance, graph_encoding encoding, goto fail; } - WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); if (wasi_nn_ctx->is_backend_ctx_initialized) { if (wasi_nn_ctx->backend != loaded_backend) { res = unsupported_operation; @@ -430,7 +434,6 @@ ensure_backend(wasm_module_inst_t instance, graph_encoding encoding, wasi_nn_ctx->is_backend_ctx_initialized = true; } - *wasi_nn_ctx_ptr = wasi_nn_ctx; return success; fail: return res; @@ -458,17 +461,23 @@ wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder, if (!instance) return runtime_error; + WASINNContext *wasi_nn_ctx = lock_ctx(instance); + if (wasi_nn_ctx == NULL) { + res = busy; + goto fail; + } + graph_builder_array builder_native = { 0 }; #if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 if (success != (res = graph_builder_array_app_native( instance, builder, builder_wasm_size, &builder_native))) - return res; + goto fail; #else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ if (success != (res = graph_builder_array_app_native(instance, builder, &builder_native))) - return res; + goto fail; #endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ if (!wasm_runtime_validate_native_addr(instance, g, @@ -478,8 +487,7 @@ wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder, goto fail; } - WASINNContext *wasi_nn_ctx; - res = ensure_backend(instance, encoding, &wasi_nn_ctx); + res = ensure_backend(instance, encoding, wasi_nn_ctx); if (res != success) goto fail; @@ -494,14 +502,40 @@ fail: // XXX: Free intermediate structure pointers if (builder_native.buf) wasm_runtime_free(builder_native.buf); + unlock_ctx(wasi_nn_ctx); return res; } +static wasi_nn_error +copyin_and_nul_terminate(wasm_module_inst_t inst, char *name, uint32_t name_len, + char **resultp) +{ + char *nul_terminated_name; + if (!wasm_runtime_validate_native_addr(inst, name, name_len)) { + return invalid_argument; + } + nul_terminated_name = wasm_runtime_malloc(name_len + 1); + if (nul_terminated_name == NULL) { + return runtime_error; + } + bh_memcpy_s(nul_terminated_name, name_len + 1, name, name_len); + nul_terminated_name[name_len] = '\0'; /* ensure NUL termination */ + if (strlen(nul_terminated_name) != name_len) { + /* reject names containing '\0' for now */ + wasm_runtime_free(nul_terminated_name); + return invalid_argument; + } + *resultp = nul_terminated_name; + return success; +} + wasi_nn_error wasi_nn_load_by_name(wasm_exec_env_t exec_env, char *name, uint32_t name_len, graph *g) { + WASINNContext *wasi_nn_ctx = NULL; + char *nul_terminated_name = NULL; wasi_nn_error res; wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); @@ -509,36 +543,46 @@ wasi_nn_load_by_name(wasm_exec_env_t exec_env, char *name, uint32_t name_len, return runtime_error; } - if (!wasm_runtime_validate_native_addr(instance, name, name_len)) { - NN_ERR_PRINTF("name is invalid"); - return invalid_argument; - } - if (!wasm_runtime_validate_native_addr(instance, g, (uint64)sizeof(graph))) { NN_ERR_PRINTF("graph is invalid"); return invalid_argument; } - if (name_len == 0 || name[name_len] != '\0') { - NN_ERR_PRINTF("Invalid filename"); - return invalid_argument; + res = copyin_and_nul_terminate(instance, name, name_len, + &nul_terminated_name); + if (res != success) { + goto fail; } - NN_DBG_PRINTF("[WASI NN] LOAD_BY_NAME %s...", name); + NN_DBG_PRINTF("[WASI NN] LOAD_BY_NAME %s...", nul_terminated_name); - WASINNContext *wasi_nn_ctx; - res = ensure_backend(instance, autodetect, &wasi_nn_ctx); + wasi_nn_ctx = lock_ctx(instance); + if (wasi_nn_ctx == NULL) { + res = busy; + goto fail; + } + + res = ensure_backend(instance, autodetect, wasi_nn_ctx); if (res != success) - return res; + goto fail; call_wasi_nn_func(wasi_nn_ctx->backend, load_by_name, res, - wasi_nn_ctx->backend_ctx, name, name_len, g); + wasi_nn_ctx->backend_ctx, nul_terminated_name, name_len, + g); if (res != success) - return res; + goto fail; wasi_nn_ctx->is_model_loaded = true; - return success; + res = success; +fail: + if (nul_terminated_name != NULL) { + wasm_runtime_free(nul_terminated_name); + } + if (wasi_nn_ctx != NULL) { + unlock_ctx(wasi_nn_ctx); + } + return res; } wasi_nn_error @@ -546,6 +590,9 @@ wasi_nn_load_by_name_with_config(wasm_exec_env_t exec_env, char *name, int32_t name_len, char *config, int32_t config_len, graph *g) { + WASINNContext *wasi_nn_ctx = NULL; + char *nul_terminated_name = NULL; + char *nul_terminated_config = NULL; wasi_nn_error res; wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env); @@ -553,42 +600,56 @@ wasi_nn_load_by_name_with_config(wasm_exec_env_t exec_env, char *name, return runtime_error; } - if (!wasm_runtime_validate_native_addr(instance, name, name_len)) { - NN_ERR_PRINTF("name is invalid"); - return invalid_argument; - } - if (!wasm_runtime_validate_native_addr(instance, g, (uint64)sizeof(graph))) { NN_ERR_PRINTF("graph is invalid"); return invalid_argument; } - if (name_len == 0 || name[name_len] != '\0') { - NN_ERR_PRINTF("Invalid filename"); - return invalid_argument; + res = copyin_and_nul_terminate(instance, name, name_len, + &nul_terminated_name); + if (res != success) { + goto fail; + } + res = copyin_and_nul_terminate(instance, config, config_len, + &nul_terminated_config); + if (res != success) { + goto fail; } - if (!config || config_len == 0 || config[config_len] != '\0') { - NN_ERR_PRINTF("Invalid config"); - return invalid_argument; + NN_DBG_PRINTF("[WASI NN] LOAD_BY_NAME_WITH_CONFIG %s %s...", + nul_terminated_name, nul_terminated_config); + + wasi_nn_ctx = lock_ctx(instance); + if (wasi_nn_ctx == NULL) { + res = busy; + goto fail; } - NN_DBG_PRINTF("[WASI NN] LOAD_BY_NAME_WITH_CONFIG %s %s...", name, config); - - WASINNContext *wasi_nn_ctx; - res = ensure_backend(instance, autodetect, &wasi_nn_ctx); + res = ensure_backend(instance, autodetect, wasi_nn_ctx); if (res != success) - return res; + goto fail; + ; call_wasi_nn_func(wasi_nn_ctx->backend, load_by_name_with_config, res, - wasi_nn_ctx->backend_ctx, name, name_len, config, - config_len, g); + wasi_nn_ctx->backend_ctx, nul_terminated_name, name_len, + nul_terminated_config, config_len, g); if (res != success) - return res; + goto fail; wasi_nn_ctx->is_model_loaded = true; - return success; + res = success; +fail: + if (nul_terminated_name != NULL) { + wasm_runtime_free(nul_terminated_name); + } + if (nul_terminated_config != NULL) { + wasm_runtime_free(nul_terminated_config); + } + if (wasi_nn_ctx != NULL) { + unlock_ctx(wasi_nn_ctx); + } + return res; } wasi_nn_error @@ -602,20 +663,27 @@ wasi_nn_init_execution_context(wasm_exec_env_t exec_env, graph g, return runtime_error; } - WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); - wasi_nn_error res; + WASINNContext *wasi_nn_ctx = lock_ctx(instance); + if (wasi_nn_ctx == NULL) { + res = busy; + goto fail; + } + if (success != (res = is_model_initialized(wasi_nn_ctx))) - return res; + goto fail; if (!wasm_runtime_validate_native_addr( instance, ctx, (uint64)sizeof(graph_execution_context))) { NN_ERR_PRINTF("ctx is invalid"); - return invalid_argument; + res = invalid_argument; + goto fail; } call_wasi_nn_func(wasi_nn_ctx->backend, init_execution_context, res, wasi_nn_ctx->backend_ctx, g, ctx); +fail: + unlock_ctx(wasi_nn_ctx); return res; } @@ -630,17 +698,21 @@ wasi_nn_set_input(wasm_exec_env_t exec_env, graph_execution_context ctx, return runtime_error; } - WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); - wasi_nn_error res; + WASINNContext *wasi_nn_ctx = lock_ctx(instance); + if (wasi_nn_ctx == NULL) { + res = busy; + goto fail; + } + if (success != (res = is_model_initialized(wasi_nn_ctx))) - return res; + goto fail; tensor input_tensor_native = { 0 }; if (success != (res = tensor_app_native(instance, input_tensor, &input_tensor_native))) - return res; + goto fail; call_wasi_nn_func(wasi_nn_ctx->backend, set_input, res, wasi_nn_ctx->backend_ctx, ctx, index, @@ -648,7 +720,8 @@ wasi_nn_set_input(wasm_exec_env_t exec_env, graph_execution_context ctx, // XXX: Free intermediate structure pointers if (input_tensor_native.dimensions) wasm_runtime_free(input_tensor_native.dimensions); - +fail: + unlock_ctx(wasi_nn_ctx); return res; } @@ -662,26 +735,32 @@ wasi_nn_compute(wasm_exec_env_t exec_env, graph_execution_context ctx) return runtime_error; } - WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); - wasi_nn_error res; + WASINNContext *wasi_nn_ctx = lock_ctx(instance); + if (wasi_nn_ctx == NULL) { + res = busy; + goto fail; + } + if (success != (res = is_model_initialized(wasi_nn_ctx))) - return res; + goto fail; call_wasi_nn_func(wasi_nn_ctx->backend, compute, res, wasi_nn_ctx->backend_ctx, ctx); +fail: + unlock_ctx(wasi_nn_ctx); return res; } #if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 wasi_nn_error wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx, - uint32_t index, tensor_data output_tensor, + uint32_t index, void *output_tensor, uint32_t output_tensor_len, uint32_t *output_tensor_size) #else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ wasi_nn_error wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx, - uint32_t index, tensor_data output_tensor, + uint32_t index, void *output_tensor, uint32_t *output_tensor_size) #endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ { @@ -692,28 +771,36 @@ wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx, return runtime_error; } - WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance); - wasi_nn_error res; + WASINNContext *wasi_nn_ctx = lock_ctx(instance); + if (wasi_nn_ctx == NULL) { + res = busy; + goto fail; + } + if (success != (res = is_model_initialized(wasi_nn_ctx))) - return res; + goto fail; if (!wasm_runtime_validate_native_addr(instance, output_tensor_size, (uint64)sizeof(uint32_t))) { NN_ERR_PRINTF("output_tensor_size is invalid"); - return invalid_argument; + res = invalid_argument; + goto fail; } + tensor_data tensor = { + .buf = output_tensor, #if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + .size = output_tensor_len, +#else + .size = *output_tensor_size, +#endif + }; call_wasi_nn_func(wasi_nn_ctx->backend, get_output, res, - wasi_nn_ctx->backend_ctx, ctx, index, output_tensor, - &output_tensor_len); - *output_tensor_size = output_tensor_len; -#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ - call_wasi_nn_func(wasi_nn_ctx->backend, get_output, res, - wasi_nn_ctx->backend_ctx, ctx, index, output_tensor, + wasi_nn_ctx->backend_ctx, ctx, index, &tensor, output_tensor_size); -#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ +fail: + unlock_ctx(wasi_nn_ctx); return res; } @@ -728,6 +815,12 @@ static NativeSymbol native_symbols_wasi_nn[] = { #if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 REG_NATIVE_FUNC(load, "(*iii*)i"), REG_NATIVE_FUNC(load_by_name, "(*i*)i"), + + /* load_by_name_with_config is intented to be compatible with + * a wasmedge extension. + * https://github.com/second-state/wasmedge-wasi-nn/pull/2 + * https://github.com/WasmEdge/WasmEdge/blob/5553924e8cdccdc2cbd2a6a6d0ed9b11250c353e/plugins/wasi_nn/wasinnmodule.cpp#L13-L14 + */ REG_NATIVE_FUNC(load_by_name_with_config, "(*i*i*)i"), REG_NATIVE_FUNC(init_execution_context, "(i*)i"), REG_NATIVE_FUNC(set_input, "(ii*)i"), diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.h b/core/iwasm/libraries/wasi-nn/src/wasi_nn_backend.h similarity index 64% rename from core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.h rename to core/iwasm/libraries/wasi-nn/src/wasi_nn_backend.h index ea03a226f..8cd03f121 100644 --- a/core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.h +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn_backend.h @@ -3,15 +3,26 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#ifndef WASI_NN_OPENVINO_HPP -#define WASI_NN_OPENVINO_HPP +#ifndef WASI_NN_BACKEND_H +#define WASI_NN_BACKEND_H #include "wasi_nn_types.h" +#ifdef __cplusplus +extern "C" { +#endif + __attribute__((visibility("default"))) wasi_nn_error load(void *ctx, graph_builder_array *builder, graph_encoding encoding, execution_target target, graph *g); +__attribute__((visibility("default"))) wasi_nn_error +load_by_name(void *tflite_ctx, const char *name, uint32_t namelen, graph *g); + +__attribute__((visibility("default"))) wasi_nn_error +load_by_name_with_config(void *ctx, const char *name, uint32_t namelen, + const char *config, uint32_t config_len, graph *g); + __attribute__((visibility("default"))) wasi_nn_error init_execution_context(void *ctx, graph g, graph_execution_context *exec_ctx); @@ -24,7 +35,7 @@ compute(void *ctx, graph_execution_context exec_ctx); __attribute__((visibility("default"))) wasi_nn_error get_output(void *ctx, graph_execution_context exec_ctx, uint32_t index, - tensor_data output_tensor, uint32_t *output_tensor_size); + tensor_data *output_tensor, uint32_t *output_tensor_size); __attribute__((visibility("default"))) wasi_nn_error init_backend(void **ctx); @@ -32,4 +43,8 @@ init_backend(void **ctx); __attribute__((visibility("default"))) wasi_nn_error deinit_backend(void *ctx); -#endif /* WASI_NN_OPENVINO_HPP */ \ No newline at end of file +#ifdef __cplusplus +} +#endif + +#endif /* WASI_NN_BACKEND_H */ diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn_llamacpp.c b/core/iwasm/libraries/wasi-nn/src/wasi_nn_llamacpp.c index 23c867b0a..2e1e64936 100644 --- a/core/iwasm/libraries/wasi-nn/src/wasi_nn_llamacpp.c +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn_llamacpp.c @@ -2,7 +2,10 @@ * Copyright (C) 2019 Intel Corporation. All rights reserved. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#include "wasi_nn_types.h" + +#include + +#include "wasi_nn_backend.h" #include "utils/logger.h" #include "llama.h" #include "ggml.h" @@ -14,6 +17,10 @@ extern char const *LLAMA_COMMIT; extern char const *LLAMA_COMPILER; extern char const *LLAMA_BUILD_TARGET; +#if WASM_ENABLE_WASI_EPHEMERAL_NN == 0 +#error This backend doesn't support legacy "wasi_nn" abi. Please enable WASM_ENABLE_WASI_EPHEMERAL_NN. +#endif + // compatible with WasmEdge // https://github.com/second-state/WasmEdge-WASINN-examples/blob/master/wasmedge-ggml/README.md#parameters // https://github.com/WasmEdge/WasmEdge/blob/master/plugins/wasi_nn/ggml.cpp @@ -286,7 +293,7 @@ deinit_backend(void *ctx) llama_backend_free(); - os_free(backend_ctx); + free(backend_ctx); return success; } @@ -302,6 +309,11 @@ __load_by_name_with_configuration(void *ctx, const char *filename, graph *g) { struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; + if (backend_ctx->model != NULL) { + // we only implement a single graph + return unsupported_operation; + } + // make sure backend_ctx->config is initialized struct llama_model_params model_params = @@ -320,6 +332,7 @@ __load_by_name_with_configuration(void *ctx, const char *filename, graph *g) #endif backend_ctx->model = model; + *g = 0; return success; } @@ -360,6 +373,16 @@ init_execution_context(void *ctx, graph g, graph_execution_context *exec_ctx) { struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; + if (g != 0 || backend_ctx->model == NULL) { + // we only implement a single graph + return runtime_error; + } + + if (backend_ctx->ctx != NULL) { + // we only implement a single context + return unsupported_operation; + } + struct llama_context_params ctx_params = llama_context_params_from_wasi_nn_llama_config(&backend_ctx->config); struct llama_context *llama_ctx = @@ -370,6 +393,7 @@ init_execution_context(void *ctx, graph g, graph_execution_context *exec_ctx) } backend_ctx->ctx = llama_ctx; + *exec_ctx = 0; NN_INFO_PRINTF("n_predict = %d, n_ctx = %d", backend_ctx->config.n_predict, llama_n_ctx(backend_ctx->ctx)); @@ -381,18 +405,41 @@ set_input(void *ctx, graph_execution_context exec_ctx, uint32_t index, tensor *wasi_nn_tensor) { struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; - // tensor->data is the prompt string. ends with \0 - char *prompt_text = (char *)wasi_nn_tensor->data; + + if (exec_ctx != 0 || backend_ctx->ctx == NULL) { + // we only implement a single context + return runtime_error; + } + + if (index != 0) { + NN_ERR_PRINTF("Invalid input index %d", index); + return invalid_argument; + } + + // tensor->data is the prompt string. + char *prompt_text = (char *)wasi_nn_tensor->data.buf; + uint32_t prompt_text_len = wasi_nn_tensor->data.size; + + // note: buf[0] == 1 is a workaround for + // https://github.com/second-state/WasmEdge-WASINN-examples/issues/196. + // we may remove it in future. + if (wasi_nn_tensor->type != u8 || wasi_nn_tensor->dimensions->size != 1 + || !(wasi_nn_tensor->dimensions->buf[0] == 1 + || wasi_nn_tensor->dimensions->buf[0] == prompt_text_len)) { + return invalid_argument; + } + if (wasi_nn_tensor->dimensions->buf[0] == 1 && prompt_text_len != 1) { + NN_WARN_PRINTF("Ignoring seemingly wrong input tensor dimensions."); + } #ifndef NDEBUG NN_DBG_PRINTF("--------------------------------------------------"); - NN_DBG_PRINTF("prompt_text: %s", prompt_text); + NN_DBG_PRINTF("prompt_text: %.*s", (int)prompt_text_len, prompt_text); NN_DBG_PRINTF("--------------------------------------------------"); #endif // tokenize the prompt uint32_t n_token_max = llama_n_ctx(backend_ctx->ctx); - uint32_t prompt_text_len = strlen(prompt_text); if (backend_ctx->prompt == NULL) { backend_ctx->prompt = calloc(n_token_max, sizeof(llama_token)); @@ -430,6 +477,11 @@ compute(void *ctx, graph_execution_context exec_ctx) struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; wasi_nn_error ret = runtime_error; + if (exec_ctx != 0 || backend_ctx->ctx == NULL) { + // we only implement a single context + return runtime_error; + } + // reset the generation buffer if (backend_ctx->generation == NULL) { backend_ctx->generation = @@ -477,7 +529,6 @@ compute(void *ctx, graph_execution_context exec_ctx) // main loop int32_t n_cur = batch.n_tokens; - int n_decode = 0; int32_t n_vocab = llama_n_vocab(backend_ctx->model); llama_token_data *candidates = NULL; @@ -528,7 +579,6 @@ compute(void *ctx, graph_execution_context exec_ctx) // push this new token for next evaluation llama_batch_add(&batch, new_token_id, n_cur, seq_ids, sizeof(seq_ids) / sizeof(seq_ids[0]), true); - n_decode++; n_cur++; if (llama_decode(backend_ctx->ctx, batch) != 0) { @@ -549,10 +599,15 @@ fail: __attribute__((visibility("default"))) wasi_nn_error get_output(void *ctx, graph_execution_context exec_ctx, uint32_t index, - tensor_data output_tensor, uint32_t *output_tensor_size) + tensor_data *output_tensor, uint32_t *output_tensor_size) { struct LlamaContext *backend_ctx = (struct LlamaContext *)ctx; + if (exec_ctx != 0 || backend_ctx->ctx == NULL) { + // we only implement a single context + return runtime_error; + } + // Compatibility with WasmEdge if (index > 1) { NN_ERR_PRINTF("Invalid output index %d", index); @@ -568,7 +623,7 @@ get_output(void *ctx, graph_execution_context exec_ctx, uint32_t index, printf("%s\n", output_metadata); } - memcpy(output_tensor, output_metadata, strlen(output_metadata)); + memcpy(output_tensor->buf, output_metadata, strlen(output_metadata)); *output_tensor_size = strlen(output_metadata); return success; } @@ -588,7 +643,7 @@ get_output(void *ctx, graph_execution_context exec_ctx, uint32_t index, printf("%s", buf); } - memcpy(output_tensor + end_pos, buf, strlen(buf)); + memcpy(output_tensor->buf + end_pos, buf, strlen(buf)); end_pos += strlen(buf); } diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.c b/core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.c index 1cd1625d6..899e06ee3 100644 --- a/core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.c +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.c @@ -3,13 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#include "wasi_nn_types.h" -#include "wasi_nn_openvino.h" +#include "wasi_nn_backend.h" #include "utils/logger.h" #include "bh_platform.h" #include "openvino/c/openvino.h" +#if WASM_ENABLE_WASI_EPHEMERAL_NN == 0 +#error This backend doesn't support legacy "wasi_nn" abi. Please enable WASM_ENABLE_WASI_EPHEMERAL_NN. +#endif + /* * refer to * https://docs.openvino.ai/2024/openvino-workflow/running-inference/integrate-openvino-with-your-application.html @@ -26,15 +29,25 @@ * from 4. to 6. is the Inference Loop */ +/* these limits are arbitrary. */ +#define MAX_GRAPHS 4 +#define MAX_EXECUTION_CONTEXTS 4 + typedef struct { ov_core_t *core; /* keep input model files */ - void *weight_data; - ov_tensor_t *weights_tensor; - ov_model_t *model; - ov_compiled_model_t *compiled_model; - ov_infer_request_t *infer_request; - ov_tensor_t *input_tensor; + struct OpenVINOGraph { + void *weight_data; + ov_tensor_t *weights_tensor; + ov_model_t *model; + ov_compiled_model_t *compiled_model; + } graphs[MAX_GRAPHS]; + struct OpenVINOExecutionContext { + struct OpenVINOGraph *graph; + ov_infer_request_t *infer_request; + } execution_contexts[MAX_EXECUTION_CONTEXTS]; + unsigned int n_graphs; + unsigned int n_execution_contexts; } OpenVINOContext; /* @@ -134,7 +147,7 @@ print_model_input_output_info(ov_model_t *model) output_port = NULL; } - ov_error = ov_error; + (void)ov_error; fail: if (friendly_name) ov_free(friendly_name); @@ -179,6 +192,29 @@ wasi_nn_tensor_type_to_openvino_element_type(tensor_type wasi_nn_type) return UNDEFINED; } +static void +free_graph(struct OpenVINOGraph *graph) +{ + if (graph->weight_data) + os_free(graph->weight_data); + + if (graph->weights_tensor) + ov_tensor_free(graph->weights_tensor); + + if (graph->model) + ov_model_free(graph->model); + + if (graph->compiled_model) + ov_compiled_model_free(graph->compiled_model); +} + +static void +free_execution_context(struct OpenVINOExecutionContext *c) +{ + if (c->infer_request) + ov_infer_request_free(c->infer_request); +} + static wasi_nn_error uint32_array_to_int64_array(uint32_t array_size, uint32_t *src, int64_t **dst) { @@ -198,6 +234,8 @@ load(void *ctx, graph_builder_array *builder, graph_encoding encoding, execution_target target, graph *g) { OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + struct OpenVINOGraph *graph; + unsigned int graph_idx; wasi_nn_error ret = unsupported_operation; if (encoding != openvino) { @@ -223,33 +261,47 @@ load(void *ctx, graph_builder_array *builder, graph_encoding encoding, graph_builder xml = builder->buf[0]; graph_builder weight = builder->buf[1]; + graph_idx = ov_ctx->n_graphs; + if (graph_idx >= MAX_GRAPHS) { + return runtime_error; + } + graph = &ov_ctx->graphs[graph_idx]; + memset(graph, 0, sizeof(*graph)); + /* transfer weight to an ov tensor */ { - ov_ctx->weight_data = os_malloc(weight.size); - if (!ov_ctx->weight_data) + graph->weight_data = os_malloc(weight.size); + if (!graph->weight_data) goto fail; - memcpy(ov_ctx->weight_data, weight.buf, weight.size); + memcpy(graph->weight_data, weight.buf, weight.size); ov_element_type_e type = U8; int64_t dims[1] = { weight.size }; ov_shape_t shape = { 1, dims }; CHECK_OV_STATUS(ov_tensor_create_from_host_ptr(type, shape, - ov_ctx->weight_data, - &ov_ctx->weights_tensor), + graph->weight_data, + &graph->weights_tensor), ret); } /* load model from buffer */ CHECK_OV_STATUS(ov_core_read_model_from_memory_buffer( ov_ctx->core, (char *)xml.buf, xml.size, - ov_ctx->weights_tensor, &ov_ctx->model), + graph->weights_tensor, &graph->model), ret); #ifndef NDEBUG - print_model_input_output_info(ov_ctx->model); + print_model_input_output_info(graph->model); #endif - ret = success; + CHECK_OV_STATUS(ov_core_compile_model(ov_ctx->core, graph->model, "CPU", 0, + &graph->compiled_model), + ret); + + *g = graph_idx; + ov_ctx->n_graphs++; + return success; fail: + free_graph(graph); return ret; } @@ -257,20 +309,62 @@ __attribute__((visibility("default"))) wasi_nn_error load_by_name(void *ctx, const char *filename, uint32_t filename_len, graph *g) { OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + struct OpenVINOGraph *graph; + unsigned int graph_idx; wasi_nn_error ret = unsupported_operation; - CHECK_OV_STATUS( - ov_core_read_model(ov_ctx->core, filename, NULL, &ov_ctx->model), ret); + graph_idx = ov_ctx->n_graphs; + if (graph_idx >= MAX_GRAPHS) { + return runtime_error; + } + graph = &ov_ctx->graphs[graph_idx]; - ret = success; + memset(graph, 0, sizeof(*graph)); + CHECK_OV_STATUS( + ov_core_read_model(ov_ctx->core, filename, NULL, &graph->model), ret); + + CHECK_OV_STATUS(ov_core_compile_model(ov_ctx->core, graph->model, "CPU", 0, + &graph->compiled_model), + ret); + + *g = graph_idx; + ov_ctx->n_graphs++; + return success; fail: + free_graph(graph); return ret; } __attribute__((visibility("default"))) wasi_nn_error init_execution_context(void *ctx, graph g, graph_execution_context *exec_ctx) { + OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + struct OpenVINOGraph *graph; + struct OpenVINOExecutionContext *exec; + unsigned int exec_idx; + wasi_nn_error ret; + + if (g >= ov_ctx->n_graphs) + return runtime_error; + graph = &ov_ctx->graphs[g]; + + exec_idx = ov_ctx->n_execution_contexts; + if (exec_idx >= MAX_EXECUTION_CONTEXTS) + return runtime_error; + exec = &ov_ctx->execution_contexts[exec_idx]; + + memset(exec, 0, sizeof(*exec)); + exec->graph = graph; + + CHECK_OV_STATUS(ov_compiled_model_create_infer_request( + graph->compiled_model, &exec->infer_request), + ret); + + *exec_ctx = exec_idx; + ov_ctx->n_execution_contexts++; return success; +fail: + return ret; } __attribute__((visibility("default"))) wasi_nn_error @@ -278,10 +372,16 @@ set_input(void *ctx, graph_execution_context exec_ctx, uint32_t index, tensor *wasi_nn_tensor) { OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + struct OpenVINOExecutionContext *exec; wasi_nn_error ret = unsupported_operation; ov_shape_t input_shape = { 0 }; + ov_tensor_t *input_tensor = NULL; int64_t *ov_dims = NULL; + if (exec_ctx >= ov_ctx->n_execution_contexts) + return runtime_error; + exec = &ov_ctx->execution_contexts[exec_ctx]; + /* wasi_nn_tensor -> ov_tensor */ { ret = uint32_array_to_int64_array(wasi_nn_tensor->dimensions->size, @@ -305,28 +405,21 @@ set_input(void *ctx, graph_execution_context exec_ctx, uint32_t index, shape_info); CHECK_OV_STATUS(ov_tensor_create_from_host_ptr(input_type, input_shape, - wasi_nn_tensor->data, - &ov_ctx->input_tensor), + wasi_nn_tensor->data.buf, + &input_tensor), ret); } - CHECK_OV_STATUS(ov_core_compile_model(ov_ctx->core, ov_ctx->model, "CPU", 0, - &ov_ctx->compiled_model), - ret); - - CHECK_OV_STATUS(ov_compiled_model_create_infer_request( - ov_ctx->compiled_model, &ov_ctx->infer_request), - ret); - /* install ov_tensor -> infer_request */ CHECK_OV_STATUS(ov_infer_request_set_input_tensor_by_index( - ov_ctx->infer_request, index, ov_ctx->input_tensor), + exec->infer_request, index, input_tensor), ret); ret = success; - fail: if (ov_dims) os_free(ov_dims); + if (input_tensor) + ov_tensor_free(input_tensor); ov_shape_free(&input_shape); return ret; @@ -336,9 +429,14 @@ __attribute__((visibility("default"))) wasi_nn_error compute(void *ctx, graph_execution_context exec_ctx) { OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + struct OpenVINOExecutionContext *exec; wasi_nn_error ret = unsupported_operation; - CHECK_OV_STATUS(ov_infer_request_infer(ov_ctx->infer_request), ret); + if (exec_ctx >= ov_ctx->n_execution_contexts) + return runtime_error; + exec = &ov_ctx->execution_contexts[exec_ctx]; + + CHECK_OV_STATUS(ov_infer_request_infer(exec->infer_request), ret); ret = success; fail: return ret; @@ -346,28 +444,33 @@ fail: __attribute__((visibility("default"))) wasi_nn_error get_output(void *ctx, graph_execution_context exec_ctx, uint32_t index, - tensor_data output_tensor, uint32_t *output_tensor_size) + tensor_data *output_tensor, uint32_t *output_tensor_size) { OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + struct OpenVINOExecutionContext *exec; wasi_nn_error ret = unsupported_operation; ov_tensor_t *ov_tensor = NULL; void *data = NULL; size_t byte_size = 0; + if (exec_ctx >= ov_ctx->n_execution_contexts) + return runtime_error; + exec = &ov_ctx->execution_contexts[exec_ctx]; + CHECK_OV_STATUS(ov_infer_request_get_output_tensor_by_index( - ov_ctx->infer_request, index, &ov_tensor), + exec->infer_request, index, &ov_tensor), ret); CHECK_OV_STATUS(ov_tensor_get_byte_size(ov_tensor, &byte_size), ret); - if (byte_size > *output_tensor_size) { + if (byte_size > output_tensor->size) { ret = too_large; goto fail; } CHECK_OV_STATUS(ov_tensor_data(ov_tensor, &data), ret); - memcpy(output_tensor, data, byte_size); + memcpy(output_tensor->buf, data, byte_size); *output_tensor_size = (uint32_t)byte_size; @@ -421,27 +524,16 @@ __attribute__((visibility("default"))) wasi_nn_error deinit_backend(void *ctx) { OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx; + unsigned int i; if (!ov_ctx) return invalid_argument; - if (ov_ctx->weight_data) - os_free(ov_ctx->weight_data); + for (i = 0; i < ov_ctx->n_execution_contexts; i++) + free_execution_context(&ov_ctx->execution_contexts[i]); - if (ov_ctx->weights_tensor) - ov_tensor_free(ov_ctx->weights_tensor); - - if (ov_ctx->input_tensor) - ov_tensor_free(ov_ctx->input_tensor); - - if (ov_ctx->infer_request) - ov_infer_request_free(ov_ctx->infer_request); - - if (ov_ctx->compiled_model) - ov_compiled_model_free(ov_ctx->compiled_model); - - if (ov_ctx->model) - ov_model_free(ov_ctx->model); + for (i = 0; i < ov_ctx->n_graphs; i++) + free_graph(&ov_ctx->graphs[i]); if (ov_ctx->core) ov_core_free(ov_ctx->core); diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn_private.h b/core/iwasm/libraries/wasi-nn/src/wasi_nn_private.h index fcca31023..466f2cef4 100644 --- a/core/iwasm/libraries/wasi-nn/src/wasi_nn_private.h +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn_private.h @@ -9,7 +9,11 @@ #include "wasi_nn_types.h" #include "wasm_export.h" +#include "bh_platform.h" + typedef struct { + korp_mutex lock; + bool busy; bool is_backend_ctx_initialized; bool is_model_loaded; graph_encoding backend; @@ -28,7 +32,7 @@ typedef wasi_nn_error (*SET_INPUT)(void *, graph_execution_context, uint32_t, tensor *); typedef wasi_nn_error (*COMPUTE)(void *, graph_execution_context); typedef wasi_nn_error (*GET_OUTPUT)(void *, graph_execution_context, uint32_t, - tensor_data, uint32_t *); + tensor_data *, uint32_t *); /* wasi-nn general APIs */ typedef wasi_nn_error (*BACKEND_INITIALIZE)(void **); typedef wasi_nn_error (*BACKEND_DEINITIALIZE)(void *); diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp b/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp index 9c458179a..a6a53f632 100644 --- a/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp @@ -3,13 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#include "wasi_nn_tensorflowlite.hpp" #include "utils/logger.h" #include "bh_platform.h" -#include "wasi_nn_types.h" +#include "wasi_nn_backend.h" #include "wasm_export.h" +#include #include #include #include @@ -280,6 +280,22 @@ set_input(void *tflite_ctx, graph_execution_context ctx, uint32_t index, tensor *input_tensor) { TFLiteContext *tfl_ctx = (TFLiteContext *)tflite_ctx; + TfLiteType tfl_type; + + switch (input_tensor->type) { + case fp32: + tfl_type = TfLiteType::kTfLiteFloat32; + break; +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + case u8: + tfl_type = TfLiteType::kTfLiteUInt8; + break; +#endif + default: + NN_ERR_PRINTF("unsupported input tensor type %u", + input_tensor->type); + return runtime_error; + } if (input_tensor->type != fp32) { NN_ERR_PRINTF("unsupported input tensor type %u", input_tensor->type); @@ -290,19 +306,32 @@ set_input(void *tflite_ctx, graph_execution_context ctx, uint32_t index, if (success != (res = is_valid_graph_execution_context(tfl_ctx, ctx))) return res; - uint32_t num_tensors = - tfl_ctx->interpreters[ctx].interpreter->inputs().size(); + auto interpreter = tfl_ctx->interpreters[ctx].interpreter.get(); + + uint32_t num_tensors = interpreter->inputs().size(); NN_DBG_PRINTF("Number of tensors (%d)", num_tensors); if (index + 1 > num_tensors) { return runtime_error; } - auto tensor = tfl_ctx->interpreters[ctx].interpreter->input_tensor(index); + auto tensor = interpreter->input_tensor(index); if (tensor == NULL) { NN_ERR_PRINTF("Missing memory"); return too_large; } +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + if (TfLiteTensorType(tensor) != tfl_type) { + NN_ERR_PRINTF("Type mismatch"); + return runtime_error; + } + + if (TfLiteTensorCopyFromBuffer(tensor, input_tensor->data.buf, + input_tensor->data.size) + != kTfLiteOk) { + return runtime_error; + } +#else uint32_t model_tensor_size = 1; for (int i = 0; i < tensor->dims->size; ++i) model_tensor_size *= (uint32_t)tensor->dims->data[i]; @@ -324,7 +353,7 @@ set_input(void *tflite_ctx, graph_execution_context ctx, uint32_t index, index); int size = model_tensor_size * sizeof(float); - bh_memcpy_s(it, size, input_tensor->data, size); + bh_memcpy_s(it, size, input_tensor->data.buf, size); } else { // TODO: Assuming uint8 quantized networks. TfLiteAffineQuantization *quant_info = @@ -342,11 +371,12 @@ set_input(void *tflite_ctx, graph_execution_context ctx, uint32_t index, NN_DBG_PRINTF("input tensor: (scale, offset) = (%f, %f)", scale, zero_point); - float *input_tensor_f = (float *)input_tensor->data; + float *input_tensor_f = (float *)input_tensor->data.buf; for (uint32_t i = 0; i < model_tensor_size; ++i) { it[i] = (uint8_t)(input_tensor_f[i] / scale + zero_point); } } +#endif return success; } @@ -366,7 +396,7 @@ compute(void *tflite_ctx, graph_execution_context ctx) __attribute__((visibility("default"))) wasi_nn_error get_output(void *tflite_ctx, graph_execution_context ctx, uint32_t index, - tensor_data output_tensor, uint32_t *output_tensor_size) + tensor_data *output_tensor, uint32_t *output_tensor_size) { TFLiteContext *tfl_ctx = (TFLiteContext *)tflite_ctx; @@ -389,23 +419,34 @@ get_output(void *tflite_ctx, graph_execution_context ctx, uint32_t index, return too_large; } - uint32_t model_tensor_size = 1; - for (int i = 0; i < (int)tensor->dims->size; ++i) - model_tensor_size *= (uint32_t)tensor->dims->data[i]; - - if (*output_tensor_size < model_tensor_size) { +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + size_t sz = TfLiteTensorByteSize(tensor); + if (output_tensor->size < sz) { NN_ERR_PRINTF("Insufficient memory to copy tensor %d", index); return too_large; } - + if (TfLiteTensorCopyToBuffer(tensor, output_tensor->buf, sz) != kTfLiteOk) { + return runtime_error; + } + *output_tensor_size = sz; +#else if (tensor->quantization.type == kTfLiteNoQuantization) { NN_DBG_PRINTF("No quantization information"); - float *ot = - tfl_ctx->interpreters[ctx].interpreter->typed_output_tensor( - index); - - int size = model_tensor_size * sizeof(float); - bh_memcpy_s(output_tensor, size, ot, size); + /* + * for now, maintain the bug-to-bug compatibility with the old abi, + * where the size here is the number of fp32, not bytes. + */ + if (output_tensor->size < tensor->bytes / sizeof(float)) { + NN_ERR_PRINTF("Insufficient memory to copy tensor %d", index); + return too_large; + } + bh_memcpy_s(output_tensor->buf, output_tensor->size, tensor->data.data, + tensor->bytes); + /* + * for now, maintain the bug-to-bug compatibility with the old abi, + * where the size here is the number of fp32, not bytes. + */ + *output_tensor_size = tensor->bytes / sizeof(float); } else { // TODO: Assuming uint8 quantized networks. TfLiteAffineQuantization *quant_info = @@ -414,6 +455,20 @@ get_output(void *tflite_ctx, graph_execution_context ctx, uint32_t index, NN_ERR_PRINTF("Quantization per channel is not supported"); return runtime_error; } + + uint32_t model_tensor_size = 1; + for (int i = 0; i < (int)tensor->dims->size; ++i) + model_tensor_size *= (uint32_t)tensor->dims->data[i]; + + /* + * for now, maintain the bug-to-bug compatibility with the old abi, + * where the size here is the number of fp32, not bytes. + */ + if (output_tensor->size < model_tensor_size) { + NN_ERR_PRINTF("Insufficient memory to copy tensor %d", index); + return too_large; + } + uint8_t *ot = tfl_ctx->interpreters[ctx] .interpreter->typed_output_tensor(index); @@ -422,13 +477,19 @@ get_output(void *tflite_ctx, graph_execution_context ctx, uint32_t index, NN_DBG_PRINTF("output tensor: (scale, offset) = (%f, %f)", scale, zero_point); - float *output_tensor_f = (float *)output_tensor; + float *output_tensor_f = (float *)output_tensor->buf; for (uint32_t i = 0; i < model_tensor_size; ++i) { output_tensor_f[i] = (ot[i] - zero_point) * scale; } - } - *output_tensor_size = model_tensor_size; + /* + * for now, maintain the bug-to-bug compatibility with the old abi, + * where the size here is the number of fp32, not bytes. + */ + *output_tensor_size = model_tensor_size; + } +#endif + return success; } diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.hpp b/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.hpp deleted file mode 100644 index d6e04ab0e..000000000 --- a/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2019 Intel Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - */ - -#ifndef WASI_NN_TENSORFLOWLITE_HPP -#define WASI_NN_TENSORFLOWLITE_HPP - -#include "wasi_nn_types.h" - -#ifdef __cplusplus -extern "C" { -#endif - -__attribute__((visibility("default"))) wasi_nn_error -load(void *tflite_ctx, graph_builder_array *builder, graph_encoding encoding, - execution_target target, graph *g); - -__attribute__((visibility("default"))) wasi_nn_error -load_by_name(void *tflite_ctx, const char *filename, uint32_t filename_len, - graph *g); - -__attribute__((visibility("default"))) wasi_nn_error -init_execution_context(void *tflite_ctx, graph g, graph_execution_context *ctx); - -__attribute__((visibility("default"))) wasi_nn_error -set_input(void *tflite_ctx, graph_execution_context ctx, uint32_t index, - tensor *input_tensor); - -__attribute__((visibility("default"))) wasi_nn_error -compute(void *tflite_ctx, graph_execution_context ctx); - -__attribute__((visibility("default"))) wasi_nn_error -get_output(void *tflite_ctx, graph_execution_context ctx, uint32_t index, - tensor_data output_tensor, uint32_t *output_tensor_size); - -__attribute__((visibility("default"))) wasi_nn_error -init_backend(void **tflite_ctx); - -__attribute__((visibility("default"))) wasi_nn_error -deinit_backend(void *tflite_ctx); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/core/shared/platform/common/posix/posix_file.c b/core/shared/platform/common/posix/posix_file.c index 3a66bba9f..a05853f5e 100644 --- a/core/shared/platform/common/posix/posix_file.c +++ b/core/shared/platform/common/posix/posix_file.c @@ -1034,6 +1034,12 @@ os_realpath(const char *path, char *resolved_path) return realpath(path, resolved_path); } +os_raw_file_handle +os_invalid_raw_handle(void) +{ + return -1; +} + // Better to define the function here, as Linux-SGX will // use this file to implement the `_os` functions. // So we don't need to define them in the Linux-SGX platform. @@ -1061,9 +1067,3 @@ os_compare_file_handle(os_file_handle handle1, os_file_handle handle2) { return handle1 == handle2; } - -os_raw_file_handle -os_invalid_raw_handle(void) -{ - return -1; -} diff --git a/core/shared/platform/common/posix/posix_socket.c b/core/shared/platform/common/posix/posix_socket.c index ae4a340f5..469b2a50c 100644 --- a/core/shared/platform/common/posix/posix_socket.c +++ b/core/shared/platform/common/posix/posix_socket.c @@ -220,12 +220,17 @@ int os_socket_accept(bh_socket_t server_sock, bh_socket_t *sock, void *addr, unsigned int *addrlen) { - *sock = accept(server_sock, addr, addrlen); - + if (addr == NULL) { + *sock = accept(server_sock, NULL, NULL); + } + else { + socklen_t len = *addrlen; + *sock = accept(server_sock, addr, &len); + *addrlen = len; + } if (*sock < 0) { return BHT_ERROR; } - return BHT_OK; } @@ -373,8 +378,8 @@ is_addrinfo_supported(struct addrinfo *info) (info->ai_family == AF_INET || info->ai_family == AF_INET6) // Allow only UDP and TCP && (info->ai_socktype == SOCK_DGRAM || info->ai_socktype == SOCK_STREAM) - && (info->ai_protocol == IPPROTO_TCP - || info->ai_protocol == IPPROTO_UDP); + && (info->ai_protocol == IPPROTO_TCP || info->ai_protocol == IPPROTO_UDP + || info->ai_protocol == 0); } int @@ -406,12 +411,11 @@ os_socket_addr_resolve(const char *host, const char *service, res = result; while (res) { + if (!is_addrinfo_supported(res)) { + res = res->ai_next; + continue; + } if (addr_info_size > pos) { - if (!is_addrinfo_supported(res)) { - res = res->ai_next; - continue; - } - ret = sockaddr_to_bh_sockaddr(res->ai_addr, &addr_info[pos].sockaddr); diff --git a/core/shared/platform/esp-idf/espidf_file.c b/core/shared/platform/esp-idf/espidf_file.c index a44b57e25..04d31cb47 100644 --- a/core/shared/platform/esp-idf/espidf_file.c +++ b/core/shared/platform/esp-idf/espidf_file.c @@ -1034,6 +1034,12 @@ os_realpath(const char *path, char *resolved_path) return realpath(path, resolved_path); } +os_raw_file_handle +os_invalid_raw_handle(void) +{ + return -1; +} + int os_ioctl(os_file_handle handle, int request, ...) { @@ -1045,9 +1051,3 @@ os_poll(os_poll_file_handle *fds, os_nfds_t nfs, int timeout) { return BHT_ERROR; } - -os_raw_file_handle -os_invalid_raw_handle(void) -{ - return -1; -} diff --git a/core/shared/platform/esp-idf/espidf_thread.c b/core/shared/platform/esp-idf/espidf_thread.c index 768d823a8..cb68df8bd 100644 --- a/core/shared/platform/esp-idf/espidf_thread.c +++ b/core/shared/platform/esp-idf/espidf_thread.c @@ -110,6 +110,13 @@ os_thread_create_with_prio(korp_tid *tid, thread_start_routine_t start, targ->start = start; targ->arg = arg; +#ifdef CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM + esp_pthread_cfg_t default_config = esp_pthread_get_default_config(); + + default_config.stack_alloc_caps = MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM; + ESP_ERROR_CHECK(esp_pthread_set_cfg(&default_config)); +#endif + if (pthread_create(tid, &tattr, os_thread_wrapper, targ) != 0) { pthread_attr_destroy(&tattr); os_free(targ); diff --git a/core/shared/platform/windows/win_file.c b/core/shared/platform/windows/win_file.c index ef7896cc2..7cfda4cc3 100644 --- a/core/shared/platform/windows/win_file.c +++ b/core/shared/platform/windows/win_file.c @@ -1811,6 +1811,12 @@ os_realpath(const char *path, char *resolved_path) return resolved_path; } +os_raw_file_handle +os_invalid_raw_handle(void) +{ + return INVALID_HANDLE_VALUE; +} + bool os_compare_file_handle(os_file_handle handle1, os_file_handle handle2) { @@ -1846,9 +1852,3 @@ os_poll(os_poll_file_handle *fds, os_nfds_t nfs, int timeout) { return BHT_ERROR; } - -os_raw_file_handle -os_invalid_raw_handle(void) -{ - return INVALID_HANDLE_VALUE; -} diff --git a/core/shared/utils/bh_common.c b/core/shared/utils/bh_common.c index 62f36caf1..34b6f40e7 100644 --- a/core/shared/utils/bh_common.c +++ b/core/shared/utils/bh_common.c @@ -59,7 +59,7 @@ b_memcpy_wa(void *s1, unsigned int s1max, const void *s2, unsigned int n) *dest++ = *p_byte_read++; } } - /* read meaning word(s) */ + /* read remaining word(s) */ else { if ((char *)p + 4 >= src + n) { for (ps = (char *)p; ps < src + n; ps++) { diff --git a/core/shared/utils/bh_queue.c b/core/shared/utils/bh_queue.c index 7c860d11a..ddbb6fffe 100644 --- a/core/shared/utils/bh_queue.c +++ b/core/shared/utils/bh_queue.c @@ -170,7 +170,7 @@ bh_free_msg(bh_queue_node *msg) return; } - // note: sometime we just use the payload pointer for a integer value + // note: sometimes we just use the payload pointer for an integer value // len!=0 is the only indicator about the body is an allocated buffer. if (msg->body && msg->len) bh_queue_free(msg->body); diff --git a/core/shared/utils/bh_vector.c b/core/shared/utils/bh_vector.c index 352ce7192..7f0c74b9b 100644 --- a/core/shared/utils/bh_vector.c +++ b/core/shared/utils/bh_vector.c @@ -35,8 +35,8 @@ extend_vector(Vector *vector, size_t length) if (length <= vector->max_elems) return true; - if (length < vector->size_elem * 3 / 2) - length = vector->size_elem * 3 / 2; + if (length < vector->max_elems * 3 / 2) + length = vector->max_elems * 3 / 2; if (!(data = alloc_vector_data(length, vector->size_elem))) { return false; @@ -194,12 +194,12 @@ bh_vector_append(Vector *vector, const void *elem_buf) goto just_return; } - /* make sure one more slot is used by the thread who allocas it */ + /* make sure one more slot is used by the thread who allocates it */ if (vector->lock) os_mutex_lock(vector->lock); if (!extend_vector(vector, vector->num_elems + 1)) { - LOG_ERROR("Append ector elem failed: extend vector failed.\n"); + LOG_ERROR("Append vector elem failed: extend vector failed.\n"); goto unlock_return; } diff --git a/core/shared/utils/runtime_timer.c b/core/shared/utils/runtime_timer.c index 9d390c214..410f05dac 100644 --- a/core/shared/utils/runtime_timer.c +++ b/core/shared/utils/runtime_timer.c @@ -44,22 +44,22 @@ bh_get_tick_ms() uint32 bh_get_elpased_ms(uint32 *last_system_clock) { - uint32 elpased_ms; - /* attention: the bh_get_tick_ms() return 64 bits integer, but - the bh_get_elpased_ms() is designed to use 32 bits clock count */ + uint32 elapsed_ms; + /* attention: the bh_get_tick_ms() returns a 64-bit integer, but + bh_get_elpased_ms() is designed to use a 32-bit clock count */ uint32 now = (uint32)bh_get_tick_ms(); /* system clock overrun */ if (now < *last_system_clock) { PRINT("system clock overrun!\n"); - elpased_ms = now + (UINT32_MAX - *last_system_clock) + 1; + elapsed_ms = now + (UINT32_MAX - *last_system_clock) + 1; } else { - elpased_ms = now - *last_system_clock; + elapsed_ms = now - *last_system_clock; } *last_system_clock = now; - return elpased_ms; + return elapsed_ms; } static app_timer_t * @@ -162,7 +162,7 @@ reschedule_timer(timer_ctx_t ctx, app_timer_t *timer) prev->id); } else { - /* insert at the begin */ + /* insert at the beginning */ bh_assert(ctx->app_timers == NULL); ctx->app_timers = timer; PRINT("rescheduled timer [%d] as first\n", timer->id); @@ -213,7 +213,7 @@ release_timer_list(app_timer_t **p_list) timer_ctx_t create_timer_ctx(timer_callback_f timer_handler, - check_timer_expiry_f expiery_checker, int prealloc_num, + check_timer_expiry_f expiry_checker, int prealloc_num, unsigned int owner) { timer_ctx_t ctx = (timer_ctx_t)BH_MALLOC(sizeof(struct _timer_ctx)); @@ -225,7 +225,7 @@ create_timer_ctx(timer_callback_f timer_handler, ctx->timer_callback = timer_handler; ctx->pre_allocated = prealloc_num; - ctx->refresh_checker = expiery_checker; + ctx->refresh_checker = expiry_checker; ctx->owner = owner; while (prealloc_num > 0) { diff --git a/core/version.h b/core/version.h index 85e033390..261c9b8cf 100644 --- a/core/version.h +++ b/core/version.h @@ -17,7 +17,7 @@ /* clang-format off */ #define WAMR_VERSION_MAJOR 2 -#define WAMR_VERSION_MINOR 3 +#define WAMR_VERSION_MINOR 4 #define WAMR_VERSION_PATCH 1 /* clang-format on */ diff --git a/doc/build_wamr.md b/doc/build_wamr.md index 94dd96284..78e0711ec 100644 --- a/doc/build_wamr.md +++ b/doc/build_wamr.md @@ -102,6 +102,7 @@ cmake -DWAMR_BUILD_PLATFORM=linux -DWAMR_BUILD_TARGET=ARM ### **Enable lib wasi-nn** - **WAMR_BUILD_WASI_NN**=1/0, default to disable if not set +> Note: WAMR_BUILD_WASI_NN without WAMR_BUILD_WASI_EPHEMERAL_NN is deprecated and will likely be removed in future versions of WAMR. Please consider to enable WAMR_BUILD_WASI_EPHEMERAL_NN as well. > Note: See [WASI-NN](../core/iwasm/libraries/wasi-nn) for more details. ### **Enable lib wasi-nn GPU mode** @@ -113,7 +114,7 @@ cmake -DWAMR_BUILD_PLATFORM=linux -DWAMR_BUILD_TARGET=ARM - **WAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH**=Path to the external delegate shared library (e.g. `libedgetpu.so.1.0` for Coral USB) ### **Enable lib wasi-nn with `wasi_ephemeral_nn` module support** -- **WAMR_BUILD_WASI_EPHEMERAL_NN**=1/0, default to disable if not set +- **WAMR_BUILD_WASI_EPHEMERAL_NN**=1/0, default to enable if not set ### **Disable boundary check with hardware trap** - **WAMR_DISABLE_HW_BOUND_CHECK**=1/0, default to enable if not set and supported by platform @@ -292,6 +293,10 @@ Currently we only profile the memory consumption of module, module_instance and - **WAMR_BUILD_AOT_INTRINSICS**=1/0, enable the AOT intrinsic functions, default to enable if not set. These functions can be called from the AOT code when `--disable-llvm-intrinsics` flag or `--enable-builtin-intrinsics=` flag is used by wamrc to generate the AOT file. > Note: See [Tuning the XIP intrinsic functions](./xip.md#tuning-the-xip-intrinsic-functions) for more details. +### **Enable extended constant expression** +- **WAMR_BUILD_EXTENDED_CONST_EXPR**=1/0, default to disable if not set. +> Note: See [Extended Constant Expressions](https://github.com/WebAssembly/extended-const/blob/main/proposals/extended-const/Overview.md) for more details. + ### **Configurable memory access boundary check** - **WAMR_CONFIGURABLE_BOUNDS_CHECKS**=1/0, default to disable if not set > Note: If it is enabled, allow to run `iwasm --disable-bounds-checks` to disable the memory access boundary checks for interpreter mode. @@ -360,4 +365,4 @@ For Valgrind, begin with the following configurations and add additional ones as -DWAMR_DISABLE_HW_BOUND_CHECK=0 \ -DWAMR_DISABLE_WRITE_GS_BASE=0 #... -``` \ No newline at end of file +``` diff --git a/doc/build_wasm_app.md b/doc/build_wasm_app.md index 2cea78069..3c4b519b6 100644 --- a/doc/build_wasm_app.md +++ b/doc/build_wasm_app.md @@ -382,17 +382,26 @@ a non-compatible`AOT_CURRENT_VERSION`. We try our best to maintain our runtime ABI for AoT-compiled modules compatible among WAMR versions with compatible `AOT_CURRENT_VERSION` so that combinations of older wamrc and newer runtime usually work. -However, there might be minor incompatibilities time to time. -For productions, we recommend to use compatible versions of -wamrc and the runtime. -| WAMR version | AOT_CURRENT_VERSION | Compatible AOT version | -| ------------ | ------------------- | ---------------------- | -| 1.x | 3 | 3 | -| 2.0.0 | 3 | 3 | -| 2.1.x | 3 | 3 | -| 2.2.0 | 3 | 3 | -| next | 4 | 3,4 | +However, there might be minor incompatibilities from time to time. For +example, we usually avoid bumping the version when making a change which +affects only a small fraction of users. For productions, we recommend +using exactly same versions of wamrc and the runtime. + +| WAMR version | AOT_CURRENT_VERSION | Compatible AOT version | | +| ------------ | ------------------- | ---------------------- | ---------------------- | +| 1.x | 3 | 3 | | +| 2.0.0 | 3 | 3 | | +| 2.1.x | 3 | 3 | | +| 2.2.0 | 3 | 3 | | +| 2.3.0 | 4 | 3,4 | | +| 2.4.0 | 4 | 3,4 | See the following note | +| 2.4.1 | 5 | 5 | | + +Note: 2.4.0 had a broken AoT versioning. See [issue 4504] for details. +We recommend all 2.4.0 users to migrate to 2.4.1. + +[issue 4504]: https://github.com/bytecodealliance/wasm-micro-runtime/issues/4504 ## AoT compilation with 3rd-party toolchains diff --git a/doc/security_issue_runbook.md b/doc/security_issue_runbook.md new file mode 100644 index 000000000..1706b42c1 --- /dev/null +++ b/doc/security_issue_runbook.md @@ -0,0 +1,46 @@ +# Security Issue Runbook + +This runbook provides step-by-step guidance on handling a security advisory. Typically, it begins with a draft security advisory when we initiate the process outlined in this runbook. The draft security advisory is created by a contributor or a maintainer. + +For information on what types of issues are considered security vulnerabilities and require a security advisory for resolution, please refer to [identifying a security issue](./security_need_to_know.md#identifying-a-security-issue). + +## Step 1: Initial Response to Security Advisory + +- Receive Security Advisory: When a new security advisory is received, the Incident Manager, typically the maintainer who opened the advisory, becomes the first responder. If the advisory was opened by someone else, a maintainer should take on the role of Incident Manager. The Incident Manager can hand off this role to another maintainer if necessary. +- Acknowledge Receipt: The Incident Manager should promptly acknowledge receipt of the advisory and communicate that the investigation will begin immediately. Security issues are the highest priority. + +## Step 2: Investigating the Vulnerability + +- Identify the Vulnerability: Reproduce the issue to understand the vulnerability. Determine which versions and platforms are affected. Fill out the advisory details with this information. +- Accept the Report: Accept the security report and create a temporary private fork to collaborate on a fix. Invite necessary helpers and stakeholders to this fork, as their input can be valuable. + +## Step 3: Communication and Collaboration + +- Use Non-Public Channels: Communicate through non-public channels, preferably email, during the resolution process. Avoid filing issues or pull requests on third-party repositories if they are involved. +- Workaround for Third-Party Dependencies: If third-party dependencies are involved, consider a workaround to patch the issue quickly unless the third party can release a fix promptly. + +## Step 4: Finalizing and Preparing for Release + +- Finalize Details: Once a fix is developed and the vulnerability is fully understood, finalize the advisory details and prepare for public release. Ensure the security issues are resolved in the private fork. +- Request CVE: Use the Big Green Button on the advisory to request a CVE number from GitHub staff. +- Advanced Disclosure Email: Decide on a disclosure date, typically within a week, and send an email to sec-announce@bytecodealliance.org about the upcoming security release. Other ways are also available to communicate the disclosure date. + +## Step 5: Preparing and Testing Patch Releases + +- Prepare PRs for Patch Releases: Create pull requests in the private fork for each version being patched. Ensure each PR is ready to apply cleanly and includes release notes for each release branch. +- Run Full Test Suite: Run the full test suite locally for the main branch. Attempt to run as much of the CI matrix locally as possible. + +## Step 6: Public Release and Communication + +- Open Version Bump PRs: Open version bump pull requests on the public repository without including patch notes or release notes for the fix. +- Manually Make PRs from Private Fork: Transfer the necessary pull requests from the private fork to the public repository. +- Merge and Trigger Releases: Merge the version bump PRs and trigger the release process. +- Publish GitHub Advisories: Delete the private forks and use the Big Green Button to publish the advisory. +- Send Security Release Email: Send a follow-up email to sec-announce@bytecodealliance.org describing the security release. Other communication channels can also be used to inform users about the security release. + +By following these steps, you can effectively manage and resolve security issues for your open source project, ensuring timely communication and collaboration while maintaining the integrity and security of your software. + +## References + +- [Vulnerability Response Runbook](https://github.com/bytecodealliance/rfcs/blob/main/accepted/vulnerability-response-runbook.md) +- [Wasmtime Security Vulnerability Runbook](https://docs.wasmtime.dev/security-vulnerability-runbook.html) diff --git a/doc/security_need_to_know.md b/doc/security_need_to_know.md index b2b358983..b0d076d38 100644 --- a/doc/security_need_to_know.md +++ b/doc/security_need_to_know.md @@ -30,4 +30,4 @@ Before reporting an issue, particularly one related to crashing, consult [the ch Upon receiving an issue, thoroughly review [the cheat sheet](https://github.com/bytecodealliance/rfcs/blob/main/accepted/what-is-considered-a-security-bug.md#cheat-sheet-is-this-bug-considered-a-security-vulnerability) to assess and _Report a security vulnerability_ if the issue is indeed a security vulnerability. -Once a security issue is confirmed, please refer to [the runbook](https://github.com/bytecodealliance/rfcs/blob/main/accepted/vulnerability-response-runbook.md) for the subsequent steps to take. +Once a security issue is confirmed, please refer to [the runbook](./security_issue_runbook.md) for the subsequent steps to take. diff --git a/doc/socket_api.md b/doc/socket_api.md index eff937617..1ecc1213c 100644 --- a/doc/socket_api.md +++ b/doc/socket_api.md @@ -58,7 +58,8 @@ enabled. _iwasm_ accepts address ranges via an option, `--addr-pool`, to implement the capability control. All IP address the WebAssembly application may need to `bind()` or `connect()` -should be announced first. Every IP address should be in CIDR notation. +should be announced first. Every IP address should be in CIDR notation. If not, _iwasm_ will return +an error. ```bash $ iwasm --addr-pool=1.2.3.4/15,2.3.4.6/16 socket_example.wasm @@ -87,3 +88,38 @@ $ iwasm --addr-pool=1.2.3.4/15,2.3.4.6/16 socket_example.wasm ``` Refer to [socket api sample](../samples/socket-api) for the compilation of the Wasm applications and [_iwasm_ for Intel SGX](../product-mini/platforms/linux-sgx) for the Wasm runtime. + +## The background and compatibility notes + +### WASIp1 + +The WASIp1 provides a subset of the socket API. +Namely, + +* send() +* recv() +* shutdown() +* accept() + +Functionalities like connect() and listen() are intentionally omitted +there to maintain the capability-based security model, inherited from +cloudabi. The common practice for applications is to make the host code +pass already connected/listening sockets to wasm module. + +### WAMR extensions + +WAMR extends the WASIp1 with the rest of socket API functionalities +for convenience. + +* socket() +* connect() +* bind() +* listen() +* some of getsockopt/setsockopt options +* name resolution (a subset of getaddrinfo) + +### Compatibilities + +Many of runtimes (eg. Wasmer and WasmEdge) provide similar extensions. +Unfortunately, they are all incompatible. Thus, portable applications +should not rely on these extensions. diff --git a/product-mini/README.md b/product-mini/README.md index 5563d9579..65be5ce04 100644 --- a/product-mini/README.md +++ b/product-mini/README.md @@ -111,7 +111,7 @@ The Fast JIT is a lightweight JIT engine with quick startup, small footprint and (6) To enable the `Multi-tier JIT` mode: ``` Bash mkdir build && cd build -cmake .. -DWAMR_BUILD_FAST_JTI=1 -DWAMR_BUILD_JIT=1 +cmake .. -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 make ``` The Multi-tier JIT is a two level JIT tier-up engine, which launches Fast JIT to run the wasm module as soon as possible and creates backend threads to compile the LLVM JIT functions at the same time, and when the LLVM JIT functions are compiled, the runtime will switch the extecution from the Fast JIT jitted code to LLVM JIT jitted code gradually, so as to gain the best performance. diff --git a/product-mini/platforms/common/libc_wasi.c b/product-mini/platforms/common/libc_wasi.c index b0f86e5f4..51255f7ac 100644 --- a/product-mini/platforms/common/libc_wasi.c +++ b/product-mini/platforms/common/libc_wasi.c @@ -45,7 +45,7 @@ libc_wasi_print_help(void) "path, for example:\n"); printf(" --map-dir= " "--map-dir=\n"); - printf(" --addr-pool= Grant wasi access to the given network " + printf(" --addr-pool= Grant wasi access to the given network " "addresses in\n"); printf(" CIDR notation to the program, separated " "with ',',\n"); diff --git a/product-mini/platforms/posix/main.c b/product-mini/platforms/posix/main.c index 2790a12eb..342eef0fd 100644 --- a/product-mini/platforms/posix/main.c +++ b/product-mini/platforms/posix/main.c @@ -57,6 +57,10 @@ print_help(void) #else printf(" --heap-size=n Set maximum heap size in bytes, default is 16 KB when libc wasi is diabled\n"); #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + printf(" --shared-heap-size=n Create shared heap of n bytes and attach to the wasm app.\n"); + printf(" The size n will be adjusted to a minumum number aligned to page size\n"); +#endif #if WASM_ENABLE_FAST_JIT != 0 printf(" --jit-codecache-size=n Set fast jit maximum code cache size in bytes,\n"); printf(" default is %u KB\n", FAST_JIT_DEFAULT_CODE_CACHE_SIZE / 1024); @@ -578,6 +582,11 @@ main(int argc, char *argv[]) #else uint32 heap_size = 16 * 1024; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + SharedHeapInitArgs heap_init_args; + uint32 shared_heap_size = 0; + void *shared_heap = NULL; +#endif #if WASM_ENABLE_FAST_JIT != 0 uint32 jit_code_cache_size = FAST_JIT_DEFAULT_CODE_CACHE_SIZE; #endif @@ -596,6 +605,7 @@ main(int argc, char *argv[]) wasm_module_inst_t wasm_module_inst = NULL; RunningMode running_mode = 0; RuntimeInitArgs init_args; + struct InstantiationArgs2 *inst_args; char error_buf[128] = { 0 }; #if WASM_ENABLE_LOG != 0 int log_verbose_level = 2; @@ -684,6 +694,13 @@ main(int argc, char *argv[]) return print_help(); heap_size = atoi(argv[0] + 12); } +#if WASM_ENABLE_SHARED_HEAP != 0 + else if (!strncmp(argv[0], "--shared-heap-size=", 19)) { + if (argv[0][19] == '\0') + return print_help(); + shared_heap_size = atoi(argv[0] + 19); + } +#endif #if WASM_ENABLE_FAST_JIT != 0 else if (!strncmp(argv[0], "--jit-codecache-size=", 21)) { if (argv[0][21] == '\0') @@ -949,10 +966,20 @@ main(int argc, char *argv[]) libc_wasi_init(wasm_module, argc, argv, &wasi_parse_ctx); #endif + if (!wasm_runtime_instantiation_args_create(&inst_args)) { + printf("failed to create instantiate args\n"); + goto fail3; + } + wasm_runtime_instantiation_args_set_default_stack_size(inst_args, + stack_size); + wasm_runtime_instantiation_args_set_host_managed_heap_size(inst_args, + heap_size); + /* instantiate the module */ - if (!(wasm_module_inst = - wasm_runtime_instantiate(wasm_module, stack_size, heap_size, - error_buf, sizeof(error_buf)))) { + wasm_module_inst = wasm_runtime_instantiate_ex2( + wasm_module, inst_args, error_buf, sizeof(error_buf)); + wasm_runtime_instantiation_args_destroy(inst_args); + if (!wasm_module_inst) { printf("%s\n", error_buf); goto fail3; } @@ -996,6 +1023,24 @@ main(int argc, char *argv[]) } #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + if (shared_heap_size > 0) { + memset(&heap_init_args, 0, sizeof(heap_init_args)); + heap_init_args.size = shared_heap_size; + shared_heap = wasm_runtime_create_shared_heap(&heap_init_args); + if (!shared_heap) { + printf("Create preallocated shared heap failed\n"); + goto fail6; + } + + /* attach module instance 2 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(wasm_module_inst, shared_heap)) { + printf("Attach shared heap failed.\n"); + goto fail6; + } + } +#endif + ret = 0; const char *exception = NULL; if (is_repl_mode) { @@ -1039,6 +1084,9 @@ main(int argc, char *argv[]) } #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +fail6: +#endif #if WASM_ENABLE_THREAD_MGR != 0 fail5: #endif diff --git a/product-mini/platforms/windows/main.c b/product-mini/platforms/windows/main.c index 7537216dc..31680da8a 100644 --- a/product-mini/platforms/windows/main.c +++ b/product-mini/platforms/windows/main.c @@ -53,6 +53,10 @@ print_help() printf(" --gc-heap-size=n Set maximum gc heap size in bytes,\n"); printf(" default is %u KB\n", GC_HEAP_SIZE_DEFAULT / 1024); #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + printf(" --shared-heap-size=n Create shared heap of n bytes and attach to the wasm app.\n"); + printf(" The size n will be adjusted to a minumum number aligned to page size\n"); +#endif #if WASM_ENABLE_JIT != 0 printf(" --llvm-jit-size-level=n Set LLVM JIT size level, default is 3\n"); printf(" --llvm-jit-opt-level=n Set LLVM JIT optimization level, default is 3\n"); @@ -364,6 +368,13 @@ main(int argc, char *argv[]) gc_heap_size = atoi(argv[0] + 15); } #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + else if (!strncmp(argv[0], "--shared-heap-size=", 19)) { + if (argv[0][19] == '\0') + return print_help(); + shared_heap_size = atoi(argv[0] + 19); + } +#endif #if WASM_ENABLE_JIT != 0 else if (!strncmp(argv[0], "--llvm-jit-size-level=", 22)) { if (argv[0][22] == '\0') @@ -551,6 +562,23 @@ main(int argc, char *argv[]) goto fail3; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (shared_heap_size > 0) { + memset(&shared_heap_init_args, 0, sizeof(shared_heap_init_args)); + shared_heap_init_args.size = shared_heap_size; + shared_heap = wasm_runtime_create_shared_heap(&shared_heap_init_args); + + if (!shared_heap) { + printf("Create shared heap failed.\n"); + goto fail5; + } + if (!wasm_runtime_attach_shared_heap(wasm_module_inst, shared_heap)) { + printf("Attach shared heap failed.\n"); + goto fail5; + } + } +#endif + #if WASM_ENABLE_DEBUG_INTERP != 0 if (ip_addr != NULL) { wasm_exec_env_t exec_env = @@ -598,6 +626,9 @@ main(int argc, char *argv[]) if (exception) printf("%s\n", exception); +#if WASM_ENABLE_SHARED_HEAP != 0 +fail5: +#endif #if WASM_ENABLE_DEBUG_INTERP != 0 fail4: #endif diff --git a/product-mini/platforms/zephyr/simple/Dockerfile b/product-mini/platforms/zephyr/simple/Dockerfile index c3fb1325d..73171cfea 100644 --- a/product-mini/platforms/zephyr/simple/Dockerfile +++ b/product-mini/platforms/zephyr/simple/Dockerfile @@ -3,7 +3,23 @@ # Refer to https://docs.zephyrproject.org/3.7.0/develop/getting_started/index.html # for more information on how to set up the Zephyr development environment. -FROM ubuntu:22.04 + +# https://docs.zephyrproject.org/latest/develop/application/index.html#zephyr-workspace-application +# zephyrproject/ --> CI ROOT +# ├─── .west/ +# │ └─── config +# ├─── bootloader/ +# ├─── zephyr/ --> Zephyr source code +# ├─── zephyr-sdk/ +# ├─── modules/ +# │ |─── wasm-micro-runtime --> WAMR source code +# ├─── tools/ +# ├─── vendor/ +# └─── application/ --> DUMMY. keep west_lite.yml here + +# If you modify this file, you may need to sync the modifications to the +# .github/actions/setup-zephyr/action.yml +FROM ghcr.io/zephyrproject-rtos/ci-base:v0.26-branch ARG DEBIAN_FRONTEND=noninteractive ENV TZ=Asian/Shanghai @@ -11,44 +27,31 @@ ARG ZEPHYR_SDK_VERSION=0.16.9 # In west_lite.yml, the Zephyr version is set to v3.7.0 #ARG ZEPHYR_VERSION=3.7.0 -# Install dependencies for Zephyr -# hadolint ignore=DL3008 -RUN apt-get update && apt-get install -y --no-install-recommends git cmake ninja-build gperf \ - ccache dfu-util device-tree-compiler wget \ - python3-dev python3-pip python3-setuptools python3-tk python3-wheel xz-utils file \ - make gcc gcc-multilib g++-multilib libsdl2-dev libmagic1 \ - && apt-get clean -y && rm -rf /var/lib/apt/lists/* - -# Install the Zephyr Software Development Kit (SDK) -WORKDIR /opt +# Install the Zephyr Software Development Kit (SDK) minimal version +WORKDIR /root/zephyrproject/zephyr-sdk # hadolint ignore=DL4006 -RUN wget --progress=dot:giga https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZEPHYR_SDK_VERSION}/zephyr-sdk-${ZEPHYR_SDK_VERSION}_linux-x86_64.tar.xz \ +RUN wget --progress=dot:giga https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZEPHYR_SDK_VERSION}/zephyr-sdk-${ZEPHYR_SDK_VERSION}_linux-x86_64_minimal.tar.xz \ && wget --progress=dot:giga -O - https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZEPHYR_SDK_VERSION}/sha256.sum | shasum --check --ignore-missing \ - && tar xf zephyr-sdk-${ZEPHYR_SDK_VERSION}_linux-x86_64.tar.xz && rm zephyr-sdk-${ZEPHYR_SDK_VERSION}_linux-x86_64.tar.xz - -WORKDIR /opt/zephyr-sdk-${ZEPHYR_SDK_VERSION} + && tar --strip-components=1 -xf zephyr-sdk-${ZEPHYR_SDK_VERSION}_linux-x86_64_minimal.tar.xz && rm zephyr-sdk-${ZEPHYR_SDK_VERSION}_linux-x86_64_minimal.tar.xz # hadolint ignore=DL4006 -# Install host tools and Register Zephyr SDK CMake package -RUN ./setup.sh -h -c +# Install arc tools, host tools and Register Zephyr SDK CMake package +# If you want to use other toolchains, please change the -t option +RUN ./setup.sh -t arc-zephyr-elf -h -c -# Get Zephyr -WORKDIR /root/zephyrproject/smoke-test - -# hadolint ignore=DL3013 +# Install west +# hadolint ignore=DL3013,DL3059 RUN pip3 install --no-cache-dir west -COPY ./west_lite.yml ./west.yml + +# Setup a T2(Star topology) workspace +WORKDIR /root/zephyrproject/application +COPY ./west_lite.yml ./west_lite.yml # init the west workspace with a minimal manifest -RUN west init -l +RUN west init -l --mf west_lite.yml . WORKDIR /root/zephyrproject RUN west update --stats -WORKDIR /root/zephyrproject/modules/zephyr -RUN west zephyr-export && pip install --no-cache-dir -r ./scripts/requirements.txt - -ENV ZEPHYR_BASE="/root/zephyrproject/modules/zephyr" - # Git clone wamr WORKDIR /root/zephyrproject/modules/ RUN git clone https://github.com/bytecodealliance/wasm-micro-runtime.git wasm-micro-runtime diff --git a/product-mini/platforms/zephyr/simple/west_lite.yml b/product-mini/platforms/zephyr/simple/west_lite.yml index 447affdb4..7c05d7367 100644 --- a/product-mini/platforms/zephyr/simple/west_lite.yml +++ b/product-mini/platforms/zephyr/simple/west_lite.yml @@ -8,8 +8,8 @@ manifest: url: https://github.com/zephyrproject-rtos/zephyr revision: v3.7.0 clone-depth: 1 - path: modules/zephyr + path: zephyr west-commands: scripts/west-commands.yml self: - path: smoke-test + path: application diff --git a/samples/shared-heap/CMakeLists.txt b/samples/shared-heap/CMakeLists.txt index 6346d077e..03906f7c3 100644 --- a/samples/shared-heap/CMakeLists.txt +++ b/samples/shared-heap/CMakeLists.txt @@ -83,17 +83,21 @@ target_link_libraries(vmlib ${LLVM_AVAILABLE_LIBS} ${UV_A_LIBS} -lm -ldl -lpthre include_directories(${CMAKE_CURRENT_LIST_DIR}/src) include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) +add_executable (shared_heap_chain_test src/shared_heap_chain.c ${UNCOMMON_SHARED_SOURCE}) add_executable (shared_heap_test src/main.c ${UNCOMMON_SHARED_SOURCE}) check_pie_supported() set_target_properties (shared_heap_test PROPERTIES POSITION_INDEPENDENT_CODE ON) if (APPLE) - target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread) + set (LIBS vmlib -lm -ldl -lpthread) else () - target_link_libraries (shared_heap_test vmlib -lm -ldl -lpthread -lrt) + set (LIBS vmlib -lm -ldl -lpthread -lrt) endif () +target_link_libraries (shared_heap_chain_test ${LIBS}) +target_link_libraries (shared_heap_test ${LIBS}) + add_subdirectory(wasm-apps) if (WAMR_BUILD_AOT EQUAL 1) @@ -107,21 +111,31 @@ if (WAMR_BUILD_AOT EQUAL 1) ) if (WAMR_COMPILER) message (CHECK_PASS "found") - else() + else () message (CHECK_FAIL "not found") - endif() + endif () if (NOT EXISTS ${WAMR_COMPILER}) message (FATAL_ERROR "Please build wamrc under ${WAMR_ROOT_DIR}/wamr-compiler") - else() + else () message (STATUS "WAMR_COMPILER is ${WAMR_COMPILER}") - endif() + endif () + + if (WAMR_BUILD_TARGET STREQUAL "X86_32") + set (WAMR_COMPILER_FLAGS --enable-shared-heap --target=i386) + set (WAMR_COMPILER_CHAIN_FLAGS --enable-shared-chain --target=i386) + else () + set (WAMR_COMPILER_FLAGS --enable-shared-heap) + set (WAMR_COMPILER_CHAIN_FLAGS --enable-shared-chain) + endif () add_custom_target( wasm_to_aot ALL DEPENDS wasm-apps/test1.wasm wasm-apps/test2.wasm ${WAMR_COMPILER} - COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test1.aot wasm-apps/test1.wasm - COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test2.aot wasm-apps/test2.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_FLAGS} -o wasm-apps/test1.aot wasm-apps/test1.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_FLAGS} -o wasm-apps/test2.aot wasm-apps/test2.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_CHAIN_FLAGS} -o wasm-apps/test1_chain.aot wasm-apps/test1.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_CHAIN_FLAGS} -o wasm-apps/test2_chain.aot wasm-apps/test2.wasm WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) endif() diff --git a/samples/shared-heap/README.md b/samples/shared-heap/README.md new file mode 100644 index 000000000..6e43ceab9 --- /dev/null +++ b/samples/shared-heap/README.md @@ -0,0 +1,50 @@ +# Shared heap Sample introduction + +This is a sample to show how to use the shared heap feature in WAMR. The shared heap feature allows multiple WASM instances to share the same memory space. This feature is useful when you want to run multiple WASM instances in the same process and share data between them. The sandbox nature of WASM is still maintained in the shared heap by WAMR. But the data management and correct data synchronization in shared heap is relied on the user's implementation. + +> Note: The shared heap feature is experimental feature, it should be used with caution. It's optional and only available when building WAMR with the CMake cache variable `WAMR_BUILD_SHARED_HEAP` set to 1. + +## Build and run the sample + +To build the shared heap used in multi thread sample and the shared heap chain sample with following commands: + +```bash +cmake -S . -B build +cmake --build build +``` + +For the shared heap sample, it demonstrates how to create a shared heap and use it shares data between two WASM instances, which would satisfy most of the use cases. Use the following commands to run the sample: + +```bash +cd build +./shared_heap_test +``` + +For the shared heap chain sample. It chains a pre-allocated heap and a normal shared heap to one chain(linked list) as a whole and attaches/detaches all together, and pass the WASM address directly between two WASM instances. Use the following commands to run the sample: + +```bash +cd build +./shared_heap_chain_test +``` + +## How to use shared heap + +The shared heap is an advanced feature in WAMR that gives the user flexibility to share data between multiple WASM instances(it will be the same address mapping for different WASM instance) or between WebAssembly and the host without incurring any copy overhead. The shared heap can be regarded as an extension of the WebAssembly linear memory. But it also heavily relies on the user's implementation to manage the shared data correctly. The following are some takeaway points to help the user use the shared heap correctly. + +### Create and manage shared heap + +You can create a shared heap by calling the `wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args)` API. And based on the `init_args`, you can create a shared heap in two ways: + +1. WAMR managed shared heap: when only `init_args.size` is given and `init_args.pre_allocated_addr` stays as NULL, WAMR will allocate a shared heap(not from the linear memory) with the given size. The shared heap will be managed by WAMR, the wasm app or host(WAMR users) can dynamically manage memory from it by calling `wasm_runtime_shared_heap_malloc()` and `wasm_runtime_shared_heap_free()` on demand. Only the memory allocated from the shared heap is valid and can be shared, not the unallocated part of shared heap memory. And it will be automatically freed when runtime is destroyed(when `wasm_runtime_destroy()` is called). + +2. Preallocated shared heap: the user can also use a pre-allocated memory(it can be allocated from the system heap, or is a static global buffer, the correctness of its accessibility and size needs to be ensured by the user) as a shared heap by giving `init_args.pre_allocated_addr` and `init_args.size`. This kind of shared heap serves as an area for data exchange, primarily between the host and WebAssembly. Any data within this area can be directly accessed by both sides (assuming the layout of the data structure is known). For instance, the host can store large structured variables in this space, allowing the WebAssembly application to operate on them without the need for copying. And the pre-allocated memory will relies on user to manage its life cycle. + +After creation, the shared heap can be attached to a WASM instance(an additional segment appended to the end of the linear memory) by calling `wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst, wasm_shared_heap_t shared_heap)`. And it can be detached by calling `wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst)`. So that the data sharing can only happen between the WASM instances that have the same shared heap attached, complete by user's choice. + +#### Shared heap chain + +Sometimes you may want to use multiple shared heaps to attach together as a chain(linked list) and to share data more flexibly. You can call `wasm_runtime_chain_shared_heaps(wasm_shared_heap_t head, wasm_shared_heap_t body)` to chain two shared heaps together. The shared heap list remains one continuous shared heap in wasm app's point of view. To create a shared heap chain, the shared heaps can't be currently attached to any WASM instance. + +> PS: At most one shared heap in shared heap list can be WAMR managed shared heap, the rest have to be the pre-allocated shared heap. + +![shared-heap-chain](./images/shared_heap_chain.png) diff --git a/samples/shared-heap/images/shared_heap_chain.png b/samples/shared-heap/images/shared_heap_chain.png new file mode 100644 index 0000000000000000000000000000000000000000..740a709d49b3bbb2a8d8423221e18e10953e789a GIT binary patch literal 111396 zcmeFZg;$hc^fx+)hyntlgf!CK-Q5i$ty0p$fOIG+HFSq`GfGN}NDM6?NP`SBgmm}& z4EX)M?_KNOb?+Z=U5n*<=6U8k=j^@D-k<&1`^-lz4aJ98q*x#j=%KQboDK+tK?ee% z%iX^Rd{U}PLjnAU2G&uO0aXo=uK^c#ZKTzuL7*=X?2A_z!1aT-N`_z%h$0O2hXz^M zwE}^z3zg-h^*qh~&S7}!+7M&z=pGZXcFjG-@o3;C+1lcn4R!e1JoDDWgM@1~%=a6~ zR#oAa`4Fs?UOhp;F}M5rSj;Ql2Mh*R#s>_Yw^`G?hsxXsUUmDAnkbd^_`eGfXo38W z^8Z|g^WUdK`|pZ+f;#r@e;0u-f}{xky8yj@`2TS9e4hx4(`4~g3)@dWa*^wJZ&^iwYdncRoSA(yCt}%-*dGm zv@*aAZ`=oYZHcROY*AWT+o<9u^~b8FOjxG%=_fxRB#`l+bhe$hpg5gtQSYG4=9zis zrQ>>1dZ*FNP8tz|%L8%A3CJO;%6me4-oKYEir42Gmcy!Dz0eew7R?`Wa+=VSEDrx- zON`Ox%{ojrZEcEnO!c!TO||!U@0y8@EBtSN(kDyw<~U@P?4G}JQ$0fp?qFhJe$06+ z++tN^7Z2;fed=TS4W`94Y_A~(k9R7g`4m$`L(o&mfEuDSuZ2@vEW5>2qL@>sZ_0vc z2L?X-`jQ@_)t`7*IP%$ijG*`xb@Qk^|M`>|IFmSTI8F7*pJi1dC+6Nl^~bJuTC!D8 z&3%K|$vEBIVfxMa81ga$93PWBuff%9S=oI(U-o9`w3->_)r495#B<9mT2}hC`H4m0 zKbyOk$bbXeE!VwfcY}S|#}01BC_PZKMaFevM(I(a0~bnC2SqdZdlFvRY^DjSpDdFE z_%UX^K0RbIZ>Mk8k%tX@&y;Ctp>J89^rX1RxTv~Rt;g7lSI;JYa&XkDcAnj;^ln8p zt}(y$=%I{EuiuKbx1euCML&aqKsEartzm(Z0vaSce&JUJJm8XO%7BSkns&e0HOQCK zG~w{{(kJDw$`b}Kqhj??vhx1d*kE1f^BC&k&2##S8dk&L3ZBAAr+srqI)^`AErTrE zos4Vh?Qu_b;KEbzA)4HT>&j-h&lOghsHpOZl`&kgnmtj|OMcvoN}N5Har-f*X)-Lq zHpcYPpfoPSbu(V?`3QG)U#7HE8Pm2ZUksx&F=pI*PV+I8k@FNH#OxC-|M%xnjyi=gf`>GBiT;UIdx5HD9O^c>$dNw85+iILcC;TVX=WyXI!sv z{W8C#evoWTF#Ynubr7@7Q@xG2DmKSj{D@E`F0C?}4G*QXS2QRu7Rc3dM9wfg7uMUT zkFFx@E;MDprjD!MZ|k%IL&CirsbA2w-&L@qh7np3G3#oT$?Ntu+L~5F^+L%vF68_T zoZ_!H&k5d4!v~o+j*(?Ql=piF`r>C1F|;W$gRnMV<?!x#%m<{|Ni+t1F|#y>Hfnl>w&VM27n+pi6<>6B}tnM02Y zErXg=wYrktlRKW8eN~~!o=?BzkhM-G`Z=AlO0GH2foqPQ)1F6{zGB+*a3Yg;^9o#; zGknVWd1zjzJyg7hQ^hI6Fp?gxU*GrI{W89Dgnx$RG*YPA%R z0sd{h_Rzbtc!Fw2M}exc?@m`T=$6^JX{yhty9ZTks-z{xs%hX zhhP~PWL6HdwoKyA;*sU;hiRundEB$u7LWW4p|UI0miw|CR#ilTs)%((+^(-W8aeJq zY?fJ~%{+yKJ3AlmdM2-pX~kz#^bX8xc!d^Bv%{d3g*t=ag}4#xHu4JBMQek4K>zH- zO*!PA^lKBzZ1Ie7Yf$-XzmsfP&d36eCp#z17|j96Q$MFtQpq)|u-u*Il!mC5FFl;h zaw@=b|`$VQKc*;$6aU0TIuRb-_VU0~;i>H6!WAgelByLyOKlu)& zcmm>V$T1_B65BdK5`su=h(laZ8Xu|XdbfFP39DJBuf!j2k|WZQe(p|Y21Di7b74AY zC`18T`e(aB!LO)n*psaX$-kAvAFWHRT~90F@|QD#YrhKD>nBI`jS~}9!4ehCckIU~ zUMRvs4-#XQsW6?p-^!C3Os@T5QL7q(BU(ex#NnIOZmQDnhs~7aKOJ0ptTc()7&MWj z85;Fj5DKE#vtQ)%9J;^#T7E4-{(VhwEqbr*%@4SzYe??&4@0Z$ABHUFUya7LyB)2Z z4h+8v#aY=2TB$^$D@s7fXAKKA^9O$Y*<5<5XP|b({j3Edf#mGBXm+Ll-pX0S{ydhA zf70b!Ynbb7BTNjlhln{rRjO(j(bJN+HF8fy*y71q913o5-FXinC{XY#gEl%tt3kS; z(iRGy?CivHMNZ=*YDlFI^XeR?Rq1Cs6BVMvG8tUaMv=kuUgk4CXW>?dxyad+N`vf& znvyNA(eNny{T3655Bvvkyd}N8Q)hb8BI)7SSAUuP81DI;YmLm{5!^E>dO(&~dVpS= zT0Ma8-~907dfRGb-)_{K>t98%_mCWzpW2H8H+tSyd|`$>??JiX$*!IRsCoIoT2kXH zm)}ltebFEMe3@6rY_QJ$O4GFYTP~%47#R20M&T%Gm78!bMPl;4$M4KVSD$h>HRHRh zWGofKP@xUK@y{9QWtj|YMdD-glWKj<2cqVD>vGPR5ZYtH45po~awdc3(!tl4+p~f}Al^T1-X)7K+t;eXazu3~!(&>zxLd^Sc zBRe4WIt$Y>i=LK#V=_dCJwsy+;!t!DCKzW?Y1bFepp!1cxHg^6V>U~*6pdM>?E*H% zO~ZL`bnu+AtNP7f)mjK+jwCITYv^bp9O^%~8oBtQqyXzTet;mQ{EFuZfgVWfPLCpDoli#=^-TFE>?dV5|s90IK#CpPIgtM>A+ zRT-M2M9o&p^9B}EKM|Do8?zA-KX1f1ox_p*^wBD`Rw3gw{lmGlo~+bUc0*{I!%p$= zlegD#JXWL`cf2g!EEZHJnHFJSt1V)&XD9n2j%)Ir8u04{aWBpD>Ms6kjI@N|kKiW+ z_$6$mU4uXUu1|A#2%AwW5YILX(^M)f;qZ^%?kcaQXc2eWG&bR7!JXWUAdc?|t-?U~ zEMZ}^U4Cjj8jF)JOLSfCk%lhVLK(%$5%#k9(V+yhA^O-uEJsSut!Jg?~OoQDa@X?~U1LL)T^K3o4bQy(v z<|3vtd|Qu4&QA>YL_F29dFux+u1cQ|G4zloF)f~|Zc5y{t2H`L0c+HkwJQBh45$^h zt2W)hOTRW6ac7||VbTTc?vLkxotn7}MVV=n3^c^NOerb9;(CIjlt8`CGa^a|N;b4- zLHUAlWqfILxM;YsAAUIb;K9enoL_L+^B<91#K6L++jy`a_*%(8P{}eV_QhL%s<6ze zgs{-3UgW4xs0BXB{68k5Z>k>xkKm(i$tGYXf5hq^XJ-BtZ)+LH6T-vj&s_(5_c=wd zbxl0;qH0bLk;T+t@q3w(QaKP(?y5H zev{g3_Rabr;rASo+AU(QP~DfVoOY9}-N(KOS*TF}M@xD@^M7P*_x>-p{y(OA|F2*A zVF9?eK3dfK0)L?0yh2Lsu-cKzdnO?9c0I8|x9NL$nA!NY)Rmxff@(2Q-;E6a_g4rA zp2%sk3g!hCbM}6jAGS0WYTv^_jVkr$B@w|1i=5ZZl?&5qzNmQlgdQc>$})b4eTDnx znb#k%P~1VFC-u%QI5-aV*^a5y&6dRPxwty!6nFluO-UPt3hRMCaBrtcj!K!#+*dgv z3DmcDwn4X&7y46&A zRSr=F-LVxJ+$G+8CRWvF^Q8dlK4<~`R@z@yFN1~QjgC2D8WZiT!u*xAkQO3eS>VRnP? zKqvP;%!#j_a#k|^p42STg=VQv2v_(08z!yM^(K=sS6cWwE>f3YgE%Gk+B3z4s0~TZ?0G|#Jm|2G#&=z0-4i)^Q1Vt;kv@* zfcSk-Yu(TJ%Ce^EpM>)s+%Dqo9UzY?mgJ{xevM%@UFc|`G69XeeMMl2Y{HFI(q3%O z7N@X#$AbIs0SPX*A#zYzS)@TP`^Luvu1bZVz-=H}n}L%ONm{LkqL+j2BIfPh0hy!S z7z{{UqL^c3W2jv_;!993xChO(#3;sOnqT|hAgQ-n{nWYfZQ-D3`@XBO&_@Y1Qz?yT z`vh?0Szpbq>U39SQI1k->B}odztsBJQ71#SVKR?gc#u!5MQcuy&5iID?f;FoYx|Sb z)x0Hnlxjvrla=oi0LT(#2U-4&wo`W_!@%wN9_IbE{i3r0|NP~{(JGGlfU%h(8+8z-R3ymgy{pP zUQd-)YmTXlRD=Hp<^MP6PP?Dxccq#55LlI)wzs4cQQ=feY6aL@Klx_Jz~BFei;QJD zHEYv=HsjFsgto`Z6WF+#SI3NQnaQK}BK^(t)b+QKi-12T0ZAX_w)Q)7fSsK#O6_`5 z;hzDx+`X|hIwP}l$Y_Q?Ewk-)6#!x%`pth%;K`hoMkr_g&tSKyp&ZPjpAuk>unI|d zV=t~wsk|tJ6rOT!G4dLdS>1-AaO|6a-p#5WJgkk1yy?Y%z@ zFg4>B#TE*-4V=3Ftl$Jhh~hM3nOZ1>o_|-OtVLM$Z&_N~M{?bSZfZ)In;`MPC_G29 zb&Ds~7iv-*MSS#g4C`Spiz%r}&8zqS^!9`CcJduq-q&$v^=s-t{-e{>YQO68h161) znzPV|Tlb&d{*c`+fEq*0O9PcFf)z1U>RuCxGn;1o?I&Vv2uVo%rwNIF?~egUorCHr zD2h=~5q1pb8+%vsJlmugqW4e5Cs?-%Ve-Ge&}5;du+uBD4DuNnV~K`+8PZIAJ^S)M zArBA8+%XSfx(O|R_;Vg3SJkhz;iL83!u%ET!~VANLxbx&qyDDKIdw)R|#W$NwIsx!s|=**ysHgI9ZEDiV2;Q}97$PEwC#>UeZi z2nICr;uhk!AlhC(_DgTb)B$1%xHBN~Y0S2yiMvDIgML=*PD12wX^!5+4Db59Y ztk&^qjW_9YJl-a4&`pX4lKLL3T@UyOYO_BK(_5^9y3?Ee;U9wNqu_D1k8Za!ox`nm z=Dh@8vv^U6pz_4LZ-U#a`seIfopaO?Cj3DSF}D7ZVn`g#!v{0GB?S`F@)8@8R6r zhg~=pcX_?%P6c^t>hs3!k@3hF^n)$P$3*M>*VV8DS;HGf1w?R*3h;|BoaPDv_#LNT zbO1j>TAsByiMCP+MviWpR;HzB{P#5Wt*C4O+BM&x(kISu>JMDR@opohnJSeHTOLkN zeY||%lRllBdB>)vhTLt#(!Sw`e~@ll*sN|kTl6+sp`*0xm)V*qHnIKbpx7fmO}LF0 z<3?>2w}PqIpY{lzk4{xlZfc-SsgGXk{Lu*aJGomZ+M@iQ+E{PrQV!(Cxp)ak?LA5F z1dF+Kl=T4lLDuM|Ss~kvPX~Q`dJ{;(`M3H7M~hEcd)@oEs-apD^9NM&0QaMl9*zB< zANg`Cy)S>pT2^>I)X!Wzu-vyW^PY3Z2k3p>vK8KcR=DUBNVyD*{*8`iUgF?gucsA@ z-Pe!xJwri>-kZJ;XwB=^nb9Y25gq8{>Nesg_!`xQ_70n;@lYH;S_5zK6wcL#1Ke~z z@u6s}*Iq%Q>($+V7V-~m1uC7IH7VpwI?<*Ku_p<~VejrHNjyWRl*p}}Nsjj-u`VFAbC?G$8pf_p&r{;@)=JrKOa!zb zdNa|Nu|RzTDCc}q{uMXhGc>Z{B-`64@24^i#XdD_l;+IvU_?$lv?9G(oK(oon{f7C zG)E6)k;4Eo+bZT&wh!*u%tALRlVes+c@Fp4qC)K$mC;fR`pqfne@6cg6b3JUafjkq zF3+}58d||V4Lsc5?>0T~ALmoKQgn88nvybRo&qz`Z;pL50RYaeU_dAT*toP)H*q!i>ui+{{blRfzLTs)je7b9xK95fo;MWRlzK>;HS{KNK3g*Y2J&JLy0R zc#_^P)`V70FQ^y>X}6bcV2)ct)$Dt;UJmoJ+lrcu4FJ7g<*RIU3z9*dw|kK3nBVy0 z9duaf{BU39=lO>@GX4!)@iQNkEtB{k`%SQ2qkM18E3_%1i~mIRO{gaJn*q%YQa_0S z$|v&9n9;OA`G;wxg1gV8@+GIUx1a-0$O)zv0BU;&M&oBw96xDdKi6sUv+w=R-&k$w z*>-&aall3z6t!vk3p(qDLvwhnQqu}CP>>{zhhz0p5zF&HFO6r2$I-4@ZKX`@De|~I zfSHb3D~)FW;OZ;DR=Pz|*q%GVA0+&BilVr9X){`cVFTSpX|G7k!B^LF!0j5P>?mO4 z-8^`p2L3C-q#K%M_-05N?eS2@%rQ_IYUHuXbgv(<-1<`hyaDjbA$KvDfrmuxhNOpj zhou)IC!d_`)d5fGpYh$K)K-}b{Dbuv3e}EB?{4UKD-$3=d2+HIjT$P3jpuuSoXo|= z3Nk|G*%gz;`r%T*HVsCz50kD!s`sYdMCV3iRHi){(&Nti6G?X|7drrdky`Jd;n5@2Gbk%%Ba@4aSMSu*by@BGS$@s<>cejHnR zrXBGB!6ddRCqxM^f!k%FQI^PKhBil^g?S?V_kq6=lU!CTuVGN`2=nPiPf zK71dO@wY2=5?ff`9XsT#`na}4z5^f=xD7=2nbv_j%E?ZuAKGiLmX+qIR8cmx_IfF1V5gc~9~Kbc=BIfg0F7 zo=BW)T+LQrh8zF#0CMf$`erum17AScv(EPR(_>FL3FwRl5N83a1`JrwA2;eSW#GiK z)nU@$>iKTl+Y@_h$I3=STPkB_2>&opynQaS5$@?QOK%QR=w`83P!agt%wt6o_KON> z=O2Skd!_TSfqiN*+j5O)HwB-n_*(g@f%f_w#b7qpIs-ck0p6lIYjLeX!0)kwad}4^3b86P2@;r_A&DFIM@!B;$2><-XhtR#vL9jCd z?0sTs>Jr}5S;2yc@;=Ycq#GuN^lUz64Ox1^Y2QUu6_oWjKBm>JJo0~8o z30=_-#c1fEdW=oQ37CABWhho?60-4xM+IoShd3) z_Ew%3;Vg5$Zo$j>gpY0>zFMu69GuA~xgW>P|MxR{BIoG-w{}T2;bNj84fuFLJuq#9 zW$!os3hGN*jSdBqGsr_xrWgCXq=-nWtJf;qkt|npOXI7=z?yf7X@Qa$KQqqq=S1}1 z;XsYZtc@fx%E6efXV7B9Y)`_?1Q03UbYW`Xhh&}a;bE}-q|)%+1<|b_;b#~0_yJF! zJjMN`1I;Q&seMnZbxjbpZ;m++_nwaaQjyDk?oe_%hX8g}&##5jNkv~ZwaeEK7cQ!w zhmTJ79*P6OEiEEdHo7nI^*HE3zcAlq*ZDBv?~C)UqL*eam6f_9I=PuAFXCmpk5@Oh7S6|JpCR%)39y3H-1`cwPA)e!P<$?sy2Y& zj@K$vRO|OG2-j9jeh#|pIU~TCt5>SicM-hxd$(r1$kf?xe1|xl$Ce~beuv%hwfMcC zb)GOIQc-`Rhm^mA{5S24c+*OZYTmJ4ayt+ZvhAQ(nh9rQK3jv_2dLwvtIIa`_Lu&G zSNR5;XQ8%c>pn{n!Er;;c5wjt$pbl$J!tkaM>+e%%`m(itxE+8+cCtPW##F}uIeDm zc!jNLV-lNos)WB=@uU1Ao&E`NS=KwkweX0qcvKQUXFMKxFlEQ?NPH>QG>`S5%Ti~Y zu)sHcuY$Sep}u+`+BClWcVmP2%pd3kLlZ1q@$W@Zql4t_^zN4$>KnbS0Ht3z+)1~Y z9vT|wj2XqyK6BhF&%5o|9(Pk0knlE2zh5->IX3>`y2g=J&vHtVqQNWpOWCk0vk_~N z2v?-`rfO0ShF7lR%I#fAN1){gStSr;H03GT=I?qO8vGy3TY^0D5ti`9m(T$;Q22u9 zfqGR0j`#8V_@gbRj)jH_k%QTK$OlQEc-4&V1FuevGcNb{QPqxF;=F!FpX0NWOKH)G z-!dK_@w4cbCDjy^fi7i%#6jwHkDqJ6MluU%p_~abUZSlHjznsv@f4-BJ+t_b0nQN( zNUJgf`YmRK9q*oqHzD?N0w|XL)WTkZD2ZAc5>vjO>p zm`Lt*;bngTFVXQsvivqq&g(q@F58ch%ri&~`~E#C@)>e3?O&r}m7RQ$gKm~GJs2o) zISFh`M==?0%nwBA?NA_a^V;eyr|!-ADV>!ZwEN{qK&ND2XYUwtYOY86pNqr%rtV>J z3l>|V`xhRZPxOHWyj+=}C~iz?_A4Q*c}aIwJ!c8%o+=-}2!zt~1x?R3ABU6jZ7Fqh z(9~LuJ?l%OE#Fi7G2h~)=)T^NAQhJ334PJ1{DI$8Heg$wLX;1WmusLfpz7x;H}C57 z^_4#2O`5#~_2I^t;XPSxMNNbRx#&Ae>WC1FJzxDfG1O4jh zJeu{=1``!eZM$o=5Ax?1(ZmAr@SLm`Tvek%h7#yxjNK@(;gY0f*5vvsQ1~p1gOJc05#|Mf)Y`*I&~F z42`saTouNmE3%U2+;hCH?mcG z85&5IIBmT^nTg;Y7;z5>A526WhDTB=Ch!20&!B4d)Ad!eBBMrj@p!4O&e?%2OlKdB z<(Y|T{i|h-vx7C>OZStzhQ`KY&glM0TzxRud#BkJ^wy4^U_c(2IGU6c-o zS2n6m7Ur``0v0$DHTTih&Z3Roc1B$Ks!j`Id5Z;}93JC+q7cO!=}&>KogH2mj8>_v zRF7AwKeqeXA|Wg)8Xp$M=)NTq`)>PGRi@C8S5bF6o1!!v;nx@+tN0MA_r>kaAtUEP zE0XG<0fBCm&gYbVyse^6I+;7$Sp73XKVxku3zLAaNHxXiY#c7_AN6sF`{g+vdEpVo zd0bLbVX&pI{ZyXB>Dj6BYEMt8ZzDl24I)?>Y`fjosh8`^yuJX93=X^y3xkpC7#SME z7G=w@iK|rJBG=Z`Hv}9~xT@@t6MB_)!c&NA)AmBcg-3>pDwE;o55~mXX59i71DybO zFaE$mSWUcOYMVvBFWx=^?zT3{@3o)^$?0pfn=K)7Uz?5SOXN^F2D00iNjc}GMj<03 zrw3`O1h~cO8AerMZGIyCFIB`|j_f{0@*yvYepP6vIq*>xOl39Wy+Kw5!T-XQ`2DNc zkczDuLYDAv=WU4A@%?mRuy<2MW1I8r31qb3UH*m2{qsL7tVXTYtQdFiDtV@xgCE`j zLiH^9s(#A*Cl|0RlO8;pnGcdo!@kBjh{{ikcsWc;`yOmc`whH1KXqO04kBkoC(2j` zyHJ!-n@znucC7xi9=vfQR!?m@iA{!RH653_JI{m$_Ep%ifl_vLFKMHX{272rSj;M7 z?*=BZCdK)myOsWWKkIgkTu*v~IMB}3_nW=@?xdHw*6xtVZl;Jknv`1#!HfGtC`t#N zfe_Q)apt%XYejW52b-aPzl9Cc!$dumM%Kuv%~B!NPPpj#%I+so+R#H<>_|@O^`vsa z62nNR++Qx_JC!DNhV{|tOjQ$OjzvR+?<%Jc;u+esOJk$@xEHN5>i#lpe{mWV>&s$o z9>sg=GUa zE!nx9jALgHMd`j}aXVVR5qi^`US~|tHIONcUTMlNsOC;9*nt>H2zuqudFUtn#NYlG4mJ0~XvV?p)@;*hfqEtfhv%v=+fChK41EV@mp1IXA$+2X5LLy#@^*Q+gM z$QHNKyjshjdJmRPFD^XJDO-Jx-$xKJ;}<<61W^}Bs2M9!lD&%u$`YeuA_Bpl=OYPw zhbvz~>|LL_jDGQ!_cm%3ZmhMJ0ou-zkf``6wV$>>%I_`uvbLVTfBMw&qx2={HC>;d z5>HhG9OSUM-bi5P@^X1=OFHaIgkvxQo4A}~2_G}%OlV+zq|G)}DlDjQZ$+hVx!HqC zr7I@_o$it=o?bIsZDMJ8+5WTh_n6I$Fl<^cvg1ujnV}&K9qv*xg=(=zu+m%iqglz^ zOWZ5JzqMkMxqdZo{j6QVn-i6!EV@OJecWH)?qZ31A0+yh8p@~BZTvYLuhcayBWRq~ z>gws&;!5ZHh%GE)hb}DJQ{nmLgAPSm$Smsu@OLiE35;wHhR6Qe*dIM^C)$TU^}33|GY;BVO!!OjJ`- zQ=?-xoHE1c?NnhMfag{uc>prc3PE^O0Eqnyfx=K`UFtm?a&pqu+ zQiMNA_gBtdM$Mf5-UUJDd0f*J?`1TwP{s0!gzpKR^ui@3gg&z`o!{YWY2UFd$4cB< zo-58*y9Ag0txL=++=jtfzN}#kLG_8QjYW%LwA?1PZ7Ks|(oR2lQY;?b#=-)%bmU2D zOEh&X--_qc@9*Eq>8FH#*LCNjp0n8113TsSjWF%9_?tUn;evK5Q4QbB3VyZ}JM6nD zPO~0c9HL|@*T+QFP@@=MPP0PdXpuPGp!CFA8%(R4g2PBxE?t1vhR3qx8JATXN;I_7 z?+YYwtG~XE&5)+Lw6EctY7MyNrno#$td<*hVQTOmBhfaq1nw32wB53H(Sd9p{R(z;?J-~+wS4vZ7%YtCRg!( zZJmXuH6$pC8@L2S$*YckA}9=qL-Yj&P%-v60C-A4G1fYb*Gh{rr0X)MET~!TN8&P} zE_t{8f~F>BIIXKvyGMcY3k;$zr=V076YA%nu&_9T*A+< zIu)(@gmhyJ)OOxQk?`xwEN6>Pt6FVcirQE%oOB0ap@9NzpI6SasW+r>7;8FVGir~T zQ6$-*{08;2s>h*9b#!a)^!GqTk*?G*_4kbn-QjPkyYgvy717lL)w}Y&z@NKo_G_j~PNH8IAI3Ks6OL)~EV^k!!8P`Nz8sBIs+4zo^qoIHGxcrx znj>$~!&)>uC)vNcPA<3KJJY0};p^abPKf~u40jPEk7{Q5!d`J{d_x(>)u`o!0uX3C zG#-S{u1#s>v)SaRQ`hS!|88Si;ml&Zi)Ieca;ZVit5L}uk+T|+?|gVfHD1ekZ~Zt) zNgsOk455!VZ@+zzgBYfunifm%WTZ4r&q#ktChoQN$Cl?$gM|B&)p&ZWvG1n|Ov(f{ zD^J;R=f9GE*6Q#j2;7?XQ%c7mBjrCEm7FZG+Q=ptQLswfEFnk7CC(7IPTHCakgKtY ziR*Cuey_%VbeN9H0vCtVtf|bfEkkYplkEEG(mqc>hf2Dbdz+r})@;2Ai)Nb0G(Qpce5%Ip~^-5_A_8UjUkK>}N9alvf{g)+eiH3SK z#5Kr&>>r*iEER^psyZOJO%O76y! zf8JGE6uXb*4Bj;QMd$0g`-gvX&f~=qr2Jw6n#KA42{Zv+7l6dZaM@-U7 z!Ab*YEMZOLtE5i0i)_{5gRJG#O;*{)0t z>(C2Q$b#3QvZXAMdadNp;FtjC-C>er51+m95H9`izOCS#YS2em`}KV8u*jrWD2^?& zy4~`e7i!1)I%YoIv0;-vsd>H9XH`HFW~_2Y>JKWZcX?Ci--&C~@T&R1umt+z=N&$W zH|A^Q+Fdl{2A{pWe8!I0RhdJ~#>{c!Nd-4$Bn_Z)yhYJCm7%^zf9Vf_xPJCf&!x*U zXP$!IbNTa2SYGLh(RM$#Wf_(3SNz!* zwwPv8N#Upiih3sMy=bc?++&^O3l)aEqrZO!6;#K`u19CQQxT;{FhBN&E#W z!v*0A289uYrOr7+uc|RXO5}NjK!6DhN9SroN=0QzZ5$bwLNaNF!{5z59QdxzpJaO;~gO-!aUH7kzD6hkcG%|Q~1vK)D*K@_SHFAnVudba%(pxCVZlEuvaqRh)z+_CumYl z4yn_+mFmsmnO$7mpa)scR@@RyP{f+dBL3NwmTQkqfO%CM)|@5evX`j_mKxHpPGU)p z3#6hJKy^-n+03TGA#1dVzra*t^SAk-cSA`P0ogw7_dA1pjl zkTKb9KMo)G>tG=s#rI=&*0UeAu>!w-pL^#5f-Q3vs3hrpfFanM!`{tdfF6+DQI2I8 zV)LmVaK(%L9YrZBdrz)R_ncAWYe3e44HkaN9ZCs?4Yu875x&CaN4Fzckdpsyetxzq z>4$jskU~P)i{a|N_kPR=i6&NE&`Yel@znWX^ES^|5qF2G0W>8sOTI&^4XapEN;WL& zD%3%~jvR3qOZMGm1mfi8aNwVY0%cRnw?7HGuzw=heck}fV;}EsC(s(S=4kE~a1WpG z+%M0D~78{DGDJnq)Nc;^M;gAoYazTQtDbJ^8z&3>gTacmB#HsW%}%jrMks>2+$9_j`?Y+QMMQsI!6bG zL`5S6!uqD6HZV9>`jT|CZ+Ry!p)XUY=t4AvVFpx$YLoal1Q2{{3&tKLXPSgb!k}HH zx8bvfbn4HJzYI=Kt9TWJ-1NgLy&)(6%j{A0_WI=rgUwHAAfe|=vkW~8H#*soGR_>-3Q7K&I2`An0 z^_7p})^cJ%;1MOeCP)p{fk2QY>OrXmBbatpL!)N9RlpBwgCPgk>)nLr*`7lDfPhB9 zn(x>8s3||i3keR6ZEnWIz<3p01r34FAG59^i!Wv4UU+(WjrBmbTnFVCzEGmeUN!qc zToD6hYG{ERz%~^PMyW;V&3Fgv=4Rrfs%5P=368DTHK1=ozE~e^gkF$}zq_}>`6BJ? zfK2*5N#RDhkk|ShLYA|~0Joo~yksi#Elbd2eUB`OO}D?mUzcE+n=4p$QZq*)%-;eF z^lA-K?f9`R26zWG+Mnl?Kz?dzV^coxQItj{!I8JM$_04Q-FTNk^n}te;?b(PutZ_= zw}dl5K!S4{RAXRZ^jm~K<^LV+od3D&ZdinG>&eN<=*FZokW$ijTfFQI3WG2Mz|R#7 z)<=K>IdJSy!1H4Wz#_Q1MVLG2Nog|gzNfl0`IO94{#g8(=tQ@i4aG%ziAhk4hM8bH zW;=2FJpSeK;JF`=6AI^?c{cVA<*eIZ;;crjbBC`B>>PgUrxNy7*-~$n@Jxq%WbWz0 z$EA}Dh>?6>umR6a-2HR)w!!UuG9KtVk?ZI<@ERoAYd_ZV!uaumZqn5R+w&?O=-KYQ z9zwU$_&a)CJU2V5=7l2EV?Vz=*5A#O;b9^{p9aZk*WOXa#RVwO$cW?;nuh0&-r9BR zgG`!jSmQOvT2HlPQbba_KR-%X8<f zVN42ga(9p1l?3Z6j&_4!KnKfGeS~$xyYscH{~;j!kcFV(+%2Dy zo+hWuql3i`3oGU2yyh4HQ<1dXyY{$Dt_~)m>Q5>c=x^H<@tV7!Y+-9RyH4kKi#cf# zaGuH%aGEY;ldm=fHqJwjD;x^rLz9aaC3dMf?<4d{e+~w0GpWjFmG%V6A0l^vWXtl} z70yB1-?05#a|^t}PP|2`4~2}btoLpxM5VLze1LrPHaW4;yZ$Q2AS9G}krY$7%o_Ow z#&r6L64e(LtSiAd#5(-VLw4JSu|Ak;{N8P+DwhkjEd|~ODd2oeZ=;K&6&6VCQKgjY z)ZWAK{7EW!)hYg5y}E@Fm-93AY@x(G;1D%>-P^!VGnP6JsfF&Y&F_1#n3k0sK!ohO z-<)v=P?l1Xal4l3O_iST%Z6zd?LM~A{rOCC6cHU+U@qV0p}M955aqFl@}ZpsxVoM` z;^M8SM60^>Vjkdr<$R7Qa)JB3)nSm#0-{Rqk;`*PsedbJ?#zpNHJ7HhyDW7MV+q-k z0rdSmxZ<9o!XtmD9Y5dI@g?`XypTYVAZ`FXx0gBz8+P9uRVk{mgh}xhL*l(6i8P9G z$_?UA?c66#vm#1M+K6?{I|cnKIJvZpmx|6M*qV?Ara2rES0>8Ek~@6>JyX&TMK83A zne^K_pcHX?DgJPEQolY7^HO`#Ud?U71qee^g0hn@uhH_q16wafSr-AbFhh19Vq0C# z`8(1gbDe?Ha8gk!;`8L$hrrJH_w$|Q@iwooB!(5H;+@Ua>wXr@gPXN%0`fLzJg{-PETCtc|U*8vF+$rnArS$ z$#c~`?#S(d1fc}rfYhFllyHijdGsu1;qe#VgFJ1YVlUPnX z&A4)|J;t!wRS7dTU5*}ZJ@5OA9%9qO7a7V_lh=Gf zdT&#YfxHb>hbuEJC;e5ao0ZI-reRoPofsG@i$fR|WWxO7iwv_-lP4)5tC7^1KK#5Z zKqJ>}O-I!#$H~*;ewa!ChZ*~z5b;osQUb#T%}oFHU~W&ey<(~93r@HT$1%7Uk33i3vEkF(t0{+d1cB8y`dR%6CbRBGL6h+*9F$c4SLrDcnn32iobq^jRm9)AipJ8VBs_#^KKw}8J zDKF3YvP7W|lZ4%}c^uzo8puvsCiPhro_gxZuaL+cB1h)uH4xVx`@t1C(2apK-d^rh zE~ooxX=y*}H_yNGFCZqH?PF$QD*NHcIH+*jLbc16mMeJazE1R0!Ur_5mK_TIVHj_Q zx|&}lG>WqeU+BGOi)WyTv{YUZhPG>r^iQ;+8dt0PVv~VxNP1O4gS}agM6^+=|wzFSk5<-4)C>GSgwtsl^{1oV$u8CUSPPhMQL($4$=Dn!+m zVYUW-LPA(zVv4t)FDwq#bG)(Do^YLYPH=C(2}roCj8FVdF5PzP_qY#d!KrNg?H0gc z#$;(Qa88r`oaC{KF4faEJmr6xK!jw5#b1oTVCEa+CI6vUYN8?Yz*MZ-aX})B?yd+> zRnYsr)|bQCaQ7{_N{J5eOBopc7t)jqt>v4VkpZQDe|u?BfE$IyAVmKQDG*}G4{Om{Jj13xk1?^jw7w)LZv;KtGhY}VK}E*%k%7? ze~_ZULFxK>^Yo70js6r?W{4MAqw8P$lZkD@gncqXyB9y+=Sf@zz3!t*TCJeXvyUx5 zbecOO=-%bw;Lxzl6*M9p)=V%W^CVd>kImVhudFoTZ2J22xG>;~k+6Y2g7`VYZrA>V zbmjQ`{P)we)Hzv}EA`K!TjkF+=N_UMkN=e6%k_rgYE#2MHgLHR?bhqRH14I3<@&5jvH;=D4 zB#u2hu6>>7xz=Y{>$C03a+srK&)50t2=M0O-=ZD=sg+&eVsg1a7@GFhQJ8C_SH6LC z*QpMYiktDWz9{S`>4qZo6A+6z^RJVoRySD^@OH?W|h+^CyOxdQ6b1`f}8ffF(zd2h6$UBOH&a>%ia( z5K+uR_XIJ>NlM|X!5!(d2`1Oyh7I=)p~gUkEgc(F!Y=&vmu!<*q*p3f5YcGgiHf!y zW%Tf(z0-{bkc)j{z9a4vYI@xnZUXCi_fvX@!)JsLMNKVI8`2!!i}mo$g=>ETOWWZV zdBIZ~!AB3#Kb7h=MNAPtnLaCc?Ymq?<95bEfVnN3BFJdm8>hE`I87{*e{sabIwDRM0g?8g3j4AD!B5 zbuK41f@(|(e{<_IdrWxU?&@rZEQIm(YhOxXPkKV%9K9%ILNa0h`cxfd_ru{|!^Voq zbq22lHP#1{bzProqzF#0Ypw0*sHTWcCN2qHc&D zUb&%q($A?^KKY&x8jVL#xlV`1s%p|bAJ`Vx|8Ac5{rLVov1;D(s@$G_Q< z6_~M)Iz)@d2fV9pU{Z};^;oqEk~TJQCnDSY8N;X>JXw+=tI27ac1q^dEy5`1f{o|i zIH@S=6#9Cj<=7*4o<>o6X-&22J=jodxNFlHRfpNO7&nUQXhAg*q`KOi4n zHI-mCwN^uroGOm11OaLsS2!g4tEGT5g%sEkMjBSTY-?%eQad?SOL)cddry^a1xkr{ zJH1Nmu7d`!nhSrgt9hAiKGGNawj0!IoTU1Z3&~kX<)5(3x0*R48Y=~$s}!|3L1wgX zz`mR;gga>oE7ydF#Si%vat_UP{FvKNOsjO$hyrXBPDLWV_>JLL0$`Gti9f)`gm8)#KcUx zbRz5pwxoGk7?h->K;=ESQv2|*BX%BfLML)uyXKBrlRPcfUjZTTzFL@Ve&qYDQXjId z!D3a;5B)aRUNMc&h*)&quhRGPlIrE7t{bPtpbEyEi$Ox+?bUm{gXB!O>6uhhWWyH+ zrC847FCnK}3xi9iMGF$xyDz8W^Z&bWZE&KJ;ll`3x~#b-XPmcnUV2FQnqjs51q%ix z&a%10e2asF9i6pW>-*E0>)0;(&i;X(R#ra;&*U$sI@?6bRQ{pQzFhY{^tyW2k}A}9 z+B}d3GpoJ{Nu=YlJ|-{0sNQfSH%FQeMs-a#pe!}pL7a!i?J*mjhiHQR^oqE!sjf?d z_)dAThBW6Hcwu8Flk7tnpC)$ATP1j73#Fb)GSY0p-luEupst$AkVg>5{Yq{hwhf~O zzOPIorUKz1J2mQjs>5ApSyT@;9N%`A)5MCR@izlnz-`8rbb-MgE3YuYoyzp^rIxlX zPGZSP7806bJ?`&5#JI0rXbr8I$e#FNt^ag0oGt8)pb5JFiqHDS^cC?keSR1>mBRd- ziV~R8M#DzhNJJ`Se(*{AT-0hMM}zy8}b&%h;UR5b5nsKRpvcD;-% z#;gw`-=Y~+zmS}s9T?8YkWb_)rkd^76B1C416hh1v`O4w;#&D+G(I#&F^`8%w zE+O2x>t7}R4JR^Q#%Rmrm#>M0OA_5`oS&OYrE1*G_F}tLj4$kv4~2ONdyMcmxeL}F zhb)RC#9yQ)USL6bFLzg_pv}ggeLv$C)#8ch@#&*?x`#Nv^Sw8iC0#XPHm8`uw$P>f zyD9jtB~xQ(ZwxDS%WT1fcB^{&F*kyyCv$|0a{bLB9_&Q}eN)WV75TB$X0TU|gBS0` zQK$Ux7gw&j)$Zc8OHsM5Lgr29e>A6abxcH((HGwscLTGCocJ!P> zC%yX)J>Iv~MoXU}pF&QUCO4i))bhB3eInmimQPxz#V4~Qj|+phD+!}?)IVpG!uDni zO%-^BH?mK~R)QjZ%;r~jla9=VB`mk%@{z;ItI zyKf?WhevC8d>hNSj2~RV^P6{Se}O?oKl)w|x7pn1!{+_tS_?iaVS6BNOaqe$kIT?&qv(SKo&w)OnekJe$;Xcy^+?A{sXvk<_PQ_IeKrnHm70_#Wuj)}HAfwE zNZzD*LHIkPatCwu&1)i7+m|=GN5=Au;%COo>`b*9?K4@1G85pR=j(C4m-IyCvMGU? zr}4B6U4gKFM-X8KnDEK`8JNObgx?%FdR`;Fj^V~~5(ACg=A)XSpvAYy_jz1eY$`{7 zM=CWvM~RvC`f5eeZ*8jm&NGB@w|v;W({G-JwL3!qXalHBNh-xjbn5YE>b)|m=KIh{ z=7()Od%O{T#}#{sQl~6i50%FkD^({rW*ce6wA{|)PFRmWuQ}Ir<;RT8c0dY7&x- zvHD%9p-ym#tgD6@#05bBq`uZ|+Lp)APJvS)v6Q{LF z;R>Fd_`}mQS7@i~14{I7S3+$PaYUdOybf%I! zs_}h$r4bwUgALujE;He{5s3V}%_s9%VvNAZC|f?!S(c7`6|pp2;qr^!N8=r$!DX@T zdPbu6PmR^$%dUfy_#kd5k|BP`ezTVzQIMCSk2?b@c)5QQ8q ztZ88G)#C~Yqwx3b`#G5Q9mib8prUr2o1CZ9` zo2+}Rh(bFLE#k}9)x3{y)p-o7n)}e!0!Bz4FuMW=M@+-V`!;1-z zICA7j(}%eB7icg_?oJK<3|B-XIE*E2o?2|kMw!nWl12PE=U>C@QSUSDt7=xSCHI8i z*x~aVBBf*^+clAeTn^U!?i8SlV72&5f58WxX=S(C;->~sJlxt+2XdJEWi@$SU7)X9 zEPsW#v~L;=9N+-rR;}6x*Nq?ms2m=X3E65}bDi^iKdYuyA&3`lnD=7nc3(V`&^zK6 zD>BnJfZ1T zaHobLt>W22xJeKrI=_+U4T}KY@o@{lkc?B3O|)T=Bf~d6)hUf>a&#<6pk+Z;5KKUt9czB(UNlTtB<8|0V%jFNgL0)G z6;Toib?zL6Vp6WVZ5{S-8!0JbaIM~j?EMcN6fH!%#~iDX@`K#pwp8HkXIFl#99FTI znjKYy4KnNn*T6x}S{w57RB;kJn4xSBcq>26fYIyZV!s5&P$M!u1Sx%AZW2#|8XDdyvJUVP4{duFDIj=Fijg{t4;~7vU-CG^Xf6UMN#3-R&%bQF=3WZS)t+ z(zI-Y)O~&x@A&qBl5V6Qx;-p+S7rd?r`H#WIn|MzC?6|`>(^uiLm%f8;&$&v3Pe&SNOFO6{2P#?jv-2Y^Cde-L_f$bxj$%O_F^2 zE*>cqu^h;&hv(1vxFNMp$LOG|ZyV8-&75&^l|+rqCA8aKlzphEa~A?og^-y;<@XaH)-rGasS%i~ zD;-#jP(w(7Nz>~x%rDm(62rW_G8NR+dJAhX*;&c_%#AqPrl2|$^dx>m>+mfC&-L2z zn|o*q{jHbH7@)TGcx->c=#F$SXS|cJVFYH;Mo-f&UjDczvOwQ-do3%p$<`+|Ca6*a z8sz{ZEZ#yj-%}zcT`m{jx&-c8(;WWP>IYuo5v3O!z@T1_x9F8r^-fG~;Wq2-Oiy|6 zJ=w&$=^jM?`(q(bh3X&7jKZb!3Y3XXJzrFVNf-taGAs6i!+s-vK-ZjCp8<;QbM>Tp zAzT62$|#~$eVS+y4(UGF!Lh%E;<#EavbUI)FSc^oVld!}=Cq3Au|MRHG5p~aO> z{Ajj3>owoOymQ=w*Ey~FE|5U>1zIIoM0;&cp2JkO4nBG+?)yh+dma+$T)9p4NY94c z^-8a{9bvk#o%(I+Z?ixLNZulHM;{&(PFkh|xwXXa!je*IB{*UpfP%^=+|SBbizw8H z&|dsJ6#j~awzX8Cyg?O)jJ_!7@9&1WzLjY(=S9){?hNUhO9nJL`dc2aJ+A?eezxut zO@(JQsW!I&_|Whkf@*+Dfwu%)LjOwhX_0S#fDBJ1%0xF`p}}dCh|S-2@h%@> z>MOg@BCq|wc9k|m<|tz!#kIwg)9rc2IRhJ}670==p0j-|x|padfqkrm!X9)Sl9%_E zfl5$2p%4zAEAH+woTWR|sc^FRjG)r2Fv{qUACsMWPX4U`_&V_g;wT}g`c{gP3)kj? ze}D6}wuTiE7N`@BG!4SfZf};Bh0-A zRpk!DouT&@pRd}d$6>0nNJA-hV>Yxrmtl6w@IbqXlVt7OHM>t;)YN=F8|Gp+$!+Dv zeeXiZCSyH4noNxSW{eWr+Kvp-qpYW%Pzl$`rnXPrAZ%*sXXn&@18pU3{dlp)rpcDz z>kl{XFthbz&E@LT){esQdSkwP^Eu#$z%X(x6Qo@87nijz9~?h@$`+lIii4W)lPyd zUfWbzk?S;u=L%(l8cI-Dg&DMFk57jSj)xHps@NZCK{NQ#(faMp zFsAgt;s>->6hN62pn8=aCu|@&?rf}0)Mu3K&vX91F=TR@Q2J9}}7H0$`Cq~=l zr=x-{KzcQrx_om_a=Uv()!}&D^$UC>`8e>rSZA65e3qLAi!8D5!Yb22EtVEq^k{N5 z38K=VL}6%Ztl1dpejz8M#IiT@kdtf+s$u11RTP-y6=v^9iyk<~l6W1?_Y->uPK9XM zS!bQ?P>-0~_WH5nmu+^|aEWK^&FgtHnr$)4w~nWut^xuDfY9!N+3h0aU=#Me;COgsMcp)PKOFR9nDJ&90qJsa`S}=UFlx zxie{_2&9$~1s&aooH=u~7o1@##X;yX%O-u7h@$kCdv4*t5jDCeewr|lFPf>q*(|4K zSBn5OhVjlWXSTgE4!dfuf;7zD1Zz)nw5=@)$jIfU%T01X_1e=2U*c5+eMTS_!DSxH z4f?Gt1G-ij-sa`BX zA9csLURCs6Gau(&*4+G-B?1J4&4jLq<%m;Q9Y^`B_t#A0rxxO%ePYbL^K4+Z&p|{a zP3_6q)~Uao}SNOcv7HId&{Nt_JHnudn$?-_~2b{{f~&V{Xl znX1{r{=~^t3y#Cpmd3X18OoY)z4N&4`at@N%R!R#oy`KmimjIUij&2GCnrtJl3Sw} z*PNKAi~f9LE!VrB5rZ+6j_^+c?zS|*-~7RqFrvu|4Y)bDLnwrU%&$fRd)12+fiOB| z^L6wp{ji6dbL@8Bk`SpXn8yhTHA>u5x0awF)YhuG*IAz?+;f1w(=Yl z2rRFu}hLbG2-*8e9^2b^_8vN?$%Z@q@}}%JF%^!yn@+SP6mr6r%aS_Njr>T<$lhp=7FIXMeTcLvAmE6ewpEbbnW> zEVMX~A0IWp3cja!T_8~+v!?tjQw--Bc)_!)XCbX`-28z#nEG1Gqz_O)Wsl=F?TCA> z-REIP1xgY~@8JZf;s6D{CdOi^Z-V5yV zaomz(@`Q*Uev`l0(?ib&c4W8mWPl&bvTMSEZ+q^dFZvCH7iEzX?7Blsb!o&!;#FXX>?=V+7A?-YE< zek`9NpwGmwd1e{EF2p%UwU&k9B~{yfsa#-bwZmud)bX#N$isWc#ZzsREn&RGD(aPd z3GBk<=#$?X$E|gLB42L5Vg1ltYF6N7;u-=hAyN$Ex0kDR02`Zu{uL-&nI6$nH>xtz zGU66KnRw306yvRrX~Wv~5&f)#_z#va$JsOP310RRYvcAm%G|fC-Gs{Y1}S4rilwJ6!s49|{ykkMk)VsZ=Adpl&#iPf>Vg;Rk*BW9$>RpdE%90Te=16lVGQ;sv`Xz|s}iMTQL z;FxHi-}8spLlJjk7_PCdKjdNRHrngPV=ZO^MWJ#sa*umToVD(BQP%g$<^CBABQ&)C(Y>JV4S4{wF^9@Z?sniv`Rnj=R^@3ROllE_J(*X3iWyP%i3 zFCOWGe>!_ZE1e{sDA|hePlQ)n&ikYQhBgHXOp ze|IESY1wmPBcQcCmdl>^F=~)NqD22qD(16uQP+*#<>!%82P-S&0!$WS&FP0pu)fbT z(T~T=TBa~QKS>xCqWxX@VwcK(w2LH&VQGYmv!e3(tk?ykS!$?J%xT|{#c-nYNr9o4 zYm#{ncezx&7-eDot&qG8dMKTLDw7f*vx-fX4uO6NN)#H86!W0K8qtA(ScmYg99_V` zD(HN*t>55cQ2|G4lu0)-(1H*OWKb_c35?~^?6ebg_oRiPrXqreBvc#92 z>o&b_NzpTclm~FWJe#l>7S-bCwpI(ukt5CN54lB2JHcm7x7M9iiDe;)h8;ce8mV$# z{HS%lX>O+5>M|TZQ&j+TOvwz6JJFnY;529WUq)<{Xy+(W98F+yWiTtjFZxyB!kB1r zY`uh$rTkHW` zd5~N)2BuHs9}Y~9Qc53bRTC}YMD&2E(|2Tbt33MdPx_blK#mmZ&Zd+8a}NoI9Z|h1 z%BBFt!9Q6b6AMILl~xf{#xpr0dTnql^LuauJjE|#77QLqu()px^{5~f7%&+~qTua_ zRntBZek6!g0YNZhS&{1lx=GzSa6X1~KtFh+&?c3m@$?UES)&GDiN2KTOS~||N7_{; zs~cO?rsw3mDKspXae=m(G|hmNMb)QRbk*AH=&192EgY%8+W^$W-Px9&loDT3Rw|Ht## zK1;qV*Yl_knG4GxeRot{cv;jWDBQLCW9MktxR;khFElK@PS^eRaQO*|U+^Z`jLia( zXK0OTvggHd$jL+p;`<=FzpIXBH)2CTU7ks`W-5H^uMgV-o?c4fGgq?l(=F9fEXO8w z_fUtSC&C-hu1$!S@VH^ugm3UvhlhWIE1Yd~SMFyW>&_){ox(U`!;@p zCkRoZCIy5RkFWmFuEvm~QRy;bb6qh>@a5*z@Fo0L;1=-~m8r*-j#>#>&RIxT*M=KW z-o^C#PCIwvGvz?#dkm*_uF>wcctWrfv!jI1w_`?-65kUnGoaRq-x#TI6#&wmhnsJB z@}#K~b_1BnJ=pbM0;w6pd`Zk`6Yb)tr5K$@G1{S>vQO z6foej;pY-obtq(e_;B8Px*X31q(=BoBf$TNq71sY9-4(k41 z^+ZOSn?Rt*X5AUi7gjG^kUR6gELlmDt^(8yrVm-9tRHFvfxni(B=v$Lti0zVE*^L5 zSUu&b%N2*EitloG2|6C7YNC?|11P7KtAB)5-Po-oRt?(`b3>D1eg!Iiy*T->$Q709(mk+Q{MvR=10$I=(!K=s#9#Y#nXGxb}*eMZTjWx zZ+7|9k!Ull*7_-wiQVJ8Q~s4By5cZcdmc|K3h{owF@QH74tGy?m8FOmE*K< zkVOHM`;+xlJzTchPd|L59YyeQU%%d3#$?>Dyar-Sly8;9C{q&Qq#|15+DAE4{nmXw zsF4c38R14r{7$&+5=@fnfUS1BGWODV6i>r??n?0+4(0r4Pmkz^Zmo;@U2#4>$aD_C z1oVYm4y>HoJ<5Dj$JulMq$NhK`wX|hB~tDVHk#W@>8q7r_B4#8OBO~Otrxx#a$9wZ z?M!4g{Dp!VZ;fa*ixIwsNPl96Y893HISoHPbvfJ`B&U!6w45>F08EwzN{vqe$VNkg zKo`h$*L#x1C5W$~N2JEmtd6$N^loi@I+c8oJZlZ61H|%xi;&wDmZ}VxRofd)`eH9m zKS0y^)2UO@J=q&4N+IF}hXR1vGZ9*cMC^%WO5i>kTjOWy8jjogdr$volM|I(U#v}< zT7KnbqCOX>N%%MF<;hF7_4vqBj=22|&C3cYvR*2E8!MTAC@aG6x}u~B8KZYJir?lU z8kvUPb{h%_UFiU_k5NJ*!C;b*%m%C0>v+!+J?RaywytQOGq88*t2>V?yzH5>>EnI7 zeF2A*TNXaUX{(3)%6MRuU}Gq8Yf{Pd%5<=3?eE`rfN}Adeo1W}$W02Qxl9!lr=uGm zOzd8k=%sq2C^06ibv;KbIZ>Mm2kC~A)k;yFRoSK7jr*Y0sOov6d2)1w<2+x6QbIJ2 zFEPp9B&Qz$_7jg?iXeZ6T1U2C(A>}*Vo(F>5x2{uyJ;KAOwL0jaG_IM(0_dfc7Is_ z6Hz{?69fdYiXNyLOtv;AfH8i%8?&q1Az#VL)1cuDCx>3NKD# zn6{3neV~N8bliE$ter!AebD<@z|*17QWXq$ zd#(EOzN$6qNSl!sYRnwf@xNM~QULWy!L6Dya(H!JGK5%Us+2NfSedHH`Y2U3A52@y zhXAA{TxX)n3~rquD|@5e%N8)V zj!v^)xFY;`cMeFum4QM#I)95w+%TgYPp73PT?@1>+jM;1@*LDSo$Byw0C!2aW5PEd zZd{KV8(pT4WTh6s;G6GQ_}!kR);{WpV`v+1Pg*WUXEH z`}OPPc;x#>zcvg@ey7fzBo(^5l3XKzHL+L@UTm9wC2{FEkU&SH&Y-VyS44cG>#Cxs zt74LUB1=);u=e7zy$8OAce@i)VH~Z!kHrezz~**35!uVD(MEj7T~f+DQzW3LoJt~^ z-~H~h1b{dG*6gWos>&nnYU^!oAM=~m+bAyst!J0cHO1lEiqRw@aTXO?Ia)jWVVJINdU}6Eq;_qjUknL+XMMmmN0>1do(*1Kf>opNt%k?Im3VjXblhs zkSA?Fl+`)y7zl+4?D^Nv-1HIO+OHIFT&I11leoF$cTbe((K5XyP+T2|c`Ak_aZ z7Rs3&f4!sQv zYM7_{D+BruasueNF{WoK?p|VVRmO^8izx4^#b7j?=rL{^|IOY>ufU z3u{V>#9At;jZ*%aQApOp0E&XidDwyJPG;^$3u!_*2Wi!G7TZ!eRwLs_S7_T8T~cq* z@E0I>v=kIo^4U1c!z|_I*tl}9|1Mnu>yIyPbj$Js|8O0<91Y!V-rbE&{`NK7HYGnI zh?iZHvkpeiLW+<_iC|$z3l)U&SODhAiSX|#T?SDCm&)T*KjPX*VtK91?QI`jVnw!b z&I=LAw@^{SK1CNYP3!>y!q}LLqiSdHHul5#{5cgB`T=O&z`i<)erF&<1!)IeB=v%a z{8BvhbzQ^6s)(=RudJQ=yLbvm*FI4Bw*{Bdh~Fs@YV5SM3Q2t_^fE`SR$^_k_;-2e z-{}fsOY4Ct%+x3QPfkzCR=eh(l4IMy>bsL?{l{zLFC9O_OI{?$Wm#DCy4ZR7^0_szQ!nkRHoXem{>g+| zbnX0v!|Wh>`KrGep)22!@4W&xxNtx*6f5*YLcZnh1+5@*^Vspr$$7pn325IbY2>SH z4;L_3Y4QmVeSCy272~>ekK#%Y4?6=_W)%4MNV%-P&#~!aEIRy5?6!v*<7>7U)u>Yqp1Ra|LO$uo>9k9{ahe5jfKOO@YU6Ue<%NV&GQ9}G^%aY z$3IlMgB2oTFC3>6UkGllYe8VD&& z+i6$lwTTV-o7&boX8B=-zg=$pD%`RcB`Aaa(D9cYkJ1yEzwUaGM2tk)0eyEaQJgtY7=HU)$`g7@FsO`mPUe>9#oW7%=?2BV-%i zIrYHbsuHgcOpLOtGHQCMrG=Oa9^bV(FTT*FauPtym?L(^!U`nhz3v z%&-DH>ez@1!(f3APx0#DpwjfI@kb0w*XFNSR2uzgwP%GltIlyJ9=HxumY8g`gq;W7 zuhK-ElGUu)fq_;QmVe%tYh>~q{J{4h`QU|5cE)y!s&9;^`r+rK7;=4NK~jnJhc0(~H~NM(zW};Dr-1X+S$y z%~_EM=ftFSz5tC+@L~X)At|lTo*_fh93M;ETZV3B?|E;p7(ef+oo452S~uy@k?#;C zK`7=vl%A8M;iK6wnXm5$GE}t%{optD`oWH!g+E&DLtG4OOwFL67161}%z42nzHry-p_20$c2LUH9d#3OakIiMqK39E; z@Y;A$D5ZXMb0qTC9rxVvimPv$*?FM(Jnm@drPuGCdk>C=-mIo>=20qkPE?QwK${X5 z-TtgNUq5p#njKuXogqoM94h3EIuO34Nu;um(4Ai4Oq5)y?oIfCQr!dVlz!|~62n20 zLikY6VP=|SjAKfY5v1t(g9Z;pfF+K&W2r_S>8Vv7dRM=mzK=UU7`HR(!~Hp!V&g-m z6GPRq(d{~6qyT_1c;DXvgT@o_g`{4!IgT;+4MGV_L$u~H7cwcJ&5Q@YGll^{fMO&r zxd2nL?L4C&FC2DG9%$8mj^V~w&q=Sdw6tUZY7K(gE$6u*!Vf?WqMTFt+5qOVZC-1v zDI(%6IP`U<=21oPvW?sMb!~ppz6b^RzO4&f1HQaKynkS`SGQMQ6xh~~KlZGbW4T#U zSZ8g?8I&F$Tj@Rf;phB9FWs$!FJ`)i!Mpf$q8J`^|3e?dIt!H4jItJX#4u11VZ367 z;LaA^S}c3AUbr8L@R(BtMl2V!zwm?0QiaYrXgLe>>*xX%vadh+pZ9c+LLVWfyXUEj zG=HmyV(=xish0;FTehDEd6R>*0I%>l|E&Ep&IJ?PyQh*Kn>o{0Lc1U~1K2U7eqz`8 ztgKy9*YvL%HV^b%a5&GWt(4jCRyS}P&wP7S>Ke;tWe&rwwLstBS#~oizR+AxJ9uBA zZ%n%jGMTz7jeFOk|JLaD%t>pf_t99R8W$#$8f!pB%<{62??B-|LjL7!#)EF1Yf0vW z{vsB}ZEt;4GBL($80qm|^RE-@En3&V*{~cIuB>d39u=i2m~U`na*ogs-`0h=V_p3V z^#~8A;J262`qaVmW`LLWvY2c-hA{`AYgOB4MIsBUTtu*}J^SwI9Rk4B03W~iih-Xv zBd+i{QG_Sl7}vj+Gf4aurGfELWYwseZb?w24>|I{#DwV@KDhx8b{OkhhvSjrI0p?*4L0B4IIUm5rT%ci3s$S@_>@!tR|DG4>~1-*?Ie4hQ(1pXhckxrpy zk^2@@ZFzt1K_+;uRQxH0++Xw^dJDMMgBAFFNP_@otBfwO}`{4SQ-6Wy4MQ-nxbF5Lc?r{%-UpV3qnFW3uWA4k9mtHurjH6`Y&K z{5D$(lxRFMoX;U5Kg1>Dct0rI1bH`_t4KqW26yc2I&j+LfCMw?6t^%XDO-G)!d0@4K-H3phH=kU1qjXsoZZ|k z4}l2S#?==KPbxQxagBBXzEa72$lwdvfg$%#U3Y8)Hk?1|{4ZdL1h29;^|4IChYv@9 ztG5jjy8Ju=8$1WFvKZx^RO@8)Lv?<%zb1FEvx~L$yGQFG%{_DkzY_3l$OpS+T6J~6 z%^JP0Djcp{Sap5@m`Ef=bry>yi~x_Ec%Ry^Yba95W+bu-6d=zrxZ{y^VCXrTbLyo6 z)B|+gTl3Df*y9}bC)^kkn}d`u@8rtNLvTe-?DU`2>(05WQ4OsELLf0nZFiTOK&1&rFwVw?xM_LabYgX8Ywm@U(pbxhT zBNv_j1oYUi>Yun%P8KE543yFkOc)Z!gByj)|5qk@@5M%QF#7oNYDbKtd#Fp3fYT$g zzjht8cw3R|`qtB?CdS6^*#-wy91{i9W#V&XJ#hSM>ROnZZu1`B%QFe-RiW-Z_zkMh ze9z!VRAb4rvC4kO*+%c|7Zio`6jK}1HU@TpPCBSHxiFxlwENJh+Im9C^BI4;jZ*3N zW4RAkDegt!Oy>2E6aXLS-oh_V2e%8-WqAMQ8<2I%Z8PX%cDF>axSv}iySFPOb(Ktb zGN!%5k26b=`7%kCU25#70-WpU?}h9|atmQK`G%K`jsW*qsD^GnHvi{PH()9;7HIpi z7w!Jet6a{wc4m^^MnR^^fz(*A31V%=dm|09$a}6%ylm2e6_Zic{HQWr`B%t*=@nig z&{#z~ZDQ_SevXxOHlU!}n~m@1nW|zt^#DQKq^p<%E`GlkhOguUWjm2Gs3aJvM<}7$ z&I5)pzn zF!6kDh<3H{C5&AnrHdxm{?-TiF}F@|5&Hf)IwP_cwzD~uvbOb^D#u2WkdKvj60|%v zgO?3ku0QQqho!Gp7%Vf9?A87c*Bsw9;0fbrWjEdWVBZuf_{|yIZl9+kx66aIicKtU zX$*TgxakEh(}IR@n)vvhg#B%RnMe7&ows;izROk)H{H^S2lEni8~0=h2ak7~q@%{Y zL$o%E?}p$UE35s>57GZdT)Ru2rCfQk6g0g&Ss&1kaI)5Bo9D0hxcsWYmYOek4pt8xj*LPooR*Nk`Sioh<0=<$2+}bJC%C zd5$Y8ENnH*8WNQLCKG6?1A93pb@;-Z{2S3c{yGs%WRAbEUib?c+zOzTjq<^`PhHj zK0^J^{KwY@TqL!a04oRV;aNH(_&;A9gPqX(v!J=E9CIvbZgpQRWCZDdc76S#@Hf1#BMAGoTKP_9^h!aoh66yt5kFeE()&U@>43#rePL&3JTnDr^C-QlCP_+` zbqntJkbKZH^k%c}R4%k8BNZ%r12+8t4m*|r<})-hK1_Gqu~2aYLlo$!I0B;xmLHsa zPGIyZ+S(1?PQ$-ww4+g@U9=;j8gJ59w%j<^4SbF}f5hU+gk3=H7%IxcteQ<)$A{yt zyLeu>obT)A)kd;Rg0=kXx1Ij4BHx=*p*a}$M2bpWQF2++14Q&_hF{cnji{kfGp%=R zyN6VHZBe+I*f08%$gP(u+&XcVikE%-(B1N|u~f(~Hj!O-YAC&)c}J_4Gp zS=`h6e_YYB71kwYYuK{s`X)!hh3Oe&(c14g;g=R^o?aX7+Vb*yFN#0U)jz6xnv0?1 zQK-h=4S1WLiS%#R4P$I+nL$WUMS?@c*Qa>n`Q3cd{Cy(tLC&4)y3w^C;sEKDnueE} zm0wMwb$%xfmBUHpQ-i`zCJXh+nIBGUaZyNR|23j zLUw zbIGeaQB_mmX61%v)x1p1+ZPiYYP0j^LWAVV=E_7Rm`&tA1yU=$0gqR@^$zNqV$9mr za?vmNkzc=-7XC3+G(X>XTMU&Ig(QCJIoVL&Bn4aBtpy?q2MToUB}JYd$bd)*2I!rCw=Wvnr}(Z; zT_b%Mwq};d18cPqQZ4}gSQMeArska4h?Z`PHbp&gpHeR)9Z$B0Ccy3p6BT_GkA)T? z`tQd3)t@hInKowR03&FhK*3zQ4?{@Z?J<~Rnghz>T5qZEC!{Q@q4SoUq!Mi|c z1&U#VKeMwvZ^J5!Q7IGmqX6Y0NAQ9PoDb9wJ#YNqmhC&K)EBqQ?uNibYj&;In%{bL zT>Ji=c~q0*|9)o1NLx7E2uuY<3N`v)P*Ld@9D&uD4fTi6+WG#4BCdea5dQRJL^u9h z_n+zM-iDuFMIvL^A(19Sa&HJ#ru(`4f<9hgKCTH~Vy*eB8Nf z%=WImvr}u|!y^PevdAj26HxPBCU97Au7)VvSxO7F8rig$rI z5F@k^g+u{x?^sVNfe5F4fxLdyUMGMQmCEOFAyGb<;s=60!2f!0{X6^kT}98dka$Xf!GdL*MBL(-T}V%AC7kM%WhxCkr-{qinU4s=^{w9 z4Z!$!{~h_NUX!AtCd;KFf`v*KUM#kYe*o67n>5~m#7P=1E-tYC{^opH)arLUcHrYC zL02+spOQ4u2ZVt9xU($WZA%fTkE@3P!XUFb$Vi`Qc~- zAqUsjAgE8AD)G&mDD{n#-qFsB#X8xR1tAc0wUJ;kDd6m6+*=EvD2z)Zs(_N+=U|p4 zToG_sC+JL>6`uASL?K?$1*o>^M>DS0pHXFdo3h6+uKtyI7Rz)HaDUG0-GZcoazK1M zXVT6uwpmHt+tmSe2r_sZ5qNT3Fe6mP;4wG#PeI=SY%vfnb+YF#>xj@ z$z71}RFu;K@=G-1`NGjq2}p8{H}iD9*Ie`U->UNU_)opRkX(u+NPp|`e?wV33osiP zj4V2;?~pCL=}h|hUwuP2hkU6*+<{Px+jO5ZL0}oLAm%NWnUERtkZ;w^Fb65j1&60p z(~rKNq_m%hZE0B*)BRck{yZGy`#Iia8IX6C6BS+iPzaE+@7gc77=cLR6@?m9_}t(V0{``6o*xa1#u{ zNGA~lj^~2VHL$YfL;9n4r{b0rQ@7u*4*pwJ4U=};g}WQ=Nx&IYgvUl*Vy?zi){Wj) zMF5}|^Zz65y~CRPwr)`rg&#^$v4K>jNbewB1f`4g4kEn-gwU&sVyD;8q=in97CI;@ zy#@k=E}am159O?&dw=_T&bjxy_c`xhBq49!w$_?+%rVE%p4tZz%m@hwQuPm?M$$R1 zTzxoWUFytsUf_Cb8JrcohyDUP&P0MaGEJI${*^Wu3a{?YAV-iw{97im<wA{E*I@4WvNH6FQ=L>CTMcVehqO=Alvt zb#Qk8|AV5cEX;eDeRio8*c^_wM)Wa`6{yk70n^My-M7bWIzJ*$dK2ledf*ODKY81f z+;R`;tfcJc`GjCg7#J({iCi#~Z@AGpA|%V@?WgQXr94DSUu5Kt10QpMf?DAVOXb`4 zdChX4fNvtW0dp0iGyOT&6F7*4Xr_UEU}rZv2)qr%uoP*{T*I+0GrPeu48_cXhe9rG z-8BzOG9&b2-T0@xHi6w;l67A6m1Ttxd6S~NkKt85nJ@V-0%xnU46+2z&c4HxsV9Ph zwW+lYm#46|ON!sS#$U1f9!ElV$#vGz>ab}1Q`xIn(AQP*M8^92J#Auz_I_t8AQ%GF zHi)Obgr&}SWgXwRpSc-7NNsqYhtV&rvxehdB(xD2wzC6u1Yi$YnHP-+yL4Q_zV$J+ zc%)^w77}qDQJX?FZOer~74Y4!kr_K8-`R+@RwpY+)w8GD?|w-V@v&EfQyF2EoJN08g)ccN*`VUbf3oPXB|lH0jcw}qIK|2cN@mVd)8p~m~b zh*PSMPYL+A&hphbO-tB|2$QuDW>fizqF`{?If9$j6 z?8i$alR<+H!>F|C_3PI`@<7;>?((-0WPn-jN`0Z&6i92kbo%yh#?x^Y{w3#SW?UbF zAS(j?hx_lYGeX2kq||RDjNsdK9E3y>l5P<9{kf&?lFtV0l?5|VBrqKWg)6`erEo@JNtF3}~i(_d-y zdP!QrVK7OGskdzS*KwacJ?=f{Lzzr{yDY+W0}(&Pf4KmkzCrZD<-)11=$}`Y98dT6 z=f(Ykzqjc0G-y?MdO=|t=x>UX= z#~FVQwO-_OZ);-?Lto&dgocHc)2lH#0z5ze6^iW@og*dH22EeBK#>|W{y(&_v1t=J z-S=AI7tk^RG;53j2u<*9^aU;q2a)FpAHHKfecUE%&~V*5xt~l3f$N`}UeEgHS52Xy z59ba#X>|fh63Qm^OvnIU3 zgg$*l{(SbrAoxDv^!{wm{&;&Gj6_5vV0-<_9;bp2IbpvN4<0Qk9HA_SI zfao3XpI<(Gamn$&UzgM8`1BK^|M`6X{NMl2i}XK#>AxnI`)99Ty{bTfR{XTZK{=S> z$_+j~mknSj2f6}UJ37=764wHbmzj5g%C{S1L?f7N|HB}akVW73KZOsNB5vOE<_29y zstJOhV`5k#m^Jomobeh7#AL}xBryGudckX6^*B+CHysd`v{ICZ2>z_ww4ULO0V8v) z(VV&}vouWrcTok^m2}Y2*aHwbu2YPFN#;I5Sp9`Uy4vD1-~>0uk9NAP6w|+PxI5PxHBX>3NCZuN5M+1#hm0(ur>nU+&a5FYze9cRRd8_D(mF`S zKxu{4y2$nAON`1?%`QWmG~uHCU_5>PuDl0q{@x$!4R5Om5-V<2-M71u5Y)ZVO@fIC zVBm+(H;4gPFyk7R*u$zkf*2lvYyM0q{&D>mBu?LWXL`D``_F|4HdDPoe0~8?)&==h z-1qnQQQ!~=9a~$!e>&Pba@m->yH!u=jxkVS-%%2fH{ z;N0E#4A~Z{h8w$z3$uk|vW$$3hM_J?UpT7wj`jiuroWF+(;wFRb5k;k(_~DyBc=y8 zeSxmC{GN!vT32#|({o0T_2a<91>W|J&y2Su<__!ODe{4j>ln=H{izolRW+sO;d|(j zOT!+`-V6xUx)5&|a(0glRY91lnZ7ojYU97ZNvesH+=+T>58QfmvfK=36Zr1g4VUf~ zZ)LYwyytZ$k+JE&Pg6L2Rq9I{p#sTK#%pYR3qU_y<&Gl?kEHZS>89AGTcg;sfT<&s zxw$MMNhXvTI$J&$aygOl4J7U&mrbj!x_CD|=ff z5MhGki;pNbLxgW;I?Jm2YpQLrDtJJFlKwP&UT#;|1<2ka-_v^9+C{cE|3Hcy*9hfPF9TZOZ zGc-2WQ>48(c+9_Qb^+v6ytKDge3}D*nqYsyzZBdl_iMRLD_*i`B=d#o)*b=~FGuTV zpT(myVii(U)xuh(b*42GkuUnAzxiGV@_3-bE9+p};R$dxd4SaTX-ZRIAR5&NSI_F` zJ-9xY{Gq8w6zALSGmqO%>ov;35L2U|-@pg%L#;m3euMbG99 zgdM?PR#Jbbr}$kk9QZD>PCXgOo6Q04je^}M6qmax-3L#6IR&dmtj?oQ1`(U;bE>xn z`N(mMo4~n(&}Dq3tK1Rw`onde`=AKZ1B7bqrc(1+=|)OiFK20_h7eo{7^J)(0J|1- zNiX`$XE5uLatJ4OYB27unbO(sN2H-zNQMSgZ&A=^cSQ?WUG=4Y9P9yZfH?s*aAAMD z38mmW9$TBhHAWLqr|)X4&}}WOYr{m3G{oxze7(}^Va5vg_Lc^0Zvyp`*8~lrBfLi% z3oozx=ER6wN|$-18bQklz&dZm=jm8B>)ls|w9X2erG$|pTt?&NuBhep7js+wa(5%Y z2=LW8gemf~C}dn6G;Q7os?C&jl(!|{(j8OM%NU00wqALYYYzN3l}lHE=Vb4@2<=&^ z0wV!c660rdpH8amC_sZ}^JIC;wL%l0mh>x1nBn2bH*e01I6cuu?=D73@|ctvy=rWW zY3n(xqJ9u;J?6OQXzukX-$>GZiEoOHwBxY!=X}&rz%n_ZyYB4j!;9>1y9i6W(x=lU zj^$az`A9$s%P+NQ%xzsjrs~@Oj(8&33wM@)-(Aj{QUO6;$ z$0Bxize(|Nnr=R_XYXrp?!(Q4HX-xzjK5__>!RyoZ=BI3iNZ~$NSl<`Ih>T6pw}#g zP$dPeg6T=(C7j(ie|19lMJOqEz`$~*90e5&Hvsu3qEBcfXsel}ZhHemXAY0N=(0yp zCsUK8XxlP;i*5j&)W+8y>G}Q5*;vl|l)AcH`jA*lEkL3>fPS+o`T8|>|6pF}C$q_X z5pfnriSxFd)qFVd1mL-Azu92Ts@C#40WMh_&0YP>P z;%a}6)t+b{9sN|{Y;?tFf4>kRDH3i)OzIxhs4Z=CNGlRPKovv?Tdv&NVg|F`8vS%Q z!MzB~bK<#I`+r&}0V(86Biur-_RNZN?W!E**gYo;Oj}EF=Tq^5sY~i&#m`LW^|5hz z7IT~U<&mJ*Ny*79Exi=JH}!A2U7H5vd!@$UGo^AAG=&s$Yr@*b8yFQ*WDTV6rojHcC{FD2H&KHAD=GFZD(b1^HM9Bxx&2Aczabf1+xlYM~z8?F@8-ky7 zIJ8;B9zm=+3YOaz1=oE0g*s*y_Of0W}tvDiRW$b- zxaZW|RcDiT^jrL*ANZ=(_Z;XvU)$$fVugL|_QhFe;z&%Zpt38rJ@lr?x~hV)ELHuP znOq_GA~r>u)M_d{=+#*4QR%PO-qoW6`GbYF=lk5~4Su-#K=f{z8Qprkzqi+t{$H>hP$!6+DdJl*d@dQiQA!nhg%%>g zNJ~K>ACjmdFIQ-~SlN*6G=E39t1B(!`P(o$A-PPF&`@m2gm6FV(%Rvhk8|P{0em5G zXgZH;R+y$qaN2<2J8&+Q*`6Bj?*nZIKBTkX->B+=VD|npAtky`H#a};t7ERw?F~1~ z-&yo-`nk|S>r|CLdqZz`WkO6Yac6z7-jd3G0NzfG{G3>!isnvusrOnS#u|fv;6yt5{DnxLB_7%>6m4{#d`6qRH=ChBsQTvBLB(aRBbNMdXK7}spew{jcF zZFsYY)bi?E&^pn+bPbxsAX^79K{Dk8brsj&NR0va+g^+evudm4%TQ!SA!8&4?z5A^ z>3xO=+uo%}PuiyP-WnnkQt@0d8BSgJ*b_ndI{jN@ktKddTF?H+0u!Ia(0frIX}7IP zU0R94_ejS<8K}A9BQzTl_`Z-8;BH1bL+#3Z%KB`oHMBRi!zN<52GxSz{50dZT7dZW zj>HKw_$B2c{TI8v?to0y`3g9EHc$6f0qb{-#OYN`UAJ&m(De4kH;2W=W66u%CSMyE z%X~^jttmhLIRW1Z4&3;QdN70sz zYbQooFWno%QUe)W9Hf`NuT+Jd;D`3)ErKZIDe?l zs5*Y8JyG@8q_}^JITtalw5v!eMm8MxR7qm;J;N=?*&<@p(Bi?|$v%>7`TGNN#ofW- z^_mkV{#xj@;%aKyK!4>Zi5T%YOb4d7Xbf-tv#lI$GIXz=vCO_Y1raXnkE-Xyy+njG zVxot~{MHTM=j|r<(MHNrF*0&5qwz-Yu*HUUAkk+O1{R7RGAYvai~UX2pdaKJ4Sj-P zV)T;IcT54W|6WGlc`5w5HC(2U+n}=JoE3;e1-`x82?FESOWc2^GCHWR%zdxm4cSwZ z)+CPL-0iG06Q2?`ttaZ`;dA55;uo__@h(>A)r*;0S%NlCuh47ybT-`TZ24()hc7b; z)D+gKv#s_&lnX-9{z5sSl+o#%7)A4mW7+nuWPHcnA=>Dr3VZJy5n4tWf4c{)raPN_ zzRDE^2_46AKIN6US$5g@dgn2l5gEpqpQ^F#FNqbMP}kg+`VC79C|OEp5ZBlY`X<## zX~jJcGl31;!)kXNk2btZR7BC-e@fi(vxZ89R05f8d6*&vzlX6$Ls{KhR zVg)C%aW4@miE}&>ttKN?wf?htG~{zVQPFBiz%|Dr-GAh;TZgoMO@LQxX$QPd3 zm!Gicn;wzd=-v>inAhVkzbH~0^Mc5*%Arm?j(OrEt(#04E|GzcLK2m&YFQTvEvn!v zv)n3}@WPyA*Qb`Pb~IINiurZj)-LK?k@VW{s#wK#JQX{_dkU?}{(CVpP$NCl`{Yqm z{v!%U2@3HtHoXM>j|ul5#|=Dp##ML1uMe?9V9MPq$tEGzo<_Hd4jr>P?vC6Nbr$M0 z6v^pj8wHLYkszT}ntH$&Z}c8?SMbY4kvz;&sdp-z5}@!cQ+C4$=XO@R`(5WbZg!f8 z$sbf_8p!t2M=7R(Qe5Iy?s#*(LNh)t{i3L?B=v>9<{bT<|zq7$r)!dgM$R^wq^22Xi5nTx~mE zeUSK4EU=*na&i{G;g4RG^kXty`#BFeVeo}*KR$wlinHk#nUx-mwDM*=+7g-v_yv>b z=09taC(D?6aKlfM(1IwKz0sW>`a;%hF&vg3*y?>dkRsxM_PO9*Y9o71Gc~b7) zdRmcB{YKAs1WmmR&YDZ&eRb?cey=29>zBT|?wCk>?##9A?7oN{s}C3yzKCSRucb7t z)gqVYH5~;dRxG_k<7#jh+Z#T%1WxdnR$xArnUyGL?t64Fl``&3|K0;9K=clG3Oc;F zJINgm0K<15Z}tP1Kf-oD`V^QZN`E2Rj<%V=Warh$@Dr9{p;NG!NV9QV);5ym{hfX) zLNwy^Z?Kf9?cey-zxDxkcggznhX{LCp!qkXrActPJKg?2{_A13^LVYbb9W?xYEkVm z0{bZAUQ=XJWdr{zr<2rf3(OX3+;A8QvVZ<5RslO+D?wtsEd40z2R z?tj#?pXqB{EzDbO{rHBnt#&`iFsrwzavhar_f^blD?V3Gx#QZk`3=J=zugr7t@U}W zJT;Nwy7?En7rMnak1Oon1u2Y&i*Ku45vjc{EqTqWaIWP@JI9+{BU!6zR*vhN=(!IFB(4~TLQy)fW2EXj<&3JkWLc0h9_oHPTJ^Ey*{noqQ8ta;qtyHW&he^Sl}EcPV?y{MG5z5B zh0U#Uk6Z4>qx4*3p7^f=0c71TxVv^ve!NG`4ZR)RU2~4(I`A#(GE@jP^&S_BsNw_n zV7EFHiu# zb23cFr&tBGjRP~cN?P0n1J<;QkP~V)wM*_v!nL>XSeHJRM!s(kRHWY?;RAWDan;hg zn+fxmrK&bhJKczd!m$^yU$qI7{Rq8V@<4taZ=J7a>{X6PL)Mo^3P80b*L@LV>^xC# z8S}xaJLMtC7XA?lC9TEy9ZN;3Xn+$Kg@M^=((n|hj#%kP*xYcQu7I<_NMv}7Yxy{| zSj5isMcbFE8{D%+8wg$c5Mp4YtsYzqS~=QC)t8j}GL@}n^E@_2wq(_wintzL;xLB4 zHbjQFb{e}!0=)sH&XYP;m)c_(*$NPSVI?LF3oo_hYbj=)NO+=R|GCPhQ&Q@(D41uA zWDavO$nfs`Q5%SuSOCT(LJmAeUD#&&wvRCp*Uz0pKWA z6cufJ^A(nNe5|ThV-((BAvqsey^2XP!B=u?*5;Tr1j+;3I4cKky*z_Qf%}W3n^GQF zG-SVW;Ks%(M%{+d80m03S34`ZaKux&+bme^i+BL$pHf`W(5|_>8pY-9czxhbL{Pg& z#PZt2F7BwyChUxuaOiGWFZLW;j_kMIZibBX*EJKBhK0`m)McVcfnG;jW|uQhxNPJS za$S%fMAHsT@V9p)iI|6w-*W8AnszlV+hepJtH>8`pd}jV-Y%H95kJ^p4B9XW)7s>4 z1~9ZCdZ%MV4e3B=tKZE1hJNKkou%0X+t@L_mIXQfp=g{5x&F*D}_h^qo^J zo#*yX=Xh=>EQQ0&YM1g-{3xrKL|Bp}d|8=H%P4W&4RmZR_B*bUvSHkiO&#p8?Su_? zVAUP3Ey>*L!@RB5FoNpD^-gDPVs>X^3ngc_7NO|K+fmsaq83(BzdUA}<)k|uGgadr zjr(W3BAXO8Nf!3ei%)c66nFdXnS^bV4eAa@G#qp!ky3oxj(21#_?8^vPD{RoUC=@D zsK$O0^zG<>Dk33I4K8Y|Wj6CBRX|7rEc8y`qU8fair8)R3Tmz|S6+4BcE6z{u<3d! zCFJaQ(B$(?vp!6C=)CVCB3PX$*Md;QOFzXl!8Mh4-iA@I^*fvlD}r?U4v&}KjzuYn zT={Txhz@ONuKe531S3z+_tfl>d+PxORn1!2kFC*ZX-SsUmilI#@#bcT4VaI#B!%k5 z8@|1xvhE8x2{NiNTrom&sTsfxT;=7V&+Nf=qT7MmQzoD3Up$7Db_ryh6A&HjdpX|= zGu^fhmmJg;q^T^@zG3mJXwmt`$6;fmva#`aBTL;dpbJIt_s!&T@b=dqIand*rcPO#24mOcV`AJOfB zHy`8Fs7Kv_iQA5oVS(0QZCxFU+^ciw(Wo->U*RIh!nJ)y@fgL~?+;JnM_?kieh!NH z2lv{2mgGb9C{Bca=uII;PPsDOjFiqn$%tvBNZbKBX>rnil;?2BBRKI-JDi;9%F+OK zPJAU(JqjaCOBsSEPPym}8zDF%MXR1q zMbh2#9^(8gHBx~S72*F_r!};durS0pd4%6>8av!IEFras_!;Q5i<4zh3}ry7+(~st zl6=N&87r0s8Ud+-qEOT0Lmzdalc19^t4H)+?B?|*Z1JOYwhbm#Zk?O~YKh6%g((pr2WBH>$tR=}>~n1nu*-+_?$@2$CJiwu1KZnmj0I6c&_zp9Zcz-M4+CF#5M zdh6g=ak$p{t?qB5on=F{J$L|Zh{xIu{pDso;@ZHYAOYg>*Y1aV972L#3So%8wIZ`I zhbw$$QZ!rBcT2m%-F23>Z&i@cukNi+tHSzoJ0^wSQZWiyd!8P8atLg;EjAi5;?@2` zxm1lYBh=k2)i!8#v1RPMx&Qd5yE53_^WQ>0hhC9!*s?k``#fJWJR`U)H$q9=n+aB84 z^ZYV=a`_IsayhN~O!uh+)30+&I=z90{O8DsaU{E{%>ide z{8Xz9m3Eej1q|rM1!KI@`!Jr=%@i?UVQ!yL)lWF1C1XF>`PwL$bUPeGB=57cvvWATH#IJfD5JV5<^mdJV%1yPiUBb!iXl6Y`J-u%X_s5bo?E4PS%hww5Z>2~z( zH|D~3!n)(d%>z^G?t_Ah6~o|>_I_%&B|LO_q!907{&?{#y;O1b*7A63hhq*LkB}P~ z4|tmioW2E}m!EE8z=A%pJ|xj+6losVs~5?rACeCEld0`=hlF?kZ~pZ;_CM3PXKxe5 zToQ~8exNNc887(sL$A`@8w9`5eVuRw&lgw5{5E=X2ds8ei1q9{y!JPAbB*f0 z)*5b`iU%F1>n@o%Nt1m2In~sh<+Kl*Ggu`5(6q+eWI0VKCO!Qdl(Sij&nK&m++bHZf z{_WfY`JiRilcKRL1?r*Q@j@Qy8ZUCR$<9JAbzkY=Q^T)rg}NICN4^7D&0_Sdn(43g z-nT3WXwTG)%V!!7dJ{YpTU-YH_O* zyN zvY_`<4UR3XGx;pfhtp&~$AHN=F~Do<5b85CZ&3#%edzSw$2aCHqde!l(Ruq7?qCW| zy61}V)ln0&85<98W-XW>rg05Pi?sM#=SZ(OxtMpZYX`#_R`FAk z6?!)S!KZfDHd`(x#BKVNj_3TweAG1ijM%db@y!DpOb`=GJeupSH?RZBn=E%)dHRQc}ET0Ll9L_LZR(A{|{7`GxmaUe|BV^MhCO45u zCeLc$#{xHT6b;Uyo7bfIxRffzT zfwbaM905JLM7p@pRK6q{_lc&jb>_XSGDt`Y70(7s73OBY{EVJ+Bex@@O21AHGIH0g z#VmfS?ZXv=fMr_QC%SRI94c9SPV-LVug8i14CW3#Ahhrc!9Pb6kQFySrd*3C5<`Gr zQRpSAHD$#X)+p@{swBW(g<_=x4w8iZ5#=)-5uLAHXeEZt6%(12fsc%+)Y@LPO8 zH+6xf2}-_<@6U8KT?9ciDs#wrWM`~zZ46CVn>HX}-JSIg3%>XrQ{`B0<>G+B$&!cd zU2ympOPmb&X$da}$XN~iCLOV1Gn0_iyo|6B>Z@Fa9$fh-&fNbtBhs61Xv4Qw_#Arr z3%1s0vw}tW%1|cear<_>dXliJZ_wbwafnWP^a!ys#nHHLX?m_)?!30RjH2V4CH#+T z_;rM7eqQcZn(LwS62=uBf{+cqNLJNHSj`9_+phMS&m%Cl;ieG(8&`jX?Ebmjy>AJ_ ztEcI0{cjK`gaKIqTIWz5CmShrgHoNVUF{fjp`p`Od5j&O`(M(L!0>YUnK`FE~@ z=E=S+F1L+-8kCezcM7qF6M})1!rG$>fVYYBL&K!5t|rIP5Yx2Q28Tmf8=KYPi zSijws#JOLb>GR{g^#O0yF2#wt6g?H;Hth`Kgt{oI152JsN@9gy-dQTBma`LZ3l>?NJFXCL|Vje|rAR`X*) zt)n2umN_2D0@gc^@$%ey8xW5DB}urUYh2NUDq@wi{R^XFLL-lq@B->Sb_XZ9F)$dA zrJKmBFM~kpT4K|B zRVU+d(@6EUvC_BiF`0o)u^Y-Y&U5jL5NySm4UMg!mC}5fh5|+BgIL91RbF0B^!A@M zait~qzint`*l62tD(j?O44ntHrBx@Zl0sM|6N! zMV&7mZAW42MFW0>_-}V%QEg1>e9ODJg>PW=1k%!!5Fd2 z4kBk00!nkAF8gT1zOWFWux)Fpc%`^5h&shD-F#64)z=ffmh5Wr+Dt)a96Ks$Pmf3+ zvHa6dq!vsE887GzY(ki3@9*&d9DvKTYIB3G4bqqSmuh3Neww7_TL1m5 zC0f)9B|5Vn%vSdWO|E@;DOSJOYaT**%Mqpn4(f8&OWoW^zC?HfC?W`bLxECEeEm6g zjU!Xw?&HKm<5?|3KXm{qC>y^cG>`*=&6e790%MeRERztA zG3gFS+o|x2`nd*KMT;Xn!}s|kloL4|we9GYjm=A&k;I)}Zwk0Y>g4w-nB^N7*Ds+5 zU&qk?cHntHQ2;|Ouw*o=2^>AAq>*?eW4Dvg4zI_EO_>KFqLF#^wxMsww~osu=3oKK zJ@pTJ?#mIbn`vq}UI|E%T9{=#i}AHFJ*f6rDi7Ch9|&iJj1B`{L_?UHKQ5dRqrn01X-VrPyLXd2Qnw=aqbKKK$9@pf#Ee8i0U$}7n0NW8|h5wsW2u+fN(MzQSZd=vq^$Jr(?9a)_q=8 zed1H7`bUvH?3&?4`++%NTfOUCjC;cuu_&*# zc&wfxXzaU7L)+hBg7;fS)#T?c?wPJK!!^19_9Pf)UztNpV*ebC%kQ4Sx*VK$v`Lcm z@}vPUhWLu-l=$w-0+W=)z0Ox}+6y8Zp}l)Y5*t3vdNw25bcdyf=F}vZUGv>KXf`?$ zSgf1^qFBsh8HeK{A2mlUQGcXI&b&3Y!qmon4LM$5H-coaF)gB#_Wk}kJhb_txvDve zrWq>jvC&;(kMh*M*dHjfwT&q|zHfwdx^i{nU$;U3G~R2rC(1awkvG#Cm0(lBY+oo% zKhq>&2h9vP@9<=LxoT(I(c2Ockz-x`7{Ab`5}XebZdHKoyxJD0p$$;4TPy)(BKG8tWfdU{&hE)`|>d=$*%N9}^Iw;#rhc&S4j2ys}ec{8~n z-a?i)v{_QvLVVXEt@#mBAIVh)wU8yPW~~0ikl)i)RkN91Zd`7z=4Fw22k+YJQ7t*a z++BX;zp#Trj?v?>Z)Q)Usliip;m4CM0-!;}RGk5YR{m^%3d&3&eKx}wH6zkB<>=hw!cJ6IO9n)0ipEkK2rh!Y*4ZE3X|nJK0f|EKhLg@FpE=-Ll|XgsyaTD z@!e8d8LQ#D!D~{YBA-1XX*uV=@7iLgT?()cAlptr7G4;u9vd-Fvv-6q3e6d^eyj4M z3o~s9)DAg1%>Q&#L^%|&`eF#EwJX7rh3I*E_iBVkx42i>w18jXWAHvx5dqC+q697_ zX>^jFS_B@Yn#Xtqi7iHBI=SYh`p4C=8g?MUFDgc_^lNOIxyr9DyiB}|@_K1tliqN> zrA9mnqSt!4y~goMA-2J77DgylQ_@NO_VjZP@~F`-FkzqXG8I-YZ<0si+5i`Qem`-r z-k}E3tP+-18m@&jN2($<;x4XYhZn26Q|g#h;|fKw7jcyfZoX2>Bfp)4wp}5-s}42a z1t-H8yaL90(9_QJ2bHC}j|_H4c)8oLqDwL(VG+(X^@Cf|H{5&z)CM!c-ptkk-G;wy zY~VBjj>T{$a9;kTiWY>dD*w+VXeuhI#lOX5Z13dOdS8}VM9`pfiOMTfGrI;Io3$Aa z1AK6EW1~cc*uJm?%vn*??2nCEUw*h6s7Wa90o+oooW!x;!NH;XNU8w2*sSG9E%C1k z_%MTF7aB2w`kmpG6;N|Ml1a`#U$~fUHuYkx3YflUL3F5>s{}xLtNRTuR+C>Y+XVc( zK1`BbzOW?b#Id_7TdeXb|4$mH@w<69LN%@^UhWm>HL#JUC0J7QHLMP5ef^%Vu^c7@ zO48ve71Q-#Y0jVd3aaR;VJ;FX=?Y|pS0Z{X;VoWuZ{scDh;2|EKQ}TyT23|Mb=4oFS!uT-^8RjDeGy$i^T?)7F`LsDLj@d3y&u1@<#AMp; zK8^&DqL8fhlG7rlFw7m`N0ICrHh#%+a^wsXTT^&GmhUuUcXxIKEnws0e#qF-ptZ)=u6ZQn_Q|U6vbWYHO;z`#QCV%I)xF45iJk`r-eATMzY3ye`6M|#cCc4Siv7fM^;92 z%WFq+V#Un{@^mA}7f~>HuR1J{4YYj5p55O(kJG@E$+Cq%`mqj{pR^5ZWL1#RZ*v16bX*gr`H! z^F?(V?Q3Lg!fMfXUy?b~Hw}TqZ>?Mx%GAk?%knKB?1VPeOjJ{u`|sCTbtP#YZ11a7 z2aSD8qNd22?=a->1u*N(hl>CT5{F8s8Eu0y3a5?if(fXkw8u&2kc@-)R}Io4QJzl% z3FDo>B?b!OkX1J0x)jQi%|^-phAgkNm|^*6j_*=v0k11gpeG(-STS$Y32XKJAx`nc z;})2BtO)cK3T(e4Gcc@)pg}!S$Y5+fn6Uu28^7TR%7IuFlHvV1u-S&05C4uf3uEgPSVSBCf(#N2ZMj)oX zstfeqF^)qO8xrFt;K$D%xGT~=#_xHPzR3-3wp*GpSNjL?z&U>fTJDt>-yD5Z>tH8s zwTApQjv@?Kihxtd%KizpAe~OGcDCS|AwacKe?o2eeG?!-jFIwh!&+6(Z>lztocP_Y zbgkf}_$cFqW-{>3?nYq}mwM7H*$lLd@^O3X+HjK5&F)zNYtQPGlL2qkIY^?|D!(-_ zABfFR)PUi9>e?K?RVc=}jFl)qfsSWq1!viH%GlyF%Jg#3kEK8^yEibXlH<#kP07j#x-_SGxVn!gl+PAIQQrBhoZoBC~$ligUdX;Am#AF2+elV+B< z87jKzNiLcU#$rY^kM`mFqrT@xHRpP44lpbx@)m^&c3uZt;)ZoL8DyM*LF-z4E9D3c z%iCR@5Zg`ud>x@)g9Bj_?!%WJ)$;{ajk}3xHa%LwI4h~wLLc*~{nL8zW8kKkDKwR?9vF$!M~?-AsjdNU-4ECOu4|el-@8jwQZAWHS6prtnjITt&)tIg};0S^z^G~KeGkR1LQhF09k>j_1etQ zOKkItJlVd-C|&j#v|`$X;sLRVRU5vAj?_?e6j@s6rKNNDC)XmAayeb(mPk&2qab(L zj{VOM)@n~H1CR8mB^>tl+(4HQ9lCf8RMyXnU}XXi%V>nN-E*JWtTEQSTSkV|8WBSQ z>{H@3cwyhN>z@7oIvx&(?_3mjK6G~^1pEp>p9*w@#6w0r6OFtjyx)?DhX&ksF!$|( zIgQ9y^;G&w)gt0#gUqzCklmCcIMufnc=>X)=mdpoC(?i4Edk@42I&Tn;q2af6gtVT zn%DN|7ANkbpfS1wp8AcLl*%|OY^>p=QJ6zt;u(?)7|QXF*|?u zhtsQQ+2-kO3g+#~B_<(X%JLxP=1p9vXfpbR(O{Z7Uc|p&MjEG{Q6Bb)-rh9P11l&| zU4;&HUv}KI-!<<3%wSCaWD%L$)VX6<>VoqkMCK6$zRZhzzsyQkKsfj9tazPlz%Qw6 zW>W(9q8R9Ac?yLqDHSnLtzzx%a*1EXYZj7z_Wtc^+7PNkuSx4L`|oZCni6zIh#nU0 zB0$`6mwxY2;uY+UUC&%)Ew7}**35H}AatWD>AT*S{(6N*j7|YYs!trxI1K0XYIjUZ zcpAZ($MOcSojW&pw0gjbVHy{bicvV4k~hy6elERva%jwJTKz0WN;)7JQ$8-bdIoa6 z&T|aT?!X)Ja9m0}%lqqXLQxyKQr?P`zOM)RJ|aN45g~Ho9DO;14Mv1~u-|;gFMn+X zS>Lb~kiXg;?jI1)IZdmwBmok!OX8=6PfL}nd=8If(9iR&vAi~U0o!$hd(_(a$BE-1 zTMu5&cics>YotU}Qy8_(n{P!@Ci}tyZ;nYbj1QtP&VL~fW>jDz*R}lqC{R2iFibKZ z*InOm07B8zJp$;ysqEm7{=>7>+Snpowz+cfqr#5ra~AKKQk%*+$l+m%kX4xvaa#iTg6`C7~VLLs$tm zjU#E35xVPO40S#z9LOoZgzYan_;Gh?WnrV-q2Ah|-{EQg9T!Q_ee@H;=z(oJJ}toC zC?WIo-Zh>m#UK{je@+O`n}?O}sOQe>w{Mr1!EIa4IDtp&lI!VOAYf$a`nxhstJc7R zy4rq$00Xn#q?5{dYkws_EzVMVB$^|t9YiN85#RY>nqO=?lK2|B&6wu-Ox*C^3T7Hj zoo_bTRR@q<3E_|Bs}ME2LII?KJ((%)xQ+LQU4Mxs4sF&QYgVZrXdN6-7Py8!fhTdl5f1b;)_MhjSP!4k;Rq-A1 z%~n5tNZ>8@`*#3d$u{|5UY+*CY7dC**~*~U6USGBG5P3(z&j#D^ngDB4916EJJbGJ zswgz5(z{TnbAPUro*hKN-;A^u-2DP+59)LfJJriWRfBX;=^bAw#V|T1;1@1=uzLYK+ZSV{ zhriJOwC#WK_LgB$yF$OBL;(vVq{ARYx;sVb zaDV|BO1g$_7&vR=FP`gM=e#)o=eo|kB4+mNeeZkU_gd={i$+%v?I-1Q#T5Iw@OME5 zQwh{igtg)v5WX0p+@b(FU)w*u8oaOf8sifjK zUgZR8ert>uQ2l_S@asJGvoUVuqN(k9&ODat_2nZj z!Hw7zf#5-Edi5oK*QE!@-ulpUWfh-AxJ#zC%D=?Z4E!f(4 zIDM*Q%k=T0NHyU)QjQ`RO&^3R2cPwTO)Pz1h8F%6iWVusNF9}GYE`(X%ezc>f22q_&c z-YFI(g%F}vCz=gD1Z6%gQ;g-YX5N@rjnd>DiTrn=A>=jbt-ynRxm#*(yf@o=`ph7N zvKq;gVtTvx7$kB}ekw&9&=mZeBfykf*q!UR8u1%l(b>gZtPr*IyDI$O6jg#7v`EY$ z&QL(|_IH74XsW`TT<8ss*i{x4(^gQXrAYswG)BMp6vkstp;GPi+hS+=OK=#JM!{#S z7yTnycG6)g@BfaaJ7kgl&U7a^FKuDn1e{ZuI5k@7-F5TH=eUEfoZ$%h1p)uB2vpiW z!ZivwQZ}1Xm&aMp4@{nC(MIMn=Pr)S<4H&Y2apHrFG*7K;;wr*! zYlP{y7Zg2w1`(if@7xBL``8^wOYK!`$Qe|p6|+*DGngbgyG=&ur4%B8XLst7(EwpH z5Ev1Ho$7mkQap&hZ?H*x3=`0u+dXk>M>7 zFo(IN0FxqEemhv{-rA$DT{@HW5u2`JnkkhOLGP&sfO$IaR`lkvbbst%h3g1UJmJBcs_W;OI(Q5i;Y*fpA1w~|KZ5{L7KmI%(uHbWm1!^N?_c^Jb{p) z{u$&lj-*Ef_doxO^xDwZr*tIWARZ|4tbxZ96YCjA!B-$-Im>dW$+8yB7k&Ushcd|5W;c6H zS2y_a&w0OhBnVZUrs-FKJ-+tVx$_=;z!rNOCNdT!>HvB3l{;qlyXMUae*ddd%EVA4 z#QH2I3#qUHx-_if61A%U`3w?uNrrSg z^U)IExKDO%Bh}zDCirwS=lg3y9v`1%KjE9%C>)XY7ExEo!L`ou6e5H~g2WTM2*ca$ zA&+lQY@kQ?-ex{PF6CJ-zkW&v!n)EUGoLCbjFjO>(A8FMdPk$8i3Daxug{1FI6~&-+Os7dKPqZZ3xNrpHoL@ z(S7M=iU*8R!I@kDW zLe+Jnpm@>z+PEN78QwwX)Pvk?@fVEp(c(x2xG^{(g6wLE#?eihIXcJXCrCV%im0Ix zKe*Z>r5}Ot2Z#k^R-U~yuB$gjREtMsT)SXoQ6f;n;kV|3K;{T~qmiQ#jYl{n1hpy| zx@>m(ar9D3PmeCv@oheMl4egG=&|3Z#g%Z&se^E7ar z<;#CZXeiH+c@hy$`byAeA}30B_U}J*Fqtt7P^9N_Z&?B;%Jc{|I-1mO#Fdqm$*8J1 z45=aUeShaA%F?QyIs`$E`(R%K4VQra}(jW`alsm#Vao zHIWVS*H#2m`^(^fz2F<{t|oSI%_^4%5ee+yGymyi7x((2?g!xB9*uTT>x?9F(Vc@o z-b5h`dgWAk8)RGm3xmNAu&5cEA$J6M{eR~{)3qcBIO-xI*LHY{=-5{fp1J-Ia!)_s zPwAU~6&M(JTZd~1nGxdJ5@bhpkTRr{4~nN|p!OWylZJ~&ryt}166{8Kpfp;i!U2MW z{ys31g#i<9IE0zsT;hnb?6Y}_Ow^#iK7Gqc+aG1TA@mFfEZpGe!r1@X{p~Z;h^~LQ zkyFVI^~%tqjOmyo*Y>k}kBq{ySxTQq9A6Q%xp- z*82DKT#im}yVIy80x6)&M(!6quS24uv44&p(*3l7V)I*wni8J4qiGS;{y&t-p8}AF z6<+_R@%z!-qtkRpFiQ~+bS$jE;#NoObzd{%w*GYGPY%>zQRaI@((XsfhXe$_WIx?I z(dJa&5MzRffct;^cVP&+gpf2k5=A(KirEqM;?V*fM26r6zz9FVMT4);n`V$64#`^R({r9F) z|GlYfNYw|y5P|+82NT@-=mw-7-2m~zULaC(m$t{3y~#qZrNQa%{3tG@Z3YTM*pbW> z{vI^@-{%Q_B^ZtQ{Qo%AT}DE$Z|ml3BY{zS*`{V$m~x@Cgk?wBc%xGDzS!XE!Z@d78x&nQCnRpeh{sYdJQ@}Kb`JiLjQKTfc^GLAxus_x@ z+q;p5%d_L2CA$-nc0rdAi*|t$(5a>8=|IaUOIm49%(Vp5pVu zZQw)lv702{6J#&8O1}viE|`kP<~io9gQSv3g6`^{d) zLGq;G_{sB=xaitYCpDgC7Mz;e1=2=(2UUrbnHA$B&oe8PB69~)QA-xK#J_lah3|yTm*K)^j{!mL2ZxCb z`DxTOE=^3<$g}pB?4UkFXPBD8r@jY<$Zf>ZMlYo-wk7gD)P({!;0k1$@Vr1QT4DC# zMYc*H&=9dEn0QAvO_xYUpRC*|Z6W)fjWV?S$K69w_-(KBhbN+9g16|QHvHui_!?v1 zlxD?6{>>|9-iUJ@2Q@qW;X#EA;G~ZM*kki2UVmRQ^36grQq zUY@+v#xYc040o+ttRGKux@1aw-fVAN1@kc;pGB{i7BTsPLAwAYMl7EyY;kDPqfCtQ_6NQsr=ltC zt5D2(lCIqWlkMK5pu_r%3`{n%tNTj3>n`|m$d)2(J=RO6?xj(($wlo?aYd^$%GS_B zLKFcDQ0%D-jmtvSN?zOw(RciMvQ!l2da^A|6$Bx+@RP*Lm6Lr;P5mQf}(({;8lMot} zdHXOZ#NL&!v737^vT>FFc`z)32X`p86N{o|Jt+ysz8N+uwt(;z8B4Df_Y$!i8M!a0 z4e(wFk$$k$z7+;B5$D)57E8r+_BSV2#I^!9@F zSbww!stC$cs(pP6j#KPm$c6}W+k3Z=YrRny=AuVJ@5)-@RAqVhR|5qV!qyMD0o2B= z5%0l10InR*Y5U4+_jeymoAr@y+JZ)z;Yxw5Fy`5nj;NZqywlM2-q;wZv2Y+D^M!&C zFc0pzKxIzl&chQhn{rV~o`+R}t3xH({^bvsijGJmQ}?1c4Fxq0my+OPYg31eNrm$! z?PYvcO|HsS1M~1l1u-j*!`p{_FdTmSJ4U~rH^U#Lg5%$3524^un%{S~EV<&Z=2338 zch#n*m~3Z=ZdsH6Q|0Qa+!TKPHA-o?V(C3EW%={pW?oYW*Kspe}$)2fk zCliqTvX{W*d`sojDqs^#5!D{hmy&xTHJvwojg+E+f9XfGuQ=CRwM=bO3q0GN={tcH z7x&eP_mFRu;q~t=I3HH)zI|DGw?HSipW_*5ndx~Knj$!>u-}F?=QqZHq`kE$Qb5+h z{%yTlO~Q4n&I!K@HtEjAznNX@-(Ug2RhjF{un}n$ zz~7pR5+wABx%^TH$Ts5b`ax{>gdr{?lNx(dskz; zy)q>5Ag5mUpWUIcRg4M|ZtzYIP$rdxVzOeNI(C@9GsA~X zE0#N%8+Cj1@#dvd@*l0ni|aw^1rp(ZLj zf9l>=OY6%CU`J*NOxN!|xb+Ixx)AnKMd!e}Tko({^{X*E(al%G^r=^7Qn(DOV_M&n z_E^_kWLX&j9s!rwrE=wh%|Iq$d*<8NV*Ni{=0lruuuOO?MB#adh)#X_Wtb!Ov+PuS z!~q^tGoB4%?xdaTP1N}Oe3MCSxW1}Qw~B$ix@>F7FJd=*diG%-Ez3Qi)BNORe&sM@ z9wW|eif2g>^s#J*RpVVplF+bmd(yI3aKNN(>A`4!+K3YD(2k~ubNMG;sd3!|HXJ-V zPc05ARjTZxHJfL6KOW%i9#jNc0Gk%xX|S{R7`W;xdEcq6T}r)MM_~F>(QR&HoNpSJ zEaY6764ElOcRW+C{GBD!hk0WuAd!%a{N9&Cz75;bfYXbpJ2{UIR1v5V!=11P%A()Z zLwJFvc!4h<&Kin!RFn*_#-sQY9_#!P&-WS08Ny)DmL9Ih3Qo%DXI(sP0zgT~;J(>= zI+T}t$zz3*4}SHlgy6Ng2D%V^WcSS~)W{YxAkFa&LeHms`u6 zLFW7uDk~`WE)FkQ4i8;6oi?KP1DSn8wAToNZ0ZM+#WL}!3xh?SFGdz8niC|!w8I#7 zw3#l>GKXGoJBmn6W~$?6Yy_^9C9kaK(R$~Gdg)GEmYm#et-|EmKkzTl-51P`_3jc6 zV|MlaB|?s!T7LAF^tf0kbk-n^`Pb^Irjw3BhqOf!?2i=l;+(|-TKmUiX zqoULwZ9gM_cHHu(a5i;!p^Udo=#3Z%XX>e3UZlh%AK?FyW4qDYzqvEU9`wsj!6(!C zffm)w-gljO0_(aWmeWh*_;_?DtNJ?rT%_iG*+W!?5DCc{8q!^;pMrVX_nwo_fP`LL z^Cz1y4IHSW2eb)%8?R|m&4rjGxmp-IWPg1~H+v5g`izcafINByne;HsF%)kHlq>lGQh5xXjA46%zI&|_?^BcjDGT!tsG72P!p{1RK4`$V_PaZ@v3`XS8UnWd+YHlTZ=GPY~P^a=DNi+CBSP2tY2XkbzdCZGN;kBPe*f)-?4jC~~1JluP5T43h2Z)FK|ryX$*0P$;Os z<(}!5SC43K5#|)+%|WCCUT9UILKa8d+R0a?K;@MOOn{Vq5?oC%*UmaVs_-2+u{gtt zuCd+KJw`BEliYU;Q9WMao<_3oN|mB~9zY}U^$E#DmjBS5H9T(Z7Rd4F0uN0!EKGr;~X z4rNhYa*W_va8lp~>f0UVA9LdQ53^FyWv#AoQh0OR^Uj(XM)xERdL(XtH*85i$=VzG z{3WZ*uV=Q1+ZQ5P#-RObGS#z&pAKj4W|Q7Ad;@t|EFM2MFG_KCw4A~f#wPokpp(1> z=HSum%2lAzbN9lmrkk5}20+FKQJzRcs@xR$?oE4CglJVoEEZ6T)@)pBvdnlnlb7fE z%&cIoRNLpJ5CrmnVK3HF&bwyR<-3SnN~q64A`A3!45yb;Y#J21q}2(fgNu25?<-xG zsOyu@73H;YAD2oX0){M0uez|ktU`Ud!kkCy*s zn1^7GDXsHvG!7acW^#6SmAivLH>7;?5~W~Hj{Q`dJNJ@wpZ)0QIgKKZs#(FGI;`)O zt!$sZG)*W6XsW!bd}y(}{Fh}D*j+`N^pty<&G$_@Rc+D+Ie%+>T0wEL)t;1mH<-w` zlodl923$D4?)6!8EX&!1#}x#NkYy^_3|5-on+3*%%&cp4)?@YCpiqui_9^V{xPD5 z_XMs$s4HJsEWvIQgJ&2Q-H5#b$k+A9Qo3 z3m-gp$?N`XJk4CZ6bCnSFN{^gs4yvQ^AZ~G)h*D-5Y39vg&QU*(p+1`Qzgww4t1x0^!rdt*L|Xgq+K)|7`w5?AJ|8-@%$$5BauQhR5@a;F|-kBigB!B zhg`#&TnjL0J)eIZN`H6NiW}$rIKrQQFe27&{}sefNMN+Pc_`5D{4?lrxDgm?r2bw9 zqqOGy8k45}UxX>*2VPvj=(-}{kzZs)Csre6@~u&Mk1t1O-nu|RlGuk8Nh5)r8^``) zb3sHu^3BbvMD*k4s;a8w`>$>QU_qB1ROB}Stb=x*2PZAUuJ3+vIaUOEg`ihX$rDSS z&Y_F`UzJbIhC7v*Ci^v7b;7eC1!2!>G`i!c(n)JItmbgMx#D& zkcd}6-JoSA6mNW(sA^&t!LQ34h7`B`ZbosABI%857+cn*>(|uv?nC>$PQV$}NVt z0Nr~+roXkQX+RwLtBV&j2Fhj18awuON(iV8rUgCgKd8enx(l#*tQmf_=|=qi#(k)* zcKQ1VG`}k~TK-(_nkK`WXI>LS0sWk}ZH0)SfejY4!azdO^GtVC=mh^glIc8W<9H1t zUxw@z-K#b@Zq(9 zuE*fLf76c(H|<>7fG&3fyag@VPoVp{&Z>pPxbRQhvO4olSMS{wA%#>tUmsC=mKGo9 z{E5F8?zK#p>vGLCF=DN3Fvw^34NOOuEaacJm@aMV2 z_UAUjb@>H__`zFcsk^h9FY6}#Em)QZmem~M2?z@Qk-0DNggNQP`}^_|xaZ;O_s zb$80+<(I%_U}t(MW=g6C+3mzy`#`*#6riz7I1P;9J8*%{k4tQKWf3{{OrAo2&fRQJETCG5BA!cZaHe7G%%5&SR*#skvRgb>nd0c2STpz)_a4 z+5I$g0;{~*>nb0`eZ&|3jmW!cr6j|_3~XN4PFFW^N|5(P;l)S5e~UfH-HOkpt;Zq;EvxO?}7RNv~$qkF| zKdo04LXM?g{Q3Dr_~dpdS-x{O1@{aVgxB>t=jw(7ON#sZ|EIxOJ9#4&-<)Z=u9Dfz zbQ=!q=l7=o2t1>;E(0gt7FhRU^S;lwlP3YiHyn)^qkE?{&VYj3L~kRSEf*K%l5jrO zuu(~K>v{LtEl_KLQ?|LQ0Jt*p8cZxF`Ov*3@_kBY)Nz&u9=e{`$H6LeOrKrq3eI&~ zLd%k~Ix`{81NrqAt2XbV#v;m+8k$3#42rC=HfuZD3m@uJPXvrRA9!Oof--1T69DXd zw-Y}M0qivakHK4HI{8L23`&dfsbk+aah{pzMkleN$?pZp-odPR37*@1R>0qV52nb! zgVlAh_N*5&8Q&~|qm2uspKFp1(jM$ZDKa2_V6cvTqLyws5akLKL^+t)dZeBU*UH8N3 z=P9du3olbM!k`_SX2^>qU8$Q@igu^R*Q+;6sE27)!$}gC%Z4z!|o=(6+Qq!X~XFgr>6Lma9R6mV=? z$h`w&CZXpE%LY|p#66CScUnbQ2evk&bVNDDi0AFCyS{$7!e9cL>lDylD_EXkE7+P% z>GJ@$?Au^&{hmny8)Mx&o=SL5K5@6gs5SEU?di7%mk(NlR!qF9whs_iZl~a#PsIb~ zzdzgO&P$uF@6N7uR6W-WU*C3l4{;Cfl@IV~-Z&Kg3Js2caJ}5$i{ifPZxu=+U-@(-w zz$*c~O4$b$Vi})GIoSZeI}$MkCwR6O&)IipyF3iJreEIkYkVTJS*MwT1G`yNqh0Rx z<8*F6xOXB)&5PId7_QAJK=0d{F(z zbL+}2gvnv-)9&`{i9TKV?=s7zo!$W&-`SiI%{NME9tJfA<7$o)Ih1dJ@MQMz6Xp%> z4JC)%J52?JAOXkSugt{$zv|QrYb2>Bcl$W~+a9E!Y57bzZl)KZdMhft+xSx4@xvRJl-A2hzI%`KjK)3@b6HrQcWCiddBad zKL&(Eo9%w?dE-px<9uk@>xF$0rIOmt*Z9zqQa02R9Le-l4GsQQ_P#bUFZ%U@? z)kXwlik;czn~?*zk;e*J-ZQ2VioDkf17CE^Yy_fE0hF~2r=0nkC*e{ z*8BA$o550ZV%fqzCg;uFP0MKLeb(~v*pNaDzVt4QErIx8%E-H-m-T&m-X_(wQui~n zuA3{F>*(Ak_;sS_)K{I+;A#?{`hymF zrd}*?hpuz}t=`a~%?^Jf2BIOcQkhD3U?(=EHP)(-dUF#f4Q|B9@<8zHMi%Zfn|fj@ z<`e;)ORM2tVut_ssm`RnTckBlv0xFjA62L%Lmk8f%ei(J!V4o9{c8~iKZ!?tLi~i0 z*Z&*u0X9|b>StLO*fs;=1ssQuXhZP!WxY3$E*#(kCtE&PBBlvPECl#Rtq&UnN8s6Z zM5scJ{ObXv+v7iuXamUaH@YGGoGdsqlvNh=YQukkh=}Ov7V-~x3BMDg+-F0uF|d`> zgotFGP{zl{=ZG!@|MpE=0hc^(@7Q6A*&~h!{7pO>Twh|+OG-*&9uoaWaR1^}Qj*B~ zb*}E9BU?q*^A((iX%K1oL?Bk&zquF@{)6u_X{o0YjZO{A0l!TPJ+}RDfDDV*8uOVi zF0BY!4Wd*E>v5A5?oQ5^jjt#{y%C1ic*0&@4U!xSUX}ifo_7bgI${)ryHoNKFP4)C zZOQs>v^tv!Q#3?v12qwF^Rq?!9r2VkYm0kH-JQ4aZ4T21#X|$8alF&^jQ!X1c{0M+Zd7TKOLChGOmZ=3l zWlgv#fI5s>d%?+bgFm6xT{Eh4l?NSrDgnjPde=MIEyb0(dTz(v=}#D35<{ER2g@n1l@bLVPJ>+`)rRhNfzlF}0^xqg zTEHfs-#ft|v2p{>x4KcWd8kHR);BIYYV3D)6SL*E7*fQy?b}*M@MzN4l{ZOsS&Wd= zs5dUdPmfTHGA#r}-SK0{QXh&ZSGU1$1_~ps0318w39+7i+}0Ku4P~Agm{xy6DHw(y z@Oa^BwxptRT5#?@GPPf%)LuF6+VW_8i3QZH-s22KBaC?#^a)$w>Uwpo+_;lC7H+H&IW^W$;*k4W&yM1C? zG{QHq+HZ?iv=A@3a{f@rwLSmJ&(rU>U+Ui3l60J1)1B%s3(uWs4YTQYg-hQZ?y9j~ zoY@n@H=!sFAqS$ER=nWZT0X5Ac38!dUSomWWABFhWt=Vu=oR@xrF~?zYnw)Z-Slc< z{*?BqR@0Fk6Y7b&Y;U-d%6y{>Nz)E-HC2lTS;r5HkX9;?DKv-rTmEQeeOAO^XiZlJ zyE_8l#q90ngt2LvAcRA@1)EL{2G_`}bDa^*jET6^qRV59N>n_~ec>g6QE6C?H* zNMV(5f7J*~Z>i5O!mKB)0AgIW()xCWR!*6}E(vAc$Fa0%b|=&zex7c8VDzzri1 z^R(53oa5l{+h%S(=&TeQe08XNtoaVcIfxc>N5?pRKb-X9y>Zt+`SI@4-j*G_WP&`) zc9Zh$CQJmas z(X56Z&3AE>1;S@_c+CeC0q$pJ4z?+jt>=LK1lb!B*X>5Guehpsa%UQpqe3@D><5tL z=U5-dShks+v4vK7#<3`#yzPloBFhD*EFHUYt>Qd#y_KRZ&g=>&HNty$&W-(KgFAUA z$seeeTRj@E7!3a|D8-51GoQIII&C5r-oLpfu(g*iG5dv8UEv?2=eKgsUJq3~&y*WU zOwe#)sd@zSgs6uq?A!xH4>5Kv3%72!Bq4)$_FQ{;8QF~bdn}jp08UlfT+%T9@Io(> z))*M|wTf}GI%azx%stZE^PGIa?*s6fC}DRlnfcTZL)Rlx-`I-^*1njI?V=c?&$7ny zBs+rjjBr4Cd&{Y$V#}B~o4tLa#1yR(n^uHI{w1!uN#}^P7_{M(>nE2`eR0K;!Q8_o ztkApq2BO*E?7(LUP3|_yI9T^T%^(W$kg@LjrF8z`Eg(&Ynt}BpLW^);p9$UGBZ)R2 z_4dI&vEAI$Ts&;>-t%gj-X#%E^jTfWtx`0b$Dqau(~e2-?^Uwz-7mw{G`3pStOd6U zWlyhoIDez*6w^&I+`->@`(pMc%8M4E-ny-f#iWCAM4~rUzOZhWVX$$q3ii-%jV7d3 zb#&aXyRlU;dwj((F)?u}HD?mhv7Cq$Lk-$%G)@caswHKNZw=6%@*4a;i?7vMb{4HZ zbkkaY3N3;aI}$7C(i9Uuv_|E~wnQ{5IPZazgbX-+Fq8#fJKb<&3u6`WhT2oM@;)ge ze&%2-oH=a+WMkrO6#9D|P6y*1sT>Oz*sjX0l3#`9S)8Z*7|xynI5-s13_&L@BW&?d z@hoaS^y`=F+K82y#Pe;8H?53*TpyR?NQ4+lJ55wHxb=jnSSapISxHG0j?;Fnbo_~V z^hGYKQ))IRTz31%Cux*${GJ5cMr9qy$duW95hAA%K7A4#6>4`zk7eMi@to6>np0=9 ziBIWlpq8hF*-LWh?R23j?}-yN9(qw_u1lz8EPjv4t2d6gAFP85*sflt5IXg4gEN6R zQsYnlVR&o6=`T>c&04dr*bzpF^+UrRsqB$bmcE=3o-Legln3ZWt@Z0==u$m%9doqJ zc3$=$NJs~S;_G>K*jC9H+BMzaTtbkk&N$2a+q)Zg%nY zF3U*XVk7t3|D=HtJ?!dJ_p;`%%JGY0xAjSk+}pK?Ns5`PBvvU8{>UZfJv?qqo%E$u z08Jog*=PRfay}+OF`*aJ{gKNqrm=Olr?s_P)3WVjBg383J-(cGh}J!mjf0QrFQ4t= zyn9*$%WXe)UyZ*mB*TkOaLZ>w3#x4svt96v@--8|QCh6;oNBH+M zk}8Y<_G5}$ssdXK>gsixGg+T}XV_uE1;5%T1Wf$5(@%WPr?!E^C(zG>$(gfC1GbjC(8fxeG8BrJL3o6NUP zEv*#sbxKRA-x#VVvT<6+|2S{VIj5^`NxDydxTl;kg+&8Mf(D)(*FhgGDgC*$b)Xle zKgN@pZ|hoiXCnF}@%7Nm#|qurdziO7sY_OLsQ_IhZ@N$4JyHL~c%CB}pktkcBX#!Q@N2`%Ac6suF!QbH`K0+5%^=$trW6FXpA> zgg~0;#vB+vPyBqRm8w@ucD1;9?1T#RBD?(IqCNKGF%f8c=K~d-Lrqwufk;o7({_cS zH7T4yrs$R$>9Q5CSI#@cC^Pm=^wX;dIl<+fdD3wMd*ECmz2`2lX+8oB)7;`=s$v2f zPOk!(w>LND)WRkFrT!k$w{!;DIHhJ(q@A-QDMDJy%%Y1*z$I55X_=yWvNUxm23(-R zU?prx@{x}KnS5&@L zPxsJ0TU-M%n~qu0YG6gQWtXf|@yemH6k^pOPF5qzqBbjuuzG^uQE`-f3|BOY(XW32 zAY?EJQv;Xm1S@Q@7Q1#OgK4Mkosh|Vb7y1cgmiQcG05xUo!)%;VyT4RuY3(B~JTgv6@tYHEw z#87Pdz=AmKoJKS>017n?IL8^a$13ETE*(=z1h1=|(jR-PD^g1`V-2t5ubkDHXwtdY z63KEMK%^qsdPG@y)Ud$3Ms(;OdcQ|AIYk578%?3-)kx2J55@}UWs_Z@M-^|Gxhcd5 zYRN`ru_Mg0>VLd2i}V^rUd6Zf9m5a-IgiYBWLFMlGwrZ~s+cmuKAtK!+>5M9QSa

`Lcru75oB+swyxO=|RJiyw^u9Pd2)V6Agog zmQUr7N%0O)dO1MsnxK2A5wz%J&>7%a1uH`3qJh09C&dfTcDht$B^KXMhb*2|{T6F? z+B-V!d(qHsToX~+y`#bu$`ZVo0`;wYjfc3?+VV2zN=KxOaJ`#-q1O*hq(SQy#${;Z z+y2t0AeBu1JQO&54+oIN(A}{Sn*Zgr06il4*@tp2aDxUY%AB9?oH^(57cbRjl!?MV zb%|b*jl9_sb@IilGuj`ii4Svr#;K9|c3tq|tie4utKlWNPkszsbx+wY2@7OV^qJtR z#SsTf2B2S?DAF>HI9b{f^FrYxNRg-eGE^{uSBlti0q=iH_xG1LXOmJx$D-k z8zI0}i9s7(4I&~rbB(je!rQA72sWR7`6XrRR?)0W5ZxyshaQ?LNq30C2IMQCKHMG> zrNVSz^za|Jxj`L;dd(qPm8Oe62M;)Ymzr0UExlFWEu^qJ^b;0#>VYp9>2?W~rX%f# z5HVy5FMip$GxE1QSj_1I5nPyD286a0bRaA&jL^K~76*SLjf$YJr5}dA z?l+;#2z_Jwg&-?2ufvoGFawHxbNRSTgF6tKqCj&f14acfNZB(l9|1dU5?qmC0UTMY z@otnC(IyJ3u10a8#!ky52tuRY)FYwRSx2)dZUkQ6I^6Y4+4;oFpoA?_tTd8o*qXQk zp(^ShhI7+HU~CA@3(VW`{QK5eyJ~l4fyWD3k8IO!VLrS`ateof%WJ3{b-tyc(W$&` z`i-BHzlv5}WMq4TzpTWjKy}q>5b6&CQBf5eMY7Mbfh^Zz%}Q|mv%hbJ8d-Dl zCz7xN)S(JA`(uKxFh5TlL9OoWdBitAJ>io6p6=aeJq#Hj@;axU|gD;(vKVQWBVvl&+oV|ZXKASvAUl8@Mf2QUN?hSbm`|*8zdk=R6 zZS?wc@1E%9Fn(J_#JTLKF2n6?Hc~I2*10VS6;R_vA++TUPg<Ms zpBTwvbK;Cf3TIwvMLtkrcs!YZqnKlepSd$EFfc@;vsT5haK=s!OetqQF9xxP@opy> zT^6b3dF)^UJ<$-3|4y#CWH%1kHsw=`7*0W-sq$ov>M^~Hb&yIZ4mgs*5vKo?OB6|k@#y~2x0(`!M8@3b6jdx;^Um@^;Y1Yi9p7nE3M4JBfGchc&n~9L3kqTM{;ds3!*XS+W1% z-0^HPG8t;lyG&hrrDHYV^0ef+kQ_6xXVD{ib1C~NtLZ$-XqQD8-?8V1SI~5N!He@> zH7bjh@K)BH6?nAYAw*$HG|v68OQ9%JKBrx7fZc?AQ6; zjiju@yg`l1!5#wVhe?js%Qpp7%M!8Hr1QmM{f^>_s$c0{BQx);MG5+tMpi+2B!=Ir zbo;9mIK0Hzlw1c zli=6TescweJ5M6zwQ<{e*Qe2}X?YcE?AEQEOrWe(o8`Z|%aKdoJfkOx?huPWo$)58 z4R6)k>Q0uXsCld_tdjcAoE)xWBx6E*TxoLnr}bC@TLDA5+Jj0fy%t-JeW?!qogN~` zU??p{tjf!~=Z z?#62`ii&5i(^z=6k1Bk1-s`~Fu=A&Ngu2OuM_aEnCn?b^h?o^mZdTT0GJY??C9;Ic zjq$SvsZX6Y5v^l=*_OA`9M|1udD-1?(5b8|HA7g~t>HyoIeW+p(zfE6`I&hjB=j13 z>~*G$LGgwmU9+ z&m#H#7(VAcrIq&Z2rH{EA|Wp#Z;n1EEn4oNK(i=hwg}~j?YrpE^7D~i8eED88fdSr zmqzhC@=&QNWb={E-o!V*Fq=g!n0xLs%itAAn(uCy$_%AsrBAv}VkJz59*X-8#;;av z(}g&nxmL zM;U=zUL{c1uxp3KRR>n{SBj?x%h}ZT@HaxTM+p9UpEYk7C{&c{esb`M=a6WjkX(j? zL&=ie15M>mPJS(vRfIS;x1H6SO6KYA#rel?FmJswp1ZkuYMkIJ_8}>my3OUS$Yl9c zg(A|yEg1KmbwxA*euxe@JXI~~{jSCM2e+|!c{nXtB`W3r>o-5q#Pfu}!?u@RN_Sr(%^>`=AT^25m_)9U?L2{Y@)483m zpYi77`xFtkZyiI0$dmnS8gwDMV%x5%FGP1NEZ>@Yr6b|HM}y0Z)w}}cYvlGsE8~{y zLTKO~d5ptQO`zwQXuRCO$DiZYMbz^~OLMDbgwuwk7ScCTOCiLE20Xy;xq^^qefzIV z1~`r3aMtZ7^40lPWTOUPPj%XEvX6n*-x`~pR2dA4(MPyX$W-M*lVQdA$<@?N<=SP*JmVPWvg!}*2sX$3m zFv-)UtB`DFAkMp5Ms3jtuQ<=`XHkVE_PlT4)+c8WA!SJT62MR|OV~Wa^HpI0HN>!& z`k>DC>Luw&F=m;$9Bd$KboVhko=4r9UanEue0c7F%9zJ#l3i0Rf;Zm=i+k^G5Su)T z%J4>oRUGcKu7ojN3IT5gM*5qly(=uvrXMpm`{EnOlwT|&9+E)(1tgCN4-W5&xD3b- zUdhUuldtSyVI1zJsW7{D_!Aw-@cBHB{m)`PuY*skB6#BhePe(f(|xp9=1GBFDl~{0 zEeixKTx$MnwQYhMSft@^1XWnA3R;vwQq4YuO;}jO`mjZ$?c;5YOY$CEGft(mbey|> zDaTUJJa-)L8)*|PR7>T1gE?lFpS2^2KHT?rV^Xr9d6O%(xaYFbk;s_)GZ}O1^p~c= z&00fBB`AGJ72h^*4CkkXa8#{vuG%{M14{upF{;Cjy z2R(+YPPe9yYgEp~PPoYLFRrJmnS5?BR#36PIVxM87X4Ky^j5h0#HPWjzU`>=pX)JH zp}?K3zAi#Rf?;?Y#gG@Xn;>in_HwbDdWx#)9Q;2!gTO)Fx;HjBgk2}n_tEj6?<@07 zIaNjsncI07hWN=C4GHmQS2x~K|GD)nieR2nOA7pt-W=#4`_g@J$dubda)K)zjm{-a zuRe7YpUCI`y_xTA>q7;#f!i9NzdrG<0Fm?lXVv3(O^-LH?FU}anXP3N94i1U&p}

s(FcNuP zWYATbhDN|R!0(`%d!aj{Y7^8L#IjqW)lsdM-x!eCkx%&E{s7C32mN=&TgBl!(F&0PK~_N&#&i0V8jtFr32)}Bs<%XfJ{ zQ-zvS-_ocH7@^!H{`ABdi}ONpRv7?aFtCWC^p4YrM#$+M&`nbnanj#UvK^M`@EQEg zGp%@FKri!j&bXjb$#0s$U{+mT;rpqq@}?gydaaX-gcDW?A#JOofaCi)*LR;DRNQ+Z zKZN66+ZQ?&*%CuV+cat8-EJA5)pw7E=I*nHXO%W>vOW$dQk_mcSUnGWk7jmA>}>#C z`LT{jV&M?tHx_!w4=`83Ts||Rk|7~S;`1ijw7kt$omHyy3hoe`Nkc@@urzj2;Ltz{ zI+FB5&-Xqx7NH`pc~zPfZw3H-+bm~3@|P>^?n<;`3B#Y(b}pR*dgfb_;u@co@owM< z5CnM(XC0GV8ZxYvxliV4S(b*o5h_k$ud7cRmRCCUHZYLE@`(zU;nTNBuk2BRLFs4l(KHH%=(b zZ0R@ZuEj5RrFLEUd4>y{ym(vV`WD9aVp3%pgY{ufI|1i)sd@}m*cIUKCU0cCIn2U3 znEv7F)-(EiGKKN~BI>Q9qUzrFVd)S-6hs;XL>ftH5TrY#OG;Wwaww&w2kGwa?oztD zLpq1<`0aT<-}S!phs(8eoipdmIeXvtbziZzo(cv}cc%4$HtrcG*)-+{wKQ!PlqK^H zTd7`h^(8E70pWx1$BeSOWy6!nL-A8xAfh3dBL4d`b~G5Ia&RB~9ybFCxyaaWvEY<~xi)2o&+kb9k_ zaOCNC(j%OH$R_AAvuz#&Ro_CsukeD75dkZ&9CED?m!x+Tzzv3hY%{9=9w)4=<_`B`W6ip zdSNj3*u}*Dd^=QsM+;+=c=;{vmN|vgGbsv>GXf131yl@?fi!~c@?!t2Om#hGY9sMy z5f6~o82);Pp1R}lY|{gr?45uE695Z}P*H*ikeRJ?m?(R-`YRX5d-0XCIVX0Mj6wjq z>VeFGxAF6N5ke0-}x@!^8AWs$OhzgZobz=1+3|n)y}rj>b1X_&}Uy_!1XX|-G#$*4XqZc zfLBY>cmg|^DwTn2`eXpVeSfg#jQN-+f3g75M?TZ_bcQ)R|5_zcQ)0d3b7s&){&WYj z2e6~AiShjYp!ra$daM6Z6EJ(=9PpRW-Q`dPp!p9kjVA&%l0vK zBea&Nw-HtwUEV4m1;qYZ!-4$jB|1^$H(2bcr9#i{^k0j z#r8o!0iLIca{3m|I2$|#KA?Yp{(@V*1KdItXuz2*)rZ-`Emh7Z88#(>k!RmJU2Edz zF7W6jfwG%@e%Zu}edYi{K`XJHfLjue{aIaoDahlnA?yE5*KB@+4N#YrErOYkfOGX; zuE?wVB>cspi&;!MaUyoWBE?u>Z3noeEc|qL+H0b+ZPujB^Dmgvx=XVYfCKvMumQ^r z=>*_>4-kpLi`~IU0RCzqC~D9P#`QGY+|6NEH!ut~LFld;5?JC}^hG zb+ke2M2PDS9Qoe1@d4#0#@{OcRWM1DeQA2jDQ65s(myyX=Y(?r3W;Z~2p zzWfKA5AAw-;4D|UEg}>Kl}n_WItb-T)C11u_4@FX2}f9(;VKXdSArO|!75R`)*ZVs z2aOsljMKXay>}4`m;`Q0K#yr|@oD$8ze#I>KSv~I5sqCMdQF9 zIPc##G&=3x6`zNU?0B;PuYWcKgGmcmV!I~X4oEsW|1I2&3ht4(h54HkspD(Ujy&Af z_THaqaf!%lWQeh}8oBl6L~t-HIMb_fls1TZHWd!sw7Q%l_p%u;D+K{WM) z%(Wvli*weV9j@)Fc#w5p9JbN`|>cc{XQ*J!Y(`T1e=4X}9V-kWd zj@W>j$dr}TT?h&y&0>Iqnq45mo z_ofr+^fXWE^PIueYI5mkTrzkpUp!~0MIdMIb>(l99jB;CfQV){NO-J(jxHQruvzC4 z_w9Jt?8hkUKd#_*?1-|_|CRa2--B69GO7aiD0FZ0f{j)ao9?czcStr81QroN%cC!v zKNyUlyRIFGkAa|y9CL@{IW;-`sME(dgN~uHF+n0H#n;98uE2c;p!e3|q-0w2O6E)o zX0pWtQDy^9sbOQ*x$R{!=0eapudE9%?J@&{sC~G5yMg^d!=T8$tVL^-4jjx`*#2-lBwc^3qJk02}mS@zM5}MbO$Bg>=(`$#&xxmzeVEQ+) z7dU8Q1PD{_yvXF2Vu^SKJhm+d+as~Z#(|9tNRNAFGNaI%(C!GV0vVoeDnNbi8xu$C zI%UFZv9U5A${}Pwv3^v4{?{e9O3d0##t3FyA|;xum6;%%x9@_qQLZ(@)qbl?`tQOt z+cPRB{qbA{hI$7S!4#k>I*qgG2Yx{2}GAu@0NLZZq=@4qxBCiH90jeFmFkdG(#A(d}fR&mUOrg{!ooBEfC@*FZKiwJj$-M_&S6g$7aqn8ebd?XIVEt84ipp&a% z^4wyU{>R-XlS7zEEjKI=wc8zrHppm?KlYrDMs18nTTVe&qsnxo##xL#_Lhy{@Ulk% z%iISh1+B&cLj1$Ii1+qhd(2Wg*-6In-0f#9`L(j_67gFLDUY=aDa|AN(~$tNFx3Su z8ApFE`I5vx8o4h5zj{^(hgE?6Z-m%cIw!TjM~q$pfwkk#D0paU58Rv=)X36c!Zi4G zjN(-Yt9ZbDp@8?2l_gl8^??Wh#IH{x?Xh)Q^LmjS`uLNlf3a~@4%$8B0E zM88^&IvjFj&E@0|6vpIl6SHh}K#8>m5^x2lgt>%O%N~KJ8<-kn_fm}wpxcE{MP&7U zry!MlyOBHiw^MIp`68Pc;MSmX|MJiMu4-YqjI)d%^NNg*4zP9cOM&6!Us`h1r$qcf zU5-~w6fEI;W$(VqrV=%#)I=Rbqx6@}^!C_lI{=4s1ULC=xtZbkMZhabw}jyHI&om{ zD`1w6fvC@Vrw zmixdDJZ8r#J{ei&m-U0HXW$xARCRX+VSl{cM>)A1Q+umQ0!G(Z`;VdH3G<~*Cr}J> zHQ)uEn4f$>n~#vQ)WQ24q|c28)gYEX&Y+5++)|7wz{TUj8s_Yrh`qu9XNqy5gK9eX za3oC-h0W=&HHWV5o$i_Q&(uiV*)F^QJl|k-|L#H9ev^X>#(DaG9NntkTyX5H>{Apm(hMJqNRGI(V<8c&!?&gho<9%#qfpnPZ<_ z@wqq3u`Pk6^-8S9M(az=HE8Nc};dDR5KjLEaZ-49OKJcvu+rr6b4Q$-VL9Zk( zw3h$QJI@%W?Le1C91lJN2fT>>4bZ^U*A4Rh&T&gISN;_c8+(;3Xm(>Cf(eN4arw)= zmG;@w2K6TgX$IWmrR$-9`hQw;mE33$Q++BMHMi6DDvtr|9mn|C0DHU;BOuk<9L z+TY9)$nTYbRx`)nR8lKS^wxJmkqzYg!VO~s(eHKSo~ObD19{=vB0T{WyC3-G&8mzDNB=O+DhJ36^%N+>2!Jn6W9 zMQ4e96@Go2*k1Q>PgFT^ud7DHy5-)ctO`UVV3EHc4KXf;7{>sYCM}P%l>Ep#uVMOl ze!DopRnhomd2pG#zed|sWt!z#BcIHrUZUS>9*ugzll5Z(-u3PXu|1x`4L_?uqP~3sr<$a6ur;|O!MQk!wtFW~cuZCrO zmYd1mRYrAHt&%YV*tLbjbL}+xysTES6(QW`@?R(0^Ik#!es4> z!}zUM4utDrT3@y*{tWaveK~{k*~*2o_L-@T_qPyh)K&nHC@ zO&KBPm@ZeXQf2X`H<7bS+Lv9f=KhO1g(lEEst_w%e#(LOM4^I?RlAX$xACU`W-Cl% z;C_4Nt%K1Zq+aUB4d+1KlHb#21EyyHag{E0*#A*GyW^eZ{EE49(|a4Zg_rdv^93;8 zn#mOvioDdj8M7H1;BCj<{M_uRm7a@ zjr&2q|0ok@?+=DgWJ7V_=;Qk8L+59@jJ{}w>DO48qtYj)&j#}DhPFT5WGY&Cug#8z z4LMPel8SW;J{fFTWXOi$=u35l>=b1Kn)3LFjV1|NkV%e|te1&+Z(l_JUhrrdq*9F4^eDh}~*D@ugYcJrf_zL-4RnM+J>+dDx!= za9#Xh1a`=%d+&1X{iOc(lD@vyO?2f><%sSmiJDPQj0_mMO#6qtN1B;2mvn;+s2=mV zi-1iW-HDZqpZKBXwAp>w=c^{e({JLSkEjuZ=QaG+OWYfb44AB4&LoD9)4y0#Je3}qT=w-_6jEtO?|!n)S!=kuxJ&iP7ysF+UwFGRF&6=t0knWf!g&c} z)S%a!Ws?EBEHY%}jRle1tD7SPW@#SJeyCA=#u)e%qMGSbd+O$DzXSNoD@ZEtN!~e6YbaV zscTU-A`RpHuPpM(<%uK@m$RP>dS$YRcjQuYg1R)<60kWz6af(QJp?AVy{z9OvhMb& zg1~%nLAPEWK&*yjGE;du#!0MVO>AbYr-8gc;mEB5pM;JOLI2CeD31gNPmJt)LhAv! zU*Gs=+Y?G$Yy5`=G9cM1?3i2pRy;hJo_Lz6Od3oZz67jNz3uB$+XFEMogn8CJ_v*l z%^Z;$WjgF>ij`L$4;E39UP=(79ue>?=>BSSPsfY>cZmHrSvg60p}DS+56{_I+Rphg zZ*?fvjLkiE{R?fIuFFG&enq034O3$!tcCq$m5WC%7p17n-5X_!e5K|l7gs+JxlRT* zV>;J(Q&IJ0!wo+~%<16YU(D@tF%j*VGQ$lj_dVJ-Z{FbGy%9rKvz}M(LFW}6%js0k zXe)bcQvwd;KMM?ro|6$5v(G8Eg-7*}+Wq?jovWtLpI&=A021O9@}|X%zdt@u5%PNY zx`n1Vj5@qm`qmcW3=Vpt3Wm!OmoRHYXu~rpRO%g-I=AzjyYp!w`@Jb$fi5rp4;1D$ z48M*gPlCCMiQ-eWxuIQhUNH>nHhG2_rHjBKHQVL~S3%iJc2h_5$qXOQfOp$Z`@>2D z0|R?RU0q$T4^6bqBt1Ra<*naspUik?DRBl9Le~j;pm*64?z7Fea7TIRf=R2lxx+dm zuosYIK-yzeE98y_uCIl=bFHB}!(=sdlCBP`zZRJFMD-qBKZlq5`?}>RT6Oq=v+yfi z9j$seTNoKjws_>H1o^%RACABxgc4A!bIT=8MCK_`6_RNJVqh`id+nOTDR2tiu+$GU zH2o8iTP%8c-jI7YHJatiU;oSS#~7SP!AS1d(!KXnBZ91Rk<{?p#DV+f=cP@> z&~y6lvgagkRNtM;;^DogkTUMxj)Z{3L;>grrH?y{g_#`IOPZe7PI3HZ#3X0Yb&%T> z$HO^&dUfkJ@oYI!c}f|QRe4J5v1FuYcw!rlHwPPyJO0eFxo>_oU&xEu-Ou^5L2hyk^;=aFg|jF#IRA}c&>d4ckYvV)KidVvH5mYQm}uplA% zz4zd5h^0`<#<3P8J7+ zR&H@}HtV3-;+Qbk3c_9KW*wp9uqX5I?##p;;J zUT?>zLTcRxRpnZFd^+u%I@!c>#VOmF65cXGw!DA`4y6&}GwSPyW!l}@6c7{2N+AWl z7+H5-yD^IlbX$e>%QC_<-+-&DLxj$4F`KK?37&Wk*AkFnstBci5C=(!)WazP>I8KU z&&2+%`!r&@ydHe|x^vS`u=GG6I+u8Gs5Zmy+Ww`Tl3*tqaD13w*fSo#0 z>X8=Y`)#$OV7DkQLAOE-*gH4m;nBybD~RA#K|JooZ8l2ZtyY*p>!Jd7%cn-LkK)-0 zjL-Yx|NAwSj+_tgXh5KHiPc=0@L0B7cJ~ViufrBotX^|KZ-_Znw#Mk4ThTIjl-F_4 zcawOXh8&Mt86WR2#gwxo3wY(ep^`FD@*PYTPUmzpoX@@XJl!mAa=(;cYWaKW?s_XP zv}btg4x4aLEO{H2784sAyE{7NB94z(I+9LPwc~Rk=W9g#vsAzJy>0FErnT9Uf2mM^ zO`E6R`LA<-C&E`a*a1<4&oSTv2Zps%y0qUHBNyHsiY6@Y0SbHZ%8v% zHspKf=vCNO`v~1as+Xs2qrQ{6^^S@XQ=1NKyD@_2R_Uk)tcLlzwblCzH98<_stA6i zl6^$HR=x#Nv5fJLmprH-*ZV#4U&bHNE;Zy69<;D!uZZc=_sy04GeO@B+MJlP#l7xt!n4Ak-HGPE>=CN zuYo(E6q&9><7Fu^feV)_-C-v;3p7&p+s*=AL}9Unm3T3Sy7SU&2(T!$&8a0A= zc!YTyBMXS(oq=Tm;A2b`Uym(ZOA5cikB4@g(@QrwYCAmX?qvrYZma+MRw8Cx zw#zM1n{sL1Om`;KMK4A2if{aEJeQDG$}fS&yb*B zpw2+sDIpKlZN%CA-*4FovUp9phwjI+?5w5Y+49NukT@m40h!bPbxj!s`jUYo5(T}W z)Ky*l@AeNj##%4NWHCV8ouSlHEs?-5B;N{t`!iMy<}aGrzDyX?X;G)1?aqpTJiE;~ zx%2&K{&Kgsn1F~a5L4$D9xJ;ZpU-yFYqYL5V zys%nS27EEzsc^a>g&#c4d?GG?tZru*O5@-b|G~42Y;S(vvUZ|U{e*fq{v8;Fmn*oP zlf7C=5GT_&A-1LmcRbSUDJgA^+h@}GIy#lEstSW8mmm@zD11-DF9N@9pl<>wU!=3? zLrp-GbGB<`L&f)VtK437TiZA!s9_wXRBGGpC2aC^zV?tacD+Y^#6Qaoe3>^muZo?h zq{6x=)&;bFzdlEU62Oh13Vw=~-=Kz(yiN7wj8-V<>6Cbk zitf*q3AZ^oYYg+lBL4i5R+>VO&ZCw^3Qy5zQa!jEje6Ff8W(qTx}8Du&3&IOP<=3I zb1_57e-Zo!ge9qfux2#_`;my{^_BEBH+uNJQP|rVH>?3)$ea8KVvy*O zZk(c#+^40(_T$fg4}G%!tp1`TDLo(%F&5X1{h!SP{=cPZU;GF_9q8~D;Y}?JXGo>j3EBYuJ_)Wi!W-h<)MJ%&^RO{n|daKbZZW}A9mVvEGYa^^k8wd|Ckb3Dz5xfOPxh}0;x0{z@OJ1xviQR;$ z=K@v_*!P6@b9X7{&R=F&#;Mwl4;T6LVTl1u`iWcJTby0D1BnM!J{yaTm-hQtsUQ(v zCy1D~=1&-@3VgZ1_}3i`d{PUrE#HWTq~*TFbek%fe`7uW_$E{QEQUTBm|1yll_N-r zSzeNVpzjX18Z&}FXLnn`u7#fU`sSQiVDt37zU)pgBLvqpggyMy)!H+TWnlnDIDVru z5YIveSyJzGb6YcfNE$X9SUd0vIBs(ZsciaieoyLReNPB8Z(=ckJnEXIKE9a$qe4kM zEVBSt}eE0IBMp!4u)Uwev5p0zccQ!yc#_sNv2YYv|M9@D$Pm7d&5a-km zZ=5i-+~X1Kyf-Y!Ug+ztu6|qsO+~4q+Bg9iFU3CwU}yVx{%rri!WnYxBINj0)`$$2 zt#6ccthv_Mlv{pfv&It=d+yk^MJ0w_x)t-zMXS}YvC3jxR~#G0q*lmEvGbIfU z->?TSJH_;Akr_jSzw!S#CL-8zsquIXi`P%g1)Je@%TDX%9u7HE_nzI&?ZLfu|7AOE zVmE&?uA=Zt+g7#HMVR%sSE-!g)9vWNtp~TW-*vABVs~1JYQSTqwP(V*#4R*0Z+YE3 zG?jTQFx6?mWAzX+{GtESdb)Lot$Ry=&#V8c=wP|Z0|#fm)%?2UxEbqLlmPk>l_gR8 zg<|@EkL#MxvG|jvJEFDoZ<48osh1(cnk_Ytq%o<-0?owZ+lk0s99_3Vg5idz_WsJ9 zf~D`15BRmbawnJ7`wvg=ySA)_EO`d7zGV#Cc6O1DRXA}IzTa2R|6E-}w&x7CAb5!n zHov|Q@A?g-qV8Gj`NS@lxNbhrwWF(0>!v>a-R zcvPnBXYS_Ak|sMddXdKU)rSRB-wPGBj`N;QFCqD`u5oI}4^zCxg$#yxuOFp1XSr3| z|F7DD-|_!G%)k~-xnT0^+wmN%Sg=A(7~O~cw-=whQ&6{waCU~toYjQ>w{gE2K3{Fk zJ=y(37o>RQF5tgC{&9XcUxj8`6d)>@@J%4oWuG;4^%3JgDtDlat%j{9Pu4 zvy_%{riBiL3QO_@d_%y%m;QnSqoK?cgLVt}_cY#ypAo~agzR4@+V%1kMSaicUBBl_!|rvxWyy~8J)|M2gz!5*!2baA+9y2>61;K@TbpK{~}no&n(9 zeRt(rzUO^6>HNE4hbUVQKWM2TQc`j=LFs6%+}*nJm2Q^wd5&A4Zz515_IryMZPf2j zSbl(P`eqF(3WIQ2L(Y_z5cN{Me$XcvFF#PZg=lumLujN=m-{Phh<$5t=p@m6Tm2*z6Pa_8Hg#|CWUA29GW@fhy~T$1ie9lRn((y0&Y z5MDG}8=+Fs&78~#5D-GKv``elt`LJ6P29UPzIFZ@^OxrVyACQ2X)a=v*!n^%Xh7w4 zZ+eThoxy36;QQOLX?XO}wL^a}>FxULjrIRF^p(|(6wwpM4OEOo&L5!u)9lz7Dp{t) zOyTFExISK!!06>eX&-BJhHZ2P;JEcj^nUy?*aYCZms86D0s&6BDPy zB+E0d&tGRKFyxUFM-Xb3ifHrnETy#OMdU!JhVleoIPZQ7QmrT>lYw6!|4)_(iGRa{ z497X4iqG4+I6>^b5X3s2pGjW4I8tZT{Cr783k;AA{B^`cD z9)2*9BbGmR0`pQ>Kk`{Ve{H`xTe-z%(;+*KU9FYE?^@tnKU--6Wlq-at*x)n(v>ut zyMFq)Z(uffW3}@!-wwPgu@?sklKD_S8r3{S3z9VJS_{quU{U)i`*fJeWWaVDsAkVk zz-9yQlPB_ZrRJ*58FA14QqGa+$8dMy7 z?0!qSF|4v>JPXaIKkYkuxCfBPR>U2bjsKn9NeB@EKvm$&RF(C8rr0k$rBEcJmM5n+ z^Yt(0)U6iI%<`4%L;Buvyq-6lJz3r;ZGBK$BPrIF2-!#!bo!~){UadcG-Y|c_oYm% zd~ud>P)Ca|Ro8djI+tj8sS#RPOa6KuFIilZZ7R3?nVBpxogbzrtMALE~inp_WFnNO*{~Qvf?@)qLQyyoOEYR zj~w!vbJl(Nk9)J*_lOFUU)MK$c^NG(y+rZ6kd}q!Tj5ZLs!+ySl#A&Hq^_R&)7S}j z089V&L28ipKQmvSITkW$Yb#%x$LQyF{H8E|A)VpRWVoEb3-JMk|I~&p{`{BDt^<&i zNKq{Z@6tU9X5?Ro?{a@hAO)oX^18s*LkTqus5ED!uWfo1WyN&e}tBIoND6ZU4` zWFgftW#E6GoGKU>W<6x^_)GNx4Pzrir6m?Eyf?8jmRc;1^+g9m)eh$0{#z&X;AuUn zmoj;EC45)z4i6O`F2~9^JO&B1-)14?ELP3jq)1Ckb;SWBr);EaA3uc`1O&dT6GbBi zjR_~I;!VGt^lR@KoQ@H{APx0CP%{5y`MZ*L%>!gdl z%qFh0KVo2**y$P%7k1LFPtr}xnfb}3YIS&}%HYffoVHhTCT?V!-}1d2a~%4mnqBp? zO66_U*_U}eSBLu;GJ8gyumR>_br{*5Lds){nqZTM`I1C=Ys-6Bsm1bSsU36Z>>e!! zK2^d)7Ub*vXtF~p&0TVB&Rz+QLTdi%YtlrSbuIi!%ZxOvX~*aSj+j(03+987uzVp` zE8&v>X!UyUc&`0dAtY!Cg+s1L<^}z`A1fkMhE z-Ix)P{V+Gx2(G~xTt&pc!KpxUc}b}7={HtHl;eMYBd1IC3tJ6UH$Ew1w>1D^{VgHhHa?QBN`Tvi(RlAUJHWoN2yy)i?w zBqPOM4J1 zC_?s|JsE68A?K=vrk@B$N`o&Ooc?GCT6Jf3Kfv>jG^|yw0EB8jmlfvrEM^|Rz0PCV ziy{4^-|tR|2gN$6mkf1YD=;Afj zkHI(lSIekZ30_8?d{wpuh(Eor^Ho|yb-3|Agj|;|46{DU5lw$5e9d+tgq(%D^5<;E z47St^Yi3U1EG~>eS+>1hic^AQEB{sAS|VF3s`>k6jhdny z#19wwj^-+0mSf&DZHcAQo};%|0B^^uaWq>3Af~hb=%5p-D_kFLi+vB=+G{lGEq?sO zA(yxAd?ADfeM(T0cTXlVGV2hdRqTv(;fxHOj>j}V-3n9LTgEja?bKFDf95k)XJ-^Y zwZ&dNOfmDpWY=SI>D?y6X$WLYLwUhCRxyjEUkmqw{?F zVeXvTee;FfXAwqUk)BQtCAl`-IbXYPBMYp*E?Do@XBBbMTOU|Fc_ur5771RNe>f;K zQ*AbGJuPO1Tz&dKJjX@DD7O;~CS^Ip!^H{XXObVmPG(4#GuirS39%WP`LQs0G$_7J9+k)gM3aS5Q}-v}$h>UH1}g4_po*yIcyRMt(q& zOqF&1`2AXSKFL|ysDfJ$kE8dCY)>@-2cvqw~^zWza5*$BlZ(kTsqoi@thZ;ibh5=ySR(5!N3wEAY1(Wgx-=~r>@LG3ppo&V=s$0tGvm88xf@^=ayLk8;r8^ zFyB29R4KG!s0UL5wUJJxZ|zGiR7U}A044}Bxd<6uLLgxz_6_bCAmT_%1l5RUsPikl zZ+H<&To9k#^Z5%~3UohM0;hYBn!%(HLN8!lN{8$FOU`o%74SV1kNBgZa4ARdH>_fo zk+2`FW*!gWEi#%)g1n0ift=Dj-mK+4YDi3c<*Qpvw>J!UYVwf+akS3g3l9v5T{vdK zypC2PRZFJCmYQ4{7-HpO{{+32_OihZ^ga*u^pS5r*vIy@ZBz@4QOl{ z&*xsGwKb%1=(W5xm|de_4C!_oS}-XvA}fQ~PVR<|r5oAO>AWj7zhlK9<_b|$BodL{ zR}nXbJ<1YT7SB1V-_Er?=&BjKV66RgSz--GlPns)w6@z?UZ(tLS?XLipH7eMGp$;% zjeM_>>uCv65_3uqel4@Kz>e2iO?KA}m76pWIG1K#UHB$$laj{!qsI&-o+e&aAbCcq z$?Y-OuLmyO22sWA&3s=*L3V#{i>@`CAgFcV?!7_6Z5RVA2joz$Xv4^h!e}+PHDb{P zwV7!zRE%Z5?{EpDWHVh#xObyI7O}KGe82mJAm!$;Q?T=~?~}jrwxW#*ZgahdoFKm) zM9H5*Bd&@LJc;CTt;@Onp0PNk$Q-F`B144c_wfb zrD1oJkAqtO>>tt1c~k$8Er^1>p5pb?yo5SJZv-U`m`*VxSq@LxL~hwQ9{ROO4(Rjk zQr>>6-0U5zcI&A}M>&;@o1g=$A^KS>dvdBS1}RIJ7`?gL@@L~X13O{&?m1uoN zDM@y1aSGQa{I=>~7FCwR7Q~L;(83xW`O(ESo_#^>dt>B=jT=|!%On&h>; zP@Xw>4~N-~wx+$QD+&vyEtt7wQ6fMrMuA02%)|ATP_&<)RnaWFL?I*JBQka93NE5i z4_6h3$2xU9w^#icZ2lAdZtl#EyL7f2j52E8+n?52`?&5T1e zez@yhu-=LS-q(}a2Pj4TznSU5h!$p9A9}7_*5!4*Cd6y6<*;Vqq$yTux20nHnOpZlNs4X9TT$p}t> zAKzy}v2Ds%z!c1W`@8(j1vyP37fjWM4asvo z_hIIc?X@|b)9(6b1bpEf{ejn0&YvR7e+g61X;;^JC}oJ93?ZN4Ah$;#1eh5&Q|1}Z z)WQP$ibU1E80nQ`PkP3%)Rr1h8~4S@53-N_!IgDFE3op4Vr?N~>vR)-&MKc22c<4| z>O%H>iY~6jYkE_hrp4{6Cw}fFL#INmMi`iP;kk7P5tzcV7MZhoYCQk3K()8nH_~7} z=grit-gpDzzU75)8KB21%2D09?iOJtXm)(B#u7(-F!e!now(`#&bnetS%~X}q;|CE z=R~r6cFANY$SQP2YTCQcxAwF(84jTqEOnnJLttwm$p_)C70o*a3c}N|jjix`0zl16 zLvbxdL;JjsW&(9*SPiLlH1?hlU;A5x8nDEf^~X&qnLhVL>j@KMMNp$;q6s8dH_`B< zM=$XxFhF4IeZC_61f!kDeQbC48xUXAVAfrKitM;n9eFvMl2?($t1C#Sp6+9zKK>oj zK&7ed(ALjVb`VkRe{pi;92xq#LY9e5=l3 z18UB*x1mBVSCaRhV6UVC6#I+np(9pAz%Gch{qp)Y;5e}&n$^%Fwj27Sl*Q}rI~;2U zQw>z{%Fkb=87Kf)Gn;!kJXUIJCLneZheSISFLGEGYi-<*PVWM?LhG1#CS(ytcH(

6zJy4MLZY9QDJ7g)Q~xMa?7R81@k8Lt?!vZs-mn112cV`ftPH%g0;B~i`=R> zHQf5!&qyZSglP7?O4wS z+<@?>W|vomq(YCFq;vfGUmu5;?lc4$=O?gaHEc@`KN{*bG&Vo~CbohrvI25w3LyxS z)wtfw8WiCveUBlD1Ruy!`I`F=8`#LcGQ4a}Zu@H5{(NRH_`lzB+tO^^yu6PK1>J4b zZ;zL+?-j3qPf~H-u6O5G?qRmf-WL^#BZTO_(lypACRh_te8y8%HR+=O!yn?rf1JFB9h9V=XXBsYZXIrQfbIIraA|Ff*&# z36$3^TkyKSS4cCCNI#H3evU)JpHpCxDbV{)>5WVZ|Li$l2BPwIDB*&pL~2yz-yp6} zNPO@1G3Qu9$OIDlJ~*Cf|Mj%%tkT0Z2!7puzq^B;l7V?Pq zwoq3-5gI=4d{`g77!2W$JZzr6lAHaAnjQIRuP0XrOmyHM{YqBDjyyBxFXTh46xY%H zm(BGDf{ssof}Lf0Vme%qGeHN!;ydm(?~SN0Q!N`48aSIr)O zTg*MZri^5JkeFM*bWks&%az~va?44hp&rg&gw3FZE;bJ9& z!ep}HuNe?ZmiYR|IZKf4T@l3K_;4})7_|#Vn6+=cAWRP^qKrU6|Jv|$vJ&@s3V`mk zxbwO~t+-^qTE4g?*+ko$ve$h*kB*>I6x>#{v?skOyQ3OD7)~a@b?~@W>~__~AC}0f zg0eJ1DScurb;jp|mQ0onfE0lDF{|VZ)k=qVHxO}s$-rEnELQoe&@Lvc-Jq&e4UC=36Wp z`Tn}ss{h~1%x$Ww-y6+dNOnSwO5JLY{u+yDM$UxuA7V407md|hdgW_2X77rjsFjZL zJTJgm%YbaJq|mQn2(VHS7a8TYjAd!k_Uh+BcfC=RP5`Ao?G<7R)q)bZfuJHH*d1tNIqLa*P|mZ`Xh^{{Xg4K5aTo?G)x)!rrH5p zL=E=u2ik?=-pl7lMJkQZzTYJ8Eh>I9EeoH&(`m30h{pA|sE8p9*U*>qEGl+G?rIo} z&aFzIwR(Ej45vC>YH<{jr#V^TAX=kKQm$y$TfD5c+HBQUd$tC#axi8P^f=S-lAd7e zOruBJlv@^X02}*FZvxGa zloul2E)^Dku_li(i!|!>oP2#kYBbr!`+HVfQS(#?gcVD*r@81Gcecx;*qoQ_>)l~K zp4=H| zCo}5uf3kKMVDWs~)Ai{sb=Z4ACK?5rZD!gE`SPTmxI=)jn-=T8xUQ6Bi9$OeYP z$UK&o2*FTJQe+%`Q2TqN5a1c%%jDi-mY~sqFw$r^gOq5pzrT#K6|h|Y``{(9ib4ao z8JyQPt$95ERpIrdN_`Eh3lz%dK1pUw$3Uxk8qc8K zMx^+E2V&_qDzx&7Elm;&!vpS~T3LW>6p0`pr(Qa{&`97=<;rU^Ho~&B#(|g(xeZtL zk#NNH<6<)U`zzfyYTK1~4M-N(3m4%G9`L~w)G$c|SAqu!4?(Y(3fy#a9F+(KDuxMg z#1YvzZtrd?cNOW}Yh)`R-GY-C*t~Q06BzJ8DE#iRkR?+Ra!TXPRvG$`flKro3JvqSm1NF|0J_fZq9kcdoxKk3Uh+_@H>9_2iMPRW(@R& zpKT|M2KGDwmFVQa*@PO5MJF?ty+>~rSGq?}9QP;Go0n=Jr70)U+{sX3{G+yb_K#C6myteIr*Yn+K*%USf zARo`@jzg)o@0vb5+1O(<7}0RDdB3vY>wq=H8M787n4RWkC7d$>+pCp2e!)3;w;IW8 zB(gIHNduEpNrg{hDZYfzvrp+OoL| z=X%!I9-hmD15(PtT>b^Kl?LJIeASa$_m5(DieMLG2C0ZDK2n$oL;7`$`1IV+@!VlR zN;TdAT%oZXT?O;SngE?fPZ?d}j3#`(@3NXr#-YEN)z#&Xf%m-oYB)I}kTI33Hv?m9sw@u>)MT?f!47i$NP9X-Mm&y`f*nLMHWojG;iqGlrFbeCV>_ltwVKncv_dO<~FvAYKb zo@adUmcEaRoq8vDrp`P7L@3Eg-zh3~_&qJ`BDC&+4@#e{5q63<@_vOOAM9HJZW>Um zl`P3b@|MabaY)GFz$vu{?=)+s1HVg-kOf)aB7a50hS@IGNMMujzs>HOIDe4++b7gF z>p(E+*d6wm?;m;z=UD(|v?`2eZRo^c=4}F6C1WDp_j+<7zj7M&$|da2pu+X|1gIt?k)=-u|i7A^?(~MEs$!B zs$6dtkkT4?wO7(qVNB*7D=eM*n_(F>33$%xbuItVU2D0Q$TvUUp0&(_16B$fi3cs# zReygBbKRBySafyRy7aps9eq&CDf*N=bh}+XK5+{eiepAa0xn0ACz~uRJPxr??b<_L zYtga)*VS1^RTXt>Uyza%1nEXVB&18aMHJ*5x|9^@lr8~LI;Fb~-Q7s1aOjejly3Oe z(f7Ugj&J|LV2smyuf5iqb3V^+a?CFL5%hdi_Wnn>ow1klqRa7=0vJTiXab>2HP)JZ ztp{k`loVd9@3fK@ZFhc@VT=iVd^9NVc%^;zu=X?hEma=_+DfDU(Q_WF%Zh{{fj841 znT8Ip4H~e|wV6|=shD;>!rG<}X6NnENREsL8lEIY?N)7kv_N;y72ckpGQ2&w5#WDE zPImO|D0Fl+l+CbV**&Z!==VWs4Yk2d44b~bUbFiXpnMKLj>mK<;g~J&-km*Ta9${1 z*pPDzb~JVgt*T+C*o^{tW@;ZOUo{(3UBg}Gj$b*lsGqU<&%!A42GahoH{<7)OCpe? zs+wdQ&&4wP7VQ?O>6%=o&RF#ghZ9H29S}Z67mSGS3KMha!VP!1V@Kg9*z2R;(M?+@ zOWC>&F1L6S3mmQY&$c)DcjHdh78gTBYWc+L_#}L5Bk5cSL?+}`dmKD>xcR#JWF zf(y?dbOQzo?v5a%{s|ERHZ4LR*>e(68&|;CeU@>FxkTC_gSnyVJa8{n=86umaUy{d zO+I1#DT7J|%2rc@GqpUBQNkbK`=d67@sR@`PC6zFn1$ZeJoLfNopIDIm^3S3P>8gK z&{lw=G(uHB7F3-rq(HWhkPw-VTRr8sno~=)?Mz-Pi{@2C`!IVcNA#VX&po4$ju${# zRH23du2GiIb^ao@t}y@JXByV7ck$mI_c9@>-ltM_=EC>IooOzG zT=tvs!)d?M_aj!=Ghqr=TAtT97GaEo!h)=Y_c047g#cfVF({f%^%b-t1fM>^liw!o zzE7m_Ru8Z&JW(HFHD3V5&q7Rlv1nNKdRwG^O&rp$-(Yf$4j?UO!OA;B9x1j5;AAd3 zV*6hRoy_!9Yz)J16Zw>0PS|6Qn0#_dIgrk{twNbY+)<_hbq^H=XqwT9j_yx6gm{`)dsO4wd$oo};Cj+y0V{ z@dK22A!gGhf%^h`;~z&}Ba*kg!G<`Yg0;2XWyIaDoM)unu1=GWcllpZhIRPkYrPZR@Ru4&w6wwb_a9mixO_oX>d~1 zs@%$17fidh8~VdhDCJJD^fFU`?G);U!dUF6&kiE~+mFN)jOKL%0Yv34~d{{VJd_ z8{|UsPCkp{4rx~_JI#P%wNUr+9NPEPr>YwTjy}Un`|mvDfaqiptFKFd|5C;vCPW1n z>{#oVn?AHu&dCm)-dW9~D*$0`jPxRd@+|Vz!_zAu2$8?@G0ZaF-XmagV%nZnqe8Cz%>RP7m$@=!(&2ip$t&&(C$Xx@>nRw(*UzuO%*gA@=Oxg_qSuRv? zy(>i6Tu>#pjVrU!RQs`GXUld!M+>D~X{dTh(o;9*qVib>ywj@&=RwK`WRv33gx5bp ziB#W9hZP5212y|$MZbD+;_FSXyd?}wJv74V(dPH@KYxrrPvM+v)YRQy!g}4xY*3^x ztX}0+{_D);ZLx)|V@7+j07|<#go6Y9v>3XdIVhVnH)1<df$d@z38#iifksVG!wZLYP>yU^e0IQAZx9@4SNxQ1YpN^=i% zOKq|-Evy#Hr5Tk-=pJ?9L|P`s^#Ie&EB|C}*O!wIcO=u#uNv#d7rV|3E=r3%$;b2u z_vbG9SApQ^_nR*t$O|E<5;W#_RH+v-vkGG-tbd?pZCtn%JZhF$Jn5w$I*|8URGQ3P zKwsvsDL?s*=+ZoX8Y|}Hxg_#$@H+xJD}rk$bs+Da^^ z?i#n660?2Jt0r!4(VgwOq};Fxq$U;>t^vqdj)~bX?^+W47IHlLoIHdtJ>J#hQhJ>% z;F=AiF9y0uAQAY>4KQlf5COmp`G5s=bVxns_~+b$p<82?Dj?GGZnFOk5sKg$Uh))~MO(AW4H>J3;VeLF1?`BQ%d-!GNXnS~p-vc^Z{(s}Tm&@5)}p z8$ttk6a4F_8naSxB%N1*UU^-<$m=BEHp=W%OnyZUmsXfEVms<1Q z=&ao3*i1?!g{Pz-4{!s(IC;Th(Q9mrv(3i&EW-4)AB=2}b1KfXfAC6X2Jnu%O2i`> zqbr{Fh1Hr!#Jn>?rhEPo%SQ&~txB=&MnGA?Hfc+=;Ycj+sbl|Ir~AtHXkZI-0a>|} zpMrPA94*&Q*I3=q=&I!(-wgTvBER@DB}u8XCg%L-hAdFG&WjS+%e%})vNqEHRS~N! z(ArCi8VQ^+TJw&wK27?ILt9pd8BYq5w$%ORyxE>oxwijGaTwQJ4@4Vzd(|0C`bK`~;@^Bjk>Jv013l(u6Ir291196X zte<64+>XD>9j1k#wH-gc;jebODMVe)waJ86n1(qy6==-*NfKjr)@sicexNK6r=B1x z=Xs9DTzmVb#JW+h%)Cjf8Ym0%hsnYzXEHQBc&%egb!|sKTYf$@3=cs&C$Ie@Inp2T z=U0b|DT6yE!a+s!-LNY-eQ!xyl4DDQk?=Oew;HTuPgqR@j`L5#uarM^`^O)+wbtiB z%Ce$aG^gItjxOwDjlFZ#v7t$`vOsUG{ykb$lyK&9q@HKowO{X$pJ6T-F7o~Xk~!qQ zR60SbW2e~ct^FNihI9}Es0yj}I~>KXtl#_6!@YD~?f%2nkz(uUqg;WZ>}QN$Yu&QE z@H&L^XKN~~l{Qombmc4;GWqo>0L$Ho^Vd|p@wU}XD4vnYFAdA2jz7WnWX;qYR7rlw z?Wa?iwQXMvpICifXxgX=7E<&SF+E3L$2*D#SsS(8!FM9GrGAD2$a=#mb&FjkyDs;P z6TcF@giXd8i(2SkQzpGp*#lKm-16T`eKF))(e7Hx5k9)_25uA^_LqXcMInbBM{^LsrG zb_~2Bmtc3}KQFt}j&&J*^>TtG4k*M1+Y7DHItHs4!GS4V?zY~YiSg0`N&jK=f_b`lnPUBFsIdH z{ZU%yC6aOxZv7ES90>zTxTp&Uu?gJD%uIBEu2921a#O|{(RIma@Sz^iQE ztrRN%PPM$N^LnH~Tx-_3*on%5@4{!~O`GSJ!x+wKjHML5)F|M1_&i3&VSPu&(Y?(1 zK591}{UfW5;v1Gyg9j4Svny;?zBGSGN#5qZ-rJrs-=Eu6zXgL3+KzHCE{M{G5Bc@u zjAVn`D1x>CE8ent4O7l$NIs@7$=2+t)!fu&3?8IL)YhTmczbMG-Gy-Z2ad+{eYm;` zqhyt3QVI4{u5v#L5_hJv5i9=${W`az$>IiQbWnwHf1VWsH89qOIkKf08yekZScD!q z0!GM*2K5w2X|Oj<9NRco`Mh)RHX2Ho#bU*-00A@)m>rAyl9(r_+FjdjgFMi{d>Yyp zcJ8qvG6$(DyzvX324=iG@B9iAGmGhll!K3gXh%LP!I`_(I(M}M;U{Zxi=q1ONWJ#v z)QyARdwp1_ES_Ja!1rh(31<*c#>HkHL91%mc*(e{v&9dPwHph zApvAbdNv124=w7hd`hOdhz=ENqXOho$Td$brwLinG1WgJfyS;9TxLA~X zB%cpH#tbnF>lU*?r|{2&f8iAKX3lvcHW#OiPT?xNdRN? z$+Y~uZ^s~UT?-qz&0lISvPx!wDF?Tc3Y{t>v$vQ(!XUhSQ%F(+ylXAnAJ|sTf-26nYaqXsO z`=ez8yG2qvxXK&?DIsRlV_PK7qlvE|LZk{+_?z)bPOwm>t{(;eIHc3?&5<9eeh<`M zkSJy(AA9;4>`v|(C7xK+4N@L|UvG*TXaG7Mb$)fi;=l54wc8#yazriCMzMEnh;0Yn zhSpr33|eZ_h2+(yw>|&RVC=P9h(W7Rn`?*Ymz%+q$`6D5L{y^g7F8_*EMjz1Ul;>z zRN>|J;PUZtrC8q29-4Y)eNjcRVxM%;wP!~*DTbJ2F3xIJFk{O;9KB{K4cS@*x^jD$ zxhEMVpg6S3BoIUvLg0nDM~LDprEM)xw-U;?R9vx3sZNNZY>bZZfMc#y<5eDfL5^fI zQcwHlp-mDNjOv^o9@OHua5hpvh55_O+x=Gt6!vhJIcog+%)^?A@Kzb4vU}9|7u~)p z8l|p80P@VS1Q7x2qiU=fON)^_zb%q?Zv?ut z=8gv8u**jL_+l*NevZeBOFW;K;9n?-@@)RsUiID2mIo`cSokjNbDf-_+FuyDkJRvTXohj4yB_9rh>)Y<#+RSvwwIo-)%~eJAYJRL$|z1JGjxE zjqA@ox`M2%fL-6Hsy6qX$KvdRqt(s*mAw*?l2@n;t+>k<(8JXr=*mvq(0+`FusSSY z0iU&lD}&#bLJ_&V3AufAzfEi#XO?33Ou&yO%h(5>l8xqwh%~d7)W<@BC(+xSKa?xL zqOM`L<=EQy%L{54&E*N>)YaKgSYr5dBy-&JCe6^a0WomfwA)fkq)l^o_pD)E8Evos zV37~n|53M+o9b4;hk2je{PTF>{L8BB5ZI5KE2jwF0+Zg~QM2$&r@@!5FYlz#n1^Cy z`KbE5fvWn^IDJ8hSZ$nGYA>`qEt+xS7^u_qLQSiLt5Ny%@P@$U+u{Dxw&(aq3*j7FLTOUX5A`@;n zu4VSuY@_vK6)XNGacO;OV6@HPuNU=@+oHu}WwsG0Fxl3ZrEMM6H9(OX;lzG@8p>7d zi%Dp6c|u);iDH(D0kv1g&~09J7V#If@2|ZdMuk3)|I7p%$_9lVl#Y?y0!;G!klqhU z&EvEXR$)Sh(M0N*c1b86#JFpI#_g<#{5?OW(mDCLcJ=c&)zFv_76sWDeQoAEVa$3| zKDi?UaasaP5rph1ltQPZoY9suz?>gUiau>BewSUY_PLfXErvHoGxNXELKG`DdZSG3 zR@N!2Jm_WrTr_7Xv_$M5u0n_{)l23fSDYl!(WHZI#H|$66XADi>;DCr-M!Xk>-(3Z z#HK<9qx=|l?Kf}|aU!i~kO*ol8rnWxI``cV0+mN+gQv9!QbaxH2aF!U4t!`pgNHiM z-<<^JSh=2UF?ZuyLy+h!!t6KZ_IrL%GAE`py`N*19iHB#Bu&1bPWe7_I)0mfhoeUS zmkCEK*7T?I*AN*wacpqIEq8@v53QKx&XzZ=bFeuTe&6b%KgHRhPbD`edpG}%y7Sg# z7**Gwa+kiIC5l9ONSu61Ht{$Q$u}{R`9AtF|4ypit79`2D6h$N$_`tu?afMGu9m*1 zzk!%J>93mEN(D+%&caD198Ut3KM2!iR)(0@FVA83RZ;p+B;S2VvyN6)Yn}Js75J9@ zD$Eym1*(Fr*Oq;RdyaF5PtwwaKMoN%%27A*sFkR_7IHSv-{T6SjQFmQhuUwZ;b^K< z+YGc`;P~)Wj($l%4h3;>XpwG>Op`31AhL@@%xD&>mF$erQT9Fkd0{4!P`!iFRi9>nI0O zte19Wz8Tujwb`hAjU7S@$MA#2khRZAgiCt1}`slls#traR!O&7|2 zZkb_EuYJBdrR}MmYVL7?TI@l_d#kX!p(|8{2p{uS9^glCgL(ze+x4e^NO6qOCn$|t zDGifoB7%57Z3;M=E_hR6T;Y)v?HNquLp75d{a^?iMX%~QtYJsY*`gRT8KatR7PrYQ zBi$j>CXDJlTqMYtO3XGE&XuWN|E_}x#-gZMVWQk093_ODgi(JpWBQq8i7fc{d{GxU zn>nGq$@GmJEZ0O4BLS#|TPK}=A*TcPm-&kYxnI-X!)ctKwDT~bKwMfC4C47edQOBW zqa>H0gq8oRHJrxtg{lv0P#=r*tcdI{Qh%dL)DOqF*5Y`n?;0D&QSV`9$NF0uby!T* z>JsUUeiTBC9JB%M^?Bw^Ln_Y3cPqHO96-BG_%!KF)m5!$hf_S4V>8_Uek}UclUSnU z>nRhR3};Pu>9R;9cohu)6Dc08hWR-a{it)nrG;lKx}$Pke%_jZRhLm9_AY2=h)aTE!=03 zjpiqeRY_=a8&-Eo+%`I0oTMA7{R1!Mx|g#Q9_8Ev{Fss_89efvjCNq$_+v7@mO5!U-Tx%53+&)zbm3OO%Re5IlGos3nkHdk&*a1A*z zzo=mwj;k%cxOKBHnMl1~ZQN83g@^q3o#WDt^L8K%sU;A|+6x>5$p83w%=<6uhnTN5 z^(V!)K(Cde@k@(sR0-hoH^`F|Q!S$r^a*PcMPaQhHDCwkcT;u{RyDr=p{MjgbEra- zQa2UKxz0uOc0c7gxl}-?dM|F3xmxu4;MSp5(&jU76keS;w9i1=)&`yBUljwZH`Jd6 z;#@k&dc;9!7Kpo-zem8KS+9mD!B7gH#4rvDDcnlNK={aKXg`dX1_*sXm+3Jgw_7 zk6*b9@&hbMOM5I6VVokc#|;fu%GoHyI?Os5>c7dVwDL|F`k`iN*{QxJ|?{W z2;nG+&DRaodt2e8cvAD+nqF4v)UoXgOrFxq9LQ_*N?&}Z>i=x%_pD`4^V+bynTYT{pP&YRK=SY zvop_k9rw=SYWOBmykoC6lX{%HB#8M>8j~*MNv8`oNcwTxB((|N>n728MZSUM;BG>J zAS-2r#&@}(cE5-G2DteKXI`Z{F4s{?9;2j$xYoS^R0#r#Z(&rz`LU0@Is}~MxMEuV zq&xxRlgs!O6DroK3S3b!Ha|18qpR%PC!fNHCj80VoYg{Q%2~b4#r?I11L|c2yyHr9 z6Kv#{2{qo5@z~JH(OBl0OTHl`P-jq!|JdXDL&>Pr!VU#~xYGSClf8KaX(;^NO4IPS zA%o;4p3fgfQ82+Q>3@TB#W3+~qC|y}dnoR?nRt!3K5_9u5nXuL5TBv|#VW!6hFNi1H+d@~xC6GJopmZKd$wNNqpZ6)~`n z7ejl^1w(#EY(8WRj+@cfMO6}mU)ptpw5rkYg47mopZn9d8!-&MEmc?IHi@9dYvojF zF|UMo5e5dYZFo&>gq#ls>ON~ccODB;-phv?%Es^EUrB#vxQOsJ0w&pXy=7San83#_ zGg86pNt#*Q2fhYakBi?n^}h2|DJ^B4qj{9(G23NcACZ~+$&{GREJ!bfEsoT=T}5MS zwjs(KW2St`d*9+`Z8btrazj~2{VEJLZbm42Rx^DZJ@m_MrrIiNrdApW-G`iz#wPGl zNDhZK7nB9cnx8Y@w2_uzEiTclpZXd$s6bf#r$*)EG3uTS79-5InziN`EEx3qe7(yL zOtpB&XfqLmz8N}LF{MA+@YGds z*Py<(Qb+B6SGwgBaX2?b!amNc`ex%=NvF^ZBwQJYT(ORXYg zj0?VoQLl^Dm4-Mhxer z_=be%>#J4b9Wi$3q)zulQWNMxvlNmg;XL(*drLY~lg&G4K?IcSm06L0Sv0w&k)uv9xOdjBsk#%61*;r8N@kVQ%GE?65zysv^tdDjes98b)3)&ZAr+zBI=79F>Jv`p&;im zFM)N;IyGbDsd>VAsCdJdyWT0`(qU?i3_aJV?%q+Sq6jx4k4zP7H;vHZLBj_&6(%L1jJijPo(I!i4)leE zAT&;g7GlnO&lFTwV=r-L3sQ==;YEi(tmT(oiFSuFDb*cX%^YZq9c2ANMJ*{ z?PTsp^(Q0p616f$6V()J(&hfOT=4514HKz8L~61ln0Q0ragn z2)EekT6opxh1^&1F3v}tWmr+}zeyI&)oXFTc=enVU5ST4wdnw;*$Gk^-V zYvFWAV$Irl&{*Wvti_g_)2=8ph@`s&|c zVZkOH@0c{4PY^7(yKq1-8+OEOAQOf2g<>)i&{~^!*)m6$c3a0TRvdI0!=L{$Pr%Jk z5t8VXcW(uVDf7H>87>)pIzz}6Q8D)Cz0$fgxsXFnwxKaby##>CHOn%Np-e{l6(JBv zVcVT5)BlT^Nc-`PSh)lUpwkx}S%F+3^7*?Lf}b{PuaAbkTnl+2lMA4j4*EN5yv=z9 z>%Pkd{s=X~DFsdU zxD+Djf{ccu^}>jPh@J(I$r|jh4+^^$lESp8L5TFeJ2c;QZ2;(7F|TRzK0JtzQ7_hu zGaJp2?<4Fz*5HQZ&I3OLLh1louPd|x)6X`5z4tD9-4*1PbLce+uqc08>%$RrzKx{% zIG9(gTRU<_z%A<_GT}uh6P%ZFcd?;gwNs3c_f^Ha91d5oC~{ce9xhnE<@+$9us2s% z;#-G-Ld>oPziI;Z@}9ffv$)&G)Cfiyh^rKQ+GC1lY6{;L<(IIW0A+XV{$Y%&rKu>8 zB;jq zzB`J24y8}wm`IFTWK!g=^ zC^(}LOf6QtdUdN!G$PrRkqs;my$B_eWkPh$@8!)h6O6}n263EHTJUBgpyFxt3znUY z4*lffqHypYb$ro*&_>eH(ukXGg!Cas8fkwFpXEygb;WhJ4_FJbw9h1d-sb8gSt1AjXeQ^^KGUR7JND!2~eAY{oj@yhQVgs zV+Vdi$rCiXzZ0WKC01>|hcnXa-5uLo!<*N=?nEL0twZ6LqzjxYnnkCduF-Q z6x7g)Mg7p%iQ1<0uY&)?_mWCS^0@*_2#TCP_c_H6$fGHR> z@sUK`3Z=YKHHMH4CDo<*nhDyQlwq#^+NTvJ7>;3yjYPm0&lKZWU|mHTb#2D*O$$wK z*eiv~U9&`$50FXOQf@~WA)qYLXye7F>(=9h&%5Em3@nppVon;0-`$p_04)Wp;$x*) z6?)hH*pP1!Mi#DHV9a&l}Q6*w<&{Rec!Dv$MLu98=MQnt?i9tK$*KQH2G@$%G>+6v)3M?9bcd} z5Dr|H`FuPFb(Gq1FPA$Xz+FWh&OhIN&Kr0GDUpL-cc4%711b6Vu{J-O+AON2vsU{h z373WMC6&cd2fC+vYMbHW9^AZnokJrXIR}lk188L6T%kaX!K%$A!`8t0rf0_yg_4`r zAT%_r)E%rJzkw2%tJ3qIOx_0-3In%1%-3hp{gHmyNd%qCTev90OvLqmORg*H%-a_k zLSK8MpB9n5ekhjmVTQr(*D}%mfH2CKGAhZ~bTxGlwxWZ)-#v|#7wZd+NF|&@$wQXm zwm~!#C)+j9Hw&@BTQjIwo~D=0@~(v*80Fc+ke-Ii(gXRFEn%n(nGZh-raEhKj!^3X zmeCz;|A@EwpG_n(p!n?B5;2koZcXhX{>tG88XFi?bgh$l?{EsXGXPdj0kLa${A5Hj zgT|0<7-$m7^MOu;f#r{OnWVxAS@HwYq12YbYSrwoPvGTRFM+rXG`wtb4cUQ;-}`hr zmrWq*F7SIgvnc`n{>*mgVwL}HozwhdUNJ=02m`-;+gaL}%ZV>UD&}VG*@T-Z&)!^> zKD40=@Q2Gwoj0U4@G!b&Oy;#D(_Tmg@UW;7^~*_1pZ*d^WZS?4O?fy)y7oaifRrWL zWt4Tl59|-YtkO!;kKe`$S*w~HqVqKZi88fh_*|Xan-Hn+olda20aTqreEXaxwrH4M zB@hV&^~v9-p9*@#i^9i-knurfWMlqhDYzq>V3=oY)e0xi(lGmbQ*rG-Y^_|4+nVWd zF9Zs%K5-t{(?IMG&9}_JG6DMF=uH%-u=Oe(t*dJ|SN9V2Mlz}mV_Z)+n^rcAb!`1B)=%6nla)6SMVGhZZEn6c3iQ&~G2!lcgac z&FhVd2Z0ml7X*0-Cnz^9(=D4jMGl&ZdsryVx0&3q&c`<@%1Fo-IXqb}X~Pec!2kfk zgL>~27WN1~04V>j{X!)SxgNjMwQm5-4Yg z;&d0ScMwbI<{vZ(T0fPw*`8AI;kll#igA3|(Ans7cKWF=79vLFq{@37y09;Sor zah7+ayC{3p_C3OWW99YL_~CYFnqdH}`4$=-V&1nsF%SXRVZHT>HL#Qf&f(j495kEy z7$fN-az+m9ATu-LxS2hp#sf2%wIeigLcQt$!x<8DsbZLez&{uPwbrFBcj3pvN07wo znnfDr1W>JX?ZNhutu>ipsz0M(op;49cf}?OiOloH#AdJ2w^M7C{ibtP0*YZL@>gQa*YJ>A%tQwdvAEUTi%wAT+2->?s zd5I~(b$8kt7d(l-zG$xM(~?4?_Vvr7`sPhkPLHp{H51Q6BOm01ffULl%s2Hb zS61!YhLUP=q<&&{9RU#|9C+`%&kejq!kCWkjcHQjHEw;ZL!11=+n&h zea;?UJHYF3np=#5Bv}d!&r+n!bqr35LtwL@4TV|>Iu2>>%+%#vDaAi~Xm0*H2N*Yy z69@u4u1=GK_+yUmS-!1E{!7+_Ger;@V)($-*)H8jwZX_SrkVn|-GrFqXcIH_@si+= z^;Wtw2!v*m=vSh_k{ncvS@n_~^ubgSZgjz*FFkj`F+e^~tZ@9l0N;pU6M zF68ILpu$TQAJUW2V0Tgv!wfNBhLDB7LwJRzzrw-6(MmZ}#m=XkkkklSXb$BIF~qiuhwZ z-oYGD z+U1tSyP!sP#F%Q!85Ax;_R^SWlj^?M$|gI4mwGx@y+*?&D=2@DlQ*-FyU47s86MA_ z9R-+?km(#?RjIkez8$K4zzt<&aHV~dGBB4Z=&I9QuMe=FuTsl;u7pRUC+A%Hp%wRr zNVc&>+`uI5p?CS>=Jp2tWQQ7TkS*q6=@#>i8q4vR2+ALzE(kn4?@tkl>MO5^t1=Nk zv4GTh04&UwJSsF}S3)>)(}P^12(Zbusg=)?~xY&sV6>0P`>gI#WB~O+#$*D zLh{ta05Vzqd8@Zbi%+@4d1tN!_SuUaV>yN$3PHU^pz)Ow93w&9sen}yd&L7~C;Z3x zpHah`r!5lV#L&tRZ~KU4IDF9zxP6U!bTkkwS2!#dI;N!-QLNnj8Nouwr4nKXTx8JR z)GhDgnNDHC%$pI~hp?+|c-!miWIYkDw@1OPxd-u~$(OJJ`D2RE*XARhPfjUY>!}4Z zYB0U&o@)UMWz#hrGQ21|rgv(PDr>?N+h5(J3c^L76<(CA#m8Vfi~PFs-+8IUB0IF-l7HROm*lsh}Y@+)eGZD zMl<_CGdyFkl1K7eza}buVY^&;8@;&J7Y|(@rp%q7MIKX(D^itM0*2F`l;mgGG8LCB zOKu30bt$$+w<;q}OmmZP{;OF8JUzJqH!t|-nqsa(Kl z{_}RCnE(oy$MQ1|_gYZcGJWak%jg~$Jq6@wvEnQo3iWbOa z)_1!Dc;gQgrB~Y@&D$o8dX{+o*E#1IaC%+uVD$vV@ljHNVcTV3pO`^J`72E|9)Rle zhfL~ENV0_TB}{oYH*?emzryCbhjEZjG%ModffzLV)$jKRh`r$9YOm@E1-&Z|xFP?u zFNkPKvrhz0k)F>%70`xA=^>gc+wGhmF!Vige(U~>9|ce*-pwAf_F4%TC08ZwMO`}6 z8NzSb>>-q|*9=yzI;J-R8}4&S#~e=P|LZ8U(kyoVk3m$o3{F3-@kd@s(`F|fHX#VPLG)P7?epEQEIL`g|GZ23~ zNcCmJxuh<6U|Xj!IQbfYhW_ms@b3v8R^U+r^GO&30yG;_z_$gGZ~r+~LlIpD&L319 zE4RAyBJ~1Qt2}V-L6rZENW|}(I|5Moa+6&XglCxQ(Q{hSX16~oE&q19i0@qfw#t>8 zD0&WP)jPwe<-$%{N*Mn$1QD;lUI$$%?0^R=U$@|W8VbD+>Hk_ZjnwNfaaNbJ3+xQ` zwGbH~D)Q$8|A7@F-daCLNuLIxEC7B2bgkzEfVTeMZ!WEdm=deDs}Y&xXw;*w?J^&~ z|Kz_AFFkZZuvOK|(my{~c>;2*F(jNBznziM3oH^5E|h;UJK~9#^&up@nt*`9o+)>V zb8NuieuC5s-dFzDduX<5oaf)(0DsTq=wt!NwsAQ*h&_e;5f)*z@i>82rvfVLR!6F+ zy#zccRmkPLbSQ;F%fH11@l}0(7Yv67gS=g#2A}R%;6y^lrHVR?A?pRQ`jfwK^uIrd z@cUFimICO7V+3SXKY=6!Ze|+vv&y)wJ?8L${^|+=I{tfLzS=RK?HbZW&+6vzgT@+F z7m^xphNna_I%3|i|GVCQfx{d5_ACoT>^d3%^!;A^XDBg;Uf=h{oZjgCBlsWC(Iv|+ z`{x$#|GdD)Z?yrSqk0cxw$~<#8W7fUE~}k;H+vgDyt5u{4CzNkWai}IK+@eGAm*BX z2lJ+|{b|My0A4ul&2R*UU8i@+1d~W0+6Rd76+R7m!~|Ip^@{%i9&z7gbmGi1Kesboq=00|L{sxedc z2mM!I{ndM8sgE(1>0U24+YDM4E z19yZ$zGd8cYRJLnQ)ah7wfk?#U&cdui9rO5-Ab1w5lJ<6_%c3UuwWWF$Q$JAmn1|# z)rENVH6y_KfD5g2+D;{C$3QAno7)*osvA~TqSTOb2rR|fffOId>LXT%7q>X6g0M%qFVx64$}?1!(@fPwR5~#Unt1LriCgvTHi% zNYeXeX{Jj1aMLRcxFrYxTp6?m0su);pGWHGTt?;-1=>)rzOoNv#I@K0NWKomxHAIG zqSf-KP7G=To9r7nZ5^P=O~IONwta3aJGa0#ui3_g^Nn(28qaN6EI{A&UjSnt_{<18 znP@h0VuU5@m}C**?CVX)1Qg6-03iICsVyvGqz{q-tE;J|;0Fr4(-UZr1KPR$wzInX zpki;&2&zDbOKtRk;OScofTVyk39O29m0q#J@+) zw@QovP=nF-M(ivC*zw)aM+cD{w#muy+XK~)Vp-#lSe@r;0X(UO%5ETeg#knh$|<7u z5o=T|(O~1(wq~~nFf^~PZDsKUWBHUYp8_ucRr!8zRBpM=sCgC606%5$E6?qd$81^< z1a9Lt+R)7AYQbZY|N3W+LjI6fF$KtMiMpSs1@3$!coLZfoTGSx0hX7mz1$+J-}Twm zbQB(n6>D4GED7vw@sqJYCyyQNHk^sLjN~u);SFKM@gns1S^V6wu;W@MbuPThpBwat(KXq;*T)HpgzA)mGcBH+Ew2-JBBGydH|ks; z|K`>#wTWqTz~8|z6iNY2X`oLq1S$Z~x^6$i{cD27THoywR{_TK+m@0=77o4lf{Tb0 z|G3>WJI4^Jq3nrcT}xeJ;uox!{fqY(!VA^75oM$C;ES1Rzo7+iFz&m>dANmWnhuQRf9F2|Int96KWF^Q=3p zoh)PP)0CIC%>fH?xy8jkm<#Q5odxH`PC4ZYL1kNJWysn^+zIto%eV3xxb zdJ`%s*eF}{*j&uUc&M=KZC={Z&HZhh?w9JNsa2rX2nM;`Xz8y<@RJ~w$Gm;nvcAEb z#VUSe?ETSuonkiEHXCGZ^A?JJ4aN&;h@J&_M(lqs2;KjDx;aqhr!OK=p_Oygnte!Wibdderm`nw8YV9b3Dj~8uSN=pb zS#M-{`4Q4b=UV%*G(hhuf1mB9xyKY{4IT6n@MV1QH7*apXSd-Memoy^zyu$--|k%R z`)wIyj{u$)5se~BQ-MkLxz7_AE^01g?DGgJ0Ae4kR*1epoy;3!B&T~?YsVUZ<3b5H z`X>C)fC}K8!35;IYkRgpRC#89FsCPCL>(#p$6Yqb|E?&qyF%U$!F3r^hRP)TBh!QK}u3codV*b??`OGZR6hsHFKcX)&&f#U7 z_kyYp+>nACKp!E+tJm!{Zo_Threy)EQzXM8B2ENnqo+M@GOb9PICzhuV?LSR=yqqJ zto?(kY%TM%ODRWy=G{ouhLJH0fd+0$#yhod%cbANbA8AIlapylI+s@)CODCojz>Bs zw^{uf$}!VAZT@`7!b3}qfk&yrL#fld4&(>`1tQl9P7~9VO174&=fTH%8`~IE zo3p+9n)P!hKbWqS8@9~a2_X5_BC@(>gS=au7<`+bG7`P)O#?FaH!~8Ehu>oCt9LHpMNBqFqnV)D=RF0P*V4t3 H2EP9fC-k>^ literal 0 HcmV?d00001 diff --git a/samples/shared-heap/src/main.c b/samples/shared-heap/src/main.c index f4024f08c..111da3399 100644 --- a/samples/shared-heap/src/main.c +++ b/samples/shared-heap/src/main.c @@ -55,7 +55,7 @@ thread1_callback(void *arg) i + 1); printf("wasm app1 send buf: %s\n\n", buf); - if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + if (!bh_post_msg(queue, 1, buf, 1024 * (i + 1))) { printf("Failed to post message to queue\n"); wasm_runtime_shared_heap_free(module_inst, offset); break; @@ -84,7 +84,7 @@ thread1_callback(void *arg) buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); printf("wasm app1 send buf: %s\n\n", buf); - if (!bh_post_msg(queue, 1, buf, 1024 * i)) { + if (!bh_post_msg(queue, 1, buf, 1024 * (i + 1))) { printf("Failed to post message to queue\n"); wasm_runtime_shared_heap_free(module_inst, argv[0]); break; @@ -251,7 +251,7 @@ main(int argc, char **argv) heap_init_args.size = 65536; shared_heap = wasm_runtime_create_shared_heap(&heap_init_args); if (!shared_heap) { - printf("Create shared heap failed. error: %s\n", error_buf); + printf("Create shared heap failed.\n"); goto fail; } @@ -268,7 +268,7 @@ main(int argc, char **argv) } /* create thread 1 */ - struct thread_arg targ1 = { 0 }; + thread_arg targ1 = { 0 }; korp_tid tid1; targ1.queue = queue; targ1.module_inst = module_inst1; @@ -279,7 +279,7 @@ main(int argc, char **argv) } /* create thread 2 */ - struct thread_arg targ2 = { 0 }; + thread_arg targ2 = { 0 }; korp_tid tid2; targ2.queue = queue; targ2.module_inst = module_inst2; diff --git a/samples/shared-heap/src/shared_heap_chain.c b/samples/shared-heap/src/shared_heap_chain.c new file mode 100644 index 000000000..f355b5dae --- /dev/null +++ b/samples/shared-heap/src/shared_heap_chain.c @@ -0,0 +1,321 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "wasm_export.h" +#include "bh_platform.h" +#include "bh_read_file.h" + +#define BUF_SIZE 4096 +static char preallocated_buf[BUF_SIZE]; + +static bool +produce_data(wasm_module_inst_t module_inst, wasm_exec_env_t exec_env, + bh_queue *queue, wasm_function_inst_t func, uint32 *argv, + uint32 buf_size, bool free_on_fail) +{ + uint8 *buf; + + wasm_runtime_call_wasm(exec_env, func, 2, argv); + + if (wasm_runtime_get_exception(module_inst)) { + printf("Failed to call function: %s\n", + wasm_runtime_get_exception(module_inst)); + return false; + } + if (argv[0] == 0) { + printf("Failed to allocate memory from shared heap\n"); + return false; + } + + buf = wasm_runtime_addr_app_to_native(module_inst, argv[0]); + printf("wasm app1 send buf: %s\n\n", buf); + + /* Passes wasm address directly between wasm apps since memory in shared + * heap chain is viewed as single address space in wasm's perspective */ + buf = (uint8 *)(uintptr_t)argv[0]; + if (!bh_post_msg(queue, 1, buf, buf_size)) { + printf("Failed to post message to queue\n"); + if (free_on_fail) + wasm_runtime_shared_heap_free(module_inst, argv[0]); + return false; + } + + return true; +} + +static void * +wasm_producer(wasm_module_inst_t module_inst, bh_queue *queue) +{ + wasm_exec_env_t exec_env; + wasm_function_inst_t my_shared_heap_malloc_func, my_shared_heap_free_func, + produce_str_func; + uint32 i, argv[2]; + + /* lookup wasm functions */ + if (!(my_shared_heap_malloc_func = wasm_runtime_lookup_function( + module_inst, "my_shared_heap_malloc")) + || !(my_shared_heap_free_func = wasm_runtime_lookup_function( + module_inst, "my_shared_heap_free")) + || !(produce_str_func = + wasm_runtime_lookup_function(module_inst, "produce_str"))) { + printf("Failed to lookup function.\n"); + } + + /* create exec env */ + if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { + printf("Failed to create exec env.\n"); + return NULL; + } + + /* allocate memory by calling my_shared_heap_malloc function and send it + to wasm app2 */ + for (i = 0; i < 8; i++) { + argv[0] = 1024 * (i + 1); + argv[1] = i + 1; + if (!produce_data(module_inst, exec_env, queue, + my_shared_heap_malloc_func, argv, 1024 * (i + 1), + true)) { + break; + } + } + + /* use pre-allocated shared heap memory by calling produce_str function and + send it to wasm app2, the pre-allocated shared heap is the last one in + chain, so its end address is calculated from UIN32_MAX */ + uint32 wasm_start_addr = UINT32_MAX - BUF_SIZE + 1; + for (i = 8; i < 16; i++) { + argv[0] = wasm_start_addr + 512 * (i - 8); + argv[1] = i + 1; + if (!produce_data(module_inst, exec_env, queue, produce_str_func, argv, + 512, false)) { + break; + } + } + + wasm_runtime_destroy_exec_env(exec_env); + + return NULL; +} + +static void +wasm_consumer(wasm_module_inst_t module_inst, bh_queue *queue) +{ + wasm_function_inst_t print_buf_func, consume_str_func; + wasm_exec_env_t exec_env; + uint32 argv[2], i; + bh_message_t msg; + char *buf; + + /* lookup wasm function */ + if (!(print_buf_func = + wasm_runtime_lookup_function(module_inst, "print_buf")) + || !(consume_str_func = + wasm_runtime_lookup_function(module_inst, "consume_str"))) { + printf("Failed to lookup function.\n"); + return; + } + + /* create exec env */ + if (!(exec_env = wasm_runtime_create_exec_env(module_inst, 32768))) { + printf("Failed to create exec env.\n"); + return; + } + + for (i = 0; i < 16; i++) { + msg = bh_get_msg(queue, BHT_WAIT_FOREVER); + if (!msg) + return; + buf = bh_message_payload(msg); + + /* call wasm function */ + argv[0] = (uint32)(uintptr_t)buf; + if (i < 8) + wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); + else + wasm_runtime_call_wasm(exec_env, consume_str_func, 1, argv); + + if (wasm_runtime_get_exception(module_inst)) { + printf( + "Failed to call 'print_buf' or 'consumer_str' function: %s\n", + wasm_runtime_get_exception(module_inst)); + } + + bh_free_msg(msg); + } + + wasm_runtime_destroy_exec_env(exec_env); +} + +static char global_heap_buf[512 * 1024]; + +int +main(int argc, char **argv) +{ + char *wasm_file1 = NULL, *wasm_file2 = NULL; + uint8 *wasm_file1_buf = NULL, *wasm_file2_buf = NULL; + uint32 wasm_file1_size, wasm_file2_size; + wasm_module_t wasm_module1 = NULL, wasm_module2 = NULL; + wasm_module_inst_t module_inst1 = NULL; + wasm_module_inst_t module_inst2 = NULL; + wasm_shared_heap_t shared_heap = NULL, shared_heap2 = NULL, + shared_heap_chain = NULL; + bh_queue *queue = NULL; + RuntimeInitArgs init_args; + SharedHeapInitArgs heap_init_args; + char error_buf[128] = { 0 }; + bool aot_mode = false; + int ret = -1; + + if (argc > 1 && !strcmp(argv[1], "--aot")) + aot_mode = true; + + if (!aot_mode) + printf("Test shared heap in interpreter mode\n\n"); + else + printf("Test shared heap in AOT mode\n\n"); + + memset(&init_args, 0, sizeof(RuntimeInitArgs)); + + init_args.mem_alloc_type = Alloc_With_Pool; + init_args.mem_alloc_option.pool.heap_buf = global_heap_buf; + init_args.mem_alloc_option.pool.heap_size = sizeof(global_heap_buf); + + /* init wasm runtime */ + if (!wasm_runtime_full_init(&init_args)) { + printf("Init runtime environment failed.\n"); + return -1; + } + + /* create queue */ + if (!(queue = bh_queue_create())) { + printf("Create queue failed.\n"); + goto fail; + } + + /* read wasm file */ + if (!aot_mode) + wasm_file1 = "./wasm-apps/test1.wasm"; + else + wasm_file1 = "./wasm-apps/test1_chain.aot"; + if (!(wasm_file1_buf = + bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { + printf("Open wasm file %s failed.\n", wasm_file1); + goto fail; + } + + /* load wasm file */ + wasm_module1 = wasm_runtime_load((uint8 *)wasm_file1_buf, wasm_file1_size, + error_buf, sizeof(error_buf)); + if (!wasm_module1) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* instantiate module */ + module_inst1 = wasm_runtime_instantiate(wasm_module1, 65536, 0, error_buf, + sizeof(error_buf)); + if (!module_inst1) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* read wasm file */ + if (!aot_mode) + wasm_file2 = "./wasm-apps/test2.wasm"; + else + wasm_file2 = "./wasm-apps/test2_chain.aot"; + if (!(wasm_file2_buf = + bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { + printf("Open wasm file %s failed.\n", wasm_file1); + goto fail; + } + + /* load wasm file */ + wasm_module2 = wasm_runtime_load((uint8 *)wasm_file2_buf, wasm_file2_size, + error_buf, sizeof(error_buf)); + if (!wasm_module2) { + printf("Load wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* instantiate module */ + module_inst2 = wasm_runtime_instantiate(wasm_module2, 65536, 0, error_buf, + sizeof(error_buf)); + if (!module_inst2) { + printf("Instantiate wasm module failed. error: %s\n", error_buf); + goto fail; + } + + /* create shared heap */ + memset(&heap_init_args, 0, sizeof(heap_init_args)); + heap_init_args.size = 65536; + shared_heap = wasm_runtime_create_shared_heap(&heap_init_args); + if (!shared_heap) { + printf("Create shared heap failed.\n"); + goto fail; + } + + /* create a preallocated shared heap */ + memset(&heap_init_args, 0, sizeof(heap_init_args)); + heap_init_args.pre_allocated_addr = preallocated_buf; + heap_init_args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&heap_init_args); + if (!shared_heap2) { + printf("Create preallocated shared heap failed\n"); + goto fail; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + printf("Create shared heap chain failed\n"); + goto fail; + } + + /* attach module instance 1 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(module_inst1, shared_heap_chain)) { + printf("Attach shared heap failed.\n"); + goto fail; + } + + /* attach module instance 2 to the shared heap */ + if (!wasm_runtime_attach_shared_heap(module_inst2, shared_heap_chain)) { + printf("Attach shared heap failed.\n"); + goto fail; + } + + /* wasm 1 produce shared data */ + wasm_producer(module_inst1, queue); + + /* wasm 2 consume shared data */ + wasm_consumer(module_inst2, queue); + ret = 0; + +fail: + if (module_inst2) + wasm_runtime_deinstantiate(module_inst2); + + if (module_inst1) + wasm_runtime_deinstantiate(module_inst1); + + if (wasm_module2) + wasm_runtime_unload(wasm_module2); + + if (wasm_module1) + wasm_runtime_unload(wasm_module1); + + if (wasm_file2_buf) + wasm_runtime_free(wasm_file2_buf); + + if (wasm_file1_buf) + wasm_runtime_free(wasm_file1_buf); + + if (queue) + bh_queue_destroy(queue); + + wasm_runtime_destroy(); + + return ret; +} diff --git a/samples/shared-heap/wasm-apps/CMakeLists.txt b/samples/shared-heap/wasm-apps/CMakeLists.txt index c0010af6a..7bfa8cd48 100644 --- a/samples/shared-heap/wasm-apps/CMakeLists.txt +++ b/samples/shared-heap/wasm-apps/CMakeLists.txt @@ -30,9 +30,7 @@ set (CMAKE_EXE_LINKER_FLAGS -Wl,--no-entry,--strip-all, \ -Wl,--export=__heap_base,--export=__data_end \ -Wl,--export=__wasm_call_ctors \ - -Wl,--export=my_shared_heap_malloc \ - -Wl,--export=my_shared_heap_free \ - -Wl,--export=print_buf \ + -Wl,--export-all \ -Wl,--allow-undefined" ) diff --git a/samples/shared-heap/wasm-apps/test1.c b/samples/shared-heap/wasm-apps/test1.c index c8fe0c755..321f10218 100644 --- a/samples/shared-heap/wasm-apps/test1.c +++ b/samples/shared-heap/wasm-apps/test1.c @@ -58,3 +58,14 @@ my_shared_heap_free(void *ptr) { shared_heap_free(ptr); } + +void * +produce_str(char *addr, uint32_t index) +{ + char c; + snprintf(addr, 512, "Data: %u stores to pre-allocated shared heap", index); + /* Actually access it in wasm */ + c = addr[0]; + printf("In WASM: the first char is %c\n", c); + return addr; +} diff --git a/samples/shared-heap/wasm-apps/test2.c b/samples/shared-heap/wasm-apps/test2.c index b63efcd1a..44d573164 100644 --- a/samples/shared-heap/wasm-apps/test2.c +++ b/samples/shared-heap/wasm-apps/test2.c @@ -4,8 +4,7 @@ */ #include - -#include +#include extern void shared_heap_free(void *ptr); @@ -16,3 +15,14 @@ print_buf(char *buf) printf("wasm app2's wasm func received buf: %s\n\n", buf); shared_heap_free(buf); } + +void +consume_str(char *buf) +{ + /* Actually access it in wasm */ + char c = buf[0]; + printf("In WASM: wasm app2's wasm func received buf in pre-allocated " + "shared buf: " + "%s with its first char is %c\n\n", + buf, c); +} diff --git a/samples/socket-api/CMakeLists.txt b/samples/socket-api/CMakeLists.txt index e68a63eb0..a68d0caba 100644 --- a/samples/socket-api/CMakeLists.txt +++ b/samples/socket-api/CMakeLists.txt @@ -171,6 +171,7 @@ set(WAMR_BUILD_JIT 0) set(WAMR_BUILD_LIBC_BUILTIN 1) set(WAMR_BUILD_LIBC_WASI 1) set(WAMR_BUILD_LIB_PTHREAD 1) +set(WAMR_BUILD_REF_TYPES 1) # compiling and linking flags if (NOT (CMAKE_C_COMPILER MATCHES ".*clang.*" OR CMAKE_C_COMPILER_ID MATCHES ".*Clang")) diff --git a/samples/socket-api/README.md b/samples/socket-api/README.md index 522cbf47b..f172c5b17 100644 --- a/samples/socket-api/README.md +++ b/samples/socket-api/README.md @@ -79,7 +79,7 @@ Client is running... Start receiving. Start sending. Send 106 bytes successfully! -Receive 106 bytes successlly! +Receive 106 bytes successfully! Data: The stars shine down It brings us light diff --git a/samples/socket-api/wasm-src/send_recv.c b/samples/socket-api/wasm-src/send_recv.c index 0071b2a7b..176061698 100644 --- a/samples/socket-api/wasm-src/send_recv.c +++ b/samples/socket-api/wasm-src/send_recv.c @@ -25,6 +25,7 @@ static bool server_is_ready = false; void * run_as_server(void *arg) { + (void)arg; int sock = -1, on = 1; struct sockaddr_in addr = { 0 }; int addrlen = 0; @@ -109,7 +110,7 @@ run_as_server(void *arg) fail2: close(new_sock); fail1: - shutdown(sock, SHUT_RD); + shutdown(sock, SHUT_RDWR); close(sock); return NULL; } @@ -117,6 +118,7 @@ fail1: void * run_as_client(void *arg) { + (void)arg; int sock = -1; struct sockaddr_in addr = { 0 }; /* buf of server is 106 bytes */ @@ -159,7 +161,7 @@ run_as_client(void *arg) goto fail; } - printf("Receive %ld bytes successlly!\n", recv_len); + printf("Receive %ld bytes successfully!\n", recv_len); assert(recv_len == 106); printf("Data:\n"); @@ -170,7 +172,7 @@ run_as_client(void *arg) } fail: - shutdown(sock, SHUT_RD); + shutdown(sock, SHUT_RDWR); close(sock); return NULL; } @@ -178,6 +180,8 @@ fail: int main(int argc, char *argv[]) { + (void)argc; + (void)argv; pthread_t cs[2] = { 0 }; uint8_t i = 0; int ret = EXIT_SUCCESS; diff --git a/samples/wasm-c-api-imports/wasm/send_recv.c b/samples/wasm-c-api-imports/wasm/send_recv.c index d9f95573a..821914f4e 100644 --- a/samples/wasm-c-api-imports/wasm/send_recv.c +++ b/samples/wasm-c-api-imports/wasm/send_recv.c @@ -50,6 +50,7 @@ local_printf(const char *formatter, ...) void * run_as_server(void *arg) { + (void)arg; int sock = -1, on = 1; struct sockaddr_in addr = { 0 }; int addrlen = 0; @@ -134,7 +135,7 @@ run_as_server(void *arg) fail2: close(new_sock); fail1: - shutdown(sock, SHUT_RD); + shutdown(sock, SHUT_RDWR); close(sock); return NULL; } @@ -142,6 +143,7 @@ fail1: void * run_as_client(void *arg) { + (void)arg; int sock = -1; struct sockaddr_in addr = { 0 }; /* buf of server is 106 bytes */ @@ -184,7 +186,7 @@ run_as_client(void *arg) goto fail; } - local_printf("Receive %ld bytes successlly!\n", recv_len); + local_printf("Receive %ld bytes successfully!\n", recv_len); assert(recv_len == 106); local_printf("Data:\n"); @@ -195,7 +197,7 @@ run_as_client(void *arg) } fail: - shutdown(sock, SHUT_RD); + shutdown(sock, SHUT_RDWR); close(sock); return NULL; } @@ -203,6 +205,8 @@ fail: int main(int argc, char *argv[]) { + (void)argc; + (void)argv; pthread_t cs[2] = { 0 }; uint8_t i = 0; int ret = EXIT_SUCCESS; diff --git a/test-tools/aot-analyzer/include/config.h b/test-tools/aot-analyzer/include/config.h index 970d7e2cc..9f7fa49ea 100644 --- a/test-tools/aot-analyzer/include/config.h +++ b/test-tools/aot-analyzer/include/config.h @@ -15,7 +15,7 @@ #define WASM_CURRENT_VERSION 1 #define AOT_MAGIC_NUMBER 0x746f6100 -#define AOT_CURRENT_VERSION 4 +#define AOT_CURRENT_VERSION 5 /* Legal values for bin_type */ #define BIN_TYPE_ELF32L 0 /* 32-bit little endian */ diff --git a/tests/fuzz/wasm-mutator-fuzz/aot-compiler/aot_compiler_fuzz.cc b/tests/fuzz/wasm-mutator-fuzz/aot-compiler/aot_compiler_fuzz.cc index 89b4bad0a..c8ec4c0d5 100644 --- a/tests/fuzz/wasm-mutator-fuzz/aot-compiler/aot_compiler_fuzz.cc +++ b/tests/fuzz/wasm-mutator-fuzz/aot-compiler/aot_compiler_fuzz.cc @@ -58,6 +58,7 @@ LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) option.enable_simd = true; option.enable_ref_types = true; option.enable_gc = true; + option.aux_stack_frame_type = AOT_STACK_FRAME_TYPE_STANDARD; comp_data = aot_create_comp_data(module, option.target_arch, option.enable_gc); diff --git a/tests/fuzz/wasm-mutator-fuzz/server/app/main.py b/tests/fuzz/wasm-mutator-fuzz/server/app/main.py index 620625dd3..c1ee3bfc4 100644 --- a/tests/fuzz/wasm-mutator-fuzz/server/app/main.py +++ b/tests/fuzz/wasm-mutator-fuzz/server/app/main.py @@ -72,7 +72,7 @@ def to_json(inst, cls): class Fuzzing(db.Model): - __tablename__ = 'fazzing_task' + __tablename__ = 'fuzzing_task' id = db.Column(db.Integer, autoincrement=True, primary_key=True, nullable=False) repo = db.Column(db.String(200), nullable=False, default='') @@ -96,7 +96,7 @@ class TaskError(db.Model): __tablename__ = 'task_error' id = db.Column(db.Integer, autoincrement=True, primary_key=True, nullable=False) - fazzing_id = db.Column(db.Integer, db.ForeignKey("fazzing_task.id")) + fuzzing_id = db.Column(db.Integer, db.ForeignKey("fuzzing_task.id")) name = db.Column(db.String(200), nullable=False, default='') std_out = db.Column(db.Text, default='') data = db.Column(db.JSON) @@ -119,9 +119,9 @@ def to_data(data): def error_count(data): error = len(TaskError.query.filter( - TaskError.fazzing_id == data.get('id'), TaskError.status.in_([1, 2])).all()) + TaskError.fuzzing_id == data.get('id'), TaskError.status.in_([1, 2])).all()) end_error = len(TaskError.query.filter( - TaskError.fazzing_id == data.get('id'), TaskError.status == 0).all()) + TaskError.fuzzing_id == data.get('id'), TaskError.status == 0).all()) data['error'] = error data['end_error'] = end_error return data @@ -159,11 +159,11 @@ def show_fuzz_list(): id = data.get('id') if id: all_error = TaskError.query.filter( - TaskError.fazzing_id == id).with_entities(TaskError.id, TaskError.fazzing_id, + TaskError.fuzzing_id == id).with_entities(TaskError.id, TaskError.fuzzing_id, TaskError.create_time, TaskError.data, TaskError.name, TaskError.status, TaskError.update_time, TaskError.comment).order_by(TaskError.status.desc(), TaskError.update_time.desc(), TaskError.id.desc()).all() - data_message = [{'id': error['id'], "fuzzing_id": error['fazzing_id'], + data_message = [{'id': error['id'], "fuzzing_id": error['fuzzing_id'], "name": error['name'], "data": error['data'], 'create_time': error['create_time'].strftime('%Y-%m-%d %H:%M:%S'), 'update_time': error['update_time'].strftime('%Y-%m-%d %H:%M:%S'), @@ -204,7 +204,7 @@ def New_fuzzing(): # curd.set_error_status_to(list(map(lambda x: x.id, error_list)), db) # Fuzzing.query.filter_by(id=fuzz.id).delete() fuzz.data = {'error': "Clone repo Error"} - db.commit() + db.session.commit() return jsonify({"status": 0, "result": "", "msg": "Clone repo Error"}) wamr_path_parent = fuzz_dir.parent.parent @@ -277,7 +277,7 @@ def scheduler_run_task(): for fuzz in fuzz_query: all_error = TaskError.query.filter( - TaskError.fazzing_id == fuzz.id).with_entities(TaskError.name).all() + TaskError.fuzzing_id == fuzz.id).with_entities(TaskError.name).all() fuzz_cmd = wasm_mutator_dir / \ 'workspace' / f'build_{fuzz.id}' dir_list = filter(lambda x: x.startswith( @@ -287,7 +287,7 @@ def scheduler_run_task(): for dir in dir_list: cmd = f'cd {fuzz_cmd} && ./wasm_mutator_fuzz {dir}' status, resp = getstatusoutput(cmd) - task_error = TaskError(name=dir, std_out=resp, fazzing_id=fuzz.id, + task_error = TaskError(name=dir, std_out=resp, fuzzing_id=fuzz.id, create_time=datetime.utcnow() + timedelta(hours=8)) db.session.add(task_error) db.session.commit() @@ -312,7 +312,7 @@ def get_error_txt(): return jsonify({"status": 0, "results": [], 'msg': "Error"}) error = TaskError.query.get(id) fuzz_cmd = wasm_mutator_dir / \ - 'workspace' / f'build_{error.fazzing_id}' + 'workspace' / f'build_{error.fuzzing_id}' file_cmd = fuzz_cmd / error.name response = send_file(file_cmd, as_attachment=True, @@ -351,7 +351,7 @@ def get_cases_zip(): with ZipFile(memory_file, "w", ZIP_DEFLATED) as zf: for task_error in task_query: fuzz_cmd = wasm_mutator_dir / \ - 'workspace' / f'build_{task_error.fazzing_id}' + 'workspace' / f'build_{task_error.fuzzing_id}' file_cmd = fuzz_cmd / task_error.name zf.write(str(file_cmd), arcname=task_error.name) memory_file.seek(0) @@ -399,7 +399,7 @@ def error_restart(): if run_status: return jsonify({"status": 0, "results": [], 'msg': "There are already tasks in progress"}) task_query = TaskError.query.filter(TaskError.id.in_(id_list)).all() - fuzzing_id = task_query[0].fazzing_id + fuzzing_id = task_query[0].fuzzing_id fuzz_cmd = wasm_mutator_dir / \ 'workspace' / f'build_{fuzzing_id}' restart_cmd = wasm_mutator_dir / \ @@ -412,7 +412,7 @@ def error_restart(): if not Path(restart_cmd / 'wamr').exists(): print('------ error: clone repo not folder exists ------') # fuzz.data = {'error': "Clone repo Error"} - db.commit() + db.session.commit() return jsonify({"status": 0, "result": "", "msg": "Clone repo Error"}) wamr_path_parent = fuzz_dir.parent.parent wamr_path = wamr_path_parent / 'wamr' diff --git a/tests/regression/ba-issues/README.md b/tests/regression/ba-issues/README.md index dd5c3b3c5..ed423da41 100644 --- a/tests/regression/ba-issues/README.md +++ b/tests/regression/ba-issues/README.md @@ -218,22 +218,57 @@ simply run `run.py` ./run.py ``` +Specify a specific issue with option `--issues`/`-i` + +```shell +./run.py --issues 2833 # test 1 issue #2833 +./run.py -i 2833,2834,2835 # test 3 issues #2833 #2834 #2835 +``` + If everything went well, you should see similarly output in your command line output ```shell -Finish testing, 22/22 of test cases passed, no more issues should further test +==== Test results ==== + Total: 22 + Passed: 22 + Failed: 0 + Left issues in folder: no more + Cases in JSON but not found in folder: no more ``` If you add the test case under directory `issues` but forget to add the running config in json file, the output can be something like ```shell -Finish testing, 21/21 of test cases passed, {2945} issue(s) should further test +==== Test results ==== + Total: 21 + Passed: 21 + Failed: 0 + missed: 0 + Left issues in folder: #3022 + Cases in JSON but not found in folder: no more +``` + +If you add the test case in `running_config.json` but used the wrong id or forget to add the test case under directory `issues`, the output can be someting like + +```shell +==== Test results ==== + Total: 21 + Passed: 21 + Failed: 0 + missed: 0 + Left issues in folder: #2855 + Cases in JSON but not found in folder: #12345 ``` If some test case are failing, then it will be something like ```shell -Finish testing, 21/22 of test cases passed, no more issue(s) should further test +==== Test results ==== + Total: 22 + Passed: 21 + Failed: 1 + Left issues in folder: no more + Cases in JSON but not found in folder: no more ``` And a log file named `issues_tests.log` will be generated and inside it will display the details of the failing cases, for example: diff --git a/tests/regression/ba-issues/run.py b/tests/regression/ba-issues/run.py index 7f26738d1..d50bf9b14 100755 --- a/tests/regression/ba-issues/run.py +++ b/tests/regression/ba-issues/run.py @@ -10,7 +10,10 @@ import os import subprocess import glob import re -from typing import Dict +import argparse +import sys + +from typing import Dict, Optional, List WORK_DIR = os.getcwd() TEST_WASM_COMMAND = ( @@ -45,7 +48,12 @@ def dump_error_log(failing_issue_id, command_lists, exit_code_cmp, stdout_cmp): ) -def get_issue_ids_should_test(): +def get_issue_ids_should_test(selected_ids: Optional[List[int]] = None): + """Find all issue IDs that should be tested in folder issues.""" + # If specific issue IDs are provided, return them as a set + if selected_ids: + return set(selected_ids) + # Define the path pattern path_pattern = "issues/issue-*" @@ -60,8 +68,8 @@ def get_issue_ids_should_test(): # Extract the issue number using regular expression match = re.search(pattern, dir_path) if match: - issue_number = match.group(1) - issue_numbers.add(int(issue_number)) + issue_number = int(match.group(1)) + issue_numbers.add(issue_number) # Print the set of issue numbers return issue_numbers @@ -77,10 +85,10 @@ def get_and_check(d, key, default=None, nullable=False): def run_and_compare_results( - passed_ids, failed_ids, issue_id, cmd, description, ret_code, stdout_content -): + issue_id, cmd, description, ret_code, stdout_content +) -> bool: print(f"####################################") - print(f"test BA issue #{issue_id} `{description}`: {cmd}") + print(f"test BA issue #{issue_id} `{description}`...") command_list = cmd.split() result = subprocess.run( command_list, @@ -95,19 +103,21 @@ def run_and_compare_results( exit_code_cmp = f"exit code (actual, expected) : {actual_exit_code, ret_code}" stdout_cmp = f"stdout (actual, expected) : {actual_output, stdout_content}" - print(exit_code_cmp) - print(stdout_cmp) if actual_exit_code == ret_code and ( actual_output == stdout_content - or (stdout_content == "Compile success" - and actual_output.find(stdout_content) != -1) + or ( + stdout_content == "Compile success" + and actual_output.find(stdout_content) != -1 + ) or (len(stdout_content) > 30 and actual_output.find(stdout_content) != -1) ): - passed_ids.add(issue_id) print("== PASS ==") + return True else: - failed_ids.add(issue_id) + print(cmd) + print(exit_code_cmp) + print(stdout_cmp) print(f"== FAILED: {issue_id} ==") dump_error_log( issue_id, @@ -115,15 +125,11 @@ def run_and_compare_results( exit_code_cmp, stdout_cmp, ) - - print("") + return False -def run_issue_test_wamrc( - passed_ids, failed_ids, issue_id, compile_options, stdout_only_cmp_last_line=False -): +def run_issue_test_wamrc(issue_id, compile_options): compiler = get_and_check(compile_options, "compiler") - only_compile = get_and_check(compile_options, "only compile") in_file = get_and_check(compile_options, "in file") out_file = get_and_check(compile_options, "out file") options = get_and_check(compile_options, "options") @@ -145,14 +151,10 @@ def run_issue_test_wamrc( compiler=compiler, options=options, out_file=out_file_path, in_file=in_file_path ) - run_and_compare_results( - passed_ids, failed_ids, issue_id, cmd, description, ret_code, stdout_content - ) - - return only_compile + return run_and_compare_results(issue_id, cmd, description, ret_code, stdout_content) -def run_issue_test_iwasm(passed_ids, failed_ids, issue_id, test_case): +def run_issue_test_iwasm(issue_id, test_case) -> bool: runtime = get_and_check(test_case, "runtime") mode = get_and_check(test_case, "mode") file = get_and_check(test_case, "file") @@ -194,17 +196,19 @@ def run_issue_test_iwasm(passed_ids, failed_ids, issue_id, test_case): argument=argument, ) - run_and_compare_results( - passed_ids, failed_ids, issue_id, cmd, description, ret_code, stdout_content - ) + return run_and_compare_results(issue_id, cmd, description, ret_code, stdout_content) -def process_and_run_test_cases(data: Dict[str, Dict]): - issue_ids_should_test = get_issue_ids_should_test() +def process_and_run_test_cases( + data: Dict[str, Dict], selected_ids: Optional[List[int]] = None +): + issue_ids_should_test = get_issue_ids_should_test(selected_ids) passed_ids = set() failed_ids = set() + json_only_ids = set() + # Iterate through each test case in the json data for test_case in data.get("test cases", []): is_deprecated = get_and_check(test_case, "deprecated") issue_ids = get_and_check(test_case, "ids", default=[]) @@ -214,33 +218,84 @@ def process_and_run_test_cases(data: Dict[str, Dict]): continue compile_options = get_and_check(test_case, "compile_options", nullable=True) - for issue_id in issue_ids: - only_compile = False - # if this issue needs to test wamrc to compile the test case first - if compile_options: - only_compile = compile_options["only compile"] - run_issue_test_wamrc(passed_ids, failed_ids, issue_id, compile_options) - # if this issue requires to test iwasm to run the test case - if not only_compile: - run_issue_test_iwasm(passed_ids, failed_ids, issue_id, test_case) + for issue_id in issue_ids: + if issue_id not in issue_ids_should_test: + json_only_ids.add(issue_id) + continue # cross out the this issue_id in the should test set issue_ids_should_test.remove(issue_id) + only_compile = False + + # if this issue needs to test wamrc to compile the test case first + if compile_options: + only_compile = compile_options["only compile"] + compile_res = run_issue_test_wamrc(issue_id, compile_options) + if only_compile: + if compile_res: + passed_ids.add(issue_id) + else: + failed_ids.add(issue_id) + continue + else: + # if compile success, then continue to test iwasm + if not compile_res: + failed_ids.add(issue_id) + continue + + # if this issue requires to test iwasm to run the test case + if not only_compile: + if run_issue_test_iwasm(issue_id, test_case): + passed_ids.add(issue_id) + else: + failed_ids.add(issue_id) + total = len(passed_ids) + len(failed_ids) passed = len(passed_ids) failed = len(failed_ids) - issue_ids_should_test = ( - issue_ids_should_test if issue_ids_should_test else "no more" + + format_issue_ids_should_test = ( + " ".join(f"#{x}" for x in issue_ids_should_test) + if issue_ids_should_test + else "no more" ) + format_json_only_ids = ( + " ".join(f"#{x}" for x in json_only_ids) if json_only_ids else "no more" + ) + + print(f"####################################") print(f"==== Test results ====") - print(f" Total: {total}") + print(f" Total: {total}") print(f" Passed: {passed}") print(f" Failed: {failed}") + if not selected_ids: + print(f" Left issues in folder: {format_issue_ids_should_test}") + print(f" Cases in JSON but not found in folder: {format_json_only_ids}") + else: + print(f" Issues not found in folder: {format_issue_ids_should_test}") + + if failed > 0: + # Exit with error code if there are failed test for CI + print("Some tests failed, see log file for details.") + sys.exit(1) def main(): + parser = argparse.ArgumentParser(description="Run BA issue tests.") + parser.add_argument( + "-i", + "--issues", + type=str, + help="Comma separated list of issue ids to run, e.g. 1,2,3. Default: all.", + ) + args = parser.parse_args() + + selected_ids = None + if args.issues: + selected_ids = [int(x) for x in args.issues.split(",") if x.strip().isdigit()] + # Path to the JSON file file_path = "running_config.json" @@ -256,7 +311,7 @@ def main(): os.remove(LOG_FILE) # Process the data - process_and_run_test_cases(data) + process_and_run_test_cases(data, selected_ids) if __name__ == "__main__": diff --git a/tests/regression/ba-issues/running_config.json b/tests/regression/ba-issues/running_config.json index 2532decfe..9288eb970 100644 --- a/tests/regression/ba-issues/running_config.json +++ b/tests/regression/ba-issues/running_config.json @@ -96,7 +96,7 @@ "argument": "", "expected return": { "ret code": 255, - "stdout content": "WASM module load failed: END opcode expected", + "stdout content": "WASM module load failed: unexpected end opcodes from unbalanced control flow structures", "description": "no sanitizer 'Heap Buffer Overflow'" } }, @@ -979,7 +979,7 @@ "argument": "", "expected return": { "ret code": 255, - "stdout content": "WASM module load failed: unexpected end of section or function", + "stdout content": "WASM module load failed: section size mismatch: function body END opcode expected", "description": "no 'Heap out of bound read of size 1 in wasm_loader_prepare_bytecode function'" } }, diff --git a/tests/requirement-engineering/gc-aot/build_spec_interpreter.sh b/tests/requirement-engineering/gc-aot/build_spec_interpreter.sh index 48d6343b1..0ecfe93f4 100755 --- a/tests/requirement-engineering/gc-aot/build_spec_interpreter.sh +++ b/tests/requirement-engineering/gc-aot/build_spec_interpreter.sh @@ -17,7 +17,7 @@ git apply ../../../wamr-test-suites/spec-test-script/gc_ignore_cases.patch # Set OCaml compiler environment eval $(opam config env) -echo "compile the reference intepreter" +echo "compile the reference interpreter" pushd interpreter make -popd \ No newline at end of file +popd diff --git a/tests/standalone/test-running-modes/test_c_embed_api_thoroughly.py b/tests/standalone/test-running-modes/test_c_embed_api_thoroughly.py index 63e871e4a..8dcde7e74 100755 --- a/tests/standalone/test-running-modes/test_c_embed_api_thoroughly.py +++ b/tests/standalone/test-running-modes/test_c_embed_api_thoroughly.py @@ -9,7 +9,7 @@ import os from collections import OrderedDict -def CLI_ARGS_GENREATOR(running_modes_supported: list[str]) -> list[str]: +def CLI_ARGS_GENERATOR(running_modes_supported: list[str]) -> list[str]: res = [] list_2d = [["--default-running-mode={} --module-running-mode={}".format(i, j) for i in running_modes_supported] for j in running_modes_supported] @@ -35,16 +35,16 @@ def main(): ] # Python 3.7+: Dictionary iteration order is guaranteed to be in order of insertion. - # just to be safe, using orderreddict + # just to be safe, using OrderedDict # key: value -> compile mode, {"compile_flag": CMake compile flag, "iwasm_cli_args": array of CLI args tested} test_options = OrderedDict({ - "INTERP": {"compile_flag": COMPILE_FLAGS[0], "cli_args": CLI_ARGS_GENREATOR(RUNNING_MODES[:1])}, - "FAST_JIT": {"compile_flag": COMPILE_FLAGS[1], "cli_args": CLI_ARGS_GENREATOR(RUNNING_MODES[:2])}, + "INTERP": {"compile_flag": COMPILE_FLAGS[0], "cli_args": CLI_ARGS_GENERATOR(RUNNING_MODES[:1])}, + "FAST_JIT": {"compile_flag": COMPILE_FLAGS[1], "cli_args": CLI_ARGS_GENERATOR(RUNNING_MODES[:2])}, "LLVM_JIT": {"compile_flag": COMPILE_FLAGS[2], - "cli_args": CLI_ARGS_GENREATOR([RUNNING_MODES[0], RUNNING_MODES[2]])}, - "MULTI_TIER_JIT": {"compile_flag": COMPILE_FLAGS[3], "cli_args": CLI_ARGS_GENREATOR(RUNNING_MODES)}, + "cli_args": CLI_ARGS_GENERATOR([RUNNING_MODES[0], RUNNING_MODES[2]])}, + "MULTI_TIER_JIT": {"compile_flag": COMPILE_FLAGS[3], "cli_args": CLI_ARGS_GENERATOR(RUNNING_MODES)}, "EAGER_JIT_WITH_BOTH_JIT": {"compile_flag": COMPILE_FLAGS[4], - "cli_args": CLI_ARGS_GENREATOR(RUNNING_MODES[:3])} + "cli_args": CLI_ARGS_GENERATOR(RUNNING_MODES[:3])} }) build_cmd = "./build_c_embed.sh \"{build_flag}\"" diff --git a/tests/standalone/test-running-modes/test_iwasm_thoroughly.py b/tests/standalone/test-running-modes/test_iwasm_thoroughly.py index a5af29101..3c631a6d5 100755 --- a/tests/standalone/test-running-modes/test_iwasm_thoroughly.py +++ b/tests/standalone/test-running-modes/test_iwasm_thoroughly.py @@ -29,7 +29,7 @@ def main(): ] # Python 3.7+: Dictionary iteration order is guaranteed to be in order of insertion. - # just to be safe, using orderreddict + # just to be safe, using OrderedDict # key: value -> compile mode, {"compile_flag": CMake compile flag, "iwasm_cli_args": array of CLI args tested} test_options = OrderedDict({ "INTERP": {"compile_flag": COMPILE_FLAGS[0], "iwasm_cli_args": IWASM_CLI_ARGS[:1]}, diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 8c963cadd..b726f83a1 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -19,8 +19,15 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}") if(WAMR_BUILD_TARGET STREQUAL "X86_32") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32") + # 1) Force -m32 + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32" CACHE STRING "" FORCE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "" FORCE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m32" CACHE STRING "" FORCE) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -m32" CACHE STRING "" FORCE) + + # 2) Make CMake prefer i386 libraries + set(CMAKE_SYSTEM_PROCESSOR i386 CACHE STRING "" FORCE) + set(CMAKE_LIBRARY_ARCHITECTURE "i386-linux-gnu" CACHE STRING "" FORCE) endif() # Prevent overriding the parent project's compiler/linker @@ -29,12 +36,21 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) # Fetch Google test include (FetchContent) -FetchContent_Declare ( + +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24") + FetchContent_Declare ( googletest URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip - DOWNLOAD_EXTRACT_TIMESTAMP TRUE -) -FetchContent_MakeAvailable (googletest) + DOWNLOAD_EXTRACT_TIMESTAMP ON + ) +else() + FetchContent_Declare ( + googletest + URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip + ) +endif() + +FetchContent_MakeAvailable(googletest) SET(GOOGLETEST_INCLUDED 1) diff --git a/tests/unit/memory64/memory64_atomic_test.cc b/tests/unit/memory64/memory64_atomic_test.cc index 49295668d..3b2f679b5 100644 --- a/tests/unit/memory64/memory64_atomic_test.cc +++ b/tests/unit/memory64/memory64_atomic_test.cc @@ -31,7 +31,7 @@ class memory64_atomic_test_suite : public testing::TestWithParam return true; fail: - if (!module) + if (module) wasm_runtime_unload(module); return false; @@ -56,6 +56,8 @@ class memory64_atomic_test_suite : public testing::TestWithParam if (exec_env) wasm_runtime_destroy_exec_env(exec_env); if (module_inst) + wasm_runtime_deinstantiate(module_inst); + if (module) wasm_runtime_unload(module); return false; } diff --git a/tests/unit/memory64/memory64_test.cc b/tests/unit/memory64/memory64_test.cc index af36f308c..2418d70ad 100644 --- a/tests/unit/memory64/memory64_test.cc +++ b/tests/unit/memory64/memory64_test.cc @@ -31,7 +31,7 @@ class memory64_test_suite : public testing::TestWithParam return true; fail: - if (!module) + if (module) wasm_runtime_unload(module); return false; @@ -56,11 +56,13 @@ class memory64_test_suite : public testing::TestWithParam if (exec_env) wasm_runtime_destroy_exec_env(exec_env); if (module_inst) + wasm_runtime_deinstantiate(module_inst); + if (module) wasm_runtime_unload(module); return false; } - void destory_exec_env() + void destroy_exec_env() { wasm_runtime_destroy_exec_env(exec_env); wasm_runtime_deinstantiate(module_inst); @@ -201,7 +203,7 @@ TEST_P(memory64_test_suite, memory_8GB) i64 = 0xbeefdead; ASSERT_EQ(i64, GET_U64_FROM_ADDR(wasm_argv)); - destory_exec_env(); + destroy_exec_env(); } TEST_P(memory64_test_suite, mem64_from_clang) @@ -228,7 +230,7 @@ TEST_P(memory64_test_suite, mem64_from_clang) i32 = 0x109; ASSERT_EQ(i32, wasm_argv[0]); - destory_exec_env(); + destroy_exec_env(); } INSTANTIATE_TEST_CASE_P(RunningMode, memory64_test_suite, diff --git a/tests/unit/running-modes/wasm_running_modes_test.cc b/tests/unit/running-modes/wasm_running_modes_test.cc index e18e64fb1..5f370dd64 100644 --- a/tests/unit/running-modes/wasm_running_modes_test.cc +++ b/tests/unit/running-modes/wasm_running_modes_test.cc @@ -21,7 +21,7 @@ std::string TEST_WASM1 = "/hello.wasm"; std::string TEST_WASM2 = "/mytest.wasm"; char *WASM_FILE_1; char *WASM_FILE_2; -std::vector running_mode_supportted = { Mode_Interp, +std::vector running_mode_supported = { Mode_Interp, #if WASM_ENABLE_FAST_JIT != 0 Mode_Fast_JIT, #endif @@ -76,7 +76,7 @@ class wasm_running_modes_test_suite : public testing::TestWithParam return true; fail: - if (!module) + if (module) wasm_runtime_unload(module); return false; @@ -101,11 +101,13 @@ class wasm_running_modes_test_suite : public testing::TestWithParam if (exec_env) wasm_runtime_destroy_exec_env(exec_env); if (module_inst) + wasm_runtime_deinstantiate(module_inst); + if (module) wasm_runtime_unload(module); return false; } - void destory_exec_env() + void destroy_exec_env() { wasm_runtime_destroy_exec_env(exec_env); wasm_runtime_deinstantiate(module_inst); @@ -139,7 +141,7 @@ class wasm_running_modes_test_suite : public testing::TestWithParam ASSERT_TRUE(ret); ASSERT_EQ(10, wasm_argv[0]); - destory_exec_env(); + destroy_exec_env(); } void run_wasm_complex(char *filename1, char *filename2, @@ -168,7 +170,7 @@ class wasm_running_modes_test_suite : public testing::TestWithParam ASSERT_TRUE(ret); ASSERT_EQ(10, wasm_argv[0]); - destory_exec_env(); + destroy_exec_env(); /* run wasm file 2 in running_mode */ ret = load_wasm_file(filename2); @@ -184,7 +186,7 @@ class wasm_running_modes_test_suite : public testing::TestWithParam ret = wasm_runtime_call_wasm(exec_env, main, 2, wasm_argv); ASSERT_TRUE(ret); - destory_exec_env(); + destroy_exec_env(); } public: @@ -246,7 +248,7 @@ TEST_F(wasm_running_modes_test_suite, wasm_runtime_is_running_mode_supported) // normal situation ASSERT_EQ(true, wasm_runtime_is_running_mode_supported( static_cast(Mode_Default))); - for (auto running_mode : running_mode_supportted) { + for (auto running_mode : running_mode_supported) { ASSERT_EQ(true, wasm_runtime_is_running_mode_supported(running_mode)); } @@ -264,7 +266,7 @@ TEST_F(wasm_running_modes_test_suite, wasm_runtime_set_default_running_mode) // normal situation: only set up ASSERT_EQ(true, wasm_runtime_set_default_running_mode( static_cast(Mode_Default))); - for (auto running_mode : running_mode_supportted) { + for (auto running_mode : running_mode_supported) { ASSERT_EQ(true, wasm_runtime_set_default_running_mode(running_mode)); } @@ -296,13 +298,13 @@ TEST_P(wasm_running_modes_test_suite, wasm_runtime_set_and_get_running_mode_complex) { RunningMode default_running_mode = GetParam(); - for (auto running_mode : running_mode_supportted) { + for (auto running_mode : running_mode_supported) { run_wasm_complex(WASM_FILE_1, WASM_FILE_2, default_running_mode, running_mode); } } INSTANTIATE_TEST_CASE_P(RunningMode, wasm_running_modes_test_suite, - testing::ValuesIn(running_mode_supportted)); + testing::ValuesIn(running_mode_supported)); -} \ No newline at end of file +} diff --git a/tests/unit/shared-heap/CMakeLists.txt b/tests/unit/shared-heap/CMakeLists.txt index 2b06c537f..fa7067918 100644 --- a/tests/unit/shared-heap/CMakeLists.txt +++ b/tests/unit/shared-heap/CMakeLists.txt @@ -12,12 +12,20 @@ set(WAMR_BUILD_AOT 1) set(WAMR_BUILD_INTERP 1) set(WAMR_BUILD_FAST_INTERP 1) set(WAMR_BUILD_JIT 0) -set(WAMR_BUILD_MEMORY64 1) +if(WAMR_BUILD_TARGET STREQUAL "X86_32") + set(WAMR_BUILD_MEMORY64 0) +else() + set(WAMR_BUILD_MEMORY64 1) +endif() set(WAMR_BUILD_SHARED_HEAP 1) # Compile wasm modules add_subdirectory(wasm-apps) +if (WAMR_BUILD_MEMORY64 EQUAL 1) + add_subdirectory(wasm-apps/memory64) +endif () + # if only load this CMake other than load it as subdirectory include(../unit_common.cmake) @@ -56,4 +64,4 @@ add_executable(shared_heap_test ${unit_test_sources}) target_link_libraries(shared_heap_test ${LLVM_AVAILABLE_LIBS} gtest_main) -gtest_discover_tests(shared_heap_test) \ No newline at end of file +gtest_discover_tests(shared_heap_test) diff --git a/tests/unit/shared-heap/shared_heap_test.cc b/tests/unit/shared-heap/shared_heap_test.cc index deb4bbb38..47d2eb3e8 100644 --- a/tests/unit/shared-heap/shared_heap_test.cc +++ b/tests/unit/shared-heap/shared_heap_test.cc @@ -9,6 +9,8 @@ #include "bh_read_file.h" #include "wasm_runtime_common.h" +#include + class shared_heap_test : public testing::Test { protected: @@ -26,50 +28,52 @@ struct ret_env { char error_buf[128]; }; -struct ret_env -load_wasm(char *wasm_file_tested, unsigned int app_heap_size) +static void +destroy_module_env(struct ret_env module_env); + +static bool +load_wasm(char *wasm_file_tested, unsigned int app_heap_size, + ret_env &ret_module_env) { - std::string wasm_mem_page = wasm_file_tested; - const char *wasm_file = strdup(wasm_mem_page.c_str()); - wasm_module_inst_t wasm_module_inst = nullptr; - wasm_module_t wasm_module = nullptr; - wasm_exec_env_t exec_env = nullptr; - unsigned char *wasm_file_buf = nullptr; + char *wasm_file = strdup(wasm_file_tested); unsigned int wasm_file_size = 0; unsigned int stack_size = 16 * 1024, heap_size = app_heap_size; char error_buf[128] = { 0 }; - struct ret_env ret_module_env; - memset(ret_module_env.error_buf, 0, 128); - wasm_file_buf = + ret_module_env.wasm_file_buf = (unsigned char *)bh_read_file_to_buffer(wasm_file, &wasm_file_size); - if (!wasm_file_buf) { + if (!ret_module_env.wasm_file_buf) { goto fail; } - wasm_module = wasm_runtime_load(wasm_file_buf, wasm_file_size, error_buf, - sizeof(error_buf)); - if (!wasm_module) { + ret_module_env.wasm_module = + wasm_runtime_load(ret_module_env.wasm_file_buf, wasm_file_size, + error_buf, sizeof(error_buf)); + if (!ret_module_env.wasm_module) { memcpy(ret_module_env.error_buf, error_buf, 128); goto fail; } - wasm_module_inst = wasm_runtime_instantiate( - wasm_module, stack_size, heap_size, error_buf, sizeof(error_buf)); - if (!wasm_module_inst) { + ret_module_env.wasm_module_inst = + wasm_runtime_instantiate(ret_module_env.wasm_module, stack_size, + heap_size, error_buf, sizeof(error_buf)); + if (!ret_module_env.wasm_module_inst) { memcpy(ret_module_env.error_buf, error_buf, 128); goto fail; } - exec_env = wasm_runtime_create_exec_env(wasm_module_inst, stack_size); + ret_module_env.exec_env = wasm_runtime_create_exec_env( + ret_module_env.wasm_module_inst, stack_size); + if (!ret_module_env.exec_env) { + goto fail; + } + free(wasm_file); + return true; fail: - ret_module_env.exec_env = exec_env; - ret_module_env.wasm_module = wasm_module; - ret_module_env.wasm_module_inst = wasm_module_inst; - ret_module_env.wasm_file_buf = wasm_file_buf; - - return ret_module_env; + free(wasm_file); + destroy_module_env(ret_module_env); + return false; } void @@ -92,46 +96,52 @@ destroy_module_env(struct ret_env module_env) } } -static void test_shared_heap(WASMSharedHeap *shared_heap, const char *file, const char *func_name, uint32 argc, uint32 argv[]) +static void +test_shared_heap(WASMSharedHeap *shared_heap, const char *file, + const char *func_name, uint32 argc, uint32 argv[]) { struct ret_env tmp_module_env; WASMFunctionInstanceCommon *func_test = nullptr; bool ret = false; - const char *exception = nullptr; - tmp_module_env = load_wasm((char *)file, 0); - - if (!wasm_runtime_attach_shared_heap(tmp_module_env.wasm_module_inst, shared_heap)) { - printf("Failed to attach shared heap\n"); - goto test_failed; + if (!load_wasm((char *)file, 0, tmp_module_env)) { + ADD_FAILURE() << "Failed to load wasm file\n"; + goto fail0; } + + if (!wasm_runtime_attach_shared_heap(tmp_module_env.wasm_module_inst, + shared_heap)) { + ADD_FAILURE() << "Failed to attach shared heap\n"; + goto fail1; + } + func_test = wasm_runtime_lookup_function(tmp_module_env.wasm_module_inst, func_name); if (!func_test) { - printf("\nFailed to wasm_runtime_lookup_function!\n"); - goto test_failed; + ADD_FAILURE() << "Failed to wasm_runtime_lookup_function!\n"; + goto fail2; } ret = wasm_runtime_call_wasm(tmp_module_env.exec_env, func_test, argc, argv); if (!ret) { - printf("\nFailed to wasm_runtime_call_wasm!\n"); - const char *s = wasm_runtime_get_exception(tmp_module_env.wasm_module_inst); - printf("exception: %s\n", s); - goto test_failed; + const char *s = + wasm_runtime_get_exception(tmp_module_env.wasm_module_inst); + ADD_FAILURE() << "Failed to wasm_runtime_call_wasm with " + << "exception: " << s; } +fail2: wasm_runtime_detach_shared_heap(tmp_module_env.wasm_module_inst); +fail1: destroy_module_env(tmp_module_env); +fail0: return; -test_failed: - destroy_module_env(tmp_module_env); - EXPECT_EQ(1, 0); } TEST_F(shared_heap_test, test_shared_heap_basic) { - SharedHeapInitArgs args; + SharedHeapInitArgs args = { 0 }; WASMSharedHeap *shared_heap = nullptr; uint32 argv[1] = { 0 }; @@ -139,23 +149,22 @@ TEST_F(shared_heap_test, test_shared_heap_basic) shared_heap = wasm_runtime_create_shared_heap(&args); if (!shared_heap) { - printf("Failed to create shared heap\n"); - EXPECT_EQ(1, 0); + FAIL() << "Failed to create shared heap"; } - // test wasm - test_shared_heap(shared_heap, "test.wasm", "test", 1, argv); + test_shared_heap(shared_heap, "test.wasm", "test", 0, argv); EXPECT_EQ(10, argv[0]); - // test aot - test_shared_heap(shared_heap, "test.aot", "test", 1, argv); + test_shared_heap(shared_heap, "test.aot", "test", 0, argv); EXPECT_EQ(10, argv[0]); + test_shared_heap(shared_heap, "test_chain.aot", "test", 0, argv); + EXPECT_EQ(10, argv[0]); } TEST_F(shared_heap_test, test_shared_heap_malloc_fail) { - SharedHeapInitArgs args; + SharedHeapInitArgs args = { 0 }; WASMSharedHeap *shared_heap = nullptr; uint32 argv[1] = { 0 }; @@ -163,78 +172,880 @@ TEST_F(shared_heap_test, test_shared_heap_malloc_fail) shared_heap = wasm_runtime_create_shared_heap(&args); if (!shared_heap) { - printf("Failed to create shared heap\n"); - EXPECT_EQ(1, 0); + FAIL() << "Failed to create shared heap"; } - // test wasm - test_shared_heap(shared_heap, "test.wasm", "test_malloc_fail", 1, argv); + test_shared_heap(shared_heap, "test.wasm", "test_malloc_fail", 0, argv); EXPECT_EQ(1, argv[0]); - // test aot - test_shared_heap(shared_heap, "test.aot", "test_malloc_fail", 1, argv); + test_shared_heap(shared_heap, "test.aot", "test_malloc_fail", 0, argv); + EXPECT_EQ(1, argv[0]); + + test_shared_heap(shared_heap, "test_chain.aot", "test_malloc_fail", 0, + argv); EXPECT_EQ(1, argv[0]); } -#ifndef native_function -#define native_function(func_name, signature) \ - { #func_name, (void *)glue_##func_name, signature, NULL } +TEST_F(shared_heap_test, test_preallocated_shared_heap_malloc_fail) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE]; + /* create a preallocated shared heap */ + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + /* test wasm can't malloc with preallocated shared heap */ + argv[0] = 1024; + test_shared_heap(shared_heap, "test.wasm", "my_shared_heap_malloc", 1, + argv); + EXPECT_EQ(0, argv[0]); + + argv[0] = 1024; + test_shared_heap(shared_heap, "test.aot", "my_shared_heap_malloc", 1, argv); + EXPECT_EQ(0, argv[0]); + + argv[0] = 1024; + test_shared_heap(shared_heap, "test_chain.aot", "my_shared_heap_malloc", 1, + argv); + EXPECT_EQ(0, argv[0]); +} + +static void +create_test_shared_heap(uint8 *preallocated_buf, size_t size, + WASMSharedHeap **shared_heap_res) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr; + args.pre_allocated_addr = preallocated_buf; + args.size = size; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + *shared_heap_res = shared_heap; + if (!*shared_heap_res) { + FAIL() << "Create shared heap chain failed.\n"; + } +} + +static void +create_test_shared_heap_chain(uint8 *preallocated_buf, size_t size, + uint8 *preallocated_buf2, size_t size2, + WASMSharedHeap **shared_heap_chain) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr; + args.pre_allocated_addr = preallocated_buf; + args.size = size; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf2; + args.size = size2; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + *shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!*shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } +} + +TEST_F(shared_heap_test, test_shared_heap_rmw) +{ + WASMSharedHeap *shared_heap = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }; + uint32 start1, end1; + + create_test_shared_heap(preallocated_buf, BUF_SIZE, &shared_heap); + + /* app addr for shared heap */ + start1 = UINT32_MAX - BUF_SIZE + 1; + end1 = UINT32_MAX; + + argv[0] = end1; + argv[1] = 101; + test_shared_heap(shared_heap, "test.wasm", "read_modify_write_8", 2, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + argv[0] = start1; + argv[1] = 37; + test_shared_heap(shared_heap, "test.wasm", "read_modify_write_8", 2, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[0], 37); + + argv[0] = end1; + argv[1] = 81; + test_shared_heap(shared_heap, "test.aot", "read_modify_write_8", 2, argv); + EXPECT_EQ(101, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 81); + + argv[0] = start1; + argv[1] = 98; + test_shared_heap(shared_heap, "test.aot", "read_modify_write_8", 2, argv); + EXPECT_EQ(37, argv[0]); + EXPECT_EQ(preallocated_buf[0], 98); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; + + /* shared heap 1 */ + argv[0] = end1; + argv[1] = 101; + test_shared_heap(shared_heap_chain, "test.wasm", "read_modify_write_8", 2, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + /* shared heap 2 */ + argv[0] = start2; + argv[1] = 129; + test_shared_heap(shared_heap_chain, "test.wasm", "read_modify_write_8", 2, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf2[0], 129); + + argv[0] = start1; + argv[1] = 98; + test_shared_heap(shared_heap_chain, "test_chain.aot", "read_modify_write_8", + 2, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[0], 98); + + argv[0] = end2; + argv[1] = 81; + test_shared_heap(shared_heap_chain, "test_chain.aot", "read_modify_write_8", + 2, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 81); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; + + argv[0] = end1; + argv[1] = 101; + argv[2] = 1; + test_shared_heap(shared_heap_chain, "test_bulk_memory.wasm", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(end1, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + argv[0] = start1; + argv[1] = 14; + argv[2] = 1; + test_shared_heap(shared_heap_chain, "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(start1, argv[0]); + EXPECT_EQ(preallocated_buf[0], 14); + + /* nothing happen when memory fill 0 byte */ + argv[0] = start2; + argv[1] = 68; + argv[2] = 0; + test_shared_heap(shared_heap_chain, "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(start2, argv[0]); + EXPECT_EQ(preallocated_buf2[0], 0); + + argv[0] = end2; + argv[1] = 98; + argv[2] = 1; + test_shared_heap(shared_heap_chain, "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(end2, argv[0]); + EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 98); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory_oob) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; + + /* shared heap 1 */ + argv[0] = end1; + argv[1] = 101; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = end2; + argv[1] = 98; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start1; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start2; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = end1; + argv[1] = 101; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = end2; + argv[1] = 98; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start1; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start2; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); +} + +TEST_F(shared_heap_test, test_shared_heap_rmw_oob) +{ + WASMSharedHeap *shared_heap = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + uint32 start1, end1, start2, end2; + + create_test_shared_heap(preallocated_buf, BUF_SIZE, &shared_heap); + + /* app addr for shared heap */ + start1 = UINT32_MAX - BUF_SIZE + 1; + end1 = UINT32_MAX; + + /* try to rmw an u16, first u8 is in the first shared heap and second u8 is + * in the second shared heap, will be seen as oob */ + argv[0] = end1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test.wasm", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); + + argv[0] = start1 - 1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test.aot", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); + + argv[0] = end1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test.aot", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) +{ + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; + + /* try to rmw an u16, first u8 is in the first shared heap and second u8 is + * in the second shared heap, will be seen as oob */ + argv[0] = end2; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test.wasm", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); + + argv[0] = end1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_chain.aot", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); +} + +#if WASM_ENABLE_MEMORY64 != 0 +TEST_F(shared_heap_test, test_shared_heap_chain_memory64_rmw) +{ + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint64 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT64_MAX - 2 * BUF_SIZE + 1; + end1 = UINT64_MAX - BUF_SIZE; + start2 = UINT64_MAX - BUF_SIZE + 1; + end2 = UINT64_MAX; + + /* shared heap 1 */ + PUT_I64_TO_ADDR(argv, end1); + argv[2] = 101; + test_shared_heap(shared_heap_chain, "test64.wasm", "read_modify_write_8", 3, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + /* shared heap 2 */ + PUT_I64_TO_ADDR(argv, start2); + argv[2] = 129; + test_shared_heap(shared_heap_chain, "test64.wasm", "read_modify_write_8", 3, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf2[0], 129); + + PUT_I64_TO_ADDR(argv, start1); + argv[2] = 98; + test_shared_heap(shared_heap_chain, "test64_chain.aot", + "read_modify_write_8", 3, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[0], 98); + + PUT_I64_TO_ADDR(argv, end2); + argv[2] = 81; + test_shared_heap(shared_heap_chain, "test64_chain.aot", + "read_modify_write_8", 3, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 81); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_memory64_rmw_oob) +{ + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + uint64 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT64_MAX - 2 * BUF_SIZE + 1; + end1 = UINT64_MAX - BUF_SIZE; + start2 = UINT64_MAX - BUF_SIZE + 1; + end2 = UINT64_MAX; + + /* try to rmw an u16, first u8 is in the first shared heap and second u8 is + * in the second shared heap, will be seen as oob */ + PUT_I64_TO_ADDR(argv, end1); + argv[2] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test64.wasm", + "read_modify_write_16", 3, argv), + "Exception: out of bounds memory access"); + + PUT_I64_TO_ADDR(argv, end1); + argv[2] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test64_chain.aot", + "read_modify_write_16", 3, argv), + "Exception: out of bounds memory access"); +} +#endif + +#ifndef native_function +/* clang-format off */ +#define native_function(func_name, signature) \ + { #func_name, (void *)glue_## func_name, signature, NULL } +/* clang-format on */ #endif #ifndef nitems #define nitems(_a) (sizeof(_a) / sizeof(0 [(_a)])) #endif /* nitems */ -uintptr_t glue_test_addr_conv(wasm_exec_env_t env, uintptr_t addr) +uintptr_t +glue_test_addr_conv(wasm_exec_env_t env, uintptr_t addr) { - wasm_module_inst_t module_inst = get_module_inst(env); - uintptr_t ret; - void *native_addr = (void *)addr; - uintptr_t app_addr = addr_native_to_app(native_addr); + wasm_module_inst_t module_inst = get_module_inst(env); + void *native_addr = (void *)addr; + uintptr_t app_addr = addr_native_to_app(native_addr); - native_addr = addr_app_to_native(app_addr); - if (native_addr != (void *)addr) - { - EXPECT_EQ(1, 0); - } - return app_addr; + native_addr = addr_app_to_native(app_addr); + if (native_addr != (void *)addr) { + ADD_FAILURE() << "address conversion incorrect"; + return 0; + } + return app_addr; } -static NativeSymbol g_test_native_symbols[] = -{ - native_function(test_addr_conv,"(*)i"), +static NativeSymbol g_test_native_symbols[] = { + native_function(test_addr_conv, "(*)i"), }; TEST_F(shared_heap_test, test_addr_conv) { - SharedHeapInitArgs args; + SharedHeapInitArgs args = { 0 }; WASMSharedHeap *shared_heap = nullptr; uint32 argv[1] = { 0 }; - struct ret_env tmp_module_env; - WASMFunctionInstanceCommon *func_test = nullptr; bool ret = false; - const char *exception = nullptr; - wasm_module_inst_t module_inst = tmp_module_env.wasm_module_inst; ret = wasm_native_register_natives("env", g_test_native_symbols, nitems(g_test_native_symbols)); - if (!ret) - { - EXPECT_EQ(1, 0); - return; + if (!ret) { + FAIL() << "Failed to register natives"; } args.size = 1024; shared_heap = wasm_runtime_create_shared_heap(&args); if (!shared_heap) { - printf("Failed to create shared heap\n"); - EXPECT_EQ(1, 0); + FAIL() << "Failed to create shared heap"; } - // test wasm - test_shared_heap(shared_heap, "test_addr_conv.wasm", "test", 1, argv); + test_shared_heap(shared_heap, "test_addr_conv.wasm", "test", 0, argv); EXPECT_EQ(1, argv[0]); - // test aot - test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); + test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 0, argv); + EXPECT_EQ(1, argv[0]); + + test_shared_heap(shared_heap, "test_addr_conv_chain.aot", "test", 0, argv); EXPECT_EQ(1, argv[0]); } + +TEST_F(shared_heap_test, test_addr_conv_pre_allocated_oob) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(), + app_addr = 0xFFFFFFFF - BUF_SIZE; + uint8 preallocated_buf[BUF_SIZE]; + bool ret = false; + + /* create a preallocated shared heap */ + ret = wasm_native_register_natives("env", g_test_native_symbols, + nitems(g_test_native_symbols)); + if (!ret) { + FAIL() << "Failed to register natives"; + } + + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + argv[0] = app_addr; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test_addr_conv.wasm", + "test_preallocated", 1, argv), + "Exception: out of bounds memory access"); + + argv[0] = app_addr; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test_addr_conv.aot", + "test_preallocated", 1, argv), + "Exception: out of bounds memory access"); + + argv[0] = app_addr; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, + "test_addr_conv_chain.aot", + "test_preallocated", 1, argv), + "Exception: out of bounds memory access"); +} + +TEST_F(shared_heap_test, test_shared_heap_chain) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap_chain = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE]; + bool ret = false; + + ret = wasm_native_register_natives("env", g_test_native_symbols, + nitems(g_test_native_symbols)); + if (!ret) { + FAIL() << "Failed to register natives"; + } + + args.size = 1024; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + /* create a preallocated shared heap */ + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } + + test_shared_heap(shared_heap_chain, "test_addr_conv.wasm", "test", 0, argv); + EXPECT_EQ(1, argv[0]); + + test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 0, argv); + EXPECT_EQ(1, argv[0]); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_create_fail) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap_chain = nullptr; + + args.size = 1024; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + args.size = 4096; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + EXPECT_EQ(shared_heap_chain, nullptr); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_create_fail2) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap_chain = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE]; + struct ret_env tmp_module_env; + + args.size = 1024; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + if (!load_wasm((char *)"test.wasm", 0, tmp_module_env)) { + FAIL() << "Failed to load wasm file\n"; + } + + if (!wasm_runtime_attach_shared_heap(tmp_module_env.wasm_module_inst, + shared_heap)) { + FAIL() << "Failed to attach shared heap\n"; + } + + /* can't create shared heap chain when shared heap is attached to a wasm + * app */ + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + EXPECT_EQ(shared_heap_chain, nullptr); + + wasm_runtime_detach_shared_heap(tmp_module_env.wasm_module_inst); + destroy_module_env(tmp_module_env); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_create_fail3) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap3 = nullptr, *shared_heap_chain = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + + args.size = 1024; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf2; + args.size = BUF_SIZE; + shared_heap3 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap3) { + FAIL() << "Failed to create shared heap"; + } + + /* The head and body can't be already in other shared heap chain as body */ + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap3, shared_heap2); + EXPECT_EQ(shared_heap_chain, nullptr); + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap2, shared_heap); + EXPECT_EQ(shared_heap_chain, nullptr); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_unchain) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap3 = nullptr, *shared_heap_chain = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + + args.size = 1024; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } + + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf2; + args.size = BUF_SIZE; + shared_heap3 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap3) { + FAIL() << "Failed to create shared heap"; + } + + /* unchain shared heap so that the 'body' can be another chain 'body' + * again(1->2 to 1->3->2) */ + EXPECT_EQ(shared_heap2, + wasm_runtime_unchain_shared_heaps(shared_heap_chain, false)); + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap3, shared_heap2); + EXPECT_EQ(shared_heap_chain, shared_heap3); + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap3); + EXPECT_EQ(shared_heap, shared_heap_chain); + + /* break down the entire shared heap chain */ + EXPECT_EQ(shared_heap2, + wasm_runtime_unchain_shared_heaps(shared_heap_chain, true)); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap_chain = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE]; + bool ret = false; + + ret = wasm_native_register_natives("env", g_test_native_symbols, + nitems(g_test_native_symbols)); + if (!ret) { + FAIL() << "Failed to register natives"; + } + + args.size = 4096; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + /* create a preallocated shared heap */ + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } + + argv[0] = 0xFFFFFFFF; + test_shared_heap(shared_heap_chain, "test_addr_conv.wasm", + "test_preallocated", 1, argv); + EXPECT_EQ(1, argv[0]); + + argv[0] = 0xFFFFF000; + test_shared_heap(shared_heap_chain, "test_addr_conv.wasm", + "test_preallocated", 1, argv); + EXPECT_EQ(1, argv[0]); + + argv[0] = 0xFFFFFFFF; + test_shared_heap(shared_heap, "test_addr_conv_chain.aot", + "test_preallocated", 1, argv); + EXPECT_EQ(1, argv[0]); + + argv[0] = 0xFFFFF000; + test_shared_heap(shared_heap, "test_addr_conv_chain.aot", + "test_preallocated", 1, argv); + EXPECT_EQ(1, argv[0]); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv_oob) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, + *shared_heap_chain = nullptr; + uint32 argv[1] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE]; + bool ret = false; + + ret = wasm_native_register_natives("env", g_test_native_symbols, + nitems(g_test_native_symbols)); + if (!ret) { + FAIL() << "Failed to register natives"; + } + + args.size = 4096; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Failed to create shared heap"; + } + + /* create a preallocated shared heap */ + memset(&args, 0, sizeof(args)); + args.pre_allocated_addr = preallocated_buf; + args.size = BUF_SIZE; + shared_heap2 = wasm_runtime_create_shared_heap(&args); + if (!shared_heap2) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + shared_heap_chain = + wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); + if (!shared_heap_chain) { + FAIL() << "Create shared heap chain failed.\n"; + } + + /* test wasm */ + argv[0] = 0xFFFFFFFF - BUF_SIZE - 4096; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_addr_conv.wasm", + "test_preallocated", 1, argv), + "Exception: out of bounds memory access"); + + /* test aot */ + argv[0] = 0xFFFFFFFF - BUF_SIZE - 4096; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_addr_conv_chain.aot", + "test_preallocated", 1, argv), + "Exception: out of bounds memory access"); +} diff --git a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt index 097f66ae5..985cf18ae 100644 --- a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt +++ b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt @@ -29,44 +29,81 @@ set(CMAKE_EXE_LINKER_FLAGS -Wl,--allow-undefined" ) +if (WAMR_BUILD_TARGET STREQUAL "X86_32") + set (WAMR_COMPILER_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-heap --target=i386) + set (WAMR_COMPILER_CHAIN_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-chain --target=i386) +else () + set (WAMR_COMPILER_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-heap) + set (WAMR_COMPILER_CHAIN_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-chain) +endif () + +function(copy_wasm TARGET_NAME) + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMENT "Copy ${TARGET_NAME} to the same directory of google test" + ) +endfunction() + +function(compile_and_copy_aot_from TARGET_NAME) + string(REPLACE ".wasm" ".aot" AOT_TARGET ${TARGET_NAME}) + string(REPLACE ".wasm" "_chain.aot" AOT_CHAIN_TARGET ${TARGET_NAME}) + + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} + -o ${AOT_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_CHAIN_FLAGS} + -o ${AOT_CHAIN_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_CHAIN_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMENT "Compile and copy ${AOT_TARGET} to the same directory of google test" + ) +endfunction() + add_executable(test.wasm test.c) target_link_libraries(test.wasm) - -add_custom_command(TARGET test.wasm POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test.wasm - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test.wasm to the same directory of google test" - ) - -add_custom_command(TARGET test.wasm POST_BUILD - COMMAND ${WAMRC_ROOT_DIR}/wamrc --opt-level=0 --enable-shared-heap --bounds-checks=1 - -o - test.aot - test.wasm - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test.aot - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test.aot to the same directory of google test" - ) +copy_wasm(test.wasm) +compile_and_copy_aot_from(test.wasm) add_executable(test_addr_conv.wasm test_addr_conv.c) -target_link_libraries(test.wasm) +target_link_libraries(test_addr_conv.wasm) +copy_wasm(test_addr_conv.wasm) +compile_and_copy_aot_from(test_addr_conv.wasm) -add_custom_command(TARGET test_addr_conv.wasm POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test_addr_conv.wasm - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test_addr_conv.wasm to the same directory of google test" - ) +# copy and compile aot for bulk memory test +set(SOURCE_WASM ${CMAKE_CURRENT_SOURCE_DIR}/bulk-memory/test_bulk_memory.wasm) +set(BUILD_WASM ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory.wasm) +set(OUTPUT_AOT ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory.aot) +set(OUTPUT_CHAIN_AOT ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory_chain.aot) -add_custom_command(TARGET test_addr_conv.wasm POST_BUILD - COMMAND ${WAMRC_ROOT_DIR}/wamrc --opt-level=0 --enable-shared-heap --bounds-checks=1 - -o - test_addr_conv.aot - test_addr_conv.wasm - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test_addr_conv.aot - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test_addr_conv.aot to the same directory of google test" - ) +add_custom_command( + OUTPUT ${BUILD_WASM} + COMMAND ${CMAKE_COMMAND} -E copy + ${SOURCE_WASM} + ${BUILD_WASM} + DEPENDS ${SOURCE_WASM} + COMMENT "Copying bulk memory WASM to build directory" +) + +add_custom_command( + OUTPUT ${OUTPUT_AOT} + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} + -o ${OUTPUT_AOT} + ${BUILD_WASM} + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_CHAIN_FLAGS} + -o ${OUTPUT_CHAIN_AOT} + ${BUILD_WASM} + DEPENDS ${BUILD_WASM} + COMMENT "Compiling bulk memory AOT from copied WASM" +) + +add_custom_target(compile_bulk_memory_aot ALL + DEPENDS ${OUTPUT_AOT} +) diff --git a/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wasm b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wasm new file mode 100644 index 0000000000000000000000000000000000000000..eb3d60be3da992383bd142c229388824ba10439d GIT binary patch literal 63 zcmZQbEY4+QU|?WmXG~zOudiodW@2PuWo85lh%gG|rsn1sRmP`f=H$eeq!yPjFmUlQ Rax*9}C@?B8{o!Wd1^{Vt3%&pV literal 0 HcmV?d00001 diff --git a/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat new file mode 100644 index 000000000..e7a0c684d --- /dev/null +++ b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat @@ -0,0 +1,12 @@ +(module + (memory 1) + + (func $memory_fill_test (param $dst i32) (param $val i32) (param $len i32) + local.get $dst + local.get $val + local.get $len + memory.fill + ) + + (export "memory_fill_test" (func $memory_fill_test)) +) diff --git a/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt new file mode 100644 index 000000000..a82788b58 --- /dev/null +++ b/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt @@ -0,0 +1,68 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(wasm-apps-wasm64) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../../..) +set(WAMRC_ROOT_DIR ${WAMR_ROOT_DIR}/wamr-compiler/build) + +set(CMAKE_SYSTEM_PROCESSOR wasm64) +set(CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set(WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set(CMAKE_C_FLAGS "-nostdlib -pthread -Qunused-arguments") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -z stack-size=8192 -nostdlib -O0 --target=wasm64") +set(CMAKE_C_COMPILER_TARGET "wasm64") +set(CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +set(DEFINED_SYMBOLS + "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") + +set(CMAKE_EXE_LINKER_FLAGS + "-Wl,--no-entry \ + -Wl,--initial-memory=65536 \ + -Wl,--export-all \ + -Wl,--allow-undefined" + ) + +set (WAMR_COMPILER_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-heap) +set (WAMR_COMPILER_CHAIN_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-chain) + +function(copy_wasm TARGET_NAME) + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} + ${CMAKE_CURRENT_BINARY_DIR}/../../ + COMMENT "Copy ${TARGET_NAME} to the same directory of google test" + ) +endfunction() + +function(compile_and_copy_aot_from TARGET_NAME) + string(REPLACE ".wasm" ".aot" AOT_TARGET ${TARGET_NAME}) + string(REPLACE ".wasm" "_chain.aot" AOT_CHAIN_TARGET ${TARGET_NAME}) + + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} + -o ${AOT_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../../ + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_CHAIN_FLAGS} + -o ${AOT_CHAIN_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_CHAIN_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../../ + COMMENT "Compile and copy ${AOT_TARGET} ${AOT_CHAIN_TARGET} to the same directory of google test" + ) +endfunction() + +add_executable(test64.wasm ../test.c) +target_link_libraries(test64.wasm) +copy_wasm(test64.wasm) +compile_and_copy_aot_from(test64.wasm) diff --git a/tests/unit/shared-heap/wasm-apps/test.c b/tests/unit/shared-heap/wasm-apps/test.c index bd0df19c2..66df21c1b 100644 --- a/tests/unit/shared-heap/wasm-apps/test.c +++ b/tests/unit/shared-heap/wasm-apps/test.c @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#include +#define NULL 0 extern void * shared_heap_malloc(int size); @@ -32,3 +32,31 @@ test_malloc_fail() shared_heap_free(ptr); return 0; } + +void * +my_shared_heap_malloc(int size) +{ + return shared_heap_malloc(size); +} + +void +my_shared_heap_free(void *addr) +{ + shared_heap_free(addr); +} + +char +read_modify_write_8(char *addr, char value) +{ + char original_value = *addr; + *addr = value; + return original_value; +} + +short +read_modify_write_16(short *addr, short value) +{ + short original_value = *addr; + *addr = value; + return original_value; +} diff --git a/tests/unit/shared-heap/wasm-apps/test_addr_conv.c b/tests/unit/shared-heap/wasm-apps/test_addr_conv.c index f91764c84..5e64526a0 100644 --- a/tests/unit/shared-heap/wasm-apps/test_addr_conv.c +++ b/tests/unit/shared-heap/wasm-apps/test_addr_conv.c @@ -30,3 +30,17 @@ test() shared_heap_free(ptr); return 1; } + +int +test_preallocated(void *app_addr) +{ + int *ptr = (int *)app_addr; + int *ptr2 = NULL; + + ptr2 = test_addr_conv(ptr); + if (ptr2 != ptr) { + return 0; + } + + return 1; +} diff --git a/tests/wamr-test-suites/spec-test-script/all.py b/tests/wamr-test-suites/spec-test-script/all.py index 2c4725e89..970127d0b 100644 --- a/tests/wamr-test-suites/spec-test-script/all.py +++ b/tests/wamr-test-suites/spec-test-script/all.py @@ -172,6 +172,7 @@ def test_case( clean_up_flag=True, verbose_flag=True, gc_flag=False, + extended_const_flag=False, memory64_flag=False, multi_memory_flag=False, qemu_flag=False, @@ -229,6 +230,9 @@ def test_case( if gc_flag: CMD.append("--gc") + if extended_const_flag: + CMD.append("--extended-const") + if memory64_flag: CMD.append("--memory64") @@ -304,6 +308,7 @@ def test_suite( clean_up_flag=True, verbose_flag=True, gc_flag=False, + extended_const_flag=False, memory64_flag=False, multi_memory_flag=False, parl_flag=False, @@ -385,6 +390,7 @@ def test_suite( clean_up_flag, verbose_flag, gc_flag, + extended_const_flag, memory64_flag, multi_memory_flag, qemu_flag, @@ -428,6 +434,7 @@ def test_suite( clean_up_flag, verbose_flag, gc_flag, + extended_const_flag, memory64_flag, multi_memory_flag, qemu_flag, @@ -561,6 +568,13 @@ def main(): dest="gc_flag", help="Running with GC feature", ) + parser.add_argument( + "--enable-extended-const", + action="store_true", + default=False, + dest="extended_const_flag", + help="Running with extended const expression feature", + ) parser.add_argument( "--memory64", action="store_true", @@ -619,6 +633,7 @@ def main(): options.clean_up_flag, options.verbose_flag, options.gc_flag, + options.extended_const_flag, options.memory64_flag, options.multi_memory_flag, options.parl_flag, @@ -648,6 +663,7 @@ def main(): options.clean_up_flag, options.verbose_flag, options.gc_flag, + options.extended_const_flag, options.memory64_flag, options.multi_memory_flag, options.qemu_flag, diff --git a/tests/wamr-test-suites/spec-test-script/extended_const.patch b/tests/wamr-test-suites/spec-test-script/extended_const.patch new file mode 100644 index 000000000..f09427c29 --- /dev/null +++ b/tests/wamr-test-suites/spec-test-script/extended_const.patch @@ -0,0 +1,506 @@ +diff --git a/test/core/elem.wast b/test/core/elem.wast +index 92dab52..3954bca 100644 +--- a/test/core/elem.wast ++++ b/test/core/elem.wast +@@ -571,6 +571,7 @@ + + ;; Element sections across multiple modules change the same table + ++(; + (module $module1 + (type $out-i32 (func (result i32))) + (table (export "shared-table") 10 funcref) +@@ -620,7 +621,7 @@ + (assert_return (invoke $module1 "call-7") (i32.const 67)) + (assert_return (invoke $module1 "call-8") (i32.const 69)) + (assert_return (invoke $module1 "call-9") (i32.const 70)) +- ++;) + ;; Element segments must match element type of table + + (assert_invalid +@@ -659,24 +660,30 @@ + (func (export "set") (param $i i32) (param $x externref) + (table.set $t (local.get $i) (local.get $x)))) + +-(register "exporter" $m) ++;; (register "exporter" $m) + +-(assert_return (invoke $m "get" (i32.const 0)) (ref.null extern)) +-(assert_return (invoke $m "get" (i32.const 1)) (ref.null extern)) ++;; (assert_return (invoke $m "get" (i32.const 0)) (ref.null extern)) ++;; (assert_return (invoke $m "get" (i32.const 1)) (ref.null extern)) ++(assert_return (invoke "get" (i32.const 0)) (ref.null extern)) ++(assert_return (invoke "get" (i32.const 1)) (ref.null extern)) + +-(assert_return (invoke $m "set" (i32.const 0) (ref.extern 42))) +-(assert_return (invoke $m "set" (i32.const 1) (ref.extern 137))) +- +-(assert_return (invoke $m "get" (i32.const 0)) (ref.extern 42)) +-(assert_return (invoke $m "get" (i32.const 1)) (ref.extern 137)) ++;; (assert_return (invoke $m "set" (i32.const 0) (ref.extern 42))) ++;; (assert_return (invoke $m "set" (i32.const 1) (ref.extern 137))) ++(assert_return (invoke "set" (i32.const 0) (ref.extern 42))) ++(assert_return (invoke "set" (i32.const 1) (ref.extern 137))) + ++;; (assert_return (invoke $m "get" (i32.const 0)) (ref.extern 42)) ++;; (assert_return (invoke $m "get" (i32.const 1)) (ref.extern 137)) ++(assert_return (invoke "get" (i32.const 0)) (ref.extern 42)) ++(assert_return (invoke "get" (i32.const 1)) (ref.extern 137)) ++(; + (module + (import "exporter" "table" (table $t 2 externref)) + (elem (i32.const 0) externref (ref.null extern))) + + (assert_return (invoke $m "get" (i32.const 0)) (ref.null extern)) + (assert_return (invoke $m "get" (i32.const 1)) (ref.extern 137)) +- ++;) + ;; Initializing a table with imported funcref global + + (module $module4 +@@ -686,6 +693,7 @@ + (global (export "f") funcref (ref.func 0)) + ) + ++(; + (register "module4" $module4) + + (module +@@ -699,6 +707,7 @@ + ) + + (assert_return (invoke "call_imported_elem") (i32.const 42)) ++;) + + ;; Extended contant expressions + +diff --git a/test/core/ref_func.wast b/test/core/ref_func.wast +index adb5cb7..6396013 100644 +--- a/test/core/ref_func.wast ++++ b/test/core/ref_func.wast +@@ -4,7 +4,7 @@ + (register "M") + + (module +- (func $f (import "M" "f") (param i32) (result i32)) ++ (func $f (param $x i32) (result i32) (local.get $x)) + (func $g (param $x i32) (result i32) + (i32.add (local.get $x) (i32.const 1)) + ) +diff --git a/test/core/table_copy.wast b/test/core/table_copy.wast +index 380e84e..59230cf 100644 +--- a/test/core/table_copy.wast ++++ b/test/core/table_copy.wast +@@ -14,11 +14,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -106,11 +106,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -198,11 +198,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -290,11 +290,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -382,11 +382,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -474,11 +474,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -566,11 +566,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -658,11 +658,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -750,11 +750,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -842,11 +842,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -934,11 +934,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1026,11 +1026,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1118,11 +1118,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1210,11 +1210,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1302,11 +1302,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1394,11 +1394,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1486,11 +1486,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -1578,11 +1578,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +diff --git a/test/core/table_init.wast b/test/core/table_init.wast +index 0b2d26f..3c595e5 100644 +--- a/test/core/table_init.wast ++++ b/test/core/table_init.wast +@@ -14,11 +14,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -72,11 +72,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -130,11 +130,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t0) (i32.const 2) func 3 1 4 1) +@@ -196,11 +196,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -254,11 +254,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) +@@ -312,11 +312,11 @@ + + (module + (type (func (result i32))) ;; type #0 +- (import "a" "ef0" (func (result i32))) ;; index 0 +- (import "a" "ef1" (func (result i32))) +- (import "a" "ef2" (func (result i32))) +- (import "a" "ef3" (func (result i32))) +- (import "a" "ef4" (func (result i32))) ;; index 4 ++ (func (result i32) (i32.const 0)) ;; index 0 ++ (func (result i32) (i32.const 1)) ++ (func (result i32) (i32.const 2)) ++ (func (result i32) (i32.const 3)) ++ (func (result i32) (i32.const 4)) ;; index 4 + (table $t0 30 30 funcref) + (table $t1 30 30 funcref) + (elem (table $t1) (i32.const 2) func 3 1 4 1) diff --git a/tests/wamr-test-suites/spec-test-script/runtest.py b/tests/wamr-test-suites/spec-test-script/runtest.py index 158d759ed..fa9f5eb7d 100755 --- a/tests/wamr-test-suites/spec-test-script/runtest.py +++ b/tests/wamr-test-suites/spec-test-script/runtest.py @@ -336,6 +336,9 @@ parser.add_argument('--multi-thread', default=False, action='store_true', parser.add_argument('--gc', default=False, action='store_true', help='Test with GC') +parser.add_argument('--extended-const', action='store_true', + help='Enable extended const expression feature') + parser.add_argument('--memory64', default=False, action='store_true', help='Test with Memory64') @@ -1112,6 +1115,8 @@ def compile_wast_to_wasm(form, wast_tempfile, wasm_tempfile, opts): cmd = [opts.wast2wasm, "--enable-memory64", "--no-check", wast_tempfile, "-o", wasm_tempfile ] elif opts.multi_memory: cmd = [opts.wast2wasm, "--enable-multi-memory", "--no-check", wast_tempfile, "-o", wasm_tempfile ] + elif opts.extended_const: + cmd = [opts.wast2wasm, "--enable-extended-const", "--no-check", wast_tempfile, "-o", wasm_tempfile ] else: # `--enable-multi-memory` for a case in memory.wast but doesn't require runtime support cmd = [opts.wast2wasm, "--enable-multi-memory", "--enable-threads", "--no-check", @@ -1155,6 +1160,9 @@ def compile_wasm_to_aot(wasm_tempfile, aot_tempfile, runner, opts, r, output = ' cmd.append("--enable-gc") cmd.append("--enable-tail-call") + if opts.extended_const: + cmd.append("--enable-extended-const") + if output == 'object': cmd.append("--format=object") elif output == 'ir': diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index 8a72ae201..6f498653f 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -41,6 +41,7 @@ function help() echo "-j set the platform to test" echo "-T set sanitizer to use in tests(ubsan|tsan|asan|posan)" echo "-A use the specified wamrc command instead of building it" + echo "-N enable extended const expression feature" echo "-r [requirement name] [N [N ...]] specify a requirement name followed by one or more" echo " subrequirement IDs, if no subrequirement is specificed," echo " it will run all subrequirements. When this optin is used," @@ -59,6 +60,7 @@ ENABLE_MULTI_THREAD=0 COLLECT_CODE_COVERAGE=0 ENABLE_SIMD=0 ENABLE_GC=0 +ENABLE_EXTENDED_CONST_EXPR=0 ENABLE_MEMORY64=0 ENABLE_MULTI_MEMORY=0 ENABLE_XIP=0 @@ -87,7 +89,7 @@ REQUIREMENT_NAME="" # Initialize an empty array for subrequirement IDs SUBREQUIREMENT_IDS=() -while getopts ":s:cabgvt:m:MCpSXexwWEPGQF:j:T:r:A:" opt +while getopts ":s:cabgvt:m:MCpSXexwWEPGQF:j:T:r:A:N" opt do OPT_PARSED="TRUE" case $opt in @@ -191,6 +193,10 @@ do echo "enable GC feature" ENABLE_GC=1 ;; + N) + echo "enable extended const expression feature" + ENABLE_EXTENDED_CONST_EXPR=1 + ;; P) PARALLELISM=1 ;; @@ -414,7 +420,7 @@ function setup_wabt() function compile_reference_interpreter() { - echo "compile the reference intepreter" + echo "compile the reference interpreter" pushd interpreter make if [ $? -ne 0 ] @@ -485,6 +491,17 @@ function spec_test() # (func $f (param (ref null $t)) (result funcref) (local.get 0)) # compile_reference_interpreter + elif [[ ${ENABLE_EXTENDED_CONST_EXPR} == 1 ]]; then + echo "checkout spec for extended const expression proposal" + + git clone -b main --single-branch https://github.com/WebAssembly/extended-const.git spec + pushd spec + + # Jan 14, 2025. README.md: Add note that this proposal is done (#20) + git reset --hard 8d4f6aa2b00a8e7c0174410028625c6a176db8a1 + # ignore import table cases + git apply --ignore-whitespace ../../spec-test-script/extended_const.patch || exit 1 + elif [[ ${ENABLE_MEMORY64} == 1 ]]; then echo "checkout spec for memory64 proposal" @@ -587,6 +604,10 @@ function spec_test() ARGS_FOR_SPEC_TEST+="--gc " fi + if [[ ${ENABLE_EXTENDED_CONST_EXPR} == 1 ]]; then + ARGS_FOR_SPEC_TEST+="--enable-extended-const " + fi + if [[ 1 == ${ENABLE_MEMORY64} ]]; then ARGS_FOR_SPEC_TEST+="--memory64 " fi @@ -832,6 +853,7 @@ function build_wamrc() && cmake .. \ -DCOLLECT_CODE_COVERAGE=${COLLECT_CODE_COVERAGE} \ -DWAMR_BUILD_SHRUNK_MEMORY=0 \ + -DWAMR_BUILD_EXTENDED_CONST_EXPR=${ENABLE_EXTENDED_CONST_EXPR} \ && make -j 4 } @@ -1023,6 +1045,10 @@ function trigger() EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_TAIL_CALL=1" fi + if [[ ${ENABLE_EXTENDED_CONST_EXPR} == 1 ]]; then + EXTRA_COMPILE_FLAGS+=" -DWAMR_BUILD_EXTENDED_CONST_EXPR=1" + fi + if [[ ${ENABLE_DEBUG_VERSION} == 1 ]]; then EXTRA_COMPILE_FLAGS+=" -DCMAKE_BUILD_TYPE=Debug" fi diff --git a/wamr-compiler/CMakeLists.txt b/wamr-compiler/CMakeLists.txt index 0ce647394..8ee61cab4 100644 --- a/wamr-compiler/CMakeLists.txt +++ b/wamr-compiler/CMakeLists.txt @@ -53,6 +53,7 @@ add_definitions(-DWASM_ENABLE_PERF_PROFILING=1) add_definitions(-DWASM_ENABLE_LOAD_CUSTOM_SECTION=1) add_definitions(-DWASM_ENABLE_MODULE_INST_CONTEXT=1) add_definitions(-DWASM_ENABLE_MEMORY64=1) +add_definitions(-DWASM_ENABLE_EXTENDED_CONST_EXPR=1) add_definitions(-DWASM_ENABLE_GC=1) @@ -284,6 +285,7 @@ include (${IWASM_DIR}/interpreter/iwasm_interp.cmake) include (${IWASM_DIR}/aot/iwasm_aot.cmake) include (${IWASM_DIR}/compilation/iwasm_compl.cmake) include (${PROJECT_SOURCE_DIR}/../build-scripts/version.cmake) +include (${IWASM_DIR}/libraries/shared-heap/shared_heap.cmake) if (WAMR_BUILD_LIBC_BUILTIN EQUAL 1) include (${IWASM_DIR}/libraries/libc-builtin/libc_builtin.cmake) @@ -366,6 +368,7 @@ add_library (vmlib ${LIBC_WASI_SOURCE} ${LIB_PTHREAD_SOURCE} ${LIB_WASI_THREADS_SOURCE} + ${LIB_SHARED_HEAP_SOURCE} ${IWASM_COMMON_SOURCE} ${IWASM_INTERP_SOURCE} ${IWASM_AOT_SOURCE} diff --git a/wamr-compiler/build_llvm.sh b/wamr-compiler/build_llvm.sh index c3ec54b61..75285b52c 100755 --- a/wamr-compiler/build_llvm.sh +++ b/wamr-compiler/build_llvm.sh @@ -3,5 +3,17 @@ # Copyright (C) 2020 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/usr/bin/env python3 -m pip install --user -r ../build-scripts/requirements.txt +TEMP_DIR=$(mktemp -d) + +cleanup() { + local exit_code=$? + rm -rf "$TEMP_DIR" + exit $exit_code +} + +trap cleanup EXIT INT TERM + +/usr/bin/env python3 -m venv --clear "$TEMP_DIR" +source "$TEMP_DIR/bin/activate" +/usr/bin/env python3 -m pip install -r ../build-scripts/requirements.txt /usr/bin/env python3 ../build-scripts/build_llvm.py "$@" diff --git a/wamr-compiler/build_llvm_arc.sh b/wamr-compiler/build_llvm_arc.sh index d148e11ec..bc10216e1 100755 --- a/wamr-compiler/build_llvm_arc.sh +++ b/wamr-compiler/build_llvm_arc.sh @@ -3,5 +3,5 @@ # Copyright (C) 2020 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/usr/bin/env python3 -m pip install --user -r ../build-scripts/requirements.txt -/usr/bin/env python3 ../build-scripts/build_llvm.py --platform arc "$@" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +$SCRIPT_DIR/build_llvm.sh --platform arc "$@" diff --git a/wamr-compiler/build_llvm_xtensa.sh b/wamr-compiler/build_llvm_xtensa.sh index 183ea379f..29559ce35 100755 --- a/wamr-compiler/build_llvm_xtensa.sh +++ b/wamr-compiler/build_llvm_xtensa.sh @@ -3,5 +3,5 @@ # Copyright (C) 2020 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -/usr/bin/env python3 -m pip install --user -r ../build-scripts/requirements.txt -/usr/bin/env python3 ../build-scripts/build_llvm.py --platform xtensa "$@" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +$SCRIPT_DIR/build_llvm.sh --platform xtensa "$@" diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index c9c4ac5da..62b0dfb90 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -213,7 +213,9 @@ print_help() printf(" --enable-linux-perf Enable linux perf support\n"); #endif printf(" --mllvm=