From 76254183f9181f827c15f48948792b777c5f2ac0 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Thu, 21 Mar 2024 11:08:40 +0800 Subject: [PATCH 1/8] Make android platform's cmake flags configurable (#3239) Don't hardcode the cmake configurations in the Android platform's CMakeLists.txt. Fixes https://github.com/bytecodealliance/wasm-micro-runtime/issues/3238 --- .../compilation_on_android_ubuntu.yml | 23 ++++- product-mini/README.md | 13 ++- product-mini/platforms/android/CMakeLists.txt | 98 ++++++++++++------- 3 files changed, 92 insertions(+), 42 deletions(-) diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index f290df0c6..481b06676 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -202,11 +202,17 @@ jobs: make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" - # Fast-JIT and Multi-Tier-JIT mode don't support android(X86-32) + # Fast-JIT and Multi-Tier-JIT mode don't support android - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS platform: android - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS platform: android + # LLVM JIT pre-built binary wasn't compiled by Android NDK + # and isn't available for android + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + platform: android + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + platform: android include: - os: ubuntu-22.04 llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} @@ -232,13 +238,23 @@ jobs: if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true') run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - - name: Build iwasm + - name: Build iwasm for linux + if: matrix.platform == 'linux' run: | mkdir build && cd build cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} cmake --build . --config Release --parallel 4 working-directory: product-mini/platforms/${{ matrix.platform }} + - name: Build iwasm for android + if: matrix.platform == 'android' + run: | + mkdir build && cd build + cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} \ + -DWAMR_BUILD_TARGET=X86_64 + cmake --build . --config Release --parallel 4 + working-directory: product-mini/platforms/${{ matrix.platform }} + - name: Build and run unit tests run: | mkdir build-unittests && cd build-unittests @@ -456,7 +472,7 @@ jobs: cmake --build . --config Debug --parallel 4 ./iwasm wasm-apps/trap.wasm | grep "#" > call_stack.txt ./iwasm wasm-apps/trap.aot | grep "#" > call_stack_aot.txt - bash -x ../symbolicate.sh + bash -x ../symbolicate.sh test: needs: @@ -556,7 +572,6 @@ jobs: make -j AR=/opt/wasi-sdk/bin/llvm-ar NM=/opt/wasi-sdk/bin/llvm-nm CC=/opt/wasi-sdk/bin/clang THREAD_MODEL=posix echo "SYSROOT_PATH=$PWD/sysroot" >> $GITHUB_ENV - - name: set env variable(if llvm are used) if: matrix.running_mode == 'aot' || matrix.running_mode == 'jit' || matrix.running_mode == 'multi-tier-jit' run: echo "USE_LLVM=true" >> $GITHUB_ENV diff --git a/product-mini/README.md b/product-mini/README.md index 8d7e799a4..18813bef5 100644 --- a/product-mini/README.md +++ b/product-mini/README.md @@ -312,7 +312,7 @@ WAMR provides some features which can be easily configured by passing options to ## Android -able to generate a shared library support Android platform. +Able to generate a shared library support Android platform. - need an [android SDK](https://developer.android.com/studio). Go and get the "Command line tools only" - look for a command named *sdkmanager* and download below components. version numbers might need to check and pick others - "build-tools;29.0.3" @@ -326,7 +326,7 @@ able to generate a shared library support Android platform. - export ANDROID_NDK_LATEST_HOME=/the/path/of/downloaded/sdk/ndk/2x.xxx/ - ready to go -Use such commands, you are able to compile with default configurations. Any compiling requirement should be satisfied by modifying product-mini/platforms/android/CMakeList.txt. For example, chaning ${WAMR_BUILD_TARGET} in CMakeList could get different libraries support different ABIs. +Use such commands, you are able to compile with default configurations. ``` shell $ cd product-mini/platforms/android/ @@ -339,6 +339,15 @@ $ # include/ includes all necesary head files $ # lib includes libiwasm.so ``` +To change the target architecture and ABI, you can define `WAMR_BUILD_TARGET` or `ANDROID_ABI` respectively. To build for [supported Android ABIs](https://developer.android.com/ndk/guides/abis#sa): + +```shell +$ cmake .. -DWAMR_BUILD_TARGET=X86_32 -DANDROID_ABI=x86 # 32-bit Intel CPU +$ cmake .. -DWAMR_BUILD_TARGET=X86_64 -DANDROID_ABI=x86_64 # 64-bit Intel CPU +$ cmake .. -DWAMR_BUILD_TARGET=ARMV7A -DANDROID_ABI=armeabi-v7a # 32-bit ARM CPU +$ cmake .. -DWAMR_BUILD_TARGET=AARCH64 -DANDROID_ABI=arm64-v8a # 64-bit ARM CPU +``` + ## NuttX WAMR is intergrated with NuttX, just enable the WAMR in Kconfig option (Application Configuration/Interpreters). diff --git a/product-mini/platforms/android/CMakeLists.txt b/product-mini/platforms/android/CMakeLists.txt index 638b6ab0d..db60e8649 100644 --- a/product-mini/platforms/android/CMakeLists.txt +++ b/product-mini/platforms/android/CMakeLists.txt @@ -1,53 +1,59 @@ # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required (VERSION 3.4.1) - -set (CMAKE_VERBOSE_MAKEFILE on) -set (CMAKE_BUILD_TYPE Release) - -# https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-3 -set (CMAKE_TOOLCHAIN_FILE "$ENV{ANDROID_NDK_LATEST_HOME}/build/cmake/android.toolchain.cmake") -set (ANDROID_SDK $ENV{ANDROID_HOME}) -set (ANDROID_NDK $ENV{ANDROID_NDK_LATEST_HOME}) - -set (ANDROID_ABI "x86") -set (ANDROID_LD lld) -if (NOT DEFINED ANDROID_PLATFORM) - set (ANDROID_PLATFORM 24) -endif () - -project (iwasm) - -set (WAMR_BUILD_PLATFORM "android") -set (WAMR_BUILD_TARGET "X86_32") -set (WAMR_BUILD_TYPE Release) -set (WAMR_BUILD_INTERP 1) -set (WAMR_BUILD_AOT 1) -set (WAMR_BUILD_JIT 0) -set (WAMR_BUILD_LIBC_BUILTIN 1) -set (WAMR_BUILD_LIBC_WASI 1) +cmake_minimum_required (VERSION 3.14) # Reset default linker flags set (CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") set (CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") -# Set WAMR_BUILD_TARGET, currently values supported: -# "X86_64", "AMD_64", "X86_32", "AARCH64[sub]", "ARM[sub]", "THUMB[sub]", "MIPS", "XTENSA" if (NOT DEFINED WAMR_BUILD_TARGET) - if (CMAKE_SIZEOF_VOID_P EQUAL 8) - # Build as X86_64 by default in 64-bit platform - set (WAMR_BUILD_TARGET "X86_64") - elseif (CMAKE_SIZEOF_VOID_P EQUAL 4) - # Build as X86_32 by default in 32-bit platform - set (WAMR_BUILD_TARGET "X86_32") + message (FATAL_ERROR "WAMR_BUILD_TARGET isn't set") +endif () + +if (NOT (WAMR_BUILD_TARGET STREQUAL "X86_64" + OR WAMR_BUILD_TARGET STREQUAL "X86_32" + OR WAMR_BUILD_TARGET MATCHES "AARCH64.*" + OR WAMR_BUILD_TARGET MATCHES "ARM.*" + OR WAMR_BUILD_TARGET MATCHES "RISCV64.*")) + message (FATAL_ERROR "Unsupported build target platform ${WAMR_BUILD_TARGET}!") +endif () + +if (NOT DEFINED ANDROID_ABI) + if (WAMR_BUILD_TARGET STREQUAL "X86_64") + set (ANDROID_ABI "x86_64") + elseif (WAMR_BUILD_TARGET STREQUAL "X86_32") + set (ANDROID_ABI "x86") + elseif (WAMR_BUILD_TARGET MATCHES "AARCH64.*") + set (ANDROID_ABI "arm64-v8a") + elseif (WAMR_BUILD_TARGET MATCHES "ARM.*") + set (ANDROID_ABI "armeabi-v7a") else () - message(SEND_ERROR "Unsupported build target platform!") + set (ANDROID_ABI "riscv64") endif () endif () +if (NOT DEFINED ANDROID_LD) + set (ANDROID_LD lld) +endif () + +if (NOT DEFINED ANDROID_PLATFORM) + set (ANDROID_PLATFORM 24) +endif () + +# https://android.googlesource.com/platform/ndk/+/master/build/cmake/android.toolchain.cmake +set (CMAKE_TOOLCHAIN_FILE "$ENV{ANDROID_NDK_LATEST_HOME}/build/cmake/android.toolchain.cmake") +set (ANDROID_SDK $ENV{ANDROID_HOME}) +set (ANDROID_NDK $ENV{ANDROID_NDK_LATEST_HOME}) + +project (iwasm) + +set (WAMR_BUILD_PLATFORM "android") + +set (CMAKE_VERBOSE_MAKEFILE ON) + if (NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release) + set (CMAKE_BUILD_TYPE Release) endif () if (NOT DEFINED WAMR_BUILD_INTERP) @@ -55,6 +61,11 @@ if (NOT DEFINED WAMR_BUILD_INTERP) set (WAMR_BUILD_INTERP 1) endif () +if (NOT DEFINED WAMR_BUILD_FAST_INTERP) + # Enable fast interpreter + set (WAMR_BUILD_FAST_INTERP 1) +endif () + if (NOT DEFINED WAMR_BUILD_AOT) # Enable AOT by default. set (WAMR_BUILD_AOT 1) @@ -75,6 +86,21 @@ if (NOT DEFINED WAMR_BUILD_LIBC_WASI) set (WAMR_BUILD_LIBC_WASI 1) endif () +if (NOT DEFINED WAMR_BUILD_MULTI_MODULE) + # Disable multiple modules by default + set (WAMR_BUILD_MULTI_MODULE 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_PTHREAD) + # Disable pthread library by default + set (WAMR_BUILD_LIB_PTHREAD 0) +endif () + +if (NOT DEFINED WAMR_BUILD_LIB_WASI_THREADS) + # Disable wasi threads library by default + set (WAMR_BUILD_LIB_WASI_THREADS 0) +endif() + set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../..) include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) From a86eeb273c5e6d1dcbba94871e3538754c301985 Mon Sep 17 00:00:00 2001 From: Brian <89487381+b4yuan@users.noreply.github.com> Date: Thu, 21 Mar 2024 05:37:47 +0100 Subject: [PATCH 2/8] Add CodeQL Workflow for Code Security Analysis (#2812) Add CodeQL Workflow for Code Security Analysis This pull request introduces a CodeQL workflow to enhance the security analysis of our repository. CodeQL is a powerful static analysis tool that helps identify and mitigate security vulnerabilities in our codebase. By integrating this workflow into our GitHub Actions, we can proactively identify and address potential issues before they become security threats. We added a new CodeQL workflow file (.github/workflows/codeql.yml) that - Runs on nightly-run, and consider runs on every pull request to the main branch in the future. - Excludes queries with a high false positive rate or low-severity findings. - Does not display results for third-party code, focusing only on our own codebase. Testing: To validate the functionality of this workflow, we have run several test scans on the codebase and reviewed the results. The workflow successfully compiles the project, identifies issues, and provides actionable insights while reducing noise by excluding certain queries and third-party code. Deployment: Once this pull request is merged, the CodeQL workflow will be active and automatically run on every push and pull request to the main branch. To view the results of these code scans, please follow these steps: 1. Under the repository name, click on the Security tab. 2. In the left sidebar, click Code scanning alerts. Additional Information: - You can further customize the workflow to adapt to your specific needs by modifying the workflow file. - For more information on CodeQL and how to interpret its results, refer to the GitHub documentation and the CodeQL documentation. Signed-off-by: Brian --- .github/workflows/codeql.yml | 126 ++++++++++++++++++++++++ .github/workflows/codeql_buildscript.sh | 18 ++++ .github/workflows/fail_on_error.py | 34 +++++++ 3 files changed, 178 insertions(+) create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/codeql_buildscript.sh create mode 100755 .github/workflows/fail_on_error.py diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..98e58b389 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,126 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + # push: + # branches: [ "main", "master" ] + schedule: + - cron: '0 0 * * *' + pull_request: + branches: '*' + +jobs: + analyze: + name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-20.04' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'cpp' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ] + # Use only 'java' to analyze code written in Java, Kotlin or both + # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: recursive + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + queries: security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + #- name: Autobuild + # uses: github/codeql-action/autobuild@v2 + + # â„šī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + - run: | + ./.github/workflows/codeql_buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" + upload: false + id: step1 + + # Filter out rules with low severity or high false positve rate + # Also filter out warnings in third-party code + - name: Filter out unwanted errors and warnings + uses: advanced-security/filter-sarif@v1 + with: + patterns: | + -**:cpp/path-injection + -**:cpp/world-writable-file-creation + -**:cpp/poorly-documented-function + -**:cpp/potentially-dangerous-function + -**:cpp/use-of-goto + -**:cpp/integer-multiplication-cast-to-long + -**:cpp/comparison-with-wider-type + -**:cpp/leap-year/* + -**:cpp/ambiguously-signed-bit-field + -**:cpp/suspicious-pointer-scaling + -**:cpp/suspicious-pointer-scaling-void + -**:cpp/unsigned-comparison-zero + -**/cmake*/Modules/** + input: ${{ steps.step1.outputs.sarif-output }}/cpp.sarif + output: ${{ steps.step1.outputs.sarif-output }}/cpp.sarif + + - name: Upload CodeQL results to code scanning + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: ${{ steps.step1.outputs.sarif-output }} + category: "/language:${{matrix.language}}" + + - name: Upload CodeQL results as an artifact + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: codeql-results + path: ${{ steps.step1.outputs.sarif-output }} + retention-days: 5 + + - name: Fail if an error is found + run: | + ./.github/workflows/fail_on_error.py \ + ${{ steps.step1.outputs.sarif-output }}/cpp.sarif diff --git a/.github/workflows/codeql_buildscript.sh b/.github/workflows/codeql_buildscript.sh new file mode 100644 index 000000000..aceebe3e2 --- /dev/null +++ b/.github/workflows/codeql_buildscript.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +sudo apt install -y build-essential cmake g++-multilib libgcc-9-dev lib32gcc-9-dev ccache ninja-build + +cd wamr-compiler +./build_llvm.sh +mkdir build && cd build +cmake .. +make +# wamrc is generated under current directory + +cd ../.. + +cd product-mini/platforms/linux/ +mkdir build && cd build +cmake .. +make +# iwasm is generated under current directory diff --git a/.github/workflows/fail_on_error.py b/.github/workflows/fail_on_error.py new file mode 100755 index 000000000..29791742b --- /dev/null +++ b/.github/workflows/fail_on_error.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +import json +import sys + +# Return whether SARIF file contains error-level results +def codeql_sarif_contain_error(filename): + with open(filename, 'r') as f: + s = json.load(f) + + for run in s.get('runs', []): + rules_metadata = run['tool']['driver']['rules'] + if not rules_metadata: + rules_metadata = run['tool']['extensions'][0]['rules'] + + for res in run.get('results', []): + if 'ruleIndex' in res: + rule_index = res['ruleIndex'] + elif 'rule' in res and 'index' in res['rule']: + rule_index = res['rule']['index'] + else: + continue + try: + rule_level = rules_metadata[rule_index]['defaultConfiguration']['level'] + except IndexError as e: + print(e, rule_index, len(rules_metadata)) + else: + if rule_level == 'error': + return True + return False + +if __name__ == "__main__": + if codeql_sarif_contain_error(sys.argv[1]): + sys.exit(1) From e003ee1e299f03b9d2bed78cec1e69b3014d413b Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Thu, 21 Mar 2024 14:18:27 +0800 Subject: [PATCH 3/8] CodeQL: Add more build combinations and disable run on PR (#3246) Enhance CodeQL Code Security Analysis: - Add more compilation combinations to build iwasm with different kinds of features - Disable run on PR created and keep nightly run, since the whole time is very long, and will check how to restore run on PR created in the future --- .github/workflows/codeql.yml | 39 +-- .github/workflows/codeql_buildscript.sh | 256 +++++++++++++++++- ...il_on_error.py => codeql_fail_on_error.py} | 0 3 files changed, 256 insertions(+), 39 deletions(-) mode change 100644 => 100755 .github/workflows/codeql_buildscript.sh rename .github/workflows/{fail_on_error.py => codeql_fail_on_error.py} (100%) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 98e58b389..50a7db45e 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -4,20 +4,20 @@ # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# name: "CodeQL" on: - # push: - # branches: [ "main", "master" ] + #pull_request: + # types: + # - opened + # branches: '*' + #push: + # branches: [ "main" ] + # midnight UTC schedule: - cron: '0 0 * * *' - pull_request: - branches: '*' + # allow to be triggered manually + workflow_dispatch: jobs: analyze: @@ -39,9 +39,6 @@ jobs: matrix: language: [ 'cpp' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ] - # Use only 'java' to analyze code written in Java, Kotlin or both - # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository @@ -54,29 +51,19 @@ jobs: uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality queries: security-and-quality - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). - # If this step fails, then you should remove it and run the build manually (see below) - #- name: Autobuild - # uses: github/codeql-action/autobuild@v2 - - # â„šī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + # Command-line programs to run using the OS shell. + # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun # If the Autobuild fails above, remove it and uncomment the following three lines. # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - run: | ./.github/workflows/codeql_buildscript.sh - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 with: @@ -118,9 +105,9 @@ jobs: with: name: codeql-results path: ${{ steps.step1.outputs.sarif-output }} - retention-days: 5 + retention-days: 10 - name: Fail if an error is found run: | - ./.github/workflows/fail_on_error.py \ + ./.github/workflows/codeql_fail_on_error.py \ ${{ steps.step1.outputs.sarif-output }}/cpp.sarif diff --git a/.github/workflows/codeql_buildscript.sh b/.github/workflows/codeql_buildscript.sh old mode 100644 new mode 100755 index aceebe3e2..70e044dbd --- a/.github/workflows/codeql_buildscript.sh +++ b/.github/workflows/codeql_buildscript.sh @@ -1,18 +1,248 @@ #!/usr/bin/env bash -sudo apt install -y build-essential cmake g++-multilib libgcc-9-dev lib32gcc-9-dev ccache ninja-build +sudo apt install -y build-essential cmake g++-multilib libgcc-12-dev lib32gcc-12-dev ccache ninja-build ccache -cd wamr-compiler -./build_llvm.sh -mkdir build && cd build -cmake .. -make -# wamrc is generated under current directory +WAMR_DIR=${PWD} -cd ../.. - -cd product-mini/platforms/linux/ -mkdir build && cd build +# build wamrc +cd ${WAMR_DIR}/wamr-compiler +./build_llvm.sh +rm -fr build && mkdir build && cd build cmake .. -make -# iwasm is generated under current directory +make -j +if [[ $? != 0 ]]; then + echo "Failed to build wamrc!" + exit 1; +fi + +# build iwasm with default features enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -fr build && mkdir build && cd build +cmake .. +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with default features enabled!" + exit 1; +fi + +# build iwasm with default features enabled on x86_32 +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -fr build && mkdir build && cd build +cmake .. -DWAMR_BUILD_TARGET=X86_32 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with default features enabled on x86_32!" + exit 1; +fi + +# build iwasm with classic interpreter enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_INTERP=0 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with classic interpreter enabled!" + exit 1; +fi + +# build iwasm with extra features enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -fr build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug \ + -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIB_PTHREAD_SEMAPHORE=1 \ + -DWAMR_BUILD_MULTI_MODULE=1 -DWAMR_BUILD_SIMD=1 \ + -DWAMR_BUILD_TAIL_CALL=1 -DWAMR_BUILD_REF_TYPES=1 \ + -DWAMR_BUILD_CUSTOM_NAME_SECTION=1 -DWAMR_BUILD_MEMORY_PROFILING=1 \ + -DWAMR_BUILD_PERF_PROFILING=1 -DWAMR_BUILD_DUMP_CALL_STACK=1 \ + -DWAMR_BUILD_LOAD_CUSTOM_SECTION=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build wamrc iwasm with extra features enabled!" + exit 1; +fi + +# build iwasm with global heap pool enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -fr build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug \ + -DWAMR_BUILD_ALLOC_WITH_USER_DATA=1 \ + -DWAMR_DISABLE_STACK_HW_BOUND_CHECK=1 \ + -DWAMR_BUILD_GLOBAL_HEAP_POOL=1 \ + -DWAMR_BUILD_GLOBAL_HEAP_SIZE=131072 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with global heap pool enabled!" + exit 1; +fi + +# build iwasm with wasi-threads enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -fr build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_LIB_WASI_THREADS=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with wasi-threads enabled!" + exit 1; +fi + +# build iwasm with GC enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_GC=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with GC enabled!" + exit 1; +fi + +# build iwasm with hardware boundary check disabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_DISABLE_HW_BOUND_CHECK=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with hardware boundary check disabled!" + exit 1; +fi + +# build iwasm with quick AOT entry disabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_QUICK_AOT_ENTRY=0 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with quick AOT entry disabled!" + exit 1; +fi + +# build iwasm with wakeup of blocking operations disabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_DISABLE_WAKEUP_BLOCKING_OP=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with wakeup of blocking operations disabled!" + exit 1; +fi + +# build iwasm with module instance context disabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MODULE_INST_CONTEXT=0 \ + -DWAMR_BUILD_LIBC_BUILTIN=0 -DWAMR_BUILD_LIBC_WASI=0 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with module instance context disabled!" + exit 1; +fi + +# build iwasm with libc-uvwasi enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -fr build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_LIBC_UVWASI=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with libc-uvwasi enabled!" + exit 1; +fi + +# build iwasm with llvm jit lazy mode enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_JIT=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build llvm jit lazy mode enabled!" + exit 1; +fi + +# build iwasm with llvm jit eager mode enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build llvm jit eager mode enabled!" + exit 1; +fi + +# build iwasm with fast jit lazy mode enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_FAST_JIT_DUMP=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with fast jit lazy mode enabled!" + exit 1; +fi + +# build iwasm with fast jit eager mode enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_FAST_JIT_DUMP=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with fast jit eager mode enabled!" + exit 1; +fi + +# build iwasm with multi-tier jit enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_FAST_JIT=1 -DWAMR_BUILD_JIT=1 \ + -DWAMR_BUILD_FAST_JIT_DUMP=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with multi-tier jit enabled!" + exit 1; +fi + +# build iwasm with wasm mini-loader enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_MINI_LOADER=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build with wasm mini-loader enabled!" + exit 1; +fi + +# build iwasm with source debugging enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_DEBUG_INTERP=1 -DWAMR_BUILD_DEBUG_AOT=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with source debugging enabled!" + exit 1; +fi + +# build iwasm with AOT static PGO enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_STATIC_PGO=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with AOT static PGO enabled!" + exit 1; +fi + +# build iwasm with configurable bounds checks enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_CONFIGUABLE_BOUNDS_CHECKS=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with configurable bounds checks enabled!" + exit 1; +fi + +# build iwasm with linux perf support enabled +cd ${WAMR_DIR}/product-mini/platforms/linux/ +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_LINUX_PERF=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build iwasm with linux perf support enabled!" + exit 1; +fi diff --git a/.github/workflows/fail_on_error.py b/.github/workflows/codeql_fail_on_error.py similarity index 100% rename from .github/workflows/fail_on_error.py rename to .github/workflows/codeql_fail_on_error.py From cef88deedb0e1c3978f3437aff44af708cdf3a0f Mon Sep 17 00:00:00 2001 From: Xu Jinyang <72930776+AuYang261@users.noreply.github.com> Date: Thu, 21 Mar 2024 21:05:34 +0800 Subject: [PATCH 4/8] Add `wasi_ephemeral_nn` module support (#3241) Add `wasi_ephemeral_nn` module support with optional cmake variable, which was mentioned in #3229. --- build-scripts/config_common.cmake | 4 ++ core/config.h | 4 ++ core/iwasm/common/wasm_native.c | 7 ++- .../wasi-nn/src/utils/wasi_nn_app_native.c | 50 ++++++++++++++++--- .../wasi-nn/src/utils/wasi_nn_app_native.h | 14 ++++++ core/iwasm/libraries/wasi-nn/src/wasi_nn.c | 35 +++++++++++++ doc/build_wamr.md | 3 ++ 7 files changed, 109 insertions(+), 8 deletions(-) diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 773ff2837..4569648a1 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -430,6 +430,10 @@ if (WAMR_BUILD_WASI_NN EQUAL 1) if (DEFINED WAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH) add_definitions (-DWASM_WASI_NN_EXTERNAL_DELEGATE_PATH="${WAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH}") endif () + if (WAMR_BUILD_WASI_EPHEMERAL_NN EQUAL 1) + message (" WASI-NN: WASI-Ephemeral-NN enabled") + add_definitions (-DWASM_ENABLE_WASI_EPHEMERAL_NN=1) + endif() endif () if (WAMR_BUILD_ALLOC_WITH_USER_DATA EQUAL 1) add_definitions(-DWASM_MEM_ALLOC_WITH_USER_DATA=1) diff --git a/core/config.h b/core/config.h index be9ebfc3a..616c1f6e7 100644 --- a/core/config.h +++ b/core/config.h @@ -152,6 +152,10 @@ #define WASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE 0 #endif +#ifndef WASM_ENABLE_WASI_EPHEMERAL_NN +#define WASM_ENABLE_WASI_EPHEMERAL_NN 0 +#endif + /* Default disable libc emcc */ #ifndef WASM_ENABLE_LIBC_EMCC #define WASM_ENABLE_LIBC_EMCC 0 diff --git a/core/iwasm/common/wasm_native.c b/core/iwasm/common/wasm_native.c index 3cf7451e3..14b295ee7 100644 --- a/core/iwasm/common/wasm_native.c +++ b/core/iwasm/common/wasm_native.c @@ -567,7 +567,12 @@ wasm_native_init() #if WASM_ENABLE_WASI_NN != 0 n_native_symbols = get_wasi_nn_export_apis(&native_symbols); - if (!wasm_native_register_natives("wasi_nn", native_symbols, +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +#define wasi_nn_module_name "wasi_ephemeral_nn" +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ +#define wasi_nn_module_name "wasi_nn" +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ + if (!wasm_native_register_natives(wasi_nn_module_name, native_symbols, n_native_symbols)) goto fail; #endif diff --git a/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c b/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c index 28dfbad4e..44aef5359 100644 --- a/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c +++ b/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c @@ -23,24 +23,47 @@ graph_builder_app_native(wasm_module_inst_t instance, return success; } +/** + * builder_array_wasm is consisted of {builder_wasm, size} + */ +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +error +graph_builder_array_app_native(wasm_module_inst_t instance, + graph_builder_wasm *builder_wasm, uint32_t size, + graph_builder_array *builder_array) +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ error graph_builder_array_app_native(wasm_module_inst_t instance, graph_builder_array_wasm *builder_array_wasm, graph_builder_array *builder_array) +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ { +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +#define array_size size +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ +#define array_size builder_array_wasm->size + if (!wasm_runtime_validate_native_addr( instance, builder_array_wasm, (uint64)sizeof(graph_builder_array_wasm))) { NN_ERR_PRINTF("builder_array_wasm is invalid"); return invalid_argument; } +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ - NN_DBG_PRINTF("Graph builder array contains %d elements", - builder_array_wasm->size); + NN_DBG_PRINTF("Graph builder array contains %d elements", array_size); +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + if (!wasm_runtime_validate_native_addr(instance, builder_wasm, + (uint64)array_size + * sizeof(graph_builder_wasm))) { + NN_ERR_PRINTF("builder_wasm is invalid"); + return invalid_argument; + } +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ if (!wasm_runtime_validate_app_addr( instance, (uint64)builder_array_wasm->buf_offset, - (uint64)builder_array_wasm->size * sizeof(graph_builder_wasm))) { + (uint64)array_size * sizeof(graph_builder_wasm))) { NN_ERR_PRINTF("builder_array_wasm->buf_offset is invalid"); return invalid_argument; } @@ -48,13 +71,14 @@ graph_builder_array_app_native(wasm_module_inst_t instance, graph_builder_wasm *builder_wasm = (graph_builder_wasm *)wasm_runtime_addr_app_to_native( instance, (uint64)builder_array_wasm->buf_offset); +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ graph_builder *builder = (graph_builder *)wasm_runtime_malloc( - builder_array_wasm->size * sizeof(graph_builder)); + array_size * sizeof(graph_builder)); if (builder == NULL) return missing_memory; - for (uint32_t i = 0; i < builder_array_wasm->size; ++i) { + for (uint32_t i = 0; i < array_size; ++i) { error res; if (success != (res = graph_builder_app_native(instance, &builder_wasm[i], @@ -68,23 +92,31 @@ graph_builder_array_app_native(wasm_module_inst_t instance, } builder_array->buf = builder; - builder_array->size = builder_array_wasm->size; + builder_array->size = array_size; return success; +#undef array_size } static error tensor_data_app_native(wasm_module_inst_t instance, uint32_t total_elements, tensor_wasm *input_tensor_wasm, tensor_data *data) { +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +#define data_size input_tensor_wasm->data_size +#else +#define data_size total_elements +#endif + if (!wasm_runtime_validate_app_addr(instance, (uint64)input_tensor_wasm->data_offset, - (uint64)total_elements)) { + (uint64)data_size)) { NN_ERR_PRINTF("input_tensor_wasm->data_offset is invalid"); return invalid_argument; } *data = (tensor_data)wasm_runtime_addr_app_to_native( instance, (uint64)input_tensor_wasm->data_offset); return success; +#undef data_size } static error @@ -92,6 +124,9 @@ tensor_dimensions_app_native(wasm_module_inst_t instance, tensor_wasm *input_tensor_wasm, tensor_dimensions **dimensions) { +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + tensor_dimensions_wasm *dimensions_wasm = &input_tensor_wasm->dimensions; +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ if (!wasm_runtime_validate_app_addr( instance, (uint64)input_tensor_wasm->dimensions_offset, (uint64)sizeof(tensor_dimensions_wasm))) { @@ -102,6 +137,7 @@ tensor_dimensions_app_native(wasm_module_inst_t instance, tensor_dimensions_wasm *dimensions_wasm = (tensor_dimensions_wasm *)wasm_runtime_addr_app_to_native( instance, (uint64)input_tensor_wasm->dimensions_offset); +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ if (!wasm_runtime_validate_app_addr(instance, (uint64)dimensions_wasm->buf_offset, diff --git a/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.h b/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.h index 15154bd31..f0930a883 100644 --- a/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.h +++ b/core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.h @@ -34,15 +34,29 @@ typedef struct { } tensor_dimensions_wasm; typedef struct { +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + tensor_dimensions_wasm dimensions; + tensor_type type; + uint32_t data_offset; + uint32_t data_size; +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ uint32_t dimensions_offset; tensor_type type; uint32_t data_offset; +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ } tensor_wasm; +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +error +graph_builder_array_app_native(wasm_module_inst_t instance, + graph_builder_wasm *builder_wasm, uint32_t size, + graph_builder_array *builder_array); +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ error graph_builder_array_app_native(wasm_module_inst_t instance, graph_builder_array_wasm *builder, graph_builder_array *builder_native); +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ error tensor_app_native(wasm_module_inst_t instance, tensor_wasm *input_tensor, diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn.c b/core/iwasm/libraries/wasi-nn/src/wasi_nn.c index 8e17deed4..1fbb94428 100644 --- a/core/iwasm/libraries/wasi-nn/src/wasi_nn.c +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn.c @@ -189,9 +189,16 @@ is_model_initialized(WASINNContext *wasi_nn_ctx) /* WASI-NN implementation */ +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +error +wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_wasm *builder, + uint32_t builder_wasm_size, graph_encoding encoding, + execution_target target, graph *g) +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ error wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder, graph_encoding encoding, execution_target target, graph *g) +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ { NN_DBG_PRINTF("Running wasi_nn_load [encoding=%d, target=%d]...", encoding, target); @@ -206,10 +213,17 @@ wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder, error res; graph_builder_array builder_native = { 0 }; +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + if (success + != (res = graph_builder_array_app_native( + instance, builder, builder_wasm_size, &builder_native))) + return res; +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ if (success != (res = graph_builder_array_app_native(instance, builder, &builder_native))) return res; +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ if (!wasm_runtime_validate_native_addr(instance, g, (uint64)sizeof(graph))) { @@ -315,10 +329,17 @@ wasi_nn_compute(wasm_exec_env_t exec_env, graph_execution_context ctx) return res; } +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 +error +wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx, + uint32_t index, tensor_data output_tensor, + uint32_t output_tensor_len, uint32_t *output_tensor_size) +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ error wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx, uint32_t index, tensor_data output_tensor, uint32_t *output_tensor_size) +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ { NN_DBG_PRINTF("Running wasi_nn_get_output [ctx=%d, index=%d]...", ctx, index); @@ -337,8 +358,14 @@ wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx, return invalid_argument; } +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + res = lookup[wasi_nn_ctx->current_encoding].get_output( + wasi_nn_ctx->tflite_ctx, ctx, index, output_tensor, &output_tensor_len); + *output_tensor_size = output_tensor_len; +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ res = lookup[wasi_nn_ctx->current_encoding].get_output( wasi_nn_ctx->tflite_ctx, ctx, index, output_tensor, output_tensor_size); +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ NN_DBG_PRINTF("wasi_nn_get_output finished with status %d [data_size=%d]", res, *output_tensor_size); return res; @@ -352,11 +379,19 @@ wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx, /* clang-format on */ static NativeSymbol native_symbols_wasi_nn[] = { +#if WASM_ENABLE_WASI_EPHEMERAL_NN != 0 + REG_NATIVE_FUNC(load, "(*iii*)i"), + REG_NATIVE_FUNC(init_execution_context, "(i*)i"), + REG_NATIVE_FUNC(set_input, "(ii*)i"), + REG_NATIVE_FUNC(compute, "(i)i"), + REG_NATIVE_FUNC(get_output, "(ii*i*)i"), +#else /* WASM_ENABLE_WASI_EPHEMERAL_NN == 0 */ REG_NATIVE_FUNC(load, "(*ii*)i"), REG_NATIVE_FUNC(init_execution_context, "(i*)i"), REG_NATIVE_FUNC(set_input, "(ii*)i"), REG_NATIVE_FUNC(compute, "(i)i"), REG_NATIVE_FUNC(get_output, "(ii**)i"), +#endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */ }; uint32_t diff --git a/doc/build_wamr.md b/doc/build_wamr.md index 1331f9601..a9ffca204 100644 --- a/doc/build_wamr.md +++ b/doc/build_wamr.md @@ -107,6 +107,9 @@ cmake -DWAMR_BUILD_PLATFORM=linux -DWAMR_BUILD_TARGET=ARM - **WAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH**=Path to the external delegate shared library (e.g. `libedgetpu.so.1.0` for Coral USB) +#### **Enable lib wasi-nn with `wasi_ephemeral_nn` module support** +- **WAMR_BUILD_WASI_EPHEMERAL_NN**=1/0, default to disable if not set + #### **Disable boundary check with hardware trap** - **WAMR_DISABLE_HW_BOUND_CHECK**=1/0, default to enable if not set and supported by platform > Note: by default only platform [linux/darwin/android/windows/vxworks 64-bit](https://github.com/bytecodealliance/wasm-micro-runtime/blob/5fb5119239220b0803e7045ca49b0a29fe65e70e/core/shared/platform/linux/platform_internal.h#L81) will enable the boundary check with hardware trap feature, for 32-bit platforms it's automatically disabled even when the flag is set to 0, and the wamrc tool will generate AOT code without boundary check instructions in all 64-bit targets except SGX to improve performance. The boundary check includes linear memory access boundary and native stack access boundary, if `WAMR_DISABLE_STACK_HW_BOUND_CHECK` below isn't set. From b4941b0cde316d13085d5552332b2ad744f7b4e8 Mon Sep 17 00:00:00 2001 From: Yo Han Joo Date: Fri, 22 Mar 2024 11:45:14 +0900 Subject: [PATCH 5/8] Go binding: Change `C.long` to `C.int64_t` when call wasm_runtime_set_wasi_args_ex (#3235) - Change `C.long` to `C.int64_t` due to error: ```sh ./module.go:119:64: cannot use _Ctype_long(stdinfd) (value of type _Ctype_long) as _Ctype_longlong value in variable declaration ./module.go:120:43: cannot use _Ctype_long(stdoutfd) (value of type _Ctype_long) as _Ctype_longlong value in variable declaration ./module.go:120:60: cannot use _Ctype_long(stderrfd) (value of type _Ctype_long) as _Ctype_longlong value in variable declaration ``` - Change offset from `uint32` to `uint64` due to casting error ps. https://github.com/bytecodealliance/wasm-micro-runtime/issues/3220 https://stackoverflow.com/questions/70243683/how-to-convert-c-uint64-t-to-cgo-consistently-across-os --- language-bindings/go/samples/test.go | 2 +- language-bindings/go/wamr/module.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/language-bindings/go/samples/test.go b/language-bindings/go/samples/test.go index d0fc7d8b2..19b2814a8 100644 --- a/language-bindings/go/samples/test.go +++ b/language-bindings/go/samples/test.go @@ -87,7 +87,7 @@ func main() { var instance *wamr.Instance var argv []uint32 var results []interface{} - var offset uint32 + var offset uint64 var native_addr *uint8 var err error diff --git a/language-bindings/go/wamr/module.go b/language-bindings/go/wamr/module.go index 8775b3a1a..7fd131ea6 100644 --- a/language-bindings/go/wamr/module.go +++ b/language-bindings/go/wamr/module.go @@ -117,8 +117,8 @@ func (self *Module) SetWasiArgsEx(dirList [][]byte, mapDirList [][]byte, C.wasm_runtime_set_wasi_args_ex(self.module, dirPtr, dirCount, mapDirPtr, mapDirCount, envPtr, envCount, argvPtr, argc, - C.long(stdinfd), C.long(stdoutfd), - C.long(stderrfd)) + C.int64_t(stdinfd), C.int64_t(stdoutfd), + C.int64_t(stderrfd)) } /* Set module's wasi network address pool */ From 64b6c688a234b3cd624d89846bbef8ca7bbeb3b7 Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Fri, 22 Mar 2024 10:55:39 +0800 Subject: [PATCH 6/8] posix_file.c: Correct the dirfd argument that passes to fstatat (#3244) This PR fixes the random failing test case `nofollow_errors` mentioned in https://github.com/bytecodealliance/wasm-micro-runtime/issues/3222 ```C // dirfd: This is the file descriptor of the directory relative to which the pathname is interpreted. int openat(int dirfd, const char *pathname, int flags, ...); ``` The value should be a directory handle instead of a file handle (which is always -1 in this context) returned from `openat`. --- core/shared/platform/common/posix/posix_file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/shared/platform/common/posix/posix_file.c b/core/shared/platform/common/posix/posix_file.c index ad5589f73..20f94fba3 100644 --- a/core/shared/platform/common/posix/posix_file.c +++ b/core/shared/platform/common/posix/posix_file.c @@ -384,7 +384,7 @@ os_openat(os_file_handle handle, const char *path, __wasi_oflags_t oflags, // Linux returns ENXIO instead of EOPNOTSUPP when opening a socket. if (openat_errno == ENXIO) { struct stat sb; - int ret = fstatat(fd, path, &sb, + int ret = fstatat(handle, path, &sb, (lookup_flags & __WASI_LOOKUP_SYMLINK_FOLLOW) ? 0 : AT_SYMLINK_NOFOLLOW); @@ -396,7 +396,7 @@ os_openat(os_file_handle handle, const char *path, __wasi_oflags_t oflags, if (openat_errno == ENOTDIR && (open_flags & (O_NOFOLLOW | O_DIRECTORY)) != 0) { struct stat sb; - int ret = fstatat(fd, path, &sb, AT_SYMLINK_NOFOLLOW); + int ret = fstatat(handle, path, &sb, AT_SYMLINK_NOFOLLOW); if (S_ISLNK(sb.st_mode)) { return __WASI_ELOOP; } From 6a55bde5b355a9ebe4573a1a529435d0a5925089 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Fri, 22 Mar 2024 13:49:39 +0800 Subject: [PATCH 7/8] Add issue templates (#3248) Add issue templates of blank issue, improvement and reporting bug in WAMR. And fix several invalid links in ATTRIBUTIONS.md. --- .github/ISSUE_TEMPLATE/blank_issue.md | 5 ++++ .github/ISSUE_TEMPLATE/improvement.md | 28 +++++++++++++++++++++ .github/ISSUE_TEMPLATE/report_bug.md | 36 +++++++++++++++++++++++++++ ATTRIBUTIONS.md | 8 ++---- 4 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/blank_issue.md create mode 100644 .github/ISSUE_TEMPLATE/improvement.md create mode 100644 .github/ISSUE_TEMPLATE/report_bug.md diff --git a/.github/ISSUE_TEMPLATE/blank_issue.md b/.github/ISSUE_TEMPLATE/blank_issue.md new file mode 100644 index 000000000..57febe7d5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/blank_issue.md @@ -0,0 +1,5 @@ +--- +name: Blank Issue +about: Create a blank issue. +title: '' +--- diff --git a/.github/ISSUE_TEMPLATE/improvement.md b/.github/ISSUE_TEMPLATE/improvement.md new file mode 100644 index 000000000..ffdf0906f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/improvement.md @@ -0,0 +1,28 @@ +--- +name: Improvement +about: A feature request or code improvement. +title: '' +labels: '' +assignees: '' +--- + +Thanks for filing a feature request! Please fill out the TODOs below. + +#### Feature + +TODO: Brief description of the feature/improvement you'd like to see in WAMR + +#### Benefit + +TODO: What is the value of adding this in WAMR? What problems does it solve? + +#### Implementation + +TODO: Do you have an implementation plan, and/or ideas for data structures or +algorithms to use? + +#### Alternatives + +TODO: What are the alternative implementation approaches or alternative ways to +solve the problem that this feature would solve? How do these alternatives +compare to this proposal? diff --git a/.github/ISSUE_TEMPLATE/report_bug.md b/.github/ISSUE_TEMPLATE/report_bug.md new file mode 100644 index 000000000..d3058c9ca --- /dev/null +++ b/.github/ISSUE_TEMPLATE/report_bug.md @@ -0,0 +1,36 @@ +--- +name: WAMR bug or defect report +about: Report a bug or defect in WAMR +title: '' +--- + +Thanks for filing a bug or defect report! Please fill out the TODOs below. + +### Subject of the issue + +Describe the bug or defect here. + +### Test case + +Upload the related wasm file, wast file or the source files if you can. + +### Your environment + +* Host OS +* WAMR version, platform, cpu architecture, running mode, etc. + +### Steps to reproduce + +Tell us how to reproduce this bug or defect. + +### Expected behavior + +Tell us what should happen + +### Actual behavior + +Tell us what happens instead + +### Extra Info + +Anything else you'd like to add? diff --git a/ATTRIBUTIONS.md b/ATTRIBUTIONS.md index b249f7b02..2c83d6674 100644 --- a/ATTRIBUTIONS.md +++ b/ATTRIBUTIONS.md @@ -60,11 +60,11 @@ The WAMR fast interpreter is a clean room development. We would acknowledge the ### llvm -[LICENSE](./core/deps/llvm/llvm/LICENCE.txt) +[LICENSE](./LICENCE.txt) ### wasm-c-api -[LICENSE](https://github.com/bytecodealliance/wamr-app-framework/blob/main/samples/wasm-c-api/src/LICENSE) +[LICENSE](./samples/wasm-c-api/src/LICENSE) ### wasmtime @@ -78,10 +78,6 @@ The WAMR fast interpreter is a clean room development. We would acknowledge the [LICENSE](https://github.com/bytecodealliance/wamr-app-framework/blob/main/samples/gui/wasm-runtime-wgl/src/platform/zephyr/LICENSE) -### wac - -[LICENSE](./tests/wamr-test-suites/spec-test-script/LICENSE) - ### libuv [LICENSE](./core/iwasm/libraries/libc-uvwasi/LICENSE_LIBUV) From ca364eb5d7a42992748f4e3b57f2761995b9c2e4 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Fri, 22 Mar 2024 14:29:30 +0800 Subject: [PATCH 8/8] Fix nightly-run CI and CodeQL CI (#3250) - Fix nightly run CI failure which was introduced by PR #3239 and now it must set WAMR_BUILD_TARGET when building iwasm for Android platform - Remove building llvm, wamrc and jit in CodeQL CI, since it will do static code analyzing for llvm project and cause CodeQL run failed: `Oops! A fatal internal error occurred. This particular kind of error most often happens as a side effect of running out of disk space.` --- .github/workflows/codeql_buildscript.sh | 51 +++++++++++++++---------- .github/workflows/nightly_run.yml | 20 +++++++++- 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/.github/workflows/codeql_buildscript.sh b/.github/workflows/codeql_buildscript.sh index 70e044dbd..34e0ddd79 100755 --- a/.github/workflows/codeql_buildscript.sh +++ b/.github/workflows/codeql_buildscript.sh @@ -1,9 +1,14 @@ #!/usr/bin/env bash -sudo apt install -y build-essential cmake g++-multilib libgcc-12-dev lib32gcc-12-dev ccache ninja-build ccache +sudo apt update + +sudo apt install -y build-essential cmake g++-multilib libgcc-11-dev lib32gcc-11-dev ccache ninja-build ccache WAMR_DIR=${PWD} +# TODO: use pre-built llvm binary to build wamrc to +# avoid static code analysing for llvm +: ' # build wamrc cd ${WAMR_DIR}/wamr-compiler ./build_llvm.sh @@ -14,6 +19,7 @@ if [[ $? != 0 ]]; then echo "Failed to build wamrc!" exit 1; fi +' # build iwasm with default features enabled cd ${WAMR_DIR}/product-mini/platforms/linux @@ -146,26 +152,6 @@ if [[ $? != 0 ]]; then exit 1; fi -# build iwasm with llvm jit lazy mode enabled -cd ${WAMR_DIR}/product-mini/platforms/linux -rm -rf build && mkdir build && cd build -cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_JIT=1 -make -j -if [[ $? != 0 ]]; then - echo "Failed to build llvm jit lazy mode enabled!" - exit 1; -fi - -# build iwasm with llvm jit eager mode enabled -cd ${WAMR_DIR}/product-mini/platforms/linux -rm -rf build && mkdir build && cd build -cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0 -make -j -if [[ $? != 0 ]]; then - echo "Failed to build llvm jit eager mode enabled!" - exit 1; -fi - # build iwasm with fast jit lazy mode enabled cd ${WAMR_DIR}/product-mini/platforms/linux rm -rf build && mkdir build && cd build @@ -186,6 +172,28 @@ if [[ $? != 0 ]]; then exit 1; fi +# TODO: use pre-built llvm binary to build llvm-jit and multi-tier-jit +: ' +# build iwasm with llvm jit lazy mode enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_JIT=1 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build llvm jit lazy mode enabled!" + exit 1; +fi + +# build iwasm with llvm jit eager mode enabled +cd ${WAMR_DIR}/product-mini/platforms/linux +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0 +make -j +if [[ $? != 0 ]]; then + echo "Failed to build llvm jit eager mode enabled!" + exit 1; +fi + # build iwasm with multi-tier jit enabled cd ${WAMR_DIR}/product-mini/platforms/linux rm -rf build && mkdir build && cd build @@ -196,6 +204,7 @@ if [[ $? != 0 ]]; then echo "Failed to build iwasm with multi-tier jit enabled!" exit 1; fi +' # build iwasm with wasm mini-loader enabled cd ${WAMR_DIR}/product-mini/platforms/linux diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index b96072a0b..9d13e41e2 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -188,11 +188,17 @@ jobs: make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS make_options_feature: "-DWAMR_BUILD_MINI_LOADER=1" - # Fast-JIT and Multi-Tier-JIT mode don't support android(X86-32) + # Fast-JIT and Multi-Tier-JIT mode don't support android - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS platform: android - make_options_run_mode: $MULTI_TIER_JIT_BUILD_OPTIONS platform: android + # LLVM JIT pre-built binary wasn't compiled by Android NDK + # and isn't available for android + - make_options_run_mode: $LLVM_LAZY_JIT_BUILD_OPTIONS + platform: android + - make_options_run_mode: $LLVM_EAGER_JIT_BUILD_OPTIONS + platform: android include: - os: ubuntu-20.04 llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} @@ -219,13 +225,23 @@ jobs: if: endsWith(matrix.make_options_run_mode, '_JIT_BUILD_OPTIONS') && (steps.retrieve_llvm_libs.outputs.cache-hit != 'true') run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - - name: Build iwasm + - name: Build iwasm for linux + if: matrix.platform == 'linux' run: | mkdir build && cd build cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} cmake --build . --config Release --parallel 4 working-directory: product-mini/platforms/${{ matrix.platform }} + - name: Build iwasm for android + if: matrix.platform == 'android' + run: | + mkdir build && cd build + cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} \ + -DWAMR_BUILD_TARGET=X86_64 + cmake --build . --config Release --parallel 4 + working-directory: product-mini/platforms/${{ matrix.platform }} + build_iwasm_linux_gcc4_8: runs-on: ubuntu-latest container: