diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 06a921001..0245711d8 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -61,6 +61,7 @@ env: SIMD_TEST_OPTIONS: "-s spec -b -S -P" THREADS_TEST_OPTIONS: "-s spec -b -p -P" X86_32_TARGET_TEST_OPTIONS: "-m x86_32 -P" + WASI_TEST_OPTIONS: "-s wasi_certification" jobs: build_llvm_libraries: @@ -376,7 +377,7 @@ jobs: exit $? working-directory: ./samples/simple - spec_test: + test: needs: [build_iwasm, build_llvm_libraries, build_wamrc] runs-on: ubuntu-20.04 strategy: @@ -389,6 +390,7 @@ jobs: $MULTI_MODULES_TEST_OPTIONS, $SIMD_TEST_OPTIONS, $THREADS_TEST_OPTIONS, + $WASI_TEST_OPTIONS, ] exclude: # uncompatiable modes and features @@ -400,6 +402,9 @@ jobs: # aot and jit don't support multi module - running_mode: "aot" test_option: $MULTI_MODULES_TEST_OPTIONS + # aot is WAMR-specific while wasi-testsuite is generic + - running_mode: "aot" + test_option: $WASI_TEST_OPTIONS - running_mode: "jit" test_option: $MULTI_MODULES_TEST_OPTIONS # fast-jit is only tested on default mode, exclude other three @@ -441,7 +446,7 @@ jobs: if: env.USE_LLVM == 'true' && steps.cache_llvm.outputs.cache-hit != 'true' run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - - name: run spec tests default and extra + - name: run tests run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} working-directory: ./tests/wamr-test-suites @@ -456,7 +461,7 @@ jobs: sudo apt-get update && sudo apt install -y g++-multilib lib32gcc-9-dev - - name: run spec tests x86_32 + - name: run tests x86_32 if: env.TEST_ON_X86_32 == 'true' run: ./test_wamr.sh ${{ env.X86_32_TARGET_TEST_OPTIONS }} ${{ matrix.test_option }} -t ${{ matrix.running_mode }} working-directory: ./tests/wamr-test-suites diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index dca7f87c8..00797b312 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -46,6 +46,7 @@ PLATFORM=$(uname -s | tr A-Z a-z) PARALLELISM=0 ENABLE_QEMU=0 QEMU_FIRMWARE="" +WASI_TESTSUITE_COMMIT="1d913f28b3f0d92086d6f50405cf85768e648b54" while getopts ":s:cabt:m:MCpSXxPQF:" opt do @@ -447,7 +448,10 @@ function spec_test() cd ${WORK_DIR} echo "python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt" python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt - [[ ${PIPESTATUS[0]} -ne 0 ]] && exit 1 + if [[ ${PIPESTATUS[0]} -ne 0 ]];then + echo -e "\nspec tests FAILED" | tee -a ${REPORT_DIR}/spec_test_report.txt + exit 1 + fi cd - echo -e "\nFinish spec tests" | tee -a ${REPORT_DIR}/spec_test_report.txt @@ -469,6 +473,38 @@ function wasi_test() echo "Finish wasi tests" } +function wasi_certification_test() +{ + echo "Now start wasi tests" + + cd ${WORK_DIR} + if [ ! -d "wasi-testsuite" ]; then + echo "wasi not exist, clone it from github" + git clone -b prod/testsuite-base \ + --single-branch https://github.com/WebAssembly/wasi-testsuite.git + fi + cd wasi-testsuite + git reset --hard ${WASI_TESTSUITE_COMMIT} + + python3 -m venv wasi-env && source wasi-env/bin/activate + python3 -m pip install -r test-runner/requirements.txt + IWASM_PATH=$(dirname ${IWASM_CMD}) + PATH=${PATH}:${IWASM_PATH} python3 test-runner/wasi_test_runner.py \ + -r adapters/wasm-micro-runtime.sh \ + -t \ + tests/c/testsuite/ \ + tests/assemblyscript/testsuite/ \ + | tee -a ${REPORT_DIR}/wasi_test_report.txt + exit_code=${PIPESTATUS[0]} + deactivate + + if [[ ${exit_code} -ne 0 ]];then + echo -e "\nwasi tests FAILED" | tee -a ${REPORT_DIR}/wasi_test_report.txt + exit 1 + fi + echo -e "\nFinish wasi tests" | tee -a ${REPORT_DIR}/wasi_test_report.txt +} + function polybench_test() { echo "Now start polybench tests"