diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 2fea0c994..ad5cd7b3d 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -55,10 +55,10 @@ env: # LLVM LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex" # For Spec Test - DEFAULT_TEST_OPTIONS: "-s spec -P" - MULTI_MODULES_TEST_OPTIONS: "-s spec -M -P" - SIMD_TEST_OPTIONS: "-s spec -S -P" - THREADS_TEST_OPTIONS: "-s spec -p -P" + DEFAULT_TEST_OPTIONS: "-s spec -b -P" + MULTI_MODULES_TEST_OPTIONS: "-s spec -b -M -P" + SIMD_TEST_OPTIONS: "-s spec -b -S -P" + THREADS_TEST_OPTIONS: "-s spec -b -p -P" X86_32_TARGET_TEST_OPTIONS: "-m x86_32 -P" jobs: @@ -404,9 +404,9 @@ jobs: run: echo "USE_LLVM=true" >> $GITHUB_ENV - name: set env variable(if x86_32 test needed) - if: > - (matrix.test_option == '$DEFAULT_TEST_OPTIONS' || matrix.test_option == '$THREADS_TEST_OPTIONS') - && matrix.running_mode != 'fast-jit' && matrix.running_mode != 'jit' + if: > + (matrix.test_option == '$DEFAULT_TEST_OPTIONS' || matrix.test_option == '$THREADS_TEST_OPTIONS') + && matrix.running_mode != 'fast-jit' && matrix.running_mode != 'jit' run: echo "TEST_ON_X86_32=true" >> $GITHUB_ENV #only download llvm libraries in jit and aot mode @@ -427,9 +427,6 @@ jobs: if: env.USE_LLVM == 'true' && steps.cache_llvm.outputs.cache-hit != 'true' run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - - name: install Ninja - run: sudo apt install -y ninja-build - - name: run spec tests default and extra run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} working-directory: ./tests/wamr-test-suites diff --git a/.github/workflows/compilation_on_sgx.yml b/.github/workflows/compilation_on_sgx.yml index 13e4d4bbd..70eeaec02 100644 --- a/.github/workflows/compilation_on_sgx.yml +++ b/.github/workflows/compilation_on_sgx.yml @@ -187,8 +187,14 @@ jobs: #$AOT_BUILD_OPTIONS, ] os: [ubuntu-20.04] - wasi_sdk_release: ["https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz"] - wabt_release: ["https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-ubuntu.tar.gz"] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz", + ] + wabt_release: + [ + "https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-ubuntu.tar.gz", + ] steps: - name: checkout uses: actions/checkout@v3 @@ -236,8 +242,14 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - wasi_sdk_release: ["https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz"] - wabt_release: ["https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-ubuntu.tar.gz"] + wasi_sdk_release: + [ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz", + ] + wabt_release: + [ + "https://github.com/WebAssembly/wabt/releases/download/1.0.24/wabt-1.0.24-ubuntu.tar.gz", + ] steps: - name: checkout uses: actions/checkout@v3 @@ -321,13 +333,13 @@ jobs: strategy: matrix: running_mode: ["classic-interp", "fast-interp", "aot"] - test_option: ["-x -p -s spec -P", "-x -p -s spec -S -P"] + test_option: ["-x -p -s spec -b -P", "-x -p -s spec -S -b -P"] # classic-interp and fast-interp don't support simd exclude: - running_mode: "classic-interp" - test_option: "-x -p -s spec -S -P" + test_option: "-x -p -s spec -S -b -P" - running_mode: "fast-interp" - test_option: "-x -p -s spec -S -P" + test_option: "-x -p -s spec -S -b -P" steps: - name: checkout @@ -350,9 +362,6 @@ jobs: if: matrix.running_mode == 'aot' && steps.cache_llvm.outputs.cache-hit != 'true' run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - - name: install Ninja - run: sudo apt install -y ninja-build - - name: install SGX SDK and necessary libraries run: | mkdir -p /opt/intel diff --git a/tests/wamr-test-suites/spec-test-script/all.py b/tests/wamr-test-suites/spec-test-script/all.py index f016dafae..4cf464dc8 100644 --- a/tests/wamr-test-suites/spec-test-script/all.py +++ b/tests/wamr-test-suites/spec-test-script/all.py @@ -121,7 +121,7 @@ def test_case( ): return True - CMD = ["python", "runtest.py"] + CMD = ["python3", "runtest.py"] CMD.append("--wast2wasm") CMD.append(WAST2WASM_CMD) CMD.append("--interpreter") @@ -270,9 +270,9 @@ def test_suite( verbose_flag, ) successful_case += 1 - except Exception: + except Exception as e: failed_case += 1 - break + raise e print( f"IN ALL {case_count} cases: {successful_case} PASS, {failed_case} FAIL, {case_count - successful_case - failed_case} SKIP" diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index 5028b2ad2..84185a748 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -413,6 +413,7 @@ function spec_test() fi cd ${WORK_DIR} + echo "python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt" python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt [[ ${PIPESTATUS[0]} -ne 0 ]] && exit 1 cd -