From e70867c64fe64d7ddfb8e31e3dcd03032e7a8435 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Wed, 15 Dec 2021 14:39:54 +0800 Subject: [PATCH] Update spec test cases to latest version (#889) Update spec test cases to commit 2460ad02b51fb5ed5824f44de287a8638b19a5f8, and modify wamr test suite script as the SIMD cases have been added into spec main repo by default, no need to clone SIMD repo again when testing SIMD. --- core/iwasm/interpreter/wasm_loader.c | 1 + .../wamr-test-suites/spec-test-script/all.py | 158 +++++++----------- tests/wamr-test-suites/test_wamr.sh | 20 +-- 3 files changed, 68 insertions(+), 111 deletions(-) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 0d99babdf..de4a03813 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -3342,6 +3342,7 @@ create_sections(const uint8 *buf, uint32 size, WASMSection **p_section_list, if (last_section_index != (uint8)-1 && (section_index <= last_section_index)) { set_error_buf(error_buf, error_buf_size, + "unexpected content after last section or " "junk after last section"); return false; } diff --git a/tests/wamr-test-suites/spec-test-script/all.py b/tests/wamr-test-suites/spec-test-script/all.py index 9f88c416b..65a102a31 100644 --- a/tests/wamr-test-suites/spec-test-script/all.py +++ b/tests/wamr-test-suites/spec-test-script/all.py @@ -204,70 +204,59 @@ def test_suite( xip_flag=False, clean_up_flag=True, verbose_flag=True, + parl_flag=False, ): suite_path = pathlib.Path(SPEC_TEST_DIR).resolve() if not suite_path.exists(): print(f"can not find spec test cases at {suite_path}") return False - case_list = sorted(suite_path.glob("**/*.wast")) + case_list = sorted(suite_path.glob("*.wast")) + if simd_flag: + simd_case_list = sorted(suite_path.glob("simd/*.wast")) + case_list.extend(simd_case_list) + case_count = len(case_list) failed_case = 0 successful_case = 0 - for case_path in case_list: - try: - test_case( - str(case_path), - target, - aot_flag, - sgx_flag, - multi_module_flag, - multi_thread_flag, - simd_flag, - xip_flag, - clean_up_flag, - verbose_flag, - ) - successful_case += 1 - except Exception: - failed_case += 1 - break - print( - f"IN ALL {case_count} cases: {successful_case} PASS, {failed_case} FAIL, {case_count - successful_case - failed_case} SKIP" - ) + if parl_flag: + print(f"----- Run the whole spec test suite on {mp.cpu_count()} cores -----") + with mp.Pool() as pool: + results = {} + for case_path in case_list: + results[case_path.stem] = pool.apply_async( + test_case, + [ + str(case_path), + target, + aot_flag, + sgx_flag, + multi_module_flag, + multi_thread_flag, + simd_flag, + xip_flag, + clean_up_flag, + verbose_flag, + ], + ) - return 0 == failed_case - - -def test_suite_parallelly( - target, - aot_flag=False, - sgx_flag=False, - multi_module_flag=False, - multi_thread_flag=False, - simd_flag=False, - xip_flag=False, - clean_up_flag=False, - verbose_flag=False, -): - - suite_path = pathlib.Path(SPEC_TEST_DIR).resolve() - if not suite_path.exists(): - print(f"can not find spec test cases at {suite_path}") - return False - - case_list = sorted(suite_path.glob("**/*.wast")) - case_count = len(case_list) - failed_case = 0 - successful_case = 0 - print(f"----- Run the whole spec test suite on {mp.cpu_count()} cores -----") - with mp.Pool() as pool: - results = {} + for case_name, result in results.items(): + try: + # 5 min / case + result.wait(300) + if not result.successful(): + failed_case += 1 + else: + successful_case += 1 + except mp.TimeoutError: + print(f"{case_name} meets TimeoutError") + failed_case += 1 + else: + print(f"----- Run the whole spec test suite -----") for case_path in case_list: - results[case_path.stem] = pool.apply_async( - test_case, - [ + try: + test_case( str(case_path), target, aot_flag, @@ -278,20 +267,11 @@ def test_suite_parallelly( xip_flag, clean_up_flag, verbose_flag, - ], - ) - - for case_name, result in results.items(): - try: - # 5 min / case - result.wait(300) - if not result.successful(): - failed_case += 1 - else: - successful_case += 1 - except mp.TimeoutError: - print(f"{case_name} meets TimeoutError") + ) + successful_case += 1 + except Exception: failed_case += 1 + break print( f"IN ALL {case_count} cases: {successful_case} PASS, {failed_case} FAIL, {case_count - successful_case - failed_case} SKIP" @@ -396,37 +376,23 @@ def main(): options.clean_up_flag = False options.verbose_flag = False - start = time.time_ns() - ret = test_suite_parallelly( - options.target, - options.aot_flag, - options.sgx_flag, - options.multi_module_flag, - options.multi_thread_flag, - options.simd_flag, - options.xip_flag, - options.clean_up_flag, - options.verbose_flag, - ) - end = time.time_ns() - print( - f"It takes {((end - start) / 1000000):,} ms to run test_suite_parallelly" - ) - else: - start = time.time_ns() - ret = test_suite( - options.target, - options.aot_flag, - options.sgx_flag, - options.multi_module_flag, - options.multi_thread_flag, - options.simd_flag, - options.xip_flag, - options.clean_up_flag, - options.verbose_flag, - ) - end = time.time_ns() - print(f"It takes {((end - start) / 1000000):,} ms to run test_suite") + start = time.time_ns() + ret = test_suite( + options.target, + options.aot_flag, + options.sgx_flag, + options.multi_module_flag, + options.multi_thread_flag, + options.simd_flag, + options.xip_flag, + options.clean_up_flag, + options.verbose_flag, + options.parl_flag, + ) + end = time.time_ns() + print( + f"It takes {((end - start) / 1000000):,} ms to run test_suite {'parallelly' if options.parl_flag else ''}" + ) else: try: for case in options.cases: diff --git a/tests/wamr-test-suites/test_wamr.sh b/tests/wamr-test-suites/test_wamr.sh index 0815567ee..ea66ee353 100755 --- a/tests/wamr-test-suites/test_wamr.sh +++ b/tests/wamr-test-suites/test_wamr.sh @@ -279,8 +279,12 @@ function spec_test() # restore from XX_ignore_cases.patch # resotre branch git checkout -B master - git reset --hard 397399a70565609bf142d211891724e21bffd01f + # [spec] Fix instruction table (#1402) Thu Dec 2 17:21:54 2021 +0100 + git reset --hard 2460ad02b51fb5ed5824f44de287a8638b19a5f8 git apply ../../spec-test-script/ignore_cases.patch + if [[ ${ENABLE_SIMD} == 1 ]]; then + git apply ../../spec-test-script/simd_ignore_cases.patch + fi # udpate thread cases if [ ${ENABLE_MULTI_THREAD} == 1 ]; then @@ -298,20 +302,6 @@ function spec_test() git apply ../../spec-test-script/thread_proposal_ignore_cases.patch fi - # udpate SIMD cases - if [[ ${ENABLE_SIMD} == 1 ]]; then - echo "checkout spec for SIMD proposal" - # check spec test cases for simd - if [[ -z $(git remote | grep "\") ]]; then - git remote add simd https://github.com/WebAssembly/simd.git - fi - - git fetch simd - git checkout simd/main -- test/core/simd - - git apply ../../spec-test-script/simd_ignore_cases.patch - fi - popd echo $(pwd)