Merge branch 'main' into dev/multi_memory

.github/workflows/build_llvm_libraries.yml

@@ -14,6 +14,14 @@ on:
       container_image:
         required: false
         type: string
+      extra_build_llvm_options:
+        required: false
+        type: string
+        default: ""
+      cache_key_suffix:
+        required: false
+        type: string
+        default: ""
     outputs:
       cache_key:
         description: "A cached key of LLVM libraries"
@@ -45,7 +53,7 @@ jobs:
 
       - name: retrieve the last commit ID
         id: get_last_commit
-        run: echo "last_commit=$(GH_TOKEN=${{ secrets.GITHUB_TOKEN }} /usr/bin/env python3 ./build_llvm.py --llvm-ver)" >> $GITHUB_OUTPUT
+        run: echo "last_commit=$(GH_TOKEN=${{ secrets.GITHUB_TOKEN }} /usr/bin/env python3 ./build_llvm.py ${{ inputs.extra_build_llvm_options }} --llvm-ver)" >> $GITHUB_OUTPUT
         working-directory: build-scripts
 
       # Bump the prefix number to evict all previous caches and
@@ -54,7 +62,7 @@ jobs:
       # suspect.
       - name: form the cache key of libraries
         id: create_lib_cache_key
-        run: echo "key=0-llvm-libraries-${{ inputs.os }}-${{ inputs.arch }}-${{ steps.get_last_commit.outputs.last_commit }}" >> $GITHUB_OUTPUT
+        run: echo "key=0-llvm-libraries-${{ inputs.os }}-${{ inputs.arch }}-${{ steps.get_last_commit.outputs.last_commit }}${{ inputs.cache_key_suffix }}" >> $GITHUB_OUTPUT
 
       - name: Cache LLVM libraries
         id: retrieve_llvm_libs
@@ -101,5 +109,5 @@ jobs:
 
       - name: Build LLVM libraries
         if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true'
-        run: /usr/bin/env python3 ./build_llvm.py --arch ${{ inputs.arch }}
+        run: /usr/bin/env python3 ./build_llvm.py ${{ inputs.extra_build_llvm_options }} --arch ${{ inputs.arch }}
         working-directory: build-scripts

.github/workflows/nightly_run.yml

@@ -323,16 +323,13 @@ jobs:
           - make_options_run_mode: $FAST_JIT_BUILD_OPTIONS
             make_options_feature: "-DWAMR_BUILD_MEMORY64=1"
     steps:
-      - name: checkout
-        uses: actions/checkout@v3
-
       - name: Install dependencies
-        uses: nick-fields/retry@v2
+        run: |
-        with:
+          apt update && apt install -y make g++-4.8 gcc-4.8 wget git
-          timeout_minutes: 10
+
-          max_attempts: 3
+      - name: checkout
-          command: apt update && apt install -y make g++-4.8 gcc-4.8 wget git
+        run: |
-          on_retry_command: sudo rm -r /var/lib/apt/lists/*
+          git clone https://github.com/${{ github.repository }} wamr
 
       - name: Install cmake
         run: |
@@ -345,7 +342,7 @@ jobs:
           mkdir build && cd build
           cmake .. ${{ matrix.make_options_run_mode }} ${{ matrix.make_options_feature }} -DCMAKE_C_COMPILER=gcc-4.8 -DCMAKE_CXX_COMPILER=g++-4.8
           cmake --build . --config Release --parallel 4
-        working-directory: product-mini/platforms/linux
+        working-directory: wamr/product-mini/platforms/linux
 
   build_samples_wasm_c_api:
     needs:

.github/workflows/spec_test_on_nuttx.yml

@@ -39,6 +39,15 @@ jobs:
       arch: "ARM RISCV AArch64"
       container_image: ghcr.io/no1wudi/nuttx/apache-nuttx-ci-linux@sha256:8c4e00b607d4d6d66ba8f51c4544819a616eac69d3a2ac669e2af2150e2eb0f9
 
+  build_llvm_libraries_xtensa:
+    uses: ./.github/workflows/build_llvm_libraries.yml
+    with:
+      os: "ubuntu-22.04"
+      arch: "Xtensa"
+      extra_build_llvm_options: "--platform xtensa"
+      cache_key_suffix: "-xtensa"
+      container_image: ghcr.io/no1wudi/nuttx/apache-nuttx-ci-linux@sha256:8c4e00b607d4d6d66ba8f51c4544819a616eac69d3a2ac669e2af2150e2eb0f9
+
   spec_test_on_qemu:
     runs-on: ubuntu-latest
     needs: [build_llvm_libraries]

tests/wamr-test-suites/spec-test-script/all.py

@@ -129,7 +129,7 @@ def ignore_the_case(
     return False
 
 
-def preflight_check(aot_flag, eh_flag):
+def preflight_check(aot_flag, aot_compiler, eh_flag):
     if not pathlib.Path(SPEC_TEST_DIR).resolve().exists():
         print(f"Can not find {SPEC_TEST_DIR}")
         return False
@@ -138,8 +138,8 @@ def preflight_check(aot_flag, eh_flag):
         print(f"Can not find {WAST2WASM_CMD}")
         return False
 
-    if aot_flag and not pathlib.Path(WAMRC_CMD).resolve().exists():
+    if aot_flag and not pathlib.Path(aot_compiler).resolve().exists():
-        print(f"Can not find {WAMRC_CMD}")
+        print(f"Can not find {aot_compiler}")
         return False
 
     return True
@@ -149,6 +149,7 @@ def test_case(
     case_path,
     target,
     aot_flag=False,
+    aot_compiler=WAMRC_CMD,
     sgx_flag=False,
     multi_module_flag=False,
     multi_thread_flag=False,
@@ -177,7 +178,7 @@ def test_case(
     if no_pty:
         CMD.append("--no-pty")
     CMD.append("--aot-compiler")
-    CMD.append(WAMRC_CMD)
+    CMD.append(aot_compiler)
 
     if aot_flag:
         CMD.append("--aot")
@@ -274,6 +275,7 @@ def test_case(
 def test_suite(
     target,
     aot_flag=False,
+    aot_compiler=WAMRC_CMD,
     sgx_flag=False,
     multi_module_flag=False,
     multi_thread_flag=False,
@@ -348,6 +350,7 @@ def test_suite(
                         str(case_path),
                         target,
                         aot_flag,
+                        aot_compiler,
                         sgx_flag,
                         multi_module_flag,
                         multi_thread_flag,
@@ -389,6 +392,7 @@ def test_suite(
                     str(case_path),
                     target,
                     aot_flag,
+                    aot_compiler,
                     sgx_flag,
                     multi_module_flag,
                     multi_thread_flag,
@@ -470,6 +474,12 @@ def main():
         dest="aot_flag",
         help="Running with AOT mode",
     )
+    parser.add_argument(
+        "--aot-compiler",
+        default=WAMRC_CMD,
+        dest="aot_compiler",
+        help="AOT compiler",
+    )
     parser.add_argument(
         "-x",
         action="store_true",
@@ -550,7 +560,7 @@ def main():
     if options.target == "x86_32":
         options.target = "i386"
 
-    if not preflight_check(options.aot_flag, options.eh_flag):
+    if not preflight_check(options.aot_flag, options.aot_compiler, options.eh_flag):
         return False
 
     if not options.cases:
@@ -564,6 +574,7 @@ def main():
         ret = test_suite(
             options.target,
             options.aot_flag,
+            options.aot_compiler,
             options.sgx_flag,
             options.multi_module_flag,
             options.multi_thread_flag,
@@ -591,6 +602,7 @@ def main():
                     case,
                     options.target,
                     options.aot_flag,
+                    options.aot_compiler,
                     options.sgx_flag,
                     options.multi_module_flag,
                     options.multi_thread_flag,

tests/wamr-test-suites/spec-test-script/runtest.py

@@ -1158,7 +1158,13 @@ def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r):
     tmpfile = aot_tempfile if test_aot else wasm_tempfile
     log("Starting interpreter for module '%s'" % tmpfile)
 
-    cmd_iwasm = [opts.interpreter, "--heap-size=0", "-v=5" if opts.verbose else "-v=0", "--repl", tmpfile]
+    if opts.qemu:
+        tmpfile = f"/tmp/{os.path.basename(tmpfile)}"
+
+    if opts.verbose:
+        cmd_iwasm = [opts.interpreter, "--heap-size=0", "-v=5", "--repl", tmpfile]
+    else:
+        cmd_iwasm = [opts.interpreter, "--heap-size=0", "--repl", tmpfile]
 
     if opts.multi_module:
         cmd_iwasm.insert(1, "--module-path=" + (tempfile.gettempdir() if not opts.qemu else "/tmp" ))

tests/wamr-test-suites/test_wamr.sh

@@ -39,6 +39,7 @@ function help()
     echo "-C enable code coverage collect"
     echo "-j set the platform to test"
     echo "-T set sanitizer to use in tests(ubsan|tsan|asan)"
+    echo "-A use the specified wamrc command instead of building it"
     echo "-r [requirement name] [N [N ...]] specify a requirement name followed by one or more"
     echo "                                  subrequirement IDs, if no subrequirement is specificed,"
     echo "                                  it will run all subrequirements. When this optin is used,"
@@ -75,6 +76,7 @@ fi
 PARALLELISM=0
 ENABLE_QEMU=0
 QEMU_FIRMWARE=""
+WAMRC_CMD=""
 # prod/testsuite-all branch
 WASI_TESTSUITE_COMMIT="ee807fc551978490bf1c277059aabfa1e589a6c2"
 TARGET_LIST=("AARCH64" "AARCH64_VFP" "ARMV7" "ARMV7_VFP" "THUMBV7" "THUMBV7_VFP" \
@@ -83,7 +85,7 @@ REQUIREMENT_NAME=""
 # Initialize an empty array for subrequirement IDs
 SUBREQUIREMENT_IDS=()
 
-while getopts ":s:cabgvt:m:MCpSXexwWPGQF:j:T:r:" opt
+while getopts ":s:cabgvt:m:MCpSXexwWPGQF:j:T:r:A:" opt
 do
     OPT_PARSED="TRUE"
     case $opt in
@@ -92,8 +94,8 @@ do
         # get next suite if there are multiple vaule in -s
         eval "nxarg=\${$((OPTIND))}"
         # just get test cases, loop until the next symbol '-'
-        # IN  ====>  -s spec wasi unit -t fast-classic
+        # IN  ====>  -s spec unit -t fast-classic
-        # GET ====>  spec wasi unit
+        # GET ====>  spec unit
         while [[ "${nxarg}" != -* && ${nxarg} ]];
         do
             TEST_CASE_ARR+=(${nxarg})
@@ -214,6 +216,10 @@ do
         echo "Only Test requirement name: ${REQUIREMENT_NAME}"
         [[ ${#SUBREQUIREMENT_IDS[@]} -ne 0 ]] && echo "Choose subrequirement IDs: ${SUBREQUIREMENT_IDS[@]}"
         ;;
+        A)
+        echo "Using wamrc ${OPTARG}"
+        WAMRC_CMD=${OPTARG}
+        ;;
         ?)
         help
         exit 1
@@ -251,7 +257,7 @@ else
     readonly IWASM_CMD="${WAMR_DIR}/product-mini/platforms/${PLATFORM}/build/iwasm"
 fi
 
-readonly WAMRC_CMD="${WAMR_DIR}/wamr-compiler/build/wamrc"
+readonly WAMRC_CMD_DEFAULT="${WAMR_DIR}/wamr-compiler/build/wamrc"
 
 readonly CLASSIC_INTERP_COMPILE_FLAGS="\
     -DWAMR_BUILD_TARGET=${TARGET} \
@@ -550,6 +556,7 @@ function spec_test()
     # require warmc only in aot mode
     if [[ $1 == 'aot' ]]; then
         ARGS_FOR_SPEC_TEST+="-t "
+        ARGS_FOR_SPEC_TEST+="--aot-compiler ${WAMRC_CMD} "
     fi
 
     if [[ ${PARALLELISM} == 1 ]]; then
@@ -588,22 +595,6 @@ function spec_test()
     echo -e "\nFinish spec tests" | tee -a ${REPORT_DIR}/spec_test_report.txt
 }
 
-function wasi_test()
-{
-    echo "Now start wasi tests"
-    touch ${REPORT_DIR}/wasi_test_report.txt
-
-    cd ${WORK_DIR}/../../wasi
-    [[ $1 != "aot" ]] && \
-        python wasi_test.py --interpreter ${IWASM_CMD} ${SGX_OPT}\
-                            | tee ${REPORT_DIR}/wasi_test_report.txt \
-    || \
-        python wasi_test.py --aot --aot-compiler ${WAMRC_CMD} ${SGX_OPT}\
-                            --interpreter ${IWASM_CMD} \
-                            | tee ${REPORT_DIR}/wasi_test_report.txt
-    echo "Finish wasi tests"
-}
-
 function wamr_compiler_test()
 {
     if [[ $1 != "aot" ]]; then
@@ -1068,7 +1059,10 @@ function trigger()
                 if [[ ${ENABLE_QEMU} == 0 ]]; then
                     build_iwasm_with_cfg $BUILD_FLAGS
                 fi
+                if [ -z "${WAMRC_CMD}" ]; then
                    build_wamrc
+                   WAMRC_CMD=${WAMRC_CMD_DEFAULT}
+                fi
                 for suite in "${TEST_CASE_ARR[@]}"; do
                     $suite"_test" aot
                 done