Run spec test for classic/fast-interp in NuttX CI (#2817)

.github/workflows/spec_test_on_nuttx.yml

@@ -18,6 +18,8 @@ on:
 env:
   LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex"
   WASI_SDK_PATH: "/opt/wasi-sdk"
+  WAMR_COMMON_OPTION:
+    "CONFIG_INTERPRETERS_WAMR=y\\nCONFIG_INTERPRETERS_WAMR_STACKSIZE=32768\\nCONFIG_INTERPRETERS_WAMR_LOG=y\\nCONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN=y\\nCONFIG_INTERPRETERS_WAMR_REF_TYPES=y\\nCONFIG_INTERPRETERS_WAMR_ENABLE_SPEC_TEST=y\\nCONFIG_INTERPRETERS_WAMR_SHARED_MEMORY=y\\nCONFIG_INTERPRETERS_WAMR_BULK_MEMORY=y\\nCONFIG_EOL_IS_LF=y\\nCONFIG_ARM_SEMIHOSTING_HOSTFS=y\\nCONFIG_ARM_SEMIHOSTING_HOSTFS_CACHE_COHERENCE=y\\nCONFIG_RISCV_SEMIHOSTING_HOSTFS=y\\nCONFIG_FS_HOSTFS=y\\nCONFIG_LIBC_FLOATINGPOINT=y\\n"
 
 jobs:
   build_llvm_libraries:
@@ -27,36 +29,78 @@ jobs:
       arch: "ARM RISCV AArch64"
 
   spec_test_on_qemu:
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-22.04
     needs: [build_llvm_libraries]
     strategy:
       matrix:
-        os: [ubuntu-22.04]
+        target_config: [
-        nuttx_board_config: [
+          # {
-          # cortex-a9
+          #   config: "boards/arm64/qemu/qemu-armv8a/configs/nsh",
-          "boards/arm/imx6/sabre-6quad/configs/nsh",
+          #   target: "aarch64_vfp",
-          # riscv32imac
+          #   use_fpu: true
-          "boards/risc-v/qemu-rv/rv-virt/configs/nsh",
+          # },
-          # riscv64imac
+          # {
-          # "boards/risc-v/qemu-rv/rv-virt/configs/nsh64",
+          #   config: "boards/arm/imx6/sabre-6quad/configs/nsh",
+          #   target: "thumbv7",
+          #   use_fpu: false
+          # },
+          {
+            config: "boards/arm/imx6/sabre-6quad/configs/nsh",
+            target: "thumbv7_vfp",
+            use_fpu: true
+          },
+          {
+            config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh",
+            target: "riscv32",
+            use_fpu: false
+          },
+          # {
+          #   config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh",
+          #   target: "riscv32_ilp32d",
+          #   use_fpu: true
+          # },
+          # {
+          #   config: "boards/risc-v/qemu-rv/rv-virt/configs/nsh64",
+          #   target: "riscv64",
+          #   use_fpu: false
+          # },
         ]
+
         wamr_test_option: [
-          # "-t fast-interp",
+          {
-          "-t aot",
+            mode: "-t aot",
-          "-t aot -X"
+            option: "CONFIG_INTERPRETERS_WAMR_AOT=y\\n"
+          },
+          {
+            mode: "-t aot -X",
+            option: "CONFIG_INTERPRETERS_WAMR_AOT=y\\n"
+          },
+          {
+            mode: "-t classic-interp",
+            option: "CONFIG_INTERPRETERS_WAMR_CLASSIC=y\\n"
+          },
+          {
+            mode: "-t fast-interp",
+            option: "CONFIG_INTERPRETERS_WAMR_FAST=y\\n"
+          },
         ]
-        llvm_cache_key: [ "${{ needs.build_llvm_libraries.outputs.cache_key }}" ]
+
     steps:
       - name: Install Utilities
         run: |
           sudo apt install -y kconfig-frontends-nox genromfs
 
       - name: Install ARM Compilers
-        if: contains(matrix.nuttx_board_config, 'arm')
+        if: startsWith(matrix.target_config.config, 'boards/arm')
-        run: sudo apt install -y gcc-arm-none-eabi
+        run: |
+          sudo apt install -y gcc-arm-none-eabi
+          wget --quiet https://developer.arm.com/-/media/Files/downloads/gnu/11.2-2022.02/binrel/gcc-arm-11.2-2022.02-x86_64-aarch64-none-elf.tar.xz
+          xz -d gcc-arm-11.2-2022.02-x86_64-aarch64-none-elf.tar.xz
+          tar xf gcc-arm-11.2-2022.02-x86_64-aarch64-none-elf.tar
+          echo "$PWD/gcc-arm-11.2-2022.02-x86_64-aarch64-none-elf/bin" >> $GITHUB_PATH
 
       - name: Install RISC-V Compilers
-        if: contains(matrix.nuttx_board_config, 'risc-v')
+        if: startsWith(matrix.target_config.config, 'boards/risc-v')
         run: |
           curl -L https://github.com/xpack-dev-tools/riscv-none-elf-gcc-xpack/releases/download/v12.3.0-1/xpack-riscv-none-elf-gcc-12.3.0-1-linux-x64.tar.gz > riscv.tar.gz
           tar xvf riscv.tar.gz
@@ -87,6 +131,7 @@ jobs:
           path: apps/interpreters/wamr/wamr
 
       - name: Get LLVM libraries
+        if: contains(matrix.wamr_test_option.mode, 'aot')
         id: retrieve_llvm_libs
         uses: actions/cache@v3
         with:
@@ -96,61 +141,59 @@ jobs:
             ./core/deps/llvm/build/lib
             ./core/deps/llvm/build/libexec
             ./core/deps/llvm/build/share
-          key: ${{ matrix.llvm_cache_key }}
+          key: ${{ needs.build_llvm_libraries.outputs.cache_key }}
 
       - name: Quit if cache miss
-        if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true'
+        if: contains(matrix.wamr_test_option.mode, 'aot') && steps.retrieve_llvm_libs.outputs.cache-hit != 'true'
         run: echo "::error::can not get prebuilt llvm libraries" && exit 1
 
       - name: Copy LLVM
+        if: contains(matrix.wamr_test_option.mode, 'aot')
         run: cp -r core/deps/llvm apps/interpreters/wamr/wamr/core/deps/llvm
 
       - name: Enable WAMR for NuttX
         run: |
-          find nuttx/boards -name defconfig | xargs sed -i '$a\CONFIG_INTERPRETERS_WAMR=y\nCONFIG_INTERPRETERS_WAMR_STACKSIZE=32768\nCONFIG_INTERPRETERS_WAMR_AOT=y\nCONFIG_INTERPRETERS_WAMR_FAST=y\nCONFIG_INTERPRETERS_WAMR_LOG=y\nCONFIG_INTERPRETERS_WAMR_LIBC_BUILTIN=y\nCONFIG_INTERPRETERS_WAMR_REF_TYPES=y\nCONFIG_INTERPRETERS_WAMR_ENABLE_SPEC_TEST=y\nCONFIG_INTERPRETERS_WAMR_SHARED_MEMORY=y\nCONFIG_INTERPRETERS_WAMR_BULK_MEMORY=y\n'
+          find nuttx/boards -name defconfig | xargs sed -i '$a\${{ env.WAMR_COMMON_OPTION }}'
-          find nuttx/boards -name defconfig | xargs sed -i '$a\CONFIG_EOL_IS_LF=y\nCONFIG_ARM_SEMIHOSTING_HOSTFS=y\nCONFIG_ARM_SEMIHOSTING_HOSTFS_CACHE_COHERENCE=y\nCONFIG_RISCV_SEMIHOSTING_HOSTFS=y\nCONFIG_FS_HOSTFS=y\nCONFIG_LIBC_FLOATINGPOINT=y\n'
+      
+      - name: Enable WAMR Interpreter for NuttX
+        run: |
+          find nuttx/boards -name defconfig | xargs sed -i '$a\${{ matrix.wamr_test_option.option }}'
 
-      - name: Enable additional features for NuttX on RI5C-V
+      - name: Disable FPU for NuttX
-        if: startsWith(matrix.nuttx_board_config, 'boards/risc-v')
+        if: matrix.target_config.use_fpu== false
         run: |
           find nuttx/boards -name defconfig | xargs sed -i '$a\# CONFIG_ARCH_FPU is not set\n'
 
       - name: Build wamrc
+        if: contains(matrix.wamr_test_option.mode, 'aot')
         working-directory: apps/interpreters/wamr/wamr/wamr-compiler
         run: |
           cmake -Bbuild .
           cmake --build build
 
       - name: Build
+        id: build_firmware
         run: |
           cd nuttx
-          tools/configure.sh ${{ matrix.nuttx_board_config }}
+          tools/configure.sh ${{ matrix.target_config.config }}
           make -j$(nproc)
-          echo "firmware=$PWD/nuttx" >> $GITHUB_ENV
+          echo "firmware=$PWD/nuttx" >> $GITHUB_OUTPUT
 
-      - name: Test on ARM
+      - name: Install QEMU for ARM
-        if: endsWith(matrix.nuttx_board_config, 'sabre-6quad/configs/nsh')
+        if: startsWith(matrix.target_config.config, 'boards/arm')
         run: |
           curl -L https://github.com/xpack-dev-tools/qemu-arm-xpack/releases/download/v7.1.0-1/xpack-qemu-arm-7.1.0-1-linux-x64.tar.gz > xpack-qemu-arm.tar.gz
           tar xvf xpack-qemu-arm.tar.gz
-          export PATH=$PATH:$PWD/xpack-qemu-arm-7.1.0-1/bin
+          echo $PWD/xpack-qemu-arm-7.1.0-1/bin >> $GITHUB_PATH
-          cd apps/interpreters/wamr/wamr/tests/wamr-test-suites
-          ./test_wamr.sh -s spec ${{ matrix.wamr_test_option }} -m thumbv7_vfp -b -Q -P -F ${{ env.firmware }}
 
-      - name: Test on RISCV32
+      - name: Install QEMU for RISC-V
-        if: endsWith(matrix.nuttx_board_config, 'rv-virt/configs/nsh')
+        if: startsWith(matrix.target_config.config, 'boards/risc-v')
         run: |
           curl -L https://github.com/xpack-dev-tools/qemu-riscv-xpack/releases/download/v7.1.0-1/xpack-qemu-riscv-7.1.0-1-linux-x64.tar.gz > xpack-qemu-riscv.tar.gz
           tar xvf xpack-qemu-riscv.tar.gz
-          export PATH=$PATH:$PWD/xpack-qemu-riscv-7.1.0-1/bin
+          echo PATH=$PATH:$PWD/xpack-qemu-riscv-7.1.0-1/bin >> $GITHUB_PATH
-          cd apps/interpreters/wamr/wamr/tests/wamr-test-suites
+      
-          ./test_wamr.sh -s spec ${{ matrix.wamr_test_option }} -m riscv32 -b -Q -P -F ${{ env.firmware }}
+      - name: Test
-
-      - name: Test on RISCV64
-        if: endsWith(matrix.nuttx_board_config, 'rv-virt/configs/nsh64')
         run: |
-          curl -L https://github.com/xpack-dev-tools/qemu-riscv-xpack/releases/download/v7.1.0-1/xpack-qemu-riscv-7.1.0-1-linux-x64.tar.gz > xpack-qemu-riscv.tar.gz
-          tar xvf xpack-qemu-riscv.tar.gz
-          export PATH=$PATH:$PWD/xpack-qemu-riscv-7.1.0-1/bin
           cd apps/interpreters/wamr/wamr/tests/wamr-test-suites
-          ./test_wamr.sh -s spec ${{ matrix.wamr_test_option }} -m riscv64 -b -Q -P -F ${{ env.firmware }}
+          ./test_wamr.sh -s spec ${{ matrix.wamr_test_option.mode }} -m ${{ matrix.target_config.target }} -b -Q -P -F ${{ steps.build_firmware.outputs.firmware }}

product-mini/platforms/nuttx/wamr.mk

@@ -13,6 +13,8 @@ else ifeq ($(CONFIG_ARCH_ARMV7M),y)
 WAMR_BUILD_TARGET := THUMBV7EM
 else ifeq ($(CONFIG_ARCH_ARMV8M),y)
 WAMR_BUILD_TARGET := THUMBV8M
+else ifeq ($(CONFIG_ARCH_ARM64),y)
+WAMR_BUILD_TARGET := AARCH64
 else ifeq ($(CONFIG_ARCH_X86),y)
 WAMR_BUILD_TARGET := X86_32
 else ifeq ($(CONFIG_ARCH_X86_64),y)

tests/wamr-test-suites/spec-test-script/all.py

@@ -50,26 +50,23 @@ SPEC_TEST_DIR = "spec/test/core"
 WAST2WASM_CMD = exe_file_path("./wabt/out/gcc/Release/wat2wasm")
 SPEC_INTERPRETER_CMD = "spec/interpreter/wasm"
 WAMRC_CMD = "../../../wamr-compiler/build/wamrc"
-
+AVAILABLE_TARGETS = [
-
+    "I386",
-class TargetAction(argparse.Action):
+    "X86_32",
-    TARGET_MAP = {
+    "X86_64",
-        "ARMV7_VFP": "armv7",
+    "AARCH64",
-        "RISCV32": "riscv32_ilp32",
+    "AARCH64_VFP",
-        "RISCV32_ILP32": "riscv32_ilp32",
+    "ARMV7",
-        "RISCV32_ILP32D": "riscv32_ilp32d",
+    "ARMV7_VFP",
-        "RISCV64": "riscv64_lp64",
+    "RISCV32",
-        "RISCV64_LP64": "riscv64_lp64",
+    "RISCV32_ILP32F",
-        "RISCV64_LP64D": "riscv64_lp64",
+    "RISCV32_ILP32D",
-        "THUMBV7_VFP": "thumbv7",
+    "RISCV64",
-        "X86_32": "i386",
+    "RISCV64_LP64F",
-        "X86_64": "x86_64",
+    "RISCV64_LP64D",
-        "AARCH64": "arm64"
+    "THUMBV7",
-    }
+    "THUMBV7_VFP",
-
+]
-    def __call__(self, parser, namespace, values, option_string=None):
-        setattr(namespace, self.dest, self.TARGET_MAP.get(values, "x86_64"))
-
 
 def ignore_the_case(
     case_name,
@@ -404,8 +401,7 @@ def main():
     )
     parser.add_argument(
         "-m",
-        action=TargetAction,
+        choices=AVAILABLE_TARGETS,
-        choices=list(TargetAction.TARGET_MAP.keys()),
         type=str,
         dest="target",
         default="X86_64",
@@ -505,6 +501,13 @@ def main():
 
     options = parser.parse_args()
 
+    # Convert target to lower case for internal use, e.g. X86_64 -> x86_64
+    # target is always exist, so no need to check it
+    options.target = options.target.lower()
+
+    if options.target == "x86_32":
+        options.target = "i386"
+
     if not preflight_check(options.aot_flag):
         return False
 

tests/wamr-test-suites/spec-test-script/runtest.py

@@ -27,7 +27,9 @@ else:
     IS_PY_3 = True
 
 test_aot = False
-# "x86_64", "i386", "aarch64", "armv7", "thumbv7", "riscv32_ilp32", "riscv32_ilp32d", "riscv32_lp64", "riscv64_lp64d"
+# Available targets:
+#   "aarch64" "aarch64_vfp" "armv7" "armv7_vfp" "thumbv7" "thumbv7_vfp"
+#   "riscv32" "riscv32_ilp32f" "riscv32_ilp32d" "riscv64" "riscv64_lp64f" "riscv64_lp64d"
 test_target = "x86_64"
 
 debug_file = None
@@ -39,6 +41,25 @@ temp_file_repo = []
 # to save the mapping of module files in /tmp by name
 temp_module_table = {}
 
+# AOT compilation options mapping
+aot_target_options_map = {
+    "i386": ["--target=i386"],
+    "x86_32": ["--target=i386"],
+    "x86_64": ["--target=x86_64", "--cpu=skylake"],
+    "aarch64": ["--target=aarch64", "--target-abi=eabi", "--cpu=cortex-a53"],
+    "aarch64_vfp": ["--target=aarch64", "--target-abi=gnueabihf", "--cpu=cortex-a53"],
+    "armv7": ["--target=armv7", "--target-abi=eabi", "--cpu=cortex-a9", "--cpu-features=-neon"],
+    "armv7_vfp": ["--target=armv7", "--target-abi=gnueabihf", "--cpu=cortex-a9"],
+    "thumbv7": ["--target=thumbv7", "--target-abi=eabi", "--cpu=cortex-a9", "--cpu-features=-neon,-vfpv3"],
+    "thumbv7_vfp": ["--target=thumbv7", "--target-abi=gnueabihf", "--cpu=cortex-a9", "--cpu-features=-neon"],
+    "riscv32": ["--target=riscv32", "--target-abi=ilp32", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c"],
+    "riscv32_ilp32f": ["--target=riscv32", "--target-abi=ilp32f", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c,+f"],
+    "riscv32_ilp32d": ["--target=riscv32", "--target-abi=ilp32d", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c,+f,+d"],
+    "riscv64": ["--target=riscv64", "--target-abi=lp64", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c"],
+    "riscv64_lp64f": ["--target=riscv64", "--target-abi=lp64f", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c,+f"],
+    "riscv64_lp64d": ["--target=riscv64", "--target-abi=lp64d", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c,+f,+d"],
+}
+
 def debug(data):
     if debug_file:
         debug_file.write(data)
@@ -1025,27 +1046,8 @@ def compile_wasm_to_aot(wasm_tempfile, aot_tempfile, runner, opts, r, output = '
     log("Compiling AOT to '%s'" % aot_tempfile)
     cmd = [opts.aot_compiler]
 
-    if test_target == "x86_64":
+    if test_target in aot_target_options_map:
-        cmd.append("--target=x86_64")
+        cmd += aot_target_options_map[test_target]
-        cmd.append("--cpu=skylake")
-    elif test_target == "i386":
-        cmd.append("--target=i386")
-    elif test_target == "aarch64":
-        cmd += ["--target=aarch64", "--cpu=cortex-a57"]
-    elif test_target == "armv7":
-        cmd += ["--target=armv7", "--target-abi=gnueabihf"]
-    elif test_target == "thumbv7":
-        cmd += ["--target=thumbv7", "--target-abi=gnueabihf", "--cpu=cortex-a9", "--cpu-features=-neon"]
-    elif test_target == "riscv32_ilp32":
-        cmd += ["--target=riscv32", "--target-abi=ilp32", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c"]
-    elif test_target == "riscv32_ilp32d":
-        cmd += ["--target=riscv32", "--target-abi=ilp32d", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c"]
-    elif test_target == "riscv64_lp64":
-        cmd += ["--target=riscv64", "--target-abi=lp64", "--cpu=generic-rv64", "--cpu-features=+m,+a,+c"]
-    elif test_target == "riscv64_lp64d":
-        cmd += ["--target=riscv64", "--target-abi=lp64d", "--cpu=generic-rv32", "--cpu-features=+m,+a,+c"]
-    else:
-        pass
 
     if opts.sgx:
         cmd.append("-sgx")
@@ -1094,12 +1096,20 @@ def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r):
         if opts.qemu_firmware == '':
             raise Exception("QEMU firmware missing")
 
-        if opts.target == "thumbv7":
+        if opts.target.startswith("aarch64"):
-            cmd = ["qemu-system-arm", "-semihosting", "-M", "sabrelite", "-m", "1024", "-smp", "4", "-nographic", "-kernel", opts.qemu_firmware]
+            cmd = "qemu-system-aarch64 -cpu cortex-a53 -nographic -machine virt,virtualization=on,gic-version=3 -net none -chardev stdio,id=con,mux=on -serial chardev:con -mon chardev=con,mode=readline -kernel".split()
-        elif opts.target == "riscv32_ilp32":
+            cmd.append(opts.qemu_firmware)
-            cmd = ["qemu-system-riscv32", "-semihosting", "-M", "virt,aclint=on", "-cpu", "rv32", "-smp", "8", "-nographic", "-bios", "none", "-kernel", opts.qemu_firmware]
+        elif opts.target.startswith("thumbv7"):
-        elif opts.target == "riscv64_lp64":
+            cmd = "qemu-system-arm -semihosting -M sabrelite -m 1024 -smp 4 -nographic -kernel".split()
-            cmd = ["qemu-system-riscv64", "-semihosting", "-M", "virt,aclint=on", "-cpu", "rv64", "-smp", "8", "-nographic", "-bios", "none", "-kernel", opts.qemu_firmware]
+            cmd.append(opts.qemu_firmware)
+        elif opts.target.startswith("riscv32"):
+            cmd = "qemu-system-riscv32 -semihosting -M virt,aclint=on -cpu rv32 -smp 8 -nographic -bios none -kernel".split()
+            cmd.append(opts.qemu_firmware)
+        elif opts.target.startswith("riscv64"):
+            cmd = "qemu-system-riscv64 -semihosting -M virt,aclint=on -cpu rv64 -smp 8 -nographic -bios none -kernel".split()
+            cmd.append(opts.qemu_firmware)
+        else:
+            raise Exception("Unknwon target for QEMU: %s" % opts.target)
 
     else:
         cmd = cmd_iwasm

tests/wamr-test-suites/test_wamr.sh

@@ -15,7 +15,9 @@ function help()
     echo "test_wamr.sh [options]"
     echo "-c clean previous test results, not start test"
     echo "-s {suite_name} test only one suite (spec|wasi_certification|wamr_compiler)"
-    echo "-m set compile target of iwasm(x86_64|x86_32|armv7_vfp|thumbv7_vfp|riscv64_lp64d|riscv64_lp64|aarch64)"
+    echo "-m set compile target of iwasm(x86_64|x86_32|armv7|armv7_vfp|thumbv7|thumbv7_vfp|"
+    echo "                               riscv32|riscv32_ilp32f|riscv32_ilp32d|riscv64|"
+    echo "                               riscv64_lp64f|riscv64_lp64d|aarch64|aarch64_vfp)"
     echo "-t set compile type of iwasm(classic-interp|fast-interp|jit|aot|fast-jit|multi-tier-jit)"
     echo "-M enable multi module feature"
     echo "-p enable multi thread feature"
@@ -65,6 +67,8 @@ ENABLE_QEMU=0
 QEMU_FIRMWARE=""
 # prod/testsuite-all branch
 WASI_TESTSUITE_COMMIT="ee807fc551978490bf1c277059aabfa1e589a6c2"
+TARGET_LIST=("AARCH64" "AARCH64_VFP" "ARMV7" "ARMV7_VFP" "THUMBV7" "THUMBV7_VFP" \
+             "RISCV32" "RISCV32_ILP32F" "RISCV32_ILP32D" "RISCV64" "RISCV64_LP64F" "RISCV64_LP64D")
 
 while getopts ":s:cabgvt:m:MCpSXxwPGQF:j:T:" opt
 do
@@ -727,9 +731,7 @@ function build_iwasm_with_cfg()
 
 function build_wamrc()
 {
-    if [[ $TARGET == "ARMV7_VFP" || $TARGET == "THUMBV7_VFP"
+    if [[ "${TARGET_LIST[*]}" =~ "${TARGET}" ]]; then
-          || $TARGET == "RISCV32" || $TARGET == "RISCV32_ILP32" || $TARGET == "RISCV32_ILP32D"
-          || $TARGET == "RISCV64" || $TARGET == "RISCV64_LP64D" || $TARGET == "RISCV64_LP64" ]];then
         echo "suppose wamrc is already built"
         return
     fi