Enable running spec tests on Windows (#2423)

Update wamr-test-suites scripts to enable running spec tests on Windows.
We don't enable those tests in CI yet as not all of them are passing.

core/iwasm/interpreter/wasm_runtime.c

@@ -308,8 +308,9 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
     }
 
 #ifdef BH_PLATFORM_WINDOWS
-    if (!os_mem_commit(mapped_mem, memory_data_size,
+    if (memory_data_size > 0
-                       MMAP_PROT_READ | MMAP_PROT_WRITE)) {
+        && !os_mem_commit(mapped_mem, memory_data_size,
+                          MMAP_PROT_READ | MMAP_PROT_WRITE)) {
         set_error_buf(error_buf, error_buf_size, "commit memory failed");
         os_munmap(mapped_mem, map_size);
         goto fail1;

product-mini/platforms/windows/main.c

@@ -142,7 +142,8 @@ app_instance_repl(wasm_module_inst_t module_inst)
     char *cmd;
     size_t n;
 
-    while ((printf("webassembly> "), cmd = fgets(buffer, sizeof(buffer), stdin))
+    while ((printf("webassembly> "), fflush(stdout),
+            cmd = fgets(buffer, sizeof(buffer), stdin))
            != NULL) {
         bh_assert(cmd);
         n = strlen(cmd);

tests/wamr-test-suites/spec-test-script/all.py

@@ -6,7 +6,7 @@
 
 import argparse
 import multiprocessing as mp
-import os
+import platform
 import pathlib
 import subprocess
 import sys
@@ -28,12 +28,26 @@ To run a single GC case:
     --aot-compiler wamrc --gc spec/test/core/xxx.wast
 """
 
-PLATFORM_NAME = os.uname().sysname.lower()
+def exe_file_path(base_path: str) -> str:
-IWASM_CMD = "../../../product-mini/platforms/" + PLATFORM_NAME + "/build/iwasm"
+    if platform.system().lower() == "windows":
+        base_path += ".exe"
+    return base_path
+
+def get_iwasm_cmd(platform: str) -> str:
+    build_path = "../../../product-mini/platforms/" + platform + "/build/"
+    exe_name = "iwasm"
+
+    if platform == "windows":
+        build_path += "RelWithDebInfo/"
+
+    return exe_file_path(build_path + exe_name)
+
+PLATFORM_NAME = platform.uname().system.lower()
+IWASM_CMD = get_iwasm_cmd(PLATFORM_NAME)
 IWASM_SGX_CMD = "../../../product-mini/platforms/linux-sgx/enclave-sample/iwasm"
 IWASM_QEMU_CMD = "iwasm"
 SPEC_TEST_DIR = "spec/test/core"
-WAST2WASM_CMD = "./wabt/out/gcc/Release/wat2wasm"
+WAST2WASM_CMD = exe_file_path("./wabt/out/gcc/Release/wat2wasm")
 SPEC_INTERPRETER_CMD = "spec/interpreter/wasm"
 WAMRC_CMD = "../../../wamr-compiler/build/wamrc"
 
@@ -133,6 +147,7 @@ def test_case(
     qemu_flag=False,
     qemu_firmware='',
     log='',
+    no_pty=False
 ):
     case_path = pathlib.Path(case_path).resolve()
     case_name = case_path.stem
@@ -151,7 +166,7 @@ def test_case(
     ):
         return True
 
-    CMD = ["python3", "runtest.py"]
+    CMD = [sys.executable, "runtest.py"]
     CMD.append("--wast2wasm")
     CMD.append(WAST2WASM_CMD if not gc_flag else SPEC_INTERPRETER_CMD)
     CMD.append("--interpreter")
@@ -161,6 +176,8 @@ def test_case(
         CMD.append(IWASM_QEMU_CMD)
     else:
         CMD.append(IWASM_CMD)
+    if no_pty:
+        CMD.append("--no-pty")
     CMD.append("--aot-compiler")
     CMD.append(WAMRC_CMD)
 
@@ -261,6 +278,7 @@ def test_suite(
     qemu_flag=False,
     qemu_firmware='',
     log='',
+    no_pty=False
 ):
     suite_path = pathlib.Path(SPEC_TEST_DIR).resolve()
     if not suite_path.exists():
@@ -302,6 +320,7 @@ def test_suite(
                         qemu_flag,
                         qemu_firmware,
                         log,
+                        no_pty,
                     ],
                 )
 
@@ -339,6 +358,7 @@ def test_suite(
                     qemu_flag,
                     qemu_firmware,
                     log,
+                    no_pty,
                 )
                 successful_case += 1
             except Exception as e:
@@ -460,6 +480,8 @@ def main():
         nargs="*",
         help=f"Specify all wanted cases. If not the script will go through all cases under {SPEC_TEST_DIR}",
     )
+    parser.add_argument('--no-pty', action='store_true',
+        help="Use direct pipes instead of pseudo-tty")
 
     options = parser.parse_args()
     print(options)
@@ -490,6 +512,7 @@ def main():
             options.qemu_flag,
             options.qemu_firmware,
             options.log,
+            options.no_pty
         )
         end = time.time_ns()
         print(
@@ -512,7 +535,8 @@ def main():
                     options.gc_flag,
                     options.qemu_flag,
                     options.qemu_firmware,
-                    options.log
+                    options.log,
+                    options.no_pty
                 )
             else:
                 ret = True

tests/wamr-test-suites/spec-test-script/runtest.py

@@ -5,22 +5,21 @@ from __future__ import print_function
 import argparse
 import array
 import atexit
-import fcntl
 import math
 import os
-# Pseudo-TTY and terminal manipulation
-import pty
 import re
 import shutil
 import struct
 import subprocess
 import sys
 import tempfile
-import termios
 import time
+import threading
 import traceback
 from select import select
+from queue import Queue
 from subprocess import PIPE, STDOUT, Popen
+from typing import BinaryIO, Optional, Tuple
 
 if sys.version_info[0] == 2:
     IS_PY_3 = False
@@ -52,6 +51,10 @@ def log(data, end='\n'):
     print(data, end=end)
     sys.stdout.flush()
 
+def create_tmp_file(suffix: str) -> str:
+    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp_file:
+        return tmp_file.name
+
 # TODO: do we need to support '\n' too
 import platform
 
@@ -62,6 +65,34 @@ else:
     sep = "\r\n"
 rundir = None
 
+
+class AsyncStreamReader:
+    def __init__(self, stream: BinaryIO) -> None:
+        self._queue = Queue()
+        self._reader_thread = threading.Thread(
+            daemon=True,
+            target=AsyncStreamReader._stdout_reader,
+            args=(self._queue, stream))
+        self._reader_thread.start()
+
+    def read(self) -> Optional[bytes]:
+        return self._queue.get()
+
+    def cleanup(self) -> None:
+        self._reader_thread.join()
+
+    @staticmethod
+    def _stdout_reader(queue: Queue, stdout: BinaryIO) -> None:
+        while True:
+            try:
+                queue.put(stdout.read(1))
+            except ValueError as e:
+                if stdout.closed:
+                    queue.put(None)
+                    break
+                raise e
+
+
 class Runner():
     def __init__(self, args, no_pty=False):
         self.no_pty = no_pty
@@ -77,11 +108,14 @@ class Runner():
         if no_pty:
             self.process = Popen(args, bufsize=0,
                            stdin=PIPE, stdout=PIPE, stderr=STDOUT,
-                           preexec_fn=os.setsid,
                            env=env)
             self.stdin = self.process.stdin
             self.stdout = self.process.stdout
         else:
+            import fcntl
+            # Pseudo-TTY and terminal manipulation
+            import pty
+            import termios
             # Use tty to setup an interactive environment
             master, slave = pty.openpty()
 
@@ -101,35 +135,53 @@ class Runner():
             self.stdin = os.fdopen(master, 'r+b', 0)
             self.stdout = self.stdin
 
+        if platform.system().lower() == "windows":
+            self._stream_reader = AsyncStreamReader(self.stdout)
+        else:
+            self._stream_reader = None
+
         self.buf = ""
 
+    def _read_stdout_byte(self) -> Tuple[bool, Optional[bytes]]:
+        if self._stream_reader:
+            return True, self._stream_reader.read()
+        else:
+            # select doesn't work on file descriptors on Windows.
+            # however, this method is much faster than using
+            # queue, so we keep it for non-windows platforms.
+            [outs,_,_] = select([self.stdout], [], [], 1)
+            if self.stdout in outs:
+                return True, self.stdout.read(1)
+            else:
+                return False, None
+
     def read_to_prompt(self, prompts, timeout):
         wait_until = time.time() + timeout
         while time.time() < wait_until:
-            [outs,_,_] = select([self.stdout], [], [], 1)
+            has_value, read_byte = self._read_stdout_byte()
-            if self.stdout in outs:
+            if not has_value:
-                read_byte = self.stdout.read(1)
+                continue
-                if not read_byte:
+            if not read_byte:
-                    # EOF on macOS ends up here.
+                # EOF on macOS ends up here.
-                    break
+                break
-                read_byte = read_byte.decode('utf-8') if IS_PY_3 else read_byte
+            read_byte = read_byte.decode('utf-8') if IS_PY_3 else read_byte
 
-                debug(read_byte)
+            debug(read_byte)
-                if self.no_pty:
+            if self.no_pty:
-                    self.buf += read_byte.replace('\n', '\r\n')
+                self.buf += read_byte.replace('\n', '\r\n')
-                else:
+            else:
-                    self.buf += read_byte
+                self.buf += read_byte
-                self.buf = self.buf.replace('\r\r', '\r')
+            self.buf = self.buf.replace('\r\r', '\r')
 
-                # filter the prompts
+            # filter the prompts
-                for prompt in prompts:
+            for prompt in prompts:
-                    pattern = re.compile(prompt)
+                pattern = re.compile(prompt)
-                    match = pattern.search(self.buf)
+                match = pattern.search(self.buf)
-                    if match:
+                if match:
-                        end = match.end()
+                    end = match.end()
-                        buf = self.buf[0:end-len(prompt)]
+                    buf = self.buf[0:end-len(prompt)]
-                        self.buf = self.buf[end:]
+                    self.buf = self.buf[end:]
-                        return buf
+                    return buf
         return None
 
     def writeline(self, str):
@@ -140,6 +192,8 @@ class Runner():
         self.stdin.write(str_to_write)
 
     def cleanup(self):
+        atexit.unregister(self.cleanup)
+
         if self.process:
             try:
                 self.writeline("__exit__")
@@ -157,6 +211,8 @@ class Runner():
             self.stdout = None
             if not IS_PY_3:
                 sys.exc_clear()
+            if self._stream_reader:
+                self._stream_reader.cleanup()
 
 def assert_prompt(runner, prompts, timeout, is_need_execute_result):
     # Wait for the initial prompt
@@ -402,9 +458,9 @@ def cast_v128_to_i64x2(numbers, type, lane_type):
     unpacked = struct.unpack("Q Q", packed)
     return unpacked, "[{} {}]:{}:v128".format(unpacked[0], unpacked[1], lane_type)
 
-
 def parse_simple_const_w_type(number, type):
     number = number.replace('_', '')
+    number = re.sub(r"nan\((ind|snan)\)", "nan", number)
     if type in ["i32", "i64"]:
         number = int(number, 16) if '0x' in number else int(number)
         return number, "0x{:x}:{}".format(number, type) \
@@ -948,7 +1004,8 @@ def skip_test(form, skip_list):
 
 def compile_wast_to_wasm(form, wast_tempfile, wasm_tempfile, opts):
     log("Writing WAST module to '%s'" % wast_tempfile)
-    open(wast_tempfile, 'w').write(form)
+    with open(wast_tempfile, 'w') as file:
+        file.write(form)
     log("Compiling WASM to '%s'" % wasm_tempfile)
 
     # default arguments
@@ -1070,13 +1127,10 @@ def run_wasm_with_repl(wasm_tempfile, aot_tempfile, opts, r):
 def create_tmpfiles(wast_name):
     tempfiles = []
 
-    (t1fd, wast_tempfile) = tempfile.mkstemp(suffix=".wast")
+    tempfiles.append(create_tmp_file(".wast"))
-    (t2fd, wasm_tempfile) = tempfile.mkstemp(suffix=".wasm")
+    tempfiles.append(create_tmp_file(".wasm"))
-    tempfiles.append(wast_tempfile)
-    tempfiles.append(wasm_tempfile)
     if test_aot:
-        (t3fd, aot_tempfile) = tempfile.mkstemp(suffix=".aot")
+        tempfiles.append(create_tmp_file(".aot"))
-        tempfiles.append(aot_tempfile)
 
     # add these temp file to temporal repo, will be deleted when finishing the test
     temp_file_repo.extend(tempfiles)
@@ -1145,10 +1199,10 @@ if __name__ == "__main__":
     else:
         SKIP_TESTS = C_SKIP_TESTS
 
-    (t1fd, wast_tempfile) = tempfile.mkstemp(suffix=".wast")
+    wast_tempfile = create_tmp_file(".wast")
-    (t2fd, wasm_tempfile) = tempfile.mkstemp(suffix=".wasm")
+    wasm_tempfile = create_tmp_file(".wasm")
     if test_aot:
-        (t3fd, aot_tempfile) = tempfile.mkstemp(suffix=".aot")
+        aot_tempfile = create_tmp_file(".aot")
 
     ret_code = 0
     try:
@@ -1179,17 +1233,16 @@ if __name__ == "__main__":
                     # workaround: spec test changes error message to "malformed" while iwasm still use "invalid"
                     error_msg = m.group(2).replace("malformed", "invalid")
                     log("Testing(malformed)")
-                    f = open(wasm_tempfile, 'wb')
+                    with open(wasm_tempfile, 'wb') as f:
-                    s = m.group(1)
+                        s = m.group(1)
-                    while s:
+                        while s:
-                        res = re.match("[^\"]*\"([^\"]*)\"(.*)", s, re.DOTALL)
+                            res = re.match("[^\"]*\"([^\"]*)\"(.*)", s, re.DOTALL)
-                        if IS_PY_3:
+                            if IS_PY_3:
-                            context = res.group(1).replace("\\", "\\x").encode("latin1").decode("unicode-escape").encode("latin1")
+                                context = res.group(1).replace("\\", "\\x").encode("latin1").decode("unicode-escape").encode("latin1")
-                            f.write(context)
+                                f.write(context)
-                        else:
+                            else:
-                            f.write(res.group(1).replace("\\", "\\x").decode("string-escape"))
+                                f.write(res.group(1).replace("\\", "\\x").decode("string-escape"))
-                        s = res.group(2)
+                            s = res.group(2)
-                    f.close()
 
                     # compile wasm to aot
                     if test_aot:

tests/wamr-test-suites/test_wamr.sh

@@ -51,7 +51,13 @@ ENABLE_GC_HEAP_VERIFY=0
 #unit test case arrary
 TEST_CASE_ARR=()
 SGX_OPT=""
-PLATFORM=$(uname -s | tr A-Z a-z)
+if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
+    PLATFORM=windows
+    PYTHON_EXE=python
+else
+    PLATFORM=$(uname -s | tr A-Z a-z)
+    PYTHON_EXE=python3
+fi
 PARALLELISM=0
 ENABLE_QEMU=0
 QEMU_FIRMWARE=""
@@ -385,15 +391,18 @@ function spec_test()
                 darwin)
                     WABT_PLATFORM=macos
                     ;;
+                windows)
+                    WABT_PLATFORM=windows
+                    ;;
                 *)
                     echo "wabt platform for ${PLATFORM} in unknown"
                     exit 1
                     ;;
             esac
             if [ ! -f /tmp/wabt-1.0.31-${WABT_PLATFORM}.tar.gz ]; then
-                wget \
+                curl -L \
                     https://github.com/WebAssembly/wabt/releases/download/1.0.31/wabt-1.0.31-${WABT_PLATFORM}.tar.gz \
-                    -P /tmp
+                    -o /tmp/wabt-1.0.31-${WABT_PLATFORM}.tar.gz
             fi
 
             cd /tmp \
@@ -471,12 +480,16 @@ function spec_test()
         ARGS_FOR_SPEC_TEST+="--qemu-firmware ${QEMU_FIRMWARE} "
     fi
 
+    if [[ ${PLATFORM} == "windows" ]]; then
+        ARGS_FOR_SPEC_TEST+="--no-pty "
+    fi
+
     # set log directory
     ARGS_FOR_SPEC_TEST+="--log ${REPORT_DIR}"
 
     cd ${WORK_DIR}
-    echo "python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt"
+    echo "${PYTHON_EXE} ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt"
-    python3 ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt
+    ${PYTHON_EXE} ./all.py ${ARGS_FOR_SPEC_TEST} | tee -a ${REPORT_DIR}/spec_test_report.txt
     if [[ ${PIPESTATUS[0]} -ne 0 ]];then
         echo -e "\nspec tests FAILED" | tee -a ${REPORT_DIR}/spec_test_report.txt
         exit 1
@@ -645,7 +658,7 @@ function build_iwasm_with_cfg()
         && if [ -d build ]; then rm -rf build/*; else mkdir build; fi \
         && cd build \
         && cmake $* .. \
-        && make -j 4
+        && cmake --build . -j 4 --config RelWithDebInfo
     fi
 
     if [ "$?" != 0 ];then