Add tsan for fast interp and aot (#2679)

To run it locally:
```bash
export TSAN_OPTIONS=suppressions=<path_to_tsan_suppressions.txt>
./test_wamr.sh <your flags> -T tsan
```

An example for wasi-threads would look like:
```bash
export TSAN_OPTIONS=suppressions=<path_to_tsan_suppressions.txt>
./test_wamr.sh -w -s wasi_certification -t fast-interp -T tsan
```

.github/workflows/nightly_run.yml

@@ -49,10 +49,17 @@ jobs:
     with:
       os: "ubuntu-20.04"
       arch: "X86"
+  build_llvm_libraries_on_ubuntu_2204:
+    uses: ./.github/workflows/build_llvm_libraries.yml
+    with:
+      os: "ubuntu-22.04"
+      arch: "X86"
   
   build_wamrc:
     needs:
-      [build_llvm_libraries_on_ubuntu_2004]
+      [
+        build_llvm_libraries_on_ubuntu_2004,
+      ]
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -90,7 +97,9 @@ jobs:
 
   build_iwasm:
     needs:
-      [build_llvm_libraries_on_ubuntu_2004]
+      [
+        build_llvm_libraries_on_ubuntu_2004,
+      ]
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -497,14 +506,15 @@ jobs:
       [
         build_iwasm,
         build_llvm_libraries_on_ubuntu_2004,
+        build_llvm_libraries_on_ubuntu_2204,
         build_wamrc,
       ]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04]
+        os: [ubuntu-20.04, ubuntu-22.04]
-        sanitizer: ["", "ubsan", "asan"]
+        sanitizer: ["", "ubsan", "asan", "tsan"]
         running_mode:
           [
             "classic-interp",
@@ -530,11 +540,14 @@ jobs:
           - os: ubuntu-20.04
             llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }}
             ubuntu_version: "20.04"
+          - os: ubuntu-22.04
+            llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }}
+            ubuntu_version: "22.04"
 
         exclude:
           # uncompatiable modes and features
           - os: ubuntu-20.04
-            sanitizer: asan
+            sanitizer: tsan
           # asan works only for aot now
           - running_mode: "classic-interp"
             sanitizer: asan
@@ -546,6 +559,14 @@ jobs:
             sanitizer: asan
           - running_mode: "multi-tier-jit"
             sanitizer: asan
+          - running_mode: "classic-interp"
+            sanitizer: tsan
+          - running_mode: "jit"
+            sanitizer: tsan
+          - running_mode: "fast-jit"
+            sanitizer: tsan
+          - running_mode: "multi-tier-jit"
+            sanitizer: tsan
           # classic-interp and fast-interp don't support simd
           - running_mode: "classic-interp"
             test_option: $SIMD_TEST_OPTIONS
@@ -596,8 +617,9 @@ jobs:
           && matrix.running_mode != 'fast-jit' && matrix.running_mode != 'jit' && matrix.running_mode != 'multi-tier-jit'
         run: echo "TEST_ON_X86_32=true" >> $GITHUB_ENV
 
-      - name: set sanitizer
+      - name: set additional tsan options
-        run: echo "WAMR_BUILD_SANITIZER=${{ matrix.sanitizer }}" >> $GITHUB_ENV
+        run: echo "TSAN_OPTIONS=suppressions=$PWD/tsan_suppressions.txt" >> $GITHUB_ENV
+        working-directory: tests/wamr-test-suites
 
       #only download llvm libraries in jit and aot mode
       - name: Get LLVM libraries
@@ -638,7 +660,7 @@ jobs:
 
       - name: run tests
         timeout-minutes: 40
-        run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }}
+        run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} -T %{{matrix.sanitizer}}
         working-directory: ./tests/wamr-test-suites
 
       #only install x32 support libraries when to run x86_32 cases

core/iwasm/common/wasm_runtime_common.h

@@ -12,6 +12,7 @@
 #include "wasm_native.h"
 #include "../include/wasm_export.h"
 #include "../interpreter/wasm.h"
+
 #if WASM_ENABLE_LIBC_WASI != 0
 #if WASM_ENABLE_UVWASI == 0
 #include "wasmtime_ssp.h"
@@ -44,15 +45,16 @@ extern "C" {
 
 /* For STORE opcodes */
 #define STORE_I64 PUT_I64_TO_ADDR
-#define STORE_U32(addr, value)               \
+static inline void
-    do {                                     \
+STORE_U32(void *addr, uint32_t value)
-        *(uint32 *)(addr) = (uint32)(value); \
+{
-    } while (0)
+    *(uint32_t *)(addr) = (uint32_t)(value);
-#define STORE_U16(addr, value)               \
+}
-    do {                                     \
+static inline void
-        *(uint16 *)(addr) = (uint16)(value); \
+STORE_U16(void *addr, uint16_t value)
-    } while (0)
+{
-
+    *(uint16_t *)(addr) = (uint16_t)(value);
+}
 /* For LOAD opcodes */
 #define LOAD_I64(addr) (*(int64 *)(addr))
 #define LOAD_F64(addr) (*(float64 *)(addr))
@@ -147,42 +149,42 @@ GET_F64_FROM_ADDR(uint32 *addr)
         }                                           \
     } while (0)
 
-#define STORE_U32(addr, value)                    \
+static inline void
-    do {                                          \
+STORE_U32(void *addr, uint32_t value)
-        uintptr_t addr_ = (uintptr_t)(addr);      \
+{
-        union {                                   \
+    uintptr_t addr_ = (uintptr_t)(addr);
-            uint32 val;                           \
+    union {
-            uint16 u16[2];                        \
+        uint32_t val;
-            uint8 u8[4];                          \
+        uint16_t u16[2];
-        } u;                                      \
+        uint8_t u8[4];
-        if ((addr_ & (uintptr_t)3) == 0)          \
+    } u;
-            *(uint32 *)(addr) = (uint32)(value);  \
+    if ((addr_ & (uintptr_t)3) == 0)
-        else {                                    \
+        *(uint32_t *)(addr) = (uint32_t)(value);
-            u.val = (uint32)(value);              \
+    else {
-            if ((addr_ & (uintptr_t)1) == 0) {    \
+        u.val = (uint32_t)(value);
-                ((uint16 *)(addr))[0] = u.u16[0]; \
+        if ((addr_ & (uintptr_t)1) == 0) {
-                ((uint16 *)(addr))[1] = u.u16[1]; \
+            ((uint16_t *)(addr))[0] = u.u16[0];
-            }                                     \
+            ((uint16_t *)(addr))[1] = u.u16[1];
-            else {                                \
+        }
-                ((uint8 *)(addr))[0] = u.u8[0];   \
+        else {
-                ((uint8 *)(addr))[1] = u.u8[1];   \
+            ((uint8_t *)(addr))[0] = u.u8[0];
-                ((uint8 *)(addr))[2] = u.u8[2];   \
+            ((uint8_t *)(addr))[1] = u.u8[1];
-                ((uint8 *)(addr))[3] = u.u8[3];   \
+            ((uint8_t *)(addr))[2] = u.u8[2];
-            }                                     \
+            ((uint8_t *)(addr))[3] = u.u8[3];
-        }                                         \
+        }
-    } while (0)
+    }
-
+}
-#define STORE_U16(addr, value)          \
+static inline void
-    do {                                \
+STORE_U16(void *addr, uint16_t value)
-        union {                         \
+{
-            uint16 val;                 \
+    union {
-            uint8 u8[2];                \
+        uint16_t val;
-        } u;                            \
+        uint8_t u8[2];
-        u.val = (uint16)(value);        \
+    } u;
-        ((uint8 *)(addr))[0] = u.u8[0]; \
+    u.val = (uint16_t)(value);
-        ((uint8 *)(addr))[1] = u.u8[1]; \
+    ((uint8_t *)(addr))[0] = u.u8[0];
-    } while (0)
+    ((uint8_t *)(addr))[1] = u.u8[1];
-
+}
 /* For LOAD opcodes */
 static inline int64
 LOAD_I64(void *addr)

tests/wamr-test-suites/test_wamr.sh

@@ -32,6 +32,7 @@ function help()
     echo "-F set the firmware path used by qemu"
     echo "-C enable code coverage collect"
     echo "-j set the platform to test"
+    echo "-T set sanitizer to use in tests(ubsan|tsan|asan)"
 }
 
 OPT_PARSED=""
@@ -59,7 +60,7 @@ QEMU_FIRMWARE=""
 # prod/testsuite-all branch
 WASI_TESTSUITE_COMMIT="ee807fc551978490bf1c277059aabfa1e589a6c2"
 
-while getopts ":s:cabgvt:m:MCpSXxwPGQF:j:" opt
+while getopts ":s:cabgvt:m:MCpSXxwPGQF:j:T:" opt
 do
     OPT_PARSED="TRUE"
     case $opt in
@@ -165,9 +166,14 @@ do
         echo "test platform " ${OPTARG}
         PLATFORM=${OPTARG}
         ;;
+        T)
+        echo "sanitizer is " ${OPTARG}
+        WAMR_BUILD_SANITIZER=${OPTARG}
+        ;;
         ?)
         help
-        exit 1;;
+        exit 1
+        ;;
     esac
 done
 
@@ -553,7 +559,7 @@ function wasi_certification_test()
     cd wasi-testsuite
     git reset --hard ${WASI_TESTSUITE_COMMIT}
 
-    bash ../../wasi-test-script/run_wasi_tests.sh $1 $TARGET \
+    TSAN_OPTIONS=${TSAN_OPTIONS} bash ../../wasi-test-script/run_wasi_tests.sh $1 $TARGET \
         | tee -a ${REPORT_DIR}/wasi_test_report.txt
     ret=${PIPESTATUS[0]}
 

tests/wamr-test-suites/tsan_suppressions.txt

@@ -0,0 +1,11 @@
+# Proposing to accept this risk for now. It might be wasi-libc related.
+# https://github.com/bytecodealliance/wasm-micro-runtime/pull/1963#issuecomment-1455342931
+race:STORE_U32
+
+
+# https://github.com/bytecodealliance/wasm-micro-runtime/issues/2680
+race:execute_post_instantiate_functions
+
+# Suppressing signal-unsafe inside of a signal for AOT mode 
+# see https://github.com/bytecodealliance/wasm-micro-runtime/issues/2248#issuecomment-1630189656
+signal:*

tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh

@@ -67,7 +67,7 @@ if [[ $MODE != "aot" ]];then
     python3 -m pip install -r test-runner/requirements.txt
 
     export TEST_RUNTIME_EXE="${IWASM_CMD}"
-    python3 ${THIS_DIR}/pipe.py | python3 test-runner/wasi_test_runner.py \
+    python3 ${THIS_DIR}/pipe.py | TSAN_OPTIONS=${TSAN_OPTIONS} python3 test-runner/wasi_test_runner.py \
             -r adapters/wasm-micro-runtime.py \
             -t \
                 ${C_TESTS} \
@@ -79,7 +79,7 @@ if [[ $MODE != "aot" ]];then
 
     ret=${PIPESTATUS[1]}
 
-    TEST_RUNTIME_EXE="${IWASM_CMD_STRESS}" python3 test-runner/wasi_test_runner.py \
+    TEST_RUNTIME_EXE="${IWASM_CMD_STRESS}" TSAN_OPTIONS=${TSAN_OPTIONS}  python3 test-runner/wasi_test_runner.py \
             -r adapters/wasm-micro-runtime.py \
             -t \
                 ${THREAD_STRESS_TESTS}