Add mutex stress test (#2472)

As a part of stress-testing we want to ensure that mutex implementation is working
correctly and protecting shared resource to be allocated from other threads when
mutex is locked.

This test covers the most common situations that happen when some program uses
mutexes like locks from various threads, locks from the same thread etc.

.github/workflows/nightly_run.yml

@@ -599,6 +599,11 @@ jobs:
         run: bash build.sh --sysroot "$SYSROOT_PATH"
         working-directory: ./core/iwasm/libraries/lib-wasi-threads/test/
 
+      - name: Build WASI thread stress tests
+        if: matrix.test_option == '$WASI_TEST_OPTIONS'
+        run: bash build.sh --sysroot "$SYSROOT_PATH"
+        working-directory: ./core/iwasm/libraries/lib-wasi-threads/stress-test/
+
       - name: build socket api tests
         if: matrix.test_option == '$WASI_TEST_OPTIONS'
         run: bash build.sh

core/iwasm/libraries/lib-wasi-threads/stress-test/build.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+#
+# Copyright (C) 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+
+set -eo pipefail
+CC=${CC:=/opt/wasi-sdk/bin/clang}
+WAMR_DIR=../../../../..
+
+show_usage() {
+    echo "Usage: $0 [--sysroot PATH_TO_SYSROOT]"
+    echo "--sysroot PATH_TO_SYSROOT specify to build with custom sysroot for wasi-libc"
+}
+
+while [[ $# -gt 0 ]]; do
+    key="$1"
+    case $key in
+        --sysroot)
+            sysroot_path="$2"
+            shift
+            shift
+            ;;
+        --help)
+            show_usage
+            exit
+            ;;
+        *)
+            echo "Unknown option: $1"
+            exit 1
+            ;;
+    esac
+done
+
+rm -rf *.wasm
+rm -rf *.aot
+
+for test_c in *.c; do
+    test_wasm="$(basename $test_c .c).wasm"
+
+    if [[ -n "$sysroot_path" ]]; then 
+        if [ ! -d "$sysroot_path" ]; then 
+            echo "Directory $sysroot_path  doesn't exist. Aborting"
+            exit 1
+        fi
+        sysroot_command="--sysroot $sysroot_path"
+    fi
+    
+    echo "Compiling $test_c to $test_wasm"
+    $CC \
+        -target wasm32-wasi-threads \
+        -O2 \
+        -Wall \
+        -pthread \
+        -z stack-size=32768 \
+        -Wl,--export=__heap_base \
+        -Wl,--export=__data_end \
+        -Wl,--shared-memory,--max-memory=1966080 \
+        -Wl,--export=wasi_thread_start \
+        -Wl,--export=malloc \
+        -Wl,--export=free \
+        $sysroot_command \
+        $test_c -o $test_wasm
+done

core/iwasm/libraries/lib-wasi-threads/stress-test/errorcheck_mutex_stress_test.c

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <pthread.h>
+#include <errno.h>
+#include "mutex_common.h"
+
+int
+main()
+{
+    pthread_mutex_t mutex;
+
+    // Set mutex type to errorcheck. This type provides some additional checks
+    // (for example returns EDEADLK instead of deadlocking in some cases)
+    pthread_mutexattr_t mutex_attr;
+    pthread_mutexattr_init(&mutex_attr);
+    pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_ERRORCHECK);
+
+    pthread_mutex_init(&mutex, &mutex_attr);
+    pthread_mutexattr_destroy(&mutex_attr);
+
+    run_common_tests(&mutex);
+    fprintf(stderr, "Errorcheck mutex test is completed\n");
+    pthread_mutex_destroy(&mutex);
+}

core/iwasm/libraries/lib-wasi-threads/stress-test/manifest.json

@@ -0,0 +1,3 @@
+{
+    "name": "lib-wasi-threads stress tests"
+}

core/iwasm/libraries/lib-wasi-threads/stress-test/mutex_common.h

@@ -0,0 +1,229 @@
+/*
+ * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#ifndef MUTEX_COMMON_H
+#define MUTEX_COMMON_H
+
+#include <pthread.h>
+#include <stdio.h>
+#include <assert.h>
+#include <errno.h>
+#include <unistd.h>
+#include <stdbool.h>
+#include <time.h>
+#include <stdlib.h>
+#include <string.h>
+
+enum Constants {
+    NUM_ITER = 250000,
+    NUM_THREADS = 12,
+    NUM_RETRY = 8,
+    RETRY_SLEEP_TIME_US = 1000,
+};
+
+// We're counting how many times each thread was called using this array
+// Main thread is also counted here so we need to make arrays bigger
+typedef struct {
+    int tids[NUM_THREADS + 1];
+    int calls[NUM_THREADS + 1];
+} StatCollector;
+
+typedef struct {
+    pthread_mutex_t *mutex;
+    StatCollector stat;
+    int counter;
+    bool is_sleeping;
+} MutexCounter;
+
+// This enum defines whether thread should sleep to increase contention
+enum SleepState {
+    NON_SLEEP = 0,
+    SLEEP = 1,
+};
+
+void
+mutex_counter_init(MutexCounter *mutex_counter, pthread_mutex_t *mutex,
+                   enum SleepState is_sleeping)
+{
+    memset(mutex_counter, 0, sizeof(*mutex_counter));
+    mutex_counter->mutex = mutex;
+    mutex_counter->is_sleeping = is_sleeping;
+}
+
+// This function spawns the thread using exponential retries if it receives
+// EAGAIN
+static inline void
+spawn_thread(pthread_t *tid, void *func, void *arg)
+{
+    int status_code = -1;
+    int timeout_us = RETRY_SLEEP_TIME_US;
+    for (int tries = 0; status_code != 0 && tries < NUM_RETRY; ++tries) {
+        status_code = pthread_create(tid, NULL, (void *(*)(void *))func, arg);
+        assert(status_code == 0 || status_code == EAGAIN);
+        if (status_code == EAGAIN) {
+            usleep(timeout_us);
+            timeout_us *= 2;
+        }
+    }
+
+    assert(status_code == 0 && "Thread creation should succeed");
+}
+
+// This function adds tid to our stat
+static inline void
+add_to_stat(StatCollector *stat, int tid)
+{
+    int tid_num = 0;
+    for (; tid_num < NUM_THREADS + 1 && stat->tids[tid_num] != 0; ++tid_num) {
+        if (stat->tids[tid_num] == tid) {
+            stat->calls[tid_num]++;
+            return;
+        }
+    }
+
+    assert(tid_num < NUM_THREADS + 1);
+    stat->tids[tid_num] = tid;
+    stat->calls[tid_num] = 1;
+}
+
+// This function prints number of calls by TID
+static inline void
+print_stat(StatCollector *stat)
+{
+    fprintf(stderr, "Thread calls count by TID\n");
+    for (int i = 0; i < NUM_THREADS + 1; ++i) {
+        if (stat->tids[i] != 0) {
+            fprintf(stderr, "TID: %d; Calls: %d\n", stat->tids[i],
+                    stat->calls[i]);
+        }
+    }
+}
+
+// This function is run by the threads, it increases counter in a loop and then
+// sleeps after unlocking the mutex to provide better contention
+static inline void *
+inc_shared_variable(void *arg)
+{
+    MutexCounter *mutex_counter = (MutexCounter *)(arg);
+    int sleep_us = 0;
+    while (!pthread_mutex_lock(mutex_counter->mutex)
+           && mutex_counter->counter < NUM_ITER) {
+        mutex_counter->counter++;
+        add_to_stat(&mutex_counter->stat, (int)(pthread_self()));
+        if (mutex_counter->is_sleeping) {
+            sleep_us = rand() % 1000;
+        }
+
+        assert(pthread_mutex_unlock(mutex_counter->mutex) == 0
+               && "Should be able to unlock a mutex");
+        if (mutex_counter->is_sleeping) {
+            usleep(sleep_us);
+        }
+    }
+
+    assert(mutex_counter->counter == NUM_ITER);
+    assert(pthread_mutex_unlock(mutex_counter->mutex) == 0
+           && "Should be able to unlock the mutex after test execution");
+
+    return NULL;
+}
+
+// Locking and unlocking a mutex in a single thread.
+static inline void *
+same_thread_lock_unlock_test(void *mutex)
+{
+    for (int i = 0; i < NUM_ITER; ++i) {
+        assert(pthread_mutex_lock(mutex) == 0
+               && "Main thread should be able to lock a mutex");
+        assert(pthread_mutex_unlock(mutex) == 0
+               && "Main thread should be able to unlock a mutex");
+    }
+
+    return NULL;
+}
+
+// This function spawns a thread that locks and unlocks a mutex `NUM_ITER` times
+// in a row
+static inline void
+same_non_main_thread_lock_unlock_test(pthread_mutex_t *mutex)
+{
+    pthread_t tid = 0;
+    spawn_thread(&tid, same_thread_lock_unlock_test, mutex);
+
+    assert(tid != 0 && "TID can't be 0 after successful thread creation");
+    assert(pthread_join(tid, NULL) == 0
+           && "Thread should be joined successfully");
+}
+
+// This function checks basic contention between main and non-main thread
+// increasing the shared variable
+static inline void
+two_threads_inc_test(pthread_mutex_t *mutex)
+{
+    MutexCounter mutex_counter;
+    mutex_counter_init(&mutex_counter, mutex, false);
+
+    pthread_t tid = 0;
+    spawn_thread(&tid, inc_shared_variable, &mutex_counter);
+
+    assert(tid != 0 && "TID can't be 0 after successful thread creation");
+    inc_shared_variable(&mutex_counter);
+    assert(pthread_join(tid, NULL) == 0
+           && "Thread should be joined without errors");
+    assert(mutex_counter.counter == NUM_ITER);
+}
+
+// This function creates number of threads specified by NUM_THREADS and run
+// concurrent increasing of shared variable
+static inline void
+max_threads_inc_test(pthread_mutex_t *mutex, int threads_num,
+                     enum SleepState is_sleeping)
+{
+    MutexCounter mutex_counter;
+    mutex_counter_init(&mutex_counter, mutex, is_sleeping);
+
+    pthread_t tids[threads_num];
+    for (int i = 0; i < threads_num; ++i) {
+        spawn_thread(&tids[i], inc_shared_variable, &mutex_counter);
+    }
+
+    inc_shared_variable(&mutex_counter);
+
+    for (int i = 0; i < threads_num; ++i) {
+        assert(pthread_join(tids[i], NULL) == 0
+               && "Thread should be joined without errors");
+    }
+
+    print_stat(&mutex_counter.stat);
+}
+
+// This function just runs all the tests described above
+static inline void
+run_common_tests(pthread_mutex_t *mutex)
+{
+    srand(time(NULL));
+
+    fprintf(stderr, "Starting same_thread_lock_unlock_test test\n");
+    same_thread_lock_unlock_test(mutex);
+    fprintf(stderr, "Finished same_thread_lock_unlock_test test\n");
+
+    fprintf(stderr, "Starting same_non_main_thread_lock_unlock_test test\n");
+    same_non_main_thread_lock_unlock_test(mutex);
+    fprintf(stderr, "Finished same_non_main_thread_lock_unlock_test test\n");
+
+    fprintf(stderr, "Starting two_threads_inc_test test\n");
+    two_threads_inc_test(mutex);
+    fprintf(stderr, "Finished two_threads_inc_test test\n");
+
+    fprintf(stderr, "Starting max_threads_inc_test_sleep test\n");
+    max_threads_inc_test(mutex, NUM_THREADS, SLEEP);
+    fprintf(stderr, "Finished concurrent_inc sleep test\n");
+
+    fprintf(stderr, "Starting max_threads_inc_test_non_sleep test\n");
+    max_threads_inc_test(mutex, NUM_THREADS, NON_SLEEP);
+    fprintf(stderr, "Finished max_threads_inc_test test\n");
+}
+
+#endif // MUTEX_COMMON_H

core/iwasm/libraries/lib-wasi-threads/stress-test/normal_mutex_stress_test.c

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <pthread.h>
+#include <errno.h>
+#include "mutex_common.h"
+
+int
+main()
+{
+    pthread_mutex_t mutex;
+    pthread_mutex_init(&mutex, NULL);
+
+    run_common_tests(&mutex);
+
+    fprintf(stderr, "Normal mutex test is completed\n");
+    pthread_mutex_destroy(&mutex);
+}

core/iwasm/libraries/lib-wasi-threads/stress-test/recursive_mutex_stress_test.c

@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <pthread.h>
+#include <errno.h>
+#include "mutex_common.h"
+
+void
+multiple_same_thread_lock(void *mutex)
+{
+    for (int i = 0; i < 100; ++i) {
+        assert(pthread_mutex_lock(mutex) == 0
+               && "Recursive mutex should allow multiple locking");
+    }
+
+    for (int i = 0; i < 100; ++i) {
+        assert(pthread_mutex_unlock(mutex) == 0
+               && "Recursive mutex should allow multiple unlocking");
+    }
+}
+
+void *
+same_thread_multiple_rec_mutex_lock(void *mutex)
+{
+    for (int i = 0; i < NUM_ITER; ++i) {
+        multiple_same_thread_lock(mutex);
+    }
+
+    return NULL;
+}
+
+int
+main()
+{
+    pthread_mutex_t mutex;
+
+    // Set mutex type to recursive. This type allows multiple locking and
+    // unlocking within the same thread
+    pthread_mutexattr_t mutex_attr;
+    pthread_mutexattr_init(&mutex_attr);
+    pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_RECURSIVE);
+
+    pthread_mutex_init(&mutex, &mutex_attr);
+    pthread_mutexattr_destroy(&mutex_attr);
+
+    run_common_tests(&mutex);
+
+    fprintf(stderr, "Starting same_thread_multiple_rec_mutex_lock test\n");
+    same_thread_multiple_rec_mutex_lock(&mutex);
+    fprintf(stderr, "Finished same_thread_multiple_rec_mutex_lock test\n");
+
+    fprintf(stderr, "Starting same_thread_multiple_rec_mutex_lock test in "
+                    "non-main thread\n");
+    pthread_t tid;
+    spawn_thread(&tid, same_thread_multiple_rec_mutex_lock, &mutex);
+    assert(pthread_join(tid, NULL) == 0
+           && "Non-main thread should be joined successfully");
+    fprintf(stderr, "Finished same_thread_multiple_rec_mutex_lock test in "
+                    "non-main thread\n");
+
+    fprintf(stderr, "Recursive mutex test is completed\n");
+    pthread_mutex_destroy(&mutex);
+}

core/iwasm/libraries/lib-wasi-threads/stress-test/spawn_stress_test.c

@@ -19,7 +19,7 @@
 enum CONSTANTS {
     NUM_ITER = 100000,
     NUM_RETRY = 8,
-    MAX_NUM_THREADS = 8,
+    MAX_NUM_THREADS = 12,
     RETRY_SLEEP_TIME_US = 2000,
 };
 

core/iwasm/libraries/lib-wasi-threads/stress-test/stress_test_threads_creation.c

@@ -12,7 +12,7 @@
 enum CONSTANTS {
     NUM_ITER = 200000,
     NUM_RETRY = 8,
-    MAX_NUM_THREADS = 8,
+    MAX_NUM_THREADS = 12,
     RETRY_SLEEP_TIME_US = 4000,
     SECOND = 1000 * 1000 * 1000
 };
@@ -72,7 +72,9 @@ main(int argc, char **argv)
     }
 
     while ((__atomic_load_n(&threads_in_use, __ATOMIC_SEQ_CST) != 0)) {
-        __builtin_wasm_memory_atomic_wait32(&threads_in_use, 0, SECOND);
+        // Casting to int* to supress compiler warning
+        __builtin_wasm_memory_atomic_wait32((int *)(&threads_in_use), 0,
+                                            SECOND);
     }
 
     assert(__atomic_load_n(&threads_in_use, __ATOMIC_SEQ_CST) == 0);

core/iwasm/libraries/lib-wasi-threads/test/build.sh

@@ -34,7 +34,10 @@ while [[ $# -gt 0 ]]; do
 done
 
 # Stress tests names
-thread_start_file_exclusions=("spawn_stress_test.wasm" "linear_memory_size_update.wasm" "stress_test_threads_creation.wasm")
+thread_start_file_exclusions=("linear_memory_size_update.wasm")
+
+rm -rf *.wasm
+rm -rf *.aot
 
 for test_c in *.c; do
     test_wasm="$(basename $test_c .c).wasm"

core/iwasm/libraries/lib-wasi-threads/test/skip.json

@@ -1,6 +0,0 @@
-{
-    "lib-wasi-threads tests": {
-        "spawn_stress_test": "Stress tests are incompatible with the other part and executed differently",
-        "stress_test_threads_creation": "Stress tests are incompatible with the other part and executed differently"
-    }
-}

tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh

@@ -15,25 +15,22 @@ readonly IWASM_CMD="${WORK_DIR}/../../../../product-mini/platforms/${PLATFORM}/b
     --allow-resolve=google-public-dns-a.google.com \
     --addr-pool=::1/128,127.0.0.1/32"
 
-readonly IWASM_CMD_STRESS="${IWASM_CMD} --max-threads=8"
+readonly IWASM_CMD_STRESS="${IWASM_CMD} --max-threads=12"
 readonly WAMRC_CMD="${WORK_DIR}/../../../../wamr-compiler/build/wamrc"
 readonly C_TESTS="tests/c/testsuite/"
 readonly ASSEMBLYSCRIPT_TESTS="tests/assemblyscript/testsuite/"
 readonly THREAD_PROPOSAL_TESTS="tests/proposals/wasi-threads/"
 readonly THREAD_INTERNAL_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-wasi-threads/test/"
+readonly THREAD_STRESS_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-wasi-threads/stress-test/"
 readonly LIB_SOCKET_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-socket/test/"
-readonly STRESS_TESTS=("spawn_stress_test.wasm" "stress_test_threads_creation.wasm")
 
 run_aot_tests () {
     local tests=("$@")
-    local iwasm="${IWASM_CMD}"
     for test_wasm in ${tests[@]}; do
-        local extra_stress_flags=""
-        for stress_test in "${STRESS_TESTS[@]}"; do
-            if [ "$test_wasm" == "$stress_test" ]; then
-                iwasm="${IWASM_CMD_STRESS}"
-            fi  
-        done
+        local iwasm="${IWASM_CMD}"
+        if [[ $test_wasm =~ "stress" ]]; then
+            iwasm="${IWASM_CMD_STRESS}"
+        fi
 
         test_aot="${test_wasm%.wasm}.aot"
         test_json="${test_wasm%.wasm}.json"
@@ -52,7 +49,7 @@ run_aot_tests () {
             expected=$(jq .exit_code ${test_json})
         fi
 
-        ${IWASM_CMD} $extra_stress_flags $test_aot
+        ${iwasm} $test_aot
         ret=${PIPESTATUS[0]}
 
         echo "expected=$expected, actual=$ret"
@@ -66,19 +63,6 @@ if [[ $MODE != "aot" ]];then
     python3 -m venv wasi-env && source wasi-env/bin/activate
     python3 -m pip install -r test-runner/requirements.txt
 
-    # Stress tests require max-threads=8 so they're executed separately
-    for stress_test in "${STRESS_TESTS[@]}"; do
-        if [[ -e "${THREAD_INTERNAL_TESTS}${stress_test}" ]]; then
-            echo "${stress_test}" is a stress test
-            ${IWASM_CMD_STRESS} ${THREAD_INTERNAL_TESTS}${stress_test}
-            ret=${PIPESTATUS[0]}
-            if [ "${ret}" -ne 0 ]; then
-                echo "Stress test ${stress_test} FAILED with code " ${ret}
-                exit_code=${ret}
-            fi
-        fi
-    done
-
     TEST_RUNTIME_EXE="${IWASM_CMD}" python3 test-runner/wasi_test_runner.py \
             -r adapters/wasm-micro-runtime.py \
             -t \
@@ -87,12 +71,20 @@ if [[ $MODE != "aot" ]];then
                 ${THREAD_PROPOSAL_TESTS} \
                 ${THREAD_INTERNAL_TESTS} \
                 ${LIB_SOCKET_TESTS} \
-            --exclude-filter "${THREAD_INTERNAL_TESTS}skip.json"
 
     ret=${PIPESTATUS[0]}
-    if [ "${ret}" -ne 0 ]; then
-        exit_code=${ret}
+
+    TEST_RUNTIME_EXE="${IWASM_CMD_STRESS}" python3 test-runner/wasi_test_runner.py \
+            -r adapters/wasm-micro-runtime.py \
+            -t \
+                ${THREAD_STRESS_TESTS}
+
+    if [ "${ret}" -eq 0 ]; then
+        ret=${PIPESTATUS[0]}
     fi
+    
+    exit_code=${ret}
+    
     deactivate
 else
     target_option=""
@@ -101,7 +93,7 @@ else
     fi
 
     exit_code=0
-    for testsuite in ${THREAD_PROPOSAL_TESTS} ${THREAD_INTERNAL_TESTS}; do
+    for testsuite in ${THREAD_STRESS_TESTS} ${THREAD_PROPOSAL_TESTS} ${THREAD_INTERNAL_TESTS}; do
         tests=$(ls ${testsuite}*.wasm)
         tests_array=($tests)
         run_aot_tests "${tests_array[@]}"