update following the reivew comments

1, WAMR_BUILD_MEM_ALLOC_WITH_USAGE -> WAMR_BUILD_ALLOC_WITH_USAGE
2, redefine the malloc, free, realloc signature.
3, let customer handle full_size_mmaped also in its realloc.
4, update the doc

build-scripts/config_common.cmake

@@ -554,6 +554,6 @@ else ()
   # Disable aot intrinsics for interp, fast-jit and llvm-jit
   add_definitions (-DWASM_ENABLE_AOT_INTRINSICS=0)
 endif ()
-if (WAMR_BUILD_MEM_ALLOC_WITH_USAGE EQUAL 1)
+if (WAMR_BUILD_ALLOC_WITH_USAGE EQUAL 1)
   add_definitions(-DWASM_MEM_ALLOC_WITH_USAGE=1)
 endif()

core/iwasm/common/wasm_memory.c

@@ -27,23 +27,37 @@ static mem_allocator_t pool_allocator = NULL;
 static enlarge_memory_error_callback_t enlarge_memory_error_cb;
 static void *enlarge_memory_error_user_data;
 
-#if WASM_MEM_ALLOC_WITH_USAGE != 0
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
-static void *(*malloc_func)(mem_alloc_usage_t usage, unsigned int size) = NULL;
-static void *(*realloc_func)(mem_alloc_usage_t usage, void *ptr,
-                             unsigned int size) = NULL;
-static void (*free_func)(mem_alloc_usage_t usage, void *ptr) = NULL;
-#elif WASM_MEM_ALLOC_WITH_USER_DATA != 0
 static void *allocator_user_data = NULL;
-static void *(*malloc_func)(void *user_data, unsigned int size) = NULL;
-static void *(*realloc_func)(void *user_data, void *ptr,
-                             unsigned int size) = NULL;
-static void (*free_func)(void *user_data, void *ptr) = NULL;
-#else
-static void *(*malloc_func)(unsigned int size) = NULL;
-static void *(*realloc_func)(void *ptr, unsigned int size) = NULL;
-static void (*free_func)(void *ptr) = NULL;
 #endif
 
+static void *(*malloc_func)(
+#if WASM_MEM_ALLOC_WITH_USAGE != 0
+    mem_alloc_usage_t usage,
+#endif
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
+    void *user_data,
+#endif
+    unsigned int size) = NULL;
+
+static void *(*realloc_func)(
+#if WASM_MEM_ALLOC_WITH_USAGE != 0
+    mem_alloc_usage_t usage, bool full_size_mmaped,
+#endif
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
+    void *user_data,
+#endif
+    void *ptr, unsigned int size) = NULL;
+
+static void (*free_func)(
+#if WASM_MEM_ALLOC_WITH_USAGE != 0
+    mem_alloc_usage_t usage,
+#endif
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
+    void *user_data,
+#endif
+    void *ptr) = NULL;
+
 static unsigned int global_pool_size;
 
 static uint64
@@ -182,13 +196,14 @@ wasm_runtime_malloc_internal(unsigned int size)
         return mem_allocator_malloc(pool_allocator, size);
     }
     else if (memory_mode == MEMORY_MODE_ALLOCATOR) {
+        return malloc_func(
 #if WASM_MEM_ALLOC_WITH_USAGE != 0
-        return malloc_func(Alloc_For_Runtime, size);
+            Alloc_For_Runtime,
-#elif WASM_MEM_ALLOC_WITH_USER_DATA != 0
-        return malloc_func(allocator_user_data, size);
-#else
-        return malloc_func(size);
 #endif
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
+            allocator_user_data,
+#endif
+            size);
     }
     else {
         return os_malloc(size);
@@ -208,13 +223,14 @@ wasm_runtime_realloc_internal(void *ptr, unsigned int size)
     }
     else if (memory_mode == MEMORY_MODE_ALLOCATOR) {
         if (realloc_func)
+            return realloc_func(
 #if WASM_MEM_ALLOC_WITH_USAGE != 0
-            return realloc_func(Alloc_For_Runtime, ptr, size);
+                Alloc_For_Runtime, false,
-#elif WASM_MEM_ALLOC_WITH_USER_DATA != 0
-            return realloc_func(allocator_user_data, ptr, size);
-#else
-            return realloc_func(ptr, size);
 #endif
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
+                allocator_user_data,
+#endif
+                ptr, size);
         else
             return NULL;
     }
@@ -242,13 +258,14 @@ wasm_runtime_free_internal(void *ptr)
         mem_allocator_free(pool_allocator, ptr);
     }
     else if (memory_mode == MEMORY_MODE_ALLOCATOR) {
+        free_func(
 #if WASM_MEM_ALLOC_WITH_USAGE != 0
-        free_func(Alloc_For_Runtime, ptr);
+            Alloc_For_Runtime,
-#elif WASM_MEM_ALLOC_WITH_USER_DATA != 0
-        free_func(allocator_user_data, ptr);
-#else
-        free_func(ptr);
 #endif
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
+            allocator_user_data,
+#endif
+            ptr);
     }
     else {
         os_free(ptr);
@@ -777,12 +794,24 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count)
               <= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64));
 
 #if WASM_MEM_ALLOC_WITH_USAGE != 0
-    (void)full_size_mmaped;
+    if (!(memory_data_new =
-    if (!(memory_data_new = realloc_func(Alloc_For_LinearMemory,
+              realloc_func(Alloc_For_LinearMemory, full_size_mmaped,
-                                         memory_data_old, total_size_new))) {
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
+                           NULL,
+#endif
+                           memory_data_old, total_size_new))) {
         ret = false;
         goto return_func;
     }
+    if (heap_size > 0) {
+        if (mem_allocator_migrate(memory->heap_handle,
+                                  (char *)heap_data_old
+                                      + (memory_data_new - memory_data_old),
+                                  heap_size)
+            != 0) {
+            ret = false;
+        }
+    }
     memory->heap_data = memory_data_new + (heap_data_old - memory_data_old);
     memory->heap_data_end = memory->heap_data + heap_size;
     memory->memory_data = memory_data_new;
@@ -929,7 +958,11 @@ wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst)
 
 #if WASM_MEM_ALLOC_WITH_USAGE != 0
     (void)map_size;
-    free_func(Alloc_For_LinearMemory, memory_inst->memory_data);
+    free_func(Alloc_For_LinearMemory,
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
+              NULL,
+#endif
+              memory_inst->memory_data);
 #else
     wasm_munmap_linear_memory(memory_inst->memory_data,
                               memory_inst->memory_data_size, map_size);
@@ -986,7 +1019,11 @@ wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory,
     if (map_size > 0) {
 #if WASM_MEM_ALLOC_WITH_USAGE != 0
         (void)wasm_mmap_linear_memory;
-        if (!(*data = malloc_func(Alloc_For_LinearMemory, *memory_data_size))) {
+        if (!(*data = malloc_func(Alloc_For_LinearMemory,
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
+                                  NULL,
+#endif
+                                  *memory_data_size))) {
             return BHT_ERROR;
         }
 #else

core/iwasm/include/wasm_export.h

@@ -119,6 +119,9 @@ typedef union MemAllocOption {
         uint32_t heap_size;
     } pool;
     struct {
+        /* the function signature is varied when
+        WASM_MEM_ALLOC_WITH_USER_DATA and
+        WASM_MEM_ALLOC_WITH_USAGE are defined */
         void *malloc_func;
         void *realloc_func;
         void *free_func;

doc/build_wamr.md

@@ -254,6 +254,10 @@ Currently we only profile the memory consumption of module, module_instance and
 
 > See [Enable segue optimization for wamrc when generating the aot file](./perf_tune.md#3-enable-segue-optimization-for-wamrc-when-generating-the-aot-file) for more details.
 
+#### **User defined linear memory allocator**
+- **WAMR_BUILD_ALLOC_WITH_USAGE**=1/0, default to disable if not set
+> Notes: by default, the linear memory is allocated by system. when it's set to 1 and Alloc_With_Allocator is selected, it will be allocated by customer.
+
 #### **Enable running PGO(Profile-Guided Optimization) instrumented AOT file**
 - **WAMR_BUILD_STATIC_PGO**=1/0, default to disable if not set
 > Note: See [Use the AOT static PGO method](./perf_tune.md#5-use-the-aot-static-pgo-method) for more details.

product-mini/platforms/posix/main.c

@@ -447,7 +447,8 @@ static void *
 malloc_func(
 #if WASM_MEM_ALLOC_WITH_USAGE != 0
     mem_alloc_usage_t usage,
-#elif WASM_MEM_ALLOC_WITH_USER_DATA != 0
+#endif
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
     void *user_data,
 #endif
     unsigned int size)
@@ -458,8 +459,9 @@ malloc_func(
 static void *
 realloc_func(
 #if WASM_MEM_ALLOC_WITH_USAGE != 0
-    mem_alloc_usage_t usage,
+    mem_alloc_usage_t usage, bool full_size_mmaped,
-#elif WASM_MEM_ALLOC_WITH_USER_DATA != 0
+#endif
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
     void *user_data,
 #endif
     void *ptr, unsigned int size)
@@ -471,7 +473,8 @@ static void
 free_func(
 #if WASM_MEM_ALLOC_WITH_USAGE != 0
     mem_alloc_usage_t usage,
-#elif WASM_MEM_ALLOC_WITH_USER_DATA != 0
+#endif
+#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
     void *user_data,
 #endif
     void *ptr)