gnh1201/wasm-micro-runtime
1
0
Fork 0
mirror of https://github.com/bytecodealliance/wasm-micro-runtime.git synced 2025-02-06 06:55:07 +00:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Always allocate linear memory using mmap (#3052)

Browse Source 
With this approach we can omit using memset() for the newly allocated memory
therefore the physical pages are not being used unless touched by the program.

This also simplifies the implementation.
This commit is contained in:
Marcin Kolny 2024-02-02 14:17:44 +00:00 committed by GitHub
parent 2eb60060d8
commit a27ddece7f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 384 additions and 425 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/compilation_on_android_ubuntu.yml vendored
View File

@ -145,7 +145,6 @@ jobs:
"-DWAMR_BUILD_SIMD=1",
"-DWAMR_BUILD_TAIL_CALL=1",
"-DWAMR_DISABLE_HW_BOUND_CHECK=1",
"-DWAMR_ENABLE_SHARED_MEMORY_MMAP=1",
]
os: [ubuntu-22.04]
platform: [android, linux]

6
build-scripts/config_common.cmake
View File

@ -248,12 +248,6 @@ if (WAMR_BUILD_SHARED_MEMORY EQUAL 1)
else ()
add_definitions (-DWASM_ENABLE_SHARED_MEMORY=0)
endif ()
if (WAMR_ENABLE_SHARED_MEMORY_MMAP EQUAL 1)
add_definitions (-DWASM_ENABLE_SHARED_MEMORY_MMAP=1)
message (" Shared memory allocated using mmap enabled")
else ()
add_definitions (-DWASM_ENABLE_SHARED_MEMORY_MMAP=0)
endif ()
if (WAMR_BUILD_THREAD_MGR EQUAL 1)
message (" Thread manager enabled")
endif ()

9
core/config.h
View File

@ -188,6 +188,10 @@
#define WASM_ENABLE_APP_FRAMEWORK 0
#endif
#ifndef WASM_HAVE_MREMAP
#define WASM_HAVE_MREMAP 0
#endif
/* Bulk memory operation */
#ifndef WASM_ENABLE_BULK_MEMORY
#define WASM_ENABLE_BULK_MEMORY 0
@ -521,9 +525,4 @@
#define WASM_ENABLE_QUICK_AOT_ENTRY 1
#endif
/* Disable mmap based shared memory by default */
#ifndef WASM_ENABLE_SHARED_MEMORY_MMAP
#define WASM_ENABLE_SHARED_MEMORY_MMAP 0
#endif
#endif /* end of _CONFIG_H_ */

90
core/iwasm/aot/aot_runtime.c
View File

@ -340,9 +340,6 @@ tables_instantiate(AOTModuleInstance *module_inst, AOTModule *module,
static void
memories_deinstantiate(AOTModuleInstance *module_inst)
{
#ifdef WASM_LINEAR_MEMORY_MMAP
uint64 map_size;
#endif
uint32 i;
AOTMemoryInstance *memory_inst;
@ -364,23 +361,7 @@ memories_deinstantiate(AOTModuleInstance *module_inst)
}
if (memory_inst->memory_data) {
#ifndef OS_ENABLE_HW_BOUND_CHECK
#ifdef WASM_LINEAR_MEMORY_MMAP
if (shared_memory_is_shared(memory_inst)) {
map_size = (uint64)memory_inst->num_bytes_per_page
* memory_inst->max_page_count;
wasm_munmap_linear_memory(memory_inst->memory_data,
map_size, map_size);
}
else
#endif
wasm_runtime_free(memory_inst->memory_data);
#else
map_size = (uint64)memory_inst->num_bytes_per_page
* memory_inst->cur_page_count;
wasm_munmap_linear_memory(memory_inst->memory_data, map_size,
8 * (uint64)BH_GB);
#endif
wasm_deallocate_linear_memory(memory_inst);
}
}
}
@ -402,14 +383,10 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent,
uint32 heap_offset = num_bytes_per_page * init_page_count;
uint64 memory_data_size, max_memory_data_size;
uint8 *p = NULL, *global_addr;
#ifdef WASM_LINEAR_MEMORY_MMAP
uint8 *mapped_mem = NULL;
uint64 map_size;
#endif
bool is_shared_memory = false;
#if WASM_ENABLE_SHARED_MEMORY != 0
bool is_shared_memory = memory->memory_flags & 0x02 ? true : false;
is_shared_memory = memory->memory_flags & 0x02 ? true : false;
/* Shared memory */
if (is_shared_memory && parent != NULL) {
AOTMemoryInstance *shared_memory_instance;
@ -519,55 +496,18 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent,
module->aux_stack_size);
LOG_VERBOSE(" heap offset: %u, heap size: %d\n", heap_offset, heap_size);
memory_data_size = (uint64)num_bytes_per_page * init_page_count;
max_memory_data_size = (uint64)num_bytes_per_page * max_page_count;
bh_assert(memory_data_size <= UINT32_MAX);
bh_assert(max_memory_data_size <= 4 * (uint64)BH_GB);
(void)max_memory_data_size;
#ifndef OS_ENABLE_HW_BOUND_CHECK
#if WASM_ENABLE_SHARED_MEMORY != 0
if (is_shared_memory) {
#if WASM_ENABLE_SHARED_MEMORY_MMAP != 0
map_size = max_memory_data_size;
if (max_memory_data_size > 0
&& !(p = mapped_mem =
wasm_mmap_linear_memory(map_size, &max_memory_data_size,
error_buf, error_buf_size))) {
return NULL;
}
#else
/* Allocate maximum memory size when memory is shared */
if (max_memory_data_size > 0
&& !(p = runtime_malloc(max_memory_data_size, error_buf,
error_buf_size))) {
return NULL;
}
#endif
}
else
#endif /* end of WASM_ENABLE_SHARED_MEMORY != 0 */
{
/* Allocate initial memory size when memory is not shared */
if (memory_data_size > 0
&& !(p = runtime_malloc(memory_data_size, error_buf,
error_buf_size))) {
return NULL;
}
}
#else /* else of OS_ENABLE_HW_BOUND_CHECK */
/* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
* ea = i + memarg.offset
* both i and memarg.offset are u32 in range 0 to 4G
* so the range of ea is 0 to 8G
*/
map_size = 8 * (uint64)BH_GB;
if (!(p = mapped_mem = wasm_mmap_linear_memory(
map_size, &memory_data_size, error_buf, error_buf_size))) {
set_error_buf(error_buf, error_buf_size, "mmap memory failed");
if (wasm_allocate_linear_memory(&p, is_shared_memory, num_bytes_per_page,
init_page_count, max_page_count,
&memory_data_size)
!= BHT_OK) {
set_error_buf(error_buf, error_buf_size,
"allocate linear memory failed");
return NULL;
}
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
memory_inst->module_type = Wasm_Module_AoT;
memory_inst->num_bytes_per_page = num_bytes_per_page;
@ -617,16 +557,8 @@ fail2:
if (heap_size > 0)
wasm_runtime_free(memory_inst->heap_handle);
fail1:
#ifdef WASM_LINEAR_MEMORY_MMAP
if (mapped_mem)
wasm_munmap_linear_memory(mapped_mem, memory_data_size, map_size);
else
#endif
{
if (memory_inst->memory_data)
wasm_runtime_free(memory_inst->memory_data);
}
memory_inst->memory_data = NULL;
wasm_deallocate_linear_memory(memory_inst);
return NULL;
}

344
core/iwasm/common/wasm_memory.c
View File

@ -42,6 +42,14 @@ static void (*free_func)(void *ptr) = NULL;
static unsigned int global_pool_size;
static uint32
align_as_and_cast(uint64 size, uint64 alignment)
{
uint64 aligned_size = (size + alignment - 1) & ~(alignment - 1);
return aligned_size > UINT32_MAX ? UINT32_MAX : (uint32)aligned_size;
}
static bool
wasm_memory_init_with_pool(void *mem, unsigned int bytes)
{
@ -629,7 +637,61 @@ wasm_runtime_set_mem_bound_check_bytes(WASMMemoryInstance *memory,
#endif
}
#ifndef OS_ENABLE_HW_BOUND_CHECK
static void
wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size)
{
#ifdef BH_PLATFORM_WINDOWS
os_mem_decommit(mapped_mem, commit_size);
#else
(void)commit_size;
#endif
os_munmap(mapped_mem, map_size);
}
static void *
wasm_mremap_linear_memory(void *mapped_mem, uint64 old_size, uint64 new_size,
uint64 commit_size)
{
void *new_mem;
bh_assert(new_size > 0);
bh_assert(new_size > old_size);
if (mapped_mem) {
new_mem = os_mremap(mapped_mem, old_size, new_size);
}
else {
new_mem = os_mmap(NULL, new_size, MMAP_PROT_NONE, MMAP_MAP_NONE,
os_get_invalid_handle());
}
if (!new_mem) {
return NULL;
}
#ifdef BH_PLATFORM_WINDOWS
if (commit_size > 0
&& !os_mem_commit(new_mem, commit_size,
MMAP_PROT_READ | MMAP_PROT_WRITE)) {
os_munmap(new_mem, new_size);
return NULL;
}
#endif
if (os_mprotect(new_mem, commit_size, MMAP_PROT_READ | MMAP_PROT_WRITE)
!= 0) {
wasm_munmap_linear_memory(new_mem, new_size, new_size);
return NULL;
}
return new_mem;
}
static void *
wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size)
{
return wasm_mremap_linear_memory(NULL, 0, map_size, commit_size);
}
bool
wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count)
{
@ -638,7 +700,7 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count)
uint32 num_bytes_per_page, heap_size, total_size_old = 0;
uint32 cur_page_count, max_page_count, total_page_count;
uint64 total_size_new;
bool ret = true;
bool ret = true, full_size_mmaped;
enlarge_memory_error_reason_t failure_reason = INTERNAL_ERROR;
if (!memory) {
@ -646,12 +708,20 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count)
goto return_func;
}
heap_data_old = memory->heap_data;
heap_size = (uint32)(memory->heap_data_end - memory->heap_data);
#ifdef OS_ENABLE_HW_BOUND_CHECK
full_size_mmaped = true;
#elif WASM_ENABLE_SHARED_MEMORY != 0
full_size_mmaped = shared_memory_is_shared(memory);
#else
full_size_mmaped = false;
#endif
memory_data_old = memory->memory_data;
total_size_old = memory->memory_data_size;
heap_data_old = memory->heap_data;
heap_size = (uint32)(memory->heap_data_end - memory->heap_data);
num_bytes_per_page = memory->num_bytes_per_page;
cur_page_count = memory->cur_page_count;
max_page_count = memory->max_page_count;
@ -681,166 +751,65 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count)
total_size_new = UINT32_MAX;
}
#if WASM_ENABLE_SHARED_MEMORY != 0
if (shared_memory_is_shared(memory)) {
memory->num_bytes_per_page = num_bytes_per_page;
memory->cur_page_count = total_page_count;
memory->max_page_count = max_page_count;
SET_LINEAR_MEMORY_SIZE(memory, (uint32)total_size_new);
memory->memory_data_end = memory->memory_data + (uint32)total_size_new;
wasm_runtime_set_mem_bound_check_bytes(memory, total_size_new);
return true;
}
if (full_size_mmaped) {
#ifdef BH_PLATFORM_WINDOWS
if (!os_mem_commit(memory->memory_data_end,
(uint32)total_size_new - total_size_old,
MMAP_PROT_READ | MMAP_PROT_WRITE)) {
ret = false;
goto return_func;
}
#endif
if (heap_size > 0) {
if (mem_allocator_is_heap_corrupted(memory->heap_handle)) {
wasm_runtime_show_app_heap_corrupted_prompt();
ret = false;
goto return_func;
}
}
if (!(memory_data_new =
wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) {
if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) {
ret = false;
goto return_func;
}
if (memory_data_old) {
bh_memcpy_s(memory_data_new, (uint32)total_size_new,
memory_data_old, total_size_old);
wasm_runtime_free(memory_data_old);
}
}
memset(memory_data_new + total_size_old, 0,
(uint32)total_size_new - total_size_old);
if (heap_size > 0) {
if (mem_allocator_migrate(memory->heap_handle,
(char *)heap_data_old
+ (memory_data_new - memory_data_old),
heap_size)
if (os_mprotect(memory->memory_data_end,
(uint32)total_size_new - total_size_old,
MMAP_PROT_READ | MMAP_PROT_WRITE)
!= 0) {
/* Don't return here as memory->memory_data is obsolete and
must be updated to be correctly used later. */
#ifdef BH_PLATFORM_WINDOWS
os_mem_decommit(memory->memory_data_end,
(uint32)total_size_new - total_size_old);
#endif
ret = false;
goto return_func;
}
}
else {
if (heap_size > 0) {
if (mem_allocator_is_heap_corrupted(memory->heap_handle)) {
wasm_runtime_show_app_heap_corrupted_prompt();
ret = false;
goto return_func;
}
}
memory->heap_data = memory_data_new + (heap_data_old - memory_data_old);
memory->heap_data_end = memory->heap_data + heap_size;
if (!(memory_data_new = wasm_mremap_linear_memory(
memory_data_old, total_size_old, (uint32)total_size_new,
(uint32)total_size_new))) {
ret = false;
goto return_func;
}
memory->num_bytes_per_page = num_bytes_per_page;
memory->cur_page_count = total_page_count;
memory->max_page_count = max_page_count;
memory->memory_data_size = (uint32)total_size_new;
memory->memory_data = memory_data_new;
memory->memory_data_end = memory_data_new + (uint32)total_size_new;
wasm_runtime_set_mem_bound_check_bytes(memory, total_size_new);
if (heap_size > 0) {
if (mem_allocator_migrate(memory->heap_handle,
(char *)heap_data_old
+ (memory_data_new - memory_data_old),
heap_size)
!= 0) {
/* Don't return here as memory->memory_data is obsolete and
must be updated to be correctly used later. */
ret = false;
}
}
memory->heap_data = memory_data_new + (heap_data_old - memory_data_old);
memory->heap_data_end = memory->heap_data + heap_size;
memory->memory_data = memory_data_new;
#if defined(os_writegsbase)
/* write base addr of linear memory to GS segment register */
os_writegsbase(memory_data_new);
/* write base addr of linear memory to GS segment register */
os_writegsbase(memory_data_new);
#endif
return_func:
if (!ret && enlarge_memory_error_cb) {
WASMExecEnv *exec_env = NULL;
#if WASM_ENABLE_INTERP != 0
if (module->module_type == Wasm_Module_Bytecode)
exec_env =
((WASMModuleInstanceExtra *)module->e)->common.cur_exec_env;
#endif
#if WASM_ENABLE_AOT != 0
if (module->module_type == Wasm_Module_AoT)
exec_env =
((AOTModuleInstanceExtra *)module->e)->common.cur_exec_env;
#endif
enlarge_memory_error_cb(inc_page_count, total_size_old, 0,
failure_reason,
(WASMModuleInstanceCommon *)module, exec_env,
enlarge_memory_error_user_data);
}
return ret;
}
#else
bool
wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count)
{
WASMMemoryInstance *memory = wasm_get_default_memory(module);
uint32 num_bytes_per_page, total_size_old = 0;
uint32 cur_page_count, max_page_count, total_page_count;
uint64 total_size_new;
bool ret = true;
enlarge_memory_error_reason_t failure_reason = INTERNAL_ERROR;
if (!memory) {
ret = false;
goto return_func;
}
num_bytes_per_page = memory->num_bytes_per_page;
cur_page_count = memory->cur_page_count;
max_page_count = memory->max_page_count;
total_size_old = num_bytes_per_page * cur_page_count;
total_page_count = inc_page_count + cur_page_count;
total_size_new = num_bytes_per_page * (uint64)total_page_count;
if (inc_page_count <= 0)
/* No need to enlarge memory */
return true;
if (total_page_count < cur_page_count) { /* integer overflow */
ret = false;
goto return_func;
}
if (total_page_count > max_page_count) {
failure_reason = MAX_SIZE_REACHED;
ret = false;
goto return_func;
}
bh_assert(total_size_new <= 4 * (uint64)BH_GB);
if (total_size_new > UINT32_MAX) {
/* Resize to 1 page with size 4G-1 */
num_bytes_per_page = UINT32_MAX;
total_page_count = max_page_count = 1;
total_size_new = UINT32_MAX;
}
#ifdef BH_PLATFORM_WINDOWS
if (!os_mem_commit(memory->memory_data_end,
(uint32)total_size_new - total_size_old,
MMAP_PROT_READ | MMAP_PROT_WRITE)) {
ret = false;
goto return_func;
}
#endif
if (os_mprotect(memory->memory_data_end,
(uint32)total_size_new - total_size_old,
MMAP_PROT_READ | MMAP_PROT_WRITE)
!= 0) {
#ifdef BH_PLATFORM_WINDOWS
os_mem_decommit(memory->memory_data_end,
(uint32)total_size_new - total_size_old);
#endif
ret = false;
goto return_func;
}
/* The increased pages are filled with zero by the OS when os_mmap,
no need to memset it again here */
memory->num_bytes_per_page = num_bytes_per_page;
memory->cur_page_count = total_page_count;
memory->max_page_count = max_page_count;
@ -872,7 +841,6 @@ return_func:
return ret;
}
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
void
wasm_runtime_set_enlarge_mem_error_callback(
@ -899,3 +867,75 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count)
return ret;
}
void
wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst)
{
uint64 map_size;
bh_assert(memory_inst);
bh_assert(memory_inst->memory_data);
#ifndef OS_ENABLE_HW_BOUND_CHECK
#if WASM_ENABLE_SHARED_MEMORY != 0
if (shared_memory_is_shared(memory_inst)) {
map_size = (uint64)memory_inst->num_bytes_per_page
* memory_inst->max_page_count;
}
else
#endif
{
map_size = (uint64)memory_inst->num_bytes_per_page
* memory_inst->cur_page_count;
}
#else
map_size = 8 * (uint64)BH_GB;
#endif
wasm_munmap_linear_memory(memory_inst->memory_data,
memory_inst->memory_data_size, map_size);
memory_inst->memory_data = NULL;
}
int
wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory,
uint64 num_bytes_per_page, uint64 init_page_count,
uint64 max_page_count, uint64 *memory_data_size)
{
uint64 map_size, page_size;
bh_assert(data);
bh_assert(memory_data_size);
#ifndef OS_ENABLE_HW_BOUND_CHECK
#if WASM_ENABLE_SHARED_MEMORY != 0
if (is_shared_memory) {
/* Allocate maximum memory size when memory is shared */
map_size = max_page_count * num_bytes_per_page;
}
else
#endif
{
map_size = init_page_count * num_bytes_per_page;
}
#else /* else of OS_ENABLE_HW_BOUND_CHECK */
/* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
* ea = i + memarg.offset
* both i and memarg.offset are u32 in range 0 to 4G
* so the range of ea is 0 to 8G
*/
map_size = 8 * (uint64)BH_GB;
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
page_size = os_getpagesize();
*memory_data_size = init_page_count * num_bytes_per_page;
bh_assert(*memory_data_size <= UINT32_MAX);
align_as_and_cast(*memory_data_size, page_size);
if (map_size > 0) {
if (!(*data = wasm_mmap_linear_memory(map_size, *memory_data_size))) {
return BHT_ERROR;
}
}
return BHT_OK;
}

8
core/iwasm/common/wasm_memory.h
View File

@ -42,6 +42,14 @@ void
wasm_runtime_set_enlarge_mem_error_callback(
const enlarge_memory_error_callback_t callback, void *user_data);
void
wasm_deallocate_linear_memory(WASMMemoryInstance *memory_inst);
int
wasm_allocate_linear_memory(uint8 **data, bool is_shared_memory,
uint64 num_bytes_per_page, uint64 init_page_count,
uint64 max_page_count, uint64 *memory_data_size);
#ifdef __cplusplus
}
#endif

64
core/iwasm/common/wasm_runtime_common.c
View File

@ -6269,67 +6269,3 @@ wasm_runtime_set_linux_perf(bool flag)
enable_linux_perf = flag;
}
#endif
#ifdef WASM_LINEAR_MEMORY_MMAP
void
wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size, uint64 map_size)
{
#ifdef BH_PLATFORM_WINDOWS
os_mem_decommit(mapped_mem, commit_size);
#else
(void)commit_size;
#endif
os_munmap(mapped_mem, map_size);
}
void *
wasm_mmap_linear_memory(uint64_t map_size, uint64 *io_memory_data_size,
char *error_buf, uint32 error_buf_size)
{
uint64 page_size = os_getpagesize();
void *mapped_mem = NULL;
uint64 memory_data_size;
bh_assert(io_memory_data_size);
memory_data_size =
(*io_memory_data_size + page_size - 1) & ~(page_size - 1);
if (memory_data_size > UINT32_MAX)
memory_data_size = UINT32_MAX;
if (!(mapped_mem = os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE,
os_get_invalid_handle()))) {
set_error_buf(error_buf, error_buf_size, "mmap memory failed");
goto fail1;
}
#ifdef BH_PLATFORM_WINDOWS
if (memory_data_size > 0
&& !os_mem_commit(mapped_mem, memory_data_size,
MMAP_PROT_READ | MMAP_PROT_WRITE)) {
set_error_buf(error_buf, error_buf_size, "commit memory failed");
os_munmap(mapped_mem, map_size);
goto fail1;
}
#endif
if (os_mprotect(mapped_mem, memory_data_size,
MMAP_PROT_READ | MMAP_PROT_WRITE)
!= 0) {
set_error_buf(error_buf, error_buf_size, "mprotect memory failed");
goto fail2;
}
/* Newly allocated pages are filled with zero by the OS, we don't fill it
* again here */
*io_memory_data_size = memory_data_size;
return mapped_mem;
fail2:
wasm_munmap_linear_memory(mapped_mem, memory_data_size, map_size);
fail1:
return NULL;
}
#endif

23
core/iwasm/common/wasm_runtime_common.h
View File

@ -320,11 +320,6 @@ LOAD_I16(void *addr)
#define SHARED_MEMORY_UNLOCK(memory) (void)0
#endif
#if defined(OS_ENABLE_HW_BOUND_CHECK) \
|| (WASM_ENABLE_SHARED_MEMORY != 0 && WASM_ENABLE_SHARED_MEMORY_MMAP != 0)
#define WASM_LINEAR_MEMORY_MMAP
#endif
typedef struct WASMModuleCommon {
/* Module type, for module loaded from WASM bytecode binary,
this field is Wasm_Module_Bytecode, and this structure should
@ -421,16 +416,6 @@ typedef struct WASMRegisteredModule {
} WASMRegisteredModule;
#endif
typedef struct WASMMemoryInstanceCommon {
uint32 module_type;
/* The following uint8[1] member is a dummy just to indicate
some module_type dependent members follow.
Typically it should be accessed by casting to the corresponding
actual module_type dependent structure, not via this member. */
uint8 memory_inst_data[1];
} WASMMemoryInstanceCommon;
typedef package_type_t PackageType;
typedef wasm_section_t WASMSection, AOTSection;
@ -1098,14 +1083,6 @@ wasm_runtime_quick_invoke_c_api_native(WASMModuleInstanceCommon *module_inst,
void
wasm_runtime_show_app_heap_corrupted_prompt();
void
wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size,
uint64 map_size);
void *
wasm_mmap_linear_memory(uint64_t map_size, uint64 *io_memory_data_size,
char *error_buf, uint32 error_buf_size);
#if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
void
wasm_runtime_destroy_custom_sections(WASMCustomSection *section_list);

109
core/iwasm/interpreter/wasm_runtime.c
View File

@ -115,9 +115,6 @@ static void
memories_deinstantiate(WASMModuleInstance *module_inst,
WASMMemoryInstance **memories, uint32 count)
{
#ifdef WASM_LINEAR_MEMORY_MMAP
uint64 map_size;
#endif
uint32 i;
if (memories) {
for (i = 0; i < count; i++) {
@ -144,23 +141,7 @@ memories_deinstantiate(WASMModuleInstance *module_inst,
memories[i]->heap_handle = NULL;
}
if (memories[i]->memory_data) {
#ifndef OS_ENABLE_HW_BOUND_CHECK
#ifdef WASM_LINEAR_MEMORY_MMAP
if (shared_memory_is_shared(memories[i])) {
map_size = (uint64)memories[i]->num_bytes_per_page
* memories[i]->max_page_count;
wasm_munmap_linear_memory(memories[i]->memory_data,
map_size, map_size);
}
else
#endif
wasm_runtime_free(memories[i]->memory_data);
#else
map_size = (uint64)memories[i]->num_bytes_per_page
* memories[i]->cur_page_count;
wasm_munmap_linear_memory(memories[i]->memory_data,
map_size, 8 * (uint64)BH_GB);
#endif
wasm_deallocate_linear_memory(memories[i]);
}
}
}
@ -182,13 +163,10 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
uint32 inc_page_count, aux_heap_base, global_idx;
uint32 bytes_of_last_page, bytes_to_page_end;
uint8 *global_addr;
#ifdef WASM_LINEAR_MEMORY_MMAP
uint8 *mapped_mem = NULL;
uint64 map_size;
#endif
bool is_shared_memory = false;
#if WASM_ENABLE_SHARED_MEMORY != 0
bool is_shared_memory = flags & 0x02 ? true : false;
is_shared_memory = flags & 0x02 ? true : false;
/* shared memory */
if (is_shared_memory && parent != NULL) {
@ -197,6 +175,10 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
shared_memory_inc_reference(memory);
return memory;
}
#else
(void)parent;
(void)memory_idx;
(void)flags;
#endif /* end of WASM_ENABLE_SHARED_MEMORY */
if (heap_size > 0 && module_inst->module->malloc_function != (uint32)-1
@ -296,57 +278,20 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
num_bytes_per_page, init_page_count, max_page_count);
LOG_VERBOSE(" heap offset: %u, heap size: %d\n", heap_offset, heap_size);
memory_data_size = (uint64)num_bytes_per_page * init_page_count;
max_memory_data_size = (uint64)num_bytes_per_page * max_page_count;
bh_assert(memory_data_size <= UINT32_MAX);
bh_assert(max_memory_data_size <= 4 * (uint64)BH_GB);
(void)max_memory_data_size;
bh_assert(memory != NULL);
#ifndef OS_ENABLE_HW_BOUND_CHECK
#if WASM_ENABLE_SHARED_MEMORY != 0
if (is_shared_memory) {
/* Allocate maximum memory size when memory is shared */
#if WASM_ENABLE_SHARED_MEMORY_MMAP != 0
map_size = max_memory_data_size;
if (max_memory_data_size > 0
&& !(memory->memory_data = mapped_mem =
wasm_mmap_linear_memory(map_size, &max_memory_data_size,
error_buf, error_buf_size))) {
goto fail1;
}
#else
if (max_memory_data_size > 0
&& !(memory->memory_data = runtime_malloc(
max_memory_data_size, error_buf, error_buf_size))) {
goto fail1;
}
#endif
if (wasm_allocate_linear_memory(&memory->memory_data, is_shared_memory,
num_bytes_per_page, init_page_count,
max_page_count, &memory_data_size)
!= BHT_OK) {
set_error_buf(error_buf, error_buf_size,
"allocate linear memory failed");
return NULL;
}
else
#endif /* end of WASM_ENABLE_SHARED_MEMORY != 0 */
{
/* Allocate initial memory size when memory is not shared */
if (memory_data_size > 0
&& !(memory->memory_data = runtime_malloc(
memory_data_size, error_buf, error_buf_size))) {
goto fail1;
}
}
#else /* else of OS_ENABLE_HW_BOUND_CHECK */
/* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
* ea = i + memarg.offset
* both i and memarg.offset are u32 in range 0 to 4G
* so the range of ea is 0 to 8G
*/
map_size = 8 * (uint64)BH_GB;
if (!(memory->memory_data = mapped_mem = wasm_mmap_linear_memory(
map_size, &memory_data_size, error_buf, error_buf_size))) {
set_error_buf(error_buf, error_buf_size, "mmap memory failed");
goto fail1;
}
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
memory->module_type = Wasm_Module_Bytecode;
memory->num_bytes_per_page = num_bytes_per_page;
@ -364,13 +309,13 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
if (!(memory->heap_handle = runtime_malloc(
(uint64)heap_struct_size, error_buf, error_buf_size))) {
goto fail2;
goto fail1;
}
if (!mem_allocator_create_with_struct_and_pool(
memory->heap_handle, heap_struct_size, memory->heap_data,
heap_size)) {
set_error_buf(error_buf, error_buf_size, "init app heap failed");
goto fail3;
goto fail2;
}
}
@ -388,20 +333,13 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
LOG_VERBOSE("Memory instantiate success.");
return memory;
fail3:
fail2:
if (heap_size > 0)
wasm_runtime_free(memory->heap_handle);
fail2:
#ifdef WASM_LINEAR_MEMORY_MMAP
if (mapped_mem)
wasm_munmap_linear_memory(mapped_mem, memory_data_size, map_size);
else
#endif
{
if (memory->memory_data)
wasm_runtime_free(memory->memory_data);
}
fail1:
if (memory->memory_data)
wasm_deallocate_linear_memory(memory);
return NULL;
}
@ -633,7 +571,7 @@ fail:
* Destroy function instances.
*/
static void
functions_deinstantiate(WASMFunctionInstance *functions, uint32 count)
functions_deinstantiate(WASMFunctionInstance *functions)
{
if (functions) {
wasm_runtime_free(functions);
@ -1561,6 +1499,7 @@ set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode,
#if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0) /* No possible multi-tier JIT */
(void)first_time_set;
module_inst->e->running_mode = running_mode;
if (running_mode == Mode_Interp) {
@ -2342,12 +2281,10 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst)
}
tables_deinstantiate(module_inst);
functions_deinstantiate(module_inst->e->functions,
module_inst->e->function_count);
functions_deinstantiate(module_inst->e->functions);
#if WASM_ENABLE_TAGS != 0
tags_deinstantiate(module_inst->e->tags, module_inst->e->import_tag_ptrs);
#endif
globals_deinstantiate(module_inst->e->globals);
export_functions_deinstantiate(module_inst->export_functions);
#if WASM_ENABLE_TAGS != 0

12
core/shared/platform/common/memory/mremap.c Normal file
View File

@ -0,0 +1,12 @@
/*
* Copyright (C) 2024 Amazon Inc. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#include "bh_memutils.h"
void *
os_mremap(void *old_addr, size_t old_size, size_t new_size)
{
return bh_memory_remap_slow(old_addr, old_size, new_size);
}

4
core/shared/platform/common/memory/platform_api_memory.cmake Normal file
View File

@ -0,0 +1,4 @@
# Copyright (C) 2024 Amazon Inc. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
file (GLOB_RECURSE PLATFORM_COMMON_MEMORY_SOURCE ${CMAKE_CURRENT_LIST_DIR}/*.c)

15
core/shared/platform/common/posix/platform_api_posix.cmake
View File

@ -16,4 +16,19 @@ else()
set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE})
endif()
# This is to support old CMake version. Newer version of CMake could use
# list APPEND/POP_BACK methods.
include(CheckSymbolExists)
set (CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE ${CMAKE_REQUIRED_DEFINITIONS})
check_symbol_exists (mremap "sys/mman.h" MREMAP_EXISTS)
list (REMOVE_AT CMAKE_REQUIRED_DEFINITIONS 0)
if(MREMAP_EXISTS)
add_definitions (-DWASM_HAVE_MREMAP=1)
else()
add_definitions (-DWASM_HAVE_MREMAP=0)
include (${CMAKE_CURRENT_LIST_DIR}/../memory/platform_api_memory.cmake)
set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE})
endif()
set (PLATFORM_COMMON_POSIX_SOURCE ${source_all} )

23
core/shared/platform/common/posix/posix_memmap.c
View File

@ -3,6 +3,12 @@
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#if !defined(_GNU_SOURCE) && WASM_HAVE_MREMAP != 0
/* Enable mremap */
#define _GNU_SOURCE
#include "bh_memutils.h"
#endif
#include "platform_api_vmcore.h"
#if defined(__APPLE__) || defined(__MACH__)
@ -236,6 +242,23 @@ os_munmap(void *addr, size_t size)
}
}
#if WASM_HAVE_MREMAP != 0
void *
os_mremap(void *old_addr, size_t old_size, size_t new_size)
{
void *ptr = mremap(old_addr, old_size, new_size, MREMAP_MAYMOVE);
if (ptr == MAP_FAILED) {
#if BH_ENABLE_TRACE_MMAP != 0
os_printf("mremap failed: %d\n", errno);
#endif
return bh_memory_remap_slow(old_addr, old_size, new_size);
}
return ptr;
}
#endif
int
os_mprotect(void *addr, size_t size, int prot)
{

3
core/shared/platform/darwin/shared_platform.cmake
View File

@ -12,6 +12,9 @@ include (${CMAKE_CURRENT_LIST_DIR}/../common/posix/platform_api_posix.cmake)
file (GLOB_RECURSE source_all ${PLATFORM_SHARED_DIR}/*.c)
include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake)
set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE})
set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_POSIX_SOURCE})
file (GLOB header ${PLATFORM_SHARED_DIR}/../include/*.h)

7
core/shared/platform/include/platform_api_vmcore.h
View File

@ -132,7 +132,7 @@ enum {
MMAP_MAP_32BIT = 1,
/* Don't interpret addr as a hint: place the mapping at exactly
that address. */
MMAP_MAP_FIXED = 2
MMAP_MAP_FIXED = 2,
};
void *
@ -142,6 +142,11 @@ os_munmap(void *addr, size_t size);
int
os_mprotect(void *addr, size_t size, int prot);
/* Doesn't guarantee that protection flags will be preserved.
os_mprotect() must be called after remapping. */
void *
os_mremap(void *old_addr, size_t old_size, size_t new_size);
#if (WASM_MEM_DUAL_BUS_MIRROR != 0)
void *
os_get_dbus_mirror(void *ibus);

2
core/shared/platform/linux-sgx/platform_internal.h
View File

@ -79,6 +79,8 @@ os_get_invalid_handle()
return -1;
}
#define os_getpagesize getpagesize
#ifdef __cplusplus
}
#endif

3
core/shared/platform/linux-sgx/shared_platform.cmake
View File

@ -37,6 +37,9 @@ else()
set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE})
endif()
include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake)
set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE})
file (GLOB source_all_untrusted ${PLATFORM_SHARED_DIR}/untrusted/*.c)
set (PLATFORM_SHARED_SOURCE ${source_all})

2
core/shared/platform/nuttx/platform_internal.h
View File

@ -44,6 +44,8 @@ typedef pthread_t korp_thread;
typedef pthread_rwlock_t korp_rwlock;
typedef sem_t korp_sem;
#define os_getpagesize getpagesize
#define OS_THREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
#define BH_APPLET_PRESERVED_STACK_SIZE (2 * BH_KB)

3
core/shared/platform/nuttx/shared_platform.cmake
View File

@ -16,5 +16,6 @@ if (WAMR_BUILD_LIBC_WASI EQUAL 1)
set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE})
endif ()
set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE})
include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake)
set (PLATFORM_SHARED_SOURCE ${source_all} ${PLATFORM_COMMON_MATH_SOURCE} ${PLATFORM_COMMON_MEMORY_SOURCE})

3
core/shared/platform/windows/shared_platform.cmake
View File

@ -20,6 +20,9 @@ else()
set(source_all ${source_all} ${PLATFORM_COMMON_LIBC_UTIL_SOURCE})
endif()
include (${CMAKE_CURRENT_LIST_DIR}/../common/memory/platform_api_memory.cmake)
set (source_all ${source_all} ${PLATFORM_COMMON_MEMORY_SOURCE})
set (PLATFORM_SHARED_SOURCE ${source_all})
file (GLOB header ${PLATFORM_SHARED_DIR}/../include/*.h)

24
core/shared/utils/bh_memutils.c Normal file
View File

@ -0,0 +1,24 @@
/*
* Copyright (C) 2024 Amazon Inc. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#include "bh_memutils.h"
void *
bh_memory_remap_slow(void *old_addr, size_t old_size, size_t new_size)
{
void *new_memory =
os_mmap(NULL, new_size, MMAP_PROT_WRITE | MMAP_PROT_READ, 0, -1);
if (!new_memory) {
return NULL;
}
/*
* bh_memcpy_s can't be used as it doesn't support values bigger than
* UINT32_MAX
*/
memcpy(new_memory, old_addr, new_size < old_size ? new_size : old_size);
os_munmap(old_addr, old_size);
return new_memory;
}

35
core/shared/utils/bh_memutils.h Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright (C) 2024 Amazon Inc. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#ifndef _BH_MEMUTILS_H
#define _BH_MEMUTILS_H
#include "bh_platform.h"
#ifdef __cplusplus
extern "C" {
#endif
/**
* Remaps memory by mapping a new region, copying data from the old
* region and umapping the old region.
*
* Unless the behavior is desired, in most cases os_mremap should be used
* as it's at worst equally slow as this function, and on some platforms
* (e.g. posix with mremap) os_mremap will perform better.
*
* @param old_addr an old address.
* @param old_size a size of the old address.
* @param new_size a size of the new memory region.
* @return a pointer to the new memory region.
*/
void *
bh_memory_remap_slow(void *old_addr, size_t old_size, size_t new_size);
#ifdef __cplusplus
}
#endif
#endif /* end of _BH_MEMUTILS_H */

9
doc/memory_usage.md
View File

@ -51,14 +51,9 @@ They are in the number of WASM pages, each of which is of 65536 bytes.
The `max` is optional for non-shared memory. When omitted, it effectivily
means unlimited.
If `OS_ENABLE_HW_BOUND_CHECK` is enabled, the memory is allocated via
`os_mmap` and `os_mem_commit`/`os_mprotect`.
Otherwise, it's allocated from the global heap.
The linear memory is allocated via `os_mmap` and `os_mem_commit`/`os_mprotect`.
If the memory is shared and `OS_ENABLE_HW_BOUND_CHECK` is not enabled,
the `max` size of memory is allocated on instantiation.
Otherwise, the `min` size of memory is allocated on instantiation.
The `min` size of memory is allocated on instantiation.
It can later grow up to the `max` size via the `memory.grow` instruction.
Libc heap

7
product-mini/platforms/linux-sgx/CMakeLists.txt
View File

@ -176,3 +176,10 @@ else()
OUTPUT_VARIABLE cmdOutput
)
endif()
if (WAMR_BUILD_SPEC_TEST EQUAL 1)
execute_process(
COMMAND bash -c "sed -i -E 's/<ReservedMemMaxSize>0x1000000<\\/ReservedMemMaxSize>/<ReservedMemMaxSize>0x8000000<\\/ReservedMemMaxSize>/g' ${CMAKE_CURRENT_SOURCE_DIR}/enclave-sample/Enclave/Enclave.config.xml"
OUTPUT_VARIABLE cmdOutput
)
endif()

3
product-mini/platforms/nuttx/wamr.mk
View File

@ -391,6 +391,7 @@ CSRCS += nuttx_platform.c \
posix_thread.c \
posix_time.c \
posix_sleep.c \
mremap.c \
mem_alloc.c \
ems_kfc.c \
ems_alloc.c \
@ -401,6 +402,7 @@ CSRCS += nuttx_platform.c \
bh_hashmap.c \
bh_list.c \
bh_log.c \
bh_memutils.c \
bh_queue.c \
bh_vector.c \
bh_read_file.c \
@ -416,6 +418,7 @@ CSRCS += nuttx_platform.c \
ASRCS += $(INVOKE_NATIVE)
VPATH += $(SHARED_ROOT)/platform/nuttx
VPATH += $(SHARED_ROOT)/platform/common/memory
VPATH += $(SHARED_ROOT)/platform/common/posix
VPATH += $(SHARED_ROOT)/platform/common/libc-util
VPATH += $(SHARED_ROOT)/mem-alloc