diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 4515e4a9e..ff05456fd 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -412,18 +412,27 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, if (init_page_count == max_page_count && init_page_count == 1) { /* If only one page and at most one page, we just append the app heap to the end of linear memory, enlarge the - num_bytes_per_page, and don't change the page count*/ + num_bytes_per_page, and don't change the page count */ heap_offset = num_bytes_per_page; num_bytes_per_page += heap_size; if (num_bytes_per_page < heap_size) { set_error_buf(error_buf, error_buf_size, - "memory size must be at most 65536 pages (4GiB)"); + "failed to insert app heap into linear memory, " + "try using `--heap_size=0` option"); return NULL; } } else if (heap_size > 0) { - if (module->aux_heap_base_global_index != (uint32)-1 - && module->aux_heap_base < num_bytes_per_page * init_page_count) { + if (init_page_count == max_page_count && init_page_count == 0) { + /* If the memory data size is always 0, we resize it to + one page for app heap */ + num_bytes_per_page = heap_size; + heap_offset = 0; + inc_page_count = 1; + } + else if (module->aux_heap_base_global_index != (uint32)-1 + && module->aux_heap_base + < num_bytes_per_page * init_page_count) { /* Insert app heap before __heap_base */ aux_heap_base = module->aux_heap_base; bytes_of_last_page = aux_heap_base % num_bytes_per_page; @@ -464,13 +473,18 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, } init_page_count += inc_page_count; max_page_count += inc_page_count; - if (init_page_count > 65536) { + if (init_page_count > DEFAULT_MAX_PAGES) { set_error_buf(error_buf, error_buf_size, - "memory size must be at most 65536 pages (4GiB)"); + "failed to insert app heap into linear memory, " + "try using `--heap_size=0` option"); return NULL; } - if (max_page_count > 65536) - max_page_count = 65536; + else if (init_page_count == DEFAULT_MAX_PAGES) { + num_bytes_per_page = UINT32_MAX; + init_page_count = max_page_count = 1; + } + if (max_page_count > DEFAULT_MAX_PAGES) + max_page_count = DEFAULT_MAX_PAGES; } LOG_VERBOSE("Memory instantiate:"); @@ -487,6 +501,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, total_size = (uint64)num_bytes_per_page * max_page_count; } #endif + bh_assert(total_size <= UINT32_MAX); #ifndef OS_ENABLE_HW_BOUND_CHECK /* Allocate memory */ @@ -502,9 +517,8 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, * both i and memarg.offset are u32 in range 0 to 4G * so the range of ea is 0 to 8G */ - if (total_size >= UINT32_MAX - || !(p = mapped_mem = - os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE))) { + if (!(p = mapped_mem = + os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE))) { set_error_buf(error_buf, error_buf_size, "mmap memory failed"); return NULL; } @@ -529,15 +543,18 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, * again here */ #endif /* end of OS_ENABLE_HW_BOUND_CHECK */ + if (total_size > UINT32_MAX) + total_size = UINT32_MAX; + memory_inst->module_type = Wasm_Module_AoT; memory_inst->num_bytes_per_page = num_bytes_per_page; memory_inst->cur_page_count = init_page_count; memory_inst->max_page_count = max_page_count; + memory_inst->memory_data_size = (uint32)total_size; /* Init memory info */ memory_inst->memory_data.ptr = p; memory_inst->memory_data_end.ptr = p + (uint32)total_size; - memory_inst->memory_data_size = (uint32)total_size; /* Initialize heap info */ memory_inst->heap_data.ptr = p + heap_offset; @@ -561,21 +578,19 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, } if (total_size > 0) { - if (sizeof(uintptr_t) == sizeof(uint64)) { - memory_inst->mem_bound_check_1byte.u64 = total_size - 1; - memory_inst->mem_bound_check_2bytes.u64 = total_size - 2; - memory_inst->mem_bound_check_4bytes.u64 = total_size - 4; - memory_inst->mem_bound_check_8bytes.u64 = total_size - 8; - memory_inst->mem_bound_check_16bytes.u64 = total_size - 16; - } - else { - memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1; - memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2; - memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4; - memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8; - memory_inst->mem_bound_check_16bytes.u32[0] = - (uint32)total_size - 16; - } +#if UINTPTR_MAX == UINT64_MAX + memory_inst->mem_bound_check_1byte.u64 = total_size - 1; + memory_inst->mem_bound_check_2bytes.u64 = total_size - 2; + memory_inst->mem_bound_check_4bytes.u64 = total_size - 4; + memory_inst->mem_bound_check_8bytes.u64 = total_size - 8; + memory_inst->mem_bound_check_16bytes.u64 = total_size - 16; +#else + memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1; + memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2; + memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4; + memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8; + memory_inst->mem_bound_check_16bytes.u32[0] = (uint32)total_size - 16; +#endif } #if WASM_ENABLE_SHARED_MEMORY != 0 @@ -2032,26 +2047,29 @@ aot_get_native_addr_range(AOTModuleInstance *module_inst, uint8 *native_ptr, bool aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) { - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint32 num_bytes_per_page, cur_page_count, max_page_count; - uint32 total_page_count, total_size_old, heap_size; - uint64 total_size; - uint8 *memory_data_old, *heap_data_old, *memory_data, *heap_data; + AOTMemoryInstance *memory = aot_get_default_memory(module_inst); + uint8 *memory_data_old, *memory_data_new, *heap_data_old; + uint32 num_bytes_per_page, heap_size, total_size_old; + uint32 cur_page_count, max_page_count, total_page_count; + uint64 total_size_new; bool ret = true; - if (!memory_inst) + if (!memory) return false; - num_bytes_per_page = memory_inst->num_bytes_per_page; - cur_page_count = memory_inst->cur_page_count; - max_page_count = memory_inst->max_page_count; - total_page_count = cur_page_count + inc_page_count; - total_size_old = memory_inst->memory_data_size; - total_size = (uint64)num_bytes_per_page * total_page_count; - heap_size = (uint32)((uint8 *)memory_inst->heap_data_end.ptr - - (uint8 *)memory_inst->heap_data.ptr); - memory_data_old = (uint8 *)memory_inst->memory_data.ptr; - heap_data_old = (uint8 *)memory_inst->heap_data.ptr; + heap_data_old = (uint8 *)memory->heap_data.ptr; + heap_size = (uint32)((uint8 *)memory->heap_data_end.ptr + - (uint8 *)memory->heap_data.ptr); + + memory_data_old = (uint8 *)memory->memory_data.ptr; + total_size_old = + (uint32)((uint8 *)memory->memory_data_end.ptr - memory_data_old); + + num_bytes_per_page = memory->num_bytes_per_page; + cur_page_count = memory->cur_page_count; + max_page_count = memory->max_page_count; + total_page_count = inc_page_count + cur_page_count; + total_size_new = num_bytes_per_page * (uint64)total_page_count; if (inc_page_count <= 0) /* No need to enlarge memory */ @@ -2062,94 +2080,103 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) return false; } - if (total_size >= UINT32_MAX) { - return false; + bh_assert(total_size_new <= 4 * (uint64)BH_GB); + if (total_size_new > UINT32_MAX) { + /* Resize to 1 page with size 4G-1 */ + num_bytes_per_page = UINT32_MAX; + total_page_count = max_page_count = 1; + total_size_new = UINT32_MAX; } #if WASM_ENABLE_SHARED_MEMORY != 0 - if (memory_inst->is_shared) { - /* For shared memory, we have reserved the maximum spaces during - instantiate, only change the cur_page_count here */ - memory_inst->cur_page_count = total_page_count; + if (memory->is_shared) { + memory->num_bytes_per_page = UINT32_MAX; + memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; + memory->memory_data_size = (uint32)total_size_new; return true; } #endif if (heap_size > 0) { - if (mem_allocator_is_heap_corrupted(memory_inst->heap_handle.ptr)) { + if (mem_allocator_is_heap_corrupted(memory->heap_handle.ptr)) { wasm_runtime_show_app_heap_corrupted_prompt(); return false; } } - if (!(memory_data = - wasm_runtime_realloc(memory_data_old, (uint32)total_size))) { - if (!(memory_data = wasm_runtime_malloc((uint32)total_size))) { + if (!(memory_data_new = + wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) { + if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) { return false; } if (memory_data_old) { - bh_memcpy_s(memory_data, (uint32)total_size, memory_data_old, - total_size_old); + bh_memcpy_s(memory_data_new, (uint32)total_size_new, + memory_data_old, total_size_old); wasm_runtime_free(memory_data_old); } } - memset(memory_data + total_size_old, 0, - (uint32)total_size - total_size_old); - - memory_inst->cur_page_count = total_page_count; - memory_inst->memory_data_size = (uint32)total_size; - memory_inst->memory_data.ptr = memory_data; - memory_inst->memory_data_end.ptr = memory_data + total_size; + memset(memory_data_new + total_size_old, 0, + (uint32)total_size_new - total_size_old); if (heap_size > 0) { - if (mem_allocator_migrate(memory_inst->heap_handle.ptr, + if (mem_allocator_migrate(memory->heap_handle.ptr, (char *)heap_data_old - + (memory_data - memory_data_old), - heap_size)) { + + (memory_data_new - memory_data_old), + heap_size) + != 0) { /* Don't return here as memory->memory_data is obsolete and must be updated to be correctly used later. */ ret = false; } } - heap_data = memory_data + (heap_data_old - memory_data_old); - memory_inst->heap_data.ptr = heap_data; - memory_inst->heap_data_end.ptr = heap_data + heap_size; + memory->heap_data.ptr = memory_data_new + (heap_data_old - memory_data_old); + memory->heap_data_end.ptr = (uint8 *)memory->heap_data.ptr + heap_size; + + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; + + memory->memory_data.ptr = memory_data_new; + memory->memory_data_end.ptr = memory_data_new + (uint32)total_size_new; + memory->memory_data_size = (uint32)total_size_new; + +#if UINTPTR_MAX == UINT64_MAX + memory->mem_bound_check_1byte.u64 = total_size_new - 1; + memory->mem_bound_check_2bytes.u64 = total_size_new - 2; + memory->mem_bound_check_4bytes.u64 = total_size_new - 4; + memory->mem_bound_check_8bytes.u64 = total_size_new - 8; + memory->mem_bound_check_16bytes.u64 = total_size_new - 16; +#else + memory->mem_bound_check_1byte.u32[0] = (uint32)total_size_new - 1; + memory->mem_bound_check_2bytes.u32[0] = (uint32)total_size_new - 2; + memory->mem_bound_check_4bytes.u32[0] = (uint32)total_size_new - 4; + memory->mem_bound_check_8bytes.u32[0] = (uint32)total_size_new - 8; + memory->mem_bound_check_16bytes.u32[0] = (uint32)total_size_new - 16; +#endif - if (sizeof(uintptr_t) == sizeof(uint64)) { - memory_inst->mem_bound_check_1byte.u64 = total_size - 1; - memory_inst->mem_bound_check_2bytes.u64 = total_size - 2; - memory_inst->mem_bound_check_4bytes.u64 = total_size - 4; - memory_inst->mem_bound_check_8bytes.u64 = total_size - 8; - memory_inst->mem_bound_check_16bytes.u64 = total_size - 16; - } - else { - memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1; - memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2; - memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4; - memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8; - memory_inst->mem_bound_check_16bytes.u32[0] = (uint32)total_size - 16; - } return ret; } #else /* else of OS_ENABLE_HW_BOUND_CHECK */ bool aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) { - AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); - uint32 num_bytes_per_page, cur_page_count, max_page_count; - uint32 total_page_count; - uint64 total_size; + AOTMemoryInstance *memory = aot_get_default_memory(module_inst); + uint32 num_bytes_per_page, total_size_old; + uint32 cur_page_count, max_page_count, total_page_count; + uint64 total_size_new; - if (!memory_inst) + if (!memory) return false; - num_bytes_per_page = memory_inst->num_bytes_per_page; - cur_page_count = memory_inst->cur_page_count; - max_page_count = memory_inst->max_page_count; - total_page_count = cur_page_count + inc_page_count; - total_size = (uint64)num_bytes_per_page * total_page_count; + num_bytes_per_page = memory->num_bytes_per_page; + cur_page_count = memory->cur_page_count; + max_page_count = memory->max_page_count; + total_size_old = num_bytes_per_page * cur_page_count; + total_page_count = inc_page_count + cur_page_count; + total_size_new = num_bytes_per_page * (uint64)total_page_count; if (inc_page_count <= 0) /* No need to enlarge memory */ @@ -2160,21 +2187,29 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) return false; } + bh_assert(total_size_new <= 4 * (uint64)BH_GB); + if (total_size_new > UINT32_MAX) { + /* Resize to 1 page with size 4G-1 */ + num_bytes_per_page = UINT32_MAX; + total_page_count = max_page_count = 1; + total_size_new = UINT32_MAX; + } + #ifdef BH_PLATFORM_WINDOWS - if (!os_mem_commit(memory_inst->memory_data_end.ptr, - num_bytes_per_page * inc_page_count, + if (!os_mem_commit(memory->memory_data_end.ptr, + (uint32)total_size_new - total_size_old, MMAP_PROT_READ | MMAP_PROT_WRITE)) { return false; } #endif - if (os_mprotect(memory_inst->memory_data_end.ptr, - num_bytes_per_page * inc_page_count, + if (os_mprotect(memory->memory_data_end.ptr, + (uint32)total_size_new - total_size_old, MMAP_PROT_READ | MMAP_PROT_WRITE) != 0) { #ifdef BH_PLATFORM_WINDOWS - os_mem_decommit(memory_inst->memory_data_end.ptr, - num_bytes_per_page * inc_page_count); + os_mem_decommit(memory->memory_data_end.ptr, + (uint32)total_size_new - total_size_old); #endif return false; } @@ -2182,25 +2217,19 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) /* The increased pages are filled with zero by the OS when os_mmap, no need to memset it again here */ - memory_inst->cur_page_count = total_page_count; - memory_inst->memory_data_size = (uint32)total_size; - memory_inst->memory_data_end.ptr = - (uint8 *)memory_inst->memory_data.ptr + (uint32)total_size; + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; + memory->memory_data_size = (uint32)total_size_new; + memory->memory_data_end.ptr = + (uint8 *)memory->memory_data.ptr + (uint32)total_size_new; + + memory->mem_bound_check_1byte.u64 = total_size_new - 1; + memory->mem_bound_check_2bytes.u64 = total_size_new - 2; + memory->mem_bound_check_4bytes.u64 = total_size_new - 4; + memory->mem_bound_check_8bytes.u64 = total_size_new - 8; + memory->mem_bound_check_16bytes.u64 = total_size_new - 16; - if (sizeof(uintptr_t) == sizeof(uint64)) { - memory_inst->mem_bound_check_1byte.u64 = total_size - 1; - memory_inst->mem_bound_check_2bytes.u64 = total_size - 2; - memory_inst->mem_bound_check_4bytes.u64 = total_size - 4; - memory_inst->mem_bound_check_8bytes.u64 = total_size - 8; - memory_inst->mem_bound_check_16bytes.u64 = total_size - 16; - } - else { - memory_inst->mem_bound_check_1byte.u32[0] = (uint32)total_size - 1; - memory_inst->mem_bound_check_2bytes.u32[0] = (uint32)total_size - 2; - memory_inst->mem_bound_check_4bytes.u32[0] = (uint32)total_size - 4; - memory_inst->mem_bound_check_8bytes.u32[0] = (uint32)total_size - 8; - memory_inst->mem_bound_check_16bytes.u32[0] = (uint32)total_size - 16; - } return true; } #endif /* end of OS_ENABLE_HW_BOUND_CHECK */ diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index 1da303d85..07a553d3a 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -29,6 +29,7 @@ extern "C" { #define VALUE_TYPE_ANY 0x42 #define DEFAULT_NUM_BYTES_PER_PAGE 65536 +#define DEFAULT_MAX_PAGES 65536 #define NULL_REF (0xFFFFFFFF) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index cf106c8af..6fa0a21d1 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -1266,7 +1266,7 @@ fail: static bool check_memory_init_size(uint32 init_size, char *error_buf, uint32 error_buf_size) { - if (init_size > 65536) { + if (init_size > DEFAULT_MAX_PAGES) { set_error_buf(error_buf, error_buf_size, "memory size must be at most 65536 pages (4GiB)"); return false; @@ -1284,7 +1284,7 @@ check_memory_max_size(uint32 init_size, uint32 max_size, char *error_buf, return false; } - if (max_size > 65536) { + if (max_size > DEFAULT_MAX_PAGES) { set_error_buf(error_buf, error_buf_size, "memory size must be at most 65536 pages (4GiB)"); return false; @@ -1299,12 +1299,12 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end, char *error_buf, uint32 error_buf_size) { const uint8 *p = *p_buf, *p_end = buf_end; - uint32 pool_size = wasm_runtime_memory_pool_size(); #if WASM_ENABLE_APP_FRAMEWORK != 0 + uint32 pool_size = wasm_runtime_memory_pool_size(); uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT / DEFAULT_NUM_BYTES_PER_PAGE; #else - uint32 max_page_count = pool_size / DEFAULT_NUM_BYTES_PER_PAGE; + uint32 max_page_count = DEFAULT_MAX_PAGES; #endif /* WASM_ENABLE_APP_FRAMEWORK */ uint32 declare_max_page_count_flag = 0; uint32 declare_init_page_count = 0; @@ -1529,12 +1529,12 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, char *error_buf, uint32 error_buf_size) { const uint8 *p = *p_buf, *p_end = buf_end, *p_org; - uint32 pool_size = wasm_runtime_memory_pool_size(); #if WASM_ENABLE_APP_FRAMEWORK != 0 + uint32 pool_size = wasm_runtime_memory_pool_size(); uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT / DEFAULT_NUM_BYTES_PER_PAGE; #else - uint32 max_page_count = pool_size / DEFAULT_NUM_BYTES_PER_PAGE; + uint32 max_page_count = DEFAULT_MAX_PAGES; #endif p_org = p; @@ -3317,17 +3317,30 @@ load_from_sections(WASMModule *module, WASMSection *sections, #if WASM_ENABLE_MULTI_MODULE == 0 if (module->import_memory_count) { memory_import = &module->import_memories[0].u.memory; - /* Memory init page count cannot be larger than 65536, we don't - check integer overflow again. */ - memory_import->num_bytes_per_page *= memory_import->init_page_count; - memory_import->init_page_count = memory_import->max_page_count = 1; + if (memory_import->init_page_count < DEFAULT_MAX_PAGES) + memory_import->num_bytes_per_page *= + memory_import->init_page_count; + else + memory_import->num_bytes_per_page = UINT32_MAX; + + if (memory_import->init_page_count > 0) + memory_import->init_page_count = memory_import->max_page_count = + 1; + else + memory_import->init_page_count = memory_import->max_page_count = + 0; } if (module->memory_count) { - /* Memory init page count cannot be larger than 65536, we don't - check integer overflow again. */ memory = &module->memories[0]; - memory->num_bytes_per_page *= memory->init_page_count; - memory->init_page_count = memory->max_page_count = 1; + if (memory->init_page_count < DEFAULT_MAX_PAGES) + memory->num_bytes_per_page *= memory->init_page_count; + else + memory->num_bytes_per_page = UINT32_MAX; + + if (memory->init_page_count > 0) + memory->init_page_count = memory->max_page_count = 1; + else + memory->init_page_count = memory->max_page_count = 0; } #endif } diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 3ac80b7bd..f3c649ff5 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -534,12 +534,12 @@ load_memory_import(const uint8 **p_buf, const uint8 *buf_end, char *error_buf, uint32 error_buf_size) { const uint8 *p = *p_buf, *p_end = buf_end; - uint32 pool_size = wasm_runtime_memory_pool_size(); #if WASM_ENABLE_APP_FRAMEWORK != 0 + uint32 pool_size = wasm_runtime_memory_pool_size(); uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT / DEFAULT_NUM_BYTES_PER_PAGE; #else - uint32 max_page_count = pool_size / DEFAULT_NUM_BYTES_PER_PAGE; + uint32 max_page_count = DEFAULT_MAX_PAGES; #endif /* WASM_ENABLE_APP_FRAMEWORK */ uint32 declare_max_page_count_flag = 0; uint32 declare_init_page_count = 0; @@ -650,12 +650,12 @@ load_memory(const uint8 **p_buf, const uint8 *buf_end, WASMMemory *memory, char *error_buf, uint32 error_buf_size) { const uint8 *p = *p_buf, *p_end = buf_end, *p_org; - uint32 pool_size = wasm_runtime_memory_pool_size(); #if WASM_ENABLE_APP_FRAMEWORK != 0 + uint32 pool_size = wasm_runtime_memory_pool_size(); uint32 max_page_count = pool_size * APP_MEMORY_MAX_GLOBAL_HEAP_PERCENT / DEFAULT_NUM_BYTES_PER_PAGE; #else - uint32 max_page_count = pool_size / DEFAULT_NUM_BYTES_PER_PAGE; + uint32 max_page_count = DEFAULT_MAX_PAGES; #endif p_org = p; @@ -2153,18 +2153,31 @@ load_from_sections(WASMModule *module, WASMSection *sections, if (module->import_memory_count) { memory_import = &module->import_memories[0].u.memory; - /* Memory init page count cannot be larger than 65536, we don't - check integer overflow again. */ - memory_import->num_bytes_per_page *= memory_import->init_page_count; - memory_import->init_page_count = memory_import->max_page_count = 1; + if (memory_import->init_page_count < DEFAULT_MAX_PAGES) + memory_import->num_bytes_per_page *= + memory_import->init_page_count; + else + memory_import->num_bytes_per_page = UINT32_MAX; + + if (memory_import->init_page_count > 0) + memory_import->init_page_count = memory_import->max_page_count = + 1; + else + memory_import->init_page_count = memory_import->max_page_count = + 0; } if (module->memory_count) { - /* Memory init page count cannot be larger than 65536, we don't - check integer overflow again. */ memory = &module->memories[0]; - memory->num_bytes_per_page *= memory->init_page_count; - memory->init_page_count = memory->max_page_count = 1; + if (memory->init_page_count < DEFAULT_MAX_PAGES) + memory->num_bytes_per_page *= memory->init_page_count; + else + memory->num_bytes_per_page = UINT32_MAX; + + if (memory->init_page_count > 0) + memory->init_page_count = memory->max_page_count = 1; + else + memory->init_page_count = memory->max_page_count = 0; } } diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index e76f3b00a..b0f9a3ea3 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -206,18 +206,27 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, if (init_page_count == max_page_count && init_page_count == 1) { /* If only one page and at most one page, we just append the app heap to the end of linear memory, enlarge the - num_bytes_per_page, and don't change the page count*/ + num_bytes_per_page, and don't change the page count */ heap_offset = num_bytes_per_page; num_bytes_per_page += heap_size; if (num_bytes_per_page < heap_size) { set_error_buf(error_buf, error_buf_size, - "memory size must be at most 65536 pages (4GiB)"); + "failed to insert app heap into linear memory, " + "try using `--heap_size=0` option"); return NULL; } } else if (heap_size > 0) { - if (module->aux_heap_base_global_index != (uint32)-1 - && module->aux_heap_base < num_bytes_per_page * init_page_count) { + if (init_page_count == max_page_count && init_page_count == 0) { + /* If the memory data size is always 0, we resize it to + one page for app heap */ + num_bytes_per_page = heap_size; + heap_offset = 0; + inc_page_count = 1; + } + else if (module->aux_heap_base_global_index != (uint32)-1 + && module->aux_heap_base + < num_bytes_per_page * init_page_count) { /* Insert app heap before __heap_base */ aux_heap_base = module->aux_heap_base; bytes_of_last_page = aux_heap_base % num_bytes_per_page; @@ -259,13 +268,18 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, } init_page_count += inc_page_count; max_page_count += inc_page_count; - if (init_page_count > 65536) { + if (init_page_count > DEFAULT_MAX_PAGES) { set_error_buf(error_buf, error_buf_size, - "memory size must be at most 65536 pages (4GiB)"); + "failed to insert app heap into linear memory, " + "try using `--heap_size=0` option"); return NULL; } - if (max_page_count > 65536) - max_page_count = 65536; + else if (init_page_count == DEFAULT_MAX_PAGES) { + num_bytes_per_page = UINT32_MAX; + init_page_count = max_page_count = 1; + } + if (max_page_count > DEFAULT_MAX_PAGES) + max_page_count = DEFAULT_MAX_PAGES; } LOG_VERBOSE("Memory instantiate:"); @@ -280,6 +294,7 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, memory_data_size = (uint64)num_bytes_per_page * max_page_count; } #endif + bh_assert(memory_data_size <= 4 * (uint64)BH_GB); /* Allocate memory space, addr data and global data */ if (!(memory = runtime_malloc((uint64)sizeof(WASMMemoryInstance), error_buf, @@ -301,9 +316,8 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, * both i and memarg.offset are u32 in range 0 to 4G * so the range of ea is 0 to 8G */ - if (memory_data_size >= UINT32_MAX - || !(memory->memory_data = mapped_mem = - os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE))) { + if (!(memory->memory_data = mapped_mem = + os_mmap(NULL, map_size, MMAP_PROT_NONE, MMAP_MAP_NONE))) { set_error_buf(error_buf, error_buf_size, "mmap memory failed"); goto fail1; } @@ -327,10 +341,14 @@ memory_instantiate(WASMModuleInstance *module_inst, uint32 num_bytes_per_page, * again here */ #endif /* end of OS_ENABLE_HW_BOUND_CHECK */ + if (memory_data_size > UINT32_MAX) + memory_data_size = (uint32)memory_data_size; + memory->module_type = Wasm_Module_Bytecode; memory->num_bytes_per_page = num_bytes_per_page; memory->cur_page_count = init_page_count; memory->max_page_count = max_page_count; + memory->memory_data_size = (uint32)memory_data_size; memory->heap_data = memory->memory_data + heap_offset; memory->heap_data_end = memory->heap_data + heap_size; @@ -2419,39 +2437,49 @@ bool wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) { WASMMemoryInstance *memory = module->default_memory; - uint8 *new_memory_data, *memory_data, *heap_data_old; - uint32 heap_size, total_size_old, total_page_count; - uint64 total_size; + uint8 *memory_data_old, *memory_data_new, *heap_data_old; + uint32 num_bytes_per_page, heap_size, total_size_old; + uint32 cur_page_count, max_page_count, total_page_count; + uint64 total_size_new; bool ret = true; if (!memory) return false; - memory_data = memory->memory_data; - heap_size = (uint32)(memory->heap_data_end - memory->heap_data); - total_size_old = (uint32)(memory->memory_data_end - memory_data); - total_page_count = inc_page_count + memory->cur_page_count; - total_size = memory->num_bytes_per_page * (uint64)total_page_count; heap_data_old = memory->heap_data; + heap_size = (uint32)(memory->heap_data_end - memory->heap_data); + + memory_data_old = memory->memory_data; + total_size_old = memory->memory_data_size; + + num_bytes_per_page = memory->num_bytes_per_page; + cur_page_count = memory->cur_page_count; + max_page_count = memory->max_page_count; + total_page_count = inc_page_count + cur_page_count; + total_size_new = num_bytes_per_page * (uint64)total_page_count; if (inc_page_count <= 0) /* No need to enlarge memory */ return true; - if (total_page_count < memory->cur_page_count /* integer overflow */ - || total_page_count > memory->max_page_count) { + if (total_page_count < cur_page_count /* integer overflow */ + || total_page_count > max_page_count) { return false; } - if (total_size >= UINT32_MAX) { - return false; + bh_assert(total_size_new <= 4 * (uint64)BH_GB); + if (total_size_new > UINT32_MAX) { + /* Resize to 1 page with size 4G-1 */ + num_bytes_per_page = UINT32_MAX; + total_page_count = max_page_count = 1; + total_size_new = UINT32_MAX; } #if WASM_ENABLE_SHARED_MEMORY != 0 if (memory->is_shared) { - /* For shared memory, we have reserved the maximum spaces during - instantiate, only change the cur_page_count here */ + memory->num_bytes_per_page = UINT32_MAX; memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; return true; } #endif @@ -2463,25 +2491,25 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) } } - if (!(new_memory_data = - wasm_runtime_realloc(memory_data, (uint32)total_size))) { - if (!(new_memory_data = wasm_runtime_malloc((uint32)total_size))) { + if (!(memory_data_new = + wasm_runtime_realloc(memory_data_old, (uint32)total_size_new))) { + if (!(memory_data_new = wasm_runtime_malloc((uint32)total_size_new))) { return false; } - if (memory_data) { - bh_memcpy_s(new_memory_data, (uint32)total_size, memory_data, - total_size_old); - wasm_runtime_free(memory_data); + if (memory_data_old) { + bh_memcpy_s(memory_data_new, (uint32)total_size_new, + memory_data_old, total_size_old); + wasm_runtime_free(memory_data_old); } } - memset(new_memory_data + total_size_old, 0, - (uint32)total_size - total_size_old); + memset(memory_data_new + total_size_old, 0, + (uint32)total_size_new - total_size_old); if (heap_size > 0) { if (mem_allocator_migrate(memory->heap_handle, (char *)heap_data_old - + (new_memory_data - memory_data), + + (memory_data_new - memory_data_old), heap_size) != 0) { /* Don't return here as memory->memory_data is obsolete and @@ -2490,26 +2518,30 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) } } - memory->memory_data = new_memory_data; - memory->cur_page_count = total_page_count; - memory->heap_data = new_memory_data + (heap_data_old - memory_data); + memory->heap_data = memory_data_new + (heap_data_old - memory_data_old); memory->heap_data_end = memory->heap_data + heap_size; - memory->memory_data_end = - memory->memory_data + memory->num_bytes_per_page * total_page_count; + + memory->num_bytes_per_page = num_bytes_per_page; + memory->cur_page_count = total_page_count; + memory->max_page_count = max_page_count; + memory->memory_data_size = (uint32)total_size_new; + + memory->memory_data = memory_data_new; + memory->memory_data_end = memory_data_new + (uint32)total_size_new; #if WASM_ENABLE_FAST_JIT != 0 #if UINTPTR_MAX == UINT64_MAX - memory->mem_bound_check_1byte = total_size - 1; - memory->mem_bound_check_2bytes = total_size - 2; - memory->mem_bound_check_4bytes = total_size - 4; - memory->mem_bound_check_8bytes = total_size - 8; - memory->mem_bound_check_16bytes = total_size - 16; + memory->mem_bound_check_1byte = total_size_new - 1; + memory->mem_bound_check_2bytes = total_size_new - 2; + memory->mem_bound_check_4bytes = total_size_new - 4; + memory->mem_bound_check_8bytes = total_size_new - 8; + memory->mem_bound_check_16bytes = total_size_new - 16; #else - memory->mem_bound_check_1byte = (uint32)total_size - 1; - memory->mem_bound_check_2bytes = (uint32)total_size - 2; - memory->mem_bound_check_4bytes = (uint32)total_size - 4; - memory->mem_bound_check_8bytes = (uint32)total_size - 8; - memory->mem_bound_check_16bytes = (uint32)total_size - 16; + memory->mem_bound_check_1byte = (uint32)total_size_new - 1; + memory->mem_bound_check_2bytes = (uint32)total_size_new - 2; + memory->mem_bound_check_4bytes = (uint32)total_size_new - 4; + memory->mem_bound_check_8bytes = (uint32)total_size_new - 8; + memory->mem_bound_check_16bytes = (uint32)total_size_new - 16; #endif #endif @@ -2520,39 +2552,52 @@ bool wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) { WASMMemoryInstance *memory = module->default_memory; - uint32 num_bytes_per_page, total_page_count; + uint32 num_bytes_per_page, total_size_old; + uint32 cur_page_count, max_page_count, total_page_count; + uint64 total_size_new; if (!memory) return false; - total_page_count = inc_page_count + memory->cur_page_count; + num_bytes_per_page = memory->num_bytes_per_page; + cur_page_count = memory->cur_page_count; + max_page_count = memory->max_page_count; + total_size_old = num_bytes_per_page * cur_page_count; + total_page_count = inc_page_count + cur_page_count; + total_size_new = num_bytes_per_page * (uint64)total_page_count; if (inc_page_count <= 0) /* No need to enlarge memory */ return true; - if (total_page_count < memory->cur_page_count /* integer overflow */ - || total_page_count > memory->max_page_count) { + if (total_page_count < cur_page_count /* integer overflow */ + || total_page_count > max_page_count) { return false; } - num_bytes_per_page = memory->num_bytes_per_page; + bh_assert(total_size_new <= 4 * (uint64)BH_GB); + if (total_size_new > UINT32_MAX) { + /* Resize to 1 page with size 4G-1 */ + num_bytes_per_page = UINT32_MAX; + total_page_count = max_page_count = 1; + total_size_new = UINT32_MAX; + } #ifdef BH_PLATFORM_WINDOWS if (!os_mem_commit(memory->memory_data_end, - num_bytes_per_page * inc_page_count, + (uint32)total_size_new - total_size_old, MMAP_PROT_READ | MMAP_PROT_WRITE)) { return false; } #endif if (os_mprotect(memory->memory_data_end, - num_bytes_per_page * inc_page_count, + (uint32)total_size_new - total_size_old, MMAP_PROT_READ | MMAP_PROT_WRITE) != 0) { #ifdef BH_PLATFORM_WINDOWS os_mem_decommit(memory->memory_data_end, - num_bytes_per_page * inc_page_count); + (uint32)total_size_new - total_size_old); #endif return false; } @@ -2560,9 +2605,20 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) /* The increased pages are filled with zero by the OS when os_mmap, no need to memset it again here */ + memory->num_bytes_per_page = num_bytes_per_page; memory->cur_page_count = total_page_count; - memory->memory_data_end = - memory->memory_data + num_bytes_per_page * total_page_count; + memory->max_page_count = max_page_count; + memory->memory_data_size = (uint32)total_size_new; + memory->memory_data_end = memory->memory_data + (uint32)total_size_new; + +#if WASM_ENABLE_FAST_JIT != 0 + memory->mem_bound_check_1byte = total_size_new - 1; + memory->mem_bound_check_2bytes = total_size_new - 2; + memory->mem_bound_check_4bytes = total_size_new - 4; + memory->mem_bound_check_8bytes = total_size_new - 8; + memory->mem_bound_check_16bytes = total_size_new - 16; +#endif + return true; } #endif /* end of OS_ENABLE_HW_BOUND_CHECK */ diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index bbdf9e866..581b80ee3 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -26,12 +26,25 @@ struct WASMMemoryInstance { uint32 module_type; /* Shared memory flag */ bool is_shared; + /* Number bytes per page */ uint32 num_bytes_per_page; /* Current page count */ uint32 cur_page_count; /* Maximum page count */ uint32 max_page_count; + /* Memory data size */ + uint32 memory_data_size; + + /** + * Memory data begin address, Note: + * the app-heap might be inserted in to the linear memory, + * when memory is re-allocated, the heap data and memory data + * must be copied to new memory also + */ + uint8 *memory_data; + /* Memory data end address */ + uint8 *memory_data_end; /* Heap data base address */ uint8 *heap_data; @@ -45,14 +58,6 @@ struct WASMMemoryInstance { korp_mutex mem_lock; #endif - /* Memory data end address */ - uint8 *memory_data_end; - - /* Memory data begin address, the layout is: memory data + heap data - Note: when memory is re-allocated, the heap data and memory data - must be copied to new memory also. */ - uint8 *memory_data; - #if WASM_ENABLE_FAST_JIT != 0 #if UINTPTR_MAX == UINT64_MAX uint64 mem_bound_check_1byte;