From 91b9458ebd42711614f59c8ad54785271302337e Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Thu, 22 Oct 2020 18:52:33 +0800 Subject: [PATCH] Add more checks to enhance app heap's security (#428) --- core/iwasm/aot/aot_runtime.c | 48 ++-- core/iwasm/common/wasm_runtime_common.c | 2 - core/iwasm/interpreter/wasm_runtime.c | 46 ++-- core/shared/mem-alloc/ems/ems_alloc.c | 278 +++++++++++++++----- core/shared/mem-alloc/ems/ems_gc.h | 39 ++- core/shared/mem-alloc/ems/ems_gc_internal.h | 14 +- core/shared/mem-alloc/ems/ems_hmu.c | 28 +- core/shared/mem-alloc/ems/ems_kfc.c | 139 ++++++---- core/shared/mem-alloc/mem_alloc.c | 28 +- core/shared/mem-alloc/mem_alloc.h | 14 +- 10 files changed, 431 insertions(+), 205 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 668d03645..6507281c0 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -171,8 +171,10 @@ memories_deinstantiate(AOTModuleInstance *module_inst) continue; } #endif - if (memory_inst->heap_handle.ptr) + if (memory_inst->heap_handle.ptr) { mem_allocator_destroy(memory_inst->heap_handle.ptr); + wasm_runtime_free(memory_inst->heap_handle.ptr); + } if (memory_inst->heap_data.ptr) { #ifndef OS_ENABLE_HW_BOUND_CHECK @@ -359,13 +361,22 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, memory_inst->heap_data.ptr = p + heap_offset; memory_inst->heap_data_end.ptr = p + heap_offset + heap_size; if (heap_size > 0) { - if (!(heap_handle = mem_allocator_create(memory_inst->heap_data.ptr, - heap_size))) { - set_error_buf(error_buf, error_buf_size, - "init app heap failed"); + uint32 heap_struct_size = mem_allocator_get_heap_struct_size(); + + if (!(heap_handle = runtime_malloc((uint64)heap_struct_size, + error_buf, error_buf_size))) { goto fail1; } + memory_inst->heap_handle.ptr = heap_handle; + + if (!mem_allocator_create_with_struct_and_pool + (heap_handle, heap_struct_size, + memory_inst->heap_data.ptr, heap_size)) { + set_error_buf(error_buf, error_buf_size, + "init app heap failed"); + goto fail2; + } } if (total_size > 0) { @@ -390,7 +401,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, (WASMMemoryInstanceCommon *)memory_inst)) { set_error_buf(error_buf, error_buf_size, "allocate memory failed"); - goto fail2; + goto fail3; } } #endif @@ -398,12 +409,13 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModule *module, return memory_inst; #if WASM_ENABLE_SHARED_MEMORY != 0 -fail2: - if (heap_size > 0) { +fail3: + if (heap_size > 0) mem_allocator_destroy(memory_inst->heap_handle.ptr); - memory_inst->heap_handle.ptr = NULL; - } #endif +fail2: + if (heap_size > 0) + wasm_runtime_free(memory_inst->heap_handle.ptr); fail1: #ifndef OS_ENABLE_HW_BOUND_CHECK wasm_runtime_free(memory_inst->memory_data.ptr); @@ -1474,7 +1486,6 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) uint8 *memory_data_old = (uint8 *)memory_inst->memory_data.ptr; uint8 *heap_data_old = (uint8 *)memory_inst->heap_data.ptr; uint8 *memory_data, *heap_data; - void *heap_handle_old = memory_inst->heap_handle.ptr; if (inc_page_count <= 0) /* No need to enlarge memory */ @@ -1498,18 +1509,9 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) } #endif - if (heap_size > 0) { - /* Destroy heap's lock firstly, if its memory is re-allocated, - we cannot access its lock again. */ - mem_allocator_destroy_lock(memory_inst->heap_handle.ptr); - } if (!(memory_data = wasm_runtime_realloc(memory_data_old, (uint32)total_size))) { if (!(memory_data = wasm_runtime_malloc((uint32)total_size))) { - if (heap_size > 0) { - /* Restore heap's lock if memory re-alloc failed */ - mem_allocator_reinit_lock(memory_inst->heap_handle.ptr); - } return false; } bh_memcpy_s(memory_data, (uint32)total_size, @@ -1526,10 +1528,10 @@ aot_enlarge_memory(AOTModuleInstance *module_inst, uint32 inc_page_count) memory_inst->memory_data_end.ptr = memory_data + total_size; if (heap_size > 0) { - memory_inst->heap_handle.ptr = (uint8 *)heap_handle_old - + (memory_data - memory_data_old); if (mem_allocator_migrate(memory_inst->heap_handle.ptr, - heap_handle_old) != 0) { + (char*)heap_data_old + + (memory_data - memory_data_old), + heap_size)) { return false; } } diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 8d75eca45..5e1983fde 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -2138,8 +2138,6 @@ wasm_application_execute_main(WASMModuleInstanceCommon *module_inst, return ret; } - - #if WASM_ENABLE_MULTI_MODULE != 0 static WASMModuleInstance * get_sub_module_inst(const WASMModuleInstance *parent_module_inst, diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index bad93e1e5..d01e52813 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -107,6 +107,7 @@ memories_deinstantiate(WASMModuleInstance *module_inst, #endif if (memories[i]->heap_handle) { mem_allocator_destroy(memories[i]->heap_handle); + wasm_runtime_free(memories[i]->heap_handle); memories[i]->heap_handle = NULL; } wasm_runtime_free(memories[i]->memory_data); @@ -262,17 +263,25 @@ memory_instantiate(WASMModuleInstance *module_inst, memory->memory_data_end = memory->memory_data + (uint32)memory_data_size; /* Initialize heap */ - if (heap_size > 0 - && !(memory->heap_handle = - mem_allocator_create(memory->heap_data, heap_size))) { - set_error_buf(error_buf, error_buf_size, "init app heap failed"); - goto fail2; + if (heap_size > 0) { + uint32 heap_struct_size = mem_allocator_get_heap_struct_size(); + + if (!(memory->heap_handle = runtime_malloc((uint64)heap_struct_size, + error_buf, error_buf_size))) { + goto fail2; + } + if (!mem_allocator_create_with_struct_and_pool + (memory->heap_handle, heap_struct_size, + memory->heap_data, heap_size)) { + set_error_buf(error_buf, error_buf_size, "init app heap failed"); + goto fail3; + } } #if WASM_ENABLE_SHARED_MEMORY != 0 if (0 != os_mutex_init(&memory->mem_lock)) { set_error_buf(error_buf, error_buf_size, "init mutex failed"); - goto fail3; + goto fail4; } if (is_shared_memory) { memory->is_shared = true; @@ -281,18 +290,21 @@ memory_instantiate(WASMModuleInstance *module_inst, (WASMMemoryInstanceCommon *)memory)) { set_error_buf(error_buf, error_buf_size, "allocate memory failed"); - goto fail4; + goto fail5; } } #endif return memory; #if WASM_ENABLE_SHARED_MEMORY != 0 -fail4: +fail5: os_mutex_destroy(&memory->mem_lock); -fail3: +fail4: if (heap_size > 0) mem_allocator_destroy(memory->heap_handle); #endif +fail3: + if (heap_size > 0) + wasm_runtime_free(memory->heap_handle); fail2: wasm_runtime_free(memory->memory_data); fail1: @@ -1796,7 +1808,6 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) uint32 total_size_old = memory->memory_data_end - memory_data; uint32 total_page_count = inc_page_count + memory->cur_page_count; uint64 total_size = memory->num_bytes_per_page * (uint64)total_page_count; - void *heap_handle_old = memory->heap_handle; uint8 *heap_data_old = memory->heap_data; if (inc_page_count <= 0) @@ -1821,17 +1832,8 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) } #endif - if (heap_size > 0) { - /* Destroy heap's lock firstly, if its memory is re-allocated, - we cannot access its lock again. */ - mem_allocator_destroy_lock(memory->heap_handle); - } if (!(new_memory_data = wasm_runtime_realloc(memory_data, (uint32)total_size))) { if (!(new_memory_data = wasm_runtime_malloc((uint32)total_size))) { - if (heap_size > 0) { - /* Restore heap's lock if memory re-alloc failed */ - mem_allocator_reinit_lock(memory->heap_handle); - } return false; } bh_memcpy_s(new_memory_data, (uint32)total_size, @@ -1843,10 +1845,10 @@ wasm_enlarge_memory(WASMModuleInstance *module, uint32 inc_page_count) 0, (uint32)total_size - total_size_old); if (heap_size > 0) { - memory->heap_handle = (uint8 *)heap_handle_old + - (new_memory_data - memory_data); if (mem_allocator_migrate(memory->heap_handle, - heap_handle_old) != 0) { + (char *)heap_data_old + + (new_memory_data - memory_data), + heap_size) != 0) { return false; } } diff --git a/core/shared/mem-alloc/ems/ems_alloc.c b/core/shared/mem-alloc/ems/ems_alloc.c index e8d478d28..956bae616 100644 --- a/core/shared/mem-alloc/ems/ems_alloc.c +++ b/core/shared/mem-alloc/ems/ems_alloc.c @@ -6,12 +6,14 @@ #include "ems_gc_internal.h" -static int -hmu_is_in_heap(gc_heap_t* heap, hmu_t* hmu) +static inline bool +hmu_is_in_heap(void *hmu, + gc_uint8 *heap_base_addr, + gc_uint8 *heap_end_addr) { - return heap && hmu - && (gc_uint8*) hmu >= heap->base_addr - && (gc_uint8*) hmu < heap->base_addr + heap->current_size; + gc_uint8 *addr = (gc_uint8 *)hmu; + return (addr >= heap_base_addr && addr < heap_end_addr) + ? true : false; } /** @@ -23,21 +25,35 @@ hmu_is_in_heap(gc_heap_t* heap, hmu_t* hmu) * won't be touched. The tree will be re-organized so that the order * conditions are still satisified. */ -static void -remove_tree_node(hmu_tree_node_t *p) +static bool +remove_tree_node(gc_heap_t *heap, hmu_tree_node_t *p) { - hmu_tree_node_t *q = NULL, **slot = NULL; + hmu_tree_node_t *q = NULL, **slot = NULL, *parent; + hmu_tree_node_t *root = &heap->kfc_tree_root; + gc_uint8 *base_addr = heap->base_addr; + gc_uint8 *end_addr = base_addr + heap->current_size; bh_assert(p); - bh_assert(p->parent); /* @p can not be the ROOT node*/ + + parent = p->parent; + if (!parent || p == root /* p can not be the ROOT node */ + || !hmu_is_in_heap(p, base_addr, end_addr) + || (parent != root + && !hmu_is_in_heap(parent, base_addr, end_addr))) { + goto fail; + } /* get the slot which holds pointer to node p*/ if (p == p->parent->right) { slot = &p->parent->right; - } else { - bh_assert(p == p->parent->left); /* @p should be a child of its parent*/ + } + else if (p == p->parent->left) { + /* p should be a child of its parent*/ slot = &p->parent->left; } + else { + goto fail; + } /** * algorithms used to remove node p @@ -51,65 +67,110 @@ remove_tree_node(hmu_tree_node_t *p) if (!p->left) { /* move right child up*/ *slot = p->right; - if (p->right) + if (p->right) { + if (!hmu_is_in_heap(p->right, base_addr, end_addr)) { + goto fail; + } p->right->parent = p->parent; + } p->left = p->right = p->parent = NULL; - return; + return true; } if (!p->right) { /* move left child up*/ *slot = p->left; - p->left->parent = p->parent; /* p->left can never be NULL.*/ + if (!hmu_is_in_heap(p->left, base_addr, end_addr)) { + goto fail; + } + /* p->left can never be NULL unless it is corrupted. */ + p->left->parent = p->parent; p->left = p->right = p->parent = NULL; - return; + return true; } /* both left & right exist, find p's predecessor at first*/ q = p->left; - while (q->right) + if (!hmu_is_in_heap(q, base_addr, end_addr)) { + goto fail; + } + while (q->right) { q = q->right; + if (!hmu_is_in_heap(q, base_addr, end_addr)) { + goto fail; + } + } + /* remove from the tree*/ - remove_tree_node(q); + if (!remove_tree_node(heap, q)) + return false; *slot = q; q->parent = p->parent; q->left = p->left; q->right = p->right; - if (q->left) + if (q->left) { + if (!hmu_is_in_heap(q->left, base_addr, end_addr)) { + goto fail; + } q->left->parent = q; - if (q->right) + } + if (q->right) { + if (!hmu_is_in_heap(q->right, base_addr, end_addr)) { + goto fail; + } q->right->parent = q; + } p->left = p->right = p->parent = NULL; + + return true; +fail: + heap->is_heap_corrupted = true; + return false; } -static void +static bool unlink_hmu(gc_heap_t *heap, hmu_t *hmu) { + gc_uint8 *base_addr, *end_addr; gc_size_t size; bh_assert(gci_is_heap_valid(heap)); bh_assert(hmu && (gc_uint8*) hmu >= heap->base_addr && (gc_uint8*) hmu < heap->base_addr + heap->current_size); - bh_assert(hmu_get_ut(hmu) == HMU_FC); + if (hmu_get_ut(hmu) != HMU_FC) { + heap->is_heap_corrupted = true; + return false; + } + + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; size = hmu_get_size(hmu); if (HMU_IS_FC_NORMAL(size)) { uint32 node_idx = size >> 3; - hmu_normal_node_t *node_prev = &heap->kfc_normal_list[node_idx]; - hmu_normal_node_t *node = - get_hmu_normal_node_next(&heap->kfc_normal_list[node_idx]); + hmu_normal_node_t *node_prev = NULL, *node_next; + hmu_normal_node_t *node = heap->kfc_normal_list[node_idx].next; + while (node) { - if ((hmu_t*) node == hmu) { - set_hmu_normal_node_next(node_prev, get_hmu_normal_node_next(node)); + if (!hmu_is_in_heap(node, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return false; + } + node_next = get_hmu_normal_node_next(node); + if ((hmu_t*)node == hmu) { + if (!node_prev) /* list head */ + heap->kfc_normal_list[node_idx].next = node_next; + else + set_hmu_normal_node_next(node_prev, node_next); break; } node_prev = node; - node = get_hmu_normal_node_next(node); + node = node_next; } if (!node) { @@ -117,8 +178,10 @@ unlink_hmu(gc_heap_t *heap, hmu_t *hmu) } } else { - remove_tree_node((hmu_tree_node_t *) hmu); + if (!remove_tree_node(heap, (hmu_tree_node_t *)hmu)) + return false; } + return true; } static void @@ -140,37 +203,44 @@ hmu_set_free_size(hmu_t *hmu) * @param size should be positive and multiple of 8 * hmu with size @size will be added into KFC as a new FC. */ -void +bool gci_add_fc(gc_heap_t *heap, hmu_t *hmu, gc_size_t size) { + gc_uint8 *base_addr, *end_addr; hmu_normal_node_t *np = NULL; hmu_tree_node_t *root = NULL, *tp = NULL, *node = NULL; uint32 node_idx; bh_assert(gci_is_heap_valid(heap)); - bh_assert(hmu && (gc_uint8*) hmu >= heap->base_addr - && (gc_uint8*) hmu < heap->base_addr + heap->current_size); + bh_assert(hmu && (gc_uint8*)hmu >= heap->base_addr + && (gc_uint8*)hmu < heap->base_addr + heap->current_size); bh_assert(((gc_uint32)(uintptr_t)hmu_to_obj(hmu) & 7) == 0); bh_assert(size > 0 - && ((gc_uint8*) hmu) + size <= heap->base_addr + heap->current_size); + && ((gc_uint8*)hmu) + size <= heap->base_addr + heap->current_size); bh_assert(!(size & 7)); + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + hmu_set_ut(hmu, HMU_FC); hmu_set_size(hmu, size); hmu_set_free_size(hmu); if (HMU_IS_FC_NORMAL(size)) { - np = (hmu_normal_node_t*) hmu; + np = (hmu_normal_node_t*)hmu; + if (!hmu_is_in_heap(np, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return false; + } node_idx = size >> 3; - set_hmu_normal_node_next(np, get_hmu_normal_node_next - (&heap->kfc_normal_list[node_idx])); - set_hmu_normal_node_next(&heap->kfc_normal_list[node_idx], np); - return; + set_hmu_normal_node_next(np, heap->kfc_normal_list[node_idx].next); + heap->kfc_normal_list[node_idx].next = np; + return true; } /* big block*/ - node = (hmu_tree_node_t*) hmu; + node = (hmu_tree_node_t*)hmu; node->size = size; node->left = node->right = node->parent = NULL; @@ -195,7 +265,12 @@ gci_add_fc(gc_heap_t *heap, hmu_t *hmu, gc_size_t size) } tp = tp->left; } + if (!hmu_is_in_heap(tp, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return false; + } } + return true; } /** @@ -212,7 +287,9 @@ gci_add_fc(gc_heap_t *heap, hmu_t *hmu, gc_size_t size) static hmu_t * alloc_hmu(gc_heap_t *heap, gc_size_t size) { - hmu_normal_node_t *node = NULL, *p = NULL; + gc_uint8 *base_addr, *end_addr; + hmu_normal_list_t *normal_head = NULL; + hmu_normal_node_t *p = NULL; uint32 node_idx = 0, init_node_idx = 0; hmu_tree_node_t *root = NULL, *tp = NULL, *last_tp = NULL; hmu_t *next, *rest; @@ -220,6 +297,9 @@ alloc_hmu(gc_heap_t *heap, gc_size_t size) bh_assert(gci_is_heap_valid(heap)); bh_assert(size > 0 && !(size & 7)); + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + if (size < GC_SMALLEST_SIZE) size = GC_SMALLEST_SIZE; @@ -229,31 +309,40 @@ alloc_hmu(gc_heap_t *heap, gc_size_t size) init_node_idx = (size >> 3); for (node_idx = init_node_idx; node_idx < HMU_NORMAL_NODE_CNT; node_idx++) { - node = heap->kfc_normal_list + node_idx; - if (get_hmu_normal_node_next(node)) + normal_head = heap->kfc_normal_list + node_idx; + if (normal_head->next) break; - node = NULL; + normal_head = NULL; } - /* not found in normal list*/ - if (node) { + /* found in normal list*/ + if (normal_head) { bh_assert(node_idx >= init_node_idx); - p = get_hmu_normal_node_next(node); - set_hmu_normal_node_next(node, get_hmu_normal_node_next(p)); - bh_assert(((gc_int32)(uintptr_t)hmu_to_obj(p) & 7) == 0); + p = normal_head->next; + if (!hmu_is_in_heap(p, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return NULL; + } + normal_head->next = get_hmu_normal_node_next(p); + if (((gc_int32)(uintptr_t)hmu_to_obj(p) & 7) != 0) { + heap->is_heap_corrupted = true; + return NULL; + } if ((gc_size_t)node_idx != (uint32)init_node_idx /* with bigger size*/ && ((gc_size_t)node_idx << 3) >= size + GC_SMALLEST_SIZE) { rest = (hmu_t*) (((char *) p) + size); - gci_add_fc(heap, rest, (node_idx << 3) - size); + if (!gci_add_fc(heap, rest, (node_idx << 3) - size)) { + return NULL; + } hmu_mark_pinuse(rest); } else { size = node_idx << 3; next = (hmu_t*) ((char*) p + size); - if (hmu_is_in_heap(heap, next)) + if (hmu_is_in_heap(next, base_addr, end_addr)) hmu_mark_pinuse(next); } @@ -275,6 +364,11 @@ alloc_hmu(gc_heap_t *heap, gc_size_t size) bh_assert(root); tp = root->right; while (tp) { + if (!hmu_is_in_heap(tp, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return NULL; + } + if (tp->size < size) { tp = tp->right; continue; @@ -291,17 +385,19 @@ alloc_hmu(gc_heap_t *heap, gc_size_t size) /* alloc in last_p*/ /* remove node last_p from tree*/ - remove_tree_node(last_tp); + if (!remove_tree_node(heap, last_tp)) + return NULL; if (last_tp->size >= size + GC_SMALLEST_SIZE) { - rest = (hmu_t*) ((char*) last_tp + size); - gci_add_fc(heap, rest, last_tp->size - size); + rest = (hmu_t*)((char*)last_tp + size); + if (!gci_add_fc(heap, rest, last_tp->size - size)) + return NULL; hmu_mark_pinuse(rest); } else { size = last_tp->size; - next = (hmu_t*) ((char*) last_tp + size); - if (hmu_is_in_heap(heap, next)) + next = (hmu_t*)((char*)last_tp + size); + if (hmu_is_in_heap(next, base_addr, end_addr)) hmu_mark_pinuse(next); } @@ -309,8 +405,8 @@ alloc_hmu(gc_heap_t *heap, gc_size_t size) if ((heap->current_size - heap->total_free_size) > heap->highmark_size) heap->highmark_size = heap->current_size - heap->total_free_size; - hmu_set_size((hmu_t* ) last_tp, size); - return (hmu_t*) last_tp; + hmu_set_size((hmu_t*)last_tp, size); + return (hmu_t*)last_tp; } return NULL; @@ -365,6 +461,11 @@ gc_alloc_vo_internal(void *vheap, gc_size_t size, /* integer overflow */ return NULL; + if (heap->is_heap_corrupted) { + os_printf("[GC_ERROR]Heap is corrupted, allocate memory failed.\n"); + return NULL; + } + os_mutex_lock(&heap->lock); hmu = alloc_hmu_ex(heap, tot_size); @@ -404,6 +505,7 @@ gc_realloc_vo_internal(void *vheap, void *ptr, gc_size_t size, gc_object_t ret = (gc_object_t) NULL, obj_old = (gc_object_t)ptr; gc_size_t tot_size, tot_size_unaligned, tot_size_old = 0, tot_size_next; gc_size_t obj_size, obj_size_old; + gc_uint8 *base_addr, *end_addr; hmu_type_t ut; /* hmu header + prefix + obj + suffix */ @@ -414,6 +516,11 @@ gc_realloc_vo_internal(void *vheap, void *ptr, gc_size_t size, /* integer overflow */ return NULL; + if (heap->is_heap_corrupted) { + os_printf("[GC_ERROR]Heap is corrupted, allocate memory failed.\n"); + return NULL; + } + if (obj_old) { hmu_old = obj_to_hmu(obj_old); tot_size_old = hmu_get_size(hmu_old); @@ -422,17 +529,23 @@ gc_realloc_vo_internal(void *vheap, void *ptr, gc_size_t size, return obj_old; } + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + os_mutex_lock(&heap->lock); if (hmu_old) { hmu_next = (hmu_t*)((char *)hmu_old + tot_size_old); - if (hmu_is_in_heap(heap, hmu_next)) { + if (hmu_is_in_heap(hmu_next, base_addr, end_addr)) { ut = hmu_get_ut(hmu_next); tot_size_next = hmu_get_size(hmu_next); if (ut == HMU_FC && tot_size <= tot_size_old + tot_size_next) { /* current node and next node meets requirement */ - unlink_hmu(heap, hmu_next); + if (!unlink_hmu(heap, hmu_next)) { + os_mutex_unlock(&heap->lock); + return NULL; + } hmu_set_size(hmu_old, tot_size); memset((char*)hmu_old + tot_size_old, 0, tot_size - tot_size_old); #if BH_ENABLE_GC_VERIFY != 0 @@ -441,7 +554,10 @@ gc_realloc_vo_internal(void *vheap, void *ptr, gc_size_t size, if (tot_size < tot_size_old + tot_size_next) { hmu_next = (hmu_t*)((char*)hmu_old + tot_size); tot_size_next = tot_size_old + tot_size_next - tot_size; - gci_add_fc(heap, hmu_next, tot_size_next); + if (!gci_add_fc(heap, hmu_next, tot_size_next)) { + os_mutex_unlock(&heap->lock); + return NULL; + } } os_mutex_unlock(&heap->lock); return obj_old; @@ -507,6 +623,7 @@ gc_free_vo_internal(void *vheap, gc_object_t obj, #endif { gc_heap_t* heap = (gc_heap_t*) vheap; + gc_uint8 *base_addr, *end_addr; hmu_t *hmu = NULL; hmu_t *prev = NULL; hmu_t *next = NULL; @@ -518,14 +635,21 @@ gc_free_vo_internal(void *vheap, gc_object_t obj, return GC_SUCCESS; } + if (heap->is_heap_corrupted) { + os_printf("[GC_ERROR]Heap is corrupted, free memory failed.\n"); + return GC_ERROR; + } + hmu = obj_to_hmu(obj); + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + os_mutex_lock(&heap->lock); - if ((gc_uint8 *)hmu >= heap->base_addr - && (gc_uint8 *)hmu < heap->base_addr + heap->current_size) { + if (hmu_is_in_heap(hmu, base_addr, end_addr)) { #if BH_ENABLE_GC_VERIFY != 0 - hmu_verify(hmu); + hmu_verify(heap, hmu); #endif ut = hmu_get_ut(hmu); if (ut == HMU_VO) { @@ -544,25 +668,35 @@ gc_free_vo_internal(void *vheap, gc_object_t obj, if (!hmu_get_pinuse(hmu)) { prev = (hmu_t*) ((char*) hmu - *((int*) hmu - 1)); - if (hmu_is_in_heap(heap, prev) && hmu_get_ut(prev) == HMU_FC) { + if (hmu_is_in_heap(prev, base_addr, end_addr) + && hmu_get_ut(prev) == HMU_FC) { size += hmu_get_size(prev); hmu = prev; - unlink_hmu(heap, prev); + if (!unlink_hmu(heap, prev)) { + ret = GC_ERROR; + goto out; + } } } next = (hmu_t*) ((char*) hmu + size); - if (hmu_is_in_heap(heap, next)) { + if (hmu_is_in_heap(next, base_addr, end_addr)) { if (hmu_get_ut(next) == HMU_FC) { size += hmu_get_size(next); - unlink_hmu(heap, next); - next = (hmu_t*) ((char*) hmu + size); + if (!unlink_hmu(heap, next)) { + ret = GC_ERROR; + goto out; + } + next = (hmu_t*)((char*) hmu + size); } } - gci_add_fc(heap, hmu, size); + if (!gci_add_fc(heap, hmu, size)) { + ret = GC_ERROR; + goto out; + } - if (hmu_is_in_heap(heap, next)) { + if (hmu_is_in_heap(next, base_addr, end_addr)) { hmu_unmark_pinuse(next); } @@ -620,7 +754,11 @@ gci_dump(gc_heap_t *heap) else if (ut == HMU_FC) inuse = 'F'; - bh_assert(size > 0); + if (size == 0) { + os_printf("[GC_ERROR]Heap is corrupted, heap dump failed.\n"); + heap->is_heap_corrupted = true; + return; + } os_printf("#%d %08x %x %x %d %c %d\n", i, (int32)((char*) cur - (char*) heap->base_addr), diff --git a/core/shared/mem-alloc/ems/ems_gc.h b/core/shared/mem-alloc/ems/ems_gc.h index 206c8bd0f..c712c9087 100644 --- a/core/shared/mem-alloc/ems/ems_gc.h +++ b/core/shared/mem-alloc/ems/ems_gc.h @@ -51,7 +51,10 @@ typedef enum { } GC_STAT_INDEX; /** - * GC initialization from a buffer + * GC initialization from a buffer, which is separated into + * two parts: the beginning of the buffer is used to create + * the heap structure, and the left is used to create the + * actual pool data * * @param buf the buffer to be initialized to a heap * @param buf_size the size of buffer @@ -61,6 +64,20 @@ typedef enum { gc_handle_t gc_init_with_pool(char *buf, gc_size_t buf_size); +/** + * GC initialization from heap struct buffer and pool buffer + * + * @param struct_buf the struct buffer to create the heap structure + * @param struct_buf_size the size of struct buffer + * @param pool_buf the pool buffer to create pool data + * @param pool_buf_size the size of poll buffer + * + * @return gc handle if success, NULL otherwise + */ +gc_handle_t +gc_init_with_struct_and_pool(char *struct_buf, gc_size_t struct_buf_size, + char *pool_buf, gc_size_t pool_buf_size); + /** * Destroy heap which is initilized from a buffer * @@ -73,25 +90,23 @@ int gc_destroy_with_pool(gc_handle_t handle); /** - * Migrate heap from one place to another place - * - * @param handle handle of the new heap - * @param handle_old handle of the old heap - * - * @return GC_SUCCESS if success, GC_ERROR otherwise + * Return heap struct size */ -int -gc_migrate(gc_handle_t handle, gc_handle_t handle_old); +uint32 +gc_get_heap_struct_size(void); /** - * Re-initialize lock of heap + * Migrate heap from one pool buf to another pool buf * - * @param handle the heap handle + * @param handle handle of the new heap + * @param pool_buf_new the new pool buffer + * @param pool_buf_size the size of new pool buffer * * @return GC_SUCCESS if success, GC_ERROR otherwise */ int -gc_reinit_lock(gc_handle_t handle); +gc_migrate(gc_handle_t handle, + char *pool_buf_new, gc_size_t pool_buf_size); /** * Destroy lock of heap diff --git a/core/shared/mem-alloc/ems/ems_gc_internal.h b/core/shared/mem-alloc/ems/ems_gc_internal.h index 680d70e74..aaf49068d 100644 --- a/core/shared/mem-alloc/ems/ems_gc_internal.h +++ b/core/shared/mem-alloc/ems/ems_gc_internal.h @@ -52,7 +52,7 @@ hmu_init_prefix_and_suffix(hmu_t *hmu, gc_size_t tot_size, const char *file_name, int line_no); void -hmu_verify(hmu_t *hmu); +hmu_verify(void *vheap, hmu_t *hmu); #define SKIP_OBJ_PREFIX(p) ((void*)((gc_uint8*)(p) + OBJ_PREFIX_SIZE)) #define SKIP_OBJ_SUFFIX(p) ((void*)((gc_uint8*)(p) + OBJ_SUFFIX_SIZE)) @@ -159,6 +159,10 @@ typedef struct hmu_normal_node { gc_int32 next_offset; } hmu_normal_node_t; +typedef struct hmu_normal_list { + hmu_normal_node_t *next; +} hmu_normal_list_t; + static inline hmu_normal_node_t * get_hmu_normal_node_next(hmu_normal_node_t *node) { @@ -197,11 +201,15 @@ typedef struct gc_heap_struct { korp_mutex lock; - hmu_normal_node_t kfc_normal_list[HMU_NORMAL_NODE_CNT]; + hmu_normal_list_t kfc_normal_list[HMU_NORMAL_NODE_CNT]; /* order in kfc_tree is: size[left] <= size[cur] < size[right]*/ hmu_tree_node_t kfc_tree_root; + /* whether heap is corrupted, e.g. the hmu nodes are modified + by user */ + bool is_heap_corrupted; + gc_size_t init_size; gc_size_t highmark_size; gc_size_t total_free_size; @@ -211,7 +219,7 @@ typedef struct gc_heap_struct { * MISC internal used APIs */ -void +bool gci_add_fc(gc_heap_t *heap, hmu_t *hmu, gc_size_t size); int diff --git a/core/shared/mem-alloc/ems/ems_hmu.c b/core/shared/mem-alloc/ems/ems_hmu.c index cd55dc176..d7cf34189 100644 --- a/core/shared/mem-alloc/ems/ems_hmu.c +++ b/core/shared/mem-alloc/ems/ems_hmu.c @@ -45,8 +45,9 @@ hmu_init_prefix_and_suffix(hmu_t *hmu, gc_size_t tot_size, } void -hmu_verify(hmu_t *hmu) +hmu_verify(void *vheap, hmu_t *hmu) { + gc_heap_t *heap = (gc_heap_t *)vheap; gc_object_prefix_t *prefix = NULL; gc_object_suffix_t *suffix = NULL; gc_uint32 i = 0; @@ -62,32 +63,27 @@ hmu_verify(hmu_t *hmu) size = prefix->size; suffix = (gc_object_suffix_t *)((gc_uint8*)hmu + size - OBJ_SUFFIX_SIZE); - if(ut == HMU_VO || ut == HMU_JO) - { + if (ut == HMU_VO || ut == HMU_JO) { /* check padding*/ - for(i = 0;i < GC_OBJECT_PREFIX_PADDING_CNT;i++) - { - if(prefix->padding[i] != GC_OBJECT_PADDING_VALUE) - { + for (i = 0;i < GC_OBJECT_PREFIX_PADDING_CNT;i++) { + if (prefix->padding[i] != GC_OBJECT_PADDING_VALUE) { is_padding_ok = 0; break; } } - for(i = 0;i < GC_OBJECT_SUFFIX_PADDING_CNT;i++) - { - if(suffix->padding[i] != GC_OBJECT_PADDING_VALUE) - { + for (i = 0;i < GC_OBJECT_SUFFIX_PADDING_CNT;i++) { + if (suffix->padding[i] != GC_OBJECT_PADDING_VALUE) { is_padding_ok = 0; break; } } - if(!is_padding_ok) - { - os_printf("Invalid padding for object created at %s:%d", - (prefix->file_name ? prefix->file_name : ""), prefix->line_no); + if (!is_padding_ok) { + os_printf("Invalid padding for object created at %s:%d\n", + (prefix->file_name ? prefix->file_name : ""), + prefix->line_no); + heap->is_heap_corrupted = true; } - bh_assert(is_padding_ok); } } diff --git a/core/shared/mem-alloc/ems/ems_kfc.c b/core/shared/mem-alloc/ems/ems_kfc.c index fcf487bb9..ddfc5e4fe 100644 --- a/core/shared/mem-alloc/ems/ems_kfc.c +++ b/core/shared/mem-alloc/ems/ems_kfc.c @@ -5,26 +5,11 @@ #include "ems_gc_internal.h" -gc_handle_t -gc_init_with_pool(char *buf, gc_size_t buf_size) +static gc_handle_t +gc_init_internal(gc_heap_t *heap, char *base_addr, gc_size_t heap_max_size) { - char *buf_end = buf + buf_size; - char *buf_aligned = (char*)(((uintptr_t) buf + 7) & (uintptr_t)~7); - char *base_addr = buf_aligned + sizeof(gc_heap_t); - gc_heap_t *heap = (gc_heap_t*)buf_aligned; - gc_size_t heap_max_size; - hmu_normal_node_t *p = NULL; hmu_tree_node_t *root = NULL, *q = NULL; - int i = 0, ret; - - if (buf_size < APP_HEAP_SIZE_MIN) { - os_printf("[GC_ERROR]heap init buf size (%u) < %u\n", - buf_size, APP_HEAP_SIZE_MIN); - return NULL; - } - - base_addr = (char*) (((uintptr_t) base_addr + 7) & (uintptr_t)~7) + GC_HEAD_PADDING; - heap_max_size = (uint32)(buf_end - base_addr) & (uint32)~7; + int ret; memset(heap, 0, sizeof *heap); memset(base_addr, 0, heap_max_size); @@ -43,14 +28,6 @@ gc_init_with_pool(char *buf, gc_size_t buf_size) heap->total_free_size = heap->current_size; heap->highmark_size = 0; - for (i = 0; i < HMU_NORMAL_NODE_CNT; i++) { - /* make normal node look like a FC*/ - p = &heap->kfc_normal_list[i]; - memset(p, 0, sizeof *p); - hmu_set_ut(&p->hmu_header, HMU_FC); - hmu_set_size(&p->hmu_header, sizeof *p); - } - root = &heap->kfc_tree_root; memset(root, 0, sizeof *root); root->size = sizeof *root; @@ -79,6 +56,63 @@ gc_init_with_pool(char *buf, gc_size_t buf_size) return heap; } +gc_handle_t +gc_init_with_pool(char *buf, gc_size_t buf_size) +{ + char *buf_end = buf + buf_size; + char *buf_aligned = (char*)(((uintptr_t) buf + 7) & (uintptr_t)~7); + char *base_addr = buf_aligned + sizeof(gc_heap_t); + gc_heap_t *heap = (gc_heap_t*)buf_aligned; + gc_size_t heap_max_size; + + if (buf_size < APP_HEAP_SIZE_MIN) { + os_printf("[GC_ERROR]heap init buf size (%u) < %u\n", + buf_size, APP_HEAP_SIZE_MIN); + return NULL; + } + + base_addr = (char*) (((uintptr_t) base_addr + 7) & (uintptr_t)~7) + GC_HEAD_PADDING; + heap_max_size = (uint32)(buf_end - base_addr) & (uint32)~7; + + return gc_init_internal(heap, base_addr, heap_max_size); +} + +gc_handle_t +gc_init_with_struct_and_pool(char *struct_buf, gc_size_t struct_buf_size, + char *pool_buf, gc_size_t pool_buf_size) +{ + gc_heap_t *heap = (gc_heap_t*)struct_buf; + char *base_addr = pool_buf + GC_HEAD_PADDING; + char *pool_buf_end = pool_buf + pool_buf_size; + gc_size_t heap_max_size; + + if ((((uintptr_t)struct_buf) & 7) != 0) { + os_printf("[GC_ERROR]heap init struct buf not 8-byte aligned\n"); + return NULL; + } + + if (struct_buf_size < sizeof(gc_handle_t)) { + os_printf("[GC_ERROR]heap init struct buf size (%u) < %u\n", + struct_buf_size, sizeof(gc_handle_t)); + return NULL; + } + + if ((((uintptr_t)pool_buf) & 7) != 0) { + os_printf("[GC_ERROR]heap init pool buf not 8-byte aligned\n"); + return NULL; + } + + if (pool_buf_size < APP_HEAP_SIZE_MIN) { + os_printf("[GC_ERROR]heap init buf size (%u) < %u\n", + pool_buf_size, APP_HEAP_SIZE_MIN); + return NULL; + } + + heap_max_size = (uint32)(pool_buf_end - base_addr) & (uint32)~7; + + return gc_init_internal(heap, base_addr, heap_max_size); +} + int gc_destroy_with_pool(gc_handle_t handle) { @@ -86,7 +120,8 @@ gc_destroy_with_pool(gc_handle_t handle) #if BH_ENABLE_GC_VERIFY != 0 hmu_t *cur = (hmu_t*)heap->base_addr; hmu_t *end = (hmu_t*)((char*)heap->base_addr + heap->current_size); - if ((hmu_t*)((char *)cur + hmu_get_size(cur)) != end) { + if (!heap->is_heap_corrupted + && (hmu_t*)((char *)cur + hmu_get_size(cur)) != end) { os_printf("Memory leak detected:\n"); gci_dump(heap); #if WASM_ENABLE_SPEC_TEST != 0 @@ -100,6 +135,12 @@ gc_destroy_with_pool(gc_handle_t handle) return GC_SUCCESS; } +uint32 +gc_get_heap_struct_size() +{ + return sizeof(gc_heap_t); +} + static void adjust_ptr(uint8 **p_ptr, intptr_t offset) { @@ -108,21 +149,34 @@ adjust_ptr(uint8 **p_ptr, intptr_t offset) } int -gc_migrate(gc_handle_t handle, gc_handle_t handle_old) +gc_migrate(gc_handle_t handle, + char *pool_buf_new, gc_size_t pool_buf_size) { - gc_heap_t *heap = (gc_heap_t *) handle; - intptr_t offset = (uint8*)handle - (uint8*)handle_old; + gc_heap_t *heap = (gc_heap_t *)handle; + char *base_addr_new = pool_buf_new + GC_HEAD_PADDING; + char *pool_buf_end = pool_buf_new + pool_buf_size; + intptr_t offset = (uint8*)base_addr_new - (uint8*)heap->base_addr; hmu_t *cur = NULL, *end = NULL; hmu_tree_node_t *tree_node; - gc_size_t size; + gc_size_t heap_max_size, size; - os_mutex_init(&heap->lock); + if ((((uintptr_t)pool_buf_new) & 7) != 0) { + os_printf("[GC_ERROR]heap migrate pool buf not 8-byte aligned\n"); + return GC_ERROR; + } + + heap_max_size = (uint32)(pool_buf_end - base_addr_new) & (uint32)~7; + + if (pool_buf_end < base_addr_new + || heap_max_size < heap->current_size) { + os_printf("[GC_ERROR]heap migrate invlaid pool buf size\n"); + return GC_ERROR; + } if (offset == 0) return 0; - heap->heap_id = (gc_handle_t)heap; - heap->base_addr += offset; + heap->base_addr = (uint8*)base_addr_new; adjust_ptr((uint8**)&heap->kfc_tree_root.left, offset); adjust_ptr((uint8**)&heap->kfc_tree_root.right, offset); adjust_ptr((uint8**)&heap->kfc_tree_root.parent, offset); @@ -138,7 +192,10 @@ gc_migrate(gc_handle_t handle, gc_handle_t handle_old) tree_node = (hmu_tree_node_t *)cur; adjust_ptr((uint8**)&tree_node->left, offset); adjust_ptr((uint8**)&tree_node->right, offset); - adjust_ptr((uint8**)&tree_node->parent, offset); + if (tree_node->parent != &heap->kfc_tree_root) + /* The root node belongs to heap structure, + it is fixed part and isn't changed. */ + adjust_ptr((uint8**)&tree_node->parent, offset); } cur = (hmu_t*)((char *)cur + size); } @@ -147,13 +204,6 @@ gc_migrate(gc_handle_t handle, gc_handle_t handle_old) return 0; } -int -gc_reinit_lock(gc_handle_t handle) -{ - gc_heap_t *heap = (gc_heap_t *) handle; - return os_mutex_init(&heap->lock); -} - void gc_destroy_lock(gc_handle_t handle) { @@ -170,9 +220,8 @@ gci_verify_heap(gc_heap_t *heap) bh_assert(heap && gci_is_heap_valid(heap)); cur = (hmu_t *)heap->base_addr; end = (hmu_t *)(heap->base_addr + heap->current_size); - while(cur < end) - { - hmu_verify(cur); + while(cur < end) { + hmu_verify(heap, cur); cur = (hmu_t *)((gc_uint8*)cur + hmu_get_size(cur)); } bh_assert(cur == end); diff --git a/core/shared/mem-alloc/mem_alloc.c b/core/shared/mem-alloc/mem_alloc.c index 32d07513f..f2304f8e4 100644 --- a/core/shared/mem-alloc/mem_alloc.c +++ b/core/shared/mem-alloc/mem_alloc.c @@ -14,11 +14,29 @@ mem_allocator_t mem_allocator_create(void *mem, uint32_t size) return gc_init_with_pool((char *) mem, size); } +mem_allocator_t +mem_allocator_create_with_struct_and_pool(void *struct_buf, + uint32_t struct_buf_size, + void *pool_buf, + uint32_t pool_buf_size) +{ + return gc_init_with_struct_and_pool((char *)struct_buf, + struct_buf_size, + pool_buf, + pool_buf_size); +} + void mem_allocator_destroy(mem_allocator_t allocator) { gc_destroy_with_pool((gc_handle_t) allocator); } +uint32 +mem_allocator_get_heap_struct_size() +{ + return gc_get_heap_struct_size(); +} + void * mem_allocator_malloc(mem_allocator_t allocator, uint32_t size) { @@ -39,16 +57,10 @@ void mem_allocator_free(mem_allocator_t allocator, void *ptr) int mem_allocator_migrate(mem_allocator_t allocator, - mem_allocator_t allocator_old) + char *pool_buf_new, uint32 pool_buf_size) { return gc_migrate((gc_handle_t) allocator, - (gc_handle_t) allocator_old); -} - -int -mem_allocator_reinit_lock(mem_allocator_t allocator) -{ - return gc_reinit_lock((gc_handle_t) allocator); + pool_buf_new, pool_buf_size); } void diff --git a/core/shared/mem-alloc/mem_alloc.h b/core/shared/mem-alloc/mem_alloc.h index eb1032e5c..21e97f5bc 100644 --- a/core/shared/mem-alloc/mem_alloc.h +++ b/core/shared/mem-alloc/mem_alloc.h @@ -17,9 +17,18 @@ typedef void *mem_allocator_t; mem_allocator_t mem_allocator_create(void *mem, uint32_t size); +mem_allocator_t +mem_allocator_create_with_struct_and_pool(void *struct_buf, + uint32_t struct_buf_size, + void *pool_buf, + uint32_t pool_buf_size); + void mem_allocator_destroy(mem_allocator_t allocator); +uint32 +mem_allocator_get_heap_struct_size(void); + void * mem_allocator_malloc(mem_allocator_t allocator, uint32_t size); @@ -31,10 +40,7 @@ mem_allocator_free(mem_allocator_t allocator, void *ptr); int mem_allocator_migrate(mem_allocator_t allocator, - mem_allocator_t allocator_old); - -int -mem_allocator_reinit_lock(mem_allocator_t allocator); + char *pool_buf_new, uint32 pool_buf_size); void mem_allocator_destroy_lock(mem_allocator_t allocator);