Add the fast-interp tail call support (#409)

And also fix one bug in loader for tail-call

Signed-off-by: Xiaokang Qin <xiaokang.qxk@antgroup.com>

README.md

@@ -35,6 +35,7 @@ iwasm VM core
 - [Shared memory](https://github.com/WebAssembly/threads/blob/main/proposals/threads/Overview.md#shared-linear-memory)
 - [Multi-value](https://github.com/WebAssembly/multi-value)
 - [wasm-c-api](https://github.com/WebAssembly/wasm-c-api)
+- [Tail-call](https://github.com/WebAssembly/tail-call)
 
 ### Performance and memory usage
 The WAMR performance, footprint and memory usage data are available at the [performance](../../wiki/Performance) wiki page.

core/iwasm/interpreter/wasm_interp_fast.c

@@ -1301,10 +1301,16 @@ recover_br_info:
         goto return_func;
 
       HANDLE_OP (WASM_OP_CALL_INDIRECT):
+#if WASM_ENABLE_TAIL_CALL != 0
+      HANDLE_OP (WASM_OP_RETURN_CALL_INDIRECT):
+#endif
         {
           WASMType *cur_type, *cur_func_type;
           WASMTableInstance *cur_table_inst;
 
+#if WASM_ENABLE_TAIL_CALL != 0
+          GET_OPCODE();
+#endif
 #if WASM_ENABLE_THREAD_MGR != 0
           CHECK_SUSPEND_FLAGS();
 #endif
@@ -1360,6 +1366,10 @@ recover_br_info:
             wasm_set_exception(module, "indirect call type mismatch");
             goto got_exception;
           }
+#if WASM_ENABLE_TAIL_CALL != 0
+          if (opcode == WASM_OP_RETURN_CALL_INDIRECT)
+              goto call_func_from_return_call;
+#endif
           goto call_func_from_interp;
         }
 
@@ -3112,6 +3122,22 @@ recover_br_info:
         cur_func = module->functions + fidx;
         goto call_func_from_interp;
 
+#if WASM_ENABLE_TAIL_CALL != 0
+    HANDLE_OP (WASM_OP_RETURN_CALL):
+#if WASM_ENABLE_THREAD_MGR != 0
+        CHECK_SUSPEND_FLAGS();
+#endif
+        fidx = read_uint32(frame_ip);
+#if WASM_ENABLE_MULTI_MODULE != 0
+        if (fidx >= module->function_count) {
+          wasm_set_exception(module, "unknown function");
+          goto got_exception;
+        }
+#endif
+        cur_func = module->functions + fidx;
+        goto call_func_from_return_call;
+#endif /* WASM_ENABLE_TAIL_CALL */
+
 #if WASM_ENABLE_LABELS_AS_VALUES == 0
       default:
         wasm_set_exception(module, "unsupported opcode");
@@ -3125,8 +3151,10 @@ recover_br_info:
     HANDLE_OP (WASM_OP_UNUSED_0x08):
     HANDLE_OP (WASM_OP_UNUSED_0x09):
     HANDLE_OP (WASM_OP_UNUSED_0x0a):
+#if WASM_ENABLE_TAIL_CALL == 0
     HANDLE_OP (WASM_OP_RETURN_CALL):
     HANDLE_OP (WASM_OP_RETURN_CALL_INDIRECT):
+#endif
     HANDLE_OP (WASM_OP_UNUSED_0x14):
     HANDLE_OP (WASM_OP_UNUSED_0x15):
     HANDLE_OP (WASM_OP_UNUSED_0x16):
@@ -3169,6 +3197,40 @@ recover_br_info:
     FETCH_OPCODE_AND_DISPATCH ();
 #endif
 
+#if WASM_ENABLE_TAIL_CALL !=0
+  call_func_from_return_call:
+  {
+      uint32 *lp_base;
+      uint32 *lp;
+      int i;
+
+      if (!(lp_base = lp = wasm_runtime_malloc(cur_func->param_cell_num * sizeof(uint32)))) {
+          wasm_set_exception(module, "allocate memory failed");
+          goto got_exception;
+      }
+      for (i = 0; i < cur_func->param_count; i++) {
+          if (cur_func->param_types[i] == VALUE_TYPE_I64
+              || cur_func->param_types[i] == VALUE_TYPE_F64) {
+              *(int64*)(lp) =
+                GET_OPERAND(int64, (2 * (cur_func->param_count - i - 1)));
+              lp += 2;
+          }
+          else {
+              *(lp) = GET_OPERAND(int32, (2 * (cur_func->param_count - i - 1)));
+              lp ++;
+          }
+      }
+      frame->lp = frame->operand + cur_func->const_cell_num;
+      bh_memcpy_s(frame->lp, (lp - lp_base) * sizeof(uint32),
+                  lp_base, (lp - lp_base) * sizeof(uint32));
+      wasm_runtime_free(lp_base);
+      FREE_FRAME(exec_env, frame);
+      frame_ip += cur_func->param_count * sizeof(int16);
+      wasm_exec_env_set_cur_frame(exec_env,
+                                  (WASMRuntimeFrame *)prev_frame);
+      goto call_func_from_entry;
+  }
+#endif /* WASM_ENABLE_TAIL_CALL */
   call_func_from_interp:
     /* Only do the copy when it's called from interpreter.  */
     {

core/iwasm/interpreter/wasm_loader.c

@@ -5970,6 +5970,9 @@ handle_op_block_and_loop:
 
                 read_leb_uint32(p, p_end, type_idx);
 #if WASM_ENABLE_FAST_INTERP != 0
+#if WASM_ENABLE_TAIL_CALL != 0
+                emit_byte(loader_ctx, opcode);
+#endif
                 /* we need to emit func_idx before arguments */
                 emit_uint32(loader_ctx, type_idx);
 #endif
@@ -6023,12 +6026,13 @@ handle_op_block_and_loop:
                     }
                     for (i = 0; i < func_type->result_count; i++) {
                         type = func->func_type->types[func->func_type->param_count + i];
-                        if (func_type->types[func_type->param_count + i] != type)
+                        if (func_type->types[func_type->param_count + i] != type) {
-                            set_error_buf_v(error_buf, error_buf_size, "%s%s%s",
+                            set_error_buf_v(error_buf, error_buf_size,
-                                            "type mismatch: expect ",
+                                            "%s%s%s", "type mismatch: expect ",
                                             type_str[type - VALUE_TYPE_F64],
                                             " but got other");
-                        goto fail;
+                            goto fail;
+                        }
                     }
                     RESET_STACK();
                     SET_CUR_BLOCK_STACK_POLYMORPHIC_STATE(true);

doc/build_wamr.md

@@ -83,6 +83,9 @@ Currently we only profile the memory consumption of module, module_instance and
 - **WAMR_APP_THREAD_STACK_SIZE_MAX**=n, default to 8 MB (8388608) if not set
 > Note: the AOT boundary check with hardware trap mechanism might consume large stack since the OS may lazily grow the stack mapping as a guard page is hit, we may use this configuration to reduce the total stack usage, e.g. -DWAMR_APP_THREAD_STACK_SIZE_MAX=131072 (128 KB).
 
+#### **Enable tail call feature**
+- **WAMR_BUILD_TAIL_CALL**=1/0, default to disable if not set
+
 **Combination of configurations:**
 
 We can combine the configurations. For example, if we want to disable interpreter, enable AOT and WASI, we can run command: