wamr-python: Enable debugging WASM and grant dir access (#2449)

- Enable debugging a WASM loaded and executed from Python.
- Expose API to enable access to list of host directories. Similar to --dir in iwasm.
- Add another python language binding sample: native-symbol.

language-bindings/python/src/wamr/wamrapi/wamr.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from ctypes import Array
+from ctypes import addressof
 from ctypes import c_char
 from ctypes import c_uint
 from ctypes import c_uint8
@@ -10,6 +11,8 @@ from ctypes import cast
 from ctypes import create_string_buffer
 from ctypes import POINTER
 from ctypes import pointer
+from typing import List
+from typing import Tuple
 from wamr.wamrapi.iwasm import String
 from wamr.wamrapi.iwasm import Alloc_With_Pool
 from wamr.wamrapi.iwasm import RuntimeInitArgs
@@ -31,6 +34,14 @@ from wamr.wamrapi.iwasm import wasm_runtime_module_malloc
 from wamr.wamrapi.iwasm import wasm_runtime_module_free
 from wamr.wamrapi.iwasm import wasm_runtime_register_natives
 from wamr.wamrapi.iwasm import NativeSymbol
+from wamr.wamrapi.iwasm import wasm_runtime_start_debug_instance
+from wamr.wamrapi.iwasm import wasm_runtime_call_indirect
+from wamr.wamrapi.iwasm import wasm_runtime_get_module_inst
+from wamr.wamrapi.iwasm import wasm_runtime_addr_app_to_native
+from wamr.wamrapi.iwasm import wasm_runtime_addr_native_to_app
+from wamr.wamrapi.iwasm import wasm_runtime_set_wasi_args
+
+ID_TO_EXEC_ENV_MAPPING = {}
 
 
 class Engine:
@@ -43,16 +54,26 @@ class Engine:
         print("deleting Engine")
         wasm_runtime_destroy()
 
-    def _get_init_args(self, heap_size: int = 1024 * 512) -> RuntimeInitArgs:
+    def _get_init_args(
+        self,
+        heap_size: int = 1024 * 1024 * 2,
+        ip_addr: str = "127.0.0.1",
+        instance_port: int = 1234,
+    ) -> RuntimeInitArgs:
         init_args = RuntimeInitArgs()
         init_args.mem_alloc_type = Alloc_With_Pool
         init_args.mem_alloc_option.pool.heap_buf = cast(
             (c_char * heap_size)(), c_void_p
         )
         init_args.mem_alloc_option.pool.heap_size = heap_size
+        # Debug port setting
+        init_args.ip_addr = bytes(ip_addr, "utf-8")
+        init_args.instance_port = instance_port
         return init_args
 
-    def register_natives(self, module_name: str, native_symbols: list[NativeSymbol]) -> None:
+    def register_natives(
+        self, module_name: str, native_symbols: List[NativeSymbol]
+    ) -> None:
         module_name = String.from_param(module_name)
         # WAMR does not copy the symbols. We must store them.
         for native in native_symbols:
@@ -62,12 +83,13 @@ class Engine:
             module_name,
             cast(
                 (NativeSymbol * len(native_symbols))(*native_symbols),
-                POINTER(NativeSymbol)
+                POINTER(NativeSymbol),
             ),
-            len(native_symbols)
+            len(native_symbols),
         ):
             raise Exception("Error while registering symbols")
 
+
 class Module:
     __create_key = object()
 
@@ -86,7 +108,7 @@ class Module:
         print("deleting Module")
         wasm_runtime_unload(self.module)
 
-    def _create_module(self, fp: str) -> tuple[wasm_module_t, Array[c_uint]]:
+    def _create_module(self, fp: str) -> Tuple[wasm_module_t, "Array[c_uint]"]:
         with open(fp, "rb") as f:
             data = f.read()
             data = (c_uint8 * len(data))(*data)
@@ -99,14 +121,52 @@ class Module:
 
 
 class Instance:
-    def __init__(self, module: Module, stack_size: int = 65536, heap_size: int = 16384):
+    def __init__(
+        self,
+        module: Module,
+        stack_size: int = 65536,
+        heap_size: int = 16384,
+        dir_list: List[str] | None = None,
+        preinitialized_module_inst: wasm_module_inst_t | None = None,
+    ):
+        # Store module ensures GC does not remove it
         self.module = module
-        self.module_inst = self._create_module_inst(module, stack_size, heap_size)
+        if preinitialized_module_inst is None:
+            self.module_inst = self._create_module_inst(module, stack_size, heap_size)
+        else:
+            self.module_inst = preinitialized_module_inst
+        if dir_list:
+            self._set_wasi_args(module, dir_list)
 
     def __del__(self):
         print("deleting Instance")
         wasm_runtime_deinstantiate(self.module_inst)
 
+    def _set_wasi_args(self, module: Module, dir_list: List[str]) -> None:
+        LP_c_char = POINTER(c_char)
+        LP_LP_c_char = POINTER(LP_c_char)
+
+        p = (LP_c_char * len(dir_list))()
+        for i, dir in enumerate(dir_list):
+            enc_dir = dir.encode("utf-8")
+            p[i] = create_string_buffer(enc_dir)
+
+        na = cast(p, LP_LP_c_char)
+        wasm_runtime_set_wasi_args(
+            module.module, na, len(dir_list), None, 0, None, 0, None, 0
+        )
+
+    def _create_module_inst(
+        self, module: Module, stack_size: int, heap_size: int
+    ) -> wasm_module_inst_t:
+        error_buf = create_string_buffer(128)
+        module_inst = wasm_runtime_instantiate(
+            module.module, stack_size, heap_size, error_buf, len(error_buf)
+        )
+        if not module_inst:
+            raise Exception("Error while creating module instance")
+        return module_inst
+
     def malloc(self, nbytes: int, native_handler) -> c_uint:
         return wasm_runtime_module_malloc(self.module_inst, nbytes, native_handler)
 
@@ -119,31 +179,70 @@ class Instance:
             raise Exception("Error while looking-up function")
         return func
 
-    def _create_module_inst(self, module: Module, stack_size: int, heap_size: int) -> wasm_module_inst_t:
+    def native_addr_to_app_addr(self, native_addr) -> c_void_p:
-        error_buf = create_string_buffer(128)
+        return wasm_runtime_addr_native_to_app(self.module_inst, native_addr)
-        module_inst = wasm_runtime_instantiate(
+
-            module.module, stack_size, heap_size, error_buf, len(error_buf)
+    def app_addr_to_native_addr(self, app_addr) -> c_void_p:
-        )
+        return wasm_runtime_addr_app_to_native(self.module_inst, app_addr)
-        if not module_inst:
-            raise Exception("Error while creating module instance")
-        return module_inst
 
 
 class ExecEnv:
     def __init__(self, module_inst: Instance, stack_size: int = 65536):
         self.module_inst = module_inst
         self.exec_env = self._create_exec_env(module_inst, stack_size)
+        self.env = addressof(self.exec_env.contents)
+        self.own_c = True
+
+        ID_TO_EXEC_ENV_MAPPING[str(self.env)] = self
 
     def __del__(self):
-        print("deleting ExecEnv")
+        if self.own_c:
-        wasm_runtime_destroy_exec_env(self.exec_env)
+            print("deleting ExecEnv")
+            wasm_runtime_destroy_exec_env(self.exec_env)
+            del ID_TO_EXEC_ENV_MAPPING[str(self.env)]
+
+    def _create_exec_env(
+        self, module_inst: Instance, stack_size: int
+    ) -> wasm_exec_env_t:
+        exec_env = wasm_runtime_create_exec_env(module_inst.module_inst, stack_size)
+        if not exec_env:
+            raise Exception("Error while creating execution environment")
+        return exec_env
 
     def call(self, func: wasm_function_inst_t, argc: int, argv: "POINTER[c_uint]"):
         if not wasm_runtime_call_wasm(self.exec_env, func, argc, argv):
             raise Exception("Error while calling function")
 
-    def _create_exec_env(self, module_inst: Instance, stack_size: int) -> wasm_exec_env_t:
+    def get_module_inst(self) -> Instance:
-        exec_env = wasm_runtime_create_exec_env(module_inst.module_inst, stack_size)
+        return self.module_inst
-        if not exec_env:
+
-            raise Exception("Error while creating execution environment")
+    def start_debugging(self) -> int:
-        return exec_env
+        return wasm_runtime_start_debug_instance(self.exec_env)
+
+    def call_indirect(self, element_index: int, argc: int, argv: "POINTER[c_uint]"):
+        if not wasm_runtime_call_indirect(self.exec_env, element_index, argc, argv):
+            raise Exception("Error while calling function")
+
+    @staticmethod
+    def wrap(env: int) -> "ExecEnv":
+        if str(env) in ID_TO_EXEC_ENV_MAPPING:
+            return ID_TO_EXEC_ENV_MAPPING[str(env)]
+        return InternalExecEnv(env)
+
+
+class InternalExecEnv(ExecEnv):
+    """
+    Generate Python ExecEnv-like object from a `wasm_exec_env_t` index.
+    """
+
+    def __init__(self, env: int):
+        self.env = env
+        self.exec_env = cast(env, wasm_exec_env_t)
+        self.module_inst = Instance(
+            module=object(),
+            preinitialized_module_inst=wasm_runtime_get_module_inst(self.exec_env),
+        )
+        ID_TO_EXEC_ENV_MAPPING[str(env)] = self
+
+    def __del__(self):
+        del ID_TO_EXEC_ENV_MAPPING[str(self.env)]

language-bindings/python/utils/create_lib.sh

@@ -12,7 +12,12 @@ WAMR_BUILD_PLATFORM=${WAMR_BUILD_PLATFORM:-${UNAME}}
 cd ${ROOT_DIR}/product-mini/platforms/${WAMR_BUILD_PLATFORM}
 
 mkdir -p build && cd build
-cmake ..
+cmake \
+    -DWAMR_BUILD_DEBUG_INTERP=1 \
+    -DWAMR_BUILD_LIB_PTHREAD=1 \
+    -DWAMR_BUILD_LIB_WASI_THREADS=1 \
+    -DWAMR_BUILD_LIB_WASI=1 \
+    ..
 make -j
 
 case ${UNAME} in

language-bindings/python/wamr-api/README.md

@@ -22,10 +22,7 @@ bash language-bindings/python/utils/create_lib.sh
 
 This will build and copy libiwasm into the package.
 
-## Examples
+## Samples
 
-There is a [simple example](./samples/main.py) to show how to use bindings.
+- **[basic](./samples/basic)**: Demonstrating how to use basic python bindings.
-
+- **[native-symbol](./samples/native-symbol)**: Desmostrate how to call WASM from Python and how to export Python functions into WASM.
-```
-python samples/main.py
-```

language-bindings/python/wamr-api/samples/native-symbol/README.md

@@ -0,0 +1,44 @@
+# Native Symbol
+
+This sample demonstrates how to declare a Python function as `NativeSymbol`.
+
+Steps of the example:
+1. Load WASM from Python
+2. Call `c_func` from WASM.
+3. `c_func` calls `python_func` from Python.
+4. `python_func` calls `add` from WASM.
+5. Result shown by Python.
+
+## Build
+
+Follow instructions [build wamr Python package](../../README.md). 
+
+Compile WASM app example,
+
+```sh
+./compile.sh
+```
+
+## Run sample
+
+```sh
+python main.py
+```
+
+Output:
+
+```
+python: calling c_func(10)
+c: in c_func with input: 10
+c: calling python_func(11)
+python: in python_func with input: 11
+python: calling add(11, 1000)
+python: result from add: 1011
+c: result from python_func: 1012
+c: returning 1013
+python: result from c_func: 1013
+deleting ExecEnv
+deleting Instance
+deleting Module
+deleting Engine
+```

language-bindings/python/wamr-api/samples/native-symbol/compile.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+/opt/wasi-sdk/bin/clang     \
+    -O0 -z stack-size=4096 -Wl,--initial-memory=65536 \
+    -Wl,--export=main -Wl,--export=__main_argc_argv \
+    -Wl,--export=__data_end -Wl,--export=__heap_base \
+    -Wl,--strip-all,--no-entry \
+    -Wl,--allow-undefined \
+    -Wl,--export=c_func\
+    -Wl,--export=add\
+    -o func.wasm func.c

language-bindings/python/wamr-api/samples/native-symbol/func.c

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2019 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include <stdio.h>
+
+int
+python_func(int val);
+
+int
+add(int val1, int val2)
+{
+    return val1 + val2;
+}
+
+int
+c_func(int val)
+{
+    printf("c: in c_func with input: %d\n", val);
+    printf("c: calling python_func(%d)\n", val + 1);
+    int res = python_func(val + 1);
+    printf("c: result from python_func: %d\n", res);
+    printf("c: returning %d\n", res + 1);
+    return res + 1;
+}
+
+int
+main()
+{}

language-bindings/python/wamr-api/samples/native-symbol/main.py

@@ -0,0 +1,59 @@
+# Copyright (C) 2019 Intel Corporation.  All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from wamr.wamrapi.wamr import Engine, Module, Instance, ExecEnv
+from ctypes import c_uint
+import pathlib
+from ctypes import c_int32
+from ctypes import c_uint
+from ctypes import c_void_p
+from ctypes import cast
+from ctypes import CFUNCTYPE
+
+from wamr.wamrapi.iwasm import NativeSymbol
+from wamr.wamrapi.iwasm import String
+from wamr.wamrapi.wamr import ExecEnv
+
+def python_func(env: int, value: int) -> int:
+    print("python: in python_func with input:", value)
+    # Example of generating ExecEnv from `wasm_exec_env_t``
+    exec_env = ExecEnv.wrap(env)
+    add = exec_env.get_module_inst().lookup_function("add")
+    const = 1000
+    argv = (c_uint * 2)(value, const)
+    print(f"python: calling add({value}, {const})")
+    exec_env.call(add, 2, argv)
+    res = argv[0]
+    print("python: result from add:", res)
+    return res + 1
+
+
+native_symbols = (NativeSymbol * 1)(
+    *[
+        NativeSymbol(
+            symbol=String.from_param("python_func"),
+            func_ptr=cast(
+                CFUNCTYPE(c_int32, c_void_p, c_int32)(python_func), c_void_p
+            ),
+            signature=String.from_param("(i)i"),
+        )
+    ]
+)
+
+def main():
+    engine = Engine()
+    engine.register_natives("env", native_symbols)
+    module = Module.from_file(engine, pathlib.Path(__file__).parent / "func.wasm")
+    module_inst = Instance(module)
+    exec_env = ExecEnv(module_inst)
+
+    func = module_inst.lookup_function("c_func")
+
+    inp = 10
+    print(f"python: calling c_func({inp})")
+    argv = (c_uint)(inp)
+    exec_env.call(func, 1, argv)
+    print("python: result from c_func:", argv.value)
+
+if __name__ == "__main__":
+    main()