[wasi-nn] Add a new wasi-nn backend openvino (#3603)

.gitignore

@@ -5,7 +5,8 @@
 .venv
 /.idea
 **/cmake-build-*/
-**/*build/
+**/*build*/
+!/build-scripts
 *.obj
 *.a
 *.so

build-scripts/config_common.cmake

@@ -434,6 +434,19 @@ endif ()
 if (WAMR_BUILD_WASI_NN EQUAL 1)
   message ("     WASI-NN enabled")
   add_definitions (-DWASM_ENABLE_WASI_NN=1)
+  # Variant backends
+  if (NOT WAMR_BUILD_WASI_NN_TFLITE EQUAL 1 AND NOT WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1)
+    message (FATAL_ERROR "   Need to select a backend for WASI-NN")
+  endif ()
+  if (WAMR_BUILD_WASI_NN_TFLITE EQUAL 1)
+    message ("     WASI-NN backend tflite enabled")
+    add_definitions (-DWASM_ENABLE_WASI_NN_TFLITE)
+  endif ()
+  if (WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1)
+    message ("     WASI-NN backend openvino enabled")
+    add_definitions (-DWASM_ENABLE_WASI_NN_OPENVINO)
+  endif ()
+  # Variant devices
   if (WAMR_BUILD_WASI_NN_ENABLE_GPU EQUAL 1)
       message ("     WASI-NN: GPU enabled")
       add_definitions (-DWASM_ENABLE_WASI_NN_GPU=1)

core/iwasm/libraries/wasi-nn/README.md

@@ -19,6 +19,13 @@ $ cmake -DWAMR_BUILD_WASI_NN=1 <other options> ...
 > ![Caution]
 > If enable `WAMR_BUID_WASI_NN`, iwasm will link a shared WAMR library instead of a static one. Wasi-nn backends will be loaded dynamically at runtime. Users shall specify the path of the backend library and register it to the iwasm runtime with `--native-lib=<path of backend library>`. All shared libraries should be placed in the `LD_LIBRARY_PATH`.
 
+#### Compilation options
+
+- `WAMR_BUILD_WASI_NN`. enable wasi-nn support. can't work alone. need to identify a backend. Match legacy wasi-nn spec naming convention. use `wasi_nn` as import module names.
+- `WAMR_BUILD_WASI_EPHEMERAL_NN`. Match latest wasi-nn spec naming convention. use `wasi_ephemeral_nn` as import module names.
+- `WAMR_BUILD_WASI_NN_TFLITE`. identify the backend as TensorFlow Lite.
+- `WAMR_BUILD_WASI_NN_OPENVINO`. identify the backend as OpenVINO.
+
 ### Wasm
 
 The definition of functions provided by WASI-NN (Wasm imports) is in the header file [wasi_nn.h](_core/iwasm/libraries/wasi-nn/wasi_nn.h_). By only including this file in a WASM application you will bind WASI-NN into your module.
@@ -37,6 +44,12 @@ typedef enum { fp16 = 0, fp32, up8, ip32 } tensor_type;
 
 It is required to recompile the Wasm application if you want to switch between the two sets of functions.
 
+#### Openvino
+
+If you're planning to use OpenVINO backends, the first step is to install OpenVINO on your computer. To do this correctly, please follow the official installation guide which you can find at this link: https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-linux.html.
+
+After you've installed OpenVINO, you'll need to let cmake system know where to find it. You can do this by setting an environment variable named `OpenVINO_DIR`. This variable should point to the place on your computer where OpenVINO is installed. By setting this variable, your system will be able to locate and use OpenVINO when needed. You can find installation path by running the following command if using APT `$dpkg -L openvino`. The path should be _/opt/intel/openvino/_ or _/usr/lib/openvino_.
+
 ## Tests
 
 To run the tests we assume that the current directory is the root of the repository.
@@ -167,7 +180,7 @@ Due to the different requirements of each backend, we'll use a Docker container
 $ pwd
 /workspaces/wasm-micro-runtime/
 
-$ docker build -t wasi-nn-smoke:v1.0 -f Dockerfile.wasi-nn-smoke .
+$ docker build -t wasi-nn-smoke:v1.0 -f ./core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke .
 ```
 
 #### Execute

core/iwasm/libraries/wasi-nn/cmake/wasi_nn.cmake

@@ -3,13 +3,30 @@
 
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR})
 
-# Find tensorflow-lite
+if(WAMR_BUILD_WASI_NN_TFLITE EQUAL 1)
-find_package(tensorflow_lite REQUIRED)
+  # Find tensorflow-lite
+  find_package(tensorflow_lite REQUIRED)
+endif()
 
-set(WASI_NN_ROOT ${CMAKE_CURRENT_LIST_DIR}/..)
+if(WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1)
+  if(NOT DEFINED ENV{OpenVINO_DIR})
+    message(FATAL_ERROR
+        "OpenVINO_DIR is not defined. "
+        "Please follow https://docs.openvino.ai/2024/get-started/install-openvino.html,"
+        "install openvino, and set environment variable OpenVINO_DIR."
+        "Like OpenVINO_DIR=/usr/lib/openvino-2023.2/ cmake ..."
+        "Or OpenVINO_DIR=/opt/intel/openvino/ cmake ..."
+    )
+  endif()
+
+  list(APPEND CMAKE_MODULE_PATH $ENV{OpenVINO_DIR})
+  # Find OpenVINO
+  find_package(OpenVINO REQUIRED COMPONENTS Runtime)
+endif()
 
 #
 # wasi-nn general
+set(WASI_NN_ROOT ${CMAKE_CURRENT_LIST_DIR}/..)
 add_library(
   wasi-nn-general
   SHARED
@@ -37,20 +54,34 @@ target_compile_definitions(
 
 #
 # wasi-nn backends
-add_library(
+
-  wasi-nn-tflite
+# - tflite
-  SHARED
+if(WAMR_BUILD_WASI_NN_TFLITE EQUAL 1)
-    ${WASI_NN_ROOT}/src/wasi_nn_tensorflowlite.cpp
+  add_library(
-)
+    wasi-nn-tflite
-#target_link_options(
+    SHARED
-#  wasi-nn-tflite
+      ${WASI_NN_ROOT}/src/wasi_nn_tensorflowlite.cpp
-#  PRIVATE
+  )
-#    -Wl,--whole-archive libwasi-nn-general.a
+  target_link_libraries(
-#    -Wl,--no-whole-archive
+    wasi-nn-tflite
-#)
+    PUBLIC
-target_link_libraries(
+      tensorflow-lite
-  wasi-nn-tflite
+      wasi-nn-general
-  PUBLIC
+  )
-    tensorflow-lite
+endif()
-    wasi-nn-general
+
-)
+# - openvino
+if(WAMR_BUILD_WASI_NN_OPENVINO EQUAL 1)
+  add_library(
+    wasi-nn-openvino
+    SHARED
+      ${WASI_NN_ROOT}/src/wasi_nn_openvino.c
+  )
+  target_link_libraries(
+    wasi-nn-openvino
+    PUBLIC
+      openvino::runtime
+      openvino::runtime::c
+      wasi-nn-general
+  )
+endif()

core/iwasm/libraries/wasi-nn/include/wasi_nn_types.h

@@ -164,6 +164,10 @@ typedef struct {
 bool
 wasi_nn_register_backend(api_function apis);
 
+void
+wasi_nn_dump_tensor_dimension(tensor_dimensions *dim, int32_t output_len,
+                              char *output);
+
 #ifdef __cplusplus
 }
 #endif

core/iwasm/libraries/wasi-nn/src/utils/wasi_nn_app_native.c

@@ -88,7 +88,7 @@ graph_builder_array_app_native(wasm_module_inst_t instance,
         }
 
         NN_DBG_PRINTF("Graph builder %d contains %d elements", i,
-                      builder->size);
+                      builder[i].size);
     }
 
     builder_array->buf = builder;

core/iwasm/libraries/wasi-nn/src/wasi_nn.c

@@ -26,15 +26,17 @@
 // so, just keep one `api_function` is enough
 static api_function lookup = { 0 };
 
-#define call_wasi_nn_func(wasi_error, func, ...)                 \
+#define call_wasi_nn_func(wasi_error, func, ...)                  \
-    do {                                                         \
+    do {                                                          \
-        if (lookup.func) {                                       \
+        if (lookup.func) {                                        \
-            wasi_error = lookup.func(__VA_ARGS__);               \
+            wasi_error = lookup.func(__VA_ARGS__);                \
-        }                                                        \
+            if (wasi_error != success)                            \
-        else {                                                   \
+                NN_ERR_PRINTF("Error %s: %d", #func, wasi_error); \
-            NN_ERR_PRINTF("Error: %s is not registered", #func); \
+        }                                                         \
-            wasi_error = unsupported_operation;                  \
+        else {                                                    \
-        }                                                        \
+            NN_ERR_PRINTF("Error %s is not registered", #func);   \
+            wasi_error = unsupported_operation;                   \
+        }                                                         \
     } while (0)
 
 static HashMap *hashmap;
@@ -68,7 +70,9 @@ key_equal_func(void *key1, void *key2)
 
 static void
 key_destroy_func(void *key1)
-{}
+{
+    /* key type is wasm_module_inst_t*. do nothing */
+}
 
 static void
 value_destroy_func(void *value)
@@ -79,7 +83,8 @@ value_destroy_func(void *value)
 static WASINNContext *
 wasi_nn_initialize_context()
 {
-    NN_DBG_PRINTF("Initializing wasi-nn context");
+    NN_DBG_PRINTF("[WASI NN] INIT...");
+
     WASINNContext *wasi_nn_ctx =
         (WASINNContext *)wasm_runtime_malloc(sizeof(WASINNContext));
     if (wasi_nn_ctx == NULL) {
@@ -92,7 +97,6 @@ wasi_nn_initialize_context()
     wasi_nn_error res;
     call_wasi_nn_func(res, init, &wasi_nn_ctx->backend_ctx);
     if (res != success) {
-        NN_ERR_PRINTF("Error while initializing backend");
         wasm_runtime_free(wasi_nn_ctx);
         return NULL;
     }
@@ -103,7 +107,7 @@ wasi_nn_initialize_context()
 static bool
 wasi_nn_initialize()
 {
-    NN_DBG_PRINTF("Initializing wasi-nn");
+    NN_DBG_PRINTF("[WASI NN General] Initializing wasi-nn");
     // hashmap { instance: wasi_nn_ctx }
     hashmap = bh_hash_map_create(HASHMAP_INITIAL_SIZE, true, hash_func,
                                  key_equal_func, key_destroy_func,
@@ -133,13 +137,15 @@ wasm_runtime_get_wasi_nn_ctx(wasm_module_inst_t instance)
             return NULL;
         }
     }
-    NN_DBG_PRINTF("Returning ctx");
+
     return wasi_nn_ctx;
 }
 
 static void
 wasi_nn_ctx_destroy(WASINNContext *wasi_nn_ctx)
 {
+    NN_DBG_PRINTF("[WASI NN] DEINIT...");
+
     if (wasi_nn_ctx == NULL) {
         NN_ERR_PRINTF(
             "Error when deallocating memory. WASI-NN context is NULL");
@@ -152,9 +158,6 @@ wasi_nn_ctx_destroy(WASINNContext *wasi_nn_ctx)
     /* only one backend can be registered */
     wasi_nn_error res;
     call_wasi_nn_func(res, deinit, wasi_nn_ctx->backend_ctx);
-    if (res != success) {
-        NN_ERR_PRINTF("Error while destroyging backend");
-    }
 
     wasm_runtime_free(wasi_nn_ctx);
 }
@@ -191,7 +194,7 @@ wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder,
              graph_encoding encoding, execution_target target, graph *g)
 #endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */
 {
-    NN_DBG_PRINTF("Running wasi_nn_load [encoding=%d, target=%d]...", encoding,
+    NN_DBG_PRINTF("[WASI NN] LOAD [encoding=%d, target=%d]...", encoding,
                   target);
 
     wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env);
@@ -222,7 +225,6 @@ wasi_nn_load(wasm_exec_env_t exec_env, graph_builder_array_wasm *builder,
     WASINNContext *wasi_nn_ctx = wasm_runtime_get_wasi_nn_ctx(instance);
     call_wasi_nn_func(res, load, wasi_nn_ctx->backend_ctx, &builder_native,
                       encoding, target, g);
-    NN_DBG_PRINTF("wasi_nn_load finished with status %d [graph=%d]", res, *g);
     if (res != success)
         goto fail;
 
@@ -241,7 +243,7 @@ wasi_nn_error
 wasi_nn_load_by_name(wasm_exec_env_t exec_env, char *name, uint32_t name_len,
                      graph *g)
 {
-    NN_DBG_PRINTF("Running wasi_nn_load_by_name ...");
+    NN_DBG_PRINTF("[WASI NN] LOAD_BY_NAME %s...", name);
 
     wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env);
     if (!instance) {
@@ -261,7 +263,6 @@ wasi_nn_load_by_name(wasm_exec_env_t exec_env, char *name, uint32_t name_len,
     wasi_nn_error res;
     call_wasi_nn_func(res, load_by_name, wasi_nn_ctx->backend_ctx, name,
                       name_len, g);
-    NN_DBG_PRINTF("wasi_nn_load_by_name finished with status %d", *g);
     if (res != success)
         return res;
 
@@ -274,7 +275,7 @@ wasi_nn_error
 wasi_nn_init_execution_context(wasm_exec_env_t exec_env, graph g,
                                graph_execution_context *ctx)
 {
-    NN_DBG_PRINTF("Running wasi_nn_init_execution_context [graph=%d]...", g);
+    NN_DBG_PRINTF("[WASI NN] INIT_EXECUTION_CONTEXT...");
 
     wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env);
     if (!instance) {
@@ -295,9 +296,6 @@ wasi_nn_init_execution_context(wasm_exec_env_t exec_env, graph g,
 
     call_wasi_nn_func(res, init_execution_context, wasi_nn_ctx->backend_ctx, g,
                       ctx);
-    NN_DBG_PRINTF(
-        "wasi_nn_init_execution_context finished with status %d [ctx=%d]", res,
-        *ctx);
     return res;
 }
 
@@ -305,8 +303,7 @@ wasi_nn_error
 wasi_nn_set_input(wasm_exec_env_t exec_env, graph_execution_context ctx,
                   uint32_t index, tensor_wasm *input_tensor)
 {
-    NN_DBG_PRINTF("Running wasi_nn_set_input [ctx=%d, index=%d]...", ctx,
+    NN_DBG_PRINTF("[WASI NN] SET_INPUT [ctx=%d, index=%d]...", ctx, index);
-                  index);
 
     wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env);
     if (!instance) {
@@ -331,14 +328,13 @@ wasi_nn_set_input(wasm_exec_env_t exec_env, graph_execution_context ctx,
     if (input_tensor_native.dimensions)
         wasm_runtime_free(input_tensor_native.dimensions);
 
-    NN_DBG_PRINTF("wasi_nn_set_input finished with status %d", res);
     return res;
 }
 
 wasi_nn_error
 wasi_nn_compute(wasm_exec_env_t exec_env, graph_execution_context ctx)
 {
-    NN_DBG_PRINTF("Running wasi_nn_compute [ctx=%d]...", ctx);
+    NN_DBG_PRINTF("[WASI NN] COMPUTE [ctx=%d]...", ctx);
 
     wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env);
     if (!instance) {
@@ -352,7 +348,6 @@ wasi_nn_compute(wasm_exec_env_t exec_env, graph_execution_context ctx)
         return res;
 
     call_wasi_nn_func(res, compute, wasi_nn_ctx->backend_ctx, ctx);
-    NN_DBG_PRINTF("wasi_nn_compute finished with status %d", res);
     return res;
 }
 
@@ -368,8 +363,7 @@ wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx,
                    uint32_t *output_tensor_size)
 #endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */
 {
-    NN_DBG_PRINTF("Running wasi_nn_get_output [ctx=%d, index=%d]...", ctx,
+    NN_DBG_PRINTF("[WASI NN] GET_OUTPUT [ctx=%d, index=%d]...", ctx, index);
-                  index);
 
     wasm_module_inst_t instance = wasm_runtime_get_module_inst(exec_env);
     if (!instance) {
@@ -396,8 +390,6 @@ wasi_nn_get_output(wasm_exec_env_t exec_env, graph_execution_context ctx,
     call_wasi_nn_func(res, get_output, wasi_nn_ctx->backend_ctx, ctx, index,
                       output_tensor, output_tensor_size);
 #endif /* WASM_ENABLE_WASI_EPHEMERAL_NN != 0 */
-    NN_DBG_PRINTF("wasi_nn_get_output finished with status %d [data_size=%d]",
-                  res, *output_tensor_size);
     return res;
 }
 
@@ -435,7 +427,7 @@ get_wasi_nn_export_apis(NativeSymbol **p_native_symbols)
 __attribute__((used)) uint32_t
 get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols)
 {
-    NN_DBG_PRINTF("--|> get_native_lib");
+    NN_DBG_PRINTF("[Native Register] get_native_lib");
 
 #if WASM_ENABLE_WASI_EPHEMERAL_NN != 0
     *p_module_name = "wasi_ephemeral_nn";
@@ -449,7 +441,7 @@ get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols)
 __attribute__((used)) int
 init_native_lib()
 {
-    NN_DBG_PRINTF("--|> init_native_lib");
+    NN_DBG_PRINTF("[Native Register] init_native_lib");
 
     if (!wasi_nn_initialize())
         return 1;
@@ -460,7 +452,7 @@ init_native_lib()
 __attribute__((used)) void
 deinit_native_lib()
 {
-    NN_DBG_PRINTF("--|> deinit_native_lib");
+    NN_DBG_PRINTF("[Native Register] deinit_native_lib");
 
     wasi_nn_destroy();
 }
@@ -468,7 +460,7 @@ deinit_native_lib()
 __attribute__((used)) bool
 wasi_nn_register_backend(api_function apis)
 {
-    NN_DBG_PRINTF("--|> wasi_nn_register_backend");
+    NN_DBG_PRINTF("[Native Register] wasi_nn_register_backend");
     lookup = apis;
     return true;
-}
+}

core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.c

@@ -0,0 +1,559 @@
+/*
+ * Copyright (C) 2019 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include "wasi_nn_types.h"
+#include "wasi_nn_openvino.h"
+#include "logger.h"
+#include "bh_platform.h"
+
+#include "openvino/c/openvino.h"
+
+/*
+ * refer to
+ * https://docs.openvino.ai/2024/openvino-workflow/running-inference/integrate-openvino-with-your-application.html
+ *
+ * Steps about integrating OpenVINO are:
+ *
+ * 1. Create OpenVINO Runtime Core
+ * 2. Compile Model
+ * 3. Create Inference Request
+ * 4. Set Inputs
+ * 5. Start Inference
+ * 6. Process Inference Results
+ *
+ * from 4. to 6. is the Inference Loop
+ */
+
+typedef struct {
+    ov_core_t *core;
+    /* keep input model files */
+    void *weight_data;
+    ov_tensor_t *weights_tensor;
+    ov_model_t *model;
+    /* add prepostprocess */
+    ov_model_t *new_model;
+    ov_compiled_model_t *compiled_model;
+    ov_infer_request_t *infer_request;
+    ov_tensor_t *input_tensor;
+} OpenVINOContext;
+
+/*
+ * BE AWARE OF "goto fail"
+ */
+#define CHECK_OV_STATUS(status, error_code)                \
+    do {                                                   \
+        ov_status_e s = status;                            \
+        if (s != OK) {                                     \
+            NN_ERR_PRINTF("return status \"%s\", line %d", \
+                          ov_get_error_info(s), __LINE__); \
+            error_code = runtime_error;                    \
+            goto fail;                                     \
+        }                                                  \
+    } while (0)
+
+static void
+dump_ov_shape_t(const ov_shape_t *shape, int32_t output_len, char *output)
+{
+    int ret = 0;
+
+    ret = snprintf(output, output_len, "%ld,[", shape->rank);
+    if (!ret)
+        return;
+
+    output_len -= ret;
+    output += ret;
+
+    for (unsigned i = 0; i < shape->rank && output_len; i++) {
+        ret = snprintf(output, output_len, " %ld", shape->dims[i]);
+        if (!ret)
+            return;
+
+        output_len -= ret;
+        output += ret;
+    }
+
+    snprintf(output, output_len, "]");
+    return;
+}
+
+#ifndef NDEBUG
+static void
+print_model_input_output_info(ov_model_t *model)
+{
+    wasi_nn_error ov_error;
+    char *friendly_name = NULL;
+    size_t input_size = 0;
+    ov_output_const_port_t *input_port = NULL;
+    ov_shape_t input_shape = { 0 };
+    ov_element_type_e input_type;
+    char shape_info[64] = { 0 };
+    ov_output_const_port_t *output_port = NULL;
+    ov_shape_t output_shape = { 0 };
+    ov_element_type_e output_type;
+
+    CHECK_OV_STATUS(ov_model_get_friendly_name(model, &friendly_name),
+                    ov_error);
+    NN_DBG_PRINTF("model name: %s", friendly_name);
+
+    ov_model_inputs_size(model, &input_size);
+    for (unsigned i = 0; i < input_size; i++) {
+        CHECK_OV_STATUS(ov_model_const_input_by_index(model, i, &input_port),
+                        ov_error);
+        CHECK_OV_STATUS(ov_const_port_get_shape(input_port, &input_shape),
+                        ov_error);
+        CHECK_OV_STATUS(ov_port_get_element_type(input_port, &input_type),
+                        ov_error);
+
+        dump_ov_shape_t(&input_shape, 60, shape_info);
+        NN_DBG_PRINTF("model input[%u]. element_type: %d, shape: %s", i,
+                      input_type, shape_info);
+
+        ov_shape_free(&input_shape);
+        memset(&input_shape, 0, sizeof(input_shape));
+        ov_output_const_port_free(input_port);
+        input_port = NULL;
+    }
+
+    size_t output_size = 0;
+    ov_model_outputs_size(model, &output_size);
+    for (unsigned i = 0; i < output_size; i++) {
+        CHECK_OV_STATUS(ov_model_const_output_by_index(model, i, &output_port),
+                        ov_error);
+        CHECK_OV_STATUS(ov_const_port_get_shape(output_port, &output_shape),
+                        ov_error);
+        CHECK_OV_STATUS(ov_port_get_element_type(output_port, &output_type),
+                        ov_error);
+
+        dump_ov_shape_t(&output_shape, 60, shape_info);
+        NN_DBG_PRINTF("model output[%u]. element_type: %d, shape: %s", i,
+                      output_type, shape_info);
+
+        ov_shape_free(&output_shape);
+        memset(&output_shape, 0, sizeof(output_shape));
+        ov_output_const_port_free(output_port);
+        output_port = NULL;
+    }
+
+fail:
+    if (friendly_name)
+        ov_free(friendly_name);
+    ov_shape_free(&input_shape);
+    if (input_port)
+        ov_output_const_port_free(input_port);
+    ov_shape_free(&output_shape);
+    if (output_port)
+        ov_output_const_port_free(output_port);
+    return;
+}
+#endif
+
+static ov_element_type_e
+wasi_nn_tensor_type_to_openvino_element_type(tensor_type wasi_nn_type)
+{
+    switch (wasi_nn_type) {
+        case fp16:
+            return F16;
+        case fp32:
+            return F32;
+        case fp64:
+            return F64;
+        case bf16:
+            return BF16;
+        case u8:
+            return U8;
+        case i32:
+            return I32;
+        case i64:
+            return I64;
+        default:
+            break;
+    }
+
+    NN_ERR_PRINTF("%d is an undefined tensor type", wasi_nn_type);
+    return UNDEFINED;
+}
+
+static wasi_nn_error
+uint32_array_to_int64_array(uint32_t array_size, uint32_t *src, int64_t **dst)
+{
+    *dst = malloc(array_size * sizeof(int64_t));
+    if (!(*dst))
+        return runtime_error;
+
+    for (unsigned i = 0; i < array_size; i++) {
+        (*dst)[i] = src[i];
+    }
+
+    return success;
+}
+
+wasi_nn_error
+openvino_load(void *ctx, graph_builder_array *builder, graph_encoding encoding,
+              execution_target target, graph *g)
+{
+    OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx;
+    wasi_nn_error ret = unsupported_operation;
+
+    if (encoding != openvino) {
+        NN_ERR_PRINTF("Unexpected encoding %d.", encoding);
+        return invalid_argument;
+    }
+
+    /*FIXME: unblock non-cpu device after supporting */
+    if (target != cpu) {
+        NN_ERR_PRINTF("Unexpected device %d.", target);
+        return invalid_argument;
+    }
+
+    if (builder->size != 2) {
+        NN_ERR_PRINTF("Unexpected builder format.");
+        return invalid_argument;
+    }
+
+    /*
+     * The first builder is the XML file.
+     * The second builder is the weight file.
+     */
+    graph_builder xml = builder->buf[0];
+    graph_builder weight = builder->buf[1];
+
+    /* if xml is a String with a model in IR */
+    if (!(xml.buf[xml.size] == '\0' && xml.buf[xml.size - 1] != '\0')) {
+        NN_ERR_PRINTF("Invalid xml string.");
+        return invalid_argument;
+    }
+
+    /* transfer weight to an ov tensor */
+    {
+        ov_ctx->weight_data = malloc(weight.size);
+        if (!ov_ctx->weight_data)
+            goto fail;
+        memcpy(ov_ctx->weight_data, weight.buf, weight.size);
+
+        ov_element_type_e type = U8;
+        int64_t dims[1] = { weight.size };
+        ov_shape_t shape = { 1, dims };
+        CHECK_OV_STATUS(ov_tensor_create_from_host_ptr(type, shape,
+                                                       ov_ctx->weight_data,
+                                                       &ov_ctx->weights_tensor),
+                        ret);
+    }
+
+    /* load model from buffer */
+    CHECK_OV_STATUS(ov_core_read_model_from_memory_buffer(
+                        ov_ctx->core, (char *)xml.buf, xml.size,
+                        ov_ctx->weights_tensor, &ov_ctx->model),
+                    ret);
+#ifndef NDEBUG
+    print_model_input_output_info(ov_ctx->model);
+#endif
+
+    ret = success;
+fail:
+    return ret;
+}
+
+wasi_nn_error
+openvino_load_by_name(void *ctx, const char *filename, uint32_t filename_len,
+                      graph *g)
+{
+    OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx;
+    wasi_nn_error ret = unsupported_operation;
+
+    CHECK_OV_STATUS(
+        ov_core_read_model(ov_ctx->core, filename, NULL, &ov_ctx->model), ret);
+
+    ret = success;
+fail:
+    return ret;
+}
+
+wasi_nn_error
+openvino_init_execution_context(void *ctx, graph g,
+                                graph_execution_context *exec_ctx)
+{
+    return success;
+}
+
+wasi_nn_error
+openvino_set_input(void *ctx, graph_execution_context exec_ctx, uint32_t index,
+                   tensor *wasi_nn_tensor)
+{
+    OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx;
+    wasi_nn_error ret = unsupported_operation;
+    ov_shape_t input_shape = { 0 };
+    int64_t *ov_dims = NULL;
+
+    ov_preprocess_prepostprocessor_t *ppp = NULL;
+    ov_preprocess_input_info_t *input_info = NULL;
+    ov_preprocess_input_tensor_info_t *input_tensor_info = NULL;
+    ov_layout_t *input_layout = NULL;
+    ov_preprocess_preprocess_steps_t *input_process = NULL;
+    ov_preprocess_input_model_info_t *p_input_model = NULL;
+    ov_layout_t *model_layout = NULL;
+    ov_preprocess_output_info_t *output_info = NULL;
+    ov_preprocess_output_tensor_info_t *output_tensor_info = NULL;
+
+    /* wasi_nn_tensor -> ov_tensor */
+    {
+        ret = uint32_array_to_int64_array(wasi_nn_tensor->dimensions->size,
+                                          wasi_nn_tensor->dimensions->buf,
+                                          &ov_dims);
+        if (ret != success)
+            goto fail;
+
+        /* NCHW -> NHWC */
+        if (wasi_nn_tensor->dimensions->size == 4 || ov_dims[1] == 3) {
+            /* N */
+            /* H */
+            ov_dims[1] = ov_dims[2];
+            /* W */
+            ov_dims[2] = ov_dims[3];
+            /* C */
+            ov_dims[3] = 3;
+        }
+
+        CHECK_OV_STATUS(ov_shape_create(wasi_nn_tensor->dimensions->size,
+                                        ov_dims, &input_shape),
+                        ret);
+
+        ov_element_type_e input_type =
+            wasi_nn_tensor_type_to_openvino_element_type(wasi_nn_tensor->type);
+        if (input_type == UNDEFINED)
+            goto fail;
+
+        char shape_info[64] = { 0 };
+        dump_ov_shape_t(&input_shape, 60, shape_info);
+        NN_DBG_PRINTF("input tensor. element_type: %d, shape: %s", input_type,
+                      shape_info);
+
+        CHECK_OV_STATUS(ov_tensor_create_from_host_ptr(input_type, input_shape,
+                                                       wasi_nn_tensor->data,
+                                                       &ov_ctx->input_tensor),
+                        ret);
+    }
+
+    /* set preprocess based on wasi_nn_tensor */
+    {
+        CHECK_OV_STATUS(
+            ov_preprocess_prepostprocessor_create(ov_ctx->model, &ppp), ret);
+
+        /* reuse user' created tensor's info */
+        CHECK_OV_STATUS(ov_preprocess_prepostprocessor_get_input_info_by_index(
+                            ppp, index, &input_info),
+                        ret);
+        CHECK_OV_STATUS(ov_preprocess_input_info_get_tensor_info(
+                            input_info, &input_tensor_info),
+                        ret);
+        CHECK_OV_STATUS(ov_preprocess_input_tensor_info_set_from(
+                            input_tensor_info, ov_ctx->input_tensor),
+                        ret);
+        /* ! HAS TO BE NHWC. Match previous layout conversion */
+        CHECK_OV_STATUS(ov_layout_create("NHWC", &input_layout), ret);
+        CHECK_OV_STATUS(ov_preprocess_input_tensor_info_set_layout(
+                            input_tensor_info, input_layout),
+                        ret);
+
+        /* add RESIZE */
+        CHECK_OV_STATUS(ov_preprocess_input_info_get_preprocess_steps(
+                            input_info, &input_process),
+                        ret);
+        CHECK_OV_STATUS(
+            ov_preprocess_preprocess_steps_resize(input_process, RESIZE_LINEAR),
+            ret);
+
+        /* input model */
+        CHECK_OV_STATUS(
+            ov_preprocess_input_info_get_model_info(input_info, &p_input_model),
+            ret);
+        // TODO: what if not?
+        CHECK_OV_STATUS(ov_layout_create("NCHW", &model_layout), ret);
+        CHECK_OV_STATUS(ov_preprocess_input_model_info_set_layout(p_input_model,
+                                                                  model_layout),
+                        ret);
+
+        /* output -> F32(possibility) */
+        CHECK_OV_STATUS(ov_preprocess_prepostprocessor_get_output_info_by_index(
+                            ppp, index, &output_info),
+                        ret);
+        CHECK_OV_STATUS(ov_preprocess_output_info_get_tensor_info(
+                            output_info, &output_tensor_info),
+                        ret);
+        CHECK_OV_STATUS(
+            ov_preprocess_output_set_element_type(output_tensor_info, F32),
+            ret);
+
+        CHECK_OV_STATUS(
+            ov_preprocess_prepostprocessor_build(ppp, &ov_ctx->new_model), ret);
+    }
+
+    CHECK_OV_STATUS(ov_core_compile_model(ov_ctx->core, ov_ctx->new_model,
+                                          "CPU", 0, &ov_ctx->compiled_model),
+                    ret);
+
+    CHECK_OV_STATUS(ov_compiled_model_create_infer_request(
+                        ov_ctx->compiled_model, &ov_ctx->infer_request),
+                    ret);
+
+    /* install ov_tensor -> infer_request */
+    CHECK_OV_STATUS(ov_infer_request_set_input_tensor_by_index(
+                        ov_ctx->infer_request, index, ov_ctx->input_tensor),
+                    ret);
+    ret = success;
+
+fail:
+    if (ov_dims)
+        free(ov_dims);
+    ov_shape_free(&input_shape);
+    if (ppp)
+        ov_preprocess_prepostprocessor_free(ppp);
+    if (input_info)
+        ov_preprocess_input_info_free(input_info);
+    if (input_tensor_info)
+        ov_preprocess_input_tensor_info_free(input_tensor_info);
+    if (input_layout)
+        ov_layout_free(input_layout);
+    if (input_process)
+        ov_preprocess_preprocess_steps_free(input_process);
+    if (p_input_model)
+        ov_preprocess_input_model_info_free(p_input_model);
+    if (model_layout)
+        ov_layout_free(model_layout);
+    if (output_info)
+        ov_preprocess_output_info_free(output_info);
+    if (output_tensor_info)
+        ov_preprocess_output_tensor_info_free(output_tensor_info);
+
+    return ret;
+}
+
+wasi_nn_error
+openvino_compute(void *ctx, graph_execution_context exec_ctx)
+{
+    OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx;
+    wasi_nn_error ret = unsupported_operation;
+
+    CHECK_OV_STATUS(ov_infer_request_infer(ov_ctx->infer_request), ret);
+    ret = success;
+fail:
+    return ret;
+}
+
+wasi_nn_error
+openvino_get_output(void *ctx, graph_execution_context exec_ctx, uint32_t index,
+                    tensor_data output_tensor, uint32_t *output_tensor_size)
+{
+    OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx;
+    wasi_nn_error ret = unsupported_operation;
+    ov_tensor_t *ov_tensor = NULL;
+    void *data = NULL;
+    size_t byte_size = 0;
+
+    CHECK_OV_STATUS(ov_infer_request_get_output_tensor_by_index(
+                        ov_ctx->infer_request, index, &ov_tensor),
+                    ret);
+
+    CHECK_OV_STATUS(ov_tensor_get_byte_size(ov_tensor, &byte_size), ret);
+
+    CHECK_OV_STATUS(ov_tensor_data(ov_tensor, &data), ret);
+
+    memcpy(output_tensor, data, byte_size);
+
+    *output_tensor_size = (uint32_t)byte_size;
+
+    ret = success;
+
+fail:
+    if (ov_tensor)
+        ov_tensor_free(ov_tensor);
+    return ret;
+}
+
+wasi_nn_error
+openvino_initialize(void **ctx)
+{
+    ov_version_t version;
+    OpenVINOContext *ov_ctx = NULL;
+    wasi_nn_error ret = unsupported_operation;
+
+    if (!ctx) {
+        ret = invalid_argument;
+        goto fail;
+    }
+
+    /* Get OpenVINO runtime version */
+    CHECK_OV_STATUS(ov_get_openvino_version(&version), ret);
+    NN_INFO_PRINTF("OpenVINO INFO:");
+    NN_INFO_PRINTF("  Description : %s", version.description);
+    NN_INFO_PRINTF("  Build Number: %s", version.buildNumber);
+    ov_version_free(&version);
+
+    ov_ctx = (OpenVINOContext *)os_malloc(sizeof(OpenVINOContext));
+    if (!ov_ctx) {
+        NN_ERR_PRINTF("Allocate for OpenVINOContext failed");
+        ret = runtime_error;
+        goto fail;
+    }
+
+    memset(ov_ctx, 0, sizeof(OpenVINOContext));
+
+    /* Initialize OpenVINO Runtime Core */
+    CHECK_OV_STATUS(ov_core_create(&ov_ctx->core), ret);
+
+    *ctx = (void *)ov_ctx;
+    return success;
+fail:
+    openvino_destroy((void *)ov_ctx);
+    return ret;
+}
+
+wasi_nn_error
+openvino_destroy(void *ctx)
+{
+    OpenVINOContext *ov_ctx = (OpenVINOContext *)ctx;
+
+    if (!ov_ctx)
+        return invalid_argument;
+
+    if (ov_ctx->weight_data)
+        free(ov_ctx->weight_data);
+
+    if (ov_ctx->weights_tensor)
+        ov_tensor_free(ov_ctx->weights_tensor);
+
+    if (ov_ctx->input_tensor)
+        ov_tensor_free(ov_ctx->input_tensor);
+
+    if (ov_ctx->infer_request)
+        ov_infer_request_free(ov_ctx->infer_request);
+
+    if (ov_ctx->compiled_model)
+        ov_compiled_model_free(ov_ctx->compiled_model);
+
+    if (ov_ctx->model)
+        ov_model_free(ov_ctx->model);
+
+    if (ov_ctx->core)
+        ov_core_free(ov_ctx->core);
+
+    os_free(ov_ctx);
+    return success;
+}
+
+__attribute__((constructor(200))) void
+openvino_register_backend()
+{
+    api_function apis = {
+        .load = openvino_load,
+        .load_by_name = openvino_load_by_name,
+        .init_execution_context = openvino_init_execution_context,
+        .set_input = openvino_set_input,
+        .compute = openvino_compute,
+        .get_output = openvino_get_output,
+        .init = openvino_initialize,
+        .deinit = openvino_destroy,
+    };
+    wasi_nn_register_backend(apis);
+}

core/iwasm/libraries/wasi-nn/src/wasi_nn_openvino.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2019 Intel Corporation.  All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#ifndef WASI_NN_OPENVINO_HPP
+#define WASI_NN_OPENVINO_HPP
+
+#include "wasi_nn_types.h"
+
+wasi_nn_error
+openvino_load(void *ctx, graph_builder_array *builder, graph_encoding encoding,
+              execution_target target, graph *g);
+
+wasi_nn_error
+openvino_init_execution_context(void *ctx, graph g,
+                                graph_execution_context *exec_ctx);
+
+wasi_nn_error
+openvino_set_input(void *ctx, graph_execution_context exec_ctx, uint32_t index,
+                   tensor *input_tensor);
+
+wasi_nn_error
+openvino_compute(void *ctx, graph_execution_context exec_ctx);
+
+wasi_nn_error
+openvino_get_output(void *ctx, graph_execution_context exec_ctx, uint32_t index,
+                    tensor_data output_tensor, uint32_t *output_tensor_size);
+
+wasi_nn_error
+openvino_initialize(void **ctx);
+
+wasi_nn_error
+openvino_destroy(void *ctx);
+
+#endif /* WASI_NN_OPENVINO_HPP */

core/iwasm/libraries/wasi-nn/test/Dockerfile.wasi-nn-smoke

@@ -1,6 +1,8 @@
 # Copyright (C) 2019 Intel Corporation.  All rights reserved.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+# hadolint global ignore=DL3003,DL3008,DL3009,DL3059
+
 FROM mcr.microsoft.com/devcontainers/rust:1-1-bullseye@sha256:ddc1ee022d327f024c07484c9333db3fbbfd504bc096cdb66635653a2bebb33e
 
 ARG DEBIAN_FRONTEND=noninteractive
@@ -8,7 +10,10 @@ ENV TZ=Asian/Shanghai
 
 # hadolint ignore=DL3009
 RUN apt-get update \
-  && apt-get upgrade -y
+  && apt-get upgrade -y \
+  && apt-get install -y --no-install-recommends cmake
+
+RUN rustup target add wasm32-wasi
 
 #
 # Openvino
@@ -17,33 +22,27 @@ RUN apt-get update \
 #   - https://docs.openvino.ai/2023.3/openvino_docs_install_guides_installing_openvino_from_archive_linux.html
 #   - https://docs.openvino.ai/2024/get-started/install-openvino/install-openvino-archive-linux.html
 #
-# FIXME: upgrade to 2024.1 or latest after wasi-nn(rust binding) is ready
+RUN wget -q https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
-WORKDIR /opt/intel
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN wget -q https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3.2/linux/l_openvino_toolkit_ubuntu20_2022.3.2.9279.e2c7e4d7b4d_x86_64.tgz
+RUN apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
-RUN tar -xf l_openvino_toolkit_ubuntu20_2022.3.2.9279.e2c7e4d7b4d_x86_64.tgz \
+  && echo "deb https://apt.repos.intel.com/openvino/2023 ubuntu20 main" | tee /etc/apt/sources.list.d/intel-openvino-2023.list
-  && rm l_openvino_toolkit_ubuntu20_2022.3.2.9279.e2c7e4d7b4d_x86_64.tgz \
+RUN apt-get update \
-  && mv l_openvino_toolkit_ubuntu20_2022.3.2.9279.e2c7e4d7b4d_x86_64 /opt/intel/openvino
+  && apt-get upgrade -y \
-
+  && apt-get install --no-install-recommends -y openvino-2023.2.0
-WORKDIR /opt/intel/openvino
-RUN ./install_dependencies/install_openvino_dependencies.sh -y \
-  && ./setupvars.sh
-
-#
-# wasmtime
-WORKDIR /opt
-RUN wget -q https://github.com/bytecodealliance/wasmtime/releases/download/v21.0.0/wasmtime-v21.0.0-x86_64-linux.tar.xz
-RUN tar -xf wasmtime-v21.0.0-x86_64-linux.tar.xz \
-  && rm wasmtime-v21.0.0-x86_64-linux.tar.xz \
-  && ln -sf "$(realpath ./wasmtime-v21.0.0-x86_64-linux/wasmtime)" /usr/local/bin/wasmtime
 
 #
 # wasi-nn
 # compilation requirements
-RUN rustup target add wasm32-wasi wasm32-unknown-unknown
 WORKDIR /workspaces/wasi-nn
 RUN git clone --depth 1 https://github.com/bytecodealliance/wasi-nn.git .
-# hadolint ignore=DL3059
+
-#RUN ./build.sh rust
+WORKDIR /workspaces/wasi-nn/rust/examples/classification-example/
+RUN cargo build --target=wasm32-wasi
+
+WORKDIR /workspaces/wasi-nn/rust/examples/classification-example/build
+RUN cp ../target/wasm32-wasi/debug/wasi-nn-example.wasm . \
+  && wget -q --no-clobber https://github.com/intel/openvino-rs/raw/main/crates/openvino/tests/fixtures/mobilenet/mobilenet.xml \
+  && wget -q --no-clobber https://github.com/intel/openvino-rs/raw/main/crates/openvino/tests/fixtures/mobilenet/mobilenet.bin
 # There are model files(mobilenet*) and wasm files(wasi-nn-example.wasm) in the directory,
 # /workspaces/wasi-nn/rust/examples/classification-example/build
 
@@ -52,14 +51,35 @@ RUN git clone --depth 1 https://github.com/bytecodealliance/wasi-nn.git .
 WORKDIR /tmp
 RUN wget -q https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh \
   && chmod a+x ./install.sh
-RUN ./install.sh -p /opt/wasmedge --plugins wasi_nn-tensorflowlite
+# RUN ./install.sh -p /opt/wasmedge --plugins wasi_nn-tensorflowlite wasi_nn-openvino
+RUN ./install.sh -r yes -D -p /opt/wasmedge --plugins wasi_nn-openvino --dist ubuntu20.04 \
+  && /opt/wasmedge/bin/wasmedge --version
 ENV PATH=/opt/wasmedge/bin:${PATH}
-ENV WASMEDGE_LIB_DIR=/opt/wasmedge/lib
+# ENV WASMEDGE_LIB_DIR=/opt/wasmedge/lib
 
 #
 # wasmedge-wasinn-examples
 WORKDIR /workspaces/wasmedge-wasinn-examples
 RUN git clone --depth 1 https://github.com/second-state/WasmEdge-WASINN-examples.git .
+COPY core/iwasm/libraries/wasi-nn/test/bump_wasi_nn_to_0_6_0.patch .
+RUN git apply ./bump_wasi_nn_to_0_6_0.patch
+# recompile with wasi-nn 0.6.0
+RUN cd openvino-mobilenet-image/rust && cargo build --target=wasm32-wasi
+RUN cd openvino-mobilenet-raw/rust && cargo build --target=wasm32-wasi
+RUN cd openvino-road-segmentation-adas/openvino-road-seg-adas && cargo build --target=wasm32-wasi
+RUN cd tflite-birds_v1-image/rust && cargo build --target=wasm32-wasi
+
+# preparation
+RUN cd openvino-mobilenet-image \
+  && ./download_mobilenet.sh . \
+  && ls -l mobilenet.xml mobilenet.bin
+
+RUN cd openvino-mobilenet-raw \
+  && ./download_mobilenet.sh . \
+  && ls -l mobilenet.xml mobilenet.bin tensor-1x224x224x3-f32.bgr
+
+# RUN apt update \
+#   && apt install -y valgrind
 
 #
 # iwasm. build from source
@@ -67,7 +87,10 @@ WORKDIR /workspaces/wamr
 COPY . .
 
 WORKDIR /workspaces/wamr/product-mini/platforms/linux
-RUN cmake -S . -B build -DWAMR_BUILD_WASI_NN=1 -DWAMR_BUILD_WASI_EPHEMERAL_NN=1 \
+RUN OpenVINO_DIR=/usr/lib/openvino-2023.2.0 \
+    cmake -S . -B build \
+    -DWAMR_BUILD_WASI_NN=1 -DWAMR_BUILD_WASI_EPHEMERAL_NN=1 \
+    -DWAMR_BUILD_WASI_NN_OPENVINO=1 -DWAMR_BUILD_WASI_NN_TFLITE=1 \
   && cmake --build build
 RUN ln -sf "$(realpath ./build/iwasm)" /usr/local/bin/iwasm
 

core/iwasm/libraries/wasi-nn/test/bump_wasi_nn_to_0_6_0.patch

@@ -0,0 +1,47 @@
+diff --git a/openvino-mobilenet-image/rust/Cargo.toml b/openvino-mobilenet-image/rust/Cargo.toml
+index d09e0a4..c7083fb 100644
+--- a/openvino-mobilenet-image/rust/Cargo.toml
++++ b/openvino-mobilenet-image/rust/Cargo.toml
+@@ -8,6 +8,6 @@ publish = false
+ 
+ [dependencies]
+ image = { version = "0.23.14", default-features = false, features = ["gif", "jpeg", "ico", "png", "pnm", "tga", "tiff", "webp", "bmp", "hdr", "dxt", "dds", "farbfeld"]  }
+-wasi-nn = { version = "0.4.0" }
++wasi-nn = { version = "0.6.0" }
+ 
+ [workspace]
+diff --git a/openvino-mobilenet-raw/rust/Cargo.toml b/openvino-mobilenet-raw/rust/Cargo.toml
+index 8eab25b..3f00aec 100644
+--- a/openvino-mobilenet-raw/rust/Cargo.toml
++++ b/openvino-mobilenet-raw/rust/Cargo.toml
+@@ -7,6 +7,6 @@ edition = "2021"
+ publish = false
+ 
+ [dependencies]
+-wasi-nn = { version = "0.4.0" }
++wasi-nn = { version = "0.6.0" }
+ 
+ [workspace]
+diff --git a/openvino-road-segmentation-adas/openvino-road-seg-adas/Cargo.toml b/openvino-road-segmentation-adas/openvino-road-seg-adas/Cargo.toml
+index 998f391..93f91e0 100644
+--- a/openvino-road-segmentation-adas/openvino-road-seg-adas/Cargo.toml
++++ b/openvino-road-segmentation-adas/openvino-road-seg-adas/Cargo.toml
+@@ -5,5 +5,5 @@ name = "openvino-road-seg-adas"
+ version = "0.2.0"
+ 
+ [dependencies]
+-wasi-nn = "0.4.0"
++wasi-nn = "0.6.0"
+ image = { version = "0.23.14", default-features = false, features = ["gif", "jpeg", "ico", "png", "pnm", "tga", "tiff", "webp", "bmp", "hdr", "dxt", "dds", "farbfeld"]  }
+diff --git a/tflite-birds_v1-image/rust/Cargo.toml b/tflite-birds_v1-image/rust/Cargo.toml
+index 572ecb9..9e89e87 100644
+--- a/tflite-birds_v1-image/rust/Cargo.toml
++++ b/tflite-birds_v1-image/rust/Cargo.toml
+@@ -8,6 +8,6 @@ publish = false
+ 
+ [dependencies]
+ image = { version = "0.23.14", default-features = false, features = ["gif", "jpeg", "ico", "png", "pnm", "tga", "tiff", "webp", "bmp", "hdr", "dxt", "dds", "farbfeld"]  }
+-wasi-nn = "0.4.0"
++wasi-nn = "0.6.0"
+ 
+ [workspace]

core/iwasm/libraries/wasi-nn/test/run_smoke_test.py

@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 
+from dataclasses import dataclass
 from pathlib import Path
 from pprint import pprint
 import re
@@ -13,72 +14,151 @@ import subprocess
 from typing import List
 
 
-def execute_tflite_birds_v1_image_once(
+@dataclass
-    runtime_bin: str, runtime_args: List[str], cwd: Path
+class WasmEdgeExampleResult:
+    class_id: int
+    possibility: float
+
+
+def execute_once(
+    runtime_bin: str,
+    runtime_args: List[str],
+    wasm_file: str,
+    wasm_args: List[str],
+    cwd: Path,
 ) -> str:
-    """
-    execute tflite_birds_v1_image example with
-
-    ```
-    iwasm --native-lib=somewhere/libwasi-nn-tflite.so --map-dir=.:. \
-        ./wasmedge-wasinn-example-tflite-bird-image.wasm  \
-        lite-model_aiy_vision_classifier_birds_V1_3.tflite \
-        bird.jpg
-    ```
-
-    or
-
-    ```
-    wasmedge --dir=.:. \
-        ./wasmedge-wasinn-example-tflite-bird-image.wasm  \
-        lite-model_aiy_vision_classifier_birds_V1_3.tflite \
-        bird.jpg
-    ```
-
-    assumption:
-    - under the right directory, tflite-birds_v1-image
-    - every materials are ready
-    """
-
-    wasm_file = "./wasmedge-wasinn-example-tflite-bird-image.wasm"
-    wasm_args = ["lite-model_aiy_vision_classifier_birds_V1_3.tflite", "bird.jpg"]
-
     cmd = [runtime_bin]
     cmd.extend(runtime_args)
     cmd.append(wasm_file)
     cmd.extend(wasm_args)
 
-    try:
+    # print(f'Execute: {" ".join(cmd)}')
-        p = subprocess.run(
+
-            cmd,
+    p = subprocess.run(
-            cwd=cwd,
+        cmd,
-            capture_output=True,
+        cwd=cwd,
-            check=True,
+        capture_output=True,
-            text=True,
+        check=True,
-            universal_newlines=True,
+        text=True,
-        )
+        universal_newlines=True,
-        return p.stdout
+    )
-    except subprocess.CalledProcessError as e:
+    return p.stdout
-        print(e.stderr)
-        print()
-        print(e.stdout)
 
 
-def filter_output_tflite_birds_v1_image(output: str) -> List[str]:
+def execute_openvino_road_segmentation_adas_once(
+    runtime_bin: str, runtime_args: List[str], cwd: Path
+) -> str:
+    """
+    execute openvino-road-segmentation-adas with iwasm and wasmedge
     """
-    not all output is needed for comparision
 
-    pick lines like: "   1.) [526](136)Cathartes burrovianus"
+    wasm_file = (
+        "./openvino-road-seg-adas/target/wasm32-wasi/debug/openvino-road-seg-adas.wasm"
+    )
+    wasm_args = [
+        "./model/road-segmentation-adas-0001.xml",
+        "./model/road-segmentation-adas-0001.bin",
+        "./image/empty_road_mapillary.jpg",
+    ]
+    return execute_once(runtime_bin, runtime_args, wasm_file, wasm_args, cwd)
+
+
+def execute_openvino_mobilenet_raw_once(
+    runtime_bin: str, runtime_args: List[str], cwd: Path
+) -> str:
+    """
+    execute openvino-mobilenet-image with iwasm and wasmedge
+    """
+
+    wasm_file = "./rust/target/wasm32-wasi/debug/wasmedge-wasinn-example-mobilenet.wasm"
+    wasm_args = [
+        "mobilenet.xml",
+        "mobilenet.bin",
+        "./tensor-1x224x224x3-f32.bgr",
+    ]
+    return execute_once(runtime_bin, runtime_args, wasm_file, wasm_args, cwd)
+
+
+def execute_openvino_mobilenet_image_once(
+    runtime_bin: str, runtime_args: List[str], cwd: Path
+) -> str:
+    """
+    execute openvino-mobilenet-image with iwasm and wasmedge
+    """
+
+    wasm_file = (
+        "./rust/target/wasm32-wasi/debug/wasmedge-wasinn-example-mobilenet-image.wasm"
+    )
+    wasm_args = [
+        "mobilenet.xml",
+        "mobilenet.bin",
+        "input.jpg",
+    ]
+    return execute_once(runtime_bin, runtime_args, wasm_file, wasm_args, cwd)
+
+
+def execute_tflite_birds_v1_image_once(
+    runtime_bin: str, runtime_args: List[str], cwd: Path
+) -> str:
+    """
+    execute openvino-mobilenet-image with iwasm and wasmedge
+    """
+
+    wasm_file = (
+        "rust/target/wasm32-wasi/debug/wasmedge-wasinn-example-tflite-bird-image.wasm"
+    )
+    wasm_args = ["lite-model_aiy_vision_classifier_birds_V1_3.tflite", "bird.jpg"]
+    return execute_once(runtime_bin, runtime_args, wasm_file, wasm_args, cwd)
+
+
+def filter_output(output: str) -> List[WasmEdgeExampleResult]:
+    """
+    not all output is required for comparison
+
+    pick lines like: " 1.) [166](198)Aix galericulata"
     """
     filtered = []
-    PATTERN = re.compile(r"^\s+\d\.\)\s+\[\d+\]\(\d+\)\w+")
+    PATTERN = re.compile(r"^\s+\d\.\)\s+\[(\d+)\]\(([.0-9]+)\)\w+")
     for line in output.split("\n"):
-        if PATTERN.search(line):
+        m = PATTERN.search(line)
-            filtered.append(line.strip())
+        if m:
+            class_id, possibility = m.groups()
+            filtered.append(WasmEdgeExampleResult(class_id, possibility))
 
+    assert len(filtered)
     return filtered
 
 
+def compare_output(
+    iwasm_output: List[WasmEdgeExampleResult],
+    wasmedge_output: List[WasmEdgeExampleResult],
+) -> bool:
+    """
+    only compare top 2 and ignore possibility
+    """
+    return (iwasm_output[0].class_id, iwasm_output[1].class_id) == (
+        wasmedge_output[0].class_id,
+        wasmedge_output[1].class_id,
+    )
+
+
+def summarizer_result(
+    example_name: str,
+    iwasm_output: List[WasmEdgeExampleResult],
+    wasmedge_output: List[WasmEdgeExampleResult],
+):
+    if compare_output(iwasm_output, wasmedge_output):
+        print(f"- {example_name}. PASS")
+        return
+
+    print(f"- {example_name}. FAILED")
+    print("------------------------------------------------------------")
+    pprint(iwasm_output)
+    print("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
+    pprint(wasmedge_output)
+    print("------------------------------------------------------------")
+
+
 def execute_tflite_birds_v1_image(iwasm_bin: str, wasmedge_bin: str, cwd: Path):
     iwasm_output = execute_tflite_birds_v1_image_once(
         iwasm_bin,
@@ -88,33 +168,124 @@ def execute_tflite_birds_v1_image(iwasm_bin: str, wasmedge_bin: str, cwd: Path):
         ],
         cwd,
     )
-    iwasm_output = filter_output_tflite_birds_v1_image(iwasm_output)
+    iwasm_output = filter_output(iwasm_output)
 
     wasmedge_output = execute_tflite_birds_v1_image_once(
         wasmedge_bin, ["--dir=.:."], cwd
     )
-    wasmedge_output = filter_output_tflite_birds_v1_image(wasmedge_output)
+    wasmedge_output = filter_output(wasmedge_output)
 
-    if iwasm_output == wasmedge_output:
+    summarizer_result("tf_lite_birds_v1_image", iwasm_output, wasmedge_output)
-        print("- tflite_birds_v1_image. PASS")
+
+
+def execute_openvino_mobilenet_image(iwasm_bin: str, wasmedge_bin: str, cwd: Path):
+    iwasm_output = execute_openvino_mobilenet_image_once(
+        iwasm_bin,
+        [
+            "--native-lib=/workspaces/wamr/product-mini/platforms/linux/build/libwasi-nn-openvino.so",
+            "--map-dir=.:.",
+        ],
+        cwd,
+    )
+    iwasm_output = filter_output(iwasm_output)
+
+    wasmedge_output = execute_openvino_mobilenet_image_once(
+        wasmedge_bin, ["--dir=.:."], cwd
+    )
+    wasmedge_output = filter_output(wasmedge_output)
+
+    summarizer_result("openvino_mobile_image", iwasm_output, wasmedge_output)
+
+
+def execute_openvino_mobilenet_raw(iwasm_bin: str, wasmedge_bin: str, cwd: Path):
+    iwasm_output = execute_openvino_mobilenet_raw_once(
+        iwasm_bin,
+        [
+            "--native-lib=/workspaces/wamr/product-mini/platforms/linux/build/libwasi-nn-openvino.so",
+            "--map-dir=.:.",
+        ],
+        cwd,
+    )
+    iwasm_output = filter_output(iwasm_output)
+
+    wasmedge_output = execute_openvino_mobilenet_raw_once(
+        wasmedge_bin, ["--dir=.:."], cwd
+    )
+    wasmedge_output = filter_output(wasmedge_output)
+
+    summarizer_result("openvino_mobile_raw", iwasm_output, wasmedge_output)
+
+
+def execute_openvino_road_segmentation_adas(
+    iwasm_bin: str, wasmedge_bin: str, cwd: Path
+):
+    def filter_output(output: str) -> str:
+        """
+        focus on lines:
+           The size of the output buffer is 7340032 bytes
+           dump tensor to "wasinn-openvino-inference-output-1x4x512x896xf32.tensor"
+        """
+        for line in output.split("\n"):
+            if "The size of the output buffer is" in line:
+                dump_tensor_size = int(line.split(" ")[-2])
+                continue
+
+            if "dump tensor to " in line:
+                dump_tensor_file = line.split(" ")[-1]
+                continue
+
+        return (dump_tensor_file, dump_tensor_size)
+
+    iwasm_output = execute_openvino_road_segmentation_adas_once(
+        iwasm_bin,
+        [
+            "--native-lib=/workspaces/wamr/product-mini/platforms/linux/build/libwasi-nn-openvino.so",
+            "--map-dir=.:.",
+        ],
+        cwd,
+    )
+    iwasm_tensor_file, iwasm_tensor_size = filter_output(iwasm_output)
+
+    wasmedge_output = execute_openvino_road_segmentation_adas_once(
+        wasmedge_bin, ["--dir=.:."], cwd
+    )
+    wasmedge_tensor_file, wasmedge_tensor_size = filter_output(wasmedge_output)
+
+    # TODO: binary compare?
+    if iwasm_tensor_size == wasmedge_tensor_size:
+        print(f"- openvino_road_segmentation_adas. PASS")
         return
 
-    print("- tflite_birds_v1_image. FAILED")
+    print(f"- openvino_road_segmentation_adas. FAILED")
     print("------------------------------------------------------------")
-    pprint(iwasm_output)
+    print(f"FILE:{iwasm_tensor_file}, SIZE:{iwasm_tensor_size}")
     print("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
-    pprint(wasmedge_output)
+    print(f"FILE:{wasmedge_tensor_file}, SIZE:{wasmedge_tensor_size}")
     print("------------------------------------------------------------")
 
 
-def execute_wasmedge_wasinn_exmaples(iwasm_bin: str, wasmedge_bin: str):
+def execute_wasmedge_wasinn_examples(iwasm_bin: str, wasmedge_bin: str):
     assert Path.cwd().name == "wasmedge-wasinn-examples"
     assert shutil.which(iwasm_bin)
     assert shutil.which(wasmedge_bin)
 
-    tflite_birds_v1_image_dir = Path.cwd().joinpath("./tflite-birds_v1-image")
+    # TODO: keep commenting until https://github.com/bytecodealliance/wasm-micro-runtime/pull/3597 is merged
-    execute_tflite_birds_v1_image(iwasm_bin, wasmedge_bin, tflite_birds_v1_image_dir)
+    # tflite_birds_v1_image_dir = Path.cwd().joinpath("./tflite-birds_v1-image")
+    # execute_tflite_birds_v1_image(iwasm_bin, wasmedge_bin, tflite_birds_v1_image_dir)
+
+    openvino_mobile_image_dir = Path.cwd().joinpath("./openvino-mobilenet-image")
+    execute_openvino_mobilenet_image(iwasm_bin, wasmedge_bin, openvino_mobile_image_dir)
+
+    openvino_mobile_raw_dir = Path.cwd().joinpath("./openvino-mobilenet-raw")
+    execute_openvino_mobilenet_raw(iwasm_bin, wasmedge_bin, openvino_mobile_raw_dir)
+
+    openvino_road_segmentation_adas_dir = Path.cwd().joinpath(
+        "./openvino-road-segmentation-adas"
+    )
+    execute_openvino_road_segmentation_adas(
+        iwasm_bin, wasmedge_bin, openvino_road_segmentation_adas_dir
+    )
 
 
 if __name__ == "__main__":
-    execute_wasmedge_wasinn_exmaples("iwasm", "wasmedge")
+    execute_wasmedge_wasinn_examples("iwasm", "wasmedge")