wasi-nn: Add wasmedge-wasinn-example as smoke test (#3554)

This commit is contained in:
liang.he 2024-06-24 12:03:08 +08:00 committed by GitHub
parent cfffb62ad2
commit d36160b294
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 190 additions and 35 deletions

16
.dockerignore Normal file
View File

@ -0,0 +1,16 @@
# for now, it is used to speed up wasi-nn tests only.
# you shall adapt below rules to incoming requirements
build
*/build
*/*/build
*/*/*/build
*/*/*/*/build
*/*/*/*/*/build
*/*/*/*/*/*/build
.*
core/deps
!core/deps/tensorflow-src
samples
tests

View File

@ -147,39 +147,35 @@ Supported:
## Smoke test
Use [classification-example](https://github.com/bytecodealliance/wasi-nn/tree/main/rust/examples/classification-example) as a smoke test case to make sure the wasi-nn support in WAMR is working properly.
### Testing with WasmEdge-WASINN Examples
> [!Important]
> It requires openvino.
To ensure everything is set up correctly, use the examples from [WasmEdge-WASINN-examples](https://github.com/second-state/WasmEdge-WASINN-examples/tree/master). These examples help verify that WASI-NN support in WAMR is functioning as expected.
### Prepare the model and the wasm
> Note: The repository contains two types of examples. Some use the [standard wasi-nn](https://github.com/WebAssembly/wasi-nn), while others use [WasmEdge's version of wasi-nn](https://github.com/second-state/wasmedge-wasi-nn), which is enhanced to meet specific customer needs.
The examples test the following machine learning backends:
- OpenVINO
- PyTorch
- TensorFlow Lite
Due to the different requirements of each backend, we'll use a Docker container for a hassle-free testing environment.
#### Prepare the execution environment
```bash
$ pwd
/workspaces/wasm-micro-runtime/core/iwasm/libraries/wasi-nn/test
/workspaces/wasm-micro-runtime/
$ docker build -t wasi-nn-example:v1.0 -f Dockerfile.wasi-nn-example .
$ docker build -t wasi-nn-smoke:v1.0 -f Dockerfile.wasi-nn-smoke .
```
There are model files(\*mobilenet\**) and wasm files(*wasi-nn-example.wasm*) in the directory */workspaces/wasi-nn/rust/examples/classification-example/build\* in the image of wasi-nn-example:v1.0.
### build iwasm and test
_TODO: May need alternative steps to build the iwasm and test in the container of wasi-nn-example:v1.0_
#### Execute
```bash
$ pwd
/workspaces/wasm-micro-runtime
$ docker run --rm -it -v $(pwd):/workspaces/wasm-micro-runtime wasi-nn-example:v1.0 /bin/bash
$ docker run --rm wasi-nn-smoke:v1.0
```
> [!Caution]
> The following steps are executed in the container of wasi-nn-example:v1.0.
### Testing with bytecodealliance wasi-nn
```bash
$ cd /workspaces/wasm-micro-runtime/product-mini/platforms/linux
$ cmake -S . -B build -DWAMR_BUILD_WASI_NN=1 -DWAMR_BUILD_WASI_EPHEMERAL_NN=1
$ cmake --build build
$ ./build/iwasm -v=5 --map-dir=/workspaces/wasi-nn/rust/examples/classification-example/build/::fixture /workspaces/wasi-nn/rust/examples/classification-example/build/wasi-nn-example.wasm
```
For another example, check out [classification-example](https://github.com/bytecodealliance/wasi-nn/tree/main/rust/examples/classification-example), which focuses on OpenVINO. You can run it using the same Docker container mentioned above.

View File

@ -1,7 +1,7 @@
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
FROM mcr.microsoft.com/devcontainers/rust:1-1-bullseye
FROM mcr.microsoft.com/devcontainers/rust:1-1-bullseye@sha256:ddc1ee022d327f024c07484c9333db3fbbfd504bc096cdb66635653a2bebb33e
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=Asian/Shanghai
@ -10,10 +10,6 @@ ENV TZ=Asian/Shanghai
RUN apt-get update \
&& apt-get upgrade -y
#
# Rust targets
RUN rustup target add wasm32-wasi wasm32-unknown-unknown
#
# Openvino
# Refer to
@ -42,16 +38,43 @@ RUN tar -xf wasmtime-v21.0.0-x86_64-linux.tar.xz \
#
# wasi-nn
# compilation requirements
RUN rustup target add wasm32-wasi wasm32-unknown-unknown
WORKDIR /workspaces/wasi-nn
RUN git clone --depth 1 https://github.com/bytecodealliance/wasi-nn.git .
# hadolint ignore=DL3059
RUN ./build.sh rust
#RUN ./build.sh rust
# There are model files(mobilenet*) and wasm files(wasi-nn-example.wasm) in the directory,
# /workspaces/wasi-nn/rust/examples/classification-example/build
RUN apt-get autoremove -y \
&& apt-get clean -y \
&& rm -rf /tmp/*
#
# wasmedge
WORKDIR /tmp
RUN wget -q https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh \
&& chmod a+x ./install.sh
RUN ./install.sh -p /opt/wasmedge --plugins wasi_nn-tensorflowlite
ENV PATH=/opt/wasmedge/bin:${PATH}
ENV WASMEDGE_LIB_DIR=/opt/wasmedge/lib
WORKDIR /workspaces
#
# wasmedge-wasinn-examples
WORKDIR /workspaces/wasmedge-wasinn-examples
RUN git clone --depth 1 https://github.com/second-state/WasmEdge-WASINN-examples.git .
#
# iwasm. build from source
WORKDIR /workspaces/wamr
COPY . .
WORKDIR /workspaces/wamr/product-mini/platforms/linux
RUN cmake -S . -B build -DWAMR_BUILD_WASI_NN=1 -DWAMR_BUILD_WASI_EPHEMERAL_NN=1 \
&& cmake --build build
RUN ln -sf "$(realpath ./build/iwasm)" /usr/local/bin/iwasm
#
# add smoke test script
COPY core/iwasm/libraries/wasi-nn/test/run_smoke_test.py /
#
WORKDIR /workspaces/wasmedge-wasinn-examples
CMD ["python3", "/run_smoke_test.py"]

View File

@ -0,0 +1,120 @@
#!/usr/bin/env python3
#
# Copyright (C) 2019 Intel Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
from pathlib import Path
from pprint import pprint
import re
import shlex
import shutil
import subprocess
from typing import List
def execute_tflite_birds_v1_image_once(
runtime_bin: str, runtime_args: List[str], cwd: Path
) -> str:
"""
execute tflite_birds_v1_image example with
```
iwasm --native-lib=somewhere/libwasi-nn-tflite.so --map-dir=.:. \
./wasmedge-wasinn-example-tflite-bird-image.wasm \
lite-model_aiy_vision_classifier_birds_V1_3.tflite \
bird.jpg
```
or
```
wasmedge --dir=.:. \
./wasmedge-wasinn-example-tflite-bird-image.wasm \
lite-model_aiy_vision_classifier_birds_V1_3.tflite \
bird.jpg
```
assumption:
- under the right directory, tflite-birds_v1-image
- every materials are ready
"""
wasm_file = "./wasmedge-wasinn-example-tflite-bird-image.wasm"
wasm_args = ["lite-model_aiy_vision_classifier_birds_V1_3.tflite", "bird.jpg"]
cmd = [runtime_bin]
cmd.extend(runtime_args)
cmd.append(wasm_file)
cmd.extend(wasm_args)
try:
p = subprocess.run(
cmd,
cwd=cwd,
capture_output=True,
check=True,
text=True,
universal_newlines=True,
)
return p.stdout
except subprocess.CalledProcessError as e:
print(e.stderr)
print()
print(e.stdout)
def filter_output_tflite_birds_v1_image(output: str) -> List[str]:
"""
not all output is needed for comparision
pick lines like: " 1.) [526](136)Cathartes burrovianus"
"""
filtered = []
PATTERN = re.compile(r"^\s+\d\.\)\s+\[\d+\]\(\d+\)\w+")
for line in output.split("\n"):
if PATTERN.search(line):
filtered.append(line.strip())
return filtered
def execute_tflite_birds_v1_image(iwasm_bin: str, wasmedge_bin: str, cwd: Path):
iwasm_output = execute_tflite_birds_v1_image_once(
iwasm_bin,
[
"--native-lib=/workspaces/wamr/product-mini/platforms/linux/build/libwasi-nn-tflite.so",
"--map-dir=.:.",
],
cwd,
)
iwasm_output = filter_output_tflite_birds_v1_image(iwasm_output)
wasmedge_output = execute_tflite_birds_v1_image_once(
wasmedge_bin, ["--dir=.:."], cwd
)
wasmedge_output = filter_output_tflite_birds_v1_image(wasmedge_output)
if iwasm_output == wasmedge_output:
print("- tflite_birds_v1_image. PASS")
return
print("- tflite_birds_v1_image. FAILED")
print("------------------------------------------------------------")
pprint(iwasm_output)
print("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
pprint(wasmedge_output)
print("------------------------------------------------------------")
def execute_wasmedge_wasinn_exmaples(iwasm_bin: str, wasmedge_bin: str):
assert Path.cwd().name == "wasmedge-wasinn-examples"
assert shutil.which(iwasm_bin)
assert shutil.which(wasmedge_bin)
tflite_birds_v1_image_dir = Path.cwd().joinpath("./tflite-birds_v1-image")
execute_tflite_birds_v1_image(iwasm_bin, wasmedge_bin, tflite_birds_v1_image_dir)
if __name__ == "__main__":
execute_wasmedge_wasinn_exmaples("iwasm", "wasmedge")