mirror of
https://github.com/bytecodealliance/wasm-micro-runtime.git
synced 2025-06-18 02:59:21 +00:00
7be0d385a6
Implement the latest SIMD opcodes and update LLVM 13.0, update the llvm build scripts, update the sample workloads‘ build scripts, and build customized wasi-sdk to build some workloads. Also refine the CI rules. Signed-off-by: Wenyong Huang <wenyong.huang@intel.com>
300 lines
9.6 KiB
Python
Executable File
300 lines
9.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (C) 2019 Intel Corporation. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
#
|
|
|
|
"""
|
|
The script operates on such directories and files
|
|
|-- core
|
|
| `-- deps
|
|
| |-- emscripten
|
|
| `-- wasi-sdk
|
|
| `-- src
|
|
| |-- llvm-project
|
|
| `-- wasi-libc
|
|
`-- test-tools
|
|
|-- build-wasi-sdk
|
|
| |-- build_wasi_sdk.py
|
|
| |-- include
|
|
| `-- patches
|
|
`-- wasi-sdk
|
|
|-- bin
|
|
|-- lib
|
|
`-- share
|
|
`-- wasi-sysroot
|
|
"""
|
|
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import pathlib
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import urllib
|
|
import urllib.request
|
|
|
|
logger = logging.getLogger("build_wasi_sdk")
|
|
|
|
external_repos = {
|
|
"config": {
|
|
"sha256": "302e5e7f3c4996976c58efde8b2f28f71d51357e784330eeed738e129300dc33",
|
|
"store_dir": "core/deps/wasi-sdk/src/config",
|
|
"strip_prefix": "config-191bcb948f7191c36eefe634336f5fc5c0c4c2be",
|
|
"url": "https://git.savannah.gnu.org/cgit/config.git/snapshot/config-191bcb948f7191c36eefe634336f5fc5c0c4c2be.tar.gz",
|
|
},
|
|
"emscripten": {
|
|
"sha256": "0904a65379aea3ea94087b8c12985b2fee48599b473e3bef914fec2e3941532d",
|
|
"store_dir": "core/deps/emscripten",
|
|
"strip_prefix": "emscripten-2.0.28",
|
|
"url": "https://github.com/emscripten-core/emscripten/archive/refs/tags/2.0.28.tar.gz",
|
|
},
|
|
"llvm-project": {
|
|
"sha256": "dc5169e51919f2817d06615285e9da6a804f0f881dc55d6247baa25aed3cc143",
|
|
"store_dir": "core/deps/wasi-sdk/src/llvm-project",
|
|
"strip_prefix": "llvm-project-34ff6a75f58377f32a5046a29f55c4c0e58bee9e",
|
|
"url": "https://github.com/llvm/llvm-project/archive/34ff6a75f58377f32a5046a29f55c4c0e58bee9e.tar.gz",
|
|
},
|
|
"wasi-sdk": {
|
|
"sha256": "fc4fdb0e97b915241f32209492a7d0fab42c24216f87c1d5d75f46f7c70a553d",
|
|
"store_dir": "core/deps/wasi-sdk",
|
|
"strip_prefix": "wasi-sdk-1a953299860bbcc198ad8c12a21d1b2e2f738355",
|
|
"url": "https://github.com/WebAssembly/wasi-sdk/archive/1a953299860bbcc198ad8c12a21d1b2e2f738355.tar.gz",
|
|
},
|
|
"wasi-libc": {
|
|
"sha256": "f6316ca9479d3463eb1c4f6a1d1f659bf15f67cb3c1e2e83d9d11f188dccd864",
|
|
"store_dir": "core/deps/wasi-sdk/src/wasi-libc",
|
|
"strip_prefix": "wasi-libc-a78cd329aec717f149934d7362f57050c9401f60",
|
|
"url": "https://github.com/WebAssembly/wasi-libc/archive/a78cd329aec717f149934d7362f57050c9401f60.tar.gz",
|
|
},
|
|
}
|
|
|
|
# TOOD: can we use headers from wasi-libc and clang directly ?
|
|
emscripten_headers_src_dst = [
|
|
("include/compat/emmintrin.h", "sse/emmintrin.h"),
|
|
("include/compat/immintrin.h", "sse/immintrin.h"),
|
|
("include/compat/smmintrin.h", "sse/smmintrin.h"),
|
|
("include/compat/xmmintrin.h", "sse/xmmintrin.h"),
|
|
("lib/libc/musl/include/pthread.h", "libc/musl/pthread.h"),
|
|
("lib/libc/musl/include/signal.h", "libc/musl/signal.h"),
|
|
("lib/libc/musl/include/netdb.h", "libc/musl/netdb.h"),
|
|
("lib/libc/musl/include/sys/wait.h", "libc/musl/sys/wait.h"),
|
|
("lib/libc/musl/include/sys/socket.h", "libc/musl/sys/socket.h"),
|
|
("lib/libc/musl/include/setjmp.h", "libc/musl/setjmp.h"),
|
|
("lib/libc/musl/arch/emscripten/bits/setjmp.h", "libc/musl/bits/setjmp.h"),
|
|
]
|
|
|
|
|
|
def checksum(name, local_file):
|
|
sha256 = hashlib.sha256()
|
|
with open(local_file, "rb") as f:
|
|
bytes = f.read(4096)
|
|
while bytes:
|
|
sha256.update(bytes)
|
|
bytes = f.read(4096)
|
|
|
|
return sha256.hexdigest() == external_repos[name]["sha256"]
|
|
|
|
|
|
def download(url, local_file):
|
|
logger.debug(f"download from {url}")
|
|
urllib.request.urlretrieve(url, local_file)
|
|
return local_file.exists()
|
|
|
|
|
|
def unpack(tar_file, strip_prefix, dest_dir):
|
|
# extract .tar.gz to /tmp, then move back without strippred prefix directories
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
with tarfile.open(tar_file) as tar:
|
|
logger.debug(f"extract to {tmp}")
|
|
tar.extractall(tmp)
|
|
|
|
strip_prefix_dir = (
|
|
pathlib.Path(tmp).joinpath(strip_prefix + os.path.sep).resolve()
|
|
)
|
|
if not strip_prefix_dir.exists():
|
|
logger.error(f"extract {tar_file.name} failed")
|
|
return False
|
|
|
|
# mv /tmp/${strip_prefix} dest_dir/*
|
|
logger.debug(f"move {strip_prefix_dir} to {dest_dir}")
|
|
shutil.copytree(
|
|
str(strip_prefix_dir),
|
|
str(dest_dir),
|
|
copy_function=shutil.move,
|
|
dirs_exist_ok=True,
|
|
)
|
|
|
|
return True
|
|
|
|
|
|
def download_repo(name, root):
|
|
if not name in external_repos:
|
|
logger.error(f"{name} is not a known repository")
|
|
return False
|
|
|
|
store_dir = root.joinpath(f'{external_repos[name]["store_dir"]}').resolve()
|
|
download_flag = store_dir.joinpath("DOWNLOADED")
|
|
if store_dir.exists() and download_flag.exists():
|
|
logger.info(
|
|
f"keep using '{store_dir.relative_to(root)}'. Or to remove it and try again"
|
|
)
|
|
return True
|
|
|
|
# download only when the target is neither existed nor broken
|
|
download_dir = pathlib.Path("/tmp/build_wasi_sdk/")
|
|
download_dir.mkdir(exist_ok=True)
|
|
|
|
tar_name = pathlib.Path(external_repos[name]["url"]).name
|
|
tar_file = download_dir.joinpath(tar_name)
|
|
if tar_file.exists():
|
|
if checksum(name, tar_file):
|
|
logger.debug(f"use pre-downloaded {tar_file}")
|
|
else:
|
|
logger.debug(f"{tar_file} is broken, remove it")
|
|
tar_file.unlink()
|
|
|
|
if not tar_file.exists():
|
|
if not download(external_repos[name]["url"], tar_file) or not checksum(
|
|
name, tar_file
|
|
):
|
|
logger.error(f"download {name} failed")
|
|
return False
|
|
|
|
# unpack and removing *strip_prefix*
|
|
if not unpack(tar_file, external_repos[name]["strip_prefix"], store_dir):
|
|
return False
|
|
|
|
# leave a FLAG
|
|
download_flag.touch()
|
|
|
|
# leave download files in /tmp
|
|
return True
|
|
|
|
|
|
def run_patch(patch_file, cwd):
|
|
if not patch_file.exists():
|
|
logger.error(f"{patch_file} not found")
|
|
return False
|
|
|
|
with open(patch_file, "r") as f:
|
|
try:
|
|
PATCH_DRY_RUN_CMD = "patch -f -p1 --dry-run"
|
|
if subprocess.check_call(shlex.split(PATCH_DRY_RUN_CMD), stdin=f, cwd=cwd):
|
|
logger.error(f"patch dry-run {cwd} failed")
|
|
return False
|
|
|
|
PATCH_CMD = "patch -f -p1"
|
|
f.seek(0)
|
|
if subprocess.check_call(shlex.split(PATCH_CMD), stdin=f, cwd=cwd):
|
|
logger.error(f"patch {cwd} failed")
|
|
return False
|
|
except subprocess.CalledProcessError:
|
|
logger.error(f"patch {cwd} failed")
|
|
return False
|
|
return True
|
|
|
|
|
|
def build_and_install_wasi_sdk(root):
|
|
store_dir = root.joinpath(f'{external_repos["wasi-sdk"]["store_dir"]}').resolve()
|
|
if not store_dir.exists():
|
|
logger.error(f"{store_dir} does not found")
|
|
return False
|
|
|
|
# patch wasi-libc and wasi-sdk
|
|
patch_flag = store_dir.joinpath("PATCHED")
|
|
if not patch_flag.exists():
|
|
if not run_patch(
|
|
root.joinpath("test-tools/build-wasi-sdk/patches/wasi_libc.patch"),
|
|
store_dir.joinpath("src/wasi-libc"),
|
|
):
|
|
return False
|
|
|
|
if not run_patch(
|
|
root.joinpath("test-tools/build-wasi-sdk/patches/wasi_sdk.patch"), store_dir
|
|
):
|
|
return False
|
|
|
|
patch_flag.touch()
|
|
else:
|
|
logger.info("bypass the patch phase")
|
|
|
|
# build
|
|
build_flag = store_dir.joinpath("BUILDED")
|
|
if not build_flag.exists():
|
|
BUILD_CMD = "make build"
|
|
if subprocess.check_call(shlex.split(BUILD_CMD), cwd=store_dir):
|
|
logger.error(f"build wasi-sdk failed")
|
|
return False
|
|
|
|
build_flag.touch()
|
|
else:
|
|
logger.info("bypass the build phase")
|
|
|
|
# install
|
|
install_flag = store_dir.joinpath("INSTALLED")
|
|
binary_path = root.joinpath("test-tools").resolve()
|
|
if not install_flag.exists():
|
|
shutil.copytree(
|
|
str(store_dir.joinpath("build/install/opt").resolve()),
|
|
str(binary_path),
|
|
dirs_exist_ok=True,
|
|
)
|
|
|
|
# install headers
|
|
emscripten_headers = (
|
|
root.joinpath(external_repos["emscripten"]["store_dir"])
|
|
.joinpath("system")
|
|
.resolve()
|
|
)
|
|
wasi_sysroot_headers = binary_path.joinpath(
|
|
"wasi-sdk/share/wasi-sysroot/include"
|
|
).resolve()
|
|
for (src, dst) in emscripten_headers_src_dst:
|
|
src = emscripten_headers.joinpath(src)
|
|
dst = wasi_sysroot_headers.joinpath(dst)
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy(src, dst)
|
|
|
|
install_flag.touch()
|
|
else:
|
|
logger.info("bypass the install phase")
|
|
|
|
return True
|
|
|
|
|
|
def main():
|
|
console = logging.StreamHandler()
|
|
console.setFormatter(logging.Formatter("%(asctime)s - %(message)s"))
|
|
logger.setLevel(logging.INFO)
|
|
logger.addHandler(console)
|
|
logger.propagate = False
|
|
|
|
# locate the root of WAMR
|
|
current_file = pathlib.Path(__file__)
|
|
if current_file.is_symlink():
|
|
current_file = pathlib.Path(os.readlink(current_file))
|
|
root = current_file.parent.joinpath("../..").resolve()
|
|
logger.info(f"The root of WAMR is {root}")
|
|
|
|
# download repos
|
|
for repo in external_repos.keys():
|
|
if not download_repo(repo, root):
|
|
return False
|
|
|
|
# build wasi_sdk and install
|
|
if not build_and_install_wasi_sdk(root):
|
|
return False
|
|
|
|
# TODO install headers from emscripten
|
|
|
|
return True
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(0 if main() else 1)
|