mirror of
https://github.com/bytecodealliance/wasm-micro-runtime.git
synced 2025-03-11 16:35:33 +00:00
Enhance workflows by caching llvm directories directly (#805)
Enhance workflows by caching llvm directories directly instead of caching the llvm-xxx.tar.gz packages, so as to reduce the time consumption of unpacking the cached LLVM packages.
This commit is contained in:
liang.he
GitHub
parent
51a00a4c2b
commit
3d5a0bff0f
|
|
@ -28,11 +28,11 @@ concurrency:
|
|||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
AOT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
|
||||
AOT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
|
||||
CLASSIC_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
|
||||
FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
|
||||
JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0"
|
||||
LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1"
|
||||
FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
|
||||
JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0"
|
||||
LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1"
|
||||
LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex"
|
||||
|
||||
jobs:
|
||||
|
|
@ -118,9 +118,13 @@ jobs:
|
|||
if: ${{ matrix.light == 'green' }}
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
|
||||
path: |
|
||||
./core/deps/llvm/build/bin
|
||||
./core/deps/llvm/build/include
|
||||
./core/deps/llvm/build/lib
|
||||
./core/deps/llvm/build/libexec
|
||||
./core/deps/llvm/build/share
|
||||
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
restore-keys: ${{ matrix.os }}-${{env.LLVM_CACHE_SUFFIX }}
|
||||
|
||||
- name: Build llvm and clang from source on ubuntu
|
||||
id: build_llvm_ubuntu
|
||||
|
|
@ -134,11 +138,6 @@ jobs:
|
|||
run: /usr/bin/env python3 ./build_llvm.py --arch X86 WebAssembly
|
||||
working-directory: build-scripts
|
||||
|
||||
- name: package LLVM
|
||||
if: ${{ matrix.light == 'green' && (steps.build_llvm_ubuntu.conclusion == 'success' || steps.build_llvm_macos.conclusion == 'success')}}
|
||||
run: mv LLVM-13.0.0-*.tar.gz LLVM-13.0.0.tar.gz
|
||||
working-directory: core/deps/llvm/build/
|
||||
|
||||
build_wamrc:
|
||||
needs: [build_llvm_libraries, check_repo]
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
|
@ -165,19 +164,18 @@ jobs:
|
|||
if: ${{ matrix.light == 'green' }}
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
|
||||
path: |
|
||||
./core/deps/llvm/build/bin
|
||||
./core/deps/llvm/build/include
|
||||
./core/deps/llvm/build/lib
|
||||
./core/deps/llvm/build/libexec
|
||||
./core/deps/llvm/build/share
|
||||
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
|
||||
- name: Quit if cache miss
|
||||
if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }}
|
||||
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
|
||||
|
||||
- name: Extract the LLVM package
|
||||
if: ${{ matrix.light == 'green' }}
|
||||
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
|
||||
working-directory: ./core/deps/llvm/build
|
||||
|
||||
- name: Build wamrc
|
||||
if: ${{ matrix.light == 'green' }}
|
||||
run: |
|
||||
|
|
@ -293,19 +291,18 @@ jobs:
|
|||
if: ${{ matrix.light == 'green' }}
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
|
||||
path: |
|
||||
./core/deps/llvm/build/bin
|
||||
./core/deps/llvm/build/include
|
||||
./core/deps/llvm/build/lib
|
||||
./core/deps/llvm/build/libexec
|
||||
./core/deps/llvm/build/share
|
||||
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
|
||||
- name: Quit if cache miss
|
||||
if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }}
|
||||
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
|
||||
|
||||
- name: Extract the LLVM package
|
||||
if: ${{ matrix.light == 'green' }}
|
||||
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
|
||||
working-directory: ./core/deps/llvm/build
|
||||
|
||||
- name: Build iwasm
|
||||
if: ${{ matrix.light == 'green' }}
|
||||
run: |
|
||||
|
|
@ -362,19 +359,18 @@ jobs:
|
|||
if: ${{ matrix.light == 'green' }}
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
|
||||
path: |
|
||||
./core/deps/llvm/build/bin
|
||||
./core/deps/llvm/build/include
|
||||
./core/deps/llvm/build/lib
|
||||
./core/deps/llvm/build/libexec
|
||||
./core/deps/llvm/build/share
|
||||
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
|
||||
- name: Quit if cache miss
|
||||
if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }}
|
||||
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
|
||||
|
||||
- name: Extract the LLVM package
|
||||
if: ${{ matrix.light == 'green' }}
|
||||
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
|
||||
working-directory: ./core/deps/llvm/build
|
||||
|
||||
- name: download and install wabt
|
||||
if: ${{ matrix.light == 'green' }}
|
||||
run: |
|
||||
|
|
@ -440,19 +436,18 @@ jobs:
|
|||
if: ${{ matrix.light == 'green' }}
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
|
||||
path: |
|
||||
./core/deps/llvm/build/bin
|
||||
./core/deps/llvm/build/include
|
||||
./core/deps/llvm/build/lib
|
||||
./core/deps/llvm/build/libexec
|
||||
./core/deps/llvm/build/share
|
||||
key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
|
||||
- name: Quit if cache miss
|
||||
if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }}
|
||||
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
|
||||
|
||||
- name: Extract the LLVM package
|
||||
if: ${{ matrix.light == 'green' }}
|
||||
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
|
||||
working-directory: ./core/deps/llvm/build
|
||||
|
||||
- name: download and install wasi-sdk
|
||||
if: ${{ matrix.light == 'green' }}
|
||||
run: |
|
||||
|
|
|
|||
78
.github/workflows/spec_test.yml
vendored
78
.github/workflows/spec_test.yml
vendored
|
|
@ -49,38 +49,12 @@ jobs:
|
|||
with:
|
||||
access_token: ${{ github.token }}
|
||||
|
||||
build_llvm_libraries:
|
||||
needs: cancel_previous
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Cache LLVM libraries
|
||||
id: cache_llvm
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
|
||||
key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
|
||||
- name: Build llvm and clang from source
|
||||
id: build_llvm
|
||||
if: ${{ steps.cache_llvm.outputs.cache-hit != 'true' }}
|
||||
run: /usr/bin/env python3 ./build_llvm.py --arch X86 WebAssembly
|
||||
working-directory: build-scripts
|
||||
|
||||
- name: package LLVM
|
||||
if: ${{ steps.build_llvm.conclusion == 'success' }}
|
||||
run: mv LLVM-13.0.0-*.tar.gz LLVM-13.0.0.tar.gz
|
||||
working-directory: core/deps/llvm/build/
|
||||
|
||||
spec_test_default:
|
||||
needs: build_llvm_libraries
|
||||
needs: cancel_previous
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
test_option: [ $DEFAULT_TEST_OPTIONS, $SIMD_TEST_OPTIONS ]
|
||||
test_option: [$DEFAULT_TEST_OPTIONS, $SIMD_TEST_OPTIONS]
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
|
|
@ -89,17 +63,13 @@ jobs:
|
|||
id: cache_llvm
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
|
||||
path: |
|
||||
./core/deps/llvm/build/bin
|
||||
./core/deps/llvm/build/include
|
||||
./core/deps/llvm/build/lib
|
||||
./core/deps/llvm/build/libexec
|
||||
./core/deps/llvm/build/share
|
||||
key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
|
||||
- name: Quit if cache miss
|
||||
if: steps.cache_llvm.outputs.cache-hit != 'true'
|
||||
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
|
||||
|
||||
- name: Extract the LLVM package
|
||||
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
|
||||
working-directory: ./core/deps/llvm/build
|
||||
|
||||
- name: install Ninja
|
||||
run: sudo apt install -y ninja-build
|
||||
|
|
@ -109,12 +79,12 @@ jobs:
|
|||
working-directory: ./tests/wamr-test-suites
|
||||
|
||||
spec_test_extra:
|
||||
needs: build_llvm_libraries
|
||||
needs: cancel_previous
|
||||
if: ${{ endsWith(github.repository, 'wasm-micro-runtime') }}
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
test_option: [ $MULTI_MODULES_TEST_OPTIONS, $THREADS_TEST_OPTIONS ]
|
||||
test_option: [$MULTI_MODULES_TEST_OPTIONS, $THREADS_TEST_OPTIONS]
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
|
|
@ -123,18 +93,18 @@ jobs:
|
|||
id: cache_llvm
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
|
||||
path: |
|
||||
./core/deps/llvm/build/bin
|
||||
./core/deps/llvm/build/include
|
||||
./core/deps/llvm/build/lib
|
||||
./core/deps/llvm/build/libexec
|
||||
./core/deps/llvm/build/share
|
||||
key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
|
||||
- name: Quit if cache miss
|
||||
if: steps.cache_llvm.outputs.cache-hit != 'true'
|
||||
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
|
||||
|
||||
- name: Extract the LLVM package
|
||||
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
|
||||
working-directory: ./core/deps/llvm/build
|
||||
|
||||
- name: install Ninja
|
||||
run: sudo apt install -y ninja-build
|
||||
|
||||
|
|
@ -144,11 +114,11 @@ jobs:
|
|||
|
||||
spec_test_x86_32:
|
||||
if: ${{ endsWith(github.repository, 'wasm-micro-runtime') }}
|
||||
needs: build_llvm_libraries
|
||||
needs: cancel_previous
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
test_option: [ $DEFAULT_TEST_OPTIONS, $THREADS_TEST_OPTIONS ]
|
||||
test_option: [$DEFAULT_TEST_OPTIONS, $THREADS_TEST_OPTIONS]
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
|
|
@ -157,18 +127,18 @@ jobs:
|
|||
id: cache_llvm
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
|
||||
path: |
|
||||
./core/deps/llvm/build/bin
|
||||
./core/deps/llvm/build/include
|
||||
./core/deps/llvm/build/lib
|
||||
./core/deps/llvm/build/libexec
|
||||
./core/deps/llvm/build/share
|
||||
key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
|
||||
|
||||
- name: Quit if cache miss
|
||||
if: steps.cache_llvm.outputs.cache-hit != 'true'
|
||||
run: echo "::error::can not get prebuilt llvm libraries" && exit 1
|
||||
|
||||
- name: Extract the LLVM package
|
||||
run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
|
||||
working-directory: ./core/deps/llvm/build
|
||||
|
||||
- name: install Ninja and x32 support libraries
|
||||
run: sudo apt install -y g++-multilib libgcc-9-dev lib32gcc-9-dev ninja-build
|
||||
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ def build_llvm(llvm_dir, platform, backends, projects):
|
|||
lib_llvm_core_library = build_dir.joinpath("lib/libLLVMCore.a").resolve()
|
||||
if lib_llvm_core_library.exists():
|
||||
print(f"Please remove {build_dir} manually and try again")
|
||||
return
|
||||
return build_dir
|
||||
|
||||
compile_options = " ".join(
|
||||
LLVM_COMPILE_OPTIONS
|
||||
|
|
@ -126,6 +126,28 @@ def build_llvm(llvm_dir, platform, backends, projects):
|
|||
return build_dir
|
||||
|
||||
|
||||
def repackage_llvm(llvm_dir):
|
||||
build_dir = llvm_dir.joinpath("./build").resolve()
|
||||
|
||||
packs = [f for f in build_dir.glob("LLVM-13*.tar.gz")]
|
||||
if len(packs) > 1:
|
||||
raise Exception("Find more than one LLVM-13*.tar.gz")
|
||||
|
||||
if not packs:
|
||||
return
|
||||
|
||||
llvm_package = packs[0].name
|
||||
# mv build/LLVM-13.0.0*.gz .
|
||||
shutil.move(str(build_dir.joinpath(llvm_package).resolve()), str(llvm_dir))
|
||||
# rm -r build
|
||||
shutil.rmtree(str(build_dir))
|
||||
# mkdir build
|
||||
build_dir.mkdir()
|
||||
# tar xf ./LLVM-13.0.0-*.tar.gz --strip-components=1 --directory=build
|
||||
CMD = f"tar xf {llvm_dir.joinpath(llvm_package).resolve()} --strip-components=1 --directory={build_dir}"
|
||||
subprocess.check_call(shlex.split(CMD), cwd=llvm_dir)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="build necessary LLVM libraries")
|
||||
parser.add_argument(
|
||||
|
|
@ -198,16 +220,24 @@ def main():
|
|||
current_dir = current_file.parent.resolve()
|
||||
deps_dir = current_dir.joinpath("../core/deps").resolve()
|
||||
|
||||
print(f"==================== CLONE LLVM ====================")
|
||||
llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"])
|
||||
llvm_dir = clone_llvm(deps_dir, llvm_info["repo"], llvm_info["branch"])
|
||||
try:
|
||||
print(f"==================== CLONE LLVM ====================")
|
||||
llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"])
|
||||
llvm_dir = clone_llvm(deps_dir, llvm_info["repo"], llvm_info["branch"])
|
||||
|
||||
print()
|
||||
print(f"==================== BUILD LLVM ====================")
|
||||
build_llvm(llvm_dir, platform, options.arch, options.project)
|
||||
print()
|
||||
print(f"==================== BUILD LLVM ====================")
|
||||
build_llvm(llvm_dir, platform, options.arch, options.project)
|
||||
|
||||
print()
|
||||
print()
|
||||
print(f"==================== PACKAGE LLVM ====================")
|
||||
repackage_llvm(llvm_dir)
|
||||
|
||||
print()
|
||||
return True
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
sys.exit(0 if main() else 1)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user