Enhance workflows by caching llvm directories directly (#805)

Enhance workflows by caching llvm directories directly instead of
caching the llvm-xxx.tar.gz packages, so as to reduce the time
consumption of unpacking the cached LLVM packages.

.github/workflows/compilation_on_android_ubuntu_macos.yml

@@ -28,11 +28,11 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  AOT_BUILD_OPTIONS:            "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
+  AOT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
   CLASSIC_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
-  FAST_INTERP_BUILD_OPTIONS:    "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
+  FAST_INTERP_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=0 -DWAMR_BUILD_FAST_INTERP=1 -DWAMR_BUILD_INTERP=1 -DWAMR_BUILD_JIT=0 -DWAMR_BUILD_LAZY_JIT=0"
-  JIT_BUILD_OPTIONS:            "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0"
+  JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=0"
-  LAZY_JIT_BUILD_OPTIONS:       "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1"
+  LAZY_JIT_BUILD_OPTIONS: "-DWAMR_BUILD_AOT=1 -DWAMR_BUILD_FAST_INTERP=0 -DWAMR_BUILD_INTERP=0 -DWAMR_BUILD_JIT=1 -DWAMR_BUILD_LAZY_JIT=1"
   LLVM_CACHE_SUFFIX: "build-llvm_libraries_ex"
 
 jobs:
@@ -118,9 +118,13 @@ jobs:
         if: ${{ matrix.light == 'green' }}
         uses: actions/cache@v2
         with:
-          path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
+          path: |
+            ./core/deps/llvm/build/bin
+            ./core/deps/llvm/build/include
+            ./core/deps/llvm/build/lib
+            ./core/deps/llvm/build/libexec
+            ./core/deps/llvm/build/share
           key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
-          restore-keys: ${{ matrix.os }}-${{env.LLVM_CACHE_SUFFIX }}
 
       - name: Build llvm and clang from source on ubuntu
         id: build_llvm_ubuntu
@@ -134,11 +138,6 @@ jobs:
         run: /usr/bin/env python3 ./build_llvm.py --arch X86 WebAssembly
         working-directory: build-scripts
 
-      - name: package LLVM
-        if: ${{ matrix.light == 'green' && (steps.build_llvm_ubuntu.conclusion == 'success' || steps.build_llvm_macos.conclusion == 'success')}}
-        run: mv LLVM-13.0.0-*.tar.gz LLVM-13.0.0.tar.gz
-        working-directory: core/deps/llvm/build/
-
   build_wamrc:
     needs: [build_llvm_libraries, check_repo]
     runs-on: ${{ matrix.os }}
@@ -165,19 +164,18 @@ jobs:
         if: ${{ matrix.light == 'green' }}
         uses: actions/cache@v2
         with:
-          path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
+          path: |
+            ./core/deps/llvm/build/bin
+            ./core/deps/llvm/build/include
+            ./core/deps/llvm/build/lib
+            ./core/deps/llvm/build/libexec
+            ./core/deps/llvm/build/share
           key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
-          restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
 
       - name: Quit if cache miss
         if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }}
         run: echo "::error::can not get prebuilt llvm libraries" && exit 1
 
-      - name: Extract the LLVM package
-        if: ${{ matrix.light == 'green' }}
-        run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
-        working-directory: ./core/deps/llvm/build
-
       - name: Build wamrc
         if: ${{ matrix.light == 'green' }}
         run: |
@@ -293,19 +291,18 @@ jobs:
         if: ${{ matrix.light == 'green' }}
         uses: actions/cache@v2
         with:
-          path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
+          path: |
+            ./core/deps/llvm/build/bin
+            ./core/deps/llvm/build/include
+            ./core/deps/llvm/build/lib
+            ./core/deps/llvm/build/libexec
+            ./core/deps/llvm/build/share
           key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
-          restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
 
       - name: Quit if cache miss
         if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }}
         run: echo "::error::can not get prebuilt llvm libraries" && exit 1
 
-      - name: Extract the LLVM package
-        if: ${{ matrix.light == 'green' }}
-        run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
-        working-directory: ./core/deps/llvm/build
-
       - name: Build iwasm
         if: ${{ matrix.light == 'green' }}
         run: |
@@ -362,19 +359,18 @@ jobs:
         if: ${{ matrix.light == 'green' }}
         uses: actions/cache@v2
         with:
-          path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
+          path: |
+            ./core/deps/llvm/build/bin
+            ./core/deps/llvm/build/include
+            ./core/deps/llvm/build/lib
+            ./core/deps/llvm/build/libexec
+            ./core/deps/llvm/build/share
           key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
-          restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
 
       - name: Quit if cache miss
         if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }}
         run: echo "::error::can not get prebuilt llvm libraries" && exit 1
 
-      - name: Extract the LLVM package
-        if: ${{ matrix.light == 'green' }}
-        run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
-        working-directory: ./core/deps/llvm/build
-
       - name: download and install wabt
         if: ${{ matrix.light == 'green' }}
         run: |
@@ -440,19 +436,18 @@ jobs:
         if: ${{ matrix.light == 'green' }}
         uses: actions/cache@v2
         with:
-          path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
+          path: |
+            ./core/deps/llvm/build/bin
+            ./core/deps/llvm/build/include
+            ./core/deps/llvm/build/lib
+            ./core/deps/llvm/build/libexec
+            ./core/deps/llvm/build/share
           key: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
-          restore-keys: ${{ matrix.os }}-${{ env.LLVM_CACHE_SUFFIX }}
 
       - name: Quit if cache miss
         if: ${{ matrix.light == 'green' && steps.cache_llvm.outputs.cache-hit != 'true' }}
         run: echo "::error::can not get prebuilt llvm libraries" && exit 1
 
-      - name: Extract the LLVM package
-        if: ${{ matrix.light == 'green' }}
-        run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
-        working-directory: ./core/deps/llvm/build
-
       - name: download and install wasi-sdk
         if: ${{ matrix.light == 'green' }}
         run: |

.github/workflows/spec_test.yml

@@ -49,38 +49,12 @@ jobs:
         with:
           access_token: ${{ github.token }}
 
-  build_llvm_libraries:
-    needs: cancel_previous
-    runs-on: ubuntu-20.04
-    steps:
-      - name: checkout
-        uses: actions/checkout@v2
-
-      - name: Cache LLVM libraries
-        id: cache_llvm
-        uses: actions/cache@v2
-        with:
-          path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
-          key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
-          restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
-
-      - name: Build llvm and clang from source
-        id: build_llvm
-        if: ${{ steps.cache_llvm.outputs.cache-hit != 'true' }}
-        run: /usr/bin/env python3 ./build_llvm.py  --arch X86 WebAssembly
-        working-directory: build-scripts
-
-      - name: package LLVM
-        if: ${{ steps.build_llvm.conclusion == 'success' }}
-        run: mv LLVM-13.0.0-*.tar.gz LLVM-13.0.0.tar.gz
-        working-directory: core/deps/llvm/build/
-
   spec_test_default:
-    needs: build_llvm_libraries
+    needs: cancel_previous
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        test_option: [ $DEFAULT_TEST_OPTIONS, $SIMD_TEST_OPTIONS ]
+        test_option: [$DEFAULT_TEST_OPTIONS, $SIMD_TEST_OPTIONS]
     steps:
       - name: checkout
         uses: actions/checkout@v2
@@ -89,17 +63,13 @@ jobs:
         id: cache_llvm
         uses: actions/cache@v2
         with:
-          path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
+          path: |
+            ./core/deps/llvm/build/bin
+            ./core/deps/llvm/build/include
+            ./core/deps/llvm/build/lib
+            ./core/deps/llvm/build/libexec
+            ./core/deps/llvm/build/share
           key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
-          restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
-
-      - name: Quit if cache miss
-        if: steps.cache_llvm.outputs.cache-hit != 'true'
-        run: echo "::error::can not get prebuilt llvm libraries" && exit 1
-
-      - name: Extract the LLVM package
-        run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
-        working-directory: ./core/deps/llvm/build
 
       - name: install Ninja
         run: sudo apt install -y ninja-build
@@ -109,12 +79,12 @@ jobs:
         working-directory: ./tests/wamr-test-suites
 
   spec_test_extra:
-    needs: build_llvm_libraries
+    needs: cancel_previous
     if: ${{ endsWith(github.repository, 'wasm-micro-runtime') }}
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        test_option: [ $MULTI_MODULES_TEST_OPTIONS, $THREADS_TEST_OPTIONS ]
+        test_option: [$MULTI_MODULES_TEST_OPTIONS, $THREADS_TEST_OPTIONS]
     steps:
       - name: checkout
         uses: actions/checkout@v2
@@ -123,18 +93,18 @@ jobs:
         id: cache_llvm
         uses: actions/cache@v2
         with:
-          path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
+          path: |
+            ./core/deps/llvm/build/bin
+            ./core/deps/llvm/build/include
+            ./core/deps/llvm/build/lib
+            ./core/deps/llvm/build/libexec
+            ./core/deps/llvm/build/share
           key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
-          restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
 
       - name: Quit if cache miss
         if: steps.cache_llvm.outputs.cache-hit != 'true'
         run: echo "::error::can not get prebuilt llvm libraries" && exit 1
 
-      - name: Extract the LLVM package
-        run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
-        working-directory: ./core/deps/llvm/build
-
       - name: install Ninja
         run: sudo apt install -y ninja-build
 
@@ -144,11 +114,11 @@ jobs:
 
   spec_test_x86_32:
     if: ${{ endsWith(github.repository, 'wasm-micro-runtime') }}
-    needs: build_llvm_libraries
+    needs: cancel_previous
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        test_option: [ $DEFAULT_TEST_OPTIONS, $THREADS_TEST_OPTIONS ]
+        test_option: [$DEFAULT_TEST_OPTIONS, $THREADS_TEST_OPTIONS]
     steps:
       - name: checkout
         uses: actions/checkout@v2
@@ -157,18 +127,18 @@ jobs:
         id: cache_llvm
         uses: actions/cache@v2
         with:
-          path: ./core/deps/llvm/build/LLVM-13.0.0.tar.gz
+          path: |
+            ./core/deps/llvm/build/bin
+            ./core/deps/llvm/build/include
+            ./core/deps/llvm/build/lib
+            ./core/deps/llvm/build/libexec
+            ./core/deps/llvm/build/share
           key: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
-          restore-keys: ubuntu-20.04-${{ env.LLVM_CACHE_SUFFIX }}
 
       - name: Quit if cache miss
         if: steps.cache_llvm.outputs.cache-hit != 'true'
         run: echo "::error::can not get prebuilt llvm libraries" && exit 1
 
-      - name: Extract the LLVM package
-        run: tar xf LLVM-13.0.0.tar.gz --strip-components=1
-        working-directory: ./core/deps/llvm/build
-
       - name: install Ninja and x32 support libraries
         run: sudo apt install -y g++-multilib libgcc-9-dev lib32gcc-9-dev ninja-build
 

build-scripts/build_llvm.py

@@ -99,7 +99,7 @@ def build_llvm(llvm_dir, platform, backends, projects):
     lib_llvm_core_library = build_dir.joinpath("lib/libLLVMCore.a").resolve()
     if lib_llvm_core_library.exists():
         print(f"Please remove {build_dir} manually and try again")
-        return
+        return build_dir
 
     compile_options = " ".join(
         LLVM_COMPILE_OPTIONS
@@ -126,6 +126,28 @@ def build_llvm(llvm_dir, platform, backends, projects):
     return build_dir
 
 
+def repackage_llvm(llvm_dir):
+    build_dir = llvm_dir.joinpath("./build").resolve()
+
+    packs = [f for f in build_dir.glob("LLVM-13*.tar.gz")]
+    if len(packs) > 1:
+        raise Exception("Find more than one LLVM-13*.tar.gz")
+
+    if not packs:
+        return
+
+    llvm_package = packs[0].name
+    # mv build/LLVM-13.0.0*.gz .
+    shutil.move(str(build_dir.joinpath(llvm_package).resolve()), str(llvm_dir))
+    # rm -r build
+    shutil.rmtree(str(build_dir))
+    # mkdir build
+    build_dir.mkdir()
+    # tar xf ./LLVM-13.0.0-*.tar.gz --strip-components=1 --directory=build
+    CMD = f"tar xf {llvm_dir.joinpath(llvm_package).resolve()} --strip-components=1 --directory={build_dir}"
+    subprocess.check_call(shlex.split(CMD), cwd=llvm_dir)
+
+
 def main():
     parser = argparse.ArgumentParser(description="build necessary LLVM libraries")
     parser.add_argument(
@@ -198,16 +220,24 @@ def main():
     current_dir = current_file.parent.resolve()
     deps_dir = current_dir.joinpath("../core/deps").resolve()
 
-    print(f"==================== CLONE LLVM ====================")
+    try:
-    llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"])
+        print(f"==================== CLONE LLVM ====================")
-    llvm_dir = clone_llvm(deps_dir, llvm_info["repo"], llvm_info["branch"])
+        llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"])
+        llvm_dir = clone_llvm(deps_dir, llvm_info["repo"], llvm_info["branch"])
 
-    print()
+        print()
-    print(f"==================== BUILD LLVM ====================")
+        print(f"==================== BUILD LLVM ====================")
-    build_llvm(llvm_dir, platform, options.arch, options.project)
+        build_llvm(llvm_dir, platform, options.arch, options.project)
 
-    print()
+        print()
+        print(f"==================== PACKAGE LLVM ====================")
+        repackage_llvm(llvm_dir)
+
+        print()
+        return True
+    except subprocess.CalledProcessError:
+        return False
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(0 if main() else 1)