#!/usr/bin/env python3 # # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # import argparse import os import pathlib import requests import shlex import shutil import subprocess import sysconfig import sys def clone_llvm(dst_dir, llvm_repo, llvm_branch): """ any error will raise CallProcessError """ llvm_dir = dst_dir.joinpath("llvm").resolve() if not llvm_dir.exists(): GIT_CLONE_CMD = f"git clone --depth 1 --branch {llvm_branch} {llvm_repo} llvm" print(GIT_CLONE_CMD) subprocess.check_output(shlex.split(GIT_CLONE_CMD), cwd=dst_dir) return llvm_dir def query_llvm_version(llvm_info): github_token = os.environ['GH_TOKEN'] owner_project = llvm_info['repo'].replace("https://github.com/", "").replace(".git", "") url = f"https://api.github.com/repos/{owner_project}/commits/{llvm_info['branch']}" headers = { 'Authorization': f"Bearer {github_token}" } try: response = requests.request("GET", url, headers=headers, data={}) response.raise_for_status() except requests.exceptions.HTTPError as error: print (error) # for debugging purpose return None response = response.json() return response['sha'] def build_llvm(llvm_dir, platform, backends, projects, use_clang=False, extra_flags=''): LLVM_COMPILE_OPTIONS = [ '-DCMAKE_BUILD_TYPE:STRING="Release"', "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", "-DLLVM_APPEND_VC_REV:BOOL=ON", "-DLLVM_BUILD_EXAMPLES:BOOL=OFF", "-DLLVM_BUILD_LLVM_DYLIB:BOOL=OFF", "-DLLVM_ENABLE_BINDINGS:BOOL=OFF", "-DLLVM_ENABLE_IDE:BOOL=OFF", "-DLLVM_ENABLE_LIBEDIT=OFF", "-DLLVM_ENABLE_TERMINFO:BOOL=OFF", "-DLLVM_ENABLE_ZLIB:BOOL=ON", "-DLLVM_INCLUDE_BENCHMARKS:BOOL=OFF", "-DLLVM_INCLUDE_DOCS:BOOL=OFF", "-DLLVM_INCLUDE_EXAMPLES:BOOL=OFF", "-DLLVM_INCLUDE_UTILS:BOOL=OFF", "-DLLVM_INCLUDE_TESTS:BOOL=OFF", "-DLLVM_OPTIMIZED_TABLEGEN:BOOL=ON", ] # ccache is not available on Windows if not "windows" == platform: LLVM_COMPILE_OPTIONS.append("-DLLVM_CCACHE_BUILD:BOOL=ON") # perf support is available on Linux only if "linux" == platform: LLVM_COMPILE_OPTIONS.append("-DLLVM_USE_PERF:BOOL=ON") # use clang/clang++/lld. but macos doesn't support lld if not sys.platform.startswith("darwin") and use_clang: if shutil.which("clang") and shutil.which("clang++") and shutil.which("lld"): os.environ["CC"] = "clang" os.environ["CXX"] = "clang++" LLVM_COMPILE_OPTIONS.append('-DLLVM_USE_LINKER:STRING="lld"') print("Use the clang toolchain") else: print("Can not find clang, clang++ and lld, keep using the gcc toolchain") else: print("Use the gcc toolchain") LLVM_EXTRA_COMPILE_OPTIONS = { "arc": [ '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="ARC"', "-DLLVM_ENABLE_LIBICUUC:BOOL=OFF", "-DLLVM_ENABLE_LIBICUDATA:BOOL=OFF", ], "xtensa": [ '-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="Xtensa"', ], "windows": [ "-DCMAKE_INSTALL_PREFIX=LLVM-install", ], "default": [], } LLVM_TARGETS_TO_BUILD = [ '-DLLVM_TARGETS_TO_BUILD:STRING="' + ";".join(backends) + '"' if backends else '-DLLVM_TARGETS_TO_BUILD:STRING="AArch64;ARM;Mips;RISCV;X86"' ] LLVM_PROJECTS_TO_BUILD = [ '-DLLVM_ENABLE_PROJECTS:STRING="' + ";".join(projects) + '"' if projects else "" ] # lldb project requires libxml2 LLVM_LIBXML2_OPTION = [ "-DLLVM_ENABLE_LIBXML2:BOOL=" + ("ON" if "lldb" in projects else "OFF") ] # enabling LLVM_INCLUDE_TOOLS will increase ~300M to the final package LLVM_INCLUDE_TOOLS_OPTION = [ "-DLLVM_INCLUDE_TOOLS:BOOL=ON" if projects else "-DLLVM_INCLUDE_TOOLS:BOOL=OFF" ] if not llvm_dir.exists(): raise Exception(f"{llvm_dir} doesn't exist") build_dir = llvm_dir.joinpath( "win32build" if "windows" == platform else "build" ).resolve() build_dir.mkdir(exist_ok=True) lib_llvm_core_library = build_dir.joinpath("lib/libLLVMCore.a").resolve() if lib_llvm_core_library.exists(): print( f"It has already been fully compiled. If want to a re-build, please remove {build_dir} manually and try again" ) return None compile_options = " ".join( LLVM_COMPILE_OPTIONS + LLVM_LIBXML2_OPTION + LLVM_EXTRA_COMPILE_OPTIONS.get( platform, LLVM_EXTRA_COMPILE_OPTIONS["default"] ) + LLVM_TARGETS_TO_BUILD + LLVM_PROJECTS_TO_BUILD + LLVM_INCLUDE_TOOLS_OPTION ) CONFIG_CMD = f"cmake {compile_options} {extra_flags} ../llvm" if "windows" == platform: if "mingw" in sysconfig.get_platform().lower(): CONFIG_CMD += " -G'Unix Makefiles'" else: CONFIG_CMD += " -A x64" else: CONFIG_CMD += " -G'Ninja'" subprocess.check_call(shlex.split(CONFIG_CMD), cwd=build_dir) BUILD_CMD = "cmake --build . --target package" + ( " --config Release" if "windows" == platform else "" ) subprocess.check_call(shlex.split(BUILD_CMD), cwd=build_dir) return build_dir def repackage_llvm(llvm_dir): build_dir = llvm_dir.joinpath("./build").resolve() packs = [f for f in build_dir.glob("LLVM-*.tar.gz")] if len(packs) > 1: raise Exception("Find more than one LLVM-*.tar.gz") if not packs: return llvm_package = packs[0].name # mv build/LLVM-*.gz . shutil.move(str(build_dir.joinpath(llvm_package).resolve()), str(llvm_dir)) # rm -r build shutil.rmtree(str(build_dir)) # mkdir build build_dir.mkdir() # tar xf ./LLVM-*.tar.gz --strip-components=1 --directory=build CMD = f"tar xf {llvm_dir.joinpath(llvm_package).resolve()} --strip-components=1 --directory={build_dir}" subprocess.check_call(shlex.split(CMD), cwd=llvm_dir) # rm ./LLVM-1*.gz os.remove(llvm_dir.joinpath(llvm_package).resolve()) def main(): parser = argparse.ArgumentParser(description="build necessary LLVM libraries") parser.add_argument( "--platform", type=str, choices=["android", "arc", "darwin", "linux", "windows", "xtensa"], help="identify current platform", ) parser.add_argument( "--arch", nargs="+", type=str, choices=[ "AArch64", "ARC", "ARM", "Mips", "RISCV", "WebAssembly", "X86", "Xtensa", ], help="identify LLVM supported backends, separate by space, like '--arch ARM Mips X86'", ) parser.add_argument( "--project", nargs="+", type=str, default="", choices=["clang", "lldb"], help="identify extra LLVM projects, separate by space, like '--project clang lldb'", ) parser.add_argument( "--llvm-ver", action="store_true", help="return the version info of generated llvm libraries", ) parser.add_argument( "--use-clang", action="store_true", help="use clang instead of gcc", ) parser.add_argument( "--extra-cmake-flags", type=str, default="", help="custom extra cmake flags", ) options = parser.parse_args() # if the "platform" is not identified in the command line option, # detect it if not options.platform: if sys.platform.startswith("win32") or sys.platform.startswith("msys"): platform = "windows" elif sys.platform.startswith("darwin"): platform = "darwin" else: platform = "linux" else: platform = options.platform llvm_repo_and_branch = { "arc": { "repo": "https://github.com/llvm/llvm-project.git", "repo_ssh": "git@github.com:llvm/llvm-project.git", "branch": "release/15.x", }, "xtensa": { "repo": "https://github.com/espressif/llvm-project.git", "repo_ssh": "git@github.com:espressif/llvm-project.git", "branch": "xtensa_release_15.x", }, "default": { "repo": "https://github.com/llvm/llvm-project.git", "repo_ssh": "git@github.com:llvm/llvm-project.git", "branch": "release/15.x", }, } # retrieve the real file current_file = pathlib.Path(__file__) if current_file.is_symlink(): current_file = pathlib.Path(os.readlink(current_file)) current_dir = current_file.parent.resolve() deps_dir = current_dir.joinpath("../core/deps").resolve() try: llvm_info = llvm_repo_and_branch.get(platform, llvm_repo_and_branch["default"]) if options.llvm_ver: commit_hash = query_llvm_version(llvm_info) print(commit_hash) return commit_hash is not None repo_addr = llvm_info["repo"] if os.environ.get('USE_GIT_SSH') == "true": repo_addr = llvm_info["repo_ssh"] else: print("To use ssh for git clone, run: export USE_GIT_SSH=true") llvm_dir = clone_llvm(deps_dir, repo_addr, llvm_info["branch"]) if ( build_llvm( llvm_dir, platform, options.arch, options.project, options.use_clang, options.extra_cmake_flags ) is not None ): repackage_llvm(llvm_dir) return True except subprocess.CalledProcessError: return False if __name__ == "__main__": sys.exit(0 if main() else 1)