diff --git a/.github/workflows/ci_manylinux_x86_64_clang_llvm.yml b/.github/workflows/ci_manylinux_x86_64_clang_llvm.yml new file mode 100644 index 0000000..c4ce438 --- /dev/null +++ b/.github/workflows/ci_manylinux_x86_64_clang_llvm.yml @@ -0,0 +1,107 @@ +name: CI - manylinux x86_64 clang LLVM + +on: + workflow_call: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - main + +concurrency: + # A PR number if a pull request and otherwise the commit hash. This cancels + # queued and in-progress runs for the same PR (presubmit) or commit + # (postsubmit). The workflow name is prepended to avoid conflicts between + # different workflows. + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + manylinux_x64_clang_llvm: + runs-on: ubuntu-22.04 + + container: + image: quay.io/pypa/manylinux_2_28_x86_64 + env: + CACHE_DIR: ${{ github.workspace }}/.container-cache + # either the PR number or `branch-N` where N always increments + CACHE_KEY: manylinux_x64_clang_llvm_${{ format('{0}-{1}', github.ref_name, github.run_number) }} + + defaults: + run: + shell: bash + + steps: + - name: "Set unified TZ" + uses: szenius/set-timezone@v2.0 + with: + # this is an arbitrary choice + timezoneLinux: "Asia/Singapore" + timezoneMacos: "Asia/Singapore" + timezoneWindows: "Singapore Standard Time" + + - name: Enable cache + uses: actions/cache/restore@v3 + with: + path: ${{ env.CACHE_DIR }} + key: ${{ env.CACHE_KEY }} + restore-keys: manylinux_x64_clang_llvm_ + + - name: "Check out repository" + uses: actions/checkout@v4.2.2 + with: + submodules: true + + - name: "Setup compiler/toolchain" + uses: aminya/setup-cpp@v1 + with: + compiler: llvm-18 + cmake: true + ninja: true + cache: ccache + + - name: "Setup ccache" + run: | + echo "CCACHE_DIR=${{ env.CACHE_DIR }}" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=700M" >> $GITHUB_ENV + echo "CCACHE_COMPILERCHECK=string:$(clang --version)" >> $GITHUB_ENV + echo "CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros" >> $GITHUB_ENV + + - name: "Setup Python" + - uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: "Python deps" + run: | + pip install -r third_party/llvm-project/mlir/python/requirements.txt + + - name: "Build LLVM and MLIR" + run: | + export LLVM_BUILD_DIR="$PWD/llvm-build" + export LLVM_INSTALL_DIR="$PWD/llvm-install" + build_tools/cmake/build_llvm.sh build_llvm + build_tools/cmake/build_llvm.sh build_mlir + + - name: "Tar dist" + if: ${{ !cancelled() }} + run: | + pushd third_party/llvm-project && llvm_sha_short=$(git rev-parse --short HEAD) && popd + tar cf llvm-dist-linux-$llvm_sha_short.tar llvm-install + + - name: "Upload dist" + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: linux_x86_64_llvm_packages + path: llvm-dist-*.tar + if-no-files-found: error + + - name: "Save cache" + uses: actions/cache/save@v3 + # if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == 'main' }} + if: ${{ !cancelled() }} + with: + path: ${{ env.CACHE_DIR }} + key: ${{ env.CACHE_KEY }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4d4754e --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +.idea \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..1cc3edb --- /dev/null +++ b/.gitmodules @@ -0,0 +1,5 @@ +[submodule "third_party/llvm-project"] + path = third_party/llvm-project + url = https://github.com/llvm/llvm-project.git + ignore = dirty + shallow = true diff --git a/build_tools/cmake/build_llvm.sh b/build_tools/cmake/build_llvm.sh new file mode 100644 index 0000000..8d0963f --- /dev/null +++ b/build_tools/cmake/build_llvm.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +TD="$(cd $(dirname $0) && pwd)" +REPO_ROOT="$(cd $TD/../.. && pwd)" + +LLVM_SOURCE_DIR="${LLVM_SOURCE_DIR:-${REPO_ROOT}/third_party/llvm-project}" +LLVM_BUILD_DIR="${LLVM_BUILD_DIR:-${REPO_ROOT}/../llvm-build}" +LLVM_INSTALL_DIR="${LLVM_INSTALL_DIR:-${REPO_ROOT}/../llvm-install}" + +LLVM_SOURCE_DIR="$(realpath -m "${LLVM_SOURCE_DIR}")" +LLVM_BUILD_DIR="$(realpath -m "${LLVM_BUILD_DIR}")" +LLVM_INSTALL_DIR="$(realpath -m "${LLVM_INSTALL_DIR}")" + +echo "Paths canonicalized as:" +echo "LLVM_SOURCE_DIR=${LLVM_SOURCE_DIR}" +echo "LLVM_BUILD_DIR=${LLVM_BUILD_DIR}" +echo "LLVM_INSTALL_DIR=${LLVM_INSTALL_DIR}" + +command="$1" +shift + +has_ccache=false +if (command -v ccache &> /dev/null); then + has_ccache=true +fi + +has_clang=false +if (command -v clang &> /dev/null) && (command -v clang++ &> /dev/null); then + has_clang=true +fi + +has_lld=false +if (command -v lld &> /dev/null); then + has_lld=true +fi + +python3_command="" +if (command -v python3 &> /dev/null); then + python3_command="python3" +elif (command -v python &> /dev/null); then + python3_command="python" +fi + +set -euo pipefail + +print_toolchain_config() { + if $has_ccache; then + echo -n "-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache " + fi + if $has_clang; then + echo "-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++" + fi +} + +do_build_llvm() { + echo "*********************** BUILDING LLVM *********************************" + main_build_dir="${LLVM_BUILD_DIR}/llvm" + main_install_dir="${LLVM_INSTALL_DIR}/llvm" + targets_to_build="${LLVM_TARGETS_TO_BUILD:-X86}" + + cmake_options="-DLLVM_TARGETS_TO_BUILD='${targets_to_build}'" + cmake_options="${cmake_options} -DCMAKE_BUILD_TYPE=Release" + cmake_options="${cmake_options} -C $TD/llvm_config.cmake" + cmake_options="${cmake_options} -DCMAKE_INSTALL_PREFIX=${main_install_dir}" + cmake_options="${cmake_options} $(print_toolchain_config)" + if $has_lld; then + cmake_options="${cmake_options} -DLLVM_ENABLE_LLD=ON" + fi + + echo "Source Directory: ${LLVM_SOURCE_DIR}" + echo "Build Directory: ${main_build_dir}" + echo "CMake Options: ${cmake_options}" + cmake -GNinja -S "${LLVM_SOURCE_DIR}/llvm" -B "${main_build_dir}" \ + $cmake_options + cmake --build "${main_build_dir}" \ + --target install-toolchain-distribution \ + --target install-development-distribution +} + +do_build_mlir() { + echo "*********************** BUILDING MLIR *********************************" + main_install_dir="${LLVM_INSTALL_DIR}/llvm" + mlir_build_dir="${LLVM_BUILD_DIR}/mlir" + mlir_install_dir="${LLVM_INSTALL_DIR}/mlir" + + cmake_options="-DLLVM_DIR='${main_install_dir}/lib/cmake/llvm'" + cmake_options="${cmake_options} -DPython3_EXECUTABLE='$(which $python3_command)'" + # Note: Building the MLIR Python bindings requires the installation of + # dependencies as specified in `mlir/python/requirements.txt`, which among + # others include pybind11. + cmake_options="${cmake_options} -DMLIR_ENABLE_BINDINGS_PYTHON=ON" + cmake_options="${cmake_options} -DCMAKE_INSTALL_PREFIX=${mlir_install_dir}" + cmake_options="${cmake_options} -C $TD/mlir_config.cmake" + cmake_options="${cmake_options} $(print_toolchain_config)" + if $has_lld; then + cmake_options="${cmake_options} -DLLVM_ENABLE_LLD=ON" + fi + + echo "Source Directory: ${LLVM_SOURCE_DIR}" + echo "Build Directory: ${mlir_build_dir}" + echo "CMake Options: ${cmake_options}" + cmake -GNinja -S "${LLVM_SOURCE_DIR}/mlir" -B "${mlir_build_dir}" \ + $cmake_options + cmake --build "${mlir_build_dir}" \ + --target install-mlirdevelopment-distribution +} + +case "${command}" in + build_llvm) + do_build_llvm + ;; + + build_mlir) + do_build_mlir + ;; + + *) + echo "ERROR: Expected command of 'build_llvm', 'build_mlir' (got '${command}')" + exit 1 + ;; +esac diff --git a/build_tools/cmake/llvm_cache.cmake b/build_tools/cmake/llvm_cache.cmake new file mode 100644 index 0000000..53d0e53 --- /dev/null +++ b/build_tools/cmake/llvm_cache.cmake @@ -0,0 +1,129 @@ +### Components built ### + +set(LLVM_ENABLE_PROJECTS "clang;clang-tools-extra;lld" CACHE STRING "") + +### Target architectures ### + +# Compiler target architectures +set(LLVM_TARGETS_TO_BUILD "X86" CACHE STRING "") + +set(LLVM_ENABLE_RUNTIMES "compiler-rt" CACHE STRING "") + +# CMake builtin variables and modules are not available for this cache file +# Gather directly build/host information +execute_process(COMMAND "uname" "-m" OUTPUT_VARIABLE _UNAME_M) +string(STRIP ${_UNAME_M} BUILD_MACHINE_ARCH) + +### Default settings for the toolchain ### + +# Use the LLVM components +set(CLANG_DEFAULT_OBJCOPY llvm-objcopy CACHE STRING "") +set(CLANG_DEFAULT_LINKER lld CACHE STRING "") + +set(CLANG_ENABLE_STATIC_ANALYZER ON CACHE BOOL "") +set(LLVM_ENABLE_LIBCXX OFF CACHE BOOL "") +set(LLVM_ENABLE_ZLIB ON CACHE BOOL "") + +### Disk size optimizations ### + +# All the tools will use libllvm shared library +set(LLVM_BUILD_LLVM_DYLIB ON CACHE BOOL "") +set(LLVM_LINK_LLVM_DYLIB ON CACHE BOOL "") + +# When exceptions are disabled, unwind tables are large and useless +set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "") + +# Mildly useful misc stuff (which might also be hard to cross-compile) +set(CLANG_ENABLE_ARCMT OFF CACHE BOOL "") +set(CLANG_PLUGIN_SUPPORT OFF CACHE BOOL "") +set(LLVM_ENABLE_TERMINFO OFF CACHE BOOL "") +set(LLVM_ENABLE_Z3_SOLVER OFF CACHE BOOL "") +set(LLVM_INCLUDE_DOCS OFF CACHE BOOL "") +set(LLVM_INCLUDE_GO_TESTS OFF CACHE BOOL "") +set(LLVM_FORCE_ENABLE_STATS ON CACHE BOOL "") + +# Do not store debug information by default. +set(CMAKE_BUILD_TYPE Release CACHE STRING "") + +# Use the distributions below for the installation +set(LLVM_INSTALL_TOOLCHAIN_ONLY OFF CACHE BOOL "") + +### Distributions ### + +set(LLVM_DISTRIBUTIONS + Toolchain + Development + CACHE STRING "") + +set(LLVM_TOOLCHAIN_TOOLS + llvm-addr2line + llvm-ar + llvm-cxxfilt + llvm-dis + llvm-dwarfdump + llvm-lib + llvm-link + llvm-mc + llvm-nm + llvm-objcopy + llvm-objdump + llvm-rc + llvm-ranlib + llvm-readelf + llvm-readobj + llvm-size + llvm-strip + llvm-symbolizer + llvm-xray + CACHE STRING "") + +set(LLVM_BUILD_UTILS ON CACHE BOOL "") +set(LLVM_INSTALL_UTILS ON CACHE BOOL "") +set(LLVM_TOOLCHAIN_UTILITIES + FileCheck + count + not + CACHE STRING "") + +set(LLVM_RUNTIME_DISTRIBUTION_COMPONENTS + CACHE STRING "") + +set(LLVM_Toolchain_DISTRIBUTION_COMPONENTS + builtins + runtimes + clang + clang-cpp + clang-format + clang-offload-bundler + clang-resource-headers + clang-tidy + clangd + libclang + lld + LLVM + LTO + ${LLVM_TOOLCHAIN_TOOLS} + ${LLVM_TOOLCHAIN_UTILITIES} + CACHE STRING "") + +set(LLVM_Development_DISTRIBUTION_COMPONENTS + # LLVM + cmake-exports + development-cmake-exports + toolchain-cmake-exports + llc + llvm-config + llvm-headers + llvm-libraries + opt + Remarks + # Clang + clang-cmake-exports + clang-development-cmake-exports + clang-toolchain-cmake-exports + clang-headers + clang-libraries + # LLD + lld-cmake-exports + lld-toolchain-cmake-exports + CACHE STRING "") diff --git a/build_tools/cmake/mlir_cache.cmake b/build_tools/cmake/mlir_cache.cmake new file mode 100644 index 0000000..186f5e8 --- /dev/null +++ b/build_tools/cmake/mlir_cache.cmake @@ -0,0 +1,60 @@ +# export MLIR_BUILD_DIR=$HOME/tmp/mlirbuild +# export MLIR_INSTALL_DIR=$HOME/tmp/mlirroot +# cmake -G Ninja \ +# -B "$MLIR_BUILD_DIR" -S third_party/llvm-project/mlir \ +# -DLLVM_DIR="${LLVM_INSTALL_DIR}/lib/cmake/llvm" \ +# -C build_tools/llvm/mlir_config.cmake \ +# -DCMAKE_BUILD_TYPE="Release" \ +# -DPython3_EXECUTABLE='$(which $python3_command)' \ +# -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ +# -DCMAKE_INSTALL_PREFIX="$MLIR_INSTALL_DIR" \ +# -DCMAKE_C_COMPILER_LAUNCHER=ccache \ +# -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ +# -DCMAKE_C_COMPILER=clang \ +# -DCMAKE_CXX_COMPILER=clang++ \ +# -DCMAKE_LINKER=lld +# ninja -C $MLIR_BUILD_DIR install-mlirdevelopment-distribution + +if(NOT EXISTS ${LLVM_DIR}) + message(FATAL_ERROR "LLVM_DIR (${LLVM_DIR}) does not exist") +endif() + +# When exceptions are disabled, unwind tables are large and useless +set(LLVM_ENABLE_UNWIND_TABLES OFF CACHE BOOL "") + +# Do not store debug information by default. +set(CMAKE_BUILD_TYPE Release CACHE STRING "") + +# Use the distributions below for the installation +set(LLVM_INSTALL_TOOLCHAIN_ONLY OFF CACHE BOOL "") + +# Build tools and utils. +set(LLVM_BUILD_TOOLS ON CACHE BOOL "") +set(LLVM_BUILD_UTILS ON CACHE BOOL "") + +### Distributions ### + +set(LLVM_DISTRIBUTIONS + MlirDevelopment + CACHE STRING "") + +set(LLVM_MLIR_TOOLS + mlir-opt + mlir-reduce + mlir-tblgen + mlir-translate + CACHE STRING "") + +set(LLVM_MLIR_Python_COMPONENTS + MLIRPythonModules + mlir-python-sources + CACHE STRING "") + +set(LLVM_MlirDevelopment_DISTRIBUTION_COMPONENTS + mlir-cmake-exports + mlir-headers + mlir-libraries + MLIRPythonModules + ${LLVM_MLIR_TOOLS} + ${LLVM_MLIR_Python_COMPONENTS} + CACHE STRING "") diff --git a/third_party/llvm-project b/third_party/llvm-project new file mode 160000 index 0000000..1aff96b --- /dev/null +++ b/third_party/llvm-project @@ -0,0 +1 @@ +Subproject commit 1aff96b3dfcc58d62fda5b1452a8029f1a737cc2