ccache on for eudsl-py

llvm · Jan 1, 2025 · 88568d9 · 88568d9
1 parent 5e2252f
commit 88568d9
Show file tree

Hide file tree

Showing 4 changed files with 85 additions and 11 deletions.
diff --git a/.github/workflows/build_llvm.yml b/.github/workflows/build_llvm.yml
@@ -112,6 +112,7 @@ jobs:
       - name: "Build LLVM and MLIR"
         id: build
         run: |
+          
           export LLVM_SOURCE_DIR="$PWD/third_party/llvm-project"
           export LLVM_BUILD_DIR="$PWD/llvm-build"
           # double nested so that upload artifacts uploads a folder 

diff --git a/.github/workflows/build_test_release_eudsl.yml b/.github/workflows/build_test_release_eudsl.yml
@@ -95,7 +95,7 @@ jobs:
           os: ${{ matrix.os }}
           arch: ${{ matrix.arch }}
 
-      - name: "Download MLIR"
+      - name: "Download MLIR and configure build"
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
@@ -113,12 +113,19 @@ jobs:
             echo "LLVM_DIR=/host/$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV
             echo "MLIR_DIR=/host/$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV
             echo "Clang_DIR=/host/$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV
+            echo "CCACHE_DIR=/host/$CCACHE_DIR" >> $GITHUB_ENV
           else
             echo "LLVM_DIR=$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV
             echo "MLIR_DIR=$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV
             echo "Clang_DIR=$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV
           fi
           
+          # since linux builds execute in the cibuildwheel almalinux container
+          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
+            echo CC=clang >> $GITHUB_ENV
+            echo CXX=clang++ >> $GITHUB_ENV
+          fi
+          
           # these have to match the deployment targets of the LLVM build runners
           # ld: warning: object file (libLLVMTableGen.a[3](Error.cpp.o)) was built for newer 'macOS' version (13.7) than being linked (10.13)
           if [[ "${{ matrix.runs-on }}" == "macos-14" ]]; then
@@ -131,16 +138,20 @@ jobs:
 
       - name: "Build eudsl-tblgen"
         run: |
+          
+          ccache -z
           $python3_command -m cibuildwheel "$PWD/projects/eudsl-tblgen" --output-dir wheelhouse
+          ccache -s
 
       - name: "Build eudsl-py"
         if: ${{ ! startsWith(matrix.os, 'windows') }}
         run: |
+          
           # prevent OOM on free GHA
           export DISABLE_COMPILE_OPT="${{ inputs.debug_with_tmate }}"
-          export CC=clang
-          export CXX=clang++
+          ccache -z
           $python3_command -m cibuildwheel "$PWD/projects/eudsl-py" --output-dir wheelhouse
+          ccache -s
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
@@ -198,11 +209,13 @@ jobs:
 
       - name: "Install eudsl-tblgen"
         run: |
+          
           python -m pip install pytest
           python -m pip install eudsl-tblgen -f wheelhouse
 
       - name: "Test eudsl-tblgen"
         run: |
+          
           export TESTS_DIR="$PWD/projects/eudsl-tblgen/tests"
           if [ ${{ matrix.os }} == 'windows' ]; then
             python -m pytest -s -rA $TESTS_DIR
@@ -292,11 +305,13 @@ jobs:
 
       - name: "Install eudsl-py"
         run: |
+          
           python -m pip install pytest
           python -m pip install eudsl-py -f wheelhouse
 
       - name: "Test eudsl-py"
         run: |
+          
           export TESTS_DIR="$PWD/projects/eudsl-py/tests"
           python -m pytest -rA --capture=tee-sys $TESTS_DIR
 

diff --git a/projects/eudsl-py/pyproject.toml b/projects/eudsl-py/pyproject.toml
@@ -30,10 +30,12 @@ wheel.py-api = "cp312"
 cmake.source-dir = "."
 
 [tool.scikit-build.cmake.define]
-LLVM_DIR = {env="LLVM_DIR", default="EMPTY"}
-MLIR_DIR = {env="MLIR_DIR", default="EMPTY"}
-Clang_DIR = {env="Clang_DIR", default="EMPTY"}
-CMAKE_PREFIX_PATH = {env="CMAKE_PREFIX_PATH", default="EMPTY"}
+LLVM_DIR = { env = "LLVM_DIR", default = "EMPTY" }
+MLIR_DIR = { env = "MLIR_DIR", default = "EMPTY" }
+Clang_DIR = { env = "Clang_DIR", default = "EMPTY" }
+CMAKE_PREFIX_PATH = { env = "CMAKE_PREFIX_PATH", default = "EMPTY" }
+CMAKE_C_COMPILER_LAUNCHER = { env = "CMAKE_C_COMPILER_LAUNCHER", default = "" }
+CMAKE_CXX_COMPILER_LAUNCHER = { env = "CMAKE_CXX_COMPILER_LAUNCHER", default = "" }
 CMAKE_CXX_VISIBILITY_PRESET = "hidden"
 CMAKE_VERBOSE_MAKEFILE = "ON"
 
@@ -50,10 +52,33 @@ environment-pass = [
     "CMAKE_PREFIX_PATH",
     "DISABLE_COMPILE_OPT",
     "CC",
-    "CXX"
+    "CXX",
+    # ccache
+    "CCACHE_DIR",
+    "CCACHE_MAXSIZE=700M",
+    "CCACHE_SLOPPINESS",
+    "CCACHE_CPP2",
+    "CCACHE_UMASK",
+    "CMAKE_C_COMPILER_LAUNCHER",
+    "CMAKE_CXX_COMPILER_LAUNCHER"
+]
+before-build = [
+    "export CCACHE_DIR=$CCACHE_DIR/$(python -c 'import platform; print(platform.python_version())')",
+    "mkdir -p $CCACHE_DIR"
 ]
 
 [tool.cibuildwheel.linux]
-before-build = [
+before-all = [
     "yum install -y clang",
+    # ccache
+    "curl -sLO https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz",
+    "tar -xf ccache-4.10.2-linux-x86_64.tar.xz",
+    "pushd ccache-4.10.2-linux-x86_64 && make install && popd"
+]
+
+# override for windows because cibuildwheel runs in powershell not bash
+[tool.cibuildwheel.windows]
+before-build = [
+    "$env:python_version = python -c \"import platform; print(platform.python_version())\"",
+    "$env:CCACHE_DIR = \"$env:CCACHE_DIR\\$env:python_version\""
 ]
diff --git a/projects/eudsl-tblgen/pyproject.toml b/projects/eudsl-tblgen/pyproject.toml
@@ -22,8 +22,10 @@ wheel.py-api = "cp312"
 cmake.source-dir = "src"
 
 [tool.scikit-build.cmake.define]
-LLVM_DIR = {env="LLVM_DIR", default="EMPTY"}
+LLVM_DIR = { env = "LLVM_DIR", default = "EMPTY" }
 CMAKE_CXX_VISIBILITY_PRESET = "hidden"
+CMAKE_C_COMPILER_LAUNCHER = { env = "CMAKE_C_COMPILER_LAUNCHER", default = "" }
+CMAKE_CXX_COMPILER_LAUNCHER = { env = "CMAKE_CXX_COMPILER_LAUNCHER", default = "" }
 
 [tool.cibuildwheel]
 build-verbosity = 1
@@ -32,5 +34,36 @@ archs = ["auto64"]
 manylinux-x86_64-image = "manylinux_2_28"
 environment-pass = [
     "LLVM_DIR",
-    "CMAKE_GENERATOR"
+    "CMAKE_GENERATOR",
+    "CMAKE_PREFIX_PATH",
+    "CC",
+    "CXX",
+    # ccache
+    "CCACHE_DIR",
+    "CCACHE_MAXSIZE=700M",
+    "CCACHE_SLOPPINESS",
+    "CCACHE_CPP2",
+    "CCACHE_UMASK",
+    "CMAKE_C_COMPILER_LAUNCHER",
+    "CMAKE_CXX_COMPILER_LAUNCHER"
+]
+before-build = [
+    "export CCACHE_DIR=$CCACHE_DIR/$(python -c 'import platform; print(platform.python_version())')",
+    "mkdir -p $CCACHE_DIR"
+]
+
+[tool.cibuildwheel.linux]
+before-all = [
+    "yum install -y clang",
+    # ccache
+    "curl -sLO https://github.com/ccache/ccache/releases/download/v4.10.2/ccache-4.10.2-linux-x86_64.tar.xz",
+    "tar -xf ccache-4.10.2-linux-x86_64.tar.xz",
+    "pushd ccache-4.10.2-linux-x86_64 && make install && popd"
+]
+
+# override for windows because cibuildwheel runs in powershell not bash
+[tool.cibuildwheel.windows]
+before-build = [
+    "$env:python_version = python -c \"import platform; print(platform.python_version())\"",
+    "$env:CCACHE_DIR = \"$env:CCACHE_DIR\\$env:python_version\""
 ]