Merge branch 'main' into fiber-dist-tweak

nod-ai · Jan 7, 2025 · 062a83e · 062a83e
2 parents bf8c638 + 0e0b42a
commit 062a83e
Show file tree

Hide file tree

Showing 9 changed files with 28 additions and 67 deletions.
diff --git a/.github/workflows/ci-libshortfin.yml b/.github/workflows/ci-libshortfin.yml
@@ -25,7 +25,6 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  IREE_REPO_DIR: ${{ github.workspace }}/iree
   LIBSHORTFIN_DIR: ${{ github.workspace }}/shortfin/
 
 jobs:
@@ -95,36 +94,19 @@ jobs:
       with:
         toolchain: stable
 
-    - name: Checkout IREE repo
-      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      with:
-        repository: iree-org/iree
-        path: ${{ env.IREE_REPO_DIR }}
-        submodules: false
-        ref: iree-3.1.0rc20241220
-
-    - name: Initalize IREE submodules
-      working-directory: ${{ env.IREE_REPO_DIR }}
-      run : |
-        git submodule update --init --depth 1 -- third_party/benchmark
-        git submodule update --init --depth 1 -- third_party/cpuinfo/
-        git submodule update --init --depth 1 -- third_party/flatcc
-        git submodule update --init --depth 1 -- third_party/googletest
-        git submodule update --init --depth 1 -- third_party/hip-build-deps/
-
     - name: "Setup Python ${{ matrix.python-version }}"
       uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
       with:
         python-version: ${{ matrix.python-version }}
         cache: "pip"
         cache-dependency-path: |
           'shortfin/requirements-tests.txt'
-          'shortfin/requirements-iree-compiler.txt'
+          'requirements-iree-pinned.txt'
     - name: Install Python packages
       working-directory: ${{ env.LIBSHORTFIN_DIR }}
       run: |
         pip install -r requirements-tests.txt
-        pip install -r requirements-iree-compiler.txt
+        pip install -r ../requirements-iree-pinned.txt
         pip freeze
 
     - name: Build shortfin
@@ -134,7 +116,6 @@ jobs:
         cmake -GNinja \
           -S. \
           -Bbuild \
-          -DSHORTFIN_IREE_SOURCE_DIR="${{ env.IREE_REPO_DIR }}" \
           -DSHORTFIN_BUILD_PYTHON_BINDINGS=ON \
           ${{matrix.cmake-options}}
         cmake --build build --target all

diff --git a/.github/workflows/ci-sdxl.yaml b/.github/workflows/ci-sdxl.yaml
@@ -65,7 +65,7 @@ jobs:
       working-directory: ${{ env.LIBSHORTFIN_DIR }}
       run: |
         pip install -r requirements-tests.txt
-        pip install -r requirements-iree-compiler.txt
+        pip install -r ../requirements-iree-pinned.txt
         pip freeze
 
     - name: Install shortfin

diff --git a/.github/workflows/ci_linux_x64_asan-libshortfin.yml b/.github/workflows/ci_linux_x64_asan-libshortfin.yml
@@ -36,7 +36,6 @@ env:
   PYTHON_VER: 3.12.7
   CACHE_ASAN_VER: 2
   CACHE_DEPS_VER: 1
-  IREE_SOURCE_DIR: ${{ github.workspace }}/iree
   LIBSHORTFIN_DIR: ${{ github.workspace }}/shortfin/
 
 jobs:
@@ -100,23 +99,6 @@ jobs:
         sudo apt update
         sudo apt install clang lld cmake ninja-build
 
-    - name: Checkout IREE repo
-      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      with:
-        repository: iree-org/iree
-        path: ${{ env.IREE_SOURCE_DIR }}
-        submodules: false
-        ref: iree-3.1.0rc20241220
-
-    - name: Initalize IREE submodules
-      working-directory: ${{ env.IREE_SOURCE_DIR }}
-      run : |
-        git submodule update --init --depth 1 -- third_party/benchmark
-        git submodule update --init --depth 1 -- third_party/cpuinfo/
-        git submodule update --init --depth 1 -- third_party/flatcc
-        git submodule update --init --depth 1 -- third_party/googletest
-        git submodule update --init --depth 1 -- third_party/hip-build-deps/
-
     - name: Restore Python dependencies cache
       id: cache-python-deps-restore
       uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
@@ -142,7 +124,7 @@ jobs:
       run: |
         eval "$(pyenv init -)"
         pip install -r requirements-tests.txt
-        pip install -r requirements-iree-compiler.txt
+        pip install -r ../requirements-iree-pinned.txt
         pip freeze
 
     - name: Save Python dependencies cache
@@ -157,7 +139,6 @@ jobs:
       working-directory: ${{ env.LIBSHORTFIN_DIR }}
       run: |
         eval "$(pyenv init -)"
-        SHORTFIN_IREE_SOURCE_DIR="${{ env.IREE_SOURCE_DIR }}" \
         SHORTFIN_ENABLE_ASAN=ON \
         SHORTFIN_DEV_MODE=ON \
         SHORTFIN_RUN_CTESTS=ON \

diff --git a/.github/workflows/ci_linux_x64_nogil-libshortfin.yml b/.github/workflows/ci_linux_x64_nogil-libshortfin.yml
@@ -32,7 +32,6 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  IREE_REPO_DIR: ${{ github.workspace }}/iree
   LIBSHORTFIN_DIR: ${{ github.workspace }}/shortfin/
 
 jobs:
@@ -48,23 +47,6 @@ jobs:
         sudo apt update
         sudo apt install clang lld cmake ninja-build
 
-    - name: Checkout IREE repo
-      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      with:
-        repository: iree-org/iree
-        path: ${{ env.IREE_REPO_DIR }}
-        submodules: false
-        ref: iree-3.1.0rc20241220
-
-    - name: Initalize IREE submodules
-      working-directory: ${{ env.IREE_REPO_DIR }}
-      run : |
-        git submodule update --init --depth 1 -- third_party/benchmark
-        git submodule update --init --depth 1 -- third_party/cpuinfo/
-        git submodule update --init --depth 1 -- third_party/flatcc
-        git submodule update --init --depth 1 -- third_party/googletest
-        git submodule update --init --depth 1 -- third_party/hip-build-deps/
-
     - name: Setup Python
       uses: deadsnakes/action@e640ac8743173a67cca4d7d77cd837e514bf98e8 # v3.2.0
       with:
@@ -87,7 +69,6 @@ jobs:
           -DCMAKE_CXX_COMPILER=clang++-18 \
           -DCMAKE_LINKER_TYPE=LLD \
           -DSHORTFIN_BUNDLE_DEPS=ON \
-          -DSHORTFIN_IREE_SOURCE_DIR="${{ env.IREE_REPO_DIR }}" \
           -DSHORTFIN_BUILD_PYTHON_BINDINGS=ON
         cmake --build build --target all
         pip install -v -e build/

diff --git a/requirements-iree-pinned.txt b/requirements-iree-pinned.txt
@@ -0,0 +1,10 @@
+# Pinned versions of IREE dependencies.
+
+# Keep these versions synced with SHORTFIN_IREE_GIT_TAG in shortfin/CMakeLists.txt
+--pre
+--find-links https://iree.dev/pip-release-links.html
+iree-base-compiler==3.1.0rc20241204
+iree-base-runtime==3.1.0rc20241204
+
+# TODO(#760): include iree-turbine in this requirements file too?
+# iree-turbine==3.1.0rc20241205
diff --git a/requirements-iree-unpinned.txt b/requirements-iree-unpinned.txt
@@ -0,0 +1,9 @@
+# Unpinned versions of IREE dependencies.
+
+--pre
+--find-links https://iree.dev/pip-release-links.html
+iree-base-compiler
+iree-base-runtime
+
+# TODO(#760): include iree-turbine in this requirements file too?
+# iree-turbine
diff --git a/shortfin/CMakeLists.txt b/shortfin/CMakeLists.txt
@@ -43,7 +43,10 @@ endif()
 add_compile_options("$<$<C_COMPILER_ID:MSVC>:/utf-8>")
 add_compile_options("$<$<CXX_COMPILER_ID:MSVC>:/utf-8>")
 
-# Pins
+# Version pins for dependencies.
+# Prefer to keep the IREE git tag synced with the Python package version in the
+# requirements-iree-pinned.txt file. At a minimum, the compiler from those
+# packages must be compatible with the runtime at this source ref.
 set(SHORTFIN_IREE_GIT_TAG "iree-3.1.0rc20241220")
 
 # build options

diff --git a/shortfin/python/shortfin_apps/sd/components/config_artifacts.py b/shortfin/python/shortfin_apps/sd/components/config_artifacts.py
@@ -8,7 +8,7 @@
 from iree.build.executor import FileNamespace
 import os
 
-ARTIFACT_VERSION = "11182024"
+ARTIFACT_VERSION = "01062025"
 SDXL_CONFIG_BUCKET = f"https://sharkpublic.blob.core.windows.net/sharkpublic/sdxl/{ARTIFACT_VERSION}/configs/"
 
 

diff --git a/shortfin/requirements-iree-compiler.txt b/shortfin/requirements-iree-compiler.txt