From 2c699fd316072f4f9f4597f730357fd873d18c6b Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Tue, 7 Jan 2025 15:30:20 -0800 Subject: [PATCH] Switch presubmit CI workflows to use pinned IREE versions. (#774) Progress on https://github.com/nod-ai/shark-ai/issues/760. The idea here is that we will test with only pinned versions in all workflows that run on `pull_request` and `push` triggers, then we will create pull requests (ideally via automation like dependabot) that attempt to update the pinned versions. This will give us confidence that test regressions are _only_ due to the code changes in the pull request and not due to a dependency changing. Workflows will also be more reproducible as the versions they fetch will come from source code and not an external, time-dependent source. --- .github/workflows/ci-llama-quick-tests.yaml | 15 ++---- .github/workflows/ci-shark-ai.yml | 16 ++----- .github/workflows/ci-sharktank.yml | 52 +++++++++------------ .github/workflows/ci_eval_short.yaml | 15 ++---- requirements-iree-pinned.txt | 8 ++-- requirements-iree-unpinned.txt | 4 +- sharktank/requirements-tests.txt | 3 +- 7 files changed, 42 insertions(+), 71 deletions(-) diff --git a/.github/workflows/ci-llama-quick-tests.yaml b/.github/workflows/ci-llama-quick-tests.yaml index 7ad153924..d41f3bc23 100644 --- a/.github/workflows/ci-llama-quick-tests.yaml +++ b/.github/workflows/ci-llama-quick-tests.yaml @@ -55,18 +55,13 @@ jobs: python -m pip install --no-compile --upgrade pip # Note: We install in three steps in order to satisfy requirements - # from non default locations first. Installing the PyTorch CPU - # wheels saves multiple minutes and a lot of bandwidth on runner setup. + # from non default locations first. pip install --no-compile -r pytorch-cpu-requirements.txt + pip install -r requirements-iree-pinned.txt + pip install --no-compile \ + -r sharktank/requirements-tests.txt \ + -e sharktank/ - # Install nightly IREE packages. - # We could also pin to a known working or stable version. - pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \ - iree-base-compiler \ - iree-base-runtime \ - iree-turbine - - pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/ pip freeze - name: Run llama 8b f16 decomposed test diff --git a/.github/workflows/ci-shark-ai.yml b/.github/workflows/ci-shark-ai.yml index 211fea4ba..3957b6d11 100644 --- a/.github/workflows/ci-shark-ai.yml +++ b/.github/workflows/ci-shark-ai.yml @@ -51,18 +51,12 @@ jobs: python -m pip install --no-compile --upgrade pip # Note: We install in three steps in order to satisfy requirements - # from non default locations first. Installing the PyTorch CPU - # wheels saves multiple minutes and a lot of bandwidth on runner setup. + # from non default locations first. pip install --no-compile -r pytorch-cpu-requirements.txt - - # Install nightly IREE packages. - # We could also pin to a known working or stable version. - pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \ - iree-base-compiler \ - iree-base-runtime \ - iree-turbine - - pip install --no-compile -r requirements.txt -e sharktank/ shortfin/ + pip install -r requirements-iree-pinned.txt + pip install --no-compile \ + -r requirements.txt \ + -e sharktank/ shortfin/ pip freeze diff --git a/.github/workflows/ci-sharktank.yml b/.github/workflows/ci-sharktank.yml index 4cdd2b274..690f08d3e 100644 --- a/.github/workflows/ci-sharktank.yml +++ b/.github/workflows/ci-sharktank.yml @@ -72,18 +72,15 @@ jobs: python -m pip install --no-compile --upgrade pip # Note: We install in three steps in order to satisfy requirements - # from non default locations first. Installing the PyTorch CPU - # wheels saves multiple minutes and a lot of bandwidth on runner setup. - pip install --no-compile --index-url https://download.pytorch.org/whl/cpu torch==${{matrix.torch-version}}+cpu + # from non default locations first. + pip install --no-compile \ + --index-url https://download.pytorch.org/whl/cpu torch==${{matrix.torch-version}}+cpu + pip install -r requirements-iree-pinned.txt + pip install --no-compile \ + -r sharktank/requirements-tests.txt \ + -e sharktank/ - # Install nightly IREE packages. - # We could also pin to a known working or stable version. - pip install -f https://iree.dev/pip-release-links.html --pre \ - iree-base-compiler \ - iree-base-runtime \ - iree-turbine - - pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/ + pip freeze - name: Run sharktank tests if: ${{ !cancelled() }} @@ -121,19 +118,16 @@ jobs: run: | source ${VENV_DIR}/bin/activate python -m pip install --no-compile --upgrade pip + # Note: We install in three steps in order to satisfy requirements - # from non default locations first. Installing the PyTorch CPU - # wheels saves multiple minutes and a lot of bandwidth on runner setup. + # from non default locations first. pip install --no-compile -r pytorch-cpu-requirements.txt + pip install -r requirements-iree-pinned.txt + pip install --no-compile \ + -r sharktank/requirements-tests.txt \ + -e sharktank/ - # Install nightly IREE packages. - # We could also pin to a known working or stable version. - pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \ - iree-base-compiler \ - iree-base-runtime \ - iree-turbine - - pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/ + pip freeze - name: Run tests # TODO: unify with-*-data flags into a single flag and make it possible to run @@ -180,18 +174,14 @@ jobs: python -m pip install --no-compile --upgrade pip # Note: We install in three steps in order to satisfy requirements - # from non default locations first. Installing the PyTorch CPU - # wheels saves multiple minutes and a lot of bandwidth on runner setup. + # from non default locations first. pip install --no-compile -r pytorch-cpu-requirements.txt + pip install -r requirements-iree-pinned.txt + pip install --no-compile \ + -r sharktank/requirements-tests.txt \ + -e sharktank/ - # Install nightly IREE packages. - # We could also pin to a known working or stable version. - pip install -f https://iree.dev/pip-release-links.html --pre \ - iree-base-compiler \ - iree-base-runtime \ - iree-turbine - - pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/ + pip freeze - name: Run punet tests run: | diff --git a/.github/workflows/ci_eval_short.yaml b/.github/workflows/ci_eval_short.yaml index 05c7fa415..a02d7bf14 100644 --- a/.github/workflows/ci_eval_short.yaml +++ b/.github/workflows/ci_eval_short.yaml @@ -52,18 +52,13 @@ jobs: python -m pip install --no-compile --upgrade pip # Note: We install in three steps in order to satisfy requirements - # from non default locations first. Installing the PyTorch CPU - # wheels saves multiple minutes and a lot of bandwidth on runner setup. + # from non default locations first. pip install --no-compile -r pytorch-cpu-requirements.txt + pip install -r requirements-iree-pinned.txt + pip install --no-compile \ + -r sharktank/requirements-tests.txt \ + -e sharktank/ - # Install nightly IREE packages. - # We could also pin to a known working or stable version. - pip install -f https://iree.dev/pip-release-links.html --pre --upgrade \ - iree-base-compiler \ - iree-base-runtime \ - iree-turbine - - pip install --no-compile -r requirements.txt -r sharktank/requirements-tests.txt -e sharktank/ pip freeze - name: Run perplexity test with vmfb diff --git a/requirements-iree-pinned.txt b/requirements-iree-pinned.txt index 11d7e4258..6895fa7d4 100644 --- a/requirements-iree-pinned.txt +++ b/requirements-iree-pinned.txt @@ -3,8 +3,6 @@ # Keep these versions synced with SHORTFIN_IREE_GIT_TAG in shortfin/CMakeLists.txt --pre --find-links https://iree.dev/pip-release-links.html -iree-base-compiler==3.1.0rc20241206 -iree-base-runtime==3.1.0rc20241206 - -# TODO(#760): include iree-turbine in this requirements file too? -# iree-turbine==3.1.0rc20241206 +iree-base-compiler==3.1.0rc20250107 +iree-base-runtime==3.1.0rc20250107 +iree-turbine==3.1.0rc20250107 diff --git a/requirements-iree-unpinned.txt b/requirements-iree-unpinned.txt index 09d4688dd..878541a40 100644 --- a/requirements-iree-unpinned.txt +++ b/requirements-iree-unpinned.txt @@ -4,6 +4,4 @@ --find-links https://iree.dev/pip-release-links.html iree-base-compiler iree-base-runtime - -# TODO(#760): include iree-turbine in this requirements file too? -# iree-turbine +iree-turbine diff --git a/sharktank/requirements-tests.txt b/sharktank/requirements-tests.txt index a0ddf6117..e42d68c8c 100644 --- a/sharktank/requirements-tests.txt +++ b/sharktank/requirements-tests.txt @@ -1,5 +1,6 @@ datasets==3.0.0 +diffusers parameterized pytest==8.0.0 pytest-html -diffusers +pytest-xdist==3.5.0