Fix Documentation CI + General CI Cleanup (#7105)

* Removed cmake update from docs ci * Centralize python dependency versions to requirements files * Updated pyproject.toml to be in sync with requirements files * Added missing requirements_build.txt to dockerfile * Updated pip (23.2.1 -> 24.3.1) * Added explicit cmake install and removed Ubuntu 20.04 from comment * Fixed Documentation CI Ubuntu version to 22.04
isl-org · Dec 30, 2024 · bfb44de · bfb44de
1 parent 34d8039
commit bfb44de
Show file tree

Hide file tree

Showing 6 changed files with 40 additions and 39 deletions.
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -22,7 +22,9 @@ jobs:
   headless-docs:   # Build headless and docs
     permissions:
       contents: write  # Artifact upload and release upload
-    runs-on: ubuntu-latest  # Warn about build issues in new versions
+    # Pinning to 22.04 instead of ubuntu-latest until 24.04 is supported
+    # (see https://github.com/isl-org/Open3D/pull/7105)
+    runs-on: ubuntu-22.04
     env:
       OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML
       DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }}

diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
@@ -27,7 +27,7 @@ jobs:
           python-version: '3.8'
       - name: Install dependencies
         run: |
-          pip install -U clang-format~=10.0.0 yapf==0.30.0 nbformat
+          pip install -U -r python/requirements_style.txt
       - name: Run style check
         run: |
           python util/check_style.py
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -20,11 +20,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  PIP_VER: "23.2.1"
-  WHEEL_VER: "0.38.4"
-  STOOLS_VER: "67.3.2"
-  JEDI_VER: "0.17.2"  # https://github.com/ipython/ipython/issues/12740
-  IDNA_VER: "2.8"  # https://github.com/psf/requests/issues/5710
+  PIP_VER: "24.3.1"
   CUDA_VERSION: "12.1.0"
   SRC_DIR: "D:\\a\\open3d\\open3d"
   BUILD_DIR: "C:\\Open3D\\build"
@@ -213,15 +209,16 @@ jobs:
           cmake --build . --config ${{ matrix.CONFIG }}
           .\${{ matrix.CONFIG }}\Draw.exe --skip-for-unit-test
           Remove-Item "C:\Program Files\Open3D" -Recurse
+      - name: Install Open3D python build requirements
+        working-directory: ${{ env.SOURCE_DIR }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          python -m pip install -U pip==${{ env.PIP_VER }}
+          python -m pip install -U -c python/requirements_build.txt wheel setuptools
       - name: Install Python package
         working-directory: ${{ env.BUILD_DIR }}
         run: |
           $ErrorActionPreference = 'Stop'
-          python -m pip install --upgrade pip==${{ env.PIP_VER }} `
-            wheel==${{ env.WHEEL_VER }} `
-            setuptools==${{ env.STOOLS_VER }} `
-            jedi==${{ env.JEDI_VER }} `
-            idna==${{ env.IDNA_VER }}
           cmake --build . --config ${{ matrix.CONFIG }} --target install-pip-package
       - name: Import python package
         # If BUILD_SHARED_LIBS == ON, Open3D.dll needs to be copied, which is not recommended for python.
@@ -296,13 +293,17 @@ jobs:
             -DBUILD_JUPYTER_EXTENSION=ON `
             ${{ env.SRC_DIR }}
 
+      - name: Install Open3D python build requirements
+        working-directory: ${{ env.SOURCE_DIR }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          python -m pip install -U pip==${{ env.PIP_VER }}
+          python -m pip install -U -c python/requirements_build.txt wheel setuptools
+
       - name: Build Python package
         working-directory: ${{ env.BUILD_DIR }}
         run: |
           $ErrorActionPreference = 'Stop'
-          python -m pip install --upgrade pip==${{ env.PIP_VER }} `
-                                          wheel==${{ env.WHEEL_VER }} `
-                                          setuptools==${{ env.STOOLS_VER }}
           cmake --build . --parallel ${{ env.NPROC }} --config Release --target pip-package
           $PIP_PKG_NAME=(Get-ChildItem lib/python_package/pip_package/open3d*.whl).Name
           echo "PIP_PKG_NAME=$PIP_PKG_NAME"  | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
@@ -365,10 +366,8 @@ jobs:
           python -V
           python -m venv open3d_test_venv
           open3d_test_venv\Scripts\Activate.ps1
-
-          python -m pip install --upgrade pip==${{ env.PIP_VER }} `
-            wheel==${{ env.WHEEL_VER }} `
-            setuptools==${{ env.STOOLS_VER }}
+          python -m pip install -U pip==${{ env.PIP_VER }}
+          python -m pip install -U -c python/requirements_build.txt wheel setuptools
           python -m pip install -U -r python/requirements_test.txt
           $py_tag=(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')")
           if (Test-Path -Path "pip_package") {

diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel
@@ -111,6 +111,7 @@ COPY ./util/ci_utils.sh /root/Open3D/util/
 COPY ./python/requirements.txt /root/Open3D/python/
 COPY ./python/requirements_jupyter_build.txt /root/Open3D/python/
 COPY ./python/requirements_jupyter_install.txt /root/Open3D/python/
+COPY ./python/requirements_build.txt /root/Open3D/python/
 RUN source /root/Open3D/util/ci_utils.sh \
  && install_python_dependencies with-jupyter
 

diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -1,3 +1,10 @@
 [build-system]
-requires = ["ipywidgets>=8.0.3", "pygments>=2.7.4", "jupyter_packaging~=0.12", "jupyterlab>=3.0.0,==3.*", "setuptools>=50.3.2", "wheel==0.38.4"]
+requires = [
+    "ipywidgets>=8.0.4",
+    "pygments>=2.7.4",
+    "jupyter_packaging~=0.12",
+    "jupyterlab>=3.0.0,==3.*",
+    "setuptools>=67.3.2",
+    "wheel==0.38.4",
+]
 build-backend = "setuptools.build_meta"
diff --git a/util/ci_utils.sh b/util/ci_utils.sh
@@ -29,10 +29,7 @@ TENSORFLOW_VER="2.16.2"
 TORCH_VER="2.2.2"
 TORCH_REPO_URL="https://download.pytorch.org/whl/torch/"
 # Python
-PIP_VER="23.2.1"
-WHEEL_VER="0.38.4"
-STOOLS_VER="67.3.2"
-YAPF_VER="0.30.0"
+PIP_VER="24.3.1"
 PROTOBUF_VER="4.24.0"
 
 OPEN3D_INSTALL_DIR=~/open3d_install
@@ -42,10 +39,10 @@ install_python_dependencies() {
 
     echo "Installing Python dependencies"
     options="$(echo "$@" | tr ' ' '|')"
-    python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \
-        setuptools=="$STOOLS_VER"
+    python -m pip install -U pip=="$PIP_VER"
+    python -m pip install -U -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" wheel setuptools
     if [[ "with-unit-test" =~ ^($options)$ ]]; then
-        python -m pip install -U -r python/requirements_test.txt
+        python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_test.txt"
     fi
     if [[ "with-cuda" =~ ^($options)$ ]]; then
         TF_ARCH_NAME=tensorflow
@@ -64,7 +61,6 @@ install_python_dependencies() {
         TORCH_GLNX="torch==${TORCH_VER}+cpu"
     fi
 
-    # TODO: modify other locations to use requirements.txt
     python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements.txt"
     if [[ "with-jupyter" =~ ^($options)$ ]]; then
         python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements_jupyter_build.txt"
@@ -88,7 +84,7 @@ install_python_dependencies() {
         fi
     fi
     if [ "$BUILD_TENSORFLOW_OPS" == "ON" ] || [ "$BUILD_PYTORCH_OPS" == "ON" ]; then
-        python -m pip install -U yapf=="$YAPF_VER"
+        python -m pip install -U -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" yapf
         # Fix Protobuf compatibility issue
         # https://stackoverflow.com/a/72493690/1255535
         # https://github.com/protocolbuffers/protobuf/issues/10051
@@ -241,8 +237,8 @@ test_wheel() {
     python -m venv open3d_test.venv
     # shellcheck disable=SC1091
     source open3d_test.venv/bin/activate
-    python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \
-        setuptools=="$STOOLS_VER"
+    python -m pip install -U pip=="$PIP_VER"
+    python -m pip install -U -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" wheel setuptools
     echo -n "Using python: $(command -v python)"
     python --version
     echo -n "Using pip: "
@@ -285,7 +281,7 @@ test_wheel() {
 run_python_tests() {
     # shellcheck disable=SC1091
     source open3d_test.venv/bin/activate
-    python -m pip install -U -r python/requirements_test.txt
+    python -m pip install -U -r "$OPEN3D_SOURCE_ROOT/python/requirements_test.txt"
     echo Add --randomly-seed=SEED to the test command to reproduce test order.
     pytest_args=("$OPEN3D_SOURCE_ROOT"/python/test/)
     if [ "$BUILD_PYTORCH_OPS" == "OFF" ] && [ "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then
@@ -335,14 +331,11 @@ test_cpp_example() {
     # Now I am in Open3D/build/
 }
 
-# Install dependencies needed for building documentation (on Ubuntu 20.04)
+# Install dependencies needed for building documentation
 # Usage: install_docs_dependencies "${OPEN3D_ML_ROOT}"
 install_docs_dependencies() {
     echo
     echo Install ubuntu dependencies
-    echo Update cmake needed in Ubuntu 20.04
-    sudo apt-key adv --fetch-keys https://apt.kitware.com/keys/kitware-archive-latest.asc
-    sudo apt-add-repository --yes 'deb https://apt.kitware.com/ubuntu/ focal main'
     ./util/install_deps_ubuntu.sh assume-yes
     sudo apt-get install --yes cmake
     sudo apt-get install --yes libxml2-dev libxslt-dev python3-dev
@@ -356,9 +349,8 @@ install_docs_dependencies() {
     echo Install Python dependencies for building docs
     command -v python
     python -V
-    python -m pip install -U -q "wheel==$WHEEL_VER" \
-        "pip==$PIP_VER"
-    python -m pip install -U -q "yapf==$YAPF_VER"
+    python -m pip install -U -q "pip==$PIP_VER"
+    python -m pip install -U -q -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" wheel yapf
     if [[ -d "$1" ]]; then
         OPEN3D_ML_ROOT="$1"
         echo Installing Open3D-ML dependencies from "${OPEN3D_ML_ROOT}"