From 10892f762e71defbed8d86587719a76fabd80eb0 Mon Sep 17 00:00:00 2001 From: Michael Norris Date: Mon, 20 Jan 2025 20:23:29 -0800 Subject: [PATCH] Migration off defaults to conda-forge channel (#4126) Summary: Good resource on overriding channels to make sure we aren't using `defaults`:https://stackoverflow.com/questions/67695893/how-do-i-completely-purge-and-disable-the-default-channel-in-anaconda-and-switch Explanation of changes: - - changed to miniforge from miniconda: this ensures we only pull in from conda-defaults when creating the environment - architecture: ARM64 and aarch64 are the same thing. But there is no miniforge package for ARM64, so we need to make it check for aarch64 instead. However, mac breaks this rule, and does have macOS-arm64! So there is a conditional for mac to use arm64. https://github.com/conda-forge/miniforge/releases/ - mkl 2022.2.1 change: conda-forge and defaults have completely different dependencies. Defaults required intel-openmp, but now on conda-forge, mkl 2023.1 or higher requires llvm-openmp >=14.0.6, but this is incompatible with the pytorch build <2.5 which requires llvm-openmp<14.0. We would need to upgrade Python to 3.12 first, upgrade Pytorch build, then upgrade this mkl. (The meta.yaml changes are the ones that narrow it to 2022.2.1 during `conda build faiss`.) So, this has just been changed to 2022.2.1. - mkl now requires _openmp_mutex of type "llvm" instead of "gnu": prior non-cuVS builds all used gnu, because intel-openmp from anaconda defaults channel does not require llvm-openmp. Now we need to remove the gnu one which is automatically pulled in during miniconda setup, and only keep the llvm version of _openmp_mutex. - liblief: The above changes tried to pull in liblief 0.15. This results in an error like `AttributeError: module 'lief._lief.ELF' has no attribute 'ELF_CLASS'`. When I checked passing PR builds on defaults, they use lief 0.12, so I pinned that one. - gcc_linux-64 =11.2 for faiss-gpu on cudatoolkit-11.2: this build kept trying to reference 11.2 when 14.2 was installed. Current builds still reference 11.2, so I gave up and pinned 11.2 to keep it the same. Moving to 14.2 will take some more investigation. - conda_build_config.yaml: I removed Python 3.12 because the build fails with the below error. Python 3.12 also doesn't really work (https://github.com/facebookresearch/faiss/issues/3936) I think? So I removed it. ``` INTEL MKL ERROR: $PREFIX/lib/python3.12/site-packages/faiss/../../.././libmkl_def.so.2: undefined symbol: mkl_sparse_optimize_bsr_trsm_i8. Intel MKL FATAL ERROR: Cannot load libmkl_def.so.2. ``` - Note: test_mem_leak.cpp seems flaky. It sometimes fails, then passes with rerun. Differential Revision: D68043874 --- .github/actions/build_cmake/action.yml | 17 +- .github/actions/build_conda/action.yml | 20 +- .github/workflows/build-pull-request.yml | 385 +++++++++++++++-------- conda/conda_build_config.yaml | 1 - conda/faiss-gpu/meta.yaml | 74 ++++- conda/faiss/meta.yaml | 21 +- faiss/gpu/GpuCloner.cpp | 1 + faiss/gpu/test/test_gpu_basics.py | 5 + 8 files changed, 376 insertions(+), 148 deletions(-) diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index fa20974af5..a5f9372aec 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -23,12 +23,19 @@ runs: uses: conda-incubator/setup-miniconda@v3 with: python-version: '3.11' - miniconda-version: latest + miniforge-version: latest # ensures conda-forge channel is used. + channels: conda-forge + conda-remove-defaults: 'true' + # Set to aarch64 if we're on arm64 because there's no miniforge ARM64 package, just aarch64. + # They are the same thing, just named differently. + architecture: ${{ runner.arch == 'ARM64' && 'aarch64' || runner.arch }} - name: Configure build environment shell: bash run: | # initialize Conda conda config --set solver libmamba + # Ensure starting packages are from conda-forge. + conda list --show-channel-urls conda update -y -q conda echo "$CONDA/bin" >> $GITHUB_PATH @@ -43,7 +50,7 @@ runs: if [ "${{ runner.arch }}" = "X64" ]; then # TODO: merge this with ARM64 conda install -y -q -c conda-forge gxx_linux-64=14.2 sysroot_linux-64=2.17 - conda install -y -q mkl=2023 mkl-devel=2023 + conda install -y -q mkl=2022.2.1 mkl-devel=2022.2.1 fi # no CUDA needed for ROCm so skip this @@ -56,6 +63,7 @@ runs: elif [ "${{ inputs.cuvs }}" = "ON" ]; then conda install -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c conda-forge fi + # install test packages if [ "${{ inputs.rocm }}" = "ON" ]; then : # skip torch install via conda, we need to install via pip to get @@ -174,3 +182,8 @@ runs: with: name: test-results-arch=${{ runner.arch }}-opt=${{ inputs.opt_level }}-gpu=${{ inputs.gpu }}-cuvs=${{ inputs.cuvs }}-rocm=${{ inputs.rocm }} path: test-results + - name: Check installed packages channel + shell: bash + run: | + # Shows that all installed packages are from conda-forge. + conda list --show-channel-urls diff --git a/.github/actions/build_conda/action.yml b/.github/actions/build_conda/action.yml index ff860007b2..d7e02e606e 100644 --- a/.github/actions/build_conda/action.yml +++ b/.github/actions/build_conda/action.yml @@ -26,20 +26,25 @@ runs: else echo "shell=pwsh" >> "$GITHUB_OUTPUT" fi + echo "${{ runner.arch }}" - name: Setup miniconda uses: conda-incubator/setup-miniconda@v3 with: python-version: '3.11' - miniconda-version: latest + miniforge-version: latest # ensures conda-forge channel is used. + channels: conda-forge + conda-remove-defaults: 'true' + # Set to aarch64 if we're on arm64 because there's no miniforge ARM64 package, just aarch64. + # They are the same thing, just named differently. + architecture: ${{ (runner.arch == 'ARM64' && runner.os != 'macOS') && 'aarch64' || runner.arch }} - name: Install conda build tools shell: ${{ steps.choose_shell.outputs.shell }} run: | + # Ensure starting packages are from conda-forge. + conda list --show-channel-urls conda install -y -q "conda!=24.11.0" conda install -y -q "conda-build!=24.11.0" - - name: Fix CI failure - shell: ${{ steps.choose_shell.outputs.shell }} - if: runner.os != 'Windows' - run: conda remove conda-anaconda-telemetry + conda list --show-channel-urls - name: Enable anaconda uploads if: inputs.label != '' shell: ${{ steps.choose_shell.outputs.shell }} @@ -94,3 +99,8 @@ runs: run: | conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \ --user pytorch --label ${{ inputs.label }} -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia + - name: Check installed packages channel + shell: ${{ steps.choose_shell.outputs.shell }} + run: | + # Shows that all installed packages are from conda-forge. + conda list --show-channel-urls diff --git a/.github/workflows/build-pull-request.yml b/.github/workflows/build-pull-request.yml index bc0d2d625a..6d3d3b55d8 100644 --- a/.github/workflows/build-pull-request.yml +++ b/.github/workflows/build-pull-request.yml @@ -30,140 +30,267 @@ jobs: git --no-pager diff --color exit 1 fi - linux-x86_64-cmake: - name: Linux x86_64 (cmake) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - linux-x86_64-AVX2-cmake: - name: Linux x86_64 AVX2 (cmake) - needs: linux-x86_64-cmake - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: avx2 - linux-x86_64-AVX512-cmake: - name: Linux x86_64 AVX512 (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-aws-m7i.large - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: avx512 - linux-x86_64-AVX512_SPR-cmake: - name: Linux x86_64 AVX512_SPR (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-aws-m7i.large - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: avx512_spr - linux-x86_64-GPU-cmake: - name: Linux x86_64 GPU (cmake) - needs: linux-x86_64-cmake + # linux-x86_64-cmake: + # name: Linux x86_64 (cmake) + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Build and Test (cmake) + # uses: ./.github/actions/build_cmake + # linux-x86_64-AVX2-cmake: + # name: Linux x86_64 AVX2 (cmake) + # needs: linux-x86_64-cmake + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Build and Test (cmake) + # uses: ./.github/actions/build_cmake + # with: + # opt_level: avx2 + # linux-x86_64-AVX512-cmake: + # name: Linux x86_64 AVX512 (cmake) + # needs: linux-x86_64-cmake + # runs-on: faiss-aws-m7i.large + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Build and Test (cmake) + # uses: ./.github/actions/build_cmake + # with: + # opt_level: avx512 + # linux-x86_64-AVX512_SPR-cmake: + # name: Linux x86_64 AVX512_SPR (cmake) + # needs: linux-x86_64-cmake + # runs-on: faiss-aws-m7i.large + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Build and Test (cmake) + # uses: ./.github/actions/build_cmake + # with: + # opt_level: avx512_spr + # linux-x86_64-GPU-cmake: + # name: Linux x86_64 GPU (cmake) + # needs: linux-x86_64-cmake + # runs-on: 4-core-ubuntu-gpu-t4 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Build and Test (cmake) + # uses: ./.github/actions/build_cmake + # with: + # gpu: ON + # linux-x86_64-GPU-w-CUVS-cmake: + # name: Linux x86_64 GPU w/ cuVS (cmake) + # needs: linux-x86_64-cmake + # runs-on: 4-core-ubuntu-gpu-t4 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Build and Test (cmake) + # uses: ./.github/actions/build_cmake + # with: + # gpu: ON + # cuvs: ON + # linux-x86_64-GPU-w-ROCm-cmake: + # name: Linux x86_64 GPU w/ ROCm (cmake) + # needs: linux-x86_64-cmake + # runs-on: faiss-amd-MI200 + # container: + # image: ubuntu:22.04 + # options: --device=/dev/kfd --device=/dev/dri --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN + # steps: + # - name: Container setup + # run: | + # if [ -f /.dockerenv ]; then + # apt-get update && apt-get install -y sudo && apt-get install -y git + # git config --global --add safe.directory '*' + # else + # echo 'Skipping. Current job is not running inside a container.' + # fi + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Build and Test (cmake) + # uses: ./.github/actions/build_cmake + # with: + # gpu: ON + # rocm: ON + # linux-arm64-SVE-cmake: + # name: Linux arm64 SVE (cmake) + # needs: linux-x86_64-cmake + # runs-on: faiss-aws-r8g.large + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Build and Test (cmake) + # uses: ./.github/actions/build_cmake + # with: + # opt_level: sve + # env: + # # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction + # OPENBLAS_NUM_THREADS: '1' + # linux-x86_64-conda: + # name: Linux x86_64 (conda) + # needs: linux-x86_64-cmake + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # fetch-tags: true + # - name: Build and Package (conda) + # uses: ./.github/actions/build_conda + # windows-x86_64-conda: + # name: Windows x86_64 (conda) + # needs: linux-x86_64-cmake + # runs-on: windows-2019 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # fetch-tags: true + # - name: Build and Package (conda) + # uses: ./.github/actions/build_conda + # linux-arm64-conda: + # name: Linux arm64 (conda) + # needs: linux-x86_64-cmake + # runs-on: 2-core-ubuntu-arm + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # fetch-tags: true + # - name: Build and Package (conda) + # uses: ./.github/actions/build_conda + # linux-x86_64-nightly: + # name: Linux x86_64 nightlies + # runs-on: 4-core-ubuntu + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # fetch-tags: true + # - uses: ./.github/actions/build_conda + # env: + # ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + # with: + # label: nightly + linux-x86_64-GPU-CUDA-11-4-4-nightly: + name: Linux x86_64 GPU nightlies (CUDA 11.4.4) runs-on: 4-core-ubuntu-gpu-t4 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - gpu: ON - linux-x86_64-GPU-w-CUVS-cmake: - name: Linux x86_64 GPU w/ cuVS (cmake) - needs: linux-x86_64-cmake - runs-on: 4-core-ubuntu-gpu-t4 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - gpu: ON - cuvs: ON - linux-x86_64-GPU-w-ROCm-cmake: - name: Linux x86_64 GPU w/ ROCm (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-amd-MI200 - container: - image: ubuntu:22.04 - options: --device=/dev/kfd --device=/dev/dri --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN - steps: - - name: Container setup - run: | - if [ -f /.dockerenv ]; then - apt-get update && apt-get install -y sudo && apt-get install -y git - git config --global --add safe.directory '*' - else - echo 'Skipping. Current job is not running inside a container.' - fi - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - gpu: ON - rocm: ON - linux-arm64-SVE-cmake: - name: Linux arm64 SVE (cmake) - needs: linux-x86_64-cmake - runs-on: faiss-aws-r8g.large - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test (cmake) - uses: ./.github/actions/build_cmake - with: - opt_level: sve - env: - # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction - OPENBLAS_NUM_THREADS: '1' - linux-x86_64-conda: - name: Linux x86_64 (conda) - needs: linux-x86_64-cmake - runs-on: ubuntu-latest + env: + CUDA_ARCHS: "60-real;61-real;62-real;70-real;72-real;75-real;80;86-real" + FAISS_FLATTEN_CONDA_INCLUDES: "1" steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - windows-x86_64-conda: - name: Windows x86_64 (conda) - needs: linux-x86_64-cmake - runs-on: windows-2019 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda - linux-arm64-conda: - name: Linux arm64 (conda) - needs: linux-x86_64-cmake - runs-on: 2-core-ubuntu-arm - steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: ./.github/actions/build_conda + env: + ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} with: - fetch-depth: 0 - fetch-tags: true - - name: Build and Package (conda) - uses: ./.github/actions/build_conda + label: nightly + cuda: "11.4.4" + # linux-x86_64-GPU-CUVS-CUDA11-8-0-nightly: + # name: Linux x86_64 GPU w/ cuVS nightlies (CUDA 11.8.0) + # runs-on: 4-core-ubuntu-gpu-t4 + # env: + # CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # fetch-tags: true + # - uses: ./.github/actions/build_conda + # env: + # ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + # with: + # label: nightly + # cuvs: "ON" + # cuda: "11.8.0" + # linux-x86_64-GPU-CUDA-12-1-1-nightly: + # name: Linux x86_64 GPU nightlies (CUDA 12.1.1) + # runs-on: 4-core-ubuntu-gpu-t4 + # env: + # CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # fetch-tags: true + # - uses: ./.github/actions/build_conda + # env: + # ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + # with: + # label: nightly + # cuda: "12.1.1" + # linux-x86_64-GPU-CUVS-CUDA12-4-0-nightly: + # name: Linux x86_64 GPU w/ cuVS nightlies (CUDA 12.4.0) + # runs-on: 4-core-ubuntu-gpu-t4 + # env: + # CUDA_ARCHS: "70-real;72-real;75-real;80;86-real" + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # fetch-tags: true + # - uses: ./.github/actions/build_conda + # env: + # ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + # with: + # label: nightly + # cuvs: "ON" + # cuda: "12.4.0" + # windows-x86_64-nightly: + # name: Windows x86_64 nightlies + # runs-on: windows-2019 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # fetch-tags: true + # - uses: ./.github/actions/build_conda + # env: + # ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + # with: + # label: nightly + # osx-arm64-nightly: + # name: OSX arm64 nightlies + # runs-on: macos-14 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # fetch-tags: true + # - uses: ./.github/actions/build_conda + # env: + # ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + # with: + # label: nightly + # linux-arm64-nightly: + # name: Linux arm64 nightlies + # runs-on: 2-core-ubuntu-arm + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # fetch-tags: true + # - uses: ./.github/actions/build_conda + # env: + # ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} + # with: + # label: nightly diff --git a/conda/conda_build_config.yaml b/conda/conda_build_config.yaml index 4df05146df..77f0eec0a2 100644 --- a/conda/conda_build_config.yaml +++ b/conda/conda_build_config.yaml @@ -2,4 +2,3 @@ python: - 3.9 - 3.10 - 3.11 - - 3.12 # [not aarch64] diff --git a/conda/faiss-gpu/meta.yaml b/conda/faiss-gpu/meta.yaml index 651d42fefa..f20549f386 100644 --- a/conda/faiss-gpu/meta.yaml +++ b/conda/faiss-gpu/meta.yaml @@ -50,20 +50,51 @@ outputs: - sysroot_linux-64 =2.17 # [linux64] - llvm-openmp # [osx] - cmake >=3.24.0 - - make =4.2 # [not win] - - mkl-devel =2023 # [x86_64] + - make =4.2 # [not win and not (osx and arm64)] + - make =4.4 # [osx and arm64] + - mkl-devel =2023.0 # [x86_64] - cuda-toolkit {{ cudatoolkit }} + # - mpc =1.1 # [cudatoolkit == '11.4.4'] + # - mpfr =4.0.2 # [cudatoolkit == '11.4.4'] + - scipy =1.14.1 # [cudatoolkit == '11.4.4'] + - sympy =1.13.3 # [cudatoolkit == '11.4.4'] + - gmp =6.2.1 # [cudatoolkit == '11.4.4'] + - pycparser =2.21 # [cudatoolkit == '11.4.4'] + - py-lief =0.12.3 # [cudatoolkit == '11.4.4'] + - gmpy2 =2.1.2 # [cudatoolkit == '11.4.4'] + - tbb =2021.8.0 # [cudatoolkit == '11.4.4'] + # - mkl_fft =1.3.11 # [cudatoolkit == '11.4.4'] host: - - mkl =2023 # [x86_64] + - mkl =2023.0 # [x86_64] - openblas =0.3 # [not x86_64] + # - mpc =1.1 # [cudatoolkit == '11.4.4'] + # - mpfr =4.0.2 # [cudatoolkit == '11.4.4'] + - scipy =1.14.1 # [cudatoolkit == '11.4.4'] + - sympy =1.13.3 # [cudatoolkit == '11.4.4'] + - gmp =6.2.1 # [cudatoolkit == '11.4.4'] + - pycparser =2.21 # [cudatoolkit == '11.4.4'] + - py-lief =0.12.3 # [cudatoolkit == '11.4.4'] + - gmpy2 =2.1.2 # [cudatoolkit == '11.4.4'] + - tbb =2021.8.0 # [cudatoolkit == '11.4.4'] + # - mkl_fft =1.3.11 # [cudatoolkit == '11.4.4'] run: - - mkl =2023 # [x86_64] + - mkl =2023.0 # [x86_64] - openblas =0.3 # [not x86_64] - cuda-cudart {{ cuda_constraints }} - libcublas {{ libcublas_constraints }} test: requires: - conda-build + # - mpc =1.1 # [cudatoolkit == '11.4.4'] + # - mpfr =4.0.2 # [cudatoolkit == '11.4.4'] + - scipy =1.14.1 # [cudatoolkit == '11.4.4'] + - sympy =1.13.3 # [cudatoolkit == '11.4.4'] + - gmp =6.2.1 # [cudatoolkit == '11.4.4'] + - pycparser =2.21 # [cudatoolkit == '11.4.4'] + - py-lief =0.12.3 # [cudatoolkit == '11.4.4'] + - gmpy2 =2.1.2 # [cudatoolkit == '11.4.4'] + - tbb =2021.8.0 # [cudatoolkit == '11.4.4'] + # - mkl_fft =1.3.11 # [cudatoolkit == '11.4.4'] commands: - test -f $PREFIX/lib/libfaiss$SHLIB_EXT # [not win] - test -f $PREFIX/lib/libfaiss_avx2$SHLIB_EXT # [x86_64 and not win] @@ -83,12 +114,35 @@ outputs: - sysroot_linux-64 =2.17 # [linux64] - swig =4.0 - cmake >=3.24.0 - - make =4.2 # [not win] + - make =4.2 # [not win and not (osx and arm64)] + - make =4.4 # [osx and arm64] + - _openmp_mutex =4.5=2_kmp_llvm # [x86_64 and not win] - cuda-toolkit {{ cudatoolkit }} + # - mpc =1.1 # [cudatoolkit == '11.4.4'] + # - mpfr =4.0.2 # [cudatoolkit == '11.4.4'] + - scipy =1.14.1 # [cudatoolkit == '11.4.4'] + - sympy =1.13.3 # [cudatoolkit == '11.4.4'] + - gmp =6.2.1 # [cudatoolkit == '11.4.4'] + - pycparser =2.21 # [cudatoolkit == '11.4.4'] + - py-lief =0.12.3 # [cudatoolkit == '11.4.4'] + - gmpy2 =2.1.2 # [cudatoolkit == '11.4.4'] + - tbb =2021.8.0 # [cudatoolkit == '11.4.4'] + # - mkl_fft =1.3.11 # [cudatoolkit == '11.4.4'] host: - python {{ python }} - numpy >=1.19,<2 + - _openmp_mutex =4.5=2_kmp_llvm # [x86_64 and not win] - {{ pin_subpackage('libfaiss', exact=True) }} + # - mpc =1.1 # [cudatoolkit == '11.4.4'] + # - mpfr =4.0.2 # [cudatoolkit == '11.4.4'] + - scipy =1.14.1 # [cudatoolkit == '11.4.4'] + - sympy =1.13.3 # [cudatoolkit == '11.4.4'] + - gmp =6.2.1 # [cudatoolkit == '11.4.4'] + - pycparser =2.21 # [cudatoolkit == '11.4.4'] + - py-lief =0.12.3 # [cudatoolkit == '11.4.4'] + - gmpy2 =2.1.2 # [cudatoolkit == '11.4.4'] + - tbb =2021.8.0 # [cudatoolkit == '11.4.4'] + # - mkl_fft =1.3.11 # [cudatoolkit == '11.4.4'] run: - python {{ python }} - numpy >=1.19,<2 @@ -100,6 +154,16 @@ outputs: - scipy - pytorch <2.5 - pytorch-cuda {{ cuda_constraints }} + # - mpc =1.1 # [cudatoolkit == '11.4.4'] + # - mpfr =4.0.2 # [cudatoolkit == '11.4.4'] + - scipy =1.14.1 # [cudatoolkit == '11.4.4'] + - sympy =1.13.3 # [cudatoolkit == '11.4.4'] + - gmp =6.2.1 # [cudatoolkit == '11.4.4'] + - pycparser =2.21 # [cudatoolkit == '11.4.4'] + - py-lief =0.12.3 # [cudatoolkit == '11.4.4'] + - gmpy2 =2.1.2 # [cudatoolkit == '11.4.4'] + - tbb =2021.8.0 # [cudatoolkit == '11.4.4'] + # - mkl_fft =1.3.11 # [cudatoolkit == '11.4.4'] commands: - python -X faulthandler -m unittest discover -v -s tests/ -p "test_*" - python -X faulthandler -m unittest discover -v -s tests/ -p "torch_*" diff --git a/conda/faiss/meta.yaml b/conda/faiss/meta.yaml index fe7612c23b..8696df6ee7 100644 --- a/conda/faiss/meta.yaml +++ b/conda/faiss/meta.yaml @@ -38,15 +38,18 @@ outputs: build: - {{ compiler('cxx') }} - sysroot_linux-64 =2.17 # [linux64] - - llvm-openmp # [osx] + - llvm-openmp # [osx or linux64] - cmake >=3.24.0 - - make =4.2 # [not win] - - mkl-devel =2023 # [x86_64] + - make =4.2 # [not win and not (osx and arm64)] + - make =4.4 # [osx and arm64] + - mkl-devel =2023.0 # [x86_64] + - liblief =0.12.3 # [not win] host: - - mkl =2023 # [x86_64] + - mkl =2023.0 # [x86_64] - openblas =0.3 # [not x86_64] + - liblief =0.12.3 # [not win] run: - - mkl =2023 # [x86_64] + - mkl =2023.0 # [x86_64] - openblas =0.3 # [not x86_64] test: requires: @@ -70,16 +73,22 @@ outputs: - sysroot_linux-64 =2.17 # [linux64] - swig =4.0 - cmake >=3.24.0 - - make =4.2 # [not win] + - make =4.2 # [not win and not (osx and arm64)] + - make =4.4 # [osx and arm64] + - _openmp_mutex =4.5=2_kmp_llvm # [x86_64 and not win] + - mkl =2023.0 # [win] host: - python {{ python }} - numpy >=1.19,<2 - {{ pin_subpackage('libfaiss', exact=True) }} + - _openmp_mutex =4.5=2_kmp_llvm # [x86_64 and not win] + - mkl =2023.0 # [win] run: - python {{ python }} - numpy >=1.19,<2 - packaging - {{ pin_subpackage('libfaiss', exact=True) }} + - mkl =2023.0 # [win] test: requires: - numpy >=1.19,<2 diff --git a/faiss/gpu/GpuCloner.cpp b/faiss/gpu/GpuCloner.cpp index 575ee2e0a5..21b30df6db 100644 --- a/faiss/gpu/GpuCloner.cpp +++ b/faiss/gpu/GpuCloner.cpp @@ -514,6 +514,7 @@ GpuProgressiveDimIndexFactory::~GpuProgressiveDimIndexFactory() { Index* GpuProgressiveDimIndexFactory::operator()(int dim) { IndexFlatL2 index(dim); ncall++; + printf("Incremented ncall: %d\n", ncall); return index_cpu_to_gpu_multiple(vres, devices, &index, &options); } diff --git a/faiss/gpu/test/test_gpu_basics.py b/faiss/gpu/test/test_gpu_basics.py index 00506bf1f1..fab10e86c8 100755 --- a/faiss/gpu/test/test_gpu_basics.py +++ b/faiss/gpu/test/test_gpu_basics.py @@ -443,11 +443,16 @@ def test_with_gpu(self): fac = faiss.GpuProgressiveDimIndexFactory(1) rq1.assign_index_factory = fac rq1.train(xt) + print(f"fac.ncall (1) ={fac.ncall}") self.assertGreater(fac.ncall, 0) + print(f"fac.ncall (2) ={fac.ncall}") ncall_train = fac.ncall + print(f"fac.ncall (3) ={fac.ncall} *** ncall_train={ncall_train}") err_rq1 = eval_codec(rq1, xb) + print(f"fac.ncall (4) ={fac.ncall} *** ncall_train={ncall_train}") # codes1 = rq1.compute_codes(xb) self.assertGreater(fac.ncall, ncall_train) + print(f"fac.ncall (5) ={fac.ncall} *** ncall_train={ncall_train}") print(err_rq0, err_rq1)