From 1dd081faf91ac01081912aa0ea1bcd76e1b29d59 Mon Sep 17 00:00:00 2001 From: Michael Norris Date: Mon, 13 Jan 2025 19:33:47 -0800 Subject: [PATCH] Start migration off defaults to conda-forge channel (#4126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Good resource on overriding channels to make sure we aren't using `defaults`:https://stackoverflow.com/questions/67695893/how-do-i-completely-purge-and-disable-the-default-channel-in-anaconda-and-switch Explanation of changes: - - Python upgraded to 3.12 because otherwise we fail with the below error, and I thought we might want to keep telemetry? Let me know otherwise ``` LibMambaUnsatisfiableError: Encountered problems while solving: - package conda-anaconda-telemetry-0.1.1-py312h06a4308_0 requires python >=3.12,<3.13.0a0, but none of the providers can be installed Could not solve for environment specs The following packages are incompatible ├─ conda-anaconda-telemetry is installable and it requires │ └─ python >=3.12,<3.13.0a0 , which can be installed; └─ python 3.11** is not installable because it conflicts with any installable versions previously reported. ``` - Solver changes: https://github.com/conda/conda-libmamba-solver/issues/283#issue-1903412567 We need to manually install libmamba from conda-forge instead of defaults, then use it. If it and libarchive come from different channels, build fails with: ``` Error while loading conda entry point: conda-libmamba-solver (libarchive.so.20: cannot open shared object file: No such file or directory) CondaValueError: You have chosen a non-default solver backend (libmamba) but it was not recognized. Choose one of: classic ``` or the very cryptic: (it refers to glibc version...) ``` Your installed version is: 2.35 ``` - Why not use the classic solver, why do we need the above change to keep using libmamba? classic is not able to resolve dependencies for us. Others suggest just to use libmamba: https://github.com/conda/conda-build/issues/5516#issuecomment-2521252024. Classic fails with ``` DependencyNeedsBuildingError: Unsatisfiable dependencies for platform linux-64: {'__glibc', '__archspec', '__unix', '__conda', '__linux'} ``` - cpuinfo.py copied in because Python 3.12 removes distutils, and setuptools does not have feature parity. People have complained that they can't use FAISS with Python > 3.11 before and this is the dependency that I see. https://github.com/facebookresearch/faiss/issues/3936 - cuda-toolkit upgraded to 12.5.1 because of this error on 12.4.1: ``` nvlink fatal : Input file '/home/runner/miniconda3/bin/../lib/libcudadevrt.a:cuda_device_runtime.o' newer than toolkit (126 vs 124) ``` - Differential Revision: D68043874 --- .github/actions/build_cmake/action.yml | 49 +- .github/actions/build_conda/action.yml | 55 +- faiss/python/CMakeLists.txt | 1 + faiss/python/__init__.py | 1 + faiss/python/cpuinfo.py | 674 +++++++++++++++++++++++++ faiss/python/loader.py | 16 +- faiss/python/setup.py | 1 + 7 files changed, 763 insertions(+), 34 deletions(-) create mode 100644 faiss/python/cpuinfo.py diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml index fa20974af5..799ccb63ee 100644 --- a/.github/actions/build_cmake/action.yml +++ b/.github/actions/build_cmake/action.yml @@ -22,49 +22,80 @@ runs: - name: Setup miniconda uses: conda-incubator/setup-miniconda@v3 with: + channels: conda-forge + conda-remove-defaults: 'true' python-version: '3.11' miniconda-version: latest + conda-solver: "classic" + - name: Fix CI failure + shell: bash + run: conda remove conda-anaconda-telemetry - name: Configure build environment shell: bash run: | # initialize Conda + conda list + echo "*****1*****" + conda config --show channels + echo "*****2*****" + conda config --show-sources + # echo "*****2.1*****" + # conda remove conda-libmamba-solver + echo "*****2.2*****" + conda install -c conda-forge --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive --override-channels + echo "*****2.3*****" + conda list + echo "*****2.4*****" conda config --set solver libmamba - conda update -y -q conda + echo "*****2.5*****" + conda update -c conda-forge -y -q conda --override-channels echo "$CONDA/bin" >> $GITHUB_PATH + echo "*****3*****" + conda config --show channels + + echo "*****4.1*****" - conda install -y -q python=3.11 cmake=3.26 make=4.2 swig=4.0 "numpy<2" scipy=1.14 pytest=7.4 gflags=2.2 + conda install -c conda-forge -y -q python=3.11 cmake=3.26 make=4.2 swig=4.0 "numpy<2" scipy=1.14 pytest=7.4 gflags=2.2 --override-channels + echo "*****5*****" + conda config --show channels + echo "*****6*****" # install base packages for ARM64 if [ "${{ runner.arch }}" = "ARM64" ]; then - conda install -y -q -c conda-forge openblas=0.3 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17 + conda install -c conda-forge -y -q openblas=0.3 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17 --override-channels fi + echo "*****6.1*****" # install base packages for X86_64 if [ "${{ runner.arch }}" = "X64" ]; then # TODO: merge this with ARM64 - conda install -y -q -c conda-forge gxx_linux-64=14.2 sysroot_linux-64=2.17 - conda install -y -q mkl=2023 mkl-devel=2023 + conda install -c conda-forge -y -q gxx_linux-64=14.2 sysroot_linux-64=2.17 --override-channels + conda install -c conda-forge -y -q mkl=2023 mkl-devel=2023 --override-channels fi + echo "*****6.2*****" # no CUDA needed for ROCm so skip this if [ "${{ inputs.rocm }}" = "ON" ]; then : # regular CUDA for GPU builds elif [ "${{ inputs.gpu }}" = "ON" ] && [ "${{ inputs.cuvs }}" = "OFF" ]; then - conda install -y -q cuda-toolkit=12.4 -c "nvidia/label/cuda-12.4.0" + conda install -c conda-forge -y -q cuda-toolkit=12.4 -c "nvidia/label/cuda-12.4.0" --override-channels # and CUDA from cuVS channel for cuVS builds elif [ "${{ inputs.cuvs }}" = "ON" ]; then - conda install -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c conda-forge + conda install -c conda-forge -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai --override-channels fi + echo "*****6.3*****" # install test packages if [ "${{ inputs.rocm }}" = "ON" ]; then : # skip torch install via conda, we need to install via pip to get # ROCm-enabled version until it's supported in conda by PyTorch elif [ "${{ inputs.gpu }}" = "ON" ]; then - conda install -y -q "pytorch<2.5" pytorch-cuda=12.4 -c pytorch -c "nvidia/label/cuda-12.4.0" + conda install -c conda-forge -y -q "pytorch<2.5" pytorch-cuda=12.4 -c pytorch -c "nvidia/label/cuda-12.4.1" --override-channels else - conda install -y -q "pytorch<2.5" -c pytorch + conda install -c conda-forge -y -q "pytorch<2.5" -c pytorch --override-channels fi + echo "*****7*****" + conda config --show channels - name: ROCm - Install dependencies if: inputs.rocm == 'ON' shell: bash diff --git a/.github/actions/build_conda/action.yml b/.github/actions/build_conda/action.yml index ff860007b2..ec3601af3e 100644 --- a/.github/actions/build_conda/action.yml +++ b/.github/actions/build_conda/action.yml @@ -31,29 +31,48 @@ runs: with: python-version: '3.11' miniconda-version: latest - - name: Install conda build tools - shell: ${{ steps.choose_shell.outputs.shell }} - run: | - conda install -y -q "conda!=24.11.0" - conda install -y -q "conda-build!=24.11.0" + channels: conda-forge + conda-remove-defaults: "true" + conda-solver: "classic" - name: Fix CI failure shell: ${{ steps.choose_shell.outputs.shell }} if: runner.os != 'Windows' run: conda remove conda-anaconda-telemetry + - name: Install conda build tools + shell: ${{ steps.choose_shell.outputs.shell }} + run: | + echo "*****1*****" + conda list + # echo "*****2.1*****" + # conda remove conda-libmamba-solver + echo "*****2.2*****" + conda install -c conda-forge --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive --override-channels + echo "*****2.2*****" + conda list + echo "*****2.3*****" + conda config --set solver libmamba + echo "*****2.4*****" + # TODO: try without zlib + conda install -c conda-forge -y -q "conda!=24.11.0" --override-channels + conda install -c conda-forge -y -q "conda-build!=24.11.0" --override-channels - name: Enable anaconda uploads if: inputs.label != '' shell: ${{ steps.choose_shell.outputs.shell }} env: PACKAGE_TYPE: ${{ inputs.label }} run: | - conda install -y -q anaconda-client - conda config --set anaconda_upload yes + conda install -c conda-forge -y -q anaconda-client --override-channels + conda config -c conda-forge --set anaconda_upload yes --override-channels - name: Conda build (CPU) if: inputs.label == '' && inputs.cuda == '' shell: ${{ steps.choose_shell.outputs.shell }} working-directory: conda run: | - conda build faiss --python 3.11 -c pytorch + # TODO remove + echo "*****3.1*****" + conda list + echo "*****3.2*****" + conda build -c conda-forge faiss --python 3.11 -c pytorch --override-channels - name: Conda build (CPU) w/ anaconda upload if: inputs.label != '' && inputs.cuda == '' shell: ${{ steps.choose_shell.outputs.shell }} @@ -61,14 +80,14 @@ runs: env: PACKAGE_TYPE: ${{ inputs.label }} run: | - conda build faiss --user pytorch --label ${{ inputs.label }} -c pytorch + conda build -c conda-forge faiss --user pytorch --label ${{ inputs.label }} -c pytorch --override-channels - name: Conda build (GPU) if: inputs.label == '' && inputs.cuda != '' && inputs.cuvs == '' shell: ${{ steps.choose_shell.outputs.shell }} working-directory: conda run: | - conda build faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \ - -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia + conda build -c conda-forge faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \ + -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia --override-channels - name: Conda build (GPU) w/ anaconda upload if: inputs.label != '' && inputs.cuda != '' && inputs.cuvs == '' shell: ${{ steps.choose_shell.outputs.shell }} @@ -76,15 +95,16 @@ runs: env: PACKAGE_TYPE: ${{ inputs.label }} run: | - conda build faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \ - --user pytorch --label ${{ inputs.label }} -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia + conda build -c conda-forge faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \ + --user pytorch --label ${{ inputs.label }} -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} \ + -c nvidia --override-channels - name: Conda build (GPU w/ cuVS) if: inputs.label == '' && inputs.cuda != '' && inputs.cuvs != '' shell: ${{ steps.choose_shell.outputs.shell }} working-directory: conda run: | - conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \ - -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia + conda build -c conda-forge faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \ + -c pytorch -c rapidsai -c rapidsai-nightly -c nvidia --override-channels - name: Conda build (GPU w/ cuVS) w/ anaconda upload if: inputs.label != '' && inputs.cuda != '' && inputs.cuvs != '' shell: ${{ steps.choose_shell.outputs.shell }} @@ -92,5 +112,6 @@ runs: env: PACKAGE_TYPE: ${{ inputs.label }} run: | - conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \ - --user pytorch --label ${{ inputs.label }} -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia + conda build -c conda-forge faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \ + --user pytorch --label ${{ inputs.label }} -c pytorch -c rapidsai -c rapidsai-nightly \ + -c nvidia --override-channels diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt index 3fc46f4c56..ae5939d5fc 100644 --- a/faiss/python/CMakeLists.txt +++ b/faiss/python/CMakeLists.txt @@ -313,6 +313,7 @@ configure_file(class_wrappers.py class_wrappers.py COPYONLY) configure_file(gpu_wrappers.py gpu_wrappers.py COPYONLY) configure_file(extra_wrappers.py extra_wrappers.py COPYONLY) configure_file(array_conversions.py array_conversions.py COPYONLY) +configure_file(cpuinfo.py cpuinfo.py COPYONLY) # file(GLOB files "${PROJECT_SOURCE_DIR}/../../contrib/*.py") file(COPY ${PROJECT_SOURCE_DIR}/../../contrib DESTINATION .) diff --git a/faiss/python/__init__.py b/faiss/python/__init__.py index 9d956ebe71..87fa0f9eab 100644 --- a/faiss/python/__init__.py +++ b/faiss/python/__init__.py @@ -20,6 +20,7 @@ from faiss import class_wrappers from faiss.gpu_wrappers import * from faiss.array_conversions import * +from faiss.cpuinfo import * from faiss.extra_wrappers import kmin, kmax, pairwise_distances, rand, randint, \ lrand, randn, rand_smooth_vectors, eval_intersection, normalize_L2, \ ResultHeap, knn, Kmeans, checksum, matrix_bucket_sort_inplace, bucket_sort, \ diff --git a/faiss/python/cpuinfo.py b/faiss/python/cpuinfo.py new file mode 100644 index 0000000000..82a1551462 --- /dev/null +++ b/faiss/python/cpuinfo.py @@ -0,0 +1,674 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +# This file is copied from numpy.distutils.cpuinfo, because +# distutils was removed in Python 3.12 and cpuinfo is not +# present in setuptools. + +""" +cpuinfo + +Copyright 2002 Pearu Peterson all rights reserved, +Pearu Peterson +Permission to use, modify, and distribute this software is given under the +terms of the NumPy (BSD style) license. See LICENSE.txt that came with +this distribution for specifics. + +NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. +Pearu Peterson + +""" +__all__ = ['cpu'] + +import os +import platform +import re +import sys +import types +import warnings + +from subprocess import getstatusoutput + + +def getoutput(cmd, successful_status=(0,), stacklevel=1): + try: + status, output = getstatusoutput(cmd) + except OSError as e: + warnings.warn(str(e), UserWarning, stacklevel=stacklevel) + return False, "" + if os.WIFEXITED(status) and os.WEXITSTATUS(status) in successful_status: + return True, output + return False, output + +def command_info(successful_status=(0,), stacklevel=1, **kw): + info = {} + for key in kw: + ok, output = getoutput(kw[key], successful_status=successful_status, + stacklevel=stacklevel+1) + if ok: + info[key] = output.strip() + return info + +def command_by_line(cmd, successful_status=(0,), stacklevel=1): + ok, output = getoutput(cmd, successful_status=successful_status, + stacklevel=stacklevel+1) + if not ok: + return + for line in output.splitlines(): + yield line.strip() + +def key_value_from_command(cmd, sep, successful_status=(0,), + stacklevel=1): + d = {} + for line in command_by_line(cmd, successful_status=successful_status, + stacklevel=stacklevel+1): + l = [s.strip() for s in line.split(sep, 1)] + if len(l) == 2: + d[l[0]] = l[1] + return d + +class CPUInfoBase: + """Holds CPU information and provides methods for requiring + the availability of various CPU features. + """ + + def _try_call(self, func): + try: + return func() + except Exception: + pass + + def __getattr__(self, name): + if not name.startswith('_'): + if hasattr(self, '_'+name): + attr = getattr(self, '_'+name) + if isinstance(attr, types.MethodType): + return lambda func=self._try_call,attr=attr : func(attr) + else: + return lambda : None + raise AttributeError(name) + + def _getNCPUs(self): + return 1 + + def __get_nbits(self): + abits = platform.architecture()[0] + nbits = re.compile(r'(\d+)bit').search(abits).group(1) + return nbits + + def _is_32bit(self): + return self.__get_nbits() == '32' + + def _is_64bit(self): + return self.__get_nbits() == '64' + +class LinuxCPUInfo(CPUInfoBase): + + info = None + + def __init__(self): + if self.info is not None: + return + info = [ {} ] + ok, output = getoutput('uname -m') + if ok: + info[0]['uname_m'] = output.strip() + try: + fo = open('/proc/cpuinfo') + except OSError as e: + warnings.warn(str(e), UserWarning, stacklevel=2) + else: + for line in fo: + name_value = [s.strip() for s in line.split(':', 1)] + if len(name_value) != 2: + continue + name, value = name_value + if not info or name in info[-1]: # next processor + info.append({}) + info[-1][name] = value + fo.close() + self.__class__.info = info + + def _not_impl(self): pass + + # Athlon + + def _is_AMD(self): + return self.info[0]['vendor_id']=='AuthenticAMD' + + def _is_AthlonK6_2(self): + return self._is_AMD() and self.info[0]['model'] == '2' + + def _is_AthlonK6_3(self): + return self._is_AMD() and self.info[0]['model'] == '3' + + def _is_AthlonK6(self): + return re.match(r'.*?AMD-K6', self.info[0]['model name']) is not None + + def _is_AthlonK7(self): + return re.match(r'.*?AMD-K7', self.info[0]['model name']) is not None + + def _is_AthlonMP(self): + return re.match(r'.*?Athlon\(tm\) MP\b', + self.info[0]['model name']) is not None + + def _is_AMD64(self): + return self.is_AMD() and self.info[0]['family'] == '15' + + def _is_Athlon64(self): + return re.match(r'.*?Athlon\(tm\) 64\b', + self.info[0]['model name']) is not None + + def _is_AthlonHX(self): + return re.match(r'.*?Athlon HX\b', + self.info[0]['model name']) is not None + + def _is_Opteron(self): + return re.match(r'.*?Opteron\b', + self.info[0]['model name']) is not None + + def _is_Hammer(self): + return re.match(r'.*?Hammer\b', + self.info[0]['model name']) is not None + + # Alpha + + def _is_Alpha(self): + return self.info[0]['cpu']=='Alpha' + + def _is_EV4(self): + return self.is_Alpha() and self.info[0]['cpu model'] == 'EV4' + + def _is_EV5(self): + return self.is_Alpha() and self.info[0]['cpu model'] == 'EV5' + + def _is_EV56(self): + return self.is_Alpha() and self.info[0]['cpu model'] == 'EV56' + + def _is_PCA56(self): + return self.is_Alpha() and self.info[0]['cpu model'] == 'PCA56' + + # Intel + + #XXX + _is_i386 = _not_impl + + def _is_Intel(self): + return self.info[0]['vendor_id']=='GenuineIntel' + + def _is_i486(self): + return self.info[0]['cpu']=='i486' + + def _is_i586(self): + return self.is_Intel() and self.info[0]['cpu family'] == '5' + + def _is_i686(self): + return self.is_Intel() and self.info[0]['cpu family'] == '6' + + def _is_Celeron(self): + return re.match(r'.*?Celeron', + self.info[0]['model name']) is not None + + def _is_Pentium(self): + return re.match(r'.*?Pentium', + self.info[0]['model name']) is not None + + def _is_PentiumII(self): + return re.match(r'.*?Pentium.*?II\b', + self.info[0]['model name']) is not None + + def _is_PentiumPro(self): + return re.match(r'.*?PentiumPro\b', + self.info[0]['model name']) is not None + + def _is_PentiumMMX(self): + return re.match(r'.*?Pentium.*?MMX\b', + self.info[0]['model name']) is not None + + def _is_PentiumIII(self): + return re.match(r'.*?Pentium.*?III\b', + self.info[0]['model name']) is not None + + def _is_PentiumIV(self): + return re.match(r'.*?Pentium.*?(IV|4)\b', + self.info[0]['model name']) is not None + + def _is_PentiumM(self): + return re.match(r'.*?Pentium.*?M\b', + self.info[0]['model name']) is not None + + def _is_Prescott(self): + return self.is_PentiumIV() and self.has_sse3() + + def _is_Nocona(self): + return (self.is_Intel() + and (self.info[0]['cpu family'] == '6' + or self.info[0]['cpu family'] == '15') + and (self.has_sse3() and not self.has_ssse3()) + and re.match(r'.*?\blm\b', self.info[0]['flags']) is not None) + + def _is_Core2(self): + return (self.is_64bit() and self.is_Intel() and + re.match(r'.*?Core\(TM\)2\b', + self.info[0]['model name']) is not None) + + def _is_Itanium(self): + return re.match(r'.*?Itanium\b', + self.info[0]['family']) is not None + + def _is_XEON(self): + return re.match(r'.*?XEON\b', + self.info[0]['model name'], re.IGNORECASE) is not None + + _is_Xeon = _is_XEON + + # Varia + + def _is_singleCPU(self): + return len(self.info) == 1 + + def _getNCPUs(self): + return len(self.info) + + def _has_fdiv_bug(self): + return self.info[0]['fdiv_bug']=='yes' + + def _has_f00f_bug(self): + return self.info[0]['f00f_bug']=='yes' + + def _has_mmx(self): + return re.match(r'.*?\bmmx\b', self.info[0]['flags']) is not None + + def _has_sse(self): + return re.match(r'.*?\bsse\b', self.info[0]['flags']) is not None + + def _has_sse2(self): + return re.match(r'.*?\bsse2\b', self.info[0]['flags']) is not None + + def _has_sse3(self): + return re.match(r'.*?\bpni\b', self.info[0]['flags']) is not None + + def _has_ssse3(self): + return re.match(r'.*?\bssse3\b', self.info[0]['flags']) is not None + + def _has_3dnow(self): + return re.match(r'.*?\b3dnow\b', self.info[0]['flags']) is not None + + def _has_3dnowext(self): + return re.match(r'.*?\b3dnowext\b', self.info[0]['flags']) is not None + +class IRIXCPUInfo(CPUInfoBase): + info = None + + def __init__(self): + if self.info is not None: + return + info = key_value_from_command('sysconf', sep=' ', + successful_status=(0, 1)) + self.__class__.info = info + + def _not_impl(self): pass + + def _is_singleCPU(self): + return self.info.get('NUM_PROCESSORS') == '1' + + def _getNCPUs(self): + return int(self.info.get('NUM_PROCESSORS', 1)) + + def __cputype(self, n): + return self.info.get('PROCESSORS').split()[0].lower() == 'r%s' % (n) + def _is_r2000(self): return self.__cputype(2000) + def _is_r3000(self): return self.__cputype(3000) + def _is_r3900(self): return self.__cputype(3900) + def _is_r4000(self): return self.__cputype(4000) + def _is_r4100(self): return self.__cputype(4100) + def _is_r4300(self): return self.__cputype(4300) + def _is_r4400(self): return self.__cputype(4400) + def _is_r4600(self): return self.__cputype(4600) + def _is_r4650(self): return self.__cputype(4650) + def _is_r5000(self): return self.__cputype(5000) + def _is_r6000(self): return self.__cputype(6000) + def _is_r8000(self): return self.__cputype(8000) + def _is_r10000(self): return self.__cputype(10000) + def _is_r12000(self): return self.__cputype(12000) + def _is_rorion(self): return self.__cputype('orion') + + def get_ip(self): + try: return self.info.get('MACHINE') + except Exception: pass + def __machine(self, n): + return self.info.get('MACHINE').lower() == 'ip%s' % (n) + def _is_IP19(self): return self.__machine(19) + def _is_IP20(self): return self.__machine(20) + def _is_IP21(self): return self.__machine(21) + def _is_IP22(self): return self.__machine(22) + def _is_IP22_4k(self): return self.__machine(22) and self._is_r4000() + def _is_IP22_5k(self): return self.__machine(22) and self._is_r5000() + def _is_IP24(self): return self.__machine(24) + def _is_IP25(self): return self.__machine(25) + def _is_IP26(self): return self.__machine(26) + def _is_IP27(self): return self.__machine(27) + def _is_IP28(self): return self.__machine(28) + def _is_IP30(self): return self.__machine(30) + def _is_IP32(self): return self.__machine(32) + def _is_IP32_5k(self): return self.__machine(32) and self._is_r5000() + def _is_IP32_10k(self): return self.__machine(32) and self._is_r10000() + + +class DarwinCPUInfo(CPUInfoBase): + info = None + + def __init__(self): + if self.info is not None: + return + info = command_info(arch='arch', + machine='machine') + info['sysctl_hw'] = key_value_from_command('sysctl hw', sep='=') + self.__class__.info = info + + def _not_impl(self): pass + + def _getNCPUs(self): + return int(self.info['sysctl_hw'].get('hw.ncpu', 1)) + + def _is_Power_Macintosh(self): + return self.info['sysctl_hw']['hw.machine']=='Power Macintosh' + + def _is_i386(self): + return self.info['arch']=='i386' + def _is_ppc(self): + return self.info['arch']=='ppc' + + def __machine(self, n): + return self.info['machine'] == 'ppc%s'%n + def _is_ppc601(self): return self.__machine(601) + def _is_ppc602(self): return self.__machine(602) + def _is_ppc603(self): return self.__machine(603) + def _is_ppc603e(self): return self.__machine('603e') + def _is_ppc604(self): return self.__machine(604) + def _is_ppc604e(self): return self.__machine('604e') + def _is_ppc620(self): return self.__machine(620) + def _is_ppc630(self): return self.__machine(630) + def _is_ppc740(self): return self.__machine(740) + def _is_ppc7400(self): return self.__machine(7400) + def _is_ppc7450(self): return self.__machine(7450) + def _is_ppc750(self): return self.__machine(750) + def _is_ppc403(self): return self.__machine(403) + def _is_ppc505(self): return self.__machine(505) + def _is_ppc801(self): return self.__machine(801) + def _is_ppc821(self): return self.__machine(821) + def _is_ppc823(self): return self.__machine(823) + def _is_ppc860(self): return self.__machine(860) + + +class SunOSCPUInfo(CPUInfoBase): + + info = None + + def __init__(self): + if self.info is not None: + return + info = command_info(arch='arch', + mach='mach', + uname_i='uname_i', + isainfo_b='isainfo -b', + isainfo_n='isainfo -n', + ) + info['uname_X'] = key_value_from_command('uname -X', sep='=') + for line in command_by_line('psrinfo -v 0'): + m = re.match(r'\s*The (?P

[\w\d]+) processor operates at', line) + if m: + info['processor'] = m.group('p') + break + self.__class__.info = info + + def _not_impl(self): pass + + def _is_i386(self): + return self.info['isainfo_n']=='i386' + def _is_sparc(self): + return self.info['isainfo_n']=='sparc' + def _is_sparcv9(self): + return self.info['isainfo_n']=='sparcv9' + + def _getNCPUs(self): + return int(self.info['uname_X'].get('NumCPU', 1)) + + def _is_sun4(self): + return self.info['arch']=='sun4' + + def _is_SUNW(self): + return re.match(r'SUNW', self.info['uname_i']) is not None + def _is_sparcstation5(self): + return re.match(r'.*SPARCstation-5', self.info['uname_i']) is not None + def _is_ultra1(self): + return re.match(r'.*Ultra-1', self.info['uname_i']) is not None + def _is_ultra250(self): + return re.match(r'.*Ultra-250', self.info['uname_i']) is not None + def _is_ultra2(self): + return re.match(r'.*Ultra-2', self.info['uname_i']) is not None + def _is_ultra30(self): + return re.match(r'.*Ultra-30', self.info['uname_i']) is not None + def _is_ultra4(self): + return re.match(r'.*Ultra-4', self.info['uname_i']) is not None + def _is_ultra5_10(self): + return re.match(r'.*Ultra-5_10', self.info['uname_i']) is not None + def _is_ultra5(self): + return re.match(r'.*Ultra-5', self.info['uname_i']) is not None + def _is_ultra60(self): + return re.match(r'.*Ultra-60', self.info['uname_i']) is not None + def _is_ultra80(self): + return re.match(r'.*Ultra-80', self.info['uname_i']) is not None + def _is_ultraenterprice(self): + return re.match(r'.*Ultra-Enterprise', self.info['uname_i']) is not None + def _is_ultraenterprice10k(self): + return re.match(r'.*Ultra-Enterprise-10000', self.info['uname_i']) is not None + def _is_sunfire(self): + return re.match(r'.*Sun-Fire', self.info['uname_i']) is not None + def _is_ultra(self): + return re.match(r'.*Ultra', self.info['uname_i']) is not None + + def _is_cpusparcv7(self): + return self.info['processor']=='sparcv7' + def _is_cpusparcv8(self): + return self.info['processor']=='sparcv8' + def _is_cpusparcv9(self): + return self.info['processor']=='sparcv9' + +class Win32CPUInfo(CPUInfoBase): + + info = None + pkey = r"HARDWARE\DESCRIPTION\System\CentralProcessor" + # XXX: what does the value of + # HKEY_LOCAL_MACHINE\HARDWARE\DESCRIPTION\System\CentralProcessor\0 + # mean? + + def __init__(self): + if self.info is not None: + return + info = [] + try: + #XXX: Bad style to use so long `try:...except:...`. Fix it! + import winreg + + prgx = re.compile(r"family\s+(?P\d+)\s+model\s+(?P\d+)" + r"\s+stepping\s+(?P\d+)", re.IGNORECASE) + chnd=winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, self.pkey) + pnum=0 + while True: + try: + proc=winreg.EnumKey(chnd, pnum) + except winreg.error: + break + else: + pnum+=1 + info.append({"Processor":proc}) + phnd=winreg.OpenKey(chnd, proc) + pidx=0 + while True: + try: + name, value, vtpe=winreg.EnumValue(phnd, pidx) + except winreg.error: + break + else: + pidx=pidx+1 + info[-1][name]=value + if name=="Identifier": + srch=prgx.search(value) + if srch: + info[-1]["Family"]=int(srch.group("FML")) + info[-1]["Model"]=int(srch.group("MDL")) + info[-1]["Stepping"]=int(srch.group("STP")) + except Exception as e: + print(e, '(ignoring)') + self.__class__.info = info + + def _not_impl(self): pass + + # Athlon + + def _is_AMD(self): + return self.info[0]['VendorIdentifier']=='AuthenticAMD' + + def _is_Am486(self): + return self.is_AMD() and self.info[0]['Family']==4 + + def _is_Am5x86(self): + return self.is_AMD() and self.info[0]['Family']==4 + + def _is_AMDK5(self): + return self.is_AMD() and self.info[0]['Family']==5 \ + and self.info[0]['Model'] in [0, 1, 2, 3] + + def _is_AMDK6(self): + return self.is_AMD() and self.info[0]['Family']==5 \ + and self.info[0]['Model'] in [6, 7] + + def _is_AMDK6_2(self): + return self.is_AMD() and self.info[0]['Family']==5 \ + and self.info[0]['Model']==8 + + def _is_AMDK6_3(self): + return self.is_AMD() and self.info[0]['Family']==5 \ + and self.info[0]['Model']==9 + + def _is_AMDK7(self): + return self.is_AMD() and self.info[0]['Family'] == 6 + + # To reliably distinguish between the different types of AMD64 chips + # (Athlon64, Operton, Athlon64 X2, Semperon, Turion 64, etc.) would + # require looking at the 'brand' from cpuid + + def _is_AMD64(self): + return self.is_AMD() and self.info[0]['Family'] == 15 + + # Intel + + def _is_Intel(self): + return self.info[0]['VendorIdentifier']=='GenuineIntel' + + def _is_i386(self): + return self.info[0]['Family']==3 + + def _is_i486(self): + return self.info[0]['Family']==4 + + def _is_i586(self): + return self.is_Intel() and self.info[0]['Family']==5 + + def _is_i686(self): + return self.is_Intel() and self.info[0]['Family']==6 + + def _is_Pentium(self): + return self.is_Intel() and self.info[0]['Family']==5 + + def _is_PentiumMMX(self): + return self.is_Intel() and self.info[0]['Family']==5 \ + and self.info[0]['Model']==4 + + def _is_PentiumPro(self): + return self.is_Intel() and self.info[0]['Family']==6 \ + and self.info[0]['Model']==1 + + def _is_PentiumII(self): + return self.is_Intel() and self.info[0]['Family']==6 \ + and self.info[0]['Model'] in [3, 5, 6] + + def _is_PentiumIII(self): + return self.is_Intel() and self.info[0]['Family']==6 \ + and self.info[0]['Model'] in [7, 8, 9, 10, 11] + + def _is_PentiumIV(self): + return self.is_Intel() and self.info[0]['Family']==15 + + def _is_PentiumM(self): + return self.is_Intel() and self.info[0]['Family'] == 6 \ + and self.info[0]['Model'] in [9, 13, 14] + + def _is_Core2(self): + return self.is_Intel() and self.info[0]['Family'] == 6 \ + and self.info[0]['Model'] in [15, 16, 17] + + # Varia + + def _is_singleCPU(self): + return len(self.info) == 1 + + def _getNCPUs(self): + return len(self.info) + + def _has_mmx(self): + if self.is_Intel(): + return (self.info[0]['Family']==5 and self.info[0]['Model']==4) \ + or (self.info[0]['Family'] in [6, 15]) + elif self.is_AMD(): + return self.info[0]['Family'] in [5, 6, 15] + else: + return False + + def _has_sse(self): + if self.is_Intel(): + return ((self.info[0]['Family']==6 and + self.info[0]['Model'] in [7, 8, 9, 10, 11]) + or self.info[0]['Family']==15) + elif self.is_AMD(): + return ((self.info[0]['Family']==6 and + self.info[0]['Model'] in [6, 7, 8, 10]) + or self.info[0]['Family']==15) + else: + return False + + def _has_sse2(self): + if self.is_Intel(): + return self.is_Pentium4() or self.is_PentiumM() \ + or self.is_Core2() + elif self.is_AMD(): + return self.is_AMD64() + else: + return False + + def _has_3dnow(self): + return self.is_AMD() and self.info[0]['Family'] in [5, 6, 15] + + def _has_3dnowext(self): + return self.is_AMD() and self.info[0]['Family'] in [6, 15] + +if sys.platform.startswith('linux'): # variations: linux2,linux-i386 (any others?) + cpuinfo = LinuxCPUInfo +elif sys.platform.startswith('irix'): + cpuinfo = IRIXCPUInfo +elif sys.platform == 'darwin': + cpuinfo = DarwinCPUInfo +elif sys.platform.startswith('sunos'): + cpuinfo = SunOSCPUInfo +elif sys.platform.startswith('win32'): + cpuinfo = Win32CPUInfo +elif sys.platform.startswith('cygwin'): + cpuinfo = LinuxCPUInfo +#XXX: other OS's. Eg. use _winreg on Win32. Or os.uname on unices. +else: + cpuinfo = CPUInfoBase + +cpu = cpuinfo() diff --git a/faiss/python/loader.py b/faiss/python/loader.py index caef9e5512..f84482ab25 100644 --- a/faiss/python/loader.py +++ b/faiss/python/loader.py @@ -28,7 +28,7 @@ def supported_instruction_sets(): """ # Old numpy.core._multiarray_umath.__cpu_features__ doesn't support Arm SVE, - # so let's read Features in numpy.distutils.cpuinfo and search 'sve' entry + # so let's read Features in faiss.cpuinfo and search 'sve' entry def is_sve_supported(): if platform.machine() != "aarch64": return False @@ -39,9 +39,9 @@ def is_sve_supported(): import numpy if Version(numpy.__version__) >= Version("2.0"): return False - # platform-dependent legacy fallback using numpy.distutils.cpuinfo - import numpy.distutils.cpuinfo - return "sve" in numpy.distutils.cpuinfo.cpu.info[0].get('Features', "").split() + # platform-dependent legacy fallback using faiss.cpuinfo + import faiss.cpuinfo + return "sve" in faiss.cpuinfo.cpu.info[0].get('Features', "").split() import numpy if Version(numpy.__version__) >= Version("1.19"): @@ -61,13 +61,13 @@ def is_sve_supported(): if subprocess.check_output(["/usr/sbin/sysctl", "hw.optional.avx2_0"])[-1] == '1': return {"AVX2"} elif platform.system() == "Linux": - import numpy.distutils.cpuinfo + import faiss.cpuinfo result = set() - if "avx2" in numpy.distutils.cpuinfo.cpu.info[0].get('flags', ""): + if "avx2" in faiss.cpuinfo.cpu.info[0].get('flags', ""): result.add("AVX2") - if "avx512" in numpy.distutils.cpuinfo.cpu.info[0].get('flags', ""): + if "avx512" in faiss.cpuinfo.cpu.info[0].get('flags', ""): result.add("AVX512") - if "avx512_fp16" in numpy.distutils.cpuinfo.cpu.info[0].get('flags', ""): + if "avx512_fp16" in faiss.cpuinfo.cpu.info[0].get('flags', ""): # avx512_fp16 is supported starting SPR result.add("AVX512_SPR") if is_sve_supported(): diff --git a/faiss/python/setup.py b/faiss/python/setup.py index 90c6846a58..3874978978 100644 --- a/faiss/python/setup.py +++ b/faiss/python/setup.py @@ -21,6 +21,7 @@ shutil.copyfile("gpu_wrappers.py", "faiss/gpu_wrappers.py") shutil.copyfile("extra_wrappers.py", "faiss/extra_wrappers.py") shutil.copyfile("array_conversions.py", "faiss/array_conversions.py") +shutil.copyfile("cpuinfo.py", "faiss/cpuinfo.py") ext = ".pyd" if platform.system() == "Windows" else ".so" prefix = "Release/" * (platform.system() == "Windows")