From b051115a28dcc23770eedc09b31ebc0f07b132c2 Mon Sep 17 00:00:00 2001 From: Janusz Lisiecki Date: Thu, 24 Aug 2023 13:15:06 +0200 Subject: [PATCH] Provide a dev container for DALI Signed-off-by: Janusz Lisiecki --- CMakeLists.txt | 2 +- docker/Dockerfile.deps.ubuntu | 80 ++++++++++++ docker/Dockerfile.ubuntu | 145 ++++++++++++++++++++++ docker/bake.hcl | 57 +++++++++ qa/TL0_python-self-test-base-cuda/test.sh | 15 +-- qa/download_pip_packages.sh | 20 ++- qa/setup_packages.py | 2 +- qa/test_template.sh | 9 +- 8 files changed, 316 insertions(+), 14 deletions(-) create mode 100644 docker/Dockerfile.deps.ubuntu create mode 100644 docker/Dockerfile.ubuntu create mode 100644 docker/bake.hcl diff --git a/CMakeLists.txt b/CMakeLists.txt index 6cab422cd40..cb02e13ce1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -316,7 +316,7 @@ set(CMAKE_C_STANDARD 11) set(CMAKE_CUDA_STANDARD 17) # CXX flags -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-free-nonheap-object -Wno-unused-variable -Wno-unused-function -Wno-strict-overflow -fno-strict-aliasing -fPIC -fvisibility=hidden") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-free-nonheap-object -Wno-unused-variable -Wno-unused-function -Wno-strict-overflow -Wno-nonnull -fno-strict-aliasing -fPIC -fvisibility=hidden") if (WERROR) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") diff --git a/docker/Dockerfile.deps.ubuntu b/docker/Dockerfile.deps.ubuntu new file mode 100644 index 00000000000..b726006d54b --- /dev/null +++ b/docker/Dockerfile.deps.ubuntu @@ -0,0 +1,80 @@ + +ARG FROM_IMAGE_NAME=ubuntu:22.04 +ARG BUILDER_EXTRA_DEPS=scratch +FROM ${BUILDER_EXTRA_DEPS} as extra_deps +FROM ${FROM_IMAGE_NAME} + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt update && apt install -y --no-install-recommends \ + python3 \ + python3-distutils \ + python3-dev \ + python3-pip \ + gcc \ + g++ \ + make \ + cmake \ + git \ + git-lfs \ + wget \ + autoconf \ + automake \ + libtool \ + libtool-bin \ + patchelf \ + nasm \ + python-is-python3 \ + patch \ + zlib1g-dev \ + libxml2-dev \ + dkms dpkg-dev \ + libudev-dev \ + liburcu-dev \ + libmount-dev \ + libnuma-dev \ + libssl-dev \ + libjsoncpp-dev \ + libasan6 \ + libunwind-dev \ + curl \ + libncurses5-dev \ + lsb-core \ + wget \ + unzip \ + llvm && \ + git lfs install && \ + # crete generic symblinks to libasan.so and libstdc++.so + ln /usr/lib/x86_64-linux-gnu/libasan.so.6 /usr/lib/x86_64-linux-gnu/libasan.so && \ + ln /usr/lib/x86_64-linux-gnu/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so && \ + pip install --upgrade pip && \ + pip install --upgrade \ + future \ + setuptools \ + wheel \ + flake8 \ + clang && \ + pip install libclang && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /root/.cache/pip/ + +COPY DALI_DEPS_VERSION /tmp + +ARG DALI_DEPS_REPO +ENV DALI_DEPS_REPO=${DALI_DEPS_REPO:-https://github.com/NVIDIA/DALI_deps} + +ARG DALI_DEPS_VERSION_SHA +ENV DALI_DEPS_VERSION_SHA=${DALI_DEPS_VERSION_SHA} + +# run in /bin/bash to have more advanced features supported like list +RUN /bin/bash -c 'DALI_DEPS_VERSION_SHA=${DALI_DEPS_VERSION_SHA:-$(cat /tmp/DALI_DEPS_VERSION)} && \ + git clone ${DALI_DEPS_REPO} /tmp/dali_deps && \ + cd /tmp/dali_deps && \ + git checkout ${DALI_DEPS_VERSION_SHA} && \ + git submodule init && \ + git submodule update --depth 1 --init --recursive && \ + export CC_COMP=${CC} && \ + export CXX_COMP=${CXX} && \ + /tmp/dali_deps/build_scripts/build_deps.sh && rm -rf /tmp/dali_deps && rm -rf /tmp/DALI_DEPS_VERSION' + +# extra deps +COPY --from=extra_deps / / diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu new file mode 100644 index 00000000000..8160b358f81 --- /dev/null +++ b/docker/Dockerfile.ubuntu @@ -0,0 +1,145 @@ +ARG DEPS_IMAGE_NAME +# clean builder without source code inside +FROM ${DEPS_IMAGE_NAME} as builder + +WORKDIR /opt/dali + +# to be adjusted later, first it to run from the build dir, the second is for nose test +# so the user can `python3 -m nose_wrapper --verbose` +ENV PYTHONPATH=/opt/dali/test_comp/dali/python/:/opt/dali/qa + +# make sure that linker discovers cuda libs +RUN echo "/usr/local/cuda/targets/x86_64-linux/lib" > /etc/ld.so.conf.d/999_cuda.conf && \ + ldconfig && \ + pip install --upgrade \ + astunparse \ + gast \ + dm-tree && \ + rm -rf /root/.cache/pip/ && \ + # Install dependencies: opencv-python from 3.3.0.10 onwards uses QT which requires + # X11 and other libraries that are not present in clean docker images or bundled there + # 4.3.0.38 uses libGL.so as well so install libgl1-mesa-dev + apt-get update && \ + apt-get install -y --no-install-recommends libsm6 \ + libice6 \ + libxrender1 \ + libxext6 \ + libx11-6 \ + glib-2.0 \ + libgl1-mesa-dev && \ + # for simple audio python wheel + apt-get install -y --no-install-recommends libasound2-dev && \ + apt-get update && \ + apt-get install wget software-properties-common -y && \ + CUDNN_VERSION=8.8.1.3 && \ + CUBLASS_VERSION=12.0.2.224-1 && \ + apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub && \ + add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/ /" && \ + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin && \ + mv cuda-*.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + libcudnn8=$CUDNN_VERSION-1+cuda12.0 \ + libcudnn8-dev=$CUDNN_VERSION-1+cuda12.0 \ + libcublas-12-0=${CUBLASS_VERSION} && \ + rm -rf /var/lib/apt/lists/* + +RUN --mount=type=bind,source=qa/,target=/opt/dali/qa/ \ + export INSTALL=YES && \ + export CUDA_VERSION=$(echo $(nvcc --version) | sed 's/.*\(release \)\([0-9]\+\)\.\([0-9]\+\).*/\2\3/') && \ + cd /opt/dali/qa/ && ./download_pip_packages.sh + +RUN --mount=type=bind,source=./DALI_EXTRA_VERSION,target=/opt/dali/DALI_EXTRA_VERSION,type=bind,source=qa/,target=/opt/dali/qa/ \ + /bin/bash -c 'pushd /opt/dali/qa/ && \ + source ./setup_dali_extra.sh && \ + popd' + +ENV DALI_EXTRA_PATH=/opt/dali_extra + +ARG CC +ARG CXX +ENV CC=${CC} \ + CXX=${CXX} +# Optional build arguments + +ARG ARCH +ENV ARCH=${ARCH} +ARG CUDA_TARGET_ARCHS +ENV CUDA_TARGET_ARCHS=${CUDA_TARGET_ARCHS} +ARG CMAKE_BUILD_TYPE +ENV CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} +ARG BUILD_TEST +ENV BUILD_TEST=${BUILD_TEST} +ARG BUILD_BENCHMARK +ENV BUILD_BENCHMARK=${BUILD_BENCHMARK} +ARG BUILD_NVTX +ENV BUILD_NVTX=${BUILD_NVTX} +ARG BUILD_PYTHON +ENV BUILD_PYTHON=${BUILD_PYTHON} +ARG BUILD_LMDB +ENV BUILD_LMDB=${BUILD_LMDB} +ARG BUILD_JPEG_TURBO +ENV BUILD_JPEG_TURBO=${BUILD_JPEG_TURBO} +ARG BUILD_NVJPEG +ENV BUILD_NVJPEG=${BUILD_NVJPEG} +ARG BUILD_NVJPEG2K +ENV BUILD_NVJPEG2K=${BUILD_NVJPEG2K} +ARG BUILD_LIBTIFF +ENV BUILD_LIBTIFF=${BUILD_LIBTIFF} +ARG BUILD_LIBSND +ENV BUILD_LIBSND=${BUILD_LIBSND} +ARG BUILD_LIBTAR +ENV BUILD_LIBTAR=${BUILD_LIBTAR} +ARG BUILD_FFTS +ENV BUILD_FFTS=${BUILD_FFTS} +ARG BUILD_CFITSIO +ENV BUILD_CFITSIO=${BUILD_CFITSIO} +ARG BUILD_NVOF +ENV BUILD_NVOF=${BUILD_NVOF} +ARG BUILD_NVDEC +ENV BUILD_NVDEC=${BUILD_NVDEC} +ARG BUILD_NVML +ENV BUILD_NVML=${BUILD_NVML} +ARG BUILD_CUFILE +ENV BUILD_CUFILE=${BUILD_CUFILE:-ON} +ARG BUILD_NVCOMP +ENV BUILD_NVCOMP=${BUILD_NVCOMP} +ARG BUILD_CVCUDA +ENV BUILD_CVCUDA=${BUILD_CVCUDA} +ARG LINK_DRIVER +ENV LINK_DRIVER=${LINK_DRIVER} +ARG WITH_DYNAMIC_CUDA_TOOLKIT +ENV WITH_DYNAMIC_CUDA_TOOLKIT=${WITH_DYNAMIC_CUDA_TOOLKIT:-ON} +ARG WITH_DYNAMIC_NVJPEG +ENV WITH_DYNAMIC_NVJPEG=${WITH_DYNAMIC_NVJPEG:-ON} +ARG WITH_DYNAMIC_CUFFT +ENV WITH_DYNAMIC_CUFFT=${WITH_DYNAMIC_CUFFT:-ON} +ARG WITH_DYNAMIC_NPP +ENV WITH_DYNAMIC_NPP=${WITH_DYNAMIC_NPP:-ON} +ARG BUILD_WITH_ASAN +ENV BUILD_WITH_ASAN=${BUILD_WITH_ASAN} +ARG BUILD_WITH_LSAN +ENV BUILD_WITH_LSAN=${BUILD_WITH_LSAN} +ARG BUILD_WITH_UBSAN +ENV BUILD_WITH_UBSAN=${BUILD_WITH_UBSAN} +ARG PYTHON_VERSIONS +ENV PYTHON_VERSIONS=${PYTHON_VERSIONS} +ARG STRIP_BINARY +ENV STRIP_BINARY=${STRIP_BINARY} +ARG VERBOSE_LOGS +ENV VERBOSE_LOGS=${VERBOSE_LOGS} +ARG NVIDIA_DALI_BUILD_FLAVOR +ENV NVIDIA_DALI_BUILD_FLAVOR=${NVIDIA_DALI_BUILD_FLAVOR} +ARG GIT_SHA +ENV GIT_SHA=${GIT_SHA} +ARG DALI_TIMESTAMP +ENV DALI_TIMESTAMP=${DALI_TIMESTAMP} +ARG WHL_PLATFORM_NAME +ENV WHL_PLATFORM_NAME=${WHL_PLATFORM_NAME} +ARG WHL_COMPRESSION +ENV WHL_COMPRESSION=${WHL_COMPRESSION} +ARG EXTRA_CMAKE_OPTIONS +ENV EXTRA_CMAKE_OPTIONS=${EXTRA_CMAKE_OPTIONS} + +ARG NVIDIA_BUILD_ID +ENV NVIDIA_BUILD_ID ${NVIDIA_BUILD_ID:-0} diff --git a/docker/bake.hcl b/docker/bake.hcl new file mode 100644 index 00000000000..b98827b3c0e --- /dev/null +++ b/docker/bake.hcl @@ -0,0 +1,57 @@ +variable "ARCH" { + default = "x86_64" +} + +variable "CUDA_VERSION" { + default = "122" +} + +target "cuda_toolkit" { + dockerfile = "docker/Dockerfile.cuda${CUDA_VERSION}.${ARCH}.deps" + tags = [ + "nvidia/dali:cuda${CUDA_VERSION}_${ARCH}.toolkit" + ] +} + +target "deps_ubuntu" { + args = { + FROM_IMAGE_NAME = "ubuntu:22.04" + } + dockerfile = "docker/Dockerfile.deps.ubuntu" + tags = [ + "nvidia/dali:${ARCH}.deps" + ] +} + +target "deps_with_cuda" { + contexts = { + "nvidia/dali:${ARCH}.deps" = "target:deps_ubuntu", + "nvidia/dali:cuda${CUDA_VERSION}_${ARCH}.toolkit" = "target:cuda_toolkit" + } + args = { + FROM_IMAGE_NAME = "nvidia/dali:${ARCH}.deps", + CUDA_IMAGE = "nvidia/dali:cuda${CUDA_VERSION}_${ARCH}.toolkit" + } + dockerfile = "docker/Dockerfile.cuda.deps" + tags = [ + "nvidia/dali:cu${CUDA_VERSION}_${ARCH}.deps" + ] +} + +target "builder_image" { + contexts = { + "nvidia/dali:cu${CUDA_VERSION}_${ARCH}.deps" = "target:deps_with_cuda" + } + args = { + DEPS_IMAGE_NAME = "nvidia/dali:cu${CUDA_VERSION}_${ARCH}.deps" + } + dockerfile = "docker/Dockerfile.ubuntu" + tags = [ + "nvidia/dali:cu${CUDA_VERSION}_${ARCH}.build" + ] + target = "builder" +} + +group "default" { + targets = ["builder_image"] +} \ No newline at end of file diff --git a/qa/TL0_python-self-test-base-cuda/test.sh b/qa/TL0_python-self-test-base-cuda/test.sh index e0ba82234ff..abc4da494d2 100644 --- a/qa/TL0_python-self-test-base-cuda/test.sh +++ b/qa/TL0_python-self-test-base-cuda/test.sh @@ -11,7 +11,7 @@ version_eq "$DALI_CUDA_MAJOR_VERSION" "12" && \ mv /usr/local/cuda /usr/local/cuda_bak && \ ln -s cuda-12.0 /usr/local/cuda version_ge "$DALI_CUDA_MAJOR_VERSION" "11" && \ - pip uninstall -y `pip list | grep nvidia-cu | cut -d " " -f1` `pip list | grep nvidia-n | cut -d " " -f1` \ + pip uninstall -y `pip list | grep nvidia-cu | cut -d " " -f1` `pip list | grep nvidia-n | cut -d " " -f1` && CUDA_WHEELS_REMOVED=1 \ || true export DO_NOT_INSTALL_CUDA_WHEEL="TRUE" @@ -33,10 +33,11 @@ bash -e ./test.sh popd # restore old CUDA symlink, reinstall the latest CUDA wheels -version_eq "$DALI_CUDA_MAJOR_VERSION" "11" && \ - rm -rf /usr/local/cuda && mv /usr/local/cuda_bak /usr/local/cuda version_ge "$DALI_CUDA_MAJOR_VERSION" "11" && \ - pip install nvidia-cufft-cu${DALI_CUDA_MAJOR_VERSION} \ - nvidia-npp-cu${DALI_CUDA_MAJOR_VERSION} \ - nvidia-nvjpeg-cu${DALI_CUDA_MAJOR_VERSION} \ - || true + rm -rf /usr/local/cuda && mv /usr/local/cuda_bak /usr/local/cuda && \ + if [ -n "$CUDA_WHEELS_REMOVED" ]; then + pip install nvidia-cufft-cu${DALI_CUDA_MAJOR_VERSION} \ + nvidia-npp-cu${DALI_CUDA_MAJOR_VERSION} \ + nvidia-nvjpeg-cu${DALI_CUDA_MAJOR_VERSION} && \ + unset CUDA_WHEELS_REMOVED; + fi || true diff --git a/qa/download_pip_packages.sh b/qa/download_pip_packages.sh index 0285a09cf42..7d501a5e567 100755 --- a/qa/download_pip_packages.sh +++ b/qa/download_pip_packages.sh @@ -10,23 +10,28 @@ do # check all test files inside for test_file in $(ls -f *.sh); do + # don't parse files that don't source test_template.sh driectly as variables + # are not set up properly + if [ -z "$(grep 'test_template.sh' $test_file)" ]; then + continue + fi export pip_packages="" echo "Checking file: " ${test_file} source ${test_file} - echo "PIPs to install: " ${pip_packages} + echo "Packagess to install: " ${pip_packages} if test -n "$pip_packages" then last_config_index=$(python ../setup_packages.py -n -u $pip_packages --cuda ${CUDA_VERSION}) # get extra index url for given packages - PEP 503 Python Package Index - extra_indices=$($topdir/qa/setup_packages.py -u $pip_packages --cuda ${CUDA_VERSION} -e) + extra_indices=$(python ../setup_packages.py -u $pip_packages --cuda ${CUDA_VERSION} -e) extra_indices_string="" for e in ${extra_indices}; do extra_indices_string="${extra_indices_string} --extra-index-url=${e}" done # get link index url for given packages - a URL or path to an html file with links to archives - link_indices=$($topdir/qa/setup_packages.py -u $pip_packages --cuda ${CUDA_VERSION} -k) + link_indices=$(python ../setup_packages.py -u $pip_packages --cuda ${CUDA_VERSION} -k) link_indices_string="" for e in ${link_indices}; do link_indices_string="${link_indices_string} -f ${e}" @@ -37,7 +42,14 @@ do inst=$(python ../setup_packages.py -i $i -u $pip_packages --cuda ${CUDA_VERSION}) if [ -n "$inst" ] then - pip download $inst -d /pip-packages ${link_indices_string} ${extra_indices_string} + if [ -n "$INSTALL" ] + then + pip install $inst ${link_indices_string} ${extra_indices_string} + # only one test variant is needed + break + else + pip download $inst -d /pip-packages ${link_indices_string} ${extra_indices_string} + fi fi done fi diff --git a/qa/setup_packages.py b/qa/setup_packages.py index f8a85cc3549..787c64a4c7d 100755 --- a/qa/setup_packages.py +++ b/qa/setup_packages.py @@ -301,7 +301,7 @@ def get_name(self, cuda_version=None, idx=None): def get_all_versions(self, cuda_version): cuda_version = self.max_cuda_version(cuda_version) - return self.filter_versions(self.versions[cuda_version]) + return self.filter_versions(self.versions.get(cuda_version, [])) def max_cuda_version(self, cuda_version): """Gets a compatible, available cuda version to one asked for. diff --git a/qa/test_template.sh b/qa/test_template.sh index 89a647c0b04..870b2df95f3 100755 --- a/qa/test_template.sh +++ b/qa/test_template.sh @@ -1,7 +1,14 @@ #!/bin/bash set -o pipefail -source qa/test_template_impl.sh 2>&1 | perl -pe 'use POSIX strftime; +if [ -n "$gather_pip_packages" ] +then + # perl breaks the opulation of the outise variables from the inside of the sourced + # script. Turn this off to the gather_pip_packages process + source qa/test_template_impl.sh +else + source qa/test_template_impl.sh 2>&1 | perl -pe 'use POSIX strftime; $|=1; select((select(STDERR), $| = 1)[0]); print strftime "[%Y-%m-%d %H:%M:%S] ", localtime' +fi