From 91efb7c7f6c00a4616ed58735f7c670908d0a849 Mon Sep 17 00:00:00 2001 From: Stefano Sinigardi Date: Wed, 21 Apr 2021 18:25:46 +0200 Subject: [PATCH] Deploy CUDA 11.3 in CI (#7630) * [nuget] use personal baget repo * update cuda to 11.3 on windows * add error messages during build to not have green badge when failed * remove cygwin script, is untested now; use deploy-cuda.ps1 script also inside setup.ps1 --- .github/workflows/ccpp.yml | 77 +++++++++++++++----------- build.ps1 | 12 +++- scripts/README.md | 9 +-- scripts/deploy-cuda.ps1 | 26 +++++++++ scripts/setup.ps1 | 13 +++-- scripts/setup.sh | 6 ++ scripts/windows/win_install_cygwin.cmd | 12 ---- vcpkg.json | 2 - 8 files changed, 97 insertions(+), 60 deletions(-) create mode 100644 scripts/deploy-cuda.ps1 delete mode 100644 scripts/windows/win_install_cygwin.cmd diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml index d6c18fda3fd..ac44c9f67f1 100644 --- a/.github/workflows/ccpp.yml +++ b/.github/workflows/ccpp.yml @@ -3,6 +3,7 @@ name: Darknet Continuous Integration on: [push, pull_request, workflow_dispatch] env: + #VCPKG_BINARY_SOURCES: 'clear;nuget,githubpackages,readwrite' VCPKG_BINARY_SOURCES: 'clear;nuget,vcpkgbinarycache,readwrite' jobs: @@ -107,15 +108,20 @@ jobs: - name: 'Setup vcpkg and NuGet artifacts backend' shell: bash run: > - git clone https://github.com/microsoft/vcpkg; - ./vcpkg/bootstrap-vcpkg.sh; + git clone https://github.com/microsoft/vcpkg ; + ./vcpkg/bootstrap-vcpkg.sh ; + mono $(./vcpkg/vcpkg fetch nuget | tail -n 1) sources add + -Name "githubpackages" + -Source https://nuget.pkg.github.com/${{ github.repository_owner }} ; mono $(./vcpkg/vcpkg fetch nuget | tail -n 1) - sources add - -source "https://nuget.pkg.github.com/AlexeyAB/index.json" - -storepasswordincleartext - -name "vcpkgbinarycache" - -username "AlexeyAB" - -password "${{ secrets.GITHUB_TOKEN }}" + setapikey ${{ secrets.GITHUB_TOKEN }} + -Source https://nuget.pkg.github.com/${{ github.repository_owner }} ; + mono $(./vcpkg/vcpkg fetch nuget | tail -n 1) sources add + -Name "vcpkgbinarycache" + -Source http://93.49.111.10:5555/v3/index.json ; + mono $(./vcpkg/vcpkg fetch nuget | tail -n 1) + setapikey ${{ secrets.BAGET_API_KEY }} + -Source http://93.49.111.10:5555/v3/index.json - name: 'Build' shell: pwsh @@ -267,15 +273,20 @@ jobs: - name: 'Setup vcpkg and NuGet artifacts backend' shell: bash run: > - git clone https://github.com/microsoft/vcpkg; - ./vcpkg/bootstrap-vcpkg.sh; + git clone https://github.com/microsoft/vcpkg ; + ./vcpkg/bootstrap-vcpkg.sh ; + mono $(./vcpkg/vcpkg fetch nuget | tail -n 1) sources add + -Name "githubpackages" + -Source https://nuget.pkg.github.com/${{ github.repository_owner }} ; + mono $(./vcpkg/vcpkg fetch nuget | tail -n 1) + setapikey ${{ secrets.GITHUB_TOKEN }} + -Source https://nuget.pkg.github.com/${{ github.repository_owner }} ; + mono $(./vcpkg/vcpkg fetch nuget | tail -n 1) sources add + -Name "vcpkgbinarycache" + -Source http://93.49.111.10:5555/v3/index.json ; mono $(./vcpkg/vcpkg fetch nuget | tail -n 1) - sources add - -source "https://nuget.pkg.github.com/AlexeyAB/index.json" - -storepasswordincleartext - -name "vcpkgbinarycache" - -username "AlexeyAB" - -password "${{ secrets.GITHUB_TOKEN }}" + setapikey ${{ secrets.BAGET_API_KEY }} + -Source http://93.49.111.10:5555/v3/index.json - name: 'Build' shell: pwsh @@ -353,15 +364,20 @@ jobs: - name: 'Setup vcpkg and NuGet artifacts backend' shell: bash run: > - git clone https://github.com/microsoft/vcpkg; - ./vcpkg/bootstrap-vcpkg.sh; + git clone https://github.com/microsoft/vcpkg ; + ./vcpkg/bootstrap-vcpkg.sh ; + $(./vcpkg/vcpkg fetch nuget | tail -n 1) sources add + -Name "githubpackages" + -Source https://nuget.pkg.github.com/${{ github.repository_owner }} ; $(./vcpkg/vcpkg fetch nuget | tail -n 1) - sources add - -source "https://nuget.pkg.github.com/AlexeyAB/index.json" - -storepasswordincleartext - -name "vcpkgbinarycache" - -username "AlexeyAB" - -password "${{ secrets.GITHUB_TOKEN }}" + setapikey ${{ secrets.GITHUB_TOKEN }} + -Source https://nuget.pkg.github.com/${{ github.repository_owner }} ; + $(./vcpkg/vcpkg fetch nuget | tail -n 1) sources add + -Name "vcpkgbinarycache" + -Source http://93.49.111.10:5555/v3/index.json ; + $(./vcpkg/vcpkg fetch nuget | tail -n 1) + setapikey ${{ secrets.BAGET_API_KEY }} + -Source http://93.49.111.10:5555/v3/index.json - name: 'Build' shell: pwsh @@ -439,20 +455,15 @@ jobs: steps: - uses: actions/checkout@v2 - name: 'Install CUDA' - run: | - choco install cuda --version=10.2.89.20191206 -y - $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.." - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" - refreshenv + run: ./scripts/deploy-cuda.ps1 - uses: lukka/get-cmake@latest - name: 'Build' env: - CUDA_PATH: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2" - CUDA_PATH_V10_2: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2" - CUDA_TOOLKIT_ROOT_DIR: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2" - CUDACXX: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2\\bin\\nvcc.exe" + CUDA_PATH: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.3" + CUDA_TOOLKIT_ROOT_DIR: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.3" + CUDACXX: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.3\\bin\\nvcc.exe" shell: pwsh run: ./build.ps1 -EnableCUDA -DisableInteractive -DoNotUpdateDARKNET diff --git a/build.ps1 b/build.ps1 index e147fb878ef..9574023efd6 100755 --- a/build.ps1 +++ b/build.ps1 @@ -393,8 +393,16 @@ New-Item -Path $build_folder -ItemType directory -Force Set-Location $build_folder $cmake_args = "-G `"$generator`" ${additional_build_setup} -S .." Write-Host "CMake args: $cmake_args" -Start-Process -NoNewWindow -Wait -FilePath $CMAKE_EXE -ArgumentList $cmake_args -Start-Process -NoNewWindow -Wait -FilePath $CMAKE_EXE -ArgumentList "--build . ${selectConfig} --parallel ${number_of_build_workers} --target install" +$proc = Start-Process -NoNewWindow -Wait -PassThru -FilePath $CMAKE_EXE -ArgumentList $cmake_args +$exitCode = $proc.ExitCode +if (-not $exitCode -eq 0) { + Throw "Config failed! Exited with $exitCode." +} +$proc = Start-Process -NoNewWindow -Wait -PassThru -FilePath $CMAKE_EXE -ArgumentList "--build . ${selectConfig} --parallel ${number_of_build_workers} --target install" +$exitCode = $proc.ExitCode +if (-not $exitCode -eq 0) { + Throw "Config failed! Exited with $exitCode." +} Remove-Item DarknetConfig.cmake Remove-Item DarknetConfigVersion.cmake $dllfiles = Get-ChildItem ./${dllfolder}/*.dll diff --git a/scripts/README.md b/scripts/README.md index a641e673d0e..91b17224361 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,8 +1,6 @@ +# Datasets - -### Datasets: - -59.26TB of research data: http://academictorrents.com/ +59.26TB of research data: http://academictorrents.com/ ImageNet Torrent (Stanford): http://academictorrents.com/browse.php?search=imagenet&page=0 @@ -54,7 +52,6 @@ Visual Question Answering: https://visualqa.org/download.html Large Movie Review Dataset: http://ai.stanford.edu/~amaas/data/sentiment/ - ---- Wikipedia's List of datasets: https://en.wikipedia.org/wiki/List_of_datasets_for_machine-learning_research @@ -71,4 +68,4 @@ Pedestrian DATASETs for Vision based Detection and Tracking: https://hemprasad.w TrackingNet: https://tracking-net.org/ -RGB, RGBD, Texture-mapped 3D mesh models: http://www.ycbbenchmarks.com/ \ No newline at end of file +RGB, RGBD, Texture-mapped 3D mesh models: http://www.ycbbenchmarks.com/ diff --git a/scripts/deploy-cuda.ps1 b/scripts/deploy-cuda.ps1 new file mode 100644 index 00000000000..29847578d3e --- /dev/null +++ b/scripts/deploy-cuda.ps1 @@ -0,0 +1,26 @@ +#!/usr/bin/env pwsh + +$url = 'https://developer.download.nvidia.com/compute/cuda/11.3.0/network_installers/cuda_11.3.0_win10_network.exe' + +$CudaFeatures = 'nvcc_11.3 cuobjdump_11.3 nvprune_11.3 cupti_11.3 memcheck_11.3 nvdisasm_11.3 nvprof_11.3 ' + ` + 'visual_studio_integration_11.3 visual_profiler_11.3 visual_profiler_11.3 cublas_11.3 cublas_dev_11.3 ' + ` + 'cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 ' + ` + 'cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3 ' + ` + 'occupancy_calculator_11.3 ' + +try { + Write-Host 'Downloading CUDA...' + Invoke-WebRequest -Uri $url -OutFile "cuda_11.3.0_win10_network.exe" + Write-Host 'Installing CUDA...' + $proc = Start-Process -Wait -PassThru -FilePath "./cuda_11.3.0_win10_network.exe" -ArgumentList @('-s ' + $CudaFeatures) + $exitCode = $proc.ExitCode + if ($exitCode -eq 0) { + Write-Host 'Installation successful!' + } + else { + Throw "Installation failed! Exited with $exitCode." + } +} +catch { + Throw "Failed to install CUDA! $($_.Exception.Message)" +} diff --git a/scripts/setup.ps1 b/scripts/setup.ps1 index b846bab1656..c5c2ae22b21 100755 --- a/scripts/setup.ps1 +++ b/scripts/setup.ps1 @@ -6,15 +6,15 @@ if ($null -eq (Get-Command "choco.exe" -ErrorAction SilentlyContinue)) { # Download and install Chocolatey Set-ExecutionPolicy unrestricted -Scope CurrentUser Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) - Write-Host "Please close and re-open powershell and then re-run setup.ps1 script" - Break + Throw "Please close and re-open powershell and then re-run setup.ps1 script" } Start-Process -FilePath "choco" -Verb runAs -ArgumentList " install -y cmake ninja powershell git vscode" Start-Process -FilePath "choco" -Verb runAs -ArgumentList " install -y visualstudio2019buildtools --package-parameters `"--add Microsoft.VisualStudio.Component.VC.CoreBuildTools --includeRecommended --includeOptional --passive --locale en-US --lang en-US`"" +Push-Location $PSScriptRoot if ($install_cuda) { - Start-Process -FilePath "choco" -Verb runAs -ArgumentList " install -y cuda" + & ./deploy-cuda.ps1 $features = "full" } else { @@ -26,7 +26,10 @@ else { } } -git.exe clone https://github.com/microsoft/vcpkg -Set-Location vcpkg +git.exe clone https://github.com/microsoft/vcpkg ../vcpkg +Set-Location ..\vcpkg .\bootstrap-vcpkg.bat -disableMetrics .\vcpkg.exe install darknet[${features}]:x64-windows +Pop-Location + +Write-Host "Darknet installed in $pwd\x64-windows\tools\darknet" -ForegroundColor Yellow diff --git a/scripts/setup.sh b/scripts/setup.sh index 12a769b8881..c33379e92ef 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -77,3 +77,9 @@ if [[ ! -v VCPKG_ROOT ]]; then fi $VCPKG_ROOT/vcpkg install darknet[${features}] + +if [[ "$OSTYPE" == "darwin"* ]]; then + echo "Darknet installed in $VCPKG_ROOT/installed/x64-osx/tools/darknet" +else + echo "Darknet installed in $VCPKG_ROOT/installed/x64-linux/tools/darknet" +fi diff --git a/scripts/windows/win_install_cygwin.cmd b/scripts/windows/win_install_cygwin.cmd deleted file mode 100644 index 6a2bda4a812..00000000000 --- a/scripts/windows/win_install_cygwin.cmd +++ /dev/null @@ -1,12 +0,0 @@ -echo Download file: https://www.cygwin.com/setup-x86_64.exe - - -setup-x86_64.exe -q -P dos2unix,wget,tar,untar,gzip,unzip,qawk,bzip2,git,vim,gcc-g++,make,grep,sed,find - - -# wget rawgit.com/transcode-open/apt-cyg/master/apt-cyg -# install apt-cyg /bin - -echo Finished - -pause \ No newline at end of file diff --git a/vcpkg.json b/vcpkg.json index c57fe9175bf..30caffb7f88 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -67,7 +67,6 @@ { "name": "opencv2", "features": [ - "contrib", "ffmpeg" ] } @@ -79,7 +78,6 @@ { "name": "opencv2", "features": [ - "contrib", "cuda", "ffmpeg" ]