Deploy CUDA 11.3 in CI (AlexeyAB#7630)

* [nuget] use personal baget repo * update cuda to 11.3 on windows * add error messages during build to not have green badge when failed * remove cygwin script, is untested now; use deploy-cuda.ps1 script also inside setup.ps1
Tossy0423 · Apr 21, 2021 · 91efb7c · 91efb7c
1 parent 96971de
commit 91efb7c
Show file tree

Hide file tree

Showing 8 changed files with 97 additions and 60 deletions.
diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml
@@ -3,6 +3,7 @@ name: Darknet Continuous Integration
 on: [push, pull_request, workflow_dispatch]
 
 env:
+  #VCPKG_BINARY_SOURCES: 'clear;nuget,githubpackages,readwrite'
   VCPKG_BINARY_SOURCES: 'clear;nuget,vcpkgbinarycache,readwrite'
 
 jobs:
@@ -107,15 +108,20 @@ jobs:
     - name: 'Setup vcpkg and NuGet artifacts backend'
       shell: bash
       run: >
-        git clone https://github.com/microsoft/vcpkg;
-        ./vcpkg/bootstrap-vcpkg.sh;
+        git clone https://github.com/microsoft/vcpkg ;
+        ./vcpkg/bootstrap-vcpkg.sh ;
+        mono $(./vcpkg/vcpkg fetch nuget | tail -n 1) sources add
+        -Name "githubpackages"
+        -Source https://nuget.pkg.github.com/${{ github.repository_owner }} ;
         mono $(./vcpkg/vcpkg fetch nuget | tail -n 1)
-        sources add
-        -source "https://nuget.pkg.github.com/AlexeyAB/index.json"
-        -storepasswordincleartext
-        -name "vcpkgbinarycache"
-        -username "AlexeyAB"
-        -password "${{ secrets.GITHUB_TOKEN }}"
+        setapikey ${{ secrets.GITHUB_TOKEN }}
+        -Source https://nuget.pkg.github.com/${{ github.repository_owner }} ;
+        mono $(./vcpkg/vcpkg fetch nuget | tail -n 1) sources add
+        -Name "vcpkgbinarycache"
+        -Source http://93.49.111.10:5555/v3/index.json ;
+        mono $(./vcpkg/vcpkg fetch nuget | tail -n 1)
+        setapikey ${{ secrets.BAGET_API_KEY }}
+        -Source http://93.49.111.10:5555/v3/index.json
 
     - name: 'Build'
       shell: pwsh
@@ -267,15 +273,20 @@ jobs:
     - name: 'Setup vcpkg and NuGet artifacts backend'
       shell: bash
       run: >
-        git clone https://github.com/microsoft/vcpkg;
-        ./vcpkg/bootstrap-vcpkg.sh;
+        git clone https://github.com/microsoft/vcpkg ;
+        ./vcpkg/bootstrap-vcpkg.sh ;
+        mono $(./vcpkg/vcpkg fetch nuget | tail -n 1) sources add
+        -Name "githubpackages"
+        -Source https://nuget.pkg.github.com/${{ github.repository_owner }} ;
+        mono $(./vcpkg/vcpkg fetch nuget | tail -n 1)
+        setapikey ${{ secrets.GITHUB_TOKEN }}
+        -Source https://nuget.pkg.github.com/${{ github.repository_owner }} ;
+        mono $(./vcpkg/vcpkg fetch nuget | tail -n 1) sources add
+        -Name "vcpkgbinarycache"
+        -Source http://93.49.111.10:5555/v3/index.json ;
         mono $(./vcpkg/vcpkg fetch nuget | tail -n 1)
-        sources add
-        -source "https://nuget.pkg.github.com/AlexeyAB/index.json"
-        -storepasswordincleartext
-        -name "vcpkgbinarycache"
-        -username "AlexeyAB"
-        -password "${{ secrets.GITHUB_TOKEN }}"
+        setapikey ${{ secrets.BAGET_API_KEY }}
+        -Source http://93.49.111.10:5555/v3/index.json
 
     - name: 'Build'
       shell: pwsh
@@ -353,15 +364,20 @@ jobs:
     - name: 'Setup vcpkg and NuGet artifacts backend'
       shell: bash
       run: >
-        git clone https://github.com/microsoft/vcpkg;
-        ./vcpkg/bootstrap-vcpkg.sh;
+        git clone https://github.com/microsoft/vcpkg ;
+        ./vcpkg/bootstrap-vcpkg.sh ;
+        $(./vcpkg/vcpkg fetch nuget | tail -n 1) sources add
+        -Name "githubpackages"
+        -Source https://nuget.pkg.github.com/${{ github.repository_owner }} ;
         $(./vcpkg/vcpkg fetch nuget | tail -n 1)
-        sources add
-        -source "https://nuget.pkg.github.com/AlexeyAB/index.json"
-        -storepasswordincleartext
-        -name "vcpkgbinarycache"
-        -username "AlexeyAB"
-        -password "${{ secrets.GITHUB_TOKEN }}"
+        setapikey ${{ secrets.GITHUB_TOKEN }}
+        -Source https://nuget.pkg.github.com/${{ github.repository_owner }} ;
+        $(./vcpkg/vcpkg fetch nuget | tail -n 1) sources add
+        -Name "vcpkgbinarycache"
+        -Source http://93.49.111.10:5555/v3/index.json ;
+        $(./vcpkg/vcpkg fetch nuget | tail -n 1)
+        setapikey ${{ secrets.BAGET_API_KEY }}
+        -Source http://93.49.111.10:5555/v3/index.json
 
     - name: 'Build'
       shell: pwsh
@@ -439,20 +455,15 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: 'Install CUDA'
-      run: |
-        choco install cuda --version=10.2.89.20191206 -y
-        $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.."
-        Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
-        refreshenv
+      run: ./scripts/deploy-cuda.ps1
 
     - uses: lukka/get-cmake@latest
 
     - name: 'Build'
       env:
-        CUDA_PATH: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2"
-        CUDA_PATH_V10_2: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2"
-        CUDA_TOOLKIT_ROOT_DIR: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2"
-        CUDACXX: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2\\bin\\nvcc.exe"
+        CUDA_PATH: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.3"
+        CUDA_TOOLKIT_ROOT_DIR: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.3"
+        CUDACXX: "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.3\\bin\\nvcc.exe"
       shell: pwsh
       run: ./build.ps1 -EnableCUDA -DisableInteractive -DoNotUpdateDARKNET
 

diff --git a/build.ps1 b/build.ps1
@@ -393,8 +393,16 @@ New-Item -Path $build_folder -ItemType directory -Force
 Set-Location $build_folder
 $cmake_args = "-G `"$generator`" ${additional_build_setup} -S .."
 Write-Host "CMake args: $cmake_args"
-Start-Process -NoNewWindow -Wait -FilePath $CMAKE_EXE -ArgumentList $cmake_args
-Start-Process -NoNewWindow -Wait -FilePath $CMAKE_EXE -ArgumentList "--build . ${selectConfig} --parallel ${number_of_build_workers} --target install"
+$proc = Start-Process -NoNewWindow -Wait -PassThru -FilePath $CMAKE_EXE -ArgumentList $cmake_args
+$exitCode = $proc.ExitCode
+if (-not $exitCode -eq 0) {
+  Throw "Config failed! Exited with $exitCode."
+}
+$proc = Start-Process -NoNewWindow -Wait -PassThru -FilePath $CMAKE_EXE -ArgumentList "--build . ${selectConfig} --parallel ${number_of_build_workers} --target install"
+$exitCode = $proc.ExitCode
+if (-not $exitCode -eq 0) {
+  Throw "Config failed! Exited with $exitCode."
+}
 Remove-Item DarknetConfig.cmake
 Remove-Item DarknetConfigVersion.cmake
 $dllfiles = Get-ChildItem ./${dllfolder}/*.dll

diff --git a/scripts/README.md b/scripts/README.md
@@ -1,8 +1,6 @@
+# Datasets
 
-
-### Datasets:
-
-59.26TB of research data: http://academictorrents.com/ 
+59.26TB of research data: http://academictorrents.com/
 
 ImageNet Torrent (Stanford): http://academictorrents.com/browse.php?search=imagenet&page=0
 
@@ -54,7 +52,6 @@ Visual Question Answering: https://visualqa.org/download.html
 
 Large Movie Review Dataset: http://ai.stanford.edu/~amaas/data/sentiment/
 
-
 ----
 
 Wikipedia's List of datasets: https://en.wikipedia.org/wiki/List_of_datasets_for_machine-learning_research
@@ -71,4 +68,4 @@ Pedestrian DATASETs for Vision based Detection and Tracking: https://hemprasad.w
 
 TrackingNet: https://tracking-net.org/
 
-RGB, RGBD, Texture-mapped 3D mesh models: http://www.ycbbenchmarks.com/
+RGB, RGBD, Texture-mapped 3D mesh models: http://www.ycbbenchmarks.com/
diff --git a/scripts/deploy-cuda.ps1 b/scripts/deploy-cuda.ps1
@@ -0,0 +1,26 @@
+#!/usr/bin/env pwsh
+
+$url = 'https://developer.download.nvidia.com/compute/cuda/11.3.0/network_installers/cuda_11.3.0_win10_network.exe'
+
+$CudaFeatures = 'nvcc_11.3 cuobjdump_11.3 nvprune_11.3 cupti_11.3 memcheck_11.3 nvdisasm_11.3 nvprof_11.3 ' + `
+ 'visual_studio_integration_11.3 visual_profiler_11.3 visual_profiler_11.3 cublas_11.3 cublas_dev_11.3 ' + `
+ 'cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 ' + `
+ 'cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3 ' + `
+ 'occupancy_calculator_11.3 '
+
+try {
+  Write-Host 'Downloading CUDA...'
+  Invoke-WebRequest -Uri $url -OutFile "cuda_11.3.0_win10_network.exe"
+  Write-Host 'Installing CUDA...'
+  $proc = Start-Process -Wait -PassThru -FilePath "./cuda_11.3.0_win10_network.exe" -ArgumentList @('-s ' + $CudaFeatures)
+  $exitCode = $proc.ExitCode
+  if ($exitCode -eq 0) {
+    Write-Host 'Installation successful!'
+  }
+  else {
+    Throw "Installation failed! Exited with $exitCode."
+  }
+}
+catch {
+  Throw "Failed to install CUDA! $($_.Exception.Message)"
+}
diff --git a/scripts/setup.ps1 b/scripts/setup.ps1
@@ -6,15 +6,15 @@ if ($null -eq (Get-Command "choco.exe" -ErrorAction SilentlyContinue)) {
   # Download and install Chocolatey
   Set-ExecutionPolicy unrestricted -Scope CurrentUser
   Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
-  Write-Host "Please close and re-open powershell and then re-run setup.ps1 script"
-  Break
+  Throw "Please close and re-open powershell and then re-run setup.ps1 script"
 }
 
 Start-Process -FilePath "choco" -Verb runAs -ArgumentList " install -y cmake ninja powershell git vscode"
 Start-Process -FilePath "choco" -Verb runAs -ArgumentList " install -y visualstudio2019buildtools --package-parameters `"--add Microsoft.VisualStudio.Component.VC.CoreBuildTools --includeRecommended --includeOptional --passive --locale en-US --lang en-US`""
+Push-Location $PSScriptRoot
 
 if ($install_cuda) {
-  Start-Process -FilePath "choco" -Verb runAs -ArgumentList " install -y cuda"
+  & ./deploy-cuda.ps1
   $features = "full"
 }
 else {
@@ -26,7 +26,10 @@ else {
   }
 }
 
-git.exe clone https://github.com/microsoft/vcpkg
-Set-Location vcpkg
+git.exe clone https://github.com/microsoft/vcpkg ../vcpkg
+Set-Location ..\vcpkg
 .\bootstrap-vcpkg.bat -disableMetrics
 .\vcpkg.exe install darknet[${features}]:x64-windows
+Pop-Location
+
+Write-Host "Darknet installed in $pwd\x64-windows\tools\darknet" -ForegroundColor Yellow
diff --git a/scripts/setup.sh b/scripts/setup.sh
@@ -77,3 +77,9 @@ if [[ ! -v VCPKG_ROOT ]]; then
 fi
 
 $VCPKG_ROOT/vcpkg install darknet[${features}]
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+  echo "Darknet installed in $VCPKG_ROOT/installed/x64-osx/tools/darknet"
+else
+  echo "Darknet installed in $VCPKG_ROOT/installed/x64-linux/tools/darknet"
+fi
diff --git a/scripts/windows/win_install_cygwin.cmd b/scripts/windows/win_install_cygwin.cmd
diff --git a/vcpkg.json b/vcpkg.json
@@ -67,7 +67,6 @@
         {
           "name": "opencv2",
           "features": [
-            "contrib",
             "ffmpeg"
           ]
         }
@@ -79,7 +78,6 @@
         {
           "name": "opencv2",
           "features": [
-            "contrib",
             "cuda",
             "ffmpeg"
           ]