Add GPU driver type as CSE var, corresponding helper function and tes…

…ts (#5111)

parts/linux/cloud-init/artifacts/cse_cmd.sh

@@ -131,6 +131,7 @@ KUBELET_CONFIG_FILE_ENABLED="{{IsKubeletConfigFileEnabled}}"
 KUBELET_CONFIG_FILE_CONTENT="{{GetKubeletConfigFileContentBase64}}"
 SWAP_FILE_SIZE_MB="{{GetSwapFileSizeMB}}"
 GPU_DRIVER_VERSION="{{GPUDriverVersion}}"
+GPU_DRIVER_TYPE="{{GPUDriverType}}"
 GPU_IMAGE_SHA="{{GPUImageSHA}}"
 GPU_INSTANCE_PROFILE="{{GetGPUInstanceProfile}}"
 CUSTOM_SEARCH_DOMAIN_NAME="{{GetSearchDomainName}}"

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -149,6 +149,7 @@ DOCKER_VERSION=1.13.1-1
 export NVIDIA_DRIVER_IMAGE_SHA="${GPU_IMAGE_SHA:=}"
 export NVIDIA_DRIVER_IMAGE_TAG="${GPU_DV}-${NVIDIA_DRIVER_IMAGE_SHA}"
 export NVIDIA_DRIVER_IMAGE="mcr.microsoft.com/aks/aks-gpu"
+export NVIDIA_GPU_DRIVER_TYPE="${GPU_DRIVER_TYPE:=}"
 export CTR_GPU_INSTALL_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind"
 export DOCKER_GPU_INSTALL_CMD="docker run --privileged --net=host --pid=host -v /opt/gpu:/mnt/gpu -v /opt/actions:/mnt/actions --rm"
 APT_CACHE_DIR=/var/cache/apt/archives/

pkg/agent/baker.go

@@ -923,6 +923,9 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration
 		"GPUImageSHA": func() string {
 			return getAKSGPUImageSHA(profile.VMSize)
 		},
+		"GPUDriverType": func() string {
+			return getGPUDriverType(profile.VMSize)
+		},
 		"GetHnsRemediatorIntervalInMinutes": func() uint32 {
 			// Only need to enable HNSRemediator for Windows 2019
 			if cs.Properties.WindowsProfile != nil && profile.Distro == datamodel.AKSWindows2019Containerd {
@@ -1059,6 +1062,14 @@ func getAKSGPUImageSHA(size string) string {
 	}
 	return datamodel.AKSGPUCudaSHA
 }
+
+func getGPUDriverType(size string) string {
+	if useGridDrivers(size) {
+		return "grid"
+	}
+	return "cuda"
+}
+
 func gpuNeedsFabricManager(size string) bool {
 	return datamodel.FabricManagerGPUSizes[strings.ToLower(size)]
 }

pkg/agent/baker_test.go

@@ -2153,6 +2153,21 @@ var _ = Describe("getGPUDriverVersion", func() {
 	})
 })
 
+var _ = Describe("getGPUDriverType", func() {
+
+	It("should use cuda with nc v3", func() {
+		Expect(getGPUDriverType("standard_nc6_v3")).To(Equal("cuda"))
+	})
+	It("should use grid with nv v5", func() {
+		Expect(getGPUDriverType("standard_nv6ads_a10_v5")).To(Equal("grid"))
+		Expect(getGPUDriverType("Standard_nv36adms_A10_V5")).To(Equal("grid"))
+	})
+	// NV V1 SKUs were retired in September 2023, leaving this test just for safety
+	It("should use cuda with nv v1", func() {
+		Expect(getGPUDriverType("standard_nv6")).To(Equal("cuda"))
+	})
+})
+
 var _ = Describe("getAKSGPUImageSHA", func() {
 	It("should use newest AKSGPUGridSHA with nv v5", func() {
 		Expect(getAKSGPUImageSHA("standard_nv6ads_a10_v5")).To(Equal(datamodel.AKSGPUGridSHA))

pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData

@@ -6,7 +6,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAC/9x8e3ebSNL3/3yKGoYZSzNGErLjOM5hdrCEHV7r9gJyJo/j5WDRkhkjUKBx4rX1fPbndNNAo4tjx9k5s5s9OStR1VXVv7p2o8yPPzSv/LB55SbXgm6ajvXBsvV+x+45lq2ZtnOiGT11Hyit0xuOu44xMGzHNvr6cGyrrzLKidHTnfea3XlXUA4yyrthr+u813rGYPyHdqoPbPV1RjD1nq5Z+gaGw4xBG9mOMbBsrdcrhL7JSN1h50w316jtVoXcHb4f9IZat6QrFfqZ/mEDT7vCQ4wgfAV5r0LmMGozkPrD4w90Wc+wSpzaDKi+5ZyOTreoPuAkrO2NwdYZDmzNGOhmd52FAWeOB511IoPu7NAiDANjcFrQ9lolbc2qPQba2fhYJ2ExGNrOyXA86Kp7DCqjf7ph2V65rKfbPFJ7+1Uv9k+d0Zgzde/V2lY79ia+g3U+0yA2rrNuR4+GLrepwzXOc920jOHAMQbnWs/oqnsMys7AWN/3vlI4emQOu05XP97A1F5nGp2dOlq3yxKOoTc0Nc4lumkOTXWfwZNlKrcRuvAwj6HuyBwe6+wps3g4to/JLsnuBhnpFed7bWQ4lm6e6ybPoGxk6A4spzccno1HjK29kU37n7GprzPnwaHZ2uZcOGhxHKtJeKBwxNVAP2CGnL/rrvr2YD2w1hS/bnFJtE5lmjtjyx72HUvXzM47pzvsa8bAyvZ2yAScjsbr6w/3OKJpkMjiM+Nwf528ur/DVxyPfm50dGfUG58ag4qgg5LJGJwMHXPYdzpD0xyPbL2rHrJ0sE7/2KrnTWudh9PwRimrdFczeqV33nD1ezzqarZekvLMsXQS+OcGzSurYxojm/rJ1LVuKUlptTglpJyOR6em1tU5Ds6M0dg8ZQGvtNpFkpCCYOrEDarSYvh3DMvRLMs4HTjmcGg7o/eqoihrNBo/I93sGxaxVFUUJnak0dLVGY4+UCZVUXjJo1HvgzPSLOv90KTJdmKcqoqykrhFDxmytFCUvKJlZWwtfBTldYVhONJNzSYYZqVBUfLqZZsbiG9AyMCiSWnZWufMOdVtRzP7jj080weqkrfQVZaBbr8fmmdsL+NMsKrkHXWV3RofD3TbGZn6ifGHqpCmWiTl8djo5bVMIX0xA+W9NnI6pk7CJQMj70o8xRhY45MTo2PoAxoRZ4410jq6qpAuRbltvaePhqbdrZRNVcnrXEnPAz4j5xsxbeNE65B+Zepan3TJXI6q5DVuAxMTpiqktGWDj22PSJD/8cHpaAS3c920VSUvbCwzCEk3bUtVDorhhJlt6R0Cqt2znOPh0LZsUxsV7VT/Q+/keV9Ex0Hu4K5hHWs93TFGtnbc0y1VIWUtq5nm2LKJhPXgyotb17DIKsey3qnK6zziTaOvmR+cgdFxjBFXVJXXLPKNgaWbtmP0u5Zj6pZNQpREnzkmlgzsodPXBqc93aE2qcrr/aeuOzF6tm4W61gadQkQ+pZ1J6Tgreg7eOq6FX2vueA1deILx9T//9gw9a7azkvUYEiLgnaqWwyYdl6brIE2st4Ni2rID42tPDNGpnFOiKR/klGARGU7L1fkIR+u7dZ+njbFuN7XrDMmlNE6pt7VB7ah9bJi2904E7fyDCTjzOqo024dgCCUowgdqog1zPrXsEIjzwmqpPs7itpuHa5ycM33vWb11XbrzaNC9tS20nqUY19t57V7C8crta3kqLA5JA/yYj7VzXOSyCQbjROjQ3xhDu2sgNraqdpW9gTBn8LUDz1oIjwBGd8tEEx3A5BDd45A/EWOUYDcBIkgL2I/xCB/Sn0M7d+aHrpthmkQwAPMYrQA+RPsNHbeAr5GoQAAMLRUqZZEMQY5puKbhTSyxv18AztzF0+ua1JrF5r/rBldp2ec6WptEsUoSuoPRletNX6p16XmLrh1uIfMBByliwWKa+5F+xLci73L+ltAX3wMy50605y7/ZkWFMHybMXwADgG2YMdcacuoCBB1BI0uY5AXFEdRhimURp6ojD1BWF8PB7YY2doOQOtr6ti9l0U+ppJgqoksAclhU6Lq2TyUBRo+6In0JKhfCYK7OyjNtMkbgbRxA3oofkmvUITHAhZI8+o5LkXTW5QLKAvC4ImHdTOVVG6L8e6HPEjdSlW+HTL5rWEt77nu0xB4Saloew1FFnJVw7Oja6h5aKNvnaqO9Y7LVdZPOC1bVpDwjw383wpS/dbBD8qRRXnk7gx9ydxlERT3JhE86Z7k5C/8myRFkvJiJKNp1ld6/S7qjghYRHCzWHS8COI0xBkkkm3foBmyANZjucgyyHC8nWUYJDlzz6+lsMEFr531FzE0aSpNMOkufAJ8zxKSRzeLZB65YfebhJP1Ga0wM3ZIt31Eqw252H2JVpgPwoTNSZ8j610J5SxXJ0/qEgoNskct7bPLEI2bTBEWGV7W/ge+3gLudlHucnFQ2bAEW8NBUoUyFDc0TrvdKdrmGrz1o2bE3dyjZouWRdPrv1blDQFMuBqAzJRccxxFGHCly0Q9HN9YJND3OkpnYlyeUE0a7r/SmPU7Bce18j3hv4FozAh1jQ6aYKjuTWJ/QVuolsU4qQpdMZmjxxIR2NbbeL5ojlJ48C5RfFVlKBGlGKBFnOeI4rdZJ3D1E8NyzY/sNk0m8dpESH8zUkUTv1Z486dByAIMcLx3WTuOf7Umbp+kMaoVod7Wn4IzUeJKilv4bPrYycJEFqoUvstYH+OohSr0t5bSK79KYaff175QEVMoxh88EOQagn6BApITGj9LXgRZSF/mDiQ8g+idP/7UiSyrmLk3sDDA3wsuP0pXIDkg4w+FfLgkusd+R9aP/UvaJJi5MFHUfr9o0gWEi3J2wprjHAah6AUD4sinP+hmwepBKKgTn360YtCrmxvV7ssQQ+TIIpu0kUBOQ+zwsFMII+wGzg88DGaRLGnSvt0aYLdGFO6KtU8FyP49ack62ifr/0AAY5TVEE91w5yITXHX6Ymqi2CL1GyxROTNI5RuEVr6apajWcEmbO1Xgd5hkCq7G67M0Wb8HHxUlkWI5Kc3u6zdkaiHnni49HAXLw9DgrnPwpJtociNJ5lZhR/DcgETaLQS8RKiJV57YSRk2AXJ3+HBCf5/dL0/l45y6E1Q9jBbnzlBkGBEnZj5zGkMnaKVBoHeTpmrpbuudXLHHBxC3Qc7wp8bgwy/teUshBtT4GOk/aN8OXOOmiRsA5AniZW7xYk+jkqjfkNJK57Qfu3n5WKHGLWBUj/gB9UaMHlJmNo6ri4IqjCwGXgSz3rTONo7sRo5ic4vnPIyOSQ1vhCj5NV0yjwUEzy3o+zM1AOklj/bwwLAhssyDmODwoGA5m6GMhyNnWAdL91RFmSMOJGnJeFESfou4fRlR+68d1Xo4ixPRJIjGPh4uvVYMp1rMcTt2glpqoKV8JqS2xV11TDi4aNPF3RuTFoaNBtj5hKAFZVwjY3rsXhRslfj8cKkt87JJ8Rll8NzQ3h+XiIrizYErH0GDH1g3Kup0+e2PvJwiw+91l8vqrEHC/rq4WMZ66G2gUNNCnXBpeXRSnbXMV4US8sY8VMsrHHFSYR38MDYISq3a68y/q79DzKMI3iqtcfcXjp5D3KunBvkcfO9/QsOwmi1JP90Mcy4U0alIPyZrd3J59IMORyliJIlIM4kTmi9ZxxsRC686M+PNnhvPB4VPwV0yLlRB7IPohNYl7TE0sDy+Qobf7tNwaHsBSSuwSj+QST4KXD+/OGcUhuJ6QP5G3iW8fvwgzwXDSPQjlGQeR6T1nA7AaJmfIXTPGFbnJ8SZNStSyHkbxwZ6SwV9PvzyiNQzcgi+S0WPBN6VS6LMHR4j/OX8Tov9JZz0bV8xP3KkD/ccAyu/+e2GYVhmzlebi+FEyacHIG0t8HkFsUJ34UOjOchxlGCQZRqtFXM1PY+Sn5GO6AKP0uwgOwlz7n8ADXyPVADkGpi6CqIEqKKCyFMnr1kMSAFnpWpZivlXlQWi14BXstkLL9mHr22tOyNXtsqdI/hM2lTqkWOfgNVu6ZJUXOeBtBNBNKgKsKQA4RmUNWYGZjnAKTKA08+m7pCmW3SvydWMUTDFp/Cj/AhmsmUNpkp+1X3GZQmGWK8iT1GbcHV3eliO228N7oZilJ3cGVan/K2RL4CZbT0McJyPKUHBVk2a28iBQlpZGg+NafIHHF4GovgHynkkJCfBuY0WLBg7lW+Z4khvEyOWzfaH6L4k40X7jctX3+GlJTpVo2iZC5dZJikD0QfxXJpK3UK6zHBWt7G+vFBZnwCtlLlg7Fk+OlyKb2ysBHUgl5hXhOAPBrsxegO+R/JA/zFDyHTPm1P7tGCT7P0liVasaJpRZnkEzHUixMl6gQedp+xHTpvip0ucn6ItaWghd9Dkk57aKr0c3Mjk742Xp0dpq9JWXJTb4X75DbxaOuYeode2h+yAvt/MbzY5AXIFXI2RCeJtceSPcVyrKo5GtZR2LooNUCd4HlGcKQG8xEEPOWavaZmbb8JVMULchk6i4wvd3I17HNPdI1qtfBbwUA+pKWLFEJfmwHlbduJMC5DH5Cp/EWNzOQ2Wmd7FR2SeJG8QTRhx6alq9OyKmHbCSIJjcJfZzDIUfQXdzMjo6G2YvJoyOVlxIFXomYfLf6Akr4WuOim1be0ha13pbeZkWTnR8I4Cs4sI9EqrAUZgh3RmMtnlwXEebGk2tVqqXZ2DuvC9wZ854Qd3eXNCNd8uVgXyQKVijxnDxfP4tmeZSRhbU+y6hzL6fS2uMnmtk/2OdLLDGlVppe55Vu1qls0dbiapyfmOzCZhyXd/JZoLGrHXotobDMYNdxMZqhL+rOPxu/fmxmf48av0o7FdjK5UtQ/5eAzy8ubIbV1tMCvg/mXgRibRDNEgdHTvZSd8Vc7CY3NI/K1ycZIeMmFWXgVt4a/bQ3qAscI23MNv9mSfzpBH6yGz/tDdhlIInb3K58qmBaQm/7Usr3ZxKFToJjP5ypUg3+/EQmn3Jwk2U3ngGRkWB3vgCgxTc3aSmusQ4XKHZJthkeYWUGbGBkNZg+EZVGe2+dxXaTG4IOZGoJlhsE6QTJHrpFAYBohNMonlMD3GCdt4+SxJ2hTClpogHCyDsC6Zctcke+R3lbW+j2JvrOfQHYEUjF510enCOQuG+7ORxHILFPu8X2iQz2cZfb7RFI5ZfdfGtHILFPu8UOcs6R7+0WZucPbd9bZjnC3SxL91xc0CvR+/VfPyyl+2oULxtklVAWTpEUNxF+UClA1XrAlz9uvLkmA5CT3PgLJ/vhj+PFPpnlyzmbVPYv2Y155N2pUq28xAP5HYh9hF3Pxe4Rff0t0mF6EUdf7kD8RQTxGuPFUbOpHLxptF/tN9j/N+dsVdMPE+yGE/QPd+HL7BShtlttRW615ZbCco5LtW/ZKt1Iudd8WBIlsiUy1ZBERLDTmETzRYpRA7uzpOcnGB5g7i5qCQrQBNcatDc80KNNTbRu/MXpIu1SwAyyjSAQ34Loi/V6/aJ12bh1gxRBswni1A0SJBYrKVAZ507lBQNno8j5h02mzk16hQKEHTo6hzNngmLsT/2Ji5ETR5hG93+t4xgIm30Xb/DdxSWZcnnP0XZ5k8hMlMzwlBmeMoennOMp1uEBMldu9BWTJWadtO/GfojiYUx/kdTzw/TLSouKkryTsiYZJXRYXvk1IZmVHx42MfA/KlznWv9d4fpkUL0v9lfncAHdusHIndy4M9RlA9vY7K3sg44E98qRvOQ2I4cgSmkcbBhHgEgFMUZJGmBVuiczAfeTkKwOZlRh5X6CtzMDP0M7S7n/94lZFqV4kWJVqpGAyCOSuCx7y0ZMy1iqccfcyUgCNy35U9joURCloVU9tyapFwH2wikwq8iE++cnMt1m8v/8BJ/dJKfSJli1c20uzE8LucBXoMBB61uEUuiygp8uyGRiZj90XfHoInO5Sm9gKgErSu3qk/yYKEp7Irtrof+0UhXZr2eq7L3oM4o7bsIl732U0OMkOZNeHNGf7R5d7pDPAWE+umTJ9uNJFEP2i9td4hCpUA5+Aspho7UPbuiBGCuHrX2RPHSDJAIPTf0QeeTgQ+pCFJLOSRvmbjZySvmvff2Elkoc5TIyvZY/9wM3Du7oCardyhW1243WfgOG+BrFn/0EbRLUYCg0yi2w7KXKgVvhBp/du9KCHD4Io8/c6jKrN4Cw12hlENzuNVrPQYA5jUeAivha8FPfraR3fjrJ3cuCackKtNjwikpiJA0iogiKZeOjeEue5PcEH0WxTjtEmAYbSwkfcOtrN/2kbFPJWwnm9z6+jlLcjXAlSnPB9NYSebCTND82ms3ZTuWA+MR9p1dpiNNGzAku1S4f3zW34y3r873zqc7qOGNO/u0pXykwVUDypK/C+r0KBfunMPnNi6XW6oLw0sgUsyP2Fo/w9ooemrppgAsPvFh3Q7pnEC8brJ4n5+3HYyTDMXBxeeuWqLWt+uX4W0yozFUVXcQ0YhkdmSoUsV5fMXIRo1s/SpPeX2jsRp0Vozdy8MaTZnCb114yUtxX0b74/ZLURu4dzqbg/FWtidL9bZ4Dpfji/fdmXZtB+546VwvlvyWMH0+r7bX69t8TJReXuYfXEb/divGT8C2wLTZVVOZ+GmBfiyfXK7U526o/d2fIjo7RKCWTXVahBYAf//xER4i11TCNozklTaIQu6RtG0RG0gBjCp3qswv/sjFflXDeJpNAGGGaDru5yfjaxbvZrSt78hRhwnrUrGwp99gmQ7YWua8VuC06aFRs0POSUvbUMvYSk15YsJ5brPrjnm049D8w8LSo/tYK9WxFxUEwK0nby8Gz0M4T/6lp/zSz2Rvx6gC2/SCdz2DsVqC8JWhvnL/Ye62/2fSVHVX8KTxWewF98RPMTh8zcuK4RlDy94oSFiN29omKU9ZW9qpCNoXB95j8HpmzOCvUl7dFTpoIclx2ZvpWlaMuRShtgp9/LnpQ8W+siUEifTO1mbQiTth+0fItW2Tof3Vvf82+WLci0bkUhB/f6b2Rblr68ET4vwAAAP//UcIjrYJKAAA=
+    H4sIAAAAAAAC/9x8e3ebSNL3/3yKGoYZSzNGErLjOM5hdrCEHV7r9gJyJo/j5WDRkhkjUKBx4rX1fPbndNNAo4tjx9k5s5s9OStR1VXVv7p2o8yPPzSv/LB55SbXgm6ajvXBsvV+x+45lq2ZtnOiGT11Hyit0xuOu44xMGzHNvr6cGyrrzLKidHTnfea3XlXUA4yyrthr+u813rGYPyHdqoPbPV1RjD1nq5Z+gaGw4xBG9mOMbBsrdcrhL7JSN1h50w316jtVoXcHb4f9IZat6QrFfqZ/mEDT7vCQ4wgfAV5r0LmMGozkPrD4w90Wc+wSpzaDKi+5ZyOTreoPuAkrO2NwdYZDmzNGOhmd52FAWeOB511IoPu7NAiDANjcFrQ9lolbc2qPQba2fhYJ2ExGNrOyXA86Kp7DCqjf7ph2V65rKfbPFJ7+1Uv9k+d0Zgzde/V2lY79ia+g3U+0yA2rrNuR4+GLrepwzXOc920jOHAMQbnWs/oqnsMys7AWN/3vlI4emQOu05XP97A1F5nGp2dOlq3yxKOoTc0Nc4lumkOTXWfwZNlKrcRuvAwj6HuyBwe6+wps3g4to/JLsnuBhnpFed7bWQ4lm6e6ybPoGxk6A4spzccno1HjK29kU37n7GprzPnwaHZ2uZcOGhxHKtJeKBwxNVAP2CGnL/rrvr2YD2w1hS/bnFJtE5lmjtjyx72HUvXzM47pzvsa8bAyvZ2yAScjsbr6w/3OKJpkMjiM+Nwf528ur/DVxyPfm50dGfUG58ag4qgg5LJGJwMHXPYdzpD0xyPbL2rHrJ0sE7/2KrnTWudh9PwRimrdFczeqV33nD1ezzqarZekvLMsXQS+OcGzSurYxojm/rJ1LVuKUlptTglpJyOR6em1tU5Ds6M0dg8ZQGvtNpFkpCCYOrEDarSYvh3DMvRLMs4HTjmcGg7o/eqoihrNBo/I93sGxaxVFUUJnak0dLVGY4+UCZVUXjJo1HvgzPSLOv90KTJdmKcqoqykrhFDxmytFCUvKJlZWwtfBTldYVhONJNzSYYZqVBUfLqZZsbiG9AyMCiSWnZWufMOdVtRzP7jj080weqkrfQVZaBbr8fmmdsL+NMsKrkHXWV3RofD3TbGZn6ifGHqpCmWiTl8djo5bVMIX0xA+W9NnI6pk7CJQMj70o8xRhY45MTo2PoAxoRZ4410jq6qpAuRbltvaePhqbdrZRNVcnrXEnPAz4j5xsxbeNE65B+Zepan3TJXI6q5DVuAxMTpiqktGWDj22PSJD/8cHpaAS3c920VSUvbCwzCEk3bUtVDorhhJlt6R0Cqt2znOPh0LZsUxsV7VT/Q+/keV9Ex0Hu4K5hHWs93TFGtnbc0y1VIWUtq5nm2LKJhPXgyotb17DIKsey3qnK6zziTaOvmR+cgdFxjBFXVJXXLPKNgaWbtmP0u5Zj6pZNQpREnzkmlgzsodPXBqc93aE2qcrr/aeuOzF6tm4W61gadQkQ+pZ1J6Tgreg7eOq6FX2vueA1deILx9T//9gw9a7azkvUYEiLgnaqWwyYdl6brIE2st4Ni2rID42tPDNGpnFOiKR/klGARGU7L1fkIR+u7dZ+njbFuN7XrDMmlNE6pt7VB7ah9bJi2904E7fyDCTjzOqo024dgCCUowgdqog1zPrXsEIjzwmqpPs7itpuHa5ycM33vWb11XbrzaNC9tS20nqUY19t57V7C8crta3kqLA5JA/yYj7VzXOSyCQbjROjQ3xhDu2sgNraqdpW9gTBn8LUDz1oIjwBGd8tEEx3A5BDd45A/EWOUYDcBIkgL2I/xCB/Sn0M7d+aHrpthmkQwAPMYrQA+RPsNHbeAr5GoQAAMLRUqZZEMQY5puKbhTSyxv18AztzF0+ua1JrF5r/rBldp2ec6WptEsUoSuoPRletNX6p16XmLrh1uIfMBByliwWKa+5F+xLci73L+ltAX3wMy50605y7/ZkWFMHybMXwADgG2YMdcacuoCBB1BI0uY5AXFEdRhimURp6ojD1BWF8PB7YY2doOQOtr6ti9l0U+ppJgqoksAclhU6Lq2TyUBRo+6In0JKhfCYK7OyjNtMkbgbRxA3oofkmvUITHAhZI8+o5LkXTW5QLKAvC4ImHdTOVVG6L8e6HPEjdSlW+HTL5rWEt77nu0xB4Saloew1FFnJVw7Oja6h5aKNvnaqO9Y7LVdZPOC1bVpDwjw383wpS/dbBD8qRRXnk7gx9ydxlERT3JhE86Z7k5C/8myRri7lMLE/jPQVmMgj3moy12QzbVYMO/2uKk5ILIVwc5g0/AjiNASZpN+tH6AZ8kCW4znIcoiwfB0lGGT5s4+v5TCBhe8dNRdxNGkqzTBpLnzCPI9SErx3C6Re+aG3m8QTtRktcHO2SHe9BKvNeZh9iRbYj8JEjQnfYyvdCWUsV+cPKhKKTTJvr+0zC6tNGwwRVtneFr7HPt5CbvZRbnLxkBlwxFtDgRIFMkl3tM473ekaptq8dePmxJ1co6ZL1sWTa/8WJU2BTMXagIxhHHMcRZjwZQsE/Vwf2OTkd3pKB6lcXhDNmu6/0hg1+0WYaOR7Q/+CUZgQaxqdNMHR3JrE/gI30S0KcdIUOmOzR06xo7GtNvF80ZykceDcovgqSlAjSrFAOwDPEcVuss5h6qeGZZsf2ECbDfG08hD+5iQKp/6scefOAxCEGOH4bjL3HH/qTF0/SGNUq8M9rVmE5qNElZS38Nn1sZMECC1Uqf0WsD9HUYpVae8tJNf+FMPPP698oCKmUQw++CFItQR9AgUkJrT+FryIspA/TBxI+QdRuv99KRJZVzFyb+DhAT4W3P4ULkDyQUafCnlwyTWc/A8tuvoXNEkx8uCjKP3+USQLiZbkbYU1RjiNQ1CKh0Xlzv/QzYNUAlFQpz796EUhV+u3q12WoIdJEEU36aKAnIdZ4WAmkEfYDRwe+BhNothTpX26NMFujCldlWqeixH8+lOStcHP136AAMcpqqCeawe5kJrjL1MT1RbBlyjZ4olJGsco3KK1dFWtxjOCzNlar4M8QyBVdrfdmaJN+Lh4qSyLEUlOb/dZOyNRjzzx8WhgLt4eB4XzH4Uk20MRGs8yM4q/BmSCJlHoJWIlxMq8dsLISbCLk79DgpP8fml6f6+c5dCaIexgN75yg6BACbux8xhSGTtFKo2DPB0zV0v33OplDri4BTqOdwU+NwYZ/2tKWYi2p0DHSftG+HJnHbRIWAcgTxOrdwsS/RyVxvwGEte9oP3bz0pFDjHrAqR/wA8qtOBykzE0dVxcEVRh4DLwpZ51pnE0d2I08xMc3zlkZHJIa3yhx8mqaRR4KCZ578fZwSkHSaz/N4YFgQ0W5PDHBwWDgUxdDGQ5mzpAut86oixJGHEjzsvCiBP03cPoyg/d+O6rUcTYHgkkxrFw8fVqMOU61uOJW7QSU1WFK2G1Jbaqa6rhRcNGnq7o3Bg0NOi2R0wlAKsqYZsb1+Jwo+Svx2MFye8dks8Iy6+G5obwfDxEVxZsiVh6jJj6QTnX0ydP7P1kYRaf+yw+X1Vijpf11ULGM1dD7YIGmpRrg8vLopRtrmK8qBeWsWIm2djjCpOI7+EBMELVbldegP1deh5lmEZx1euPOLx08h5lXbi3yGPne3qWnQRR6sl+6GOZ8CYNykF5syu/k08kGHI5SxEkykGcyBzRes64WAjd+VEfnuxwXng8Kv6KaZFyIg9kH8QmMa/piaWBZXKUNv/2G4NDWArJXYLRfIJJ8NLh/XnDOCS3E9IH8jbxreN3YQZ4LppHoRyjIHK9pyxgdoPETPkLpvhCNzm+pEmpWpbDSF64M1LYq+n3Z5TGoRuQRXJaLPimdCpdluBo8R/nL2L0X+msZ6Pq+Yl7FaD/OGCZ3X9PbLMKQ7byPFxfCiZNODkD6e8DyC2KEz8KnRnOwwyjBIMo1ej7nCns/JR8DHdAlH4X4QHYm6JzeIBr5Hogh6DURVBVECVFFJZCGb16SGJACz2rUszXyjworRa8gr0WSNl+TD17V2rZmj22VOkfwuZSp1SLHPwGK/fMkiJnvI0gmgklwFUFIIeIzCErMLMxToFJlAYefSF1hbJbJf5OrOIJBq0/hR9gwzUTKG2y0/YrbjMozDJFeZL6jNuDq7tSxHZbeG90s5Sk7uBKtT/lbAn8BMtp6OMEZHlKjgqy7FbeXoqS0khQfOtPkLhicLUXQL5TSSEhvg3MaLHgwVyrfE8Sw3iZHLZvNL9FcSeaL1zu2j5/d6mpUi2bRMjcOkkxyB6Iv4pk0lbqFdbjgrW9jfXigkx4hewlS4fiyfFSZFN7ZeAjqYS8QjwnAPi12VvTHfI/kod5Cp5Dpvzan12jBJ9naaxKNePEUoszSKZjKRamS1SIPG0/Yrp0XxW63GR9EWtLwYs+h6ScdtHV6GZmRyf8bD06O81erbLkJt+LF8/t4lHXMPWOPTQ/5IV2fuP5McgLkCrkbAhPk2sPpPsKZVlU8rWsIzF00GqBu8DyDGHIDWYiiHlLNfvMTFv+kimKFmQydReY3m7k69jmHuka1evgtwIAfbNLlqgEP7aDyls3EuBcBj+h03iLmxnI7LROdiq7JHGjeILoQw9Ny1cn5NRDNhJEk5uEPs7hkCPoLm5mR0fD7MXk0ZHKS4kCr0RMvlt9ASV8rXHRTStvaYtab0tvs6LJzg8E8BUc2EciVVgKM4Q7o7EWT66LCHPjybUq1dJs7J3XBe6MeU+Iu7tLmpEu+XKwLxIFK5R4Tp6vn0WzPMrIwlqfZdS5l1Np7fETzewf7PMllphSK02v80o361S2aGtxNc5PTHZhM47LO/ks0NjVDr2WUFhmsOu4GM3QF3Xnn41fPzazv0eNX6WdCmzl8iWo/0vA5xcXNsNq62kB3wdzLwKxNohmiYMjJ3upu2IudpMbmkfl65OMkHGTijJwK2+Nftob1AWOkTZmm3+zJP50Aj/ZjZ/2BuwykMRtblc+VTAtobd9KeX7M4lCJ8GxH85UqQZ/fiKTTzm4ybIbz4DISLA7XwDQ4pubtBTXWIcLFLsk2wyPsDIDNjCyGkyfiEqjvbfOYrvJDUEHMrUEyw2CdIJkD92iAEA0wmkUz6kBbrDO20dJ4s5QppQ00QBh5B2B9MsWuSPfo7ytLXR7E33nvgDsCKTi8y4PzhFI3LfdHI4jkNin3WL7RAb7uMvt9gik8stuvrUjkNin3WIHOefI93YLs/OHtu8tsxzhbpaley4u6JXo/fqvH5bSfTWKlw2ySigLp0iKmwg/qBSgaj3gyx833lyTAchJbvyFk/1ayPFin8zy5ZxNKvuX7MY88u5UqVZe4oH8DsQ+wq7nYveIvv4W6TC9iKMvdyD+IoJ4jfHiqNlUDt402q/2G+z/m3O2qumHCXbDCfqHu/BldopQ2622IrfackthOcel2rdslW6k3Gs+LIkS2RKZakgiIthpTKL5IsWogd1Z0vMTDA8wdxe1BAVogmsN2hse6NGmJlo3/uJ0kXYpYAbZRhCIb0H0xXq9ftG6bNy6QYqg2QRx6gYJEouVFKiMc6fygoGzUeT8wyZT5ya9QgHCDh2dw5kzQTH2p/7ExciJI0yj+7/WcQyEzb6LN/ju4pJMubznaLu8SWQmSmZ4ygxPmcNTzvEU6/AAmSs3+orJErNO2ndjP0TxMKa/SOr5YfplpUVFSd5JWZOMEjosr/wEkczKDw+bGPhfIq5zrf8YcX0yqN4X+6tzuIBu3WDkTm7cGeqygW1s9lb2QUeCe+VIXnKbkUMQpTQONowjQKSCGKMkDbAq3ZOZgPtJSFYHM6qwcj/B25mBn6Gdpdz/+8Qsi1K8SLEq1UhA5BFJXJa9ZSOmZSzVuGPuZCSBm5b8KWz0KIjS0KqeW5PUiwB74RSYVWTC/fMTmW4z+X9+gs9uklNpE6zauTYX5qeFXOArUOCg9S1CKXRZwU8XZDIxs1/Hrnh0kblcpTcwlYAVpXb1SX5MFKU9kd210H+PqYrs1zNV9l70GcUdN+GS9z5K6HGSnEkvjuhvfY8ud8jngDAfXbJk+/EkiiH7me4ucYhUKAc/AeWw0doHN/RAjJXD1r5IHrpBEoGHpn6IPHLwIXUhCknnpA1zNxs5pfwnwn5CSyWOchmZXsuf+4EbB3f0BNVu5Yra7UZrvwFDfI3iz36CNglqMBQa5RZY9lLlwK1wg8/uXWlBDh+E0WdudZnVG0DYa7QyCG73Gq3nIMCcxiNARXwt+KnvVtI7P53k7mXBtGQFWmx4RSUxkgYRUQTFsvFRvCVP8nuCj6JYpx0iTIONpYQPuPW1m35StqnkrQTzex9fRynuRrgSpblgemuJPNhJmh8bzeZsp3JAfOK+06s0xGkj5gSXapeP75rb8Zb1+d75VGd1nDEn//aUrxSYKiB50ldh/V6Fgv37mfzmxVJrdUF4aWSK2RF7i0d4e0UPTd00wIUHXqy7Id0ziJcNVs+T8/bjMZLhGLi4vHVL1NpW/XL8LSZU5qqKLmIasYyOTBWKWK+vGLmI0a0fpUnvLzR2o86K0Rs5eONJM7jNay8ZKe6raF/8fklqI/cOZ1Nw/qrWROn+Ns+BUnzx/nuzrs2gfU+dq4Xy3xLGj6fV9lp9+++JkovL3MPriN9uxfhJ+BbYFpsqKnM/DbCvxZPrldqcbdWfuzNkR8dolJLJLqvQAsCPf36iI8TaapjG0ZySJlGIXdK2DSIjaYAxhU712YV/2ZivSjhvk0kgjDBNh93cZHzt4t3s1pU9eYowYT1qVraUe2yTIVuL3NcK3BYdNCo26HlJKXtqGXuJSS8sWM8tVv1xzzYc+l8leFpUf2uFerai4iCYlaTt5eBZaOeJ/9S0f5rZ7I14dQDbfpDOZzB2K1DeErQ3zl/svdbfbPrKjir+FB6rvYC++Almp48ZOXFcIyj5e0UJixE7+0TFKWsre1Uhm8Lge0x+j8xZnBXqy9siJ00EOS47M32rylGXIpQ2wc8/Fz2o+IfZxCCRvpnaTFoRJ2y/aPmWLTL0v7q3v2ZfrFuR6FwKwo/v9N5INy19eCL8XwAAAP//ikjM+7dKAAA=
 
 
 

pkg/agent/testdata/AKSUbuntu1604+Containerd/line9.sh

@@ -134,6 +134,7 @@ DOCKER_VERSION=1.13.1-1
 export NVIDIA_DRIVER_IMAGE_SHA="${GPU_IMAGE_SHA:=}"
 export NVIDIA_DRIVER_IMAGE_TAG="${GPU_DV}-${NVIDIA_DRIVER_IMAGE_SHA}"
 export NVIDIA_DRIVER_IMAGE="mcr.microsoft.com/aks/aks-gpu"
+export NVIDIA_GPU_DRIVER_TYPE="${GPU_DRIVER_TYPE:=}"
 export CTR_GPU_INSTALL_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind"
 export DOCKER_GPU_INSTALL_CMD="docker run --privileged --net=host --pid=host -v /opt/gpu:/mnt/gpu -v /opt/actions:/mnt/actions --rm"
 APT_CACHE_DIR=/var/cache/apt/archives/