Add GPU driver type as CSE var, corresponding helper function and tests

parts/linux/cloud-init/artifacts/cse_cmd.sh

@@ -131,6 +131,7 @@ KUBELET_CONFIG_FILE_ENABLED="{{IsKubeletConfigFileEnabled}}"
 KUBELET_CONFIG_FILE_CONTENT="{{GetKubeletConfigFileContentBase64}}"
 SWAP_FILE_SIZE_MB="{{GetSwapFileSizeMB}}"
 GPU_DRIVER_VERSION="{{GPUDriverVersion}}"
+GPU_DRIVER_TYPE="{{GPUDriverType}}"
 GPU_IMAGE_SHA="{{GPUImageSHA}}"
 GPU_INSTANCE_PROFILE="{{GetGPUInstanceProfile}}"
 CUSTOM_SEARCH_DOMAIN_NAME="{{GetSearchDomainName}}"

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -151,6 +151,7 @@ NVIDIA_CONTAINER_RUNTIME_VERSION="3.6.0"
 export NVIDIA_DRIVER_IMAGE_SHA="${GPU_IMAGE_SHA:=}"
 export NVIDIA_DRIVER_IMAGE_TAG="${GPU_DV}-${NVIDIA_DRIVER_IMAGE_SHA}"
 export NVIDIA_DRIVER_IMAGE="mcr.microsoft.com/aks/aks-gpu"
+export NVIDIA_GPU_DRIVER_TYPE="${GPU_DRIVER_TYPE:=}"
 export CTR_GPU_INSTALL_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind"
 export DOCKER_GPU_INSTALL_CMD="docker run --privileged --net=host --pid=host -v /opt/gpu:/mnt/gpu -v /opt/actions:/mnt/actions --rm"
 APT_CACHE_DIR=/var/cache/apt/archives/

pkg/agent/baker.go

@@ -918,6 +918,9 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration
 		"GPUImageSHA": func() string {
 			return getAKSGPUImageSHA(profile.VMSize)
 		},
+		"GPUDriverType": func() string {
+			return getGPUDriverType(profile.VMSize)
+		},
 		"GetHnsRemediatorIntervalInMinutes": func() uint32 {
 			// Only need to enable HNSRemediator for Windows 2019
 			if cs.Properties.WindowsProfile != nil && profile.Distro == datamodel.AKSWindows2019Containerd {
@@ -1054,6 +1057,14 @@ func getAKSGPUImageSHA(size string) string {
 	}
 	return datamodel.AKSGPUCudaSHA
 }
+
+func getGPUDriverType(size string) string {
+	if useGridDrivers(size) {
+		return "grid"
+	}
+	return "cuda"
+}
+
 func gpuNeedsFabricManager(size string) bool {
 	return datamodel.FabricManagerGPUSizes[strings.ToLower(size)]
 }

pkg/agent/baker_test.go

@@ -2135,6 +2135,21 @@ var _ = Describe("getGPUDriverVersion", func() {
 	})
 })
 
+var _ = Describe("getGPUDriverType", func() {
+
+	It("should use cuda with nc v3", func() {
+		Expect(getGPUDriverType("standard_nc6_v3")).To(Equal("cuda"))
+	})
+	It("should use grid with nv v5", func() {
+		Expect(getGPUDriverType("standard_nv6ads_a10_v5")).To(Equal("grid"))
+		Expect(getGPUDriverType("Standard_nv36adms_A10_V5")).To(Equal("grid"))
+	})
+	// NV V1 SKUs were retired in September 2023, leaving this test just for safety
+	It("should use cuda with nv v1", func() {
+		Expect(getGPUDriverType("standard_nv6")).To(Equal("cuda"))
+	})
+})
+
 var _ = Describe("getAKSGPUImageSHA", func() {
 	It("should use newest AKSGPUGridSHA with nv v5", func() {
 		Expect(getAKSGPUImageSHA("standard_nv6ads_a10_v5")).To(Equal(datamodel.AKSGPUGridSHA))

pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData

@@ -6,7 +6,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAA/9xc/3ebSJL/XX9FDcOMpRkjCdlxHOcxO1jCDmdJ6AA5k3O8PCxaMjECBRonXlv3t9/rpoFGXxw7yc7LbfblrURVV1d96ms3yvz8U+vKD1tXbnJd00zTsd5Ztjbo2n3HslXTdk5Uva/sA6V1+8a45+hD3XZsfaAZY1t5kVFO9L7mvFXt7puCcpBR3hj9nvNW7evD8V/qqTa0lZcZwdT6mmppGxgOMwZ1ZDv60LLVfr8Q+ioj9YzumWauUTvtCrlnvB32DbVX0uUK/Ux7t4GnU+EhShC+grxXIXMYdRhIA+P4HV3W160Spw4DamA5p6PTLVsfcBLWbGOwdY2hrepDzeytszDgzPGwu05k0J0dWoRhqA9PC9peu6StabXHQDsbH2skLIaG7ZwY42FP2WNQ6YPTDcv2ymV9zeaR2tuvenFw6ozGnKp7L9ZM7dqb+A7W+Uyd6LjOuh09GrqcUYdrnOeaaenG0NGH52pf7yl7DMruUF+3e18uHD0yjZ7T0443MHXWmUZnp47a67GEY+gZpsq5RDNNw1T2GTxZpnKG0IWHeQz1RqZxrLGnTGNjbB8TK4l1w4z0gvO9OtIdSzPPNZNnkDcy9IaW0zeMs/GIsXU2sqn/Mza1deY8OFRb3ZwLB22OYzUJD2SOuBroB0yR8ze9Vd8erAfW2sYv21wSrVPZzt2xZRsDx9JUs/vG6RkDVR9amW2HTMDpaLy+/nCPI5o6iSw+Mw7318mr9h2+4Hi0c72rOaP++FQfVgQdlEz68MRwTGPgdA3THI9sraccsnSwTv/aus+r9joPt8MruazSPVXvl955xdXv8ain2lpJyjPH0kjgn+s0r6yuqY9s6idTU3ulJLnd5jYh5XQ8OjXVnsZxcGqMxuYpC3i53SmShBQEUyNuUOQ2w7+rW45qWfrp0DENw3ZGbxVZltdoNH5GmjnQLaKpIstM7EilpatrjN5RJkWWecmjUf+dM1It661h0mQ70U8VWV5J3KKHGCwtZDmvaFkZWwsfWX5ZYTBGmqnaBMOsNMhyXr1scwPxFdQysGhSWrbaPXNONdtRzYFjG2faUJHzFrrKMtTst4Z5xmwZZ4IVOe+oq+zW+Hio2c7I1E70vxSZNNUiKY/Hej+vZTLpixkob9WR0zU1Ei4ZGHlX4in60BqfnOhdXRvSiDhzrJHa1RSZdCnKbWt9bWSYdq9SNhU5r3MlPQ/4jJwbYtr6idol/crU1AHpkrkcRc5r3AYmJkyRSWnLBh/bHpEg/+ud01UJbueaaStyXthYZhCSZtqWIh8UwwlT29K6BFS7bznHhmFbtqmOinaq/aV187wvouMgd3BPt47VvuboI1s97muWIpOyltVMc2zZRMJ6cOXFradbZJVjWW8U+WUe8aY+UM13zlDvOvqIK6rySxb5+tDSTNvRBz3LMTXLJiFKos8cE02GtuEM1OFpX3OoTor8cv+p6070vq2ZxTqWRj0ChLZl3QkpeCv7HTx13cp+L7ngNTXiC8fU/nusm1pP6eQlamjQoqCeahYDppPXJmuojqw3RlEN+aGxnWfGyNTPCZH0TzIKkKjs5OWKPOTDtdPez9OmGNcHqnXGhDJa19R62tDW1X5WbHsbZ+J2noFknFkddTrtA6jVylGEDlVEG6b9S1ihkecEVdL9HVnptA9XObjm+1a1Bkqn/epRIXtKR24/yrGvdPLavYXjhdKRc1TYHJIHeTGfauY5SWSSjfqJ3iW+MA07K6C2eqp05L1azZ/C1A89aCE8AQnfLRBMdwOQQneOQPhNilGA3AQJIC1iP8QgfUx9DJ0/Wh66bYVpEMADzGK0AOkj7DR3XgO+RmENAMCwFLGeRDEGKabiW4U0ssb9dAM7cxdPrutiexda/6zrPaevn2lKfRLFKEoaD3pPqTd/azTE1i64DbiHTAUcpYsFiuvuRecS3Iu9y8ZrQJ99DMudBts5d/szNSiC5dkbwwPgGCQPdoSdRg0FCaKaoMl1BMLK1mGEYRqloSfUpn6tNj4eD+2xY1jOUB1oipB9F2oD1SRBVRLYg5JCp8VVMnko1Gj7oifQkqF8JtTY2UdppUncCqKJG9BD8016hSY4qGWNPKOS5140uUFxDX1eEDTpoHauCOJ9OdbliB8pS6HCp1k2v0t463u+Wxue6z1dzQeG3Fud5mGzLcm1lcdyU95rypKcryoSjsy0JO0LTmGvedBsFwrku2Qa6gP1VHOsN2quefGAV3rTGpItubXnS0m83yL4USmKMJ/Ezbk/iaMkmuLmJJq33JuE/JVmi7RYSiadbMrNymN30FOECYmuEG4Ok6YfQZyGIJGEvPUDNEMeSFI8B0kKEZauowSDJH3y8bUUJrDwvaPWIo4mLbkVJq2FT5jnUUrC+W6BlCs/9HaTeKK0ogVuzRbprpdgpTUPsy/RAvtRmCgx4XtspTuhjOXq/EFFQmEk8/CanVmgbTIwRFhhti18j328hVzto1zl4iFT4IjXhgIl1Mhs3VW7bzSnp5tK69aNWxN3co1aLlkXT679W5S0amROVodkMOOY4yjChC9bUNPOtaFNzoKnp3S0yuUF0azl/iuNUWtQeFwl35vaZ4zChGjT7KYJjubWJPYXuIVuUYiTVq07NvvkXDsa20oLzxetSRoHzi2Kr6IENaMU12hP4Dmi2E3WOUztVLds8x0bcbOxntYiwt+aROHUnzXv3HkAtVqMcHw3mXuOP3Wmrh+kMao34J5WMULzUaKI8mv45PrYSQKEForYeQ3Yn6MoxYq49xqSa3+K4ddfVz5QEdMoBh/8EMR6gj6CDCIT2ngNXkRZyB8mDsT8gyDe/7kUiKyrGLk38PAA7wtufwoXIPogoY+FPLjkWlD+h5Zh7TOapBh58F4Q/3wvkIVkl+R1hTVGOI1DkIuHRS3P/1DjQSyBKKhTn370opCr/tu3XZagh0kQRTfpooCch1nmYCaQR9gNHB74GE2i2FPEfbo0wW6MKV0R656LEfz+S5I1xk/XfoAAxymqoJ7vDlIhNcdfoioqbYIv2WSLJyZpHKNwy66lq+p1nhEkTtdGA6QZArFi3XZnCjbh4+KlsixGJDm93WdZRqIeecLj0cBcvD0OCuc/CklmQxEaz1Izir8EZIImUeglQiXEyrx2wshJsIuTHyHBSX5/a3p/r5zl0Joh7GA3vnKDoEAJu7HzGFIZO0UqjYM8HTNXi/fc6mUOuLAFOo53BT43Bgn/a0pZyG5PgY6T9pXw5c46aJOwDkCaJlb/FkT6OSqV+QNErntB549f5YocotYFiP+AnxRow+UmZWjquLgiqMLAZeC3etaZxtHcidHMT3B855CRySGt8Rs9TlZNo8BDMcl7P86OUjlIQuM/MSwIbLAgx0E+KBgMZOpiIEvZ1AHi/dYRZUnCiBtxvi2MOEHfPYyu/NCN774YRYztkUBiHAsXX68GU77Hejxxi1ZiqrrhSlhtia3qmmp40bCRpit7bgwaGnTbI6YSgNUtN4rbGIcbJX85HitIfu+QfEZYfjE0N4QnPBqiKwu2RCw9Rkz9oJzr6ZMn9n6yMIvPfRafLyoxx8v6YiHjmauhdkEDTcx3g8vLopRtrmK8qG8sY8VMsrHHFSoR38MDYISq3a68EvtReh5lmEZx1euPOLx08h5lXbi3yGPne3qWnQRR6kl+6GOJ8CZNykF5s0vAk48kGHI5SwFEykGcyBzRfs64WAjd+VkzTnY4LzweFX/HtEg5kQeSD0KLqNfyhFLBMjlKnf/4g8FRW9aSuwSj+QST4KXD+/OGcUhuJ6QP5G3ia8fvQg3wXDSPQilGQeR6T1nA9AaRqfI3TPHF3uT4kibl1pIURtLCnZHCXk2/D1Eah25AFklpseCr0ql0WYKjxf87fxGl/05nPRtVz0/cq6cVqh8KWKb3j4ltVmGIKc/D9VvBpAknZSD9OIDcojjxo9CZ4TzMMEowCGKdvuGZws4vyftwBwTxTwEegL07OocHuEauB1IIckMARQFBlIXaslZGrxaSGFBDz6oU87UyD3K7DS9grw1iZo+pZW9PLVu1x5Yi/qO6rih1crXIwR+wcs8sylLG2wyiWa0EuLoBSCEic8gKzGyMk2ESpYFHX1FdoexWib8Tq3iCQetP4SfYcM0EcodY2nnBGYPCLFPkJ22fcXtwdVeK2K4L741elpLUHVyp9qecLoGfYCkNfZyAJE3JUUGS3Mr7TEGUmwmKb/0JElYUrvYCyC0VZRLi28CMFgsezLXK9yQxjJfJYXaj+S2Ku9F84XLX9vnbTFUR69kkQubWSYpB8kD4XSCTttyosB4XrJ1trBcXZMIrZC9ZOhRPjpcCm9orAx9JJeQV4jkBwK/N3qPukP+RPMxT8Byyza/92TVK8HmWxopY108spTiDZHsshUJ1kQqRpp1HVBfvq0KXm7QvYm1Z86JPISmnPXQ1upnZ0Qk/W4/OTrOXrSy5yffiVXSneNTTTa1rG+a7vNDObzw/BmkBYoWcDeFpcu2BeF+hLItKvpZ1JIYO2m1wF1iaIQy5wkwEUW+pZJ+Zasvfso2iBZlM3QWmtxv5OmbcI12jeh38ugZA3/WSJQrBj1lQeetGApzL4Cd0Gm9xMwOJndaJpZJLEjeKJ4g+9NCU8hWnHmJIEE1uEvo4h0OKoLe4mR0dGdmLyaMjhZcSBV6JmHS3+gKq9qXGRY2WX9MWtd6WXmdFk50fCOArOLCPRGptWZsh3B2N1XhyXUSYG0+uFbGeZmPvvFEodAHiPSHu7i5pRrrky8G+QDZYocRz8nz9LJrlUUaurfVZRp17OZXWHj9RzcHBPl9iiSr1UvUGv+nmPeUtu7W5GucnJruwGcflnXwWaOxqh15LyCwz2HVcjGbos7Lzz+bv71vZ36Pm7+JOBbZy+RKU/yXg84sLnWG19bSB74O5F4FoG0SzxMGRk73UXVEXu8kNzaPy9UlGyLhJRRm6lbdGv+wNGzWOkTZmm3+zJPxyAr/YzV/2huwykMRtrlc+VbBdQm/7Usr3IYlCJ8GxH84UsQ4fPpLJpxzcJMmNZ0BkJNidLwBo8c1VWgprrMYCxS7JNt0jrEyBDYysBtMngtzs7K2z2G5yQ9CBbFuC5QZBGkGyj25RACDo4TSK51QBN1jnHaAkcWco25Q00QBh5B2B+NsWuSPfo7ztLXR7E33nvgDsCMTi8y4PzhGI3LfdHI4jENmn3cJ8IoN93OWsPQKx/LKbm3YEIvu0W1iQc458b7dQO39o+94yyxHuZlm85+KCXoner//6YSneV6N42SSramXhFEhxE+AnhQJUrQd8+ePGm2syADnJjb9wst8POV7sk1m+nLNJZf9MP15F3p0i1stLPJDegDBA2PVc7B7R198CHaYXcfT5DoTfBBCuMV4ctVrywatm58V+k/1/a85WtfwwwW44Qf9wF77EThFKp92RpXZHasss57hU+xpTqSGlrfmwJIjEJDLVkEREsNOcRPNFilETu7Ok7ycYHmDuLuoJCtAE15u0NzzQo01dsG78xeki7VHAdGJGEAivQfCFRqNx0b5s3rpBiqDVAmHqBgkSipUUqIxzp/KCgdNR4PzDJlPnJr1CAcIOHZ3DmTNBMfan/sTFyIkjTKP7P9ZxDITNvos3+O7ikky5vOdou7xJJCZKYnhKDE+Jw1PK8RQa8ACZKzf6iskSsk46cGM/RLER018k9f0w/bzSoqIk76SsSUYJHZZXfpRIZuWHh00M/G8T17nWf564PhlU74v91Tm8hm7dYORObtwZ6rGBbWz2V+ygI8G9fCQtOWOkEAQxjYMN4wgQqSDEKEkDrIj3ZCbgfhKS1cGMWj7NtOL1zMDP0M5S7r8+Ms2iFC9SrIh1EhB5RBKXZW/ZiGoZSzXumDsZqcZNS/4UNnoUBNGwqufWJPUiwF44BaYVmXA/fCTTbSb/w0f45CY5lTbBqp5rc2F+WsgFvgAZDtpfI5RClxX8dEEmEzP7veyKRxeZyxV6A1MJWEHsVJ/kx0RB3BPYXQv9F5qKwH49U2XvR59Q3HUTLnnvo4QeJ8mZ9OKI/vr36HKHfA4I89ElS7afT6IYsh/u7hKHiMXm4CcgHzbb++CGHgixfNjeF8hDN0gi8NDUD5FHDj6kLkQh6Zy0Ye5mI6eY/2jYT2ipxFEuI9vX8ud+4MbBHT1Bddr5Rp1Os73fBANfo/iTn6BNgpoMhWZpAsteujlwK9zgk3tXapDDB2H0iVtdZvUGEPaa7QyC271m+zkIMKfxCFARXwp+6ruV9M5PJ7l7WTAtWYEWml5RSfSkSUQUQbFsvhduyZP8nuC9IDRohwjTYGMp4QNufe3qDeW2krcSzG99fB2luBfhSpTmgumtJfJgJ2m9b7Zas53KAfGJdqdXaYjTZswJLrddPm41Z/GW9bntfKqzOs6YV89L3z/lKwWmCkie9FVYv1ehYP+iJr95sZR6o/YVHqpGppAdsbd4hNdX8NDUTQNceOCb926K9wziZZPV8+S883iMZDgGLi5v3RKlvnV/Kf4aFSpzVWUvohrRjI5MFYrQaKwouYjRrR+lSf9vVHbjnhWlN3LwypNmcJvXXjJS3FfRvvjzktRG7h3OpuD8XakL4v1tngOl+OL99+a9NoP2PfdcLZT/ljB+PK221+rbf0+UXFzmHl5H/HYrxk/Ct8C2MKqozIM0wL4aT65XanNmqj93Z8iOjtEoJZNdVqFrAD9/+EhHiLXVMI2jOSVNohC7pG3rREbSBH0K3eqzC/+yOV+VcN4hk0AYYZoOu7nK+NrFu9mtK3vyFGG19ahZMSn32CZFtha5LxW4LXvQqNiwz7eUsqeWsW9R6RsL1nOL1WDct3WH/ncKnhbVX1uhnr1RcRDMStL2cvAstPPEf2raP01t9ka8OoBtP0jnMxi7FShvCTob5y/2XusHm76yo4o/hcdqL6DPfoLZ6WNGThzXCEr+flHCYsTOPlFxytrKXt2QTWHwPSa/R+YsTgvl29siJ00AKS47M32rylGXApQ6wa+/Fj2o+KfaRCGBvpnaTFoRt9Jz+Y77NSYy9L9o299jF+tWJDqXtdrPb7T+SDMtzTip/V8AAAD//7ZSrRjJSgAA
+    H4sIAAAAAAAA/9xc/3ebSJL/XX9FDcOMpRkjCdlxHOcxO1jCDmd9O0DO5BwvD4uWTIxAgcaJ19b97fe6aaBBkmMn2Xm5zb68lajq6qpPfe1GmZ9/al15QevKia9rmmHY5jvT0gZdq2+blmpY9omq95V9oLRufzTp2fpQt2xLH2ijiaW8SCknel+z36pW901OOUgpb0b9nv1W7evDyV/qqTa0lJcpwdD6mmpqGxgOUwZ1bNn60LTUfj8X+iol9UbdM81Yo3baJXJv9HbYH6m9gi6X6Gfauw08nRIPUYLw5eS9EpnDqMNAGoyO39Flfd0scOowoAamfTo+3bL1ASdhzTYGW3c0tFR9qBm9dRYGnDEZdteJDLqzQ5MwDPXhaU7baxe0Na32GGhnk2ONhMVwZNkno8mwp+wxqPTB6YZle8WyvmbxSO3tl704OLXHE07VvRdrpnatTXwH63yGTnRcZ92OHg1dzqjDNc5zzTD10dDWh+dqX+8pewzK7lBft3tfzh09NkY9u6cdb2DqrDONz05ttddjCcfQGxkq5xLNMEaGss/gSTOVM4QuPMxiqDc2Rscae8o0Hk2sY2IlsW6Ykl5wvlfHum1qxrlm8AzyRobe0LT7o9HZZMzYOhvZ1P+ZGNo6cxYcqqVuzoWDNsdRTcIDmSNWA/2AKXL+plf17cF6YK1t/LLNJdE6le3cnZjWaGCbmmp039i90UDVh2Zq2yETcDqerK8/3OOIhk4ii8+Mw/11ctW+wxccj3audzV73J+c6sOSoIOCSR+ejGxjNLC7I8OYjC2tpxyydDBP/9q6z6v2Og+3wyu5qNI9Ve8X3nnF1e/JuKdaWkHKMsfUSOCf6zSvzK6hjy3qJ0NTe4Ukud3mNiHldDI+NdSexnFwaownxikLeLndyZOEFARDI25Q5DbDv6ubtmqa+unQNkYjyx6/VWRZXqPR+BlrxkA3iaaKLDOxY5WWru5o/I4yKbLMSx6P++/ssWqab0cGTbYT/VSR5Uri5j1kxNJClrOKlpaxtfCR5ZclhtFYM1SLYJiWBlnOqpdlbCC+gloKFk1K01K7Z/apZtmqMbCt0Zk2VOSshVZZhpr1dmScMVsmqWBFzjpqld2cHA81yx4b2on+lyKTppon5fFE72e1TCZ9MQXlrTq2u4ZGwiUFI+tKPEUfmpOTE72ra0MaEWe2OVa7miKTLkW5La2vjUeG1SuVTUXO6lxBzwI+JWeGGJZ+onZJvzI0dUC6ZCZHkbMat4GJCVNkUtrSwceyxiTI/3pnd1WC27lmWIqcFTaWGYSkGZapyAf5cMLUNrUuAdXqm/bxaGSZlqGO83aq/aV1s7zPo+Mgc3BPN4/VvmbrY0s97mumIpOyltZMY2JaRMJ6cGXFraebZJVtmm8U+WUW8YY+UI139lDv2vqYK6rySxb5+tDUDMvWBz3TNjTTIiFKos+YEE2G1sgeqMPTvmZTnRT55f5T153ofUsz8nUsjXoECG3LuhNS8Cr7HTx1XWW/l1zwGhrxhW1o/z3RDa2ndLISNRzRoqCeaiYDppPVJnOojs03o7wa8kNjO8uMsaGfEyLpn2QUIFHZycoVeciHa6e9n6VNPq4PVPOMCWW0rqH1tKGlq/202PY2zsTtLAPJOFMddTrtA6jVilGEDlVEG6b9S6jQyHOCKun+tqx02odVDq75vlXNgdJpv3pUyJ7SkduPcuwrnax2b+F4oXTkDBU2h2RBns+nmnFOEplko36id4kvjJGVFlBLPVU68l6t5s1g5gUutBCegoTvlghmuz5IgbNAIPwmRchHTowEkJaRF2CQPiYehs4fLRfdtoLE9+EB5hFagvQRdpo7rwFfo6AGADAyFbEehxEGKaLiW7k0ssb5dAM7CwdPr+tiexda/6zrPbuvn2lKfRpGKIwbD3pPqTd/azTE1i44DbiHVAUcJssliurORecSnIu9y8ZrQJ89DKudBts5c/szNciD5dkbwwPgCCQXdoSdRg35MaKaoOl1CEJl6yDEMAuTwBVqM69WmxxPhtbEHpn2UB1oipB+F2oD1SBBVRDYg4JCp8UqmTwUarR90RNowVA8E2rs7KO0kjhq+eHU8emh+Sa5QlPs19JGnlLJczec3qCohj4vCZp0UDtXBPG+GOsyxI+UlVDi00yL3yW49VzPqQ3P9Z6uZgND5q1O87DZluRa5bHclPeasiRnq/KEIzMtSfucU9hrHjTbuQLZLqmG+kA91WzzjZppnj/gld60hmRLZu35ShLvtwh+VIoiLKZRc+FNozAOZ7g5DRct5yYmf6X5Mqku5aC13o21CtrkEa81GY/S0Titqd1BTxGmJCQDuDmMm14IURKARLL41vPRHLkgSdECJClAWLoOYwyS9MnD11IQw9Jzj1rLKJy25FYQt5YeYV6ECcmBuyVSrrzA3Y2jqdIKl7g1Xya7boyV1iJIv4RL7IVBrESE77GVzpQyFquzByUJuZEsLNbsTKNzk4EBwgqzbem57OMtZGofZSrnD5kCR7w2FCihRgbyrtp9o9k93VBat07UmjrTa9RyyLpoeu3dorhVI8O1OiTTHMcchSEmfOmCmnauDS1ygDw9pfNYJs8P5y3nX0mEWoM8TFTyval9xiiIiTbNbhLjcGFOI2+JW+gWBThu1boTo08Ow+OJpbTwYtmaJpFv36LoKoxRM0xwjTYSniOMnHidw9BOddMy3rG5OD0L0AJG+FvTMJh58+ads/ChVosQju6mC9f2ZvbM8fwkQvUG3NPSR2geihVRfg2fHA/bsY/QUhE7rwF7CxQmWBH3XkN87c0w/Ppr5QMVMQsj8MALQKzH6CPIIDKhjdfghpSF/GHiQMw+COL9nyuByLqKkHMDDw/wPuf2ZnABogcS+pjLg0uub2V/aO3WPqNpgpEL7wXxz/cCWUh2iV+XWCOEkygAOX+YN4DsDzUexAKInDrz6Ec3DLiWsX3bVQF6EPtheJMsc8h5mGUOZgJ5iB3f5oGP0DSMXEXcp0tj7ESY0hWx7joYwe+/xGk3/XTt+QhwlKAS6tnuIOVSM/wlqqLSJviSTbZ4YppEEQq27Fq4ql7nGUHidG00QJojEEvWbXemYBE+Ll5KyyJEktPdfZZlJOqRKzweDczF2+Mgd/6jkKQ25KHxLDXD6EtAxmgaBm4slEKsyGs7CO0YOzj+ERKc5Pe3pvf3ylkOrTnCNnaiK8f3c5SwE9mPIZWyU6SSyM/SMXW1eM+tXmWAC1ug43gr8DkRSPhfM8pCdnsKdJy0r4Qvc9ZBm4S1D9IsNvu3INLPYaHMHyBy3Qs6f/wql+QQtS5A/Af8pEAbLjcpQ1PHwSVBJQYuA7/Vs/YsChd2hOZejKM7m4xMNmmN3+hxsmoW+i6KSN57UXr+ykASGv+JYUFggyU5Q/JBwWAgUxcDWUqnDhDvt44oKxJG3IjzbWHECfruYXTlBU5098UoYmyPBBLjWDr4uhpM2R7r8cQtqsRUecNKWG2JrfKacnjRsJFmlT03Bg0Nuu0RUwrA8pYbxW2Mw42SvxyPJSS/d0g+Iyy/GJobwhMeDdHKgi0RS48RM88v5nr65Im9nyxM43OfxeeLUszxsr5YyHjmcqhd0EATs93g8jIvZZurGC/qG8tYPpNs7HG5SsT38AAYoXK3K+7RfpSeRxlmYVT2+iMOL5y8R1mXzi1y2fmenmWnfpi4khd4WCK8cZNyUN705vDkIwmGTM5KAJFyECcyR7SfMy7mQnd+1kYnO5wXHo+Kv2NapJzIBckDoUXUa7lCoWCRHIXOf/zB4KitavFdjNFiiknw0uH9ecM4xLdT0geyNvG143euBrgOWoSBFCE/dNynLGB6g8hU+Rum+HxvcnxJ4mJrSQpCaenMSWEvp9+HMIkCxyeLpCRf8FXpVLgsxuHy/52/iNJ/p7Oejarrxc7V0wrVDwUs0/vHxDatMMSU5+H6rWDShJNSkH4cQG5RFHthYM9xFmYYxRgEsU5fC81g55f4fbADgvinAA/AXjidwwNcI8cFKQC5IYCigCDKQm1VK6JXC0gMqIFrlor5WpkHud2GF7DXBjG1x9DSV66mpVoTUxH/UV6Xlzq5XOTgD6jcM4uylPI2/XBeKwAubwBSgMgcUoGZjXEyTMPEd+l7rSuU3irxd2IlTzBovRn8BBuumUDuEEs7LzhjUJBmivyk7VNuF67uChHbdeG90UtTkrqDK9XejNPF92IsJYGHY5CkGTkqSJJTegkqiHIzRtGtN0VCReFyL4DMUlEmIb4NzHC55MFcq3xPEsN4mRxmN1rcoqgbLpYOd22fvQJVFbGeTiJkbp0mGCQXhN8FMmnLjRLrcc7a2cZ6cUEmvFz2iqVD/uR4JbCpvTTwkVRCbi6eEwD82vTl6w75H8nDLAXPId382ptfoxifp2msiHX9xFTyM0i6x0rIVRepEGnWeUR18b4sdLVJ+zzWVjU3/BSQctpDV+ObuRWe8LP1+Ow0fUPLkpt8z99fd/JHPd3QutbIeJcV2sWN60UgLUEskdMhPImvXRDvS5RVXsnXso7E0EG7Dc4SS3OEIVOYiSDqrZT0M1Nt9Vu6Ubgkk6mzxPR2I1vHjHuka5Svg1/XAOgLYrJEIfgxC0pv3UiAcxn8hE7jLm/mILHTOrFUckjihtEU0YcumlG+/NRDDPHD6U1MH2dwSCH0ljfzo6NR+mLy6EjhpYS+WyAm3VVfQNW+1Lio0fJr2qLW29LrtGiy8wMBvIID+0ik1la1OcLd8USNptd5hDnR9FoR60k69i4auUIXIN4T4u7uimakQ74c7AtkgwolWpDn62fRNI9Scm2tzzLqws2otPZ4sWoMDvb5EktUqReqN/hNN+8pb9mtzdU4LzbYhc0kKu7k00BjVzv0WkJmmcGu4yI0R5+VnX82f3/fSv8eNX8Xd0qwFctXoPwvAZ9fnOsM1dbTBr4PZl4Eoq0fzmMbh3b6UreiLnbiG5pHxeuTlJByk4oydEpvjX7ZGzZqHCNtzBb/Zkn45QR+sZq/7A3ZZSCJ20yvbKpguwTu9qWU70McBnaMIy+YK2IdPnwkk08xuEmSE82ByIixs1gC0OKbqbQS1lhHSxQ5JNt0l7AyBTYwshpMnwhys7O3zmI58Q1BB9JtCZYbBGkEyT66RT6AoAezMFpQBRx/nXeA4tiZo3RT0kR9hJF7BOJvW+SOPZfytrfQrU30nfscsCMQ88+7PDhHIHLfdjM4jkBkn3Zz84kM9nGXs/YIxOLLbmbaEYjs025uQcY59tzdXO3soeW5qzRHuJtl8Z6LC3oler/+64eVeF+O4lWTrKoVhVMgxU2AnxQKULke8OWPG2+uyQBkxzfe0k5/dGS7kUdm+WLOJpX9M/14Fbp3ilgvLvFAegPCAGHHdbBzRF9/C3SYXkbh5zsQfhNAuMZ4edRqyQevmp0X+032/60FW9Xyghg7wRT9w1l6EjtFKJ12R5baHakts5zjUu1rTKWGFLZmw5IgEpPIVEMSEcFOcxoulglGTezM474XY3iAhbOsx8hHU1xv0t7wQI82dcG88Zany6RHAdOJGb4vvAbBExqNxkX7snnr+AmCVguEmePHSMhXUqBSzp3SCwZOR4HzD5tM7ZvkCvkI23R0Dub2FEXYm3lTByM7CjGN7v9YxzEQNvsu2uC7i0sy5fKeo+3yJpaYKInhKTE8JQ5PKcNTaMADpK7c6CsmS0g76cCJvABFo4j+IqnvBcnnSosK46yTsiYZxnRYrvySkczKDw+bGPgfNK5zrf+mcX0yKN8Xe9U5vIZuHX/sTG+cOeqxgW1i9Ct20JHgXj6SVpwxUgCCmET+hnEEiFQQIhQnPlbEezITcD8JSetgSi2eplrxeqbgp2inKfdfH5lmYYKXCVbEOgmILCKJy9K3bES1lKUcd8ydjFTjpiVvBhs9CoI4Msvn1jhxQ8BuMAOmFZlwP3wk020q/8NH+OTEGZU2wbKea3NhdlrIBL4AGQ7aXyOUQpcW/GRJJhMj/ZFtxaPL1OUKvYEpBawgdspPsmOiIO4J7K6F/rNORWC/nimz98NPKOo6MZe892FMj5PkTHpxRH8yfHS5Qz77hPnokiXbzydhBOmvfXeJQ8R8c/BikA+b7X1wAheESD5s7wvkoePHIbho5gXIJQcfUhfCgHRO2jB305FTzH5p7MW0VOIwk5Hua3oLz3ci/46eoDrtbKNOp9neb8IIX6PokxejTYKaDIVmYQLLXro5cCsc/5NzV2iQwQdB+IlbXWT1BhD2mu0Ugtu9Zvs5CDCn8QhQEV8Kfuq7Snpnp5PMvSyYVqxAC003ryR63CQi8qBYNd8Lt+RJdk/wXhAatEMEib+xlPABt762ekO5reRVgvmth6/DBPdCXIrSTDC9tUQu7MSt981Wa75TOiA+0e7kKglw0ow4wcW2q8et5izesj6znU91VscZc/W89P1TvlRgyoBkSV+G9XsVCvbPcLKbF1OpN2pf4aFyZArpEXuLR3h9BRfNnMTHuQe+ee+meM8gXjVZPY/PO4/HSIqj7+Di1i1W6lv3l6KvUaE0V5X2IqoRzejIVKIIjUZFyWWEbr0wift/o7Ib9ywpvZGDV540g9us9pKR4r6M9sWfl6Q2cu9wNgXn70pdEO9vsxwoxOfvvzfvtRm077lntVD+W8L48bTaXqtv/z1RcnGZeXgd8dutGD8J3xzb3Ki8Mg8SH3tqNL2u1ObUVG/hzJEVHqNxQia7tELXAH7+8JGOEGurYRaFC0qahgF2SNvWiYy4CfoMuuVnF95lc1GVcN4hk0AQYpoOu5nK+NrBu+mtK3vyFGG19aipmJR5bJMiW4vclwrclj1oVGzY51tK2VPL2Leo9I0F67nFajDpW7pN/+MGT4vqr61Qz94oPwimJWl7OXgW2lniPzXtn6Y2eyNeHsC2H6SzGYzdChS3BJ2N8xd7r/WDTV/pUcWbwWO1F9BnL8bs9DEnJ45rBAV/Py9hEWJnnzA/ZW1lL2/IpjD4HpPfI3MWp4Xy7W2RkyaAFBWdmb5V5agrAQqd4Ndf8x6U//tuopBA30xtJlXEVXou33G/xkSG/hdt+3vsYt2KROeqVvv5jdYfa4apjU5q/xcAAP//uFyFLP5KAAA=
 
 
 

pkg/agent/testdata/AKSUbuntu1604+Containerd/line9.sh

@@ -136,6 +136,7 @@ NVIDIA_CONTAINER_RUNTIME_VERSION="3.6.0"
 export NVIDIA_DRIVER_IMAGE_SHA="${GPU_IMAGE_SHA:=}"
 export NVIDIA_DRIVER_IMAGE_TAG="${GPU_DV}-${NVIDIA_DRIVER_IMAGE_SHA}"
 export NVIDIA_DRIVER_IMAGE="mcr.microsoft.com/aks/aks-gpu"
+export NVIDIA_GPU_DRIVER_TYPE="${GPU_DRIVER_TYPE:=}"
 export CTR_GPU_INSTALL_CMD="ctr -n k8s.io run --privileged --rm --net-host --with-ns pid:/proc/1/ns/pid --mount type=bind,src=/opt/gpu,dst=/mnt/gpu,options=rbind --mount type=bind,src=/opt/actions,dst=/mnt/actions,options=rbind"
 export DOCKER_GPU_INSTALL_CMD="docker run --privileged --net=host --pid=host -v /opt/gpu:/mnt/gpu -v /opt/actions:/mnt/actions --rm"
 APT_CACHE_DIR=/var/cache/apt/archives/