feat(lmeval): Disable online evaluation

api/lmes/v1alpha1/lmevaljob_types.go

@@ -300,8 +300,17 @@ type LMEvalJobSpec struct {
 	// Outputs specifies storage for evaluation results
 	// +optional
 	Outputs *Outputs `json:"outputs,omitempty"`
-	// Offline specifies settings for running LMEvalJobs in a offline mode
+	// Offline specifies settings for running LMEvalJobs in an offline mode
+	// +optional
 	Offline *OfflineSpec `json:"offline,omitempty"`
+	// AllowOnly specifies whether the LMEvalJob can directly download remote code, datasets and metrics. Default is false.
+	// +optional
+	// +kubebuilder:default:=false
+	AllowOnline *bool `json:"allowOnline,omitempty"`
+	// AllowCodeExecution specifies whether the LMEvalJob can execute remote code. Default is false.
+	// +optional
+	// +kubebuilder:default:=false
+	AllowCodeExecution *bool `json:"allowCodeExecution,omitempty"`
 }
 
 // IsOffline returns whether this LMEvalJob is configured to run offline

config/crd/bases/trustyai.opendatahub.io_lmevaljobs.yaml

@@ -43,6 +43,16 @@ spec:
           spec:
             description: LMEvalJobSpec defines the desired state of LMEvalJob
             properties:
+              allowCodeExecution:
+                default: false
+                description: AllowCodeExecution specifies whether the LMEvalJob can
+                  execute remote code. Default is false.
+                type: boolean
+              allowOnline:
+                default: false
+                description: AllowOnly specifies whether the LMEvalJob can directly
+                  download remote code, datasets and metrics. Default is false.
+                type: boolean
               batchSize:
                 description: |-
                   Batch size for the evaluation. This is used by the models that run and are loaded
@@ -91,7 +101,7 @@ spec:
                 type: integer
               offline:
                 description: Offline specifies settings for running LMEvalJobs in
-                  a offline mode
+                  an offline mode
                 properties:
                   storage:
                     description: OfflineStorageSpec defines the storage configuration

controllers/lmes/lmevaljob_controller.go

@@ -713,9 +713,23 @@ func CreatePod(svcOpts *serviceOptions, job *lmesv1alpha1.LMEvalJob, log logr.Lo
 		volumes = append(volumes, outputPVC)
 	}
 
-	// If the job is supposed to run offline, set the appropriate HuggingFace offline flags
-	if job.Spec.IsOffline() {
+	// Disable remote code execution by default
+	if job.Spec.AllowCodeExecution == nil || *job.Spec.AllowCodeExecution == false {
+		remoteCodeEnvVars := []corev1.EnvVar{
+			{
+				Name:  "TRUST_REMOTE_CODE",
+				Value: "0",
+			},
+			{
+				Name:  "HF_DATASETS_TRUST_REMOTE_CODE",
+				Value: "0",
+			},
+		}
+		envVars = append(envVars, remoteCodeEnvVars...)
+	}
 
+	// Enforce offline mode by default
+	if job.Spec.AllowOnline == nil || *job.Spec.AllowOnline == false {
 		offlineHuggingFaceEnvVars := []corev1.EnvVar{
 			{
 				Name:  "HF_DATASETS_OFFLINE",
@@ -725,8 +739,19 @@ func CreatePod(svcOpts *serviceOptions, job *lmesv1alpha1.LMEvalJob, log logr.Lo
 				Name:  "HF_HUB_OFFLINE",
 				Value: "1",
 			},
+			{
+				Name:  "TRANSFORMERS_OFFLINE",
+				Value: "1",
+			},
+			{
+				Name:  "HF_EVALUATE_OFFLINE",
+				Value: "1",
+			},
 		}
 		envVars = append(envVars, offlineHuggingFaceEnvVars...)
+	}
+
+	if job.Spec.IsOffline() {
 
 		// If the job is offline, a storage must be set. PVC is the only supported storage backend at the moment.
 		offlinePVCMount := corev1.VolumeMount{