diff --git a/candle-transformers/src/models/whisper/audio.rs b/candle-transformers/src/models/whisper/audio.rs
index 35f9f3df5f..8490533c4d 100644
--- a/candle-transformers/src/models/whisper/audio.rs
+++ b/candle-transformers/src/models/whisper/audio.rs
@@ -204,6 +204,7 @@ pub fn log_mel_spectrogram_<T: Float>(
 
     // ensure that the number of threads is even and less than 12
     let n_threads = std::cmp::min(get_num_threads() - get_num_threads() % 2, 12);
+    let n_threads = std::cmp::max(n_threads, 2);
 
     let hann = Arc::new(hann);
     let samples = Arc::new(samples);