Fix q3k imatrix quantization (#48)

EricLBuehler · Dec 1, 2024 · 84c89a6 · 84c89a6
1 parent e5685ce
commit 84c89a6
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/candle-core/src/quantized/k_quants.rs b/candle-core/src/quantized/k_quants.rs
@@ -1222,7 +1222,7 @@ impl GgmlType for BlockQ3K {
                 if j < 8 {
                     block.scales[j] = (l & 0xF) as u8;
                 } else {
-                    block.scales[j - 8] = ((l & 0xF) << 4) as u8;
+                    block.scales[j - 8] |= ((l & 0xF) << 4) as u8;
                 }
                 let l = l >> 4;
                 block.scales[j % 4 + 8] |= (l << (2 * (j / 4))) as u8;

diff --git a/candle-core/tests/quantized_tests.rs b/candle-core/tests/quantized_tests.rs
@@ -763,9 +763,9 @@ fn imatrix_quantize_q3k() -> Result<()> {
     let dequant1 = quant1.dequantize(cpu)?;
     let dequant2 = quant2.dequantize(cpu)?;
 
-    let _err1 = (dequant1 - &xs)?.abs()?.mean_all()?.to_scalar::<f32>()?;
+    let err1 = (dequant1 - &xs)?.abs()?.mean_all()?.to_scalar::<f32>()?;
     let err2 = (dequant2 - &xs)?.abs()?.mean_all()?.to_scalar::<f32>()?;
-    assert!(err2 < 0.16);
+    assert!(err2 < err1, "err2 {err2} > err1 {err1}");
 
     Ok(())
 }