Update modelopt config and fix running issue (#2792)

sgl-project · Jan 8, 2025 · b5fb4ef · b5fb4ef
1 parent 2e6346f
commit b5fb4ef
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/python/sglang/srt/layers/quantization/__init__.py b/python/sglang/srt/layers/quantization/__init__.py
@@ -17,12 +17,12 @@
 from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinConfig
 from vllm.model_executor.layers.quantization.gptq_marlin_24 import GPTQMarlin24Config
 from vllm.model_executor.layers.quantization.marlin import MarlinConfig
-from vllm.model_executor.layers.quantization.modelopt import ModelOptFp8Config
 from vllm.model_executor.layers.quantization.qqq import QQQConfig
 from vllm.model_executor.layers.quantization.tpu_int8 import Int8TpuConfig
 
 from sglang.srt.layers.quantization.base_config import QuantizationConfig
 from sglang.srt.layers.quantization.fp8 import Fp8Config
+from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp8Config
 
 QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
     "aqlm": AQLMConfig,

diff --git a/python/sglang/srt/layers/modelopt_quant.py → ...srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/modelopt_quant.py → ...srt/layers/quantization/modelopt_quant.py
@@ -142,6 +142,7 @@ def create_weights(
                         data=torch.full(
                             (len(output_partition_sizes),),
                             torch.finfo(torch.float32).min,
+                            dtype=torch.float32,
                         ),
                         weight_loader=weight_loader,
                     ),