From 74b3785c35811c4c3fe997bad4b506b23c2b4fe9 Mon Sep 17 00:00:00 2001 From: Joe Runde Date: Thu, 28 Mar 2024 15:02:07 -0600 Subject: [PATCH] :bug: fixup args.max_logprobs Signed-off-by: Joe Runde --- vllm/tgis_utils/args.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vllm/tgis_utils/args.py b/vllm/tgis_utils/args.py index 1951c09cd..ab1aed7f1 100644 --- a/vllm/tgis_utils/args.py +++ b/vllm/tgis_utils/args.py @@ -1,6 +1,10 @@ import argparse import os +from vllm.entrypoints.grpc.validation import MAX_TOP_N_TOKENS +from vllm.logger import init_logger + +logger = init_logger(__name__) def _to_env_var(arg_name: str): return arg_name.upper().replace("-", "_") @@ -105,5 +109,8 @@ def postprocess_tgis_args(args: argparse.Namespace) -> argparse.Namespace: "Inconsistent tensor_parallel_size and num_gpus/num_shard " "arg values") args.tensor_parallel_size = num_gpus + if args.max_logprobs < MAX_TOP_N_TOKENS + 1: + logger.info("Setting max_logprobs to %d", MAX_TOP_N_TOKENS + 1) + args.max_logprobs = MAX_TOP_N_TOKENS + 1 return args