From 93a056fe4249281107baafc2a936e34f7f59e111 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:30:45 +0100 Subject: [PATCH 1/4] Fix imports from model_config (#443) --- docs/source/using-the-python-api.mdx | 2 +- src/lighteval/main_endpoint.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/using-the-python-api.mdx b/docs/source/using-the-python-api.mdx index 2e160a679..8c44050f4 100644 --- a/docs/source/using-the-python-api.mdx +++ b/docs/source/using-the-python-api.mdx @@ -11,7 +11,7 @@ After that, simply run the pipeline and save the results. ```python import lighteval from lighteval.logging.evaluation_tracker import EvaluationTracker -from lighteval.models.model_config import VLLMModelConfig +from lighteval.models.vllm.vllm_model import VLLMModelConfig from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters from lighteval.utils.utils import EnvConfig from lighteval.utils.imports import is_accelerate_available diff --git a/src/lighteval/main_endpoint.py b/src/lighteval/main_endpoint.py index 952aae074..be75b711a 100644 --- a/src/lighteval/main_endpoint.py +++ b/src/lighteval/main_endpoint.py @@ -93,7 +93,7 @@ def openai( Evaluate OPENAI models. """ from lighteval.logging.evaluation_tracker import EvaluationTracker - from lighteval.models.model_config import OpenAIModelConfig + from lighteval.models.endpoints.openai_model import OpenAIModelConfig from lighteval.pipeline import EnvConfig, ParallelismManager, Pipeline, PipelineParameters env_config = EnvConfig(token=TOKEN, cache_dir=cache_dir) @@ -317,7 +317,7 @@ def tgi( import yaml from lighteval.logging.evaluation_tracker import EvaluationTracker - from lighteval.models.model_config import TGIModelConfig + from lighteval.models.endpoints.tgi_model import TGIModelConfig from lighteval.pipeline import EnvConfig, ParallelismManager, Pipeline, PipelineParameters env_config = EnvConfig(token=TOKEN, cache_dir=cache_dir) From 500632a097499fc7a2002ad682d605cb03072302 Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:10:21 +0100 Subject: [PATCH 2/4] Fix wrong instructions and code for custom tasks (#450) * Delete wrong instruction in custom task docs * Delete wrong code for custom tasks * Delete wrong code for extended tasks * Delete wrong code for community tasks * Delete unnecessary code for community tasks --- community_tasks/_template.py | 7 ------- community_tasks/aimo_evals.py | 8 -------- community_tasks/arabic_evals.py | 4 ---- community_tasks/german_rag_evals.py | 8 -------- community_tasks/oz_evals.py | 5 ----- community_tasks/serbian_eval.py | 4 ---- docs/source/adding-a-custom-task.mdx | 11 ----------- examples/nanotron/custom_evaluation_tasks.py | 4 ---- src/lighteval/tasks/extended/ifeval/main.py | 5 ----- src/lighteval/tasks/extended/mix_eval/main.py | 5 ----- src/lighteval/tasks/extended/mt_bench/main.py | 4 ---- src/lighteval/tasks/extended/tiny_benchmarks/main.py | 8 -------- 12 files changed, 73 deletions(-) diff --git a/community_tasks/_template.py b/community_tasks/_template.py index d0099ba26..2db28e340 100644 --- a/community_tasks/_template.py +++ b/community_tasks/_template.py @@ -116,10 +116,3 @@ def __init__( sample_level_fn=lambda x: x, # how to compute score for one sample corpus_level_fn=np.mean, # aggregation ) - -# MODULE LOGIC -# You should not need to touch this -# Convert to dict for lighteval -if __name__ == "__main__": - print(t.name for t in TASKS_TABLE) - print(len(TASKS_TABLE)) diff --git a/community_tasks/aimo_evals.py b/community_tasks/aimo_evals.py index be59950bd..885ffd8da 100644 --- a/community_tasks/aimo_evals.py +++ b/community_tasks/aimo_evals.py @@ -56,11 +56,3 @@ def aimo_prompt(line, task_name: str = None): # STORE YOUR EVALS TASKS_TABLE = [task] - - -# MODULE LOGIC -# You should not need to touch this - -if __name__ == "__main__": - print(t.name for t in TASKS_TABLE) - print(len(TASKS_TABLE)) diff --git a/community_tasks/arabic_evals.py b/community_tasks/arabic_evals.py index 07a096eca..4408f22fa 100644 --- a/community_tasks/arabic_evals.py +++ b/community_tasks/arabic_evals.py @@ -856,7 +856,3 @@ def __init__( + [toxigen_ar_task] + [sciq_ar_task] ) - -if __name__ == "__main__": - print(t.name for t in TASKS_TABLE) - print(len(TASKS_TABLE)) diff --git a/community_tasks/german_rag_evals.py b/community_tasks/german_rag_evals.py index 30d6dcb4a..78af6794e 100644 --- a/community_tasks/german_rag_evals.py +++ b/community_tasks/german_rag_evals.py @@ -221,11 +221,3 @@ def prompt_fn_context_question_match(line, task_name: str = None): # STORE YOUR EVALS TASKS_TABLE = [task1, task2, task3, task4] - - -# MODULE LOGIC -# You should not need to touch this - -if __name__ == "__main__": - print(t.name for t in TASKS_TABLE) - print(len(TASKS_TABLE)) diff --git a/community_tasks/oz_evals.py b/community_tasks/oz_evals.py index 6252a20a0..4ec70e291 100644 --- a/community_tasks/oz_evals.py +++ b/community_tasks/oz_evals.py @@ -87,8 +87,3 @@ def prompt_fn_oz_eval_task(line, task_name: str = None): # STORE YOUR EVALS TASKS_TABLE = [oz_eval_task] - - -if __name__ == "__main__": - print(t["name"] for t in TASKS_TABLE) - print(len(TASKS_TABLE)) diff --git a/community_tasks/serbian_eval.py b/community_tasks/serbian_eval.py index 3b49c4cb0..d972ac69c 100644 --- a/community_tasks/serbian_eval.py +++ b/community_tasks/serbian_eval.py @@ -784,7 +784,3 @@ def create_task_config( mmlu_world_religions, mmlu_all, ] - -if __name__ == "__main__": - print(t.name for t in TASKS_TABLE) - print(len(TASKS_TABLE)) diff --git a/docs/source/adding-a-custom-task.mdx b/docs/source/adding-a-custom-task.mdx index e1823b7b9..e5160024d 100644 --- a/docs/source/adding-a-custom-task.mdx +++ b/docs/source/adding-a-custom-task.mdx @@ -167,17 +167,6 @@ TASKS_TABLE = SUBSET_TASKS # TASKS_TABLE = [task] ``` -Finally, you need to add a module logic to convert your task to a dict for lighteval. - -```python -# MODULE LOGIC -# You should not need to touch this -# Convert to dict for lighteval -if __name__ == "__main__": - print(t.name for t in TASKS_TABLE) - print(len(TASKS_TABLE)) -``` - Once your file is created you can then run the evaluation with the following command: ```bash diff --git a/examples/nanotron/custom_evaluation_tasks.py b/examples/nanotron/custom_evaluation_tasks.py index 78c354916..e4b1b5ca6 100644 --- a/examples/nanotron/custom_evaluation_tasks.py +++ b/examples/nanotron/custom_evaluation_tasks.py @@ -671,7 +671,3 @@ def __init__( "all": ",".join(t[1] for t in _TASKS_STRINGS), "early-signal": EARLY_SIGNAL_TASKS, } - -if __name__ == "__main__": - print(t["name"] for t in TASKS_TABLE) - print(len(TASKS_TABLE)) diff --git a/src/lighteval/tasks/extended/ifeval/main.py b/src/lighteval/tasks/extended/ifeval/main.py index e6947bb6e..60d1be5fa 100644 --- a/src/lighteval/tasks/extended/ifeval/main.py +++ b/src/lighteval/tasks/extended/ifeval/main.py @@ -160,8 +160,3 @@ def agg_inst_level_acc(items): TASKS_TABLE = [ifeval] extend_enum(Metrics, "ifeval_metric", ifeval_metrics) - -if __name__ == "__main__": - # Adds the metric to the metric list! - print(t["name"] for t in TASKS_TABLE) - print(len(TASKS_TABLE)) diff --git a/src/lighteval/tasks/extended/mix_eval/main.py b/src/lighteval/tasks/extended/mix_eval/main.py index 15d7490bc..8684e910c 100644 --- a/src/lighteval/tasks/extended/mix_eval/main.py +++ b/src/lighteval/tasks/extended/mix_eval/main.py @@ -228,8 +228,3 @@ def mean_dv_5(x): TASKS_TABLE = [mixeval_multichoice_easy, mixeval_freeform_easy, mixeval_multichoice_hard, mixeval_freeform_hard] - -if __name__ == "__main__": - # Adds the metric to the metric list! - print(t["name"] for t in TASKS_TABLE) - print(len(TASKS_TABLE)) diff --git a/src/lighteval/tasks/extended/mt_bench/main.py b/src/lighteval/tasks/extended/mt_bench/main.py index e5b209982..117e363dd 100644 --- a/src/lighteval/tasks/extended/mt_bench/main.py +++ b/src/lighteval/tasks/extended/mt_bench/main.py @@ -95,7 +95,3 @@ def flow_judge_mt_bench_prompt(question, answer, options, gold): TASKS_TABLE = [task] - -if __name__ == "__main__": - print(t["name"] for t in TASKS_TABLE) - print(len(TASKS_TABLE)) diff --git a/src/lighteval/tasks/extended/tiny_benchmarks/main.py b/src/lighteval/tasks/extended/tiny_benchmarks/main.py index b283921f2..fae6e89df 100644 --- a/src/lighteval/tasks/extended/tiny_benchmarks/main.py +++ b/src/lighteval/tasks/extended/tiny_benchmarks/main.py @@ -283,11 +283,3 @@ def aggregate(self, y_input): corpus_level_fn=TinyCorpusAggregator(name).aggregate, ), ) - - -# MODULE LOGIC -# You should not need to touch this -# Convert to dict for lighteval -if __name__ == "__main__": - print(t["name"] for t in TASKS_TABLE) - print(len(TASKS_TABLE)) From 0ebc7ec016ea5814960e3bfa08bec9cc611698ab Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:39:46 +0100 Subject: [PATCH 3/4] Fix minor typos (#449) * Fix paramater -> parameter * Fix pannel -> panel * Fix refenrence -> reference --- src/lighteval/main_accelerate.py | 38 ++++++------- src/lighteval/main_baseline.py | 18 +++--- src/lighteval/main_endpoint.py | 94 ++++++++++++++++---------------- src/lighteval/main_nanotron.py | 8 +-- src/lighteval/main_vllm.py | 36 ++++++------ 5 files changed, 97 insertions(+), 97 deletions(-) diff --git a/src/lighteval/main_accelerate.py b/src/lighteval/main_accelerate.py index 3454a223b..a0a01abe8 100644 --- a/src/lighteval/main_accelerate.py +++ b/src/lighteval/main_accelerate.py @@ -33,10 +33,10 @@ TOKEN = os.getenv("HF_TOKEN") CACHE_DIR: str = os.getenv("HF_HOME", "/scratch") -HELP_PANNEL_NAME_1 = "Common Paramaters" -HELP_PANNEL_NAME_2 = "Logging Parameters" -HELP_PANNEL_NAME_3 = "Debug Paramaters" -HELP_PANNEL_NAME_4 = "Modeling Paramaters" +HELP_PANEL_NAME_1 = "Common Parameters" +HELP_PANEL_NAME_2 = "Logging Parameters" +HELP_PANEL_NAME_3 = "Debug Parameters" +HELP_PANEL_NAME_4 = "Modeling Parameters" def accelerate( # noqa C901 @@ -50,51 +50,51 @@ def accelerate( # noqa C901 tasks: Annotated[str, Argument(help="Comma-separated list of tasks to evaluate on.")], # === Common parameters === use_chat_template: Annotated[ - bool, Option(help="Use chat template for evaluation.", rich_help_panel=HELP_PANNEL_NAME_4) + bool, Option(help="Use chat template for evaluation.", rich_help_panel=HELP_PANEL_NAME_4) ] = False, system_prompt: Annotated[ - Optional[str], Option(help="Use system prompt for evaluation.", rich_help_panel=HELP_PANNEL_NAME_4) + Optional[str], Option(help="Use system prompt for evaluation.", rich_help_panel=HELP_PANEL_NAME_4) ] = None, dataset_loading_processes: Annotated[ - int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANNEL_NAME_1) + int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, custom_tasks: Annotated[ - Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANNEL_NAME_1) + Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANEL_NAME_1) ] = None, cache_dir: Annotated[ - Optional[str], Option(help="Cache directory for datasets and models.", rich_help_panel=HELP_PANNEL_NAME_1) + Optional[str], Option(help="Cache directory for datasets and models.", rich_help_panel=HELP_PANEL_NAME_1) ] = None, num_fewshot_seeds: Annotated[ - int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANNEL_NAME_1) + int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, # === saving === output_dir: Annotated[ - str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANNEL_NAME_2) + str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANEL_NAME_2) ] = "results", push_to_hub: Annotated[ - bool, Option(help="Push results to the huggingface hub.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results to the huggingface hub.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, push_to_tensorboard: Annotated[ - bool, Option(help="Push results to tensorboard.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results to tensorboard.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, public_run: Annotated[ - bool, Option(help="Push results and details to a public repo.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results and details to a public repo.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, results_org: Annotated[ - Optional[str], Option(help="Organization to push results to.", rich_help_panel=HELP_PANNEL_NAME_2) + Optional[str], Option(help="Organization to push results to.", rich_help_panel=HELP_PANEL_NAME_2) ] = None, save_details: Annotated[ - bool, Option(help="Save detailed, sample per sample, results.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Save detailed, sample per sample, results.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, # === debug === max_samples: Annotated[ - Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANNEL_NAME_3) + Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANEL_NAME_3) ] = None, override_batch_size: Annotated[ - int, Option(help="Override batch size for evaluation.", rich_help_panel=HELP_PANNEL_NAME_3) + int, Option(help="Override batch size for evaluation.", rich_help_panel=HELP_PANEL_NAME_3) ] = -1, job_id: Annotated[ - int, Option(help="Optional job id for future refenrence.", rich_help_panel=HELP_PANNEL_NAME_3) + int, Option(help="Optional job id for future reference.", rich_help_panel=HELP_PANEL_NAME_3) ] = 0, ): """ diff --git a/src/lighteval/main_baseline.py b/src/lighteval/main_baseline.py index dd4786679..2dd970ea8 100644 --- a/src/lighteval/main_baseline.py +++ b/src/lighteval/main_baseline.py @@ -30,28 +30,28 @@ CACHE_DIR: str = os.getenv("HF_HOME", "/scratch") -HELP_PANNEL_NAME_1 = "Common Paramaters" -HELP_PANNEL_NAME_2 = "Logging Parameters" -HELP_PANNEL_NAME_3 = "Debug Paramaters" -HELP_PANNEL_NAME_4 = "Modeling Paramaters" +HELP_PANEL_NAME_1 = "Common Parameters" +HELP_PANEL_NAME_2 = "Logging Parameters" +HELP_PANEL_NAME_3 = "Debug Parameters" +HELP_PANEL_NAME_4 = "Modeling Parameters" def baseline( tasks: Annotated[str, Argument(help="Comma-separated list of tasks to evaluate on.")], cache_dir: Annotated[ - str, Option(help="Cache directory for datasets and models.", rich_help_panel=HELP_PANNEL_NAME_1) + str, Option(help="Cache directory for datasets and models.", rich_help_panel=HELP_PANEL_NAME_1) ] = CACHE_DIR, custom_tasks: Annotated[ - Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANNEL_NAME_1) + Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANEL_NAME_1) ] = None, dataset_loading_processes: Annotated[ - int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANNEL_NAME_1) + int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, output_dir: Annotated[ - str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANNEL_NAME_2) + str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANEL_NAME_2) ] = "results", max_samples: Annotated[ - Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANNEL_NAME_3) + Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANEL_NAME_3) ] = None, ): """ diff --git a/src/lighteval/main_endpoint.py b/src/lighteval/main_endpoint.py index be75b711a..208cc8386 100644 --- a/src/lighteval/main_endpoint.py +++ b/src/lighteval/main_endpoint.py @@ -33,10 +33,10 @@ TOKEN = os.getenv("HF_TOKEN") CACHE_DIR: str = os.getenv("HF_HOME", "/scratch") -HELP_PANNEL_NAME_1 = "Common Paramaters" -HELP_PANNEL_NAME_2 = "Logging Parameters" -HELP_PANNEL_NAME_3 = "Debug Paramaters" -HELP_PANNEL_NAME_4 = "Modeling Paramaters" +HELP_PANEL_NAME_1 = "Common Parameters" +HELP_PANEL_NAME_2 = "Logging Parameters" +HELP_PANEL_NAME_3 = "Debug Parameters" +HELP_PANEL_NAME_4 = "Modeling Parameters" @app.command(rich_help_panel="Evaluation Backends") @@ -48,45 +48,45 @@ def openai( tasks: Annotated[str, Argument(help="Comma-separated list of tasks to evaluate on.")], # === Common parameters === system_prompt: Annotated[ - Optional[str], Option(help="Use system prompt for evaluation.", rich_help_panel=HELP_PANNEL_NAME_4) + Optional[str], Option(help="Use system prompt for evaluation.", rich_help_panel=HELP_PANEL_NAME_4) ] = None, dataset_loading_processes: Annotated[ - int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANNEL_NAME_1) + int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, custom_tasks: Annotated[ - Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANNEL_NAME_1) + Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANEL_NAME_1) ] = None, cache_dir: Annotated[ - str, Option(help="Cache directory for datasets and models.", rich_help_panel=HELP_PANNEL_NAME_1) + str, Option(help="Cache directory for datasets and models.", rich_help_panel=HELP_PANEL_NAME_1) ] = CACHE_DIR, num_fewshot_seeds: Annotated[ - int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANNEL_NAME_1) + int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, # === saving === output_dir: Annotated[ - str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANNEL_NAME_2) + str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANEL_NAME_2) ] = "results", push_to_hub: Annotated[ - bool, Option(help="Push results to the huggingface hub.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results to the huggingface hub.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, push_to_tensorboard: Annotated[ - bool, Option(help="Push results to tensorboard.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results to tensorboard.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, public_run: Annotated[ - bool, Option(help="Push results and details to a public repo.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results and details to a public repo.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, results_org: Annotated[ - Optional[str], Option(help="Organization to push results to.", rich_help_panel=HELP_PANNEL_NAME_2) + Optional[str], Option(help="Organization to push results to.", rich_help_panel=HELP_PANEL_NAME_2) ] = None, save_details: Annotated[ - bool, Option(help="Save detailed, sample per sample, results.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Save detailed, sample per sample, results.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, # === debug === max_samples: Annotated[ - Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANNEL_NAME_3) + Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANEL_NAME_3) ] = None, job_id: Annotated[ - int, Option(help="Optional job id for future refenrence.", rich_help_panel=HELP_PANNEL_NAME_3) + int, Option(help="Optional job id for future reference.", rich_help_panel=HELP_PANEL_NAME_3) ] = 0, ): """ @@ -148,51 +148,51 @@ def inference_endpoint( tasks: Annotated[str, Argument(help="Comma-separated list of tasks to evaluate on.")], # === Common parameters === use_chat_template: Annotated[ - bool, Option(help="Use chat template for evaluation.", rich_help_panel=HELP_PANNEL_NAME_4) + bool, Option(help="Use chat template for evaluation.", rich_help_panel=HELP_PANEL_NAME_4) ] = False, system_prompt: Annotated[ - Optional[str], Option(help="Use system prompt for evaluation.", rich_help_panel=HELP_PANNEL_NAME_4) + Optional[str], Option(help="Use system prompt for evaluation.", rich_help_panel=HELP_PANEL_NAME_4) ] = None, dataset_loading_processes: Annotated[ - int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANNEL_NAME_1) + int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, custom_tasks: Annotated[ - Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANNEL_NAME_1) + Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANEL_NAME_1) ] = None, cache_dir: Annotated[ - str, Option(help="Cache directory for datasets and models.", rich_help_panel=HELP_PANNEL_NAME_1) + str, Option(help="Cache directory for datasets and models.", rich_help_panel=HELP_PANEL_NAME_1) ] = CACHE_DIR, num_fewshot_seeds: Annotated[ - int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANNEL_NAME_1) + int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, # === saving === output_dir: Annotated[ - str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANNEL_NAME_2) + str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANEL_NAME_2) ] = "results", push_to_hub: Annotated[ - bool, Option(help="Push results to the huggingface hub.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results to the huggingface hub.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, push_to_tensorboard: Annotated[ - bool, Option(help="Push results to tensorboard.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results to tensorboard.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, public_run: Annotated[ - bool, Option(help="Push results and details to a public repo.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results and details to a public repo.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, results_org: Annotated[ - Optional[str], Option(help="Organization to push results to.", rich_help_panel=HELP_PANNEL_NAME_2) + Optional[str], Option(help="Organization to push results to.", rich_help_panel=HELP_PANEL_NAME_2) ] = None, save_details: Annotated[ - bool, Option(help="Save detailed, sample per sample, results.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Save detailed, sample per sample, results.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, # === debug === max_samples: Annotated[ - Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANNEL_NAME_3) + Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANEL_NAME_3) ] = None, override_batch_size: Annotated[ - int, Option(help="Override batch size for evaluation.", rich_help_panel=HELP_PANNEL_NAME_3) + int, Option(help="Override batch size for evaluation.", rich_help_panel=HELP_PANEL_NAME_3) ] = None, job_id: Annotated[ - int, Option(help="Optional job id for future refenrence.", rich_help_panel=HELP_PANNEL_NAME_3) + int, Option(help="Optional job id for future reference.", rich_help_panel=HELP_PANEL_NAME_3) ] = 0, ): """ @@ -264,51 +264,51 @@ def tgi( tasks: Annotated[str, Argument(help="Comma-separated list of tasks to evaluate on.")], # === Common parameters === use_chat_template: Annotated[ - bool, Option(help="Use chat template for evaluation.", rich_help_panel=HELP_PANNEL_NAME_4) + bool, Option(help="Use chat template for evaluation.", rich_help_panel=HELP_PANEL_NAME_4) ] = False, system_prompt: Annotated[ - Optional[str], Option(help="Use system prompt for evaluation.", rich_help_panel=HELP_PANNEL_NAME_4) + Optional[str], Option(help="Use system prompt for evaluation.", rich_help_panel=HELP_PANEL_NAME_4) ] = None, dataset_loading_processes: Annotated[ - int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANNEL_NAME_1) + int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, custom_tasks: Annotated[ - Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANNEL_NAME_1) + Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANEL_NAME_1) ] = None, cache_dir: Annotated[ - str, Option(help="Cache directory for datasets and models.", rich_help_panel=HELP_PANNEL_NAME_1) + str, Option(help="Cache directory for datasets and models.", rich_help_panel=HELP_PANEL_NAME_1) ] = CACHE_DIR, num_fewshot_seeds: Annotated[ - int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANNEL_NAME_1) + int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, # === saving === output_dir: Annotated[ - str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANNEL_NAME_2) + str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANEL_NAME_2) ] = "results", push_to_hub: Annotated[ - bool, Option(help="Push results to the huggingface hub.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results to the huggingface hub.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, push_to_tensorboard: Annotated[ - bool, Option(help="Push results to tensorboard.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results to tensorboard.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, public_run: Annotated[ - bool, Option(help="Push results and details to a public repo.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results and details to a public repo.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, results_org: Annotated[ - Optional[str], Option(help="Organization to push results to.", rich_help_panel=HELP_PANNEL_NAME_2) + Optional[str], Option(help="Organization to push results to.", rich_help_panel=HELP_PANEL_NAME_2) ] = None, save_details: Annotated[ - bool, Option(help="Save detailed, sample per sample, results.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Save detailed, sample per sample, results.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, # === debug === max_samples: Annotated[ - Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANNEL_NAME_3) + Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANEL_NAME_3) ] = None, override_batch_size: Annotated[ - int, Option(help="Override batch size for evaluation.", rich_help_panel=HELP_PANNEL_NAME_3) + int, Option(help="Override batch size for evaluation.", rich_help_panel=HELP_PANEL_NAME_3) ] = -1, job_id: Annotated[ - int, Option(help="Optional job id for future refenrence.", rich_help_panel=HELP_PANNEL_NAME_3) + int, Option(help="Optional job id for future reference.", rich_help_panel=HELP_PANEL_NAME_3) ] = 0, ): """ diff --git a/src/lighteval/main_nanotron.py b/src/lighteval/main_nanotron.py index 66826122e..94004c065 100644 --- a/src/lighteval/main_nanotron.py +++ b/src/lighteval/main_nanotron.py @@ -29,10 +29,10 @@ CACHE_DIR: str = os.getenv("HF_HOME", "/scratch") -HELP_PANNEL_NAME_1 = "Common Paramaters" -HELP_PANNEL_NAME_2 = "Logging Parameters" -HELP_PANNEL_NAME_3 = "Debug Paramaters" -HELP_PANNEL_NAME_4 = "Modeling Paramaters" +HELP_PANEL_NAME_1 = "Common Parameters" +HELP_PANEL_NAME_2 = "Logging Parameters" +HELP_PANEL_NAME_3 = "Debug Parameters" +HELP_PANEL_NAME_4 = "Modeling Parameters" SEED = 1234 diff --git a/src/lighteval/main_vllm.py b/src/lighteval/main_vllm.py index 078000da5..28c4abdc0 100644 --- a/src/lighteval/main_vllm.py +++ b/src/lighteval/main_vllm.py @@ -29,10 +29,10 @@ TOKEN = os.getenv("HF_TOKEN") CACHE_DIR: str = os.getenv("HF_HOME", "/scratch") -HELP_PANNEL_NAME_1 = "Common Paramaters" -HELP_PANNEL_NAME_2 = "Logging Parameters" -HELP_PANNEL_NAME_3 = "Debug Paramaters" -HELP_PANNEL_NAME_4 = "Modeling Paramaters" +HELP_PANEL_NAME_1 = "Common Parameters" +HELP_PANEL_NAME_2 = "Logging Parameters" +HELP_PANEL_NAME_3 = "Debug Parameters" +HELP_PANEL_NAME_4 = "Modeling Parameters" def vllm( @@ -41,48 +41,48 @@ def vllm( tasks: Annotated[str, Argument(help="Comma-separated list of tasks to evaluate on.")], # === Common parameters === use_chat_template: Annotated[ - bool, Option(help="Use chat template for evaluation.", rich_help_panel=HELP_PANNEL_NAME_4) + bool, Option(help="Use chat template for evaluation.", rich_help_panel=HELP_PANEL_NAME_4) ] = False, system_prompt: Annotated[ - Optional[str], Option(help="Use system prompt for evaluation.", rich_help_panel=HELP_PANNEL_NAME_4) + Optional[str], Option(help="Use system prompt for evaluation.", rich_help_panel=HELP_PANEL_NAME_4) ] = None, dataset_loading_processes: Annotated[ - int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANNEL_NAME_1) + int, Option(help="Number of processes to use for dataset loading.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, custom_tasks: Annotated[ - Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANNEL_NAME_1) + Optional[str], Option(help="Path to custom tasks directory.", rich_help_panel=HELP_PANEL_NAME_1) ] = None, cache_dir: Annotated[ - str, Option(help="Cache directory for datasets and models.", rich_help_panel=HELP_PANNEL_NAME_1) + str, Option(help="Cache directory for datasets and models.", rich_help_panel=HELP_PANEL_NAME_1) ] = CACHE_DIR, num_fewshot_seeds: Annotated[ - int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANNEL_NAME_1) + int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANEL_NAME_1) ] = 1, # === saving === output_dir: Annotated[ - str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANNEL_NAME_2) + str, Option(help="Output directory for evaluation results.", rich_help_panel=HELP_PANEL_NAME_2) ] = "results", push_to_hub: Annotated[ - bool, Option(help="Push results to the huggingface hub.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results to the huggingface hub.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, push_to_tensorboard: Annotated[ - bool, Option(help="Push results to tensorboard.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results to tensorboard.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, public_run: Annotated[ - bool, Option(help="Push results and details to a public repo.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Push results and details to a public repo.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, results_org: Annotated[ - Optional[str], Option(help="Organization to push results to.", rich_help_panel=HELP_PANNEL_NAME_2) + Optional[str], Option(help="Organization to push results to.", rich_help_panel=HELP_PANEL_NAME_2) ] = None, save_details: Annotated[ - bool, Option(help="Save detailed, sample per sample, results.", rich_help_panel=HELP_PANNEL_NAME_2) + bool, Option(help="Save detailed, sample per sample, results.", rich_help_panel=HELP_PANEL_NAME_2) ] = False, # === debug === max_samples: Annotated[ - Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANNEL_NAME_3) + Optional[int], Option(help="Maximum number of samples to evaluate on.", rich_help_panel=HELP_PANEL_NAME_3) ] = None, job_id: Annotated[ - int, Option(help="Optional job id for future refenrence.", rich_help_panel=HELP_PANNEL_NAME_3) + int, Option(help="Optional job id for future reference.", rich_help_panel=HELP_PANEL_NAME_3) ] = 0, ): """ From 1b9e2c3cc4e59224b96640a375e3a06b53315f2b Mon Sep 17 00:00:00 2001 From: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:30:28 +0100 Subject: [PATCH 4/4] Implement TGI model config from path (#448) Implement TGI model config from path: ```python TGIModelConfig.from_path(model_config_path) ``` Follow-up to: - #434 Related to: - #439 --- src/lighteval/main_endpoint.py | 9 +--- .../models/endpoints/endpoint_model.py | 8 ++++ src/lighteval/models/endpoints/tgi_model.py | 16 +++++++ .../{ => endpoints}/test_endpoint_model.py | 0 tests/models/endpoints/test_tgi_model.py | 42 +++++++++++++++++++ 5 files changed, 67 insertions(+), 8 deletions(-) rename tests/models/{ => endpoints}/test_endpoint_model.py (100%) create mode 100644 tests/models/endpoints/test_tgi_model.py diff --git a/src/lighteval/main_endpoint.py b/src/lighteval/main_endpoint.py index 208cc8386..47d059660 100644 --- a/src/lighteval/main_endpoint.py +++ b/src/lighteval/main_endpoint.py @@ -314,7 +314,6 @@ def tgi( """ Evaluate models using TGI as backend. """ - import yaml from lighteval.logging.evaluation_tracker import EvaluationTracker from lighteval.models.endpoints.tgi_model import TGIModelConfig @@ -332,14 +331,8 @@ def tgi( # TODO (nathan): better handling of model_args parallelism_manager = ParallelismManager.TGI - with open(model_config_path, "r") as f: - config = yaml.safe_load(f)["model"] - model_config = TGIModelConfig( - inference_server_address=config["instance"]["inference_server_address"], - inference_server_auth=config["instance"]["inference_server_auth"], - model_id=config["instance"]["model_id"], - ) + model_config = TGIModelConfig.from_path(model_config_path) pipeline_params = PipelineParameters( launcher_type=parallelism_manager, diff --git a/src/lighteval/models/endpoints/endpoint_model.py b/src/lighteval/models/endpoints/endpoint_model.py index 0bd6cbbc3..1344e2485 100644 --- a/src/lighteval/models/endpoints/endpoint_model.py +++ b/src/lighteval/models/endpoints/endpoint_model.py @@ -111,6 +111,14 @@ def __post_init__(self): @classmethod def from_path(cls, path: str) -> "InferenceEndpointModelConfig": + """Load configuration for inference endpoint model from YAML file path. + + Args: + path (`str`): Path of the model configuration YAML file. + + Returns: + [`InferenceEndpointModelConfig`]: Configuration for inference endpoint model. + """ import yaml with open(path, "r") as f: diff --git a/src/lighteval/models/endpoints/tgi_model.py b/src/lighteval/models/endpoints/tgi_model.py index d95609a50..3f20e4a57 100644 --- a/src/lighteval/models/endpoints/tgi_model.py +++ b/src/lighteval/models/endpoints/tgi_model.py @@ -51,6 +51,22 @@ class TGIModelConfig: inference_server_auth: str model_id: str + @classmethod + def from_path(cls, path: str) -> "TGIModelConfig": + """Load configuration for TGI endpoint model from YAML file path. + + Args: + path (`str`): Path of the model configuration YAML file. + + Returns: + [`TGIModelConfig`]: Configuration for TGI endpoint model. + """ + import yaml + + with open(path, "r") as f: + config = yaml.safe_load(f)["model"] + return cls(**config["instance"]) + # inherit from InferenceEndpointModel instead of LightevalModel since they both use the same interface, and only overwrite # the client functions, since they use a different client. diff --git a/tests/models/test_endpoint_model.py b/tests/models/endpoints/test_endpoint_model.py similarity index 100% rename from tests/models/test_endpoint_model.py rename to tests/models/endpoints/test_endpoint_model.py diff --git a/tests/models/endpoints/test_tgi_model.py b/tests/models/endpoints/test_tgi_model.py new file mode 100644 index 000000000..305034278 --- /dev/null +++ b/tests/models/endpoints/test_tgi_model.py @@ -0,0 +1,42 @@ +# MIT License + +# Copyright (c) 2024 The HuggingFace Team + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from dataclasses import asdict + +import pytest + +from lighteval.models.endpoints.tgi_model import TGIModelConfig + + +class TestTGIModelConfig: + @pytest.mark.parametrize( + "config_path, expected_config", + [ + ( + "examples/model_configs/tgi_model.yaml", + {"inference_server_address": "", "inference_server_auth": None, "model_id": None}, + ), + ], + ) + def test_from_path(self, config_path, expected_config): + config = TGIModelConfig.from_path(config_path) + assert asdict(config) == expected_config