From 13a867d1667bdd08c8edd0017155fa4cc9dd71b6 Mon Sep 17 00:00:00 2001
From: Xiang LIU <47224289+Dominic789654@users.noreply.github.com>
Date: Tue, 7 Jan 2025 01:52:38 +0800
Subject: [PATCH] update lora_target_modules in training scripts (#928)

---
 run_finetune_with_lora.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/run_finetune_with_lora.sh b/run_finetune_with_lora.sh
index 03210d2f..4f6c4148 100755
--- a/run_finetune_with_lora.sh
+++ b/run_finetune_with_lora.sh
@@ -2,7 +2,7 @@
 # Please run this script under ${project_id} in project directory of
 
 # Parses arguments
-model_name_or_path=gpt2
+model_name_or_path=meta-llama/Llama-2-7b-hf
 dataset_path=data/wikitext-2-raw-v1/test
 # conversation_template=llama2
 output_dir=output_models/finetune
@@ -59,6 +59,7 @@ deepspeed ${deepspeed_args} \
     --per_device_train_batch_size 1 \
     --use_lora 1 \
     --lora_r 8 \
+    --lora_target_modules="embed_tokens,q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj,lm_head" \
     --save_aggregated_lora 0\
     --deepspeed configs/ds_config_zero2.json \
     --fp16 \