Fix: add --disable_group_texts

- Otherwise short samples will be concatenated together
OptimalScale · Mar 30, 2024 · 7bea9ee · 7bea9ee
1 parent 97bb791
commit 7bea9ee
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/scripts/run_finetune.sh b/scripts/run_finetune.sh
@@ -48,6 +48,7 @@ deepspeed ${deepspeed_args} \
     --output_dir ${output_dir} --overwrite_output_dir \
     --num_train_epochs 0.01 \
     --learning_rate 2e-5 \
+    --disable_group_texts 1 \
     --block_size 512 \
     --per_device_train_batch_size 1 \
     --deepspeed configs/ds_config_zero3.json \

diff --git a/scripts/run_finetune_with_lisa.sh b/scripts/run_finetune_with_lisa.sh
@@ -85,6 +85,7 @@ deepspeed ${deepspeed_args} \
     --output_dir ${output_dir} --overwrite_output_dir \
     --num_train_epochs 1 \
     --learning_rate 2e-5 \
+    --disable_group_texts 1 \
     --block_size ${block_size} \
     --per_device_train_batch_size 1 \
     --deepspeed ${ds_config_file} \