finetune_flanv2_llama_7b.yaml

description: Finetune Llama 7b on flan v2

target:
  service: sing
  name: gcrprojvc1

# option to add key for apt: curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
# install fairseq requirements to docker image
# image is older version of pytorch because newer python version has threading error
environment:
  image: nikghosh09/transformers:qlora_v1
  registry: docker.io # any public registry can be specified here

code:
  # local directory of the code. this will be uploaded to the server.
  # $CONFIG_DIR is expanded to the directory of this config file
  local_dir: $CONFIG_DIR

jobs:
- name: flan_v2_llama_7b
  sku: G1
  command:
  - python open_instruct/prepare_subsampled_model.py
    --pretrained_model_name_or_path /mnt/default/pretrained/llama-7b
    --launcher accelerate
    --tokenizer_name huggyllama/llama-7b
    --use_fast_tokenizer False
    --output_dir $$AMLT_OUTPUT_DIR/checkpoints
    --logging_dir $$AMLT_OUTPUT_DIR/runs
    --subsamp_ratio 1.0
    --do_train
    --do_eval
    --report_to tensorboard
    --logging_strategy steps
    --logging_steps 10
    --evaluation_strategy steps
    --eval_steps 100
    --num_training_eval_samples 1000
    --num_train_epochs 2
    --save_strategy steps
    --save_steps 100
    --save_total_limit 2
    --eval_dataset_size 1000
    --per_device_train_batch_size 4
    --gradient_accumulation_steps 4
    --per_device_eval_batch_size 16
    --max_seq_length 512
    --preprocessing_num_workers 16
    --max_memory_MB 32000
    --dataloader_num_workers 3
    --use_lora
    --lora_rank 64
    --lora_alpha 16
    --lora_dropout 0.05
    --double_quant
    --quant_type nf4
    --bits 4
    --lr_scheduler_type constant
    --gradient_checkpointing
    --train_file /mnt/default/data/flan_v2_data.jsonl
    --ddp_find_unused_parameters False
    --learning_rate 0.0002
    --adam_beta2 0.999
    --max_grad_norm 0.3
    --weight_decay 0.0
- name: flan_v2_llama_7b_multi
  sku: G8
  command:
  - python open_instruct/prepare_subsampled_model.py
    --pretrained_model_name_or_path /mnt/default/pretrained/llama-7b
    --tokenizer_name huggyllama/llama-7b
    --use_fast_tokenizer False
    --output_dir $$AMLT_OUTPUT_DIR/checkpoints
    --logging_dir $$AMLT_OUTPUT_DIR/runs
    --subsamp_ratio 1.0
    --do_train
    --do_eval
    --report_to tensorboard
    --logging_strategy steps
    --logging_steps 10
    --evaluation_strategy steps
    --eval_steps 100
    --num_training_eval_samples 1000
    --num_train_epochs 2
    --save_strategy steps
    --save_steps 100
    --save_total_limit 2
    --eval_dataset_size 1000
    --per_device_train_batch_size 2
    --gradient_accumulation_steps 1
    --per_device_eval_batch_size 16
    --max_seq_length 512
    --preprocessing_num_workers 16
    --max_memory_MB 32000
    --dataloader_num_workers 3
    --use_lora
    --lora_rank 64
    --lora_alpha 16
    --lora_dropout 0.05
    --double_quant
    --quant_type nf4
    --bits 4
    --lr_scheduler_type constant
    --gradient_checkpointing
    --train_file /mnt/default/data/flan_v2_data.jsonl
    --ddp_find_unused_parameters False
    --learning_rate 0.0002
    --adam_beta2 0.999
    --max_grad_norm 0.3
    --weight_decay 0.0