# Set the environment variables first before running the command.
export HF_ALLOW_CODE_EVAL=1
export HF_DATASETS_TRUST_REMOTE_CODE=true

# task=gpqa_main_n_shot
task=humaneval
# num_fewshot=5
model_path=output_models/finetune_qwen2.5-7B-bd-size-32-lr-1e-5/checkpoint-2500

accelerate launch chengyue_script/eval_mdm_qwen_batch.py --tasks ${task} --batch_size 2 \
--confirm_run_unsafe_code --model dfm_qwen --fewshot_as_multiturn --apply_chat_template \
--model_args model_path=${model_path},show_speed=True,use_block_cache=False,small_block_size=8,threshold=1,bd_size=32


# task=humaneval

# accelerate launch chengyue_script/eval_mdm_qwen.py --tasks ${task} \
# --confirm_run_unsafe_code --model dfm_qwen --fewshot_as_multiturn --apply_chat_template \
# --model_args model_path=${model_path},show_speed=True \
# --output_path eval_results/humaneval-ns0 --log_samples

task=mbpp
# task=minerva_math
# echo "model_path: ${model_path}"
# accelerate launch --main_process_port 29501 chengyue_script/eval_mdm_qwen.py --tasks ${task} \
# --confirm_run_unsafe_code --model dfm_qwen --fewshot_as_multiturn --apply_chat_template \
# --model_args model_path=output_models/finetune_qwen_1.5B_nemotron_shizhe_no_reasoning_pad_bd_size_128_8nodes_seq_length_2k/checkpoint-10000,show_speed=True \
# --output_path eval_results/mbpp --log_samples


# accelerate launch chengyue_script/eval_dfm_qwen.py --tasks ${task} --num_fewshot ${num_fewshot} \
# --confirm_run_unsafe_code --model dfm_qwen \
# --model_args model_path=/home/chengyuew/chengyuew/workspace/pretrained_models/Qwen2.5-1.5B-Instruct,show_speed=True 
