model_path=output_models/finetune_qwen2.5-7B-ar-node-1/checkpoint-2500
log_path=eval_results/qwen2_5_7B_ar_nemo_data.log


accelerate launch -m lm_eval --confirm_run_unsafe_code --model hf \
    --model_args pretrained=${model_path} \
    --tasks ifeval \
    --trust_remote_code \
    --apply_chat_template \
    --fewshot_as_multiturn \
    --batch_size 32 >> ${log_path} 2>&1

accelerate launch -m lm_eval --confirm_run_unsafe_code --model hf \
    --model_args pretrained=${model_path} \
    --tasks minerva_math \
    --trust_remote_code \
    --apply_chat_template \
    --fewshot_as_multiturn \
    --batch_size 32 --output_path eval_results/math_1.5B \
    --log_samples >> ${log_path} 2>&1


accelerate launch -m lm_eval --confirm_run_unsafe_code --model hf \
    --model_args pretrained=../evalplus/Qwen2.5-7B-Instruct/ \
    --tasks gsm8k \
    --trust_remote_code \
    --apply_chat_template \
    --fewshot_as_multiturn \
    --batch_size 32 >> ${log_path} 2>&1


accelerate launch -m lm_eval --confirm_run_unsafe_code --model hf \
    --model_args pretrained=${model_path} \
    --tasks mmlu \
    --num_fewshot 0 \
    --trust_remote_code \
    --apply_chat_template \
    --fewshot_as_multiturn \
    --batch_size 32 >> ${log_path} 2>&1

accelerate launch -m lm_eval --confirm_run_unsafe_code --model hf \
    --model_args pretrained=${model_path} \
    --tasks gpqa_main_n_shot \
    --num_fewshot 5 \
    --trust_remote_code \
    --apply_chat_template \
    --fewshot_as_multiturn \
    --batch_size 8 >> ${log_path} 2>&1

