
# accelerate launch -m lm_eval --confirm_run_unsafe_code --model hf \
#     --model_args pretrained=/home/chengyuew/chengyuew/workspace/evalplus/Qwen2.5-1.5B-Instruct \
#     --tasks mbpp \
#     --trust_remote_code \
#     --apply_chat_template \
#     --fewshot_as_multiturn \
#     --batch_size 1 \
#     --output_path eval_results/mbpp --log_samples

# accelerate launch -m lm_eval --confirm_run_unsafe_code --model hf \
#     --model_args pretrained=/home/chengyuew/chengyuew/workspace/evalplus/Qwen3-1.7B \
#     --tasks gsm8k \
#     --num_fewshot 0 \
#     --trust_remote_code \
#     --apply_chat_template \
#     --fewshot_as_multiturn \
#     --batch_size 1 


# accelerate launch -m lm_eval --confirm_run_unsafe_code --model hf \
#     --model_args pretrained=../evalplus/Qwen3-1.7B \
#     --tasks gsm8k_cot \
#     --trust_remote_code \
#     --apply_chat_template \
#     --fewshot_as_multiturn \
#     --batch_size 1 

# accelerate launch -m lm_eval --confirm_run_unsafe_code --model hf \
#     --model_args pretrained=../evalplus/Qwen3-1.7B,enable_thinking=True,max_length=8192 \
#     --tasks gsm8k_cot \
#     --num_fewshot 4 \
#     --trust_remote_code \
#     --apply_chat_template \
#     --fewshot_as_multiturn \
#     --batch_size 8 


accelerate launch -m lm_eval --confirm_run_unsafe_code --model hf \
    --model_args pretrained=../evalplus/Qwen3-1.7B,enable_thinking=False \
    --tasks mmlu_generative \
    --trust_remote_code \
    --apply_chat_template \
    --fewshot_as_multiturn \
    --batch_size 16

accelerate launch -m lm_eval --confirm_run_unsafe_code --model hf \
    --model_args pretrained=../evalplus/Qwen3-1.7B,enable_thinking=False \
    --tasks gpqa_main_generative_n_shot \
    --trust_remote_code \
    --apply_chat_template \
    --fewshot_as_multiturn \
    --batch_size 16

accelerate launch --main_process_port 29501 -m lm_eval --confirm_run_unsafe_code --model hf \
    --model_args pretrained=/home/chengyuew/chengyuew/workspace/evalplus/Llama-3.2-1B-Instruct \
    --tasks minerva_math \
    --trust_remote_code \
    --apply_chat_template \
    --fewshot_as_multiturn \
    --batch_size 32

accelerate launch --main_process_port 29501 -m lm_eval --confirm_run_unsafe_code --model hf \
    --model_args pretrained=../evalplus/Qwen3-1.7B \
    --tasks hendrycks_math \
    --trust_remote_code \
    --apply_chat_template \
    --fewshot_as_multiturn \
    --batch_size 32

accelerate launch -m lm_eval --confirm_run_unsafe_code --model hf \
    --model_args pretrained=../evalplus/Qwen2.5-7B-Instruct \
    --tasks ifeval \
    --trust_remote_code \
    --apply_chat_template \
    --fewshot_as_multiturn \
    --batch_size 32