#!/bin/bash
#SBATCH -A nvr_elm_llm                                                                #account
#SBATCH -p batch_block1,batch_block3,batch_block4                                                         #partition
#SBATCH -t 04:00:00                                                                   #wall time limit, hr:min:sec
#SBATCH -N 1                                                                           #number of nodes
#SBATCH -J batch_eval_qwen                                                             #job name
#SBATCH --gpus-per-node 8
#SBATCH -e workdir/error.log
#SBATCH -o workdir/out.log
#SBATCH --exclusive

# Set the environment variables first before running the command.
export HF_ALLOW_CODE_EVAL=1
export HF_DATASETS_TRUST_REMOTE_CODE=true

# 任务配置
task=minerva_math
# task=mbpp
# task=humaneval
# task=minerva_math

# 模型路径
model_path=output_models/finetune_qwen2.5-1.5B-bd-size-32-lr-1e-5-causal-loss-no-padding-no-conplemenrary-mask/checkpoint-6000

# 评估参数
batch_size=32
use_block_cache=False
block_cache_refresh_interval=8
small_block_size=8
dynamic_threshold=False
factor=1
threshold=1

# 创建必要的目录
mkdir -p workdir
mkdir -p workdir/slurm_log/minerva_math_results_ablation

echo "开始执行batch evaluation..."
echo "任务: ${task}"
echo "模型路径: ${model_path}"
echo "批次大小: ${batch_size}"
echo "使用块缓存: ${use_block_cache}"
echo "动态阈值: ${dynamic_threshold}"
echo "阈值: ${threshold}"

# 构建评估命令
eval_cmd='
export HF_ALLOW_CODE_EVAL=1;
export HF_DATASETS_TRUST_REMOTE_CODE=true;
accelerate launch chengyue_script/eval_mdm_qwen_batch.py \
    --num_fewshot 0 \
    --tasks '"${task}"' \
    --batch_size '"${batch_size}"' \
    --confirm_run_unsafe_code \
    --model dfm_qwen \
    --fewshot_as_multiturn \
    --apply_chat_template \
    --model_args model_path='"${model_path}"',show_speed=True,token_shift=True,use_block_cache='"${use_block_cache}"',block_cache_refresh_interval='"${block_cache_refresh_interval}"',small_block_size='"${small_block_size}"',dynamic_threshold='"${dynamic_threshold}"',factor='"${factor}"',threshold='"${threshold}"'
'

echo "执行评估命令..."
echo $eval_cmd

# 使用srun执行评估命令，日志路径包含关键参数信息
srun -o workdir/slurm_log/minerva_math_results_ablation/slurm_%A_%j_${task}_batch${batch_size}_block_cache_${use_block_cache}_dynamic_thresh_${dynamic_threshold}_thresh${threshold}.log -e workdir/slurm_log/minerva_math_results_ablation/error_%A_%j_${task}_batch${batch_size}_block_cache_${use_block_cache}_dynamic_thresh_${dynamic_threshold}_thresh${threshold}.log \
    bash -c "${eval_cmd}"

# # 构建评估命令
# task=gpqa_main_n_shot
# eval_cmd='
# export HF_ALLOW_CODE_EVAL=1;
# export HF_DATASETS_TRUST_REMOTE_CODE=true;
# accelerate launch chengyue_script/eval_mdm_qwen_batch.py \
#     --tasks '"${task}"' \
#     --num_fewshot 5 \
#     --batch_size 1 \
#     --confirm_run_unsafe_code \
#     --model dfm_qwen \
#     --fewshot_as_multiturn \
#     --apply_chat_template \
#     --model_args model_path='"${model_path}"',show_speed=True,token_shift=True,use_block_cache='"${use_block_cache}"',block_cache_refresh_interval='"${block_cache_refresh_interval}"',small_block_size='"${small_block_size}"',dynamic_threshold='"${dynamic_threshold}"',factor='"${factor}"',threshold='"${threshold}"'
# '

# echo "执行评估命令..."
# echo $eval_cmd

# # 使用srun执行评估命令，日志路径包含关键参数信息
# srun -o workdir/slurm_log/eval_token_shift_results/slurm_%A_%j_${task}_5_shot_batch${batch_size}_block_cache_${use_block_cache}_dynamic_thresh_${dynamic_threshold}_thresh${threshold}.log -e workdir/slurm_log/eval_token_shift_results/error_%A_%j_${task}_5_shot_batch${batch_size}_block_cache_${use_block_cache}_dynamic_thresh_${dynamic_threshold}_thresh${threshold}.log \
#     bash -c "${eval_cmd}"

# # 构建评估命令
# task=mmlu
# eval_cmd='
# export HF_ALLOW_CODE_EVAL=1;
# export HF_DATASETS_TRUST_REMOTE_CODE=true;
# accelerate launch chengyue_script/eval_mdm_qwen_batch.py \
#     --tasks '"${task}"' \
#     --batch_size 1 \
#     --num_fewshot 0 \
#     --confirm_run_unsafe_code \
#     --model dfm_qwen \
#     --fewshot_as_multiturn \
#     --apply_chat_template \
#     --model_args model_path='"${model_path}"',show_speed=True,token_shift=True,use_block_cache='"${use_block_cache}"',block_cache_refresh_interval='"${block_cache_refresh_interval}"',small_block_size='"${small_block_size}"',dynamic_threshold='"${dynamic_threshold}"',factor='"${factor}"',threshold='"${threshold}"'
# '

# echo "执行评估命令..."
# echo $eval_cmd

# # 使用srun执行评估命令，日志路径包含关键参数信息
# srun -o workdir/slurm_log/eval_token_shift_results/slurm_%A_%j_${task}_0_shot_batch${batch_size}_block_cache_${use_block_cache}_dynamic_thresh_${dynamic_threshold}_thresh${threshold}.log -e workdir/slurm_log/eval_token_shift_results/error_%A_%j_${task}_0_shot_batch${batch_size}_block_cache_${use_block_cache}_dynamic_thresh_${dynamic_threshold}_thresh${threshold}.log \
#     bash -c "${eval_cmd}"

# # 构建评估命令
# task=minerva_math
# eval_cmd='
# export HF_ALLOW_CODE_EVAL=1;
# export HF_DATASETS_TRUST_REMOTE_CODE=true;
# accelerate launch chengyue_script/eval_mdm_qwen_batch.py \
#     --tasks '"${task}"' \
#     --batch_size '"${batch_size}"' \
#     --confirm_run_unsafe_code \
#     --model dfm_qwen \
#     --fewshot_as_multiturn \
#     --apply_chat_template \
#     --model_args model_path='"${model_path}"',show_speed=True,token_shift=True,use_block_cache='"${use_block_cache}"',block_cache_refresh_interval='"${block_cache_refresh_interval}"',small_block_size='"${small_block_size}"',dynamic_threshold='"${dynamic_threshold}"',factor='"${factor}"',threshold='"${threshold}"'
# '

# echo "执行评估命令..."
# echo $eval_cmd

# # 使用srun执行评估命令，日志路径包含关键参数信息
# srun -o workdir/slurm_log/eval_token_shift_results/slurm_%A_%j_${task}_batch${batch_size}_block_cache_${use_block_cache}_dynamic_thresh_${dynamic_threshold}_thresh${threshold}.log -e workdir/slurm_log/eval_token_shift_results/error_%A_%j_${task}_batch${batch_size}_block_cache_${use_block_cache}_dynamic_thresh_${dynamic_threshold}_thresh${threshold}.log \
#     bash -c "${eval_cmd}"


# # 构建评估命令
# task=ifeval
# eval_cmd='
# export HF_ALLOW_CODE_EVAL=1;
# export HF_DATASETS_TRUST_REMOTE_CODE=true;
# accelerate launch chengyue_script/eval_mdm_qwen_batch.py \
#     --tasks '"${task}"' \
#     --batch_size '"${batch_size}"' \
#     --confirm_run_unsafe_code \
#     --model dfm_qwen \
#     --fewshot_as_multiturn \
#     --apply_chat_template \
#     --model_args model_path='"${model_path}"',show_speed=True,token_shift=True,use_block_cache='"${use_block_cache}"',block_cache_refresh_interval='"${block_cache_refresh_interval}"',small_block_size='"${small_block_size}"',dynamic_threshold='"${dynamic_threshold}"',factor='"${factor}"',threshold='"${threshold}"'
# '

# echo "执行评估命令..."
# echo $eval_cmd

# # 使用srun执行评估命令，日志路径包含关键参数信息
# srun -o workdir/slurm_log/eval_token_shift_results/slurm_%A_%j_${task}_batch${batch_size}_block_cache_${use_block_cache}_dynamic_thresh_${dynamic_threshold}_thresh${threshold}.log -e workdir/slurm_log/eval_token_shift_results/error_%A_%j_${task}_batch${batch_size}_block_cache_${use_block_cache}_dynamic_thresh_${dynamic_threshold}_thresh${threshold}.log \
#     bash -c "${eval_cmd}"


# echo "评估完成！"
