#!/bin/bash
#SBATCH -A nvr_elm_llm  
#SBATCH -p batch_block1,batch_block3,batch_block4
#SBATCH -t 04:00:00
#SBATCH -N 1
#SBATCH -J eval_qwen2_5vl_full
#SBATCH --gpus-per-node 8
#SBATCH -e workdir/%x_%j.err
#SBATCH -o workdir/%x_%j.out
#SBATCH --exclusive

set -eo pipefail

echo "[driver] start at $(date), host=$(hostname)"
echo "[driver] SLURM_JOB_ID=${SLURM_JOB_ID:-}"
echo "[driver] PWD=$(pwd)"

# Set HF environment variables
export HF_ALLOW_CODE_EVAL=1
export HF_DATASETS_TRUST_REMOTE_CODE=true

# Job configuration
exp_id=eval_qwen2_5vl_full_8gpu
model_path=Qwen/Qwen2.5-VL-3B-Instruct
task=gsm8k  # Can change to: minerva_math, mbpp, humaneval, etc.
batch_size=16
save_dir=output_models/${exp_id}
log_dir=log/${exp_id}

# Ensure output/log dirs
echo "[driver] creating output and log directories..."
mkdir -p ${save_dir} ${log_dir}

# Activate conda environment
echo "[driver] activating conda environment..."
source "${PROJECT_ROOT}/scripts/cluster_env.sh"
lmflow_activate_conda

which python3 && python3 -V || true
nvidia-smi -L || true

echo "[driver] launching accelerate 8-GPU lm_eval at $(date)"
echo "[driver] Task: ${task}"
echo "[driver] Model: ${model_path}"
echo "[driver] Batch size: ${batch_size}"

set -x
accelerate launch shiyi_scripts/eval_qwen2_5_vl_3B_lm_eval.py \
    --num_fewshot 0 \
    --tasks ${task} \
    --batch_size ${batch_size} \
    --model qwen2_5_vl \
    --apply_chat_template \
    --model_args model_path=${model_path},show_speed=True,save_dir=${save_dir} \
    > >(tee ${log_dir}/eval.log) \
    2> >(tee ${log_dir}/eval.err >&2)
set +x

echo "[driver] end at $(date)"

