#!/bin/bash
#SBATCH -A nvr_elm_llm
#SBATCH -p batch_block1,batch_block3,batch_block4
#SBATCH -t 04:00:00
#SBATCH -N 1
#SBATCH --ntasks-per-node=1
#SBATCH -J eval_qwen2.5_vl_3b_baseline
#SBATCH --gpus-per-node 8
#SBATCH --cpus-per-task 64
#SBATCH --mem=512G
#SBATCH -e workdir/eval_qwen2.5_vl_3b_baseline_error.log
#SBATCH -o workdir/eval_qwen2.5_vl_3b_baseline_out.log

# ==============================================================================
# Evaluate Qwen2.5-VL-3B-Instruct baseline using VLMEvalKit
# 11 benchmarks as specified in /show-eval
# ==============================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
cd "${PROJECT_ROOT}"

source "${PROJECT_ROOT}/scripts/cluster_env.sh"
lmflow_activate_conda

DEFAULT_VLM_DIR=""
if [ -d "${PROJECT_ROOT}/third_party/VLMEvalKit" ]; then
    DEFAULT_VLM_DIR="${PROJECT_ROOT}/third_party/VLMEvalKit"
elif [ -d "${PROJECT_ROOT}/VLMEvalKit_local" ]; then
    DEFAULT_VLM_DIR="${PROJECT_ROOT}/VLMEvalKit_local"
else
    DEFAULT_VLM_DIR="${PROJECT_ROOT}/VLMEvalKit_local"
fi
VLM_DIR=${VLM_DIR:-${DEFAULT_VLM_DIR}}

export PYTHONPATH="${VLM_DIR}:${PYTHONPATH}"

# 11 benchmarks
TASKS="SEEDBench2_Plus AI2D_TEST RealWorldQA DocVQA_VAL ChartQA_TEST MathVista_MINI MMMU_DEV_VAL MMBench_DEV_EN_V11 GQA_TestDev_Balanced TextVQA_VAL POPE"

# VLMEvalKit 原生支持的模型名
MODEL_NAME="Qwen2.5-VL-3B-Instruct"
OUTPUT_DIR="${VLM_DIR}/output_models/${MODEL_NAME}"

mkdir -p "${OUTPUT_DIR}" workdir

echo "=========================================="
echo "Evaluating Qwen2.5-VL-3B-Instruct Baseline"
echo "Tasks: ${TASKS}"
echo "Output: ${OUTPUT_DIR}"
echo "=========================================="

cd "${VLM_DIR}"

# 使用 torchrun 多 GPU 并行
torchrun --nproc_per_node=8 run.py \
    --model "${MODEL_NAME}" \
    --data ${TASKS} \
    --work-dir "${OUTPUT_DIR}" \
    --verbose \
    --reuse

echo "=========================================="
echo "Evaluation complete!"
echo "Results: ${OUTPUT_DIR}"
echo "=========================================="
