#!/bin/bash
#SBATCH -A nvr_elm_llm
#SBATCH -p batch_block1,batch_block3,batch_block4
#SBATCH -t 4:00:00
#SBATCH -N 1
#SBATCH -J eval_Qwen2.5-VL-7B-Baseline
#SBATCH --gpus-per-node 8
#SBATCH --cpus-per-task 16
#SBATCH --mem=512G
#SBATCH -o eval_outputs/qwen25vl_7b_baseline/slurm_out.log
#SBATCH -e eval_outputs/qwen25vl_7b_baseline/slurm_err.log

# ==============================================================================
# User prompt: 分别7B qwen mdm/ar的实验，然后生成一个eval 原版7Bqwen的eval，然后两个都要用对的auto
# This script: Qwen2.5-VL-7B-Instruct Baseline Evaluation
# Evaluates the official HuggingFace model on all VLM benchmarks
# ==============================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
cd "${PROJECT_ROOT}"

source "${PROJECT_ROOT}/scripts/cluster_env.sh"
lmflow_activate_conda

# VLMEvalKit directory (prefer third_party)
DEFAULT_VLM_DIR=""
if [ -d "${PROJECT_ROOT}/third_party/VLMEvalKit" ]; then
    DEFAULT_VLM_DIR="${PROJECT_ROOT}/third_party/VLMEvalKit"
elif [ -d "${PROJECT_ROOT}/VLMEvalKit_local" ]; then
    DEFAULT_VLM_DIR="${PROJECT_ROOT}/VLMEvalKit_local"
else
    DEFAULT_VLM_DIR="${PROJECT_ROOT}/VLMEvalKit_local"
fi
VLM_DIR=${VLM_DIR:-${DEFAULT_VLM_DIR}}

# Create output directory
OUTPUT_BASE="${VLM_DIR}/output_models/Qwen2.5-VL-7B-Instruct"
mkdir -p "${OUTPUT_BASE}"

echo "=============================================="
echo "Qwen2.5-VL-7B-Instruct Baseline Evaluation"
echo "=============================================="
echo "Date: $(date)"
echo "Output: ${OUTPUT_BASE}"
echo ""

cd "${VLM_DIR}"

# All benchmarks to evaluate
BENCHMARKS=(
    "SEEDBench2_Plus"
    "AI2D_TEST"
    "RealWorldQA"
    "DocVQA_VAL"
    "ChartQA_TEST"
    "MathVista_MINI"
    "MMMU_DEV_VAL"
    "MMBench_DEV_EN_V11"
    "GQA_TestDev_Balanced"
    "TextVQA_VAL"
    "POPE"
)

echo "Benchmarks to evaluate: ${BENCHMARKS[*]}"
echo ""

# Run each benchmark sequentially
for benchmark in "${BENCHMARKS[@]}"; do
    echo "=========================================="
    echo "Evaluating: ${benchmark}"
    echo "=========================================="
    
    # Check if result already exists
    result_pattern="${OUTPUT_BASE}/${benchmark}/*_acc.csv"
    if ls ${result_pattern} 1> /dev/null 2>&1; then
        echo "✓ ${benchmark} already evaluated, skipping..."
        continue
    fi
    
    # Run evaluation with built-in Qwen2.5-VL-7B-Instruct
    python run.py \
        --data "${benchmark}" \
        --model Qwen2.5-VL-7B-Instruct \
        --work-dir "${OUTPUT_BASE}" \
        --verbose \
        2>&1 | tee -a "${OUTPUT_BASE}/${benchmark}_eval.log"
    
    echo "✓ ${benchmark} evaluation complete"
    echo ""
done

# Generate summary
cd "${PROJECT_ROOT}"
echo ""
echo "=========================================="
echo "Generating checkpoint_results.md..."
echo "=========================================="

python3 chengyue_script/dvlm/evaluation/generate_checkpoint_results.py "${OUTPUT_BASE}" 2>&1 || echo "Summary generation failed (might need manual run)"

echo ""
echo "=============================================="
echo "Qwen2.5-VL-7B Baseline Evaluation Complete!"
echo "Results: ${OUTPUT_BASE}"
echo "=============================================="

# Display results if available
if [ -f "${OUTPUT_BASE}/checkpoint_results.md" ]; then
    echo ""
    echo "=== Results Summary ==="
    cat "${OUTPUT_BASE}/checkpoint_results.md"
fi
