#!/bin/bash
#SBATCH -A nvr_elm_llm
#SBATCH -p batch_block1,batch_block3,batch_block4
#SBATCH -t 4:00:00
#SBATCH -N 1
#SBATCH -J eval_mdm_bd4_tps
#SBATCH --gpus-per-node 8
#SBATCH --cpus-per-task 16
#SBATCH --mem=512G
#SBATCH -o eval_outputs/mdm_bd4_best/slurm_%j.out
#SBATCH -e eval_outputs/mdm_bd4_best/slurm_%j.err

# ==============================================================================
# MDM bd4 best checkpoint evaluation (with tokens-per-step tracking)
# ==============================================================================

PROJECT_ROOT="/lustre/fs11/portfolios/nvr/projects/nvr_elm_llm/users/chengyuew/workspace/shiyi/LMFlow"
cd "${PROJECT_ROOT}"

source "${PROJECT_ROOT}/scripts/cluster_env.sh"
lmflow_activate_conda

MODEL_PATH="output_models/sharegpt4v_8node_bsz256_bd4_fix_eos_flexible_lr_1e-6_unfreeze_vision/checkpoint-2439"
OUTPUT_DIR="eval_outputs/mdm_bd4_best"
BASE_MODEL="Qwen/Qwen2.5-VL-3B-Instruct"

mkdir -p "${OUTPUT_DIR}"

echo "=============================================="
echo "MDM bd4 Best Checkpoint Evaluation"
echo "=============================================="
echo "Date: $(date)"
echo "Model: ${MODEL_PATH}"
echo "Output: ${OUTPUT_DIR}"
echo ""

python3 chengyue_script/dvlm/evaluation/eval_custom_model.py \
    --model_path "${MODEL_PATH}" \
    --base_model "${BASE_MODEL}" \
    --tasks gqa,pope,textvqa,mmmu,mmbench,mathvista \
    --num_gpus 8 \
    --output_dir "${OUTPUT_DIR}"

echo ""
echo "Results: ${OUTPUT_DIR}/results.json"

