#!/bin/bash
# 用户提示: 汇总所有实验的评测结果
# Usage: bash chengyue_script/dvlm/evaluation/summarize_all_experiments.sh

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
cd "${PROJECT_ROOT}"

# VLMEvalKit 目录（优先 third_party，其次 VLMEvalKit_local）
DEFAULT_VLM_DIR=""
if [ -d "${PROJECT_ROOT}/third_party/VLMEvalKit" ]; then
    DEFAULT_VLM_DIR="${PROJECT_ROOT}/third_party/VLMEvalKit"
elif [ -d "${PROJECT_ROOT}/VLMEvalKit_local" ]; then
    DEFAULT_VLM_DIR="${PROJECT_ROOT}/VLMEvalKit_local"
else
    DEFAULT_VLM_DIR="${PROJECT_ROOT}/VLMEvalKit_local"
fi
VLM_DIR=${VLM_DIR:-${DEFAULT_VLM_DIR}}

echo "========================================"
echo "实验综合结果汇总 $(date)"
echo "========================================"

# 定义所有实验
EXPS=(
    "sharegpt4v_8node_bsz256_bd4_fix_eos_flexible_lr_1e-6_unfreeze_vision"
    "sharegpt4v_8node_bsz256_bd8_fix_eos_flexible_lr_1e-6_unfreeze_vision"
    "sharegpt4v_8node_bsz256_bd16_fix_eos_flexible_lr_1e-6_unfreeze_vision"
    "sharegpt4v_8node_bsz256_bd4_fix_eos_flexible_lr_5e-7_unfreeze_vision"
    "sharegpt4v_8node_bsz256_bd4_fix_eos_flexible_lr_2e-6_unfreeze_vision"
    "sharegpt4v_8node_bsz256_bd4_fix_eos_flexible_lr_5e-6_unfreeze_vision"
    "j63_mdm_8node_lr_1e-6_unfreeze_vision"
)

# 先为所有有CSV结果的实验生成checkpoint_results.md
echo ""
echo "=== 生成汇总文件 ==="
total_exps=${#EXPS[@]}
current=1
for exp in "${EXPS[@]}"; do
    vlmeval_dir="${VLM_DIR}/output_models/$exp"
    short_exp=$(echo "$exp" | cut -c1-50)
    printf "\r[%d/%d] Processing: %-50s" "$current" "$total_exps" "$short_exp"
    if [ -d "$vlmeval_dir/eval_vlmeval" ]; then
        csv_count=$(find "$vlmeval_dir" -name "*.csv" 2>/dev/null | wc -l)
        if [ "$csv_count" -gt 0 ]; then
            python3 chengyue_script/dvlm/evaluation/generate_checkpoint_results.py "$vlmeval_dir" >/dev/null 2>&1 || true
        fi
    fi
    current=$((current + 1))
done
echo ""
echo "✓ 汇总文件生成完成"

# 显示所有结果
echo ""
echo "========================================"
echo "实验状态汇总"
echo "========================================"
printf "%-70s | %6s | %6s | %s\n" "实验名" "Ckpts" "CSVs" "状态"
echo "------------------------------------------------------------------------------------------------------------"

for exp in "${EXPS[@]}"; do
    ckpts=$(ls -d output_models/$exp/checkpoint-* 2>/dev/null | wc -l)
    csvs=$(find "${VLM_DIR}/output_models/$exp" -name "*.csv" 2>/dev/null | wc -l)
    
    if [ "$csvs" -eq 0 ]; then
        status="❌ 无结果"
    elif [ "$csvs" -lt 200 ]; then
        status="🔄 进行中"
    else
        status="✅ 完成"
    fi
    
    printf "%-70s | %6d | %6d | %s\n" "$exp" "$ckpts" "$csvs" "$status"
done

# 显示各实验的最佳结果
echo ""
echo "========================================"
echo "各实验最佳结果 (最后3行)"
echo "========================================"

for exp in "${EXPS[@]}"; do
    result_file="${VLM_DIR}/output_models/$exp/checkpoint_results.md"
    if [ -f "$result_file" ]; then
        echo ""
        echo "=== $exp ==="
        # 显示header和最后3行结果
        head -3 "$result_file"
        tail -3 "$result_file"
    fi
done

# 显示正在运行的评测任务
echo ""
echo "========================================"
echo "正在运行的评测任务"
echo "========================================"
running=$(squeue -u "$(whoami)" 2>/dev/null | grep -c eval || echo "0")
pending=$(squeue -u "$(whoami)" 2>/dev/null | grep eval | grep -c PD || echo "0")
echo "运行中: $running | 排队中: $pending"

