#!/usr/bin/env python3
"""
Comprehensive BD8 vs BD4 Error Analysis Script.

This script analyzes error patterns in short answer QA tasks, comparing
BD8 (block diffusion size 8) with BD4 (block diffusion size 4).

It generates detailed analysis reports including:
- Error pattern classification and statistics
- Character-level error analysis
- Truncation pattern detection
- Yes/No flip analysis
- Numerical error analysis
- Root cause hypotheses

Usage:
    python llm_analyze_errors.py --input_file <path> --output_file <path>
    python llm_analyze_errors.py  # Uses default paths
"""

import argparse
import json
import re
from pathlib import Path
from typing import List, Dict, Tuple, Optional
from collections import Counter, defaultdict
from dataclasses import dataclass, field
import csv


# Benchmark evaluation rules and characteristics
BENCHMARK_INFO = {
    'ChartQA': {
        'eval_rule': 'Relaxed Accuracy: |pred - gt| / gt < 0.05 (5% tolerance for numerical answers)',
        'answer_type': 'numerical/categorical',
        'description': 'Chart understanding and reasoning',
    },
    'TextVQA': {
        'eval_rule': 'VQA Accuracy: min(1, #matching_annotations / 3)',
        'answer_type': 'short text (OCR-based)',
        'description': 'Text recognition in images',
    },
    'GQA': {
        'eval_rule': 'Exact Match (case insensitive)',
        'answer_type': 'single word/phrase',
        'description': 'Visual reasoning and scene understanding',
    },
    'DocVQA': {
        'eval_rule': 'ANLS: Average Normalized Levenshtein Similarity',
        'answer_type': 'text extraction',
        'description': 'Document understanding',
    },
}


# Question type patterns for classification
QUESTION_TYPE_PATTERNS = {
    'yes_no': [r'^(is|are|do|does|did|was|were|can|could|will|would|has|have|had)\b'],
    'what': [r'^what\b'],
    'which': [r'^which\b'],
    'who': [r'^who\b'],
    'where': [r'^where\b'],
    'when': [r'^when\b'],
    'how_many': [r'^how many\b', r'^how much\b'],
    'how': [r'^how\b'],
    'why': [r'^why\b'],
}


@dataclass
class ErrorCase:
    """Represents a single error case."""
    index: str
    benchmark: str
    question: str
    ground_truth: List[str]
    bd4_prediction: str
    bd8_prediction: str
    bd4_score: float
    bd8_score: float
    error_pattern: str
    image_path: str


@dataclass
class BenchmarkAnalysis:
    """Analysis results for a single benchmark."""
    name: str
    total_errors: int
    error_patterns: Dict[str, int]
    truncated_cases: List[ErrorCase] = field(default_factory=list)
    char_error_cases: List[ErrorCase] = field(default_factory=list)
    yesno_flip_cases: List[ErrorCase] = field(default_factory=list)
    numerical_error_cases: List[ErrorCase] = field(default_factory=list)
    severe_truncation_cases: List[ErrorCase] = field(default_factory=list)
    # New: more detailed sub-classifications
    ocr_error_cases: List[ErrorCase] = field(default_factory=list)
    semantic_error_cases: List[ErrorCase] = field(default_factory=list)
    entity_confusion_cases: List[ErrorCase] = field(default_factory=list)
    question_type_stats: Dict[str, int] = field(default_factory=dict)
    

class ErrorAnalyzer:
    """Comprehensive error analyzer for BD8 vs BD4 comparison."""
    
    def __init__(self, errors: List[Dict]):
        """Initialize with error cases from JSON."""
        self.errors = [self._dict_to_case(e) for e in errors]
        self.by_benchmark: Dict[str, List[ErrorCase]] = defaultdict(list)
        
        for e in self.errors:
            self.by_benchmark[e.benchmark].append(e)
    
    def _dict_to_case(self, d: Dict) -> ErrorCase:
        """Convert dict to ErrorCase dataclass."""
        gt = d.get('ground_truth', [])
        if isinstance(gt, str):
            gt = [gt]
        return ErrorCase(
            index=str(d.get('index', '')),
            benchmark=d.get('benchmark', ''),
            question=d.get('question', ''),
            ground_truth=gt,
            bd4_prediction=str(d.get('bd4_prediction', '')),
            bd8_prediction=str(d.get('bd8_prediction', '')),
            bd4_score=float(d.get('bd4_score', 0)),
            bd8_score=float(d.get('bd8_score', 0)),
            error_pattern=d.get('error_pattern', ''),
            image_path=d.get('image_path', ''),
        )
    
    def is_truncated(self, case: ErrorCase, threshold: float = 0.7) -> bool:
        """Check if BD8 answer is truncated compared to BD4.
        
        Excludes:
        - Yes/No flips (these are semantic errors, not truncation)
        - Very short answers where length comparison is meaningless
        - Cases where BD8 is a prefix of BD4 (true truncation)
        """
        bd4 = case.bd4_prediction.strip()
        bd8 = case.bd8_prediction.strip()
        bd4_len = len(bd4)
        bd8_len = len(bd8)
        
        if bd4_len == 0:
            return False
        
        # Exclude yes/no flips - these are semantic errors, not truncation
        if self.is_yesno_flip(case):
            return False
        
        # For very short answers (≤4 chars), require BD8 to be a prefix of BD4
        # to count as truncation (e.g., "Coca" is prefix of "Coca-Cola")
        if bd4_len <= 4:
            return bd8.lower() in bd4.lower() and bd8_len < bd4_len
        
        # For longer answers, use length ratio but also check for prefix relationship
        if bd8_len < bd4_len * threshold:
            # True truncation: BD8 should share significant prefix with BD4
            # or BD8 should be contained in BD4
            bd4_lower = bd4.lower()
            bd8_lower = bd8.lower()
            
            # Check if BD8 is a prefix or substring of BD4
            if bd8_lower in bd4_lower:
                return True
            
            # Check if they share a common prefix (at least 50% of BD8)
            common_prefix_len = 0
            for i, (c1, c2) in enumerate(zip(bd4_lower, bd8_lower)):
                if c1 == c2:
                    common_prefix_len = i + 1
                else:
                    break
            
            if common_prefix_len >= bd8_len * 0.5:
                return True
        
        return False
    
    def is_severe_truncation(self, case: ErrorCase) -> bool:
        """Check for severe truncation (< 3 chars or single char)."""
        bd8 = case.bd8_prediction.strip()
        return len(bd8) <= 2 and len(case.bd4_prediction.strip()) > 3
    
    def is_char_level_error(self, case: ErrorCase) -> bool:
        """Check if error is at character level (similar but not exact)."""
        bd4 = case.bd4_prediction.lower().strip()
        bd8 = case.bd8_prediction.lower().strip()
        
        if len(bd4) == 0 or len(bd8) == 0:
            return False
        
        # Check character overlap
        min_len = min(len(bd4), len(bd8))
        if min_len < 3:
            return False
            
        matches = sum(1 for a, b in zip(bd4, bd8) if a == b)
        # More than 50% match but not exact
        return 0.5 <= matches / min_len < 1.0
    
    def is_yesno_flip(self, case: ErrorCase) -> bool:
        """Check if BD8 flipped a yes/no answer."""
        bd4 = case.bd4_prediction.lower().strip()
        bd8 = case.bd8_prediction.lower().strip()
        
        yes_words = {'yes', 'true', 'correct', 'right'}
        no_words = {'no', 'false', 'incorrect', 'wrong'}
        
        bd4_is_yes = bd4 in yes_words
        bd4_is_no = bd4 in no_words
        bd8_is_yes = bd8 in yes_words
        bd8_is_no = bd8 in no_words
        
        return (bd4_is_yes and bd8_is_no) or (bd4_is_no and bd8_is_yes)
    
    def is_numerical_error(self, case: ErrorCase) -> bool:
        """Check if error involves numerical values."""
        bd4 = case.bd4_prediction.strip()
        bd8 = case.bd8_prediction.strip()
        
        # Extract numbers
        bd4_nums = re.findall(r'-?\d+\.?\d*', bd4)
        bd8_nums = re.findall(r'-?\d+\.?\d*', bd8)
        
        if not bd4_nums or not bd8_nums:
            return False
        
        # Check if numbers are different
        return bd4_nums != bd8_nums
    
    def calculate_edit_distance(self, s1: str, s2: str) -> int:
        """Calculate Levenshtein edit distance between two strings."""
        if len(s1) < len(s2):
            return self.calculate_edit_distance(s2, s1)
        
        if len(s2) == 0:
            return len(s1)
        
        prev_row = range(len(s2) + 1)
        for i, c1 in enumerate(s1):
            curr_row = [i + 1]
            for j, c2 in enumerate(s2):
                insertions = prev_row[j + 1] + 1
                deletions = curr_row[j] + 1
                substitutions = prev_row[j] + (c1 != c2)
                curr_row.append(min(insertions, deletions, substitutions))
            prev_row = curr_row
        
        return prev_row[-1]
    
    def get_extra_content_cases(self, cases: List[ErrorCase], threshold: float = 1.5) -> List[ErrorCase]:
        """Find cases where BD8 generates significantly more content."""
        result = []
        for case in cases:
            bd4_len = len(case.bd4_prediction.strip())
            bd8_len = len(case.bd8_prediction.strip())
            if bd4_len > 0 and bd8_len > bd4_len * threshold:
                result.append(case)
        return result
    
    def classify_question_type(self, question: str) -> str:
        """Classify question into type based on question words."""
        question_lower = question.lower().strip()
        
        for q_type, patterns in QUESTION_TYPE_PATTERNS.items():
            for pattern in patterns:
                if re.match(pattern, question_lower):
                    return q_type
        return 'other'
    
    def is_ocr_error(self, case: ErrorCase) -> bool:
        """Check if error is likely an OCR error (character-level mistakes in text recognition).
        
        OCR errors are characterized by:
        - Similar string length
        - High character overlap but not exact
        - Common OCR confusions (0/O, 1/l, etc.)
        """
        bd4 = case.bd4_prediction.strip()
        bd8 = case.bd8_prediction.strip()
        
        if len(bd4) < 2 or len(bd8) < 2:
            return False
        
        # Length should be similar (within 30%)
        len_ratio = len(bd8) / len(bd4) if len(bd4) > 0 else 0
        if not (0.7 <= len_ratio <= 1.3):
            return False
        
        # Check for character-level similarity
        bd4_lower = bd4.lower()
        bd8_lower = bd8.lower()
        
        # Calculate character overlap
        min_len = min(len(bd4_lower), len(bd8_lower))
        matches = sum(1 for a, b in zip(bd4_lower, bd8_lower) if a == b)
        
        # OCR error: high overlap (>50%) but not perfect
        if 0.5 <= matches / min_len < 1.0:
            return True
        
        # Check for common OCR confusions
        ocr_confusions = [
            ('0', 'o'), ('1', 'l'), ('1', 'i'), ('5', 's'),
            ('8', 'b'), ('6', 'g'), ('2', 'z'), ('9', 'g'),
        ]
        for c1, c2 in ocr_confusions:
            if c1 in bd4_lower and c2 in bd8_lower:
                return True
            if c2 in bd4_lower and c1 in bd8_lower:
                return True
        
        return False
    
    def is_semantic_error(self, case: ErrorCase) -> bool:
        """Check if error is a semantic/understanding error.
        
        Semantic errors are characterized by:
        - Completely different answer
        - No character-level similarity
        - Not a yes/no flip
        """
        if self.is_yesno_flip(case):
            return False
        
        if self.is_ocr_error(case):
            return False
        
        if self.is_truncated(case):
            return False
        
        bd4 = case.bd4_prediction.lower().strip()
        bd8 = case.bd8_prediction.lower().strip()
        
        # If answers are very different, it's a semantic error
        if len(bd4) > 0 and len(bd8) > 0:
            # No overlap at all
            if bd4 not in bd8 and bd8 not in bd4:
                # Check edit distance
                edit_dist = self.calculate_edit_distance(bd4, bd8)
                max_len = max(len(bd4), len(bd8))
                # High edit distance relative to length
                if edit_dist / max_len > 0.5:
                    return True
        
        return False
    
    def is_entity_confusion(self, case: ErrorCase) -> bool:
        """Check if error is entity confusion (wrong object/entity identified).
        
        Common in GQA where model identifies wrong object.
        """
        bd4 = case.bd4_prediction.lower().strip()
        bd8 = case.bd8_prediction.lower().strip()
        
        # Both should be non-trivial answers
        if len(bd4) < 2 or len(bd8) < 2:
            return False
        
        # Exclude yes/no answers
        if bd4 in {'yes', 'no', 'true', 'false'}:
            return False
        
        # Different answers with similar length (both are entity names)
        if bd4 != bd8:
            len_ratio = len(bd8) / len(bd4) if len(bd4) > 0 else 0
            # Similar length suggests both are entity names
            if 0.5 <= len_ratio <= 2.0:
                # No overlap suggests different entities
                if bd4 not in bd8 and bd8 not in bd4:
                    return True
        
        return False
    
    def get_question_type_stats(self, cases: List[ErrorCase]) -> Dict[str, int]:
        """Get statistics of question types in error cases."""
        stats = Counter()
        for case in cases:
            q_type = self.classify_question_type(case.question)
            stats[q_type] += 1
        return dict(stats)
    
    def analyze_benchmark(self, benchmark: str) -> BenchmarkAnalysis:
        """Perform detailed analysis for a single benchmark."""
        cases = self.by_benchmark.get(benchmark, [])
        
        if not cases:
            return BenchmarkAnalysis(
                name=benchmark,
                total_errors=0,
                error_patterns={},
            )
        
        # Count error patterns
        patterns = Counter(c.error_pattern for c in cases)
        
        # Classify cases - basic categories
        truncated = [c for c in cases if self.is_truncated(c)]
        char_errors = [c for c in cases if self.is_char_level_error(c)]
        yesno_flips = [c for c in cases if self.is_yesno_flip(c)]
        numerical_errors = [c for c in cases if self.is_numerical_error(c)]
        severe_truncations = [c for c in cases if self.is_severe_truncation(c)]
        
        # New: more detailed sub-classifications
        ocr_errors = [c for c in cases if self.is_ocr_error(c)]
        semantic_errors = [c for c in cases if self.is_semantic_error(c)]
        entity_confusions = [c for c in cases if self.is_entity_confusion(c)]
        question_type_stats = self.get_question_type_stats(cases)
        
        return BenchmarkAnalysis(
            name=benchmark,
            total_errors=len(cases),
            error_patterns=dict(patterns),
            truncated_cases=truncated,
            char_error_cases=char_errors,
            yesno_flip_cases=yesno_flips,
            numerical_error_cases=numerical_errors,
            severe_truncation_cases=severe_truncations,
            ocr_error_cases=ocr_errors,
            semantic_error_cases=semantic_errors,
            entity_confusion_cases=entity_confusions,
            question_type_stats=question_type_stats,
        )
    
    def analyze_all(self) -> Dict[str, BenchmarkAnalysis]:
        """Analyze all benchmarks."""
        return {b: self.analyze_benchmark(b) for b in self.by_benchmark}


class ReportGenerator:
    """Generates markdown reports from analysis results."""
    
    def __init__(self, analyzer: ErrorAnalyzer, benchmark_analyses: Dict[str, BenchmarkAnalysis]):
        self.analyzer = analyzer
        self.analyses = benchmark_analyses
        self.errors = analyzer.errors
    
    def _format_case_example(self, case: ErrorCase, include_image: bool = False) -> str:
        """Format a single case as markdown."""
        gt_display = case.ground_truth[:2] if len(case.ground_truth) > 2 else case.ground_truth
        lines = [
            f"  - **Question**: {case.question}",
            f"  - **Ground Truth**: {gt_display}",
            f"  - **BD4**: `{case.bd4_prediction}` (score: {case.bd4_score:.2f})",
            f"  - **BD8**: `{case.bd8_prediction}` (score: {case.bd8_score:.2f})",
        ]
        if include_image and case.image_path:
            lines.append(f"  - **Image**: `{case.image_path}`")
        return "\n".join(lines)
    
    def generate_executive_summary(self) -> str:
        """Generate executive summary section."""
        total_errors = len(self.errors)
        
        # Count overall patterns
        all_patterns = Counter(e.error_pattern for e in self.errors)
        
        # Count key error types across all cases
        truncated_count = sum(len(a.truncated_cases) for a in self.analyses.values())
        yesno_count = sum(len(a.yesno_flip_cases) for a in self.analyses.values())
        char_error_count = sum(len(a.char_error_cases) for a in self.analyses.values())
        numerical_count = sum(len(a.numerical_error_cases) for a in self.analyses.values())
        
        lines = [
            "# BD8 Short Answer QA Error Analysis",
            "",
            "## Executive Summary",
            "",
            "BD8 (block diffusion size 8) shows consistent underperformance compared to BD4 (block diffusion size 4) on short answer QA benchmarks.",
            f"Analysis of **{total_errors} error cases** where BD4 succeeded but BD8 failed reveals systematic error patterns.",
            "",
            "**Key Findings:**",
            f"1. **{all_patterns.get('completely_wrong', 0)} completely wrong answers** ({all_patterns.get('completely_wrong', 0)/total_errors*100:.1f}%) - not just minor variations",
            f"2. **{all_patterns.get('partial_match', 0)} partial matches** ({all_patterns.get('partial_match', 0)/total_errors*100:.1f}%) - often truncated or incomplete",
            f"3. **{yesno_count} Yes/No flips** - BD8 gives opposite binary answers",
            f"4. **{char_error_count} character-level errors** - especially in OCR tasks",
            f"5. **{truncated_count} truncation cases** ({truncated_count/total_errors*100:.1f}%) - BD8 answers shorter than 70% of BD4",
            "",
        ]
        return "\n".join(lines)
    
    def generate_pattern_distribution(self) -> str:
        """Generate overall error pattern distribution section."""
        all_patterns = Counter(e.error_pattern for e in self.errors)
        total = len(self.errors)
        
        lines = [
            "## Overall Error Pattern Distribution",
            "",
            "| Error Pattern | Count | Percentage | Description |",
            "|--------------|-------|------------|-------------|",
        ]
        
        descriptions = {
            'completely_wrong': 'Answer is semantically incorrect',
            'partial_match': 'Answer is truncated or incomplete',
            'too_long': 'Answer has extra content',
            'too_short': 'Answer is severely truncated',
            'extra_words': 'Answer has unnecessary additions',
            'format_issue': 'Formatting differences only',
        }
        
        for pattern, count in sorted(all_patterns.items(), key=lambda x: -x[1]):
            pct = count / total * 100
            desc = descriptions.get(pattern, '')
            lines.append(f"| {pattern} | {count} | {pct:.1f}% | {desc} |")
        
        lines.append("")
        return "\n".join(lines)
    
    def generate_benchmark_section(self, analysis: BenchmarkAnalysis, all_cases: List[ErrorCase]) -> str:
        """Generate detailed section for one benchmark."""
        lines = [
            f"### {analysis.name}",
            "",
        ]
        
        # Add benchmark evaluation rule info
        if analysis.name in BENCHMARK_INFO:
            info = BENCHMARK_INFO[analysis.name]
            lines.append(f"**Description**: {info['description']}")
            lines.append(f"**Answer Type**: {info['answer_type']}")
            lines.append(f"**Evaluation Rule**: {info['eval_rule']}")
            lines.append("")
        
        lines.append(f"**Total BD8-only errors: {analysis.total_errors} cases**")
        lines.append("")
        
        # Error pattern table
        lines.append("| Error Type | Count | Percentage |")
        lines.append("|-----------|-------|------------|")
        for pattern, count in sorted(analysis.error_patterns.items(), key=lambda x: -x[1]):
            pct = count / analysis.total_errors * 100 if analysis.total_errors > 0 else 0
            lines.append(f"| {pattern} | {count} | {pct:.1f}% |")
        lines.append("")
        
        # Detailed statistics
        lines.append("**Detailed Error Statistics:**")
        lines.append(f"- Truncated answers (<70% of BD4 length): {len(analysis.truncated_cases)}")
        lines.append(f"- Severe truncation (≤2 chars): {len(analysis.severe_truncation_cases)}")
        lines.append(f"- Character-level errors: {len(analysis.char_error_cases)}")
        lines.append(f"- Yes/No flips: {len(analysis.yesno_flip_cases)}")
        lines.append(f"- Numerical errors: {len(analysis.numerical_error_cases)}")
        lines.append("")
        
        # New: Sub-classification statistics
        lines.append("**Error Sub-Classification:**")
        lines.append(f"- OCR errors (character recognition): {len(analysis.ocr_error_cases)}")
        lines.append(f"- Semantic errors (understanding): {len(analysis.semantic_error_cases)}")
        lines.append(f"- Entity confusion (wrong object): {len(analysis.entity_confusion_cases)}")
        lines.append("")
        
        # Question type breakdown
        if analysis.question_type_stats:
            lines.append("**Error Distribution by Question Type:**")
            sorted_types = sorted(analysis.question_type_stats.items(), key=lambda x: -x[1])
            for q_type, count in sorted_types:
                pct = count / analysis.total_errors * 100 if analysis.total_errors > 0 else 0
                lines.append(f"- {q_type}: {count} ({pct:.1f}%)")
            lines.append("")
        
        # Sample errors by category
        lines.append("**Sample Error Cases:**")
        lines.append("")
        
        # Truncation examples
        if analysis.truncated_cases:
            lines.append("*Truncation Examples:*")
            for case in analysis.truncated_cases[:3]:
                lines.append(f"- BD4: `{case.bd4_prediction}` → BD8: `{case.bd8_prediction}`")
            lines.append("")
        
        # Character error examples
        if analysis.char_error_cases:
            lines.append("*Character-Level Error Examples:*")
            for case in analysis.char_error_cases[:3]:
                lines.append(f"- BD4: `{case.bd4_prediction}` → BD8: `{case.bd8_prediction}`")
            lines.append("")
        
        # Yes/No flip examples
        if analysis.yesno_flip_cases:
            lines.append("*Yes/No Flip Examples:*")
            for case in analysis.yesno_flip_cases[:3]:
                lines.append(f"- Q: \"{case.question[:60]}...\" BD4: `{case.bd4_prediction}` → BD8: `{case.bd8_prediction}`")
            lines.append("")
        
        # Numerical error examples
        if analysis.numerical_error_cases:
            lines.append("*Numerical Error Examples:*")
            for case in analysis.numerical_error_cases[:3]:
                lines.append(f"- BD4: `{case.bd4_prediction}` → BD8: `{case.bd8_prediction}`")
            lines.append("")
        
        # OCR error examples
        if analysis.ocr_error_cases:
            lines.append("*OCR Error Examples (character recognition):*")
            for case in analysis.ocr_error_cases[:3]:
                lines.append(f"- BD4: `{case.bd4_prediction}` → BD8: `{case.bd8_prediction}`")
            lines.append("")
        
        # Semantic error examples
        if analysis.semantic_error_cases:
            lines.append("*Semantic Error Examples (understanding):*")
            for case in analysis.semantic_error_cases[:3]:
                lines.append(f"- Q: \"{case.question[:50]}...\" BD4: `{case.bd4_prediction}` → BD8: `{case.bd8_prediction}`")
            lines.append("")
        
        # Entity confusion examples
        if analysis.entity_confusion_cases:
            lines.append("*Entity Confusion Examples (wrong object):*")
            for case in analysis.entity_confusion_cases[:3]:
                lines.append(f"- BD4: `{case.bd4_prediction}` → BD8: `{case.bd8_prediction}`")
            lines.append("")
        
        # Full examples with context
        lines.append("**Detailed Error Cases (with full context):**")
        lines.append("")
        
        for i, case in enumerate(all_cases[:5]):
            lines.append(f"*Case {i+1} (Pattern: {case.error_pattern}):*")
            lines.append(self._format_case_example(case, include_image=True))
            lines.append("")
        
        return "\n".join(lines)
    
    def generate_cross_benchmark_analysis(self) -> str:
        """Generate cross-benchmark comparative analysis."""
        lines = [
            "## Cross-Benchmark Analysis",
            "",
            "### Error Pattern Comparison",
            "",
            "| Benchmark | Completely Wrong | Partial Match | Truncated | Yes/No Flips | Char Errors |",
            "|-----------|-----------------|---------------|-----------|--------------|-------------|",
        ]
        
        for name, analysis in self.analyses.items():
            cw = analysis.error_patterns.get('completely_wrong', 0)
            pm = analysis.error_patterns.get('partial_match', 0)
            tr = len(analysis.truncated_cases)
            yn = len(analysis.yesno_flip_cases)
            ce = len(analysis.char_error_cases)
            lines.append(f"| {name} | {cw} | {pm} | {tr} | {yn} | {ce} |")
        
        lines.append("")
        
        # New: Sub-classification comparison table
        lines.append("### Error Sub-Classification Comparison")
        lines.append("")
        lines.append("| Benchmark | OCR Errors | Semantic Errors | Entity Confusion | Numerical Errors |")
        lines.append("|-----------|------------|-----------------|------------------|------------------|")
        
        for name, analysis in self.analyses.items():
            ocr = len(analysis.ocr_error_cases)
            sem = len(analysis.semantic_error_cases)
            ent = len(analysis.entity_confusion_cases)
            num = len(analysis.numerical_error_cases)
            lines.append(f"| {name} | {ocr} | {sem} | {ent} | {num} |")
        
        lines.append("")
        
        # Analysis by error type across benchmarks
        lines.append("### Observations")
        lines.append("")
        
        # Find which benchmark has most of each error type
        if self.analyses:
            max_truncation = max(self.analyses.items(), key=lambda x: len(x[1].truncated_cases))
            max_yesno = max(self.analyses.items(), key=lambda x: len(x[1].yesno_flip_cases))
            max_char = max(self.analyses.items(), key=lambda x: len(x[1].char_error_cases))
            max_numerical = max(self.analyses.items(), key=lambda x: len(x[1].numerical_error_cases))
            max_ocr = max(self.analyses.items(), key=lambda x: len(x[1].ocr_error_cases))
            max_semantic = max(self.analyses.items(), key=lambda x: len(x[1].semantic_error_cases))
            
            lines.append(f"- **Highest truncation rate**: {max_truncation[0]} ({len(max_truncation[1].truncated_cases)} cases)")
            lines.append(f"- **Most Yes/No flips**: {max_yesno[0]} ({len(max_yesno[1].yesno_flip_cases)} cases)")
            lines.append(f"- **Most character errors**: {max_char[0]} ({len(max_char[1].char_error_cases)} cases)")
            lines.append(f"- **Most numerical errors**: {max_numerical[0]} ({len(max_numerical[1].numerical_error_cases)} cases)")
            lines.append(f"- **Most OCR errors**: {max_ocr[0]} ({len(max_ocr[1].ocr_error_cases)} cases)")
            lines.append(f"- **Most semantic errors**: {max_semantic[0]} ({len(max_semantic[1].semantic_error_cases)} cases)")
        lines.append("")
        
        return "\n".join(lines)
    
    def generate_length_analysis(self) -> str:
        """Analyze answer length patterns."""
        lines = [
            "## Answer Length Analysis",
            "",
        ]
        
        # Calculate length statistics
        bd4_lengths = [len(e.bd4_prediction.strip()) for e in self.errors]
        bd8_lengths = [len(e.bd8_prediction.strip()) for e in self.errors]
        
        avg_bd4 = sum(bd4_lengths) / len(bd4_lengths) if bd4_lengths else 0
        avg_bd8 = sum(bd8_lengths) / len(bd8_lengths) if bd8_lengths else 0
        
        lines.append(f"- Average BD4 answer length: {avg_bd4:.1f} characters")
        lines.append(f"- Average BD8 answer length: {avg_bd8:.1f} characters")
        lines.append(f"- BD8 is on average {(avg_bd8/avg_bd4 - 1)*100:+.1f}% different in length" if avg_bd4 > 0 else "")
        lines.append("")
        
        # Length difference distribution
        shorter = sum(1 for b4, b8 in zip(bd4_lengths, bd8_lengths) if b8 < b4 * 0.9)
        same = sum(1 for b4, b8 in zip(bd4_lengths, bd8_lengths) if 0.9 <= b8/b4 <= 1.1 if b4 > 0)
        longer = sum(1 for b4, b8 in zip(bd4_lengths, bd8_lengths) if b8 > b4 * 1.1)
        
        lines.append("**Length Difference Distribution:**")
        lines.append(f"- BD8 shorter (>10%): {shorter} ({shorter/len(self.errors)*100:.1f}%)")
        lines.append(f"- Similar length (±10%): {same} ({same/len(self.errors)*100:.1f}%)")
        lines.append(f"- BD8 longer (>10%): {longer} ({longer/len(self.errors)*100:.1f}%)")
        lines.append("")
        
        # Severe cases
        very_short = sum(1 for b8 in bd8_lengths if b8 <= 2)
        very_long = sum(1 for b4, b8 in zip(bd4_lengths, bd8_lengths) if b8 > b4 * 2)
        
        lines.append("**Extreme Cases:**")
        lines.append(f"- BD8 ≤ 2 characters: {very_short} cases")
        lines.append(f"- BD8 > 2x BD4 length: {very_long} cases")
        lines.append("")
        
        return "\n".join(lines)
    
    def generate_hypothesis_section(self) -> str:
        """Generate root cause hypothesis section."""
        lines = [
            "## Root Cause Hypotheses",
            "",
            "Based on the error patterns observed, we propose the following hypotheses for BD8's underperformance:",
            "",
            "### 1. Block Boundary Artifacts",
            "",
            "When generating text in blocks of 8 tokens, answers shorter than 8 tokens may experience:",
            "- Premature truncation at block boundaries",
            "- Padding artifacts affecting answer quality",
            "- Difficulty precisely ending at the optimal token",
            "",
            "**Evidence:**",
        ]
        
        # Calculate truncation evidence
        total_truncated = sum(len(a.truncated_cases) for a in self.analyses.values())
        truncation_pct = total_truncated / len(self.errors) * 100 if self.errors else 0
        lines.append(f"- {truncation_pct:.1f}% of errors show truncation (BD8 answer <70% of BD4 length)")
        
        # Find truncation examples
        all_truncated = []
        for a in self.analyses.values():
            all_truncated.extend(a.truncated_cases[:2])
        if all_truncated:
            lines.append("- Truncation examples:")
            for case in all_truncated[:3]:
                lines.append(f"  - `{case.bd4_prediction}` → `{case.bd8_prediction}`")
        lines.append("")
        
        lines.append("### 2. Character-Level Precision Loss")
        lines.append("")
        lines.append("Generating 8 tokens simultaneously reduces fine-grained control over:")
        lines.append("- Individual character sequences (OCR)")
        lines.append("- Exact numerical values")
        lines.append("- Proper nouns and codes")
        lines.append("")
        lines.append("**Evidence:**")
        
        # Find character error examples
        all_char_errors = []
        for a in self.analyses.values():
            all_char_errors.extend(a.char_error_cases[:2])
        total_char_errors = sum(len(a.char_error_cases) for a in self.analyses.values())
        lines.append(f"- {total_char_errors} character-level error cases detected")
        if all_char_errors:
            lines.append("- Character error examples:")
            for case in all_char_errors[:4]:
                lines.append(f"  - `{case.bd4_prediction}` → `{case.bd8_prediction}`")
        lines.append("")
        
        lines.append("### 3. Semantic Uncertainty Propagation")
        lines.append("")
        lines.append("With larger blocks, an error in early tokens can propagate through the entire block:")
        lines.append("- If the model is uncertain about token 1, tokens 2-8 may all be affected")
        lines.append("- This explains the high \"completely_wrong\" rate")
        lines.append("- Binary yes/no questions are particularly vulnerable")
        lines.append("")
        lines.append("**Evidence:**")
        
        cw_count = sum(a.error_patterns.get('completely_wrong', 0) for a in self.analyses.values())
        cw_pct = cw_count / len(self.errors) * 100 if self.errors else 0
        total_yesno = sum(len(a.yesno_flip_cases) for a in self.analyses.values())
        lines.append(f"- {cw_pct:.1f}% of errors are \"completely_wrong\" (not partial)")
        lines.append(f"- {total_yesno} yes/no answer flips detected")
        lines.append("")
        
        lines.append("### 4. Short Answer Sensitivity")
        lines.append("")
        lines.append("Block diffusion may be less suited for short, precise answers:")
        lines.append("- 8-token blocks are larger than typical short answers (1-3 tokens)")
        lines.append("- The model must \"waste\" capacity on padding or continuation")
        lines.append("- Short answers have less context for error recovery")
        lines.append("")
        
        return "\n".join(lines)
    
    def generate_recommendations(self) -> str:
        """Generate recommendations section."""
        lines = [
            "## Recommendations",
            "",
            "Based on this analysis, we suggest the following improvements:",
            "",
            "### 1. Block Size Tuning for Task Type",
            "- Consider using smaller block sizes (BD4 or even BD2) for short-answer QA tasks",
            "- Larger block sizes may be acceptable for longer generation tasks",
            "",
            "### 2. Adaptive Block Sizing",
            "- Dynamically adjust block size based on expected answer length",
            "- Use smaller blocks for OCR-heavy benchmarks (TextVQA)",
            "",
            "### 3. Block Boundary Training",
            "- Add training signals for clean block boundaries",
            "- Ensure short answers are properly handled in training data",
            "",
            "### 4. Answer Length Regularization",
            "- Add length-aware loss terms during training",
            "- Penalize severe truncation patterns",
            "",
            "### 5. Post-Processing (if applicable)",
            "- Consider answer validation/correction for character-level tasks",
            "- Truncation detection and completion",
            "",
        ]
        return "\n".join(lines)
    
    def generate_file_list(self) -> str:
        """Generate list of output files."""
        lines = [
            "## Generated Files",
            "",
            "- `bd8_short_answer_analysis.md` - Full error case report",
            "- `llm_analysis.md` - This comprehensive analysis",
            "- `error_cases/TextVQA_errors.json` - TextVQA error details",
            "- `error_cases/DocVQA_errors.json` - DocVQA error details",
            "- `error_cases/ChartQA_errors.json` - ChartQA error details",
            "- `error_cases/GQA_errors.json` - GQA error details",
            "- `error_cases/combined_errors_for_llm.json` - Combined errors for analysis",
            "",
        ]
        return "\n".join(lines)
    
    def generate_full_report(self) -> str:
        """Generate the complete analysis report."""
        sections = [
            self.generate_executive_summary(),
            self.generate_pattern_distribution(),
            "## Detailed Analysis by Benchmark\n",
        ]
        
        # Add each benchmark section
        for benchmark in sorted(self.analyses.keys()):
            analysis = self.analyses[benchmark]
            cases = self.analyzer.by_benchmark.get(benchmark, [])
            sections.append(self.generate_benchmark_section(analysis, cases))
        
        # Add cross-benchmark and other sections
        sections.extend([
            self.generate_cross_benchmark_analysis(),
            self.generate_length_analysis(),
            self.generate_hypothesis_section(),
            self.generate_recommendations(),
            self.generate_file_list(),
        ])
        
        return "\n".join(sections)
    
    def export_csv(self, output_path: Path):
        """Export all error cases to CSV for further analysis."""
        csv_file = output_path / "error_cases.csv"
        
        with open(csv_file, 'w', newline='', encoding='utf-8') as f:
            writer = csv.writer(f)
            
            # Header
            writer.writerow([
                'benchmark', 'index', 'question', 'question_type', 
                'ground_truth', 'bd4_prediction', 'bd8_prediction',
                'bd4_score', 'bd8_score', 'error_pattern',
                'is_truncated', 'is_yesno_flip', 'is_numerical_error',
                'is_ocr_error', 'is_semantic_error', 'is_entity_confusion',
                'bd4_length', 'bd8_length', 'image_path'
            ])
            
            # Data rows
            for case in self.errors:
                q_type = self.analyzer.classify_question_type(case.question)
                gt_str = '|'.join(case.ground_truth[:3]) if case.ground_truth else ''
                
                writer.writerow([
                    case.benchmark,
                    case.index,
                    case.question,
                    q_type,
                    gt_str,
                    case.bd4_prediction,
                    case.bd8_prediction,
                    f"{case.bd4_score:.2f}",
                    f"{case.bd8_score:.2f}",
                    case.error_pattern,
                    self.analyzer.is_truncated(case),
                    self.analyzer.is_yesno_flip(case),
                    self.analyzer.is_numerical_error(case),
                    self.analyzer.is_ocr_error(case),
                    self.analyzer.is_semantic_error(case),
                    self.analyzer.is_entity_confusion(case),
                    len(case.bd4_prediction),
                    len(case.bd8_prediction),
                    case.image_path,
                ])
        
        print(f"CSV exported to: {csv_file}")
        return csv_file
    
    def export_summary_csv(self, output_path: Path):
        """Export benchmark summary statistics to CSV."""
        csv_file = output_path / "benchmark_summary.csv"
        
        with open(csv_file, 'w', newline='', encoding='utf-8') as f:
            writer = csv.writer(f)
            
            # Header
            writer.writerow([
                'benchmark', 'total_errors', 'completely_wrong', 'partial_match',
                'truncated', 'yesno_flips', 'char_errors', 'numerical_errors',
                'ocr_errors', 'semantic_errors', 'entity_confusion'
            ])
            
            # Data rows
            for name, analysis in self.analyses.items():
                writer.writerow([
                    name,
                    analysis.total_errors,
                    analysis.error_patterns.get('completely_wrong', 0),
                    analysis.error_patterns.get('partial_match', 0),
                    len(analysis.truncated_cases),
                    len(analysis.yesno_flip_cases),
                    len(analysis.char_error_cases),
                    len(analysis.numerical_error_cases),
                    len(analysis.ocr_error_cases),
                    len(analysis.semantic_error_cases),
                    len(analysis.entity_confusion_cases),
                ])
        
        print(f"Summary CSV exported to: {csv_file}")
        return csv_file


def main():
    parser = argparse.ArgumentParser(description="Comprehensive BD8 vs BD4 error analysis")
    parser.add_argument(
        "--input_file",
        type=str,
        default="chengyue_script/dvlm/evaluation/analysis_results/error_cases/combined_errors_for_llm.json",
        help="Path to combined error cases JSON"
    )
    parser.add_argument(
        "--output_file",
        type=str,
        default="chengyue_script/dvlm/evaluation/analysis_results/llm_analysis.md",
        help="Output file for analysis report"
    )
    parser.add_argument(
        "--also_load_full_errors",
        action="store_true",
        help="Also load full error files for each benchmark"
    )
    
    args = parser.parse_args()
    
    # Load error cases
    print(f"Loading error cases from: {args.input_file}")
    with open(args.input_file) as f:
        errors = json.load(f)
    
    print(f"Loaded {len(errors)} error cases")
    
    # Optionally load full error files
    if args.also_load_full_errors:
        error_dir = Path(args.input_file).parent
        for json_file in error_dir.glob("*_errors.json"):
            if "combined" in json_file.name:
                continue
            print(f"Loading additional errors from: {json_file}")
            with open(json_file) as f:
                data = json.load(f)
                if 'cases' in data:
                    benchmark_name = data.get('benchmark', '')
                    for case in data['cases']:
                        if not case.get('benchmark'):
                            case['benchmark'] = benchmark_name
                        errors.append(case)
        print(f"Total errors after loading all files: {len(errors)}")
    
    # Create analyzer
    print("Analyzing errors...")
    analyzer = ErrorAnalyzer(errors)
    
    # Analyze each benchmark
    analyses = analyzer.analyze_all()
    
    for name, analysis in analyses.items():
        print(f"  {name}: {analysis.total_errors} errors, "
              f"{len(analysis.truncated_cases)} truncated, "
              f"{len(analysis.yesno_flip_cases)} yes/no flips")
    
    # Generate report
    print("Generating report...")
    generator = ReportGenerator(analyzer, analyses)
    report = generator.generate_full_report()
    
    # Save report
    output_path = Path(args.output_file)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    
    with open(output_path, 'w') as f:
        f.write(report)
    
    print(f"Report saved to: {output_path}")
    
    # Export CSV files
    print("Exporting CSV files...")
    generator.export_csv(output_path.parent)
    generator.export_summary_csv(output_path.parent)
    
    print("Analysis complete!")


if __name__ == "__main__":
    main()
