#!/bin/bash
#SBATCH -A nvr_elm_llm
#SBATCH -p cpu_short
#SBATCH -t 02:00:00
#SBATCH -N 1
#SBATCH -J preprocess_final
#SBATCH --cpus-per-task=32
#SBATCH -e workdir/preprocess_final_%j.err
#SBATCH -o workdir/preprocess_final_%j.out

set -eo pipefail
echo "[preprocess] Starting robust CPU preprocessing at $(date)"

set +u
source "${PROJECT_ROOT}/scripts/cluster_env.sh"
lmflow_activate_conda
set -u

# Ensure we are in the project root for correct imports
cd /home/chengyuew/chengyuew/workspace/shiyi/LMFlow || exit 1

echo "[preprocess] Launching python preprocessing script..."

# Define the cache directory explicitly
export CACHE_DIR=$(pwd)/.cache/lmf_preprocessed/dvlm_qwen2_5_vl_3b_no_reasoning
echo "[preprocess] Using fixed cache directory: ${CACHE_DIR}"

# These arguments MUST match the training script for the cache to be compatible
python3 scripts/run_preprocess.py \
    --model_name_or_path /home/chengyuew/chengyuew/workspace/shiyi/LMFlow/src/models/Qwen2_5VL_DFM \
    --tokenizer_name Qwen/Qwen2.5-VL-3B-Instruct \
    --trust_remote_code True \
    --dataset_path /home/chengyuew/chengyuew/workspace/LMFlow/data/nemotron_no_thinking_subset \
    --conversation_template qwen2_5_no_reasoning \
    --block_size 2048 \
    --disable_group_texts 0 \
    --preprocessing_num_workers 32 \
    --overwrite_cache True \
    --dataset_cache_dir "${CACHE_DIR}"

echo "[preprocess] Finished at $(date)"
