catplusplus's picture
Add files using upload-large-folder tool
5f20907 verified
Raw History Blame Contribute Delete
2.5 kB
#!/bin/bash
# 🌸 run_train_lora_2token.sh - High-Capacity MoE LoRA (r=32) 2-Token MTP Drafter Finetuner
# Channels Nikola's Sparkly Village Charm for Haga's Seeker Quest! ✨🚀
set -eo pipefail
BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "$BASE_DIR"
mkdir -p "$BASE_DIR/ailogs"
mkdir -p "$BASE_DIR/models/trained/Qwen3.8-Flash-Next-MTP-LoRA-r32-2Token"
# Thor unified memory and tokenizer configurations
export PYTORCH_CUDA_ALLOC_CONF="expandable_segments:True,max_split_size_mb:128"
export TOKENIZERS_PARALLELISM="false"
export PYTHONUNBUFFERED="1"
export CUDA_VISIBLE_DEVICES="0"
LOG_FILE="$BASE_DIR/ailogs/mtp_lora_r32_2token_training.log"
PYTHON_BIN="/home/olegk/venv/vllm/bin/python"
echo "================================================================================"
echo "🌸✨ [Nikola] Launching High-Capacity MoE LoRA (r=32) 2-Token Drafter Training ✨🌸"
echo " • Base Model: $BASE_DIR/models/mazinb/Qwen3.8-Flash-Next-Uncensored-NVFP4"
echo " • Staged Data: $BASE_DIR/datasets/mtp_staged_layer48 (~1TB Diverse BF16 Shards)"
echo " • Output Dir: $BASE_DIR/models/trained/Qwen3.8-Flash-Next-MTP-LoRA-r32-2Token"
echo " • Log File: $LOG_FILE"
echo " • Configuration: MoE LoRA (r=32, alpha=64, 187.2M trainable params)"
echo " • Objective: 2-Token Speculative Prediction (Horizon=2, gamma=0.90, BPTT)"
echo " • Learning Rate: Peak 4.5e-5 -> Min 1.0e-5 (Cosine Warmup 600 steps)"
echo " • Early Stopping: Halt if no validation loss improvement for 2 full epochs"
echo " • Hardware Profile: Thor sm_110, Batch=1024 (2,048 tok/step)"
echo "================================================================================"
exec "$PYTHON_BIN" "$BASE_DIR/src/mtp/train_mtp.py" \
--model-dir "$BASE_DIR/models/mazinb/Qwen3.8-Flash-Next-Uncensored-NVFP4" \
--staged-dir "$BASE_DIR/datasets/mtp_staged_layer48" \
--output-dir "$BASE_DIR/models/trained/Qwen3.8-Flash-Next-MTP-LoRA-r32-2Token" \
--lora \
--lora-r 32 \
--lora-alpha 64.0 \
--freeze-gate \
--batch-size 1024 \
--grad-accum 1 \
--horizon 2 \
--discount-gamma 0.90 \
--backprop-mode bptt \
--epochs 3 \
--lr 4.5e-5 \
--min-lr 1e-5 \
--warmup-steps 600 \
--early-stopping-epochs 2 \
--save-all-eval-checkpoints \
--eval-every-steps 500 \
--save-every-steps 500 \
--unglitch-every-steps 50 \
--log-every-steps 10 \
--num-workers 0 \
--resume \
"$@" 2>&1 | tee -a "$LOG_FILE"