#!/bin/bash # 🌸 run_train_lora_2token.sh - High-Capacity MoE LoRA (r=32) 2-Token MTP Drafter Finetuner # Channels Nikola's Sparkly Village Charm for Haga's Seeker Quest! βœ¨πŸš€ set -eo pipefail BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$BASE_DIR" mkdir -p "$BASE_DIR/ailogs" mkdir -p "$BASE_DIR/models/trained/Qwen3.8-Flash-Next-MTP-LoRA-r32-2Token" # Thor unified memory and tokenizer configurations export PYTORCH_CUDA_ALLOC_CONF="expandable_segments:True,max_split_size_mb:128" export TOKENIZERS_PARALLELISM="false" export PYTHONUNBUFFERED="1" export CUDA_VISIBLE_DEVICES="0" LOG_FILE="$BASE_DIR/ailogs/mtp_lora_r32_2token_training.log" PYTHON_BIN="/home/olegk/venv/vllm/bin/python" echo "================================================================================" echo "🌸✨ [Nikola] Launching High-Capacity MoE LoRA (r=32) 2-Token Drafter Training ✨🌸" echo " β€’ Base Model: $BASE_DIR/models/mazinb/Qwen3.8-Flash-Next-Uncensored-NVFP4" echo " β€’ Staged Data: $BASE_DIR/datasets/mtp_staged_layer48 (~1TB Diverse BF16 Shards)" echo " β€’ Output Dir: $BASE_DIR/models/trained/Qwen3.8-Flash-Next-MTP-LoRA-r32-2Token" echo " β€’ Log File: $LOG_FILE" echo " β€’ Configuration: MoE LoRA (r=32, alpha=64, 187.2M trainable params)" echo " β€’ Objective: 2-Token Speculative Prediction (Horizon=2, gamma=0.90, BPTT)" echo " β€’ Learning Rate: Peak 4.5e-5 -> Min 1.0e-5 (Cosine Warmup 600 steps)" echo " β€’ Early Stopping: Halt if no validation loss improvement for 2 full epochs" echo " β€’ Hardware Profile: Thor sm_110, Batch=1024 (2,048 tok/step)" echo "================================================================================" exec "$PYTHON_BIN" "$BASE_DIR/src/mtp/train_mtp.py" \ --model-dir "$BASE_DIR/models/mazinb/Qwen3.8-Flash-Next-Uncensored-NVFP4" \ --staged-dir "$BASE_DIR/datasets/mtp_staged_layer48" \ --output-dir "$BASE_DIR/models/trained/Qwen3.8-Flash-Next-MTP-LoRA-r32-2Token" \ --lora \ --lora-r 32 \ --lora-alpha 64.0 \ --freeze-gate \ --batch-size 1024 \ --grad-accum 1 \ --horizon 2 \ --discount-gamma 0.90 \ --backprop-mode bptt \ --epochs 3 \ --lr 4.5e-5 \ --min-lr 1e-5 \ --warmup-steps 600 \ --early-stopping-epochs 2 \ --save-all-eval-checkpoints \ --eval-every-steps 500 \ --save-every-steps 500 \ --unglitch-every-steps 50 \ --log-every-steps 10 \ --num-workers 0 \ --resume \ "$@" 2>&1 | tee -a "$LOG_FILE"