Download extras/scripts/run_train_lora_2token.sh from catplusplus/Qwen3.8-Flash-Next-Uncensored-NVFP4-MTP-head: direct link, hf CLI and curl.
- Browser
- Download file 2.5 kB
-
https://huggingface.co/catplusplus/Qwen3.8-Flash-Next-Uncensored-NVFP4-MTP-head/resolve/main/extras/scripts/run_train_lora_2token.sh
- Command line
-
hf download hf://catplusplus/Qwen3.8-Flash-Next-Uncensored-NVFP4-MTP-head/extras/scripts/run_train_lora_2token.sh
-
curl -L -o run_train_lora_2token.sh https://huggingface.co/catplusplus/Qwen3.8-Flash-Next-Uncensored-NVFP4-MTP-head/resolve/main/extras/scripts/run_train_lora_2token.sh
2.5 kB
| # 🌸 run_train_lora_2token.sh - High-Capacity MoE LoRA (r=32) 2-Token MTP Drafter Finetuner | |
| # Channels Nikola's Sparkly Village Charm for Haga's Seeker Quest! ✨🚀 | |
| set -eo pipefail | |
| BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" | |
| cd "$BASE_DIR" | |
| mkdir -p "$BASE_DIR/ailogs" | |
| mkdir -p "$BASE_DIR/models/trained/Qwen3.8-Flash-Next-MTP-LoRA-r32-2Token" | |
| # Thor unified memory and tokenizer configurations | |
| export PYTORCH_CUDA_ALLOC_CONF="expandable_segments:True,max_split_size_mb:128" | |
| export TOKENIZERS_PARALLELISM="false" | |
| export PYTHONUNBUFFERED="1" | |
| export CUDA_VISIBLE_DEVICES="0" | |
| LOG_FILE="$BASE_DIR/ailogs/mtp_lora_r32_2token_training.log" | |
| PYTHON_BIN="/home/olegk/venv/vllm/bin/python" | |
| echo "================================================================================" | |
| echo "🌸✨ [Nikola] Launching High-Capacity MoE LoRA (r=32) 2-Token Drafter Training ✨🌸" | |
| echo " • Base Model: $BASE_DIR/models/mazinb/Qwen3.8-Flash-Next-Uncensored-NVFP4" | |
| echo " • Staged Data: $BASE_DIR/datasets/mtp_staged_layer48 (~1TB Diverse BF16 Shards)" | |
| echo " • Output Dir: $BASE_DIR/models/trained/Qwen3.8-Flash-Next-MTP-LoRA-r32-2Token" | |
| echo " • Log File: $LOG_FILE" | |
| echo " • Configuration: MoE LoRA (r=32, alpha=64, 187.2M trainable params)" | |
| echo " • Objective: 2-Token Speculative Prediction (Horizon=2, gamma=0.90, BPTT)" | |
| echo " • Learning Rate: Peak 4.5e-5 -> Min 1.0e-5 (Cosine Warmup 600 steps)" | |
| echo " • Early Stopping: Halt if no validation loss improvement for 2 full epochs" | |
| echo " • Hardware Profile: Thor sm_110, Batch=1024 (2,048 tok/step)" | |
| echo "================================================================================" | |
| exec "$PYTHON_BIN" "$BASE_DIR/src/mtp/train_mtp.py" \ | |
| --model-dir "$BASE_DIR/models/mazinb/Qwen3.8-Flash-Next-Uncensored-NVFP4" \ | |
| --staged-dir "$BASE_DIR/datasets/mtp_staged_layer48" \ | |
| --output-dir "$BASE_DIR/models/trained/Qwen3.8-Flash-Next-MTP-LoRA-r32-2Token" \ | |
| --lora \ | |
| --lora-r 32 \ | |
| --lora-alpha 64.0 \ | |
| --freeze-gate \ | |
| --batch-size 1024 \ | |
| --grad-accum 1 \ | |
| --horizon 2 \ | |
| --discount-gamma 0.90 \ | |
| --backprop-mode bptt \ | |
| --epochs 3 \ | |
| --lr 4.5e-5 \ | |
| --min-lr 1e-5 \ | |
| --warmup-steps 600 \ | |
| --early-stopping-epochs 2 \ | |
| --save-all-eval-checkpoints \ | |
| --eval-every-steps 500 \ | |
| --save-every-steps 500 \ | |
| --unglitch-every-steps 50 \ | |
| --log-every-steps 10 \ | |
| --num-workers 0 \ | |
| --resume \ | |
| "$@" 2>&1 | tee -a "$LOG_FILE" | |