Upload train_continue_english_1b.yaml with huggingface_hub

train_continue_english_1b.yaml

@@ -0,0 +1,44 @@
+# LUNA 100M — English continuation pretraining
+# Uses same LR schedule as original 3B-token pretrain (lr=6e-4, cosine -> 6e-5)
+# auto_config: true => dynamic micro_batch via VRAM probe, auto workers, precision
+
+auto_config: true
+
+data_path: "Base/data/litdata_english_1b"
+out_dir: "out/pretrain/luna-100m-english-1b"
+tokenizer_dir: "Base/checkpoints/EleutherAI/pythia-160m"
+init_model_path: "Base/out/pretrain/luna_100m/final/lit_model.pth"
+
+model:
+  vocab_size: 50304
+  seq_len: 1024
+  n_layer: 10
+  n_embd: 768
+  n_head: 12
+
+train:
+  max_tokens: 420000000
+  lr_warmup_steps: 500
+  save_interval: 500
+  log_interval: 10
+  max_norm: 1.0
+
+optimizer:
+  lr: 0.0006
+  min_lr: 0.00006
+  weight_decay: 0.1
+  betas: [0.9, 0.95]
+  eps: 1.0e-8
+
+batch:
+  global_batch: 120
+  micro_batch: 12
+  grad_accum: 10
+
+dataloader:
+  num_workers: -1
+  pin_memory: true
+
+hardware:
+  precision: "bf16"
+  compile: true