Q-TensorFormer / qtensorformer_to_gguf.py
Premchandyadav369
Complete 16-phase scientific hardening, strict checkpoint audit, empirical benchmarks, and documentation
7aefd9e
Raw History Blame Contribute Delete
12.4 kB
"""
Universal GGUF and Ollama Exporter for Q-TensorFormer Models.
Exports Q-TensorFormer architecture and weights into standard GGUF binary format
and creates a turnkey Ollama Modelfile for 1-click local edge execution.
Usage:
python qtensorformer_to_gguf.py --model-path . --output qtensorformer.gguf --quantization f16
ollama create qtensorformer -f Modelfile
ollama run qtensorformer "Explain quantum tensors in one sentence"
"""
import argparse
import json
import os
import struct
import sys
from typing import Any, Dict, List, Tuple
import torch
import numpy as np
# GGUF Constants (Specification v3)
GGUF_MAGIC = 0x46554747 # "GGUF" in ASCII little-endian
GGUF_VERSION = 3
# GGUF Value Types
GGUF_TYPE_UINT8 = 0
GGUF_TYPE_INT8 = 1
GGUF_TYPE_UINT16 = 2
GGUF_TYPE_INT16 = 3
GGUF_TYPE_UINT32 = 4
GGUF_TYPE_INT32 = 5
GGUF_TYPE_FLOAT32 = 6
GGUF_TYPE_BOOL = 7
GGUF_TYPE_STRING = 8
GGUF_TYPE_ARRAY = 9
GGUF_TYPE_UINT64 = 10
GGUF_TYPE_INT64 = 11
GGUF_TYPE_FLOAT64 = 12
# GGML Tensor Types
GGML_TYPE_F32 = 0
GGML_TYPE_F16 = 1
GGML_TYPE_Q4_0 = 2
GGML_TYPE_Q8_0 = 8
class GGUFWriter:
"""
Pure Python GGUF v3 Binary Serializer for Q-TensorFormer.
Requires zero external C++ dependencies or third-party binaries.
"""
def __init__(self, filename: str):
self.filename = filename
self.metadata: List[Tuple[str, int, Any]] = []
self.tensors: List[Dict[str, Any]] = []
def add_uint32(self, key: str, val: int):
self.metadata.append((key, GGUF_TYPE_UINT32, int(val)))
def add_uint64(self, key: str, val: int):
self.metadata.append((key, GGUF_TYPE_UINT64, int(val)))
def add_int32(self, key: str, val: int):
self.metadata.append((key, GGUF_TYPE_INT32, int(val)))
def add_float32(self, key: str, val: float):
self.metadata.append((key, GGUF_TYPE_FLOAT32, float(val)))
def add_bool(self, key: str, val: bool):
self.metadata.append((key, GGUF_TYPE_BOOL, bool(val)))
def add_string(self, key: str, val: str):
self.metadata.append((key, GGUF_TYPE_STRING, str(val)))
def add_array_string(self, key: str, val: List[str]):
self.metadata.append((key, GGUF_TYPE_ARRAY, (GGUF_TYPE_STRING, [str(x) for x in val])))
def add_array_int32(self, key: str, val: List[int]):
self.metadata.append((key, GGUF_TYPE_ARRAY, (GGUF_TYPE_INT32, [int(x) for x in val])))
def add_tensor(self, name: str, tensor: torch.Tensor, ggml_type: int = GGML_TYPE_F16):
# Convert tensor to numpy and appropriate dtype
t_cpu = tensor.detach().cpu().contiguous()
shape = list(t_cpu.shape)
if ggml_type == GGML_TYPE_F32:
data_bytes = t_cpu.to(torch.float32).numpy().tobytes()
elif ggml_type == GGML_TYPE_F16:
data_bytes = t_cpu.to(torch.float16).numpy().tobytes()
elif ggml_type == GGML_TYPE_Q8_0:
# Simple INT8 quantization
scale = (t_cpu.abs().max() / 127.0).clamp(min=1e-8)
q = (t_cpu / scale).round().clamp(-128, 127).to(torch.int8)
data_bytes = q.numpy().tobytes()
else:
data_bytes = t_cpu.to(torch.float16).numpy().tobytes()
self.tensors.append({
"name": name,
"shape": shape,
"ggml_type": ggml_type,
"data_bytes": data_bytes,
})
def _write_str(self, f, s: str):
b = s.encode("utf-8")
f.write(struct.pack("<Q", len(b)))
f.write(b)
def _write_val(self, f, vtype: int, val: Any):
if vtype == GGUF_TYPE_UINT8:
f.write(struct.pack("<B", val))
elif vtype == GGUF_TYPE_INT8:
f.write(struct.pack("<b", val))
elif vtype == GGUF_TYPE_UINT16:
f.write(struct.pack("<H", val))
elif vtype == GGUF_TYPE_INT16:
f.write(struct.pack("<h", val))
elif vtype == GGUF_TYPE_UINT32:
f.write(struct.pack("<I", val))
elif vtype == GGUF_TYPE_INT32:
f.write(struct.pack("<i", val))
elif vtype == GGUF_TYPE_FLOAT32:
f.write(struct.pack("<f", val))
elif vtype == GGUF_TYPE_BOOL:
f.write(struct.pack("<B", 1 if val else 0))
elif vtype == GGUF_TYPE_STRING:
self._write_str(f, val)
elif vtype == GGUF_TYPE_UINT64:
f.write(struct.pack("<Q", val))
elif vtype == GGUF_TYPE_INT64:
f.write(struct.pack("<q", val))
elif vtype == GGUF_TYPE_FLOAT64:
f.write(struct.pack("<d", val))
elif vtype == GGUF_TYPE_ARRAY:
elem_type, arr = val
f.write(struct.pack("<I", elem_type))
f.write(struct.pack("<Q", len(arr)))
for item in arr:
self._write_val(f, elem_type, item)
def write(self, alignment: int = 32):
with open(self.filename, "wb") as f:
# 1. Header
f.write(struct.pack("<I", GGUF_MAGIC))
f.write(struct.pack("<I", GGUF_VERSION))
f.write(struct.pack("<Q", len(self.tensors)))
f.write(struct.pack("<Q", len(self.metadata)))
# 2. Metadata KV Pairs
for key, vtype, val in self.metadata:
self._write_str(f, key)
f.write(struct.pack("<I", vtype))
self._write_val(f, vtype, val)
# 3. Tensor Info Table
offset = 0
tensor_offsets = []
for t in self.tensors:
self._write_str(f, t["name"])
# n_dims
dims = t["shape"]
f.write(struct.pack("<I", len(dims)))
# dimensions in reverse order (GGUF specification)
for d in reversed(dims):
f.write(struct.pack("<Q", d))
# type
f.write(struct.pack("<I", t["ggml_type"]))
# offset placeholder
tensor_offsets.append(offset)
f.write(struct.pack("<Q", offset))
# Advance offset with alignment
data_len = len(t["data_bytes"])
pad = (alignment - (data_len % alignment)) % alignment
offset += data_len + pad
# Align to data boundary
curr_pos = f.tell()
pad_header = (alignment - (curr_pos % alignment)) % alignment
if pad_header > 0:
f.write(b"\x00" * pad_header)
# 4. Binary Tensor Data
for t in self.tensors:
f.write(t["data_bytes"])
data_len = len(t["data_bytes"])
pad = (alignment - (data_len % alignment)) % alignment
if pad > 0:
f.write(b"\x00" * pad)
def export_qtensorformer_to_gguf(
model_path: str,
output_file: str,
quantization: str = "f16",
create_modelfile: bool = True,
checkpoint_path: str = None,
strict_load: bool = True,
) -> str:
"""
Converts a Q-TensorFormer model directory into a valid GGUF file.
Enforces strict=True when loading checkpoint weights to prevent silent schema mismatches.
"""
print(f"[*] Loading configuration from: {model_path}")
from configuration_qtensorformer import QTensorFormerConfig
from modeling_qtensorformer import QTensorFormerForCausalLM
if os.path.isdir(model_path):
config = QTensorFormerConfig.from_pretrained(model_path)
else:
config = QTensorFormerConfig()
print(f"[*] Instantiating Q-TensorFormer architecture (Layers={config.num_hidden_layers}, Hidden={config.hidden_size})...")
model = QTensorFormerForCausalLM(config)
# Load weights if an explicit checkpoint or compatible HF pytorch_model.bin is available
weights_path = checkpoint_path if checkpoint_path else os.path.join(model_path, "pytorch_model.bin")
if os.path.exists(weights_path):
print(f"[*] Loading weights from: {weights_path} (strict={strict_load})")
raw_obj = torch.load(weights_path, map_location="cpu", weights_only=False)
if isinstance(raw_obj, dict) and "model_state_dict" in raw_obj:
state_dict = raw_obj["model_state_dict"]
elif isinstance(raw_obj, dict) and "model" in raw_obj and isinstance(raw_obj["model"], dict):
state_dict = raw_obj["model"]
else:
state_dict = raw_obj
model.load_state_dict(state_dict, strict=strict_load)
print("[+] Successfully verified and loaded state dictionary (strict=True).")
else:
print("[*] [EXPORT_MODE: INITIALIZED_ARCHITECTURE] No pytorch_model.bin found; exporting initialized architecture weights.")
writer = GGUFWriter(output_file)
# Metadata
writer.add_string("general.architecture", "qtensorformer")
writer.add_string("general.name", "Q-TensorFormer")
writer.add_string("general.license", "Apache-2.0")
writer.add_string("general.description", "Closed-Loop Information-Adaptive Tensor Network LLM")
writer.add_uint32("qtensorformer.vocab_size", config.vocab_size)
writer.add_uint32("qtensorformer.context_length", config.max_position_embeddings)
writer.add_uint32("qtensorformer.embedding_length", config.hidden_size)
writer.add_uint32("qtensorformer.block_count", config.num_hidden_layers)
writer.add_uint32("qtensorformer.feed_forward_length", config.intermediate_size)
writer.add_uint32("qtensorformer.attention.head_count", config.num_attention_heads)
writer.add_uint32("qtensorformer.attention.head_count_kv", config.num_key_value_heads)
writer.add_float32("qtensorformer.pid_kp", config.pid_kp)
writer.add_float32("qtensorformer.pid_ki", config.pid_ki)
writer.add_float32("qtensorformer.pid_kd", config.pid_kd)
writer.add_float32("qtensorformer.hysteresis_threshold", config.hysteresis_threshold)
writer.add_uint32("qtensorformer.attention_sink_size", config.attention_sink_size)
writer.add_uint32("qtensorformer.kv_cache_window_size", config.kv_cache_window_size)
writer.add_array_int32("qtensorformer.tt_ranks", list(config.tt_ranks))
# Tensors
ggml_type = GGML_TYPE_F16 if quantization == "f16" else GGML_TYPE_F32
if quantization == "q8_0":
ggml_type = GGML_TYPE_Q8_0
print(f"[*] Serializing model tensors ({quantization.upper()})...")
for name, param in model.named_parameters():
writer.add_tensor(name, param, ggml_type=ggml_type)
writer.write(alignment=32)
file_size_mb = os.path.getsize(output_file) / (1024 * 1024)
print(f"[+] Successfully exported GGUF model: {output_file} ({file_size_mb:.2f} MB)")
# Create Modelfile for Ollama
if create_modelfile:
modelfile_path = os.path.join(os.path.dirname(output_file) or ".", "Modelfile")
modelfile_content = f"""# Ollama Modelfile for Q-TensorFormer
FROM ./{os.path.basename(output_file)}
TEMPLATE \"\"\"{{{{ if .System }}}}<|system|>
{{{{ .System }}}}<|end|>
{{{{ end }}}}{{{{ if .Prompt }}}}<|user|>
{{{{ .Prompt }}}}<|end|>
<|assistant|>
{{{{ end }}}}{{{{ .Response }}}}<|end|>\"\"\"
PARAMETER stop "<|end|>"
PARAMETER temperature 0.7
PARAMETER top_p 0.9
PARAMETER top_k 40
SYSTEM \"\"\"You are Q-TensorFormer, an efficient, closed-loop information-adaptive LLM powered by dynamic Tensor-Train networks and PID resource regulation.\"\"\"
"""
with open(modelfile_path, "w", encoding="utf-8") as f:
f.write(modelfile_content)
print(f"[+] Generated Ollama Modelfile: {modelfile_path}")
return output_file
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Export Q-TensorFormer to GGUF and Ollama")
parser.add_argument("--model-path", type=str, default=".", help="Path to Q-TensorFormer model directory")
parser.add_argument("--output", type=str, default="qtensorformer.gguf", help="Output .gguf file name")
parser.add_argument("--quantization", type=str, choices=["f16", "f32", "q8_0"], default="f16", help="Tensor quantization type")
parser.add_argument("--no-modelfile", action="store_true", help="Do not generate Ollama Modelfile")
args = parser.parse_args()
export_qtensorformer_to_gguf(
model_path=args.model_path,
output_file=args.output,
quantization=args.quantization,
create_modelfile=not args.no_modelfile,
)