"""MITM diagnostic: real llama_cpp.server 0.3.35 on 127.0.0.1:7861 with a tiny GGUF, raw TCP proxy on 0.0.0.0:7860 logging both directions byte-exactly to stdout. If external 400 reproduces through the proxy, the logs show whether the proxy bytes differ from our local replay and what llama returns raw. """ import socket import threading import subprocess import sys import time import urllib.request import shutil def log(*a): print("MITM:", *a, flush=True) def pipe(src: socket.socket, dst: socket.socket, tag: str, addr: str): try: n = 0 while True: data = src.recv(65536) if not data: break if n == 0: log(f"{tag} FIRST-PACKET from {addr} ({len(data)}B): {data[:1200]!r}") n += len(data) dst.sendall(data) except Exception as e: log(f"{tag} pipe err {e!r}") finally: try: dst.shutdown(socket.SHUT_WR) except Exception: pass def handle(conn: socket.socket, addr): log(f"CONN open {addr}") try: upstream = socket.create_connection(("127.0.0.1", 7861), timeout=10) t = threading.Thread(target=pipe, args=(conn, upstream, "C->S", addr), daemon=True) t.start() pipe(upstream, conn, "S->C", addr) except Exception as e: log(f"CONN {addr} err {e!r}") finally: try: conn.close() except Exception: pass log(f"CONN close {addr}") def download_tiny_model(): paths = [ ("ggml-org/models", "tinyllamas/stories260K.gguf"), ("ggml-org/models", "tinyllamas/stories15M-q4_0.gguf"), ("ggml-org/models", "tinyllamas/stories15M.gguf"), ] from huggingface_hub import hf_hub_download for repo, fn in paths: try: p = hf_hub_download(repo_id=repo, filename=fn, local_dir="/app/models") log(f"model ok: {p} ({__import__('os').path.getsize(p)} B)") return p except Exception as e: log(f"model fail {repo}/{fn}: {e}") raise RuntimeError("no tiny model") def main(): model = download_tiny_model() import llama_cpp, uvicorn, fastapi log(f"versions: llama_cpp={llama_cpp.__version__} uvicorn={uvicorn.__version__} fastapi={fastapi.__version__}") cmd = [sys.executable, "-m", "llama_cpp.server", "--model", model, "--host", "127.0.0.1", "--port", "7861", "--n_ctx", "512", "--n_gpu_layers", "0"] env = {**__import__("os").environ, "FORWARDED_ALLOW_IPS": "*"} # NOTE: llama server is NOT given FORWARDED_ALLOW_IPS here initially — proxy is on 7860; # but uvicorn inside reads env anyway: it will trust forwarded headers from 127.0.0.1 (the proxy). proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) for i in range(120): try: urllib.request.urlopen("http://127.0.0.1:7861/v1/models", timeout=3) log(f"llama server ready after {i+1}s") break except Exception: time.sleep(1) else: log("llama server NOT ready; exiting") sys.exit(1) srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) srv.bind(("0.0.0.0", 7860)) srv.listen(64) log("mitm proxy listening 0.0.0.0:7860 -> 127.0.0.1:7861") while True: c, a = srv.accept() threading.Thread(target=handle, args=(c, a), daemon=True).start() if __name__ == "__main__": main()