| # Serve scientific_granite with vLLM (OpenAI-compatible /v1/embeddings). | |
| # | |
| # docker compose -f compose.vllm.yml up -d # needs an NVIDIA GPU + NVIDIA Container Toolkit | |
| # | |
| # Then: | |
| # curl localhost:8000/v1/embeddings -H 'Content-Type: application/json' \ | |
| # -d '{"model": "daniel-gomm/scientific_granite", "input": ["your query"]}' | |
| # | |
| # vLLM returns unnormalized embeddings for this model: L2-normalize before dot-product | |
| # similarity (cosine is unaffected). GPU memory: ~0.5 GB; the model needs no KV cache, so a | |
| # small --gpu-memory-utilization leaves the rest of the GPU free. | |
| name: scientific-granite-vllm | |
| services: | |
| vllm: | |
| build: | |
| # orjson makes vLLM's /v1/embeddings JSON encoding much faster (vLLM warns without it) | |
| dockerfile_inline: | | |
| FROM vllm/vllm-openai:${VLLM_VERSION:-v0.31.0} | |
| RUN pip install --no-cache-dir orjson | |
| image: scientific-granite-vllm:${VLLM_VERSION:-v0.31.0} | |
| entrypoint: ["vllm", "serve"] | |
| command: >- | |
| ${MODEL_ID:-daniel-gomm/scientific_granite} | |
| --runner pooling | |
| --pooler-config '{"pooling_type": "CLS"}' | |
| --max-model-len 2048 | |
| --gpu-memory-utilization ${GPU_MEM_UTIL:-0.25} | |
| --disable-uvicorn-access-log | |
| ports: | |
| - "${PORT:-8000}:8000" | |
| ipc: host | |
| volumes: | |
| - hf-cache:/root/.cache/huggingface # downloaded model files | |
| - vllm-cache:/root/.cache/vllm # torch.compile artifacts: faster restarts | |
| environment: | |
| HF_TOKEN: ${HF_TOKEN:-} | |
| healthcheck: | |
| test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"] | |
| interval: 10s | |
| start_period: 120s | |
| restart: unless-stopped | |
| deploy: | |
| resources: | |
| reservations: | |
| devices: | |
| - driver: nvidia | |
| count: 1 | |
| capabilities: [gpu] | |
| volumes: | |
| hf-cache: | |
| vllm-cache: | |