Update weights and evaluation

MERGE_REPORT.json

@@ -1,15 +1,15 @@
 {
   "snapshot": "mistralai/Ministral-3-8B-Instruct-2512-BF16@f6fae9795746f63c9be8344932f01275f3c63734",
-  "adapter": "StandardOne-8B-LoRA (training run final2-8b-long)",
-  "adapter_sha256": "a2d9f02d5dd5d3ade3fab88145fc7321315609aa52d3847319dc4ec827f0c684",
+  "adapter": "StandardOne-8B-LoRA (training run hard25-8b-r1)",
+  "adapter_sha256": "9551923c162970ea8bc2c3a5faf3ea3762b48591c9f460064e531d09427fa41a",
   "changed_params": 238,
   "unchanged_params": 293,
   "changed_outside_lm_projections": [],
-  "max_abs_delta": 0.00201416015625,
+  "max_abs_delta": 0.001983642578125,
   "changed_sample": [
     [
       "model.language_model.layers.0.self_attn.q_proj.weight",
-      0.00140380859375
+      0.00146484375
     ],
     [
       "model.language_model.layers.0.self_attn.k_proj.weight",
@@ -17,23 +17,23 @@
     ],
     [
       "model.language_model.layers.0.self_attn.v_proj.weight",
-      0.00099945068359375
+      0.0010833740234375
     ],
     [
       "model.language_model.layers.0.self_attn.o_proj.weight",
-      0.00135040283203125
+      0.001407623291015625
     ],
     [
       "model.language_model.layers.0.mlp.gate_proj.weight",
-      0.0011854171752929688
+      0.00113677978515625
     ],
     [
       "model.language_model.layers.0.mlp.up_proj.weight",
-      0.001251220703125
+      0.0011749267578125
     ],
     [
       "model.language_model.layers.0.mlp.down_proj.weight",
-      0.00054931640625
+      0.0005550384521484375
     ],
     [
       "model.language_model.layers.1.self_attn.q_proj.weight",
@@ -45,17 +45,17 @@
     "chat_template.jinja": "74eeb55fd3341286ec3fd44e902b7120721acc81cd394e96b431f85e93a1ea56",
     "config.json": "111827aa7ae6d936f48afb2f0776f51b00add45c857c708af2b893499f76d882",
     "generation_config.json": "e0923390059f84a9180b00e5501778acc45ea9856cd7f2fd68208b360927c677",
-    "model-00001-of-00004.safetensors": "41c1d711d80f96f9cd5ca8eb8b0b4375addd3174b2021bd8ab72dc1a9c9c7500",
-    "model-00002-of-00004.safetensors": "57d50fd02ebafbb491d35d445c8279dda89ad8ad5034edf7637971564857b48d",
-    "model-00003-of-00004.safetensors": "03616016aafb162714f7265f9e1a192f72bdc109e4d80d079c593fd0756c1cba",
-    "model-00004-of-00004.safetensors": "a77ea6878393937e46cead80bdfda98e090020cd93b746b89c38a5a81fbf823a",
+    "model-00001-of-00004.safetensors": "b5da315ae797d8f7c9add8bd08249968c5936df1cc3071f7fb34cfc0e91bc1f2",
+    "model-00002-of-00004.safetensors": "4157651afa8d781ce6e4ba90c57bc4606e440a09ebe19ca0e14463b65f1a3076",
+    "model-00003-of-00004.safetensors": "655d65b5bb646889bccd46166dcc31c1e4e183958dd3a5688d6434a3299d2c58",
+    "model-00004-of-00004.safetensors": "a6c3828542f1e5fefdb0658131a6d06274b7abe11794e0d9747f451bf1525a34",
     "model.safetensors.index.json": "71e022361d37d84c2b6224acbddca2c74bedcc7addc523cfc6a57420d4b22ceb",
     "params.json": "81b8377f36c5b3d60900b333214d85160a0500b84576e969092c6fd214f69538",
-    "processor_config.json": "84be30b124b50749c56d25fdbec5ccedf564446f6b3b035e88e1e07b986d2491",
+    "processor_config.json": "ece2373c2ae391bce18785a1810543bc6173a1f28b6767cffab2c35dbea5f002",
     "special_tokens_map.json": "0a5c981e8c5c6f8886ee007a6d4543a0be6b221cb9ca32a8709384a4c6fc8cbb",
     "tekken.json": "600bb27946565481ecf51ba8aee252e49b9a68507866080ac9c30185bb312843",
     "tokenizer.json": "d5f6046775b112f0e2d456ee9dba450684ab964fe5c4e231599bdc6773028135",
-    "tokenizer_config.json": "de368f6e0f14b41c380b29278159f8897f01ec6b3208a91096d2deaf2bcd19f9"
+    "tokenizer_config.json": "f59f7294e4f26383d0ea93840fe21cf197784be0842a8301a0343e8c34ed0d6d"
   },
-  "seconds": 102.7
+  "seconds": 63.8
 }

QUICKSTART.md

@@ -45,7 +45,7 @@ uv pip install --python .venv-native/bin/python -e './StandardOne-8B/server[nati
   --host 0.0.0.0 --port 30120 --max-concurrency 1 \
   --tokenizer-model mistralai/Ministral-3-8B-Instruct-2512-BF16 \
   --tokenizer-revision f6fae9795746f63c9be8344932f01275f3c63734 \
-  --prompt-wording native --native-system-prompt none --default-temperature 1.35
+  --prompt-wording native --native-system-prompt none --default-temperature 1.65
 ```
 
 6. Check health and models.
@@ -114,7 +114,7 @@ CUDA_VISIBLE_DEVICES=0 SGLANG_VLM_CACHE_SIZE_MB=0 .venv-sglang/bin/python -m sgl
   --host 0.0.0.0 --port 30120 --max-concurrency 1 \
   --tokenizer-model mistralai/Ministral-3-3B-Instruct-2512-BF16 \
   --tokenizer-revision b6d637bef2393152b3da2b2fde72eecdee30557e \
-  --prompt-wording native --native-system-prompt none --default-temperature 1.45
+  --prompt-wording native --native-system-prompt none --default-temperature 1.55
 ```
 
 5. Check health, then send a request or run the smoke script as in 8B steps 6-8, with

README.md

@@ -45,7 +45,7 @@ for the measurement conditions and limitations.
 ![Standard One benchmark card: JevBench public tiers, held-out suites, stated-distribution probability, hard-tier calibration, latency and throughput for Standard One 8B, Standard One 3B and Jev 1.13.](docs/assets/00-benchmark-card.png)
 
 *The figure combines results from different measurement paths. See [Benchmarks](#benchmarks) for
-served versus offline conditions; measured 24 September 2026.*
+served versus offline conditions; measured 24–25 September 2026.*
 
 ## At a glance
 
@@ -53,7 +53,7 @@ served versus offline conditions; measured 24 September 2026.*
   `choice` selects among labeled options, `noul` is yes/no, and `score` uses an ordinal scale.
   The endpoint scores the labels in one forward pass without decoding answer text.
 - In the same-run **offline** comparison with its untuned base, 8B improves on four of six suites,
-  ties on public easy, and declines by 5.41 percentage points on public hard. These are not
+  ties on public easy, and declines by 8.11 percentage points on public hard. These are not
   served-endpoint results.
 - Probabilities are temperature-scaled and calibration-checked (hard-tier ECE, distribution
   total-variation) — see Benchmarks below.
@@ -89,7 +89,7 @@ Adapter (`jev-adapter`, ships as `server/` in this repository):
   --host 0.0.0.0 --port 30120 --max-concurrency 1 \
   --tokenizer-model mistralai/Ministral-3-8B-Instruct-2512-BF16 \
   --tokenizer-revision f6fae9795746f63c9be8344932f01275f3c63734 \
-  --prompt-wording native --native-system-prompt none --default-temperature 1.35
+  --prompt-wording native --native-system-prompt none --default-temperature 1.65
 ```
 
 Try it:
@@ -108,7 +108,7 @@ curl -s http://127.0.0.1:30120/v1/systemone -X POST -H 'content-type: applicatio
 }'
 ```
 
-Response shape (illustrative values, default temperature applied):
+Response shape (example values, default temperature applied):
 
 ```json
 {
@@ -117,7 +117,7 @@ Response shape (illustrative values, default temperature applied):
   "usage": {"input_tokens": 96, "output_tokens": 0},
   "metadata": {
     "confidence_method": "1 - normalized_entropy",
-    "temperature": 1.35,
+    "temperature": 1.65,
     "evaluations": 1,
     "adapter_elapsed_ms": 26.1
   }
@@ -129,19 +129,19 @@ More (client command, 3B variant, request format): see [QUICKSTART.md](QUICKSTAR
 ## Benchmarks
 
 **Served endpoint results (the release configuration).** Merged BF16 weights through SGLang 0.5.20
-and `jev-adapter`, native wording, no system prompt, one option order, T=1.35. Jev 1.13 was
+and `jev-adapter`, native wording, no system prompt, one option order, T=1.65. Jev 1.13 was
 measured on the same items through its hosted endpoint; its probabilities are raw, with no
 temperature applied. These are our measurements, not official sealed-set JevBench scores.
 
 | Suite | **Standard One 8B** | Jev 1.13 |
 |---|---|---|
 | JevBench public easy (48) | **100.00 %** | 100.00 % |
-| JevBench public standard (72) | **95.83 %** | 98.61 % |
-| JevBench public hard (111) | **54.05 %** | 72.07 % |
-| judge proxy (600: routing + answer adequacy) | **90.33 %** | 90.50 % |
-| realistic transfer set (600) | **89.50 %** | 86.67 % |
-| stated-distribution probability (1,036) | **81.56 %** | 72.97 % |
-| hard proxy (600) | **48.33 %** | 54.83 % |
+| JevBench public standard (72) | **94.44 %** | 98.61 % |
+| JevBench public hard (111) | **54.95 %** | 72.07 % |
+| judge proxy (600: routing + answer adequacy) | **90.50 %** | 90.50 % |
+| realistic transfer set (600) | **90.33 %** | 86.67 % |
+| stated-distribution probability (1,036) | **81.37 %** | 72.97 % |
+| hard proxy (600) | **52.50 %** | 54.83 % |
 
 **Offline comparison with the untuned base.** This separate `transformers` runner used native
 wording, no system prompt, one option order, T=1. The base and tuned checkpoint were scored by the
@@ -151,15 +151,15 @@ same offline path; these numbers are indicative of the base-model change, not th
 |---|---|---|
 | JevBench public easy (48) | 100.00 % | **100.00 %** |
 | JevBench public standard (72) | 79.17 % | **97.22 %** |
-| JevBench public hard (111) | 60.36 % | **54.95 %** |
-| judge proxy (600: routing + answer adequacy) | 79.33 % | **88.67 %** |
-| realistic transfer set (600) | 72.83 % | **88.83 %** |
-| stated-distribution probability (1,036) | 34.85 % | **81.08 %** |
+| JevBench public hard (111) | 60.36 % | **52.25 %** |
+| judge proxy (600: routing + answer adequacy) | 79.33 % | **88.83 %** |
+| realistic transfer set (600) | 72.83 % | **90.00 %** |
+| stated-distribution probability (1,036) | 34.85 % | **81.56 %** |
 
-Against the untuned base, four suites improve, public easy ties, and public hard falls by 5.41
+Against the untuned base, four suites improve, public easy ties, and public hard falls by 8.11
 percentage points on this offline run. Served and offline probabilities differ even on identical
-prompts, so use the served table for expected endpoint behavior. Hard-tier ECE at served T is 0.178
-(T=1.35) against Jev 1.13's 0.099 raw, and mean TV to the stated distributions is 0.127 at served T
+prompts, so use the served table for expected endpoint behavior. Hard-tier ECE at served T is 0.147
+(T=1.65) against Jev 1.13's 0.099 raw, and mean TV to the stated distributions is 0.145 at served T
 against Jev's 0.192 raw; full calibration table: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#calibration).
 
 **Speed** — raw serial latency on one H200 with SGLang 0.5.20, using a 242-decision profile averaging
@@ -176,16 +176,16 @@ for speed only; no accuracy comparison is implied.
 
 Throughput on one H200 (hard+standard mix, 1,322 tokens/request): 29.3k tok/s at concurrency 1, rising to 39.5k tok/s at concurrency 64 (≈40 decisions/s at concurrency 8 on the 280-token profile above).
 
-On the public classification and decision suites (400 cases/suite, seed 13, served endpoints): AG News 84.8 %, typed decisions 69.8 %, MASSIVE intent mean 86.4 %, email spam 87.2 %, phishing 84.8 %. Full table, per-language and per-workflow breakdown: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#public-classification-and-decision-suites) and [`docs/public-classification-suites.md`](docs/public-classification-suites.md).
+On the public classification and decision suites (400 cases/suite, seed 13, served endpoints): AG News 84.2 %, typed decisions 69.3 %, MASSIVE intent mean 85.6 %, email spam 92.2 %, phishing 86.0 %. Full table, per-language and per-workflow breakdown: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#public-classification-and-decision-suites) and [`docs/public-classification-suites.md`](docs/public-classification-suites.md).
 
 A JevBench v1.4.1 run has been requested; the sealed-set result is not yet available. Full report: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md).
 
 ## Model details
 
 - **Base model:** `mistralai/Ministral-3-8B-Instruct-2512-BF16`, revision `f6fae9795746f63c9be8344932f01275f3c63734` (Apache-2.0).
-- **Adapter:** LoRA r=16, α=32, dropout 0, on `q_proj k_proj v_proj o_proj gate_proj up_proj down_proj` of the language-model projections only (vision tower and multimodal projector excluded), **44,564,480** trainable parameters, PEFT 0.21.0. Adapter file `adapter_model.safetensors`, 214,559,872 bytes, sha256 `a2d9f02d5dd5d3ade3fab88145fc7321315609aa52d3847319dc4ec827f0c684`.
-- **Merged BF16 checkpoint:** merging the adapter into the base changed **238 tensors** (293 unchanged), none outside the language-model projections, max absolute weight change **0.00201**.
-- **Serving details:** native chat-template wording, no system prompt, fixed temperature (T = 1.35, fitted on held-out calibration data); served model name `standard-one-8b` behind stock SGLang 0.5.20 via `jev-adapter` (`POST /v1/systemone`); single caller-supplied option order, no rotation ensemble; 8,192-token context.
+- **Adapter:** LoRA r=16, α=32, dropout 0, on `q_proj k_proj v_proj o_proj gate_proj up_proj down_proj` of the language-model projections only (vision tower and multimodal projector excluded), **44,564,480** trainable parameters, PEFT 0.21.0. Adapter file `adapter_model.safetensors`, 214,559,872 bytes, sha256 `9551923c162970ea8bc2c3a5faf3ea3762b48591c9f460064e531d09427fa41a`.
+- **Merged BF16 checkpoint:** merging the adapter into the base changed **238 tensors** (293 unchanged), none outside the language-model projections, max absolute weight change **0.00198**.
+- **Serving details:** native chat-template wording, no system prompt, fixed temperature (T = 1.65, fitted on held-out calibration data); served model name `standard-one-8b` behind stock SGLang 0.5.20 via `jev-adapter` (`POST /v1/systemone`); single caller-supplied option order, no rotation ensemble; 8,192-token context.
 
 | Path | Contents |
 |---|---|
@@ -197,25 +197,25 @@ A JevBench v1.4.1 run has been requested; the sealed-set result is not yet avail
 
 ## Training data
 
-Training data spans document and field normalisation, judge/routing and answer-adequacy, stated-distribution probability, verification/scoring, adequacy-rubric style, and game- and Tetris board-state cohorts — every row is synthetic and solver-generated (deterministic generators with visible checkers; a subset was teacher-reviewed), licensed Apache-2.0 (ours); the JevBench public tiers used only for evaluation carry MIT. Full per-cohort breakdown (row counts, what each covers, licence): [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#training-data-provenance).
+Training data spans document and field normalisation, judge/routing and answer-adequacy, stated-distribution probability, verification/scoring, adequacy-rubric style, abstention and robustness (paraphrase, trap, hard natural-language), and game- and Tetris board-state cohorts — every row is synthetic and solver-generated (deterministic generators with visible checkers; a subset was teacher-reviewed), licensed Apache-2.0 (ours); the JevBench public tiers used only for evaluation carry MIT. Full per-cohort breakdown (row counts, what each covers, licence): [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md#training-data-provenance).
 
-An exact-text overlap audit against the public JevBench tiers found 0 exact scenario matches and 181 exact instruction matches — rows in two adequacy-rubric cohorts whose entire instruction field, a generic 58-character adequacy question, is byte-identical to one public hard-tier instruction (0.05 % of the 359,497-row training mixture). These rows are kept and disclosed here rather than regenerated, since the overlap is limited to one rubric question's wording and never touches a scenario or an answer.
+An exact-text overlap audit against the public JevBench tiers found 0 exact scenario matches and 181 exact instruction matches — rows in two adequacy-rubric cohorts whose entire instruction field, a generic 58-character adequacy question, is byte-identical to one public hard-tier instruction (0.05 % of the 382,576-row training mixture). These rows are kept and disclosed here rather than regenerated, since the overlap is limited to one rubric question's wording and never touches a scenario or an answer.
 
 ## Limitations
 
-- **Public hard tier:** the served 8B score is **54.05 %**, versus **72.07 %** for Jev 1.13. In the
-  separate offline base comparison, Standard One 8B scores **54.95 %**, below the untuned base's
+- **Public hard tier:** the served 8B score is **54.95 %**, versus **72.07 %** for Jev 1.13. In the
+  separate offline base comparison, Standard One 8B scores **52.25 %**, below the untuned base's
   **60.36 %**.
 - Served probabilities are temperature-scaled by a single global value; if you apply this model to a
   materially different question distribution, re-fitting that temperature is advisable rather than
-  assuming 1.35 transfers.
+  assuming 1.65 transfers.
 - At most 26 options per question (one uppercase letter per option, `A`–`Z`).
 - The sealed JevBench set has not been measured for this model.
-- Served and offline probabilities can differ on identical prompts (mean total-variation ≈0.07 on the
+- Served and offline probabilities can differ on identical prompts (mean total-variation ≈0.07–0.09 on the
   hard tier); served numbers are treated as authoritative.
 - Korean is a small share of multilingual training relative to the other seven languages.
 - The card reports text benchmarks; it does not establish decision accuracy on image inputs.
-- Ten-way support triage (34–40 %) and RAG passage relevance (58–61 %) are weak zero-shot; fine-tune
+- Ten-way support triage (35–40 %) and RAG passage relevance (57–60 %) are weak zero-shot; fine-tune
   for those.
 
 ## Licence

SHA256SUMS

@@ -1,38 +1,38 @@
 1495e1e757ef4d0925a2350563cf5754bb23c51701a8ec4fb3c5cdcbedae6747  LICENSE
-91db6d8bac5fa5f6965586055b5579281b7476f1244b8fc64c1c2ca749357eaf  MERGE_REPORT.json
+1f39839e969d874d397d3842eb702f39516ec784f546ce18220e9836bffb2793  MERGE_REPORT.json
 20486cf6b2e42440b0aea913ea475c818664f34e76e4501651d6cb21987a508b  NOTICE
-cccd4aeea4b5ca9c98b6ea5f638e113175847526bfd748c488e417040ecb5930  QUICKSTART.md
-96854f10a57ef8ff7a00fa895b4580a4358fcafb6d46699fb35fe8eeb23be694  README.md
+de8701f5a28091cdbc601d0625769a1356c6e31962df4fc2c2a559fe4a67369f  QUICKSTART.md
+948299ccaca10f1c283229b51346905c69687147f9e9b89a18d807dbc9b9b41d  README.md
 66e0fda8c8ab269ccd010dca6196931d475a22c3f2eebe5b6b3a2ec33a354d0e  SYSTEM_PROMPT.txt
 74eeb55fd3341286ec3fd44e902b7120721acc81cd394e96b431f85e93a1ea56  chat_template.jinja
 111827aa7ae6d936f48afb2f0776f51b00add45c857c708af2b893499f76d882  config.json
-41f6da260cb61e59780081fa5b5e97c63413721889faf34f7ae4dc65fca7291b  docs/BENCHMARKS.md
-2300b40c53e596eacf93f152375c68bd78a9d77c03fb15ec17e5a76ca9f632bd  docs/assets/00-benchmark-card.png
-4d36cb45e5445a672de2dcbe0cfb273226354af8c842ca8d0925698033efe1a3  docs/assets/00-benchmark-card.svg
+868cb61534ddc6f57b8718113c54391b2d7fae46d9b72b6ae59b3c96d4994d2c  docs/BENCHMARKS.md
+96127670ad1e5bdd336a33118d6246330c27ac9e4fcb3f4f22767033809cd889  docs/assets/00-benchmark-card.png
+28c67a8c861f421fae5889af259d0850e7fb6eadb11f247adef72b94bf00b187  docs/assets/00-benchmark-card.svg
 7313f2b0645ee401d082572986dcee0c08d3daed1ae79fa650cfa685570ad7e2  docs/assets/02-latency-vs-qwen.png
 0b48a0dd4b4c9ac7c1842836940348edc2e1ba697184943f0c84c3e3d3c7f2d0  docs/assets/02-latency-vs-qwen.svg
 5f430f8841e393548f080833524840adb93deef1c6f75dca93d0571262a2ac68  docs/assets/04-throughput.png
 dd888be78c0e32894cf04a8e3fbf8757e7218873668fa4443ba47c7c1c48b6dc  docs/assets/04-throughput.svg
-f3211f7d1cbc624b0f558e7f9e7edac3ea51a381287cb7409719a12d3f11edaa  docs/assets/05-gain-over-base.png
-855f5e1d0871d89f5e92dbf8087282d787f9bf7c7c6a2b25b936cffd33f6c219  docs/assets/05-gain-over-base.svg
-4d41f1b782502403109c4074e7705fc42f18dfb9fc865b5aca5771f4c4991264  docs/assets/06-vs-jev.png
-946703a63a228ae328dc713186a6b16b682b193fcfbc6303e17fa143cbd58ddb  docs/assets/06-vs-jev.svg
-c3eacfddd6755a60b741d8929f3a41e84d2a862629dce7458ca96d248b4378fd  docs/public-classification-suites.md
-e61bcbcf9fd396ec69ff10200d67c77bdec18739cf172b79d4942806bd9d0f61  evidence/calibration-leaderboard-ablate-nosys.json
-5384f476f04530a271a86a023fb73134633585f55d684cd30eaa58e8f57d9c2b  evidence/calibration-leaderboard-merged-native.json
-be6dc9b31e17e07d8d7d0b13f928c7d350d6fc8cdc03b98a7d0e39e1150ab94e  evidence/served-nosys-easy-report.json
-200a2fa957969f386f9c3e4066ed6d71aca13cd9617aa05005a09604a5bfbb25  evidence/served-nosys-hard-report.json
+2d056ba120ea239ee7a2f6840187589f779d5e6266185d835c36a4693984e4b8  docs/assets/05-gain-over-base.png
+0009567d9bd46390f95b6e8e8f1defd02c86b49c62bb40fec28431bf25725f4d  docs/assets/05-gain-over-base.svg
+837f07a1934509e3dffd4bc1b824d7c983b8dccd47457e5fd881a17f6b2e047c  docs/assets/06-vs-jev.png
+085a13c149a8245f046beeca014fe90b85a8faa379639c2cdedcd443ad941a4f  docs/assets/06-vs-jev.svg
+773cdf5156019a276bb7b7153a1962f38a9439a323644cd7a7291fb08a60788f  docs/public-classification-suites.md
+b66dede01002021789916b8c519e4788041930d0dca331b838922ca071115ca0  evidence/calibration-leaderboard-ablate-nosys.json
+8a2f548daed47691ec5d830069f671042285b34939e106ebdf1318db3dd9094b  evidence/calibration-leaderboard-merged-native.json
+f68aa8f1d1bbc60b045592713188563643e07d0e8d8337df4df9387bf87254f3  evidence/served-nosys-easy-report.json
+c0eb0fde38c44b1f3f62f6a79cc45c3e89cad1f782d1c6cc40526f69e8595ff6  evidence/served-nosys-hard-report.json
 d47f43be5bd7767b80ed27a96a9a85ece1495b37fb9365e575b1df6b32fbc509  evidence/served-nosys-latency-idle-gpu.json
-26cfe2db3fa3aee577a34017d73719c8782dcf95b99bf0072e2481775c488f48  evidence/served-nosys-original-report.json
+b87e0a9a48ebb9fa933fe0db470b506cbe1850824abf65dfebf6ad6c5f58cdda  evidence/served-nosys-original-report.json
 e0923390059f84a9180b00e5501778acc45ea9856cd7f2fd68208b360927c677  generation_config.json
-41c1d711d80f96f9cd5ca8eb8b0b4375addd3174b2021bd8ab72dc1a9c9c7500  model-00001-of-00004.safetensors
-57d50fd02ebafbb491d35d445c8279dda89ad8ad5034edf7637971564857b48d  model-00002-of-00004.safetensors
-03616016aafb162714f7265f9e1a192f72bdc109e4d80d079c593fd0756c1cba  model-00003-of-00004.safetensors
-a77ea6878393937e46cead80bdfda98e090020cd93b746b89c38a5a81fbf823a  model-00004-of-00004.safetensors
+b5da315ae797d8f7c9add8bd08249968c5936df1cc3071f7fb34cfc0e91bc1f2  model-00001-of-00004.safetensors
+4157651afa8d781ce6e4ba90c57bc4606e440a09ebe19ca0e14463b65f1a3076  model-00002-of-00004.safetensors
+655d65b5bb646889bccd46166dcc31c1e4e183958dd3a5688d6434a3299d2c58  model-00003-of-00004.safetensors
+a6c3828542f1e5fefdb0658131a6d06274b7abe11794e0d9747f451bf1525a34  model-00004-of-00004.safetensors
 71e022361d37d84c2b6224acbddca2c74bedcc7addc523cfc6a57420d4b22ceb  model.safetensors.index.json
 81b8377f36c5b3d60900b333214d85160a0500b84576e969092c6fd214f69538  params.json
 ece2373c2ae391bce18785a1810543bc6173a1f28b6767cffab2c35dbea5f002  processor_config.json
-142013a2e5eee9958174aa9c277fdc87e7e99f6d4a1a865ca972b140ab11691a  release-manifest.json
+ad763bb1baa96641da64f113a8fdf28ec7ed798d081faa16749d81336b6efc68  release-manifest.json
 eef2a88a000d2ec294379755770c147327cb79d3d7278c711899bd36797d7a02  server/.gitignore
 1495e1e757ef4d0925a2350563cf5754bb23c51701a8ec4fb3c5cdcbedae6747  server/LICENSE
 0f1f023b916d38d7c12af3a71e09f5e82b819a7e156e4992118bb160ee63ede1  server/NOTICE

docs/BENCHMARKS.md

@@ -36,8 +36,8 @@ hard) are used as-is in both places.
 ## Scope
 
 - Both candidates: merged BF16 weights, LoRA r=16/α=32 on `q/k/v/o/gate/up/down` of the language model
-  only (vision tower + projector excluded), same three-stage training recipe on the **identical final
-  training mixture** (29 cohorts, 359,497 rows) — see `MODEL_CARD_8B.md` / `MODEL_CARD_3B.md` §
+  only (vision tower + projector excluded), same five-stage training recipe on the **identical final
+  training mixture** (34 cohorts, 382,576 rows) — see `MODEL_CARD_8B.md` / `MODEL_CARD_3B.md` §
   *Training* for the recipe and per-cohort breakdown, and `release-manifest.json` for the exact
   per-cohort row counts and hashes. The 3B is a smaller base/adapter pair, not a reduced-data variant.
   **Image input is supported** (the Pixtral vision tower and projector are retained; images are passed
@@ -45,7 +45,7 @@ hard) are used as-is in both places.
   detail image or content-moderation training or results further.
 - Release serving configuration for both: native wording, **no system prompt**
   (`--prompt-wording native --native-system-prompt none`), single option order, server-side default
-  temperature **T = 1.35 (8B)** / **T = 1.45 (3B)** — both refit directly on served probabilities
+  temperature **T = 1.65 (8B)** / **T = 1.55 (3B)** — both refit directly on served probabilities
   (see § Calibration). See `QUICKSTART.md` for the exact commands.
 - What is **not** covered here: any official JevBench sealed-set score (we have none yet), any additional peer-system comparison beyond Jev 1.13, and image/content-moderation
   training detail beyond the one sentence above.
@@ -60,14 +60,14 @@ Two different measurement paths appear throughout this report and are not interc
    external caller would use — see `QUICKSTART.md` for how to reproduce this endpoint.
 
 **These two paths do not agree exactly.** Served probabilities differ measurably from offline-runner
-probabilities for both candidates: mean total-variation distance ≈0.07 on the hard tier, with 8–12 of
+probabilities for both candidates: mean total-variation distance ≈0.07–0.09 on the hard tier, with 8–11 of
 111 argmax label changes, even though the rendered prompts were verified byte-identical between the two
 paths. **Treat served numbers as authoritative and offline numbers as indicative.**
 
-Temperature is **final and served-path-fit** for both candidates: T = 1.35 (8B) and T = 1.45 (3B) were
+Temperature is **final and served-path-fit** for both candidates: T = 1.65 (8B) and T = 1.55 (3B) were
 refit directly on served probabilities from the running endpoint, on
 `internal-eval-hardproxy-calib-v1` + `distprob-calib-v2`, no system prompt. This supersedes an earlier
-offline-harness fit (T = 1.5 / T = 1.3).
+offline-harness fit (T = 1.55 / T = 1.5).
 
 Most of the offline text-suite table below reflects the **with-system-prompt** configuration, while the
 release ships **without** the system prompt. Where we independently re-measured the no-system-prompt
@@ -81,14 +81,14 @@ columns are shown.
 | Suite | n | With system prompt | No system prompt (shipped) | Notes |
 |---|---|---|---|---|
 | JevBench public easy | 48 | 100.00 % | **100.00 %** | |
-| JevBench public original | 72 | 98.61 % | **97.22 %** | |
-| JevBench public hard | 111 | 56.76 % | **54.95 %** | never used for training/T-fitting/selection |
-| judge-proxy-v1 (routing + adequacy) | 600 | 89.33 % | **88.67 %** | leaderboard judge text not public; proxy only |
-| internal-eval-v2-realistic | 600 | 88.83 % | **88.83 %** | unchanged with/without system prompt |
-| distprob-val-v2 | 1,036 | 81.27 % | **81.08 %** | |
-| internal-eval-v1 | 600 | 78.50 % | not separately re-measured | |
-| internal-eval-hardproxy-v1 | 600 | 48.67 % | not separately re-measured | hard-tier proxy |
-| games-harness-eval-v1 | 360 | 97.78 % | not separately re-measured | othello/chess/tic-tac-toe/snake/graph, text |
+| JevBench public original | 72 | 95.83 % | **97.22 %** | |
+| JevBench public hard | 111 | 51.35 % | **52.25 %** | never used for training or T-fitting; informed the prompt-wording choice |
+| judge-proxy-v1 (routing + adequacy) | 600 | 89.00 % | **88.83 %** | leaderboard judge text not public; proxy only |
+| internal-eval-v2-realistic | 600 | 91.33 % | **90.00 %** | |
+| distprob-val-v2 | 1,036 | 80.89 % | **81.56 %** | |
+| internal-eval-v1 | 600 | 76.50 % | not separately re-measured | |
+| internal-eval-hardproxy-v1 | 600 | 53.00 % | not separately re-measured | hard-tier proxy |
+| games-harness-eval-v1 | 360 | 96.94 % | not separately re-measured | othello/chess/tic-tac-toe/snake/graph, text |
 | tetris-drought-eval-v1 (text) | 120 | 67.50 % | not separately re-measured | |
 
 ### Standard One 3B
@@ -96,58 +96,57 @@ columns are shown.
 | Suite | n | With system prompt | No system prompt (shipped) | Notes |
 |---|---|---|---|---|
 | JevBench public easy | 48 | 100.00 % | **100.00 %** | |
-| JevBench public original | 72 | 90.28 % | **91.67 %** | one of the few rows where dropping the prompt *helps* |
-| JevBench public hard | 111 | 52.25 % | **49.55 %** | never used for training/T-fitting/selection |
-| judge-proxy-v1 (routing + adequacy) | 600 | 86.83 % | **85.67 %** | |
-| internal-eval-v2-realistic | 600 | 88.83 % | **88.67 %** | |
-| distprob-val-v2 | 1,036 | 80.98 % | **79.73 %** | |
-| internal-eval-v1 | 600 | 77.17 % | not separately re-measured | |
-| internal-eval-hardproxy-v1 | 600 | 43.33 % | not separately re-measured | hard-tier proxy |
-| games-harness-eval-v1 | 360 | 95.83 % | not separately re-measured | |
-| tetris-drought-eval-v1 (text) | 120 | 64.17 % | not separately re-measured | |
+| JevBench public original | 72 | 93.06 % | **93.06 %** | unchanged with/without system prompt |
+| JevBench public hard | 111 | 48.65 % | **46.85 %** | never used for training or T-fitting; informed the prompt-wording choice |
+| judge-proxy-v1 (routing + adequacy) | 600 | 89.00 % | **88.83 %** | |
+| internal-eval-v2-realistic | 600 | 90.00 % | **90.67 %** | |
+| distprob-val-v2 | 1,036 | 81.18 % | **79.83 %** | |
+| internal-eval-v1 | 600 | 71.67 % | not separately re-measured | |
+| internal-eval-hardproxy-v1 | 600 | 43.50 % | not separately re-measured | hard-tier proxy |
+| games-harness-eval-v1 | 360 | 96.39 % | not separately re-measured | |
+| tetris-drought-eval-v1 (text) | 120 | 68.33 % | not separately re-measured | |
 
 ## Served parity
 
 Both offline and served measurement paths were run; served numbers below are authoritative, on the
 final served-path temperature fit:
 
-| | Standard One 8B (no system prompt, T=1.35) | Standard One 3B (no system prompt, T=1.45) |
+| | Standard One 8B (no system prompt, T=1.65) | Standard One 3B (no system prompt, T=1.55) |
 |---|---|---|
 | easy (48) | 100.00 % | 100.00 % |
-| original (72) | 95.83 % | 86.11 % |
-| hard (111) | 54.05 % | 48.65 % |
-| judge-proxy-v1 | 90.33 % | 85.67 % |
-| internal-eval-hardproxy-v1 | 48.33 % | 43.67 % |
-| internal-eval-v2-realistic | 89.50 % | 88.17 % |
-| distprob-val-v2 | 81.56 % | 80.12 % |
-| Calibration axis (leaderboard formula) | 75.9 | 77.5 |
+| original (72) | 94.44 % | 88.89 % |
+| hard (111) | 54.95 % | 45.95 % |
+| judge-proxy-v1 | 90.50 % | 89.00 % |
+| internal-eval-hardproxy-v1 | 52.50 % | 42.17 % |
+| internal-eval-v2-realistic | 90.33 % | 90.83 % |
+| distprob-val-v2 | 81.37 % | 80.50 % |
+| Calibration axis (leaderboard formula) | 78.1 | 73.3 |
 
 As noted above, served numbers differ somewhat from the corresponding offline numbers for both
-candidates (mean TV ≈0.07 hard tier, 8–12/111 argmax changes) — these are the figures used for the
+candidates (mean TV ≈0.07–0.09 hard tier, 8–11/111 argmax changes) — these are the figures used for the
 Accuracy does not depend on temperature, so the earlier T=1.0-forced served run
-(before the temperature refit) reported the same easy/original/hard accuracy: 8B 100.00/95.83/54.05, 3B
-100.00/87.50/48.65 no-system-prompt (100.00/88.89/46.85 with-system-prompt ablation).
+(before the temperature refit) reported the same easy/original/hard accuracy: 8B 100.00/94.44/54.95, 3B 100.00/88.89/45.95 no-system-prompt.
 
 ## Public classification and decision suites
 
 400 cases per suite, seed 13, sampled from public test splits, one request per case, served endpoints
 (merged BF16, SGLang 0.5.20, `jev-adapter`, native wording, no system prompt, single option order),
-measured 24 September 2026. Standard One only; no comparison column.
+measured 25 September 2026. Standard One only; no comparison column.
 
 | Suite | Standard One 8B | Standard One 3B |
 |---|---|---|
-| AG News (4 topics) | 84.8 % | 83.8 % |
-| DAIR Emotion (6 labels) | 57.0 % | 58.0 % |
-| Typed decisions (400 cases, 2,000 decisions) | 69.8 % | 69.3 % |
-| MASSIVE intent, 20 options, 9 languages, mean (en 88.0 / 85.8, ko 89.2 / 84.2) | 86.4 % | 84.0 % |
-| XNLI, 3 labels, 6 languages, mean | 61.4 % | 62.2 % |
-| Email spam | 87.2 % | 91.8 % |
-| Phishing | 84.8 % | 89.5 % |
-| Moderation (toxic vs benign) | 82.5 % | 78.8 % |
-| Model routing (domain) | 92.7 % | 87.2 % |
+| AG News (4 topics) | 84.2 % | 83.0 % |
+| DAIR Emotion (6 labels) | 56.5 % | 58.8 % |
+| Typed decisions (400 cases, 2,000 decisions) | 69.3 % | 69.0 % |
+| MASSIVE intent, 20 options, 9 languages, mean (en 88.0 / 85.5, ko 88.2 / 83.8) | 85.6 % | 83.3 % |
+| XNLI, 3 labels, 6 languages, mean | 60.1 % | 62.0 % |
+| Email spam | 92.2 % | 93.0 % |
+| Phishing | 86.0 % | 89.8 % |
+| Moderation (toxic vs benign) | 85.0 % | 81.5 % |
+| Model routing (domain) | 93.0 % | 83.7 % |
 
 Option-order flip rate (share of cases whose answer changes under a permuted option order) ranges from
-3.5 % (AG News) to 12 % (MASSIVE en) for the 8B. Full per-language and per-workflow numbers, macro-F1
+3.3 % (AG News) to 13 % (MASSIVE en) for the 8B. Full per-language and per-workflow numbers, macro-F1
 and ECE: [`public-classification-suites.md`](public-classification-suites.md).
 
 ## Latency and throughput
@@ -204,13 +203,13 @@ same host used for our own runs.
 
 | Suite (n) | Jev 1.13 | Standard One 8B | Standard One 3B |
 |---|---|---|---|
-| distprob-val-v2 (1,036) | 72.97 % | **81.56 %** | 80.12 % |
-| internal-eval-v2-realistic (600) | 86.67 % | **89.50 %** | 88.17 % |
-| judge-proxy-v1 (600) | **90.50 %** | 90.33 % | 85.67 % |
+| distprob-val-v2 (1,036) | 72.97 % | **81.37 %** | 80.50 % |
+| internal-eval-v2-realistic (600) | 86.67 % | 90.33 % | **90.83 %** |
+| judge-proxy-v1 (600) | **90.50 %** | **90.50 %** | 89.00 % |
 | JevBench easy (48) | 100.00 % | 100.00 % | 100.00 % |
-| JevBench standard (72) | **98.61 %** | 95.83 % | 86.11 % |
-| internal-eval-hardproxy-v1 (600) | **54.83 %** | 48.33 % | 43.67 % |
-| JevBench hard (111) | **72.07 %** | 54.05 % | 48.65 % |
+| JevBench standard (72) | **98.61 %** | 94.44 % | 88.89 % |
+| internal-eval-hardproxy-v1 (600) | **54.83 %** | 52.50 % | 42.17 % |
+| JevBench hard (111) | **72.07 %** | 54.95 % | 45.95 % |
 
 Jev leads on the public hard tier and the hard proxy; Standard One leads on stated-distribution probability and
 the realistic transfer set; judge and easy tie.
@@ -219,8 +218,8 @@ Calibration on the same items, Jev raw (no temperature applied):
 
 | | Jev 1.13 | Standard One 8B | Standard One 3B |
 |---|---|---|---|
-| Hard-tier ECE (10-bin, top label) | **0.099** | 0.232 raw (0.178 at T=1.35) | 0.227 raw (0.158 at T=1.45) |
-| Mean TV to stated distributions | **0.192** | 0.111 raw (0.127 at T) | 0.121 raw (0.135 at T) |
+| Hard-tier ECE (10-bin, top label) | **0.099** | 0.231 raw (0.147 at T=1.65) | 0.286 raw (0.198 at T=1.55) |
+| Mean TV to stated distributions | **0.192** | 0.109 raw (0.145 at T) | 0.112 raw (0.139 at T) |
 
 Latency, same 242-decision profile, same host:
 
@@ -234,17 +233,16 @@ Jev's latency includes network round-trip to a hosted API; Standard One's is loc
 
 ## Calibration
 
-- **T fit (release, no system prompt, served path, final)**: 8B T = 1.35, 3B T = 1.45, both refit
+- **T fit (release, no system prompt, served path, final)**: 8B T = 1.65, 3B T = 1.55, both refit
   directly on served probabilities from the running endpoint, on
   `internal-eval-hardproxy-calib-v1` + `distprob-calib-v2`. This is the final, shipped configuration.
-- **Historical reference (offline-harness fit, superseded)**: 8B T = 1.5 (no system prompt) / T = 1.4
-  (with system prompt, merged-checkpoint offline check, hard-tier ECE 0.211 at T=1, 0.148 at T=1.4);
-  3B T = 1.3 (no system prompt) / T = 1.35 (with system prompt, merged-checkpoint offline check,
-  hard-tier ECE 0.236 at T=1, 0.190 at T=1.35). These predate the served-path refit and are kept for
+- **Historical reference (offline-harness fit, superseded)**: 8B T = 1.55 (no system prompt) / T = 1.45
+  (with system prompt, merged-checkpoint offline check, hard-tier ECE 0.269 at T=1, 0.206 at T=1.45);
+  3B T = 1.5 (no system prompt) / T = 1.35 (with system prompt, merged-checkpoint offline check,
+  hard-tier ECE 0.269 at T=1, 0.212 at T=1.35). These predate the served-path refit and are kept for
   record only.
-- **Calibration axis (leaderboard formula)**: 8B **75.9**, 3B **77.5** (see § Served parity).
-- **Distribution (TV) fidelity**, `distprob-val-v2`, offline, with system prompt, at T=1: 8B mean TV
-  ≈0.107–0.128 across suite slices; 3B mean TV ≈0.118–0.128. See § *Measured against Jev 1.13* for the
+- **Calibration axis (leaderboard formula)**: 8B **78.1**, 3B **73.3** (see § Served parity).
+- **Distribution (TV) fidelity**, `distprob-val-v2`, offline, with system prompt, at T=1: 8B mean TV ≈0.103; 3B mean TV ≈0.110. See § *Measured against Jev 1.13* for the
   served-path TV comparison against Jev.
 
 ## Decision quality vs. the untuned base
@@ -254,19 +252,18 @@ Jev's latency includes network round-trip to a hosted API; Standard One's is loc
 *Accuracy change versus the untuned Ministral 3 base, same prompt, same offline runner, no system prompt, T=1. Public JevBench tiers are the 231 public items; the other suites are held-out proxies built from public data. The hard tier is placed last on purpose (see below).*
 
 The only base-model comparison in this report. On the public **hard** tier, the untuned Ministral 3 8B
-base is measured **ahead of Standard One 8B** (60.4 % vs. 55.0 %, a 6-of-111-item gap) — hard is 30 % of the
+base is measured **ahead of Standard One 8B** (60.4 % vs. 52.3 %, a 9-of-111-item gap) — hard is 30 % of the
 v1.3.0 Intelligence axis, so this is material, not a rounding artifact. All rows below come from one
-run of the same offline runner (native wording, no system prompt, T=1, single option order); the
-untuned 3B base is behind on every suite, the untuned 8B base is ahead only on the public hard tier.
+run of the same offline runner (native wording, no system prompt, T=1, single option order); the untuned 3B base is behind on every suite except the public hard tier (by 1 of 111 items), the untuned 8B base is ahead only on the public hard tier.
 
 | Suite | Base 8B | **Standard One 8B** | Base 3B | **Standard One 3B** |
 |---|---|---|---|---|
-| distprob-val-v2 (stated-distribution probability, 1,036) | 34.85 % | **81.08 %** (+46.2) | 31.56 % | **79.73 %** (+48.2) |
-| internal-eval-v2-realistic (realistic transfer set, 600) | 72.83 % | **88.83 %** (+16.0) | 62.50 % | **88.67 %** (+26.2) |
-| JevBench public standard tier (72) | 79.17 % | **97.22 %** (+18.0) | 70.83 % | **91.67 %** (+20.8) |
-| judge-proxy-v1 (routing + adequacy, 600) | 79.33 % | **88.67 %** (+9.3) | 63.17 % | **85.67 %** (+22.5) |
+| distprob-val-v2 (stated-distribution probability, 1,036) | 34.85 % | **81.56 %** (+46.7) | 31.56 % | **79.83 %** (+48.3) |
+| internal-eval-v2-realistic (realistic transfer set, 600) | 72.83 % | **90.00 %** (+17.2) | 62.50 % | **90.67 %** (+28.2) |
+| JevBench public standard tier (72) | 79.17 % | **97.22 %** (+18.0) | 70.83 % | **93.06 %** (+22.2) |
+| judge-proxy-v1 (routing + adequacy, 600) | 79.33 % | **88.83 %** (+9.5) | 63.17 % | **88.83 %** (+25.7) |
 | JevBench public easy tier (48) | 100.00 % | **100.00 %** (+0.0) | 97.92 % | **100.00 %** (+2.1) |
-| JevBench public hard tier (111) | 60.36 % | **54.95 %** (-5.4) | 47.75 % | **49.55 %** (+1.8) |
+| JevBench public hard tier (111) | 60.36 % | **52.25 %** (-8.1) | 47.75 % | **46.85 %** (-0.9) |
 
 
 ## Data provenance and licences
@@ -283,7 +280,7 @@ and in `release-manifest.json`, identical for both candidates.
 
 ## Training data provenance
 
-The following cohort families make up the identical 29-cohort, 359,497-row training mixture used by
+The following cohort families make up the identical 34-cohort, 382,576-row training mixture used by
 both Standard One 8B and Standard One 3B. Every row is synthetic and solver-generated (deterministic generators with
 visible checkers; a subset was teacher-reviewed). No JevBench evaluation scenario appears in any
 training cohort.
@@ -295,13 +292,14 @@ training cohort.
 | Stated-distribution probability (`distprob-scale30k`, `distprob-multilingual`, `distprob-pilot2k`, `narrative-prob-train`, `probability`) | Estimating a probability distribution over stated outcomes | Apache-2.0 (ours) |
 | Verification & scoring (`noul-score-10k`) | Yes/no verification and ordinal scoring | Apache-2.0 (ours) |
 | Adequacy-rubric style (`hardstyle-train`) | Hard-style adequacy questions (see § Public-overlap audit and disclosure) | Apache-2.0 (ours) |
+| Abstention & robustness (`abstain`, `paraphrase`, `trap`, `hard-natural`) | Abstaining when the state does not settle the question, paraphrased and adversarial wording, harder natural-language decisions | Apache-2.0 (ours) |
 | Game & board-state (`games-text-train`, `games-harness-train`, `games-harness-multilingual`) | Othello, chess, tic-tac-toe, snake and graph-problem decisions | Apache-2.0 (ours) |
 | Tetris board-state (`games-text-train-tetris`, `games-harness-tetris`, `tetris-drought-train`) | Tetris-specific board-state decisions | Apache-2.0 (ours) |
 | JevBench public tiers (evaluation only, not training) | Easy / standard / hard held-out accuracy measurement | MIT |
 
 ## Public-overlap audit and disclosure
 
-Run on the exact final 29-cohort, 359,497-row mixture that **both candidates train on**:
+Run on the exact final 34-cohort, 382,576-row mixture that **both candidates train on**:
 
 - **0 exact state matches** — no training row's state/scenario duplicates any public JevBench item.
 - **181 exact instruction matches**: `hardstyle-train-v1` (87 of 3,900 rows) and `hardstyle-train-v2`
@@ -313,7 +311,7 @@ Run on the exact final 29-cohort, 359,497-row mixture that **both candidates tra
   rubric sentence above (also in `judge-realistic-train-v3`, `judge-multilingual-train-v1`,
   `judge-proxy-v1`), an insurance "cancellation takes effect" boilerplate clause, and a "coverage
   reviewer" claims-adjudication phrase.
-- All other 27 cohorts: 0 exact state and 0 exact instruction matches.
+- All other 32 cohorts: 0 exact state and 0 exact instruction matches.
 
 **These 181 rows are kept and disclosed here, not regenerated.** They are 0.05 % of the mixture, and
 the exposure is limited to one hard-tier *question's* generic wording, never a *state* or an answer.
@@ -324,25 +322,25 @@ the exposure is limited to one hard-tier *question's* generic wording, never a *
 
 | File | sha256 |
 |---|---|
-| `adapter_model.safetensors` | `a2d9f02d5dd5d3ade3fab88145fc7321315609aa52d3847319dc4ec827f0c684` |
-| `adapter_config.json` | `a6a57c918f63358bc6c6b031319b124d843de612423906cdcdd777f38321fc95` |
-| `manifest.json` | `250891a115ee6c490b93fd9c9f0d63faf0a2d4cc446efd62481340c3a214b821` |
-| `MERGE_REPORT.json` | `5e98d117e0b06bf3d2272df65d49b8084469ed7718f579a9c68ee98483b04d70` |
-| `calibration-leaderboard.json` | `5384f476f04530a271a86a023fb73134633585f55d684cd30eaa58e8f57d9c2b` |
-| `latency-report-shared-gpu.json` | `2942f20a19632569828762558132706d7ac61b50660e7e6e36bed0c31d7846b5` |
-| `served-easy-report.json` | `62bd49df29c528c350bc779bf568f05a0f7715a9898ed538d5604053bc21ed29` |
-| `served-original-report.json` | `fe276e076192f037cf7e56a8495e666dbfd6ca42caad04b4548439ce70b8fa04` |
-| `served-hard-report.json` | `8a27f9cb5639eb63191dc27c08347f825556915cd6553a596c6a3ab597dbd67b` |
+| `adapter_model.safetensors` | `9551923c162970ea8bc2c3a5faf3ea3762b48591c9f460064e531d09427fa41a` |
+| `adapter_config.json` | `545101ebcafd9fb90723284518d744ca01de46534e04a5dfd4e8f86bc4de0ef2` |
+| `manifest.json` | `62bd99b6d4a49926840c258874a3b804a868626761fa57a39e8bfe10e3192904` |
+| `MERGE_REPORT.json` | `1f39839e969d874d397d3842eb702f39516ec784f546ce18220e9836bffb2793` |
+| `calibration-leaderboard.json` | `a28ada7681a3ae8edcce3f7885dec63ecfb9bc6a9c9794204fa040ae1dd1065f` |
+| `latency-report-shared-gpu.json` | `f6734e5cb4a210e9284632ef97c922876f4aaced3106461ec9cede475a640fa9` |
+| `served-easy-report.json` | `f68aa8f1d1bbc60b045592713188563643e07d0e8d8337df4df9387bf87254f3` |
+| `served-original-report.json` | `b87e0a9a48ebb9fa933fe0db470b506cbe1850824abf65dfebf6ad6c5f58cdda` |
+| `served-hard-report.json` | `c0eb0fde38c44b1f3f62f6a79cc45c3e89cad1f782d1c6cc40526f69e8595ff6` |
 
 `evidence/` (3B):
 
 | File | sha256 |
 |---|---|
-| `adapter_model.safetensors` | `ae5fe8e3be2e9186e6504e48ad2fe7eea5537e7e8382a0cca2ca49d95605f7bf` |
-| `adapter_config.json` | `8326f9e4092ed74a3e68978df37bc9c0a5d8f47b90fbea41ea27d592dd9bda64` |
-| `manifest.json` | `67a81590d59cebe6da802682d1d5c017cff04e518e7fbb752609ed038d31772a` |
-| `calibration-leaderboard.json` | `555d8f7f34a30fcadf1d428e34f8084ee9e13c9f127128e66e808959c81c69ca` |
-| `offline-native-result.json` | `a8efd79e389fc125ce38ab6001186e190eb818982290568577f9147429809f46` |
+| `adapter_model.safetensors` | `7bc8bfff4be52e0c9c6f605faf1eebc5e8a966ce103793c7f6970e29056ef5d3` |
+| `adapter_config.json` | `43b99a646a535b15e0f8bceebe2cfff03dd8cc519f5e21f68856b885657d7179` |
+| `manifest.json` | `ecfd448e212e09df55c86c93233abe857282b8d9d6154dceddb681663b6ebea2` |
+| `calibration-leaderboard.json` | `de13ebd925123ade9f065d7ae3a3955e7f55ddc29ce1d598624039fcdc75567e` |
+| `offline-native-result.json` | `2a53c2dc7c82dcf971110e2f788c750c63a09c1c0ea844a371e6abdd087adf24` |
 
 `docs/assets/` (figures referenced by this report):
 
@@ -352,5 +350,5 @@ the exposure is limited to one hard-tier *question's* generic wording, never a *
 | `04-throughput.png` | `bdb8cb392f6f288fe7ce7fde78e39759cd688340be4fe605890dc79c58e83f73` |
 
 The 3B evidence set does not yet include a separate `MERGE_REPORT.json` or served-endpoint report
-files; the merge-tensor numbers (182 tensors changed, max |Δ| 0.0025) and served numbers (§ Served
+files; the merge-tensor numbers (182 tensors changed, max |Δ| 0.0023) and served numbers (§ Served
 parity) are recorded narratively in `MODEL_CARD_3B.md`.

docs/public-classification-suites.md

@@ -1,52 +1,52 @@
 # Public classification and decision suites
 
-Measured 24 September 2026 through the served endpoints (merged BF16, SGLang 0.5.20, jev-adapter, native wording, no system prompt, single option order). Each suite: 400 cases sampled with seed 13 from the public test split, one request per case; ECE (10-bin, top label) uses the served temperature (8B T=1.35, 3B T=1.45). Accuracy and macro-F1 are argmax metrics and do not depend on the temperature. banking77 (77 labels) is not runnable: the endpoint accepts at most 26 options per question.
+Measured 25 September 2026 through the served endpoints (merged BF16, SGLang 0.5.20, jev-adapter, native wording, no system prompt, single option order). Each suite: 400 cases sampled with seed 13 from the public test split, one request per case; ECE (10-bin, top label) uses the served temperature (8B T=1.65, 3B T=1.55). Accuracy and macro-F1 are argmax metrics and do not depend on the temperature. banking77 (77 labels) is not runnable: the endpoint accepts at most 26 options per question.
 
 ## Public datasets
 
 | Suite | n | Acc 8B | Acc 3B | Macro-F1 8B | Macro-F1 3B | ECE 8B | ECE 3B |
 |---|---|---|---|---|---|---|---|
-| AG News (4 topics) | 400 | 84.8 | 83.8 | 83.3 | 82.3 | 0.095 | 0.106 |
-| DAIR Emotion (6 labels) | 400 | 57.0 | 58.0 | 47.2 | 48.3 | 0.217 | 0.052 |
+| AG News (4 topics) | 400 | 84.2 | 83.0 | 82.9 | 81.4 | 0.084 | 0.094 |
+| DAIR Emotion (6 labels) | 400 | 56.5 | 58.8 | 46.6 | 48.6 | 0.191 | 0.064 |
 | SST-5 (5-level sentiment, ordinal score) | 400 | — | — | — | — | — | — |
-| Typed decisions (400 cases, 2,000 decisions) | 400 | 69.8 | 69.3 | 61.4 | 60.5 | 0.060 | 0.070 |
+| Typed decisions (400 cases, 2,000 decisions) | 400 | 69.3 | 69.0 | 60.2 | 58.9 | 0.056 | 0.048 |
 
 ## Application workflows (400 cases each, public sources)
 
 | Suite | n | Acc 8B | Acc 3B | Macro-F1 8B | Macro-F1 3B | ECE 8B | ECE 3B |
 |---|---|---|---|---|---|---|---|
-| Email spam | 400 | 87.2 | 91.8 | 87.2 | 91.7 | 0.038 | 0.054 |
-| Phishing | 400 | 84.8 | 89.5 | 82.5 | 88.4 | 0.064 | 0.042 |
-| LLM guardrails (jailbreak vs benign) | 400 | 58.0 | 63.0 | 56.9 | 61.0 | 0.198 | 0.139 |
-| Moderation (toxic vs benign) | 400 | 82.5 | 78.8 | 82.5 | 78.5 | 0.047 | 0.080 |
-| RAG passage relevance | 400 | 60.5 | 57.2 | 57.5 | 53.4 | 0.310 | 0.292 |
-| Support triage (10 queues) | 400 | 34.0 | 40.2 | 35.0 | 37.5 | 0.463 | 0.301 |
-| Model routing (domain) | 399 | 92.7 | 87.2 | 47.0 | 45.7 | 0.042 | 0.042 |
+| Email spam | 400 | 92.2 | 93.0 | 92.2 | 93.0 | 0.047 | 0.045 |
+| Phishing | 400 | 86.0 | 89.8 | 84.2 | 88.6 | 0.048 | 0.037 |
+| LLM guardrails (jailbreak vs benign) | 400 | 50.7 | 57.5 | 50.5 | 56.7 | 0.314 | 0.181 |
+| Moderation (toxic vs benign) | 400 | 85.0 | 81.5 | 85.0 | 81.4 | 0.079 | 0.046 |
+| RAG passage relevance | 400 | 60.2 | 57.2 | 56.9 | 53.7 | 0.307 | 0.289 |
+| Support triage (10 queues) | 400 | 35.2 | 39.5 | 36.1 | 37.5 | 0.435 | 0.359 |
+| Model routing (domain) | 399 | 93.0 | 83.7 | 56.5 | 52.8 | 0.048 | 0.046 |
 
 ## MASSIVE intent, 20 options (chance = 5 %)
 
 | Language | n | Acc 8B | Acc 3B | Macro-F1 8B | Macro-F1 3B | ECE 8B | ECE 3B |
 |---|---|---|---|---|---|---|---|
-| en | 400 | 88.0 | 85.8 | 61.3 | 55.5 | 0.025 | 0.058 |
-| ja | 400 | 87.5 | 85.8 | 62.0 | 62.9 | 0.024 | 0.035 |
-| zh | 400 | 88.0 | 85.5 | 67.5 | 60.0 | 0.030 | 0.057 |
-| es | 400 | 82.5 | 82.0 | 59.0 | 52.8 | 0.020 | 0.043 |
-| fr | 400 | 85.8 | 84.5 | 59.9 | 56.7 | 0.034 | 0.061 |
-| de | 400 | 86.0 | 84.5 | 59.9 | 56.2 | 0.022 | 0.043 |
-| pt | 400 | 84.5 | 81.5 | 50.4 | 51.3 | 0.030 | 0.045 |
-| ru | 400 | 85.8 | 82.5 | 61.3 | 58.2 | 0.026 | 0.041 |
-| ko | 400 | 89.2 | 84.2 | 67.6 | 59.4 | 0.034 | 0.035 |
+| en | 400 | 88.0 | 85.5 | 61.7 | 54.3 | 0.024 | 0.055 |
+| ja | 400 | 87.0 | 86.0 | 60.1 | 65.2 | 0.032 | 0.037 |
+| zh | 400 | 86.0 | 84.0 | 63.2 | 60.7 | 0.042 | 0.054 |
+| es | 400 | 82.0 | 82.0 | 55.5 | 53.5 | 0.025 | 0.064 |
+| fr | 400 | 84.5 | 82.8 | 59.2 | 54.2 | 0.040 | 0.061 |
+| de | 400 | 85.8 | 83.8 | 62.6 | 52.8 | 0.039 | 0.043 |
+| pt | 400 | 83.5 | 80.5 | 53.4 | 48.5 | 0.044 | 0.056 |
+| ru | 400 | 85.0 | 81.8 | 62.0 | 55.5 | 0.038 | 0.062 |
+| ko | 400 | 88.2 | 83.8 | 67.2 | 58.0 | 0.034 | 0.029 |
 
 ## XNLI, 3 labels
 
 | Language | n | Acc 8B | Acc 3B | Macro-F1 8B | Macro-F1 3B | ECE 8B | ECE 3B |
 |---|---|---|---|---|---|---|---|
-| en | 400 | 65.0 | 67.8 | 65.3 | 66.3 | 0.126 | 0.094 |
-| es | 400 | 58.2 | 63.5 | 58.0 | 62.4 | 0.170 | 0.077 |
-| fr | 400 | 63.5 | 64.8 | 63.1 | 63.9 | 0.137 | 0.092 |
-| de | 400 | 61.8 | 60.8 | 61.6 | 60.1 | 0.147 | 0.054 |
-| ru | 400 | 58.0 | 60.2 | 57.0 | 59.1 | 0.196 | 0.053 |
-| zh | 400 | 62.0 | 56.2 | 61.6 | 55.6 | 0.159 | 0.032 |
+| en | 400 | 63.5 | 69.2 | 64.2 | 68.7 | 0.136 | 0.080 |
+| es | 400 | 57.2 | 64.0 | 57.5 | 63.7 | 0.186 | 0.057 |
+| fr | 400 | 61.0 | 64.2 | 61.3 | 63.7 | 0.152 | 0.067 |
+| de | 400 | 60.5 | 60.5 | 60.3 | 60.2 | 0.160 | 0.024 |
+| ru | 400 | 57.2 | 60.8 | 56.3 | 60.2 | 0.212 | 0.047 |
+| zh | 400 | 61.0 | 53.2 | 61.0 | 52.6 | 0.169 | 0.061 |
 
 ## Option-order robustness
 
@@ -54,8 +54,8 @@ Share of cases whose answer changes when the options are presented in a permuted
 
 | Suite | 8B | 3B |
 |---|---|---|
-| AG News (4 topics) | 0.035 | 0.028 |
-| DAIR Emotion (6 labels) | 0.068 | 0.160 |
-| massive_intent.en | 0.122 | 0.125 |
+| AG News (4 topics) | 0.033 | 0.037 |
+| DAIR Emotion (6 labels) | 0.072 | 0.185 |
+| massive_intent.en | 0.133 | 0.135 |
 
-SST-5 score MAE on the 0–4 scale: 8B 0.563, 3B 0.655.
+SST-5 score MAE on the 0–4 scale: 8B 0.566, 3B 0.653.

evidence/calibration-leaderboard-ablate-nosys.json

@@ -1,165 +1,237 @@
 {
  "arm_fitted": "original",
  "inputs": [
-  "runs/ablate-final8b-nosys/predictions.jsonl"
+  "runs/ablate-hard25-8b-r1-nosys/predictions.jsonl"
  ],
  "fit_suites": [
   "distprob-calib-v2",
   "internal-eval-hardproxy-calib-v1"
  ],
- "temperature": 1.5,
+ "temperature": 1.55,
  "objective": "leaderboard",
- "axis_proxy_at_T1": 82.99715770249384,
- "axis_proxy_at_T": 90.20311460318212,
+ "axis_proxy_at_T1": 82.3697836819227,
+ "axis_proxy_at_T": 88.54249285881193,
  "at_T1": {
   "jevbench-easy": {
    "original": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.004602145836125129,
-    "calibration_axis_ece_part": 99.07957083277498
+    "ece_10bin": 0.0008854583221252396,
+    "calibration_axis_ece_part": 99.82290833557495
    },
    "rotation": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.004602145836125129,
-    "calibration_axis_ece_part": 99.07957083277498
+    "ece_10bin": 0.0008854583221252396,
+    "calibration_axis_ece_part": 99.82290833557495
    }
   },
   "jevbench-original": {
    "original": {
     "questions": 72,
     "accuracy": 0.9722222222222222,
-    "ece_10bin": 0.0427086408750617,
-    "calibration_axis_ece_part": 91.45827182498766
+    "ece_10bin": 0.01631472128415063,
+    "calibration_axis_ece_part": 96.73705574316988
    },
    "rotation": {
     "questions": 72,
     "accuracy": 0.9722222222222222,
-    "ece_10bin": 0.0427086408750617,
-    "calibration_axis_ece_part": 91.45827182498766
+    "ece_10bin": 0.01631472128415063,
+    "calibration_axis_ece_part": 96.73705574316988
    }
   },
   "jevbench-hard": {
    "original": {
     "questions": 111,
-    "accuracy": 0.5495495495495496,
-    "ece_10bin": 0.2317893844883369,
-    "calibration_axis_ece_part": 53.64212310233262
+    "accuracy": 0.5225225225225225,
+    "ece_10bin": 0.2624775240597505,
+    "calibration_axis_ece_part": 47.5044951880499
    },
    "rotation": {
     "questions": 111,
-    "accuracy": 0.5495495495495496,
-    "ece_10bin": 0.2317893844883369,
-    "calibration_axis_ece_part": 53.64212310233262
+    "accuracy": 0.5225225225225225,
+    "ece_10bin": 0.2624775240597505,
+    "calibration_axis_ece_part": 47.5044951880499
    }
   },
   "judge-proxy-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.8866666666666667,
-    "ece_10bin": 0.05405669901967879,
-    "calibration_axis_ece_part": 89.18866019606423
+    "accuracy": 0.8883333333333333,
+    "ece_10bin": 0.0652446800543733,
+    "calibration_axis_ece_part": 86.95106398912534
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.8866666666666667,
-    "ece_10bin": 0.05405669901967879,
-    "calibration_axis_ece_part": 89.18866019606423
+    "accuracy": 0.8883333333333333,
+    "ece_10bin": 0.0652446800543733,
+    "calibration_axis_ece_part": 86.95106398912534
    }
   },
   "internal-eval-hardproxy-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.4816666666666667,
-    "ece_10bin": 0.1271376622702079,
-    "calibration_axis_ece_part": 74.57246754595842
+    "accuracy": 0.5133333333333333,
+    "ece_10bin": 0.11423836566848482,
+    "calibration_axis_ece_part": 77.15232686630303
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.4816666666666667,
-    "ece_10bin": 0.1271376622702079,
-    "calibration_axis_ece_part": 74.57246754595842
+    "accuracy": 0.5133333333333333,
+    "ece_10bin": 0.11423836566848482,
+    "calibration_axis_ece_part": 77.15232686630303
+   }
+  },
+  "internal-eval-hardproxy-v2": {
+   "original": {
+    "questions": 900,
+    "accuracy": 0.8422222222222222,
+    "ece_10bin": 0.0446534873004051,
+    "calibration_axis_ece_part": 91.06930253991898,
+    "probability_items": 80,
+    "mean_tv_distance": 0.11318626578882222,
+    "fidelity_part": 88.68137342111778,
+    "calibration_axis_estimate": 89.87533798051838
+   },
+   "rotation": {
+    "questions": 900,
+    "accuracy": 0.8422222222222222,
+    "ece_10bin": 0.0446534873004051,
+    "calibration_axis_ece_part": 91.06930253991898,
+    "probability_items": 80,
+    "mean_tv_distance": 0.11318626578882222,
+    "fidelity_part": 88.68137342111778,
+    "calibration_axis_estimate": 89.87533798051838
+   }
+  },
+  "internal-eval-hardproxy-v3": {
+   "original": {
+    "questions": 960,
+    "accuracy": 0.6927083333333334,
+    "ece_10bin": 0.1262817018319052,
+    "calibration_axis_ece_part": 74.74365963361895,
+    "probability_items": 120,
+    "mean_tv_distance": 0.4180075377624871,
+    "fidelity_part": 58.19924622375129,
+    "calibration_axis_estimate": 66.47145292868512
+   },
+   "rotation": {
+    "questions": 960,
+    "accuracy": 0.6927083333333334,
+    "ece_10bin": 0.1262817018319052,
+    "calibration_axis_ece_part": 74.74365963361895,
+    "probability_items": 120,
+    "mean_tv_distance": 0.4180075377624871,
+    "fidelity_part": 58.19924622375129,
+    "calibration_axis_estimate": 66.47145292868512
    }
   },
   "internal-eval-hardproxy-calib-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.5283333333333333,
-    "ece_10bin": 0.09799822260038066,
-    "calibration_axis_ece_part": 80.40035547992387
+    "accuracy": 0.5383333333333333,
+    "ece_10bin": 0.10282321627908997,
+    "calibration_axis_ece_part": 79.435356744182
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.5283333333333333,
-    "ece_10bin": 0.09799822260038066,
-    "calibration_axis_ece_part": 80.40035547992387
+    "accuracy": 0.5383333333333333,
+    "ece_10bin": 0.10282321627908997,
+    "calibration_axis_ece_part": 79.435356744182
    }
   },
   "distprob-calib-v2": {
    "original": {
     "questions": 916,
-    "accuracy": 0.7816593886462883,
-    "ece_10bin": 0.16230437584222135,
-    "calibration_axis_ece_part": 67.53912483155572,
+    "accuracy": 0.787117903930131,
+    "ece_10bin": 0.16955517360540623,
+    "calibration_axis_ece_part": 66.08896527891875,
     "probability_items": 916,
-    "mean_tv_distance": 0.14406040074936213,
-    "fidelity_part": 85.59395992506379,
-    "calibration_axis_estimate": 76.56654237830975
+    "mean_tv_distance": 0.146957893803366,
+    "fidelity_part": 85.3042106196634,
+    "calibration_axis_estimate": 75.69658794929107
    },
    "rotation": {
     "questions": 916,
-    "accuracy": 0.7816593886462883,
-    "ece_10bin": 0.16230437584222135,
-    "calibration_axis_ece_part": 67.53912483155572,
+    "accuracy": 0.787117903930131,
+    "ece_10bin": 0.16955517360540623,
+    "calibration_axis_ece_part": 66.08896527891875,
     "probability_items": 916,
-    "mean_tv_distance": 0.14406040074936213,
-    "fidelity_part": 85.59395992506379,
-    "calibration_axis_estimate": 76.56654237830975
+    "mean_tv_distance": 0.146957893803366,
+    "fidelity_part": 85.3042106196634,
+    "calibration_axis_estimate": 75.69658794929107
    }
   },
   "distprob-val-v2": {
    "original": {
     "questions": 1036,
-    "accuracy": 0.8108108108108109,
-    "ece_10bin": 0.20195528339662477,
-    "calibration_axis_ece_part": 59.608943320675046,
+    "accuracy": 0.8156370656370656,
+    "ece_10bin": 0.21033704974637754,
+    "calibration_axis_ece_part": 57.932590050724485,
     "probability_items": 1036,
-    "mean_tv_distance": 0.11085117753010042,
-    "fidelity_part": 88.91488224698996,
-    "calibration_axis_estimate": 74.2619127838325
+    "mean_tv_distance": 0.10838164218674125,
+    "fidelity_part": 89.16183578132588,
+    "calibration_axis_estimate": 73.54721291602519
    },
    "rotation": {
     "questions": 1036,
-    "accuracy": 0.8108108108108109,
-    "ece_10bin": 0.20195528339662477,
-    "calibration_axis_ece_part": 59.608943320675046,
+    "accuracy": 0.8156370656370656,
+    "ece_10bin": 0.21033704974637754,
+    "calibration_axis_ece_part": 57.932590050724485,
     "probability_items": 1036,
-    "mean_tv_distance": 0.11085117753010042,
-    "fidelity_part": 88.91488224698996,
-    "calibration_axis_estimate": 74.2619127838325
+    "mean_tv_distance": 0.10838164218674125,
+    "fidelity_part": 89.16183578132588,
+    "calibration_axis_estimate": 73.54721291602519
    }
   },
   "internal-eval-v2-realistic": {
    "original": {
     "questions": 600,
-    "accuracy": 0.8883333333333333,
-    "ece_10bin": 0.050502965552120055,
-    "calibration_axis_ece_part": 89.89940688957599,
+    "accuracy": 0.9,
+    "ece_10bin": 0.060050232687422954,
+    "calibration_axis_ece_part": 87.98995346251542,
     "probability_items": 120,
-    "mean_tv_distance": 0.11910379876800192,
-    "fidelity_part": 88.08962012319981,
-    "calibration_axis_estimate": 88.9945135063879
+    "mean_tv_distance": 0.10583699642087854,
+    "fidelity_part": 89.41630035791215,
+    "calibration_axis_estimate": 88.70312691021378
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.8883333333333333,
-    "ece_10bin": 0.050502965552120055,
-    "calibration_axis_ece_part": 89.89940688957599,
+    "accuracy": 0.9,
+    "ece_10bin": 0.060050232687422954,
+    "calibration_axis_ece_part": 87.98995346251542,
     "probability_items": 120,
-    "mean_tv_distance": 0.11910379876800192,
-    "fidelity_part": 88.08962012319981,
-    "calibration_axis_estimate": 88.9945135063879
+    "mean_tv_distance": 0.10583699642087854,
+    "fidelity_part": 89.41630035791215,
+    "calibration_axis_estimate": 88.70312691021378
+   }
+  },
+  "games-harness-eval-v1": {
+   "original": {
+    "questions": 360,
+    "accuracy": 0.9638888888888889,
+    "ece_10bin": 0.023407066864144087,
+    "calibration_axis_ece_part": 95.31858662717119
+   },
+   "rotation": {
+    "questions": 360,
+    "accuracy": 0.9638888888888889,
+    "ece_10bin": 0.023407066864144087,
+    "calibration_axis_ece_part": 95.31858662717119
+   }
+  },
+  "tetris-drought-eval-v1": {
+   "original": {
+    "questions": 120,
+    "accuracy": 0.7,
+    "ece_10bin": 0.08433566168714639,
+    "calibration_axis_ece_part": 83.13286766257072
+   },
+   "rotation": {
+    "questions": 120,
+    "accuracy": 0.7,
+    "ece_10bin": 0.08433566168714639,
+    "calibration_axis_ece_part": 83.13286766257072
    }
   }
  },
@@ -168,151 +240,223 @@
    "original": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.011259522080024077,
-    "calibration_axis_ece_part": 97.74809558399518
+    "ece_10bin": 0.005918895323043019,
+    "calibration_axis_ece_part": 98.8162209353914
    },
    "rotation": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.011259522080024077,
-    "calibration_axis_ece_part": 97.74809558399518
+    "ece_10bin": 0.005918895323043019,
+    "calibration_axis_ece_part": 98.8162209353914
    }
   },
   "jevbench-original": {
    "original": {
     "questions": 72,
     "accuracy": 0.9722222222222222,
-    "ece_10bin": 0.07466702891506422,
-    "calibration_axis_ece_part": 85.06659421698716
+    "ece_10bin": 0.039399572170556646,
+    "calibration_axis_ece_part": 92.12008556588867
    },
    "rotation": {
     "questions": 72,
     "accuracy": 0.9722222222222222,
-    "ece_10bin": 0.07466702891506422,
-    "calibration_axis_ece_part": 85.06659421698716
+    "ece_10bin": 0.039399572170556646,
+    "calibration_axis_ece_part": 92.12008556588867
    }
   },
   "jevbench-hard": {
    "original": {
     "questions": 111,
-    "accuracy": 0.5495495495495496,
-    "ece_10bin": 0.16703737634408544,
-    "calibration_axis_ece_part": 66.5925247311829
+    "accuracy": 0.5225225225225225,
+    "ece_10bin": 0.18734918932581002,
+    "calibration_axis_ece_part": 62.530162134837994
    },
    "rotation": {
     "questions": 111,
-    "accuracy": 0.5495495495495496,
-    "ece_10bin": 0.16703737634408544,
-    "calibration_axis_ece_part": 66.5925247311829
+    "accuracy": 0.5225225225225225,
+    "ece_10bin": 0.18734918932581002,
+    "calibration_axis_ece_part": 62.530162134837994
    }
   },
   "judge-proxy-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.8866666666666667,
-    "ece_10bin": 0.023871407714882973,
-    "calibration_axis_ece_part": 95.2257184570234
+    "accuracy": 0.8883333333333333,
+    "ece_10bin": 0.035292607190456925,
+    "calibration_axis_ece_part": 92.94147856190862
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.8866666666666667,
-    "ece_10bin": 0.023871407714882973,
-    "calibration_axis_ece_part": 95.2257184570234
+    "accuracy": 0.8883333333333333,
+    "ece_10bin": 0.035292607190456925,
+    "calibration_axis_ece_part": 92.94147856190862
    }
   },
   "internal-eval-hardproxy-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.4816666666666667,
-    "ece_10bin": 0.05058104123232578,
-    "calibration_axis_ece_part": 89.88379175353485
+    "accuracy": 0.5133333333333333,
+    "ece_10bin": 0.028450460462045187,
+    "calibration_axis_ece_part": 94.30990790759097
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.4816666666666667,
-    "ece_10bin": 0.05058104123232578,
-    "calibration_axis_ece_part": 89.88379175353485
+    "accuracy": 0.5133333333333333,
+    "ece_10bin": 0.028450460462045187,
+    "calibration_axis_ece_part": 94.30990790759097
+   }
+  },
+  "internal-eval-hardproxy-v2": {
+   "original": {
+    "questions": 900,
+    "accuracy": 0.8422222222222222,
+    "ece_10bin": 0.07212289971256691,
+    "calibration_axis_ece_part": 85.57542005748661,
+    "probability_items": 80,
+    "mean_tv_distance": 0.13286611561022926,
+    "fidelity_part": 86.71338843897706,
+    "calibration_axis_estimate": 86.14440424823184
+   },
+   "rotation": {
+    "questions": 900,
+    "accuracy": 0.8422222222222222,
+    "ece_10bin": 0.07212289971256691,
+    "calibration_axis_ece_part": 85.57542005748661,
+    "probability_items": 80,
+    "mean_tv_distance": 0.13286611561022926,
+    "fidelity_part": 86.71338843897706,
+    "calibration_axis_estimate": 86.14440424823184
+   }
+  },
+  "internal-eval-hardproxy-v3": {
+   "original": {
+    "questions": 960,
+    "accuracy": 0.6927083333333334,
+    "ece_10bin": 0.08424991821380648,
+    "calibration_axis_ece_part": 83.15001635723871,
+    "probability_items": 120,
+    "mean_tv_distance": 0.3456905684652154,
+    "fidelity_part": 65.43094315347847,
+    "calibration_axis_estimate": 74.29047975535859
+   },
+   "rotation": {
+    "questions": 960,
+    "accuracy": 0.6927083333333334,
+    "ece_10bin": 0.08424991821380648,
+    "calibration_axis_ece_part": 83.15001635723871,
+    "probability_items": 120,
+    "mean_tv_distance": 0.3456905684652154,
+    "fidelity_part": 65.43094315347847,
+    "calibration_axis_estimate": 74.29047975535859
    }
   },
   "internal-eval-hardproxy-calib-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.5283333333333333,
-    "ece_10bin": 0.012752801256227178,
-    "calibration_axis_ece_part": 97.44943974875456
+    "accuracy": 0.5383333333333333,
+    "ece_10bin": 0.02698693956922397,
+    "calibration_axis_ece_part": 94.6026120861552
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.5283333333333333,
-    "ece_10bin": 0.012752801256227178,
-    "calibration_axis_ece_part": 97.44943974875456
+    "accuracy": 0.5383333333333333,
+    "ece_10bin": 0.02698693956922397,
+    "calibration_axis_ece_part": 94.6026120861552
    }
   },
   "distprob-calib-v2": {
    "original": {
     "questions": 916,
-    "accuracy": 0.7816593886462883,
-    "ece_10bin": 0.22831875125860196,
-    "calibration_axis_ece_part": 54.336249748279606,
+    "accuracy": 0.787117903930131,
+    "ece_10bin": 0.240144208208916,
+    "calibration_axis_ece_part": 51.9711583582168,
     "probability_items": 916,
-    "mean_tv_distance": 0.17043210542390314,
-    "fidelity_part": 82.95678945760969,
-    "calibration_axis_estimate": 68.64651960294465
+    "mean_tv_distance": 0.17517626368531342,
+    "fidelity_part": 82.48237363146866,
+    "calibration_axis_estimate": 67.22676599484274
    },
    "rotation": {
     "questions": 916,
-    "accuracy": 0.7816593886462883,
-    "ece_10bin": 0.22831875125860196,
-    "calibration_axis_ece_part": 54.336249748279606,
+    "accuracy": 0.787117903930131,
+    "ece_10bin": 0.240144208208916,
+    "calibration_axis_ece_part": 51.9711583582168,
     "probability_items": 916,
-    "mean_tv_distance": 0.17043210542390314,
-    "fidelity_part": 82.95678945760969,
-    "calibration_axis_estimate": 68.64651960294465
+    "mean_tv_distance": 0.17517626368531342,
+    "fidelity_part": 82.48237363146866,
+    "calibration_axis_estimate": 67.22676599484274
    }
   },
   "distprob-val-v2": {
    "original": {
     "questions": 1036,
-    "accuracy": 0.8108108108108109,
-    "ece_10bin": 0.2653141316691388,
-    "calibration_axis_ece_part": 46.937173666172235,
+    "accuracy": 0.8156370656370656,
+    "ece_10bin": 0.2772134117733847,
+    "calibration_axis_ece_part": 44.55731764532306,
     "probability_items": 1036,
-    "mean_tv_distance": 0.13598684352719506,
-    "fidelity_part": 86.4013156472805,
-    "calibration_axis_estimate": 66.66924465672636
+    "mean_tv_distance": 0.13873869257846932,
+    "fidelity_part": 86.12613074215307,
+    "calibration_axis_estimate": 65.34172419373806
    },
    "rotation": {
     "questions": 1036,
-    "accuracy": 0.8108108108108109,
-    "ece_10bin": 0.2653141316691388,
-    "calibration_axis_ece_part": 46.937173666172235,
+    "accuracy": 0.8156370656370656,
+    "ece_10bin": 0.2772134117733847,
+    "calibration_axis_ece_part": 44.55731764532306,
     "probability_items": 1036,
-    "mean_tv_distance": 0.13598684352719506,
-    "fidelity_part": 86.4013156472805,
-    "calibration_axis_estimate": 66.66924465672636
+    "mean_tv_distance": 0.13873869257846932,
+    "fidelity_part": 86.12613074215307,
+    "calibration_axis_estimate": 65.34172419373806
    }
   },
   "internal-eval-v2-realistic": {
    "original": {
     "questions": 600,
-    "accuracy": 0.8883333333333333,
-    "ece_10bin": 0.0831112716252191,
-    "calibration_axis_ece_part": 83.37774567495619,
+    "accuracy": 0.9,
+    "ece_10bin": 0.08543237272014398,
+    "calibration_axis_ece_part": 82.91352545597121,
     "probability_items": 120,
-    "mean_tv_distance": 0.1160590241898743,
-    "fidelity_part": 88.39409758101257,
-    "calibration_axis_estimate": 85.88592162798437
+    "mean_tv_distance": 0.1213507615503817,
+    "fidelity_part": 87.86492384496182,
+    "calibration_axis_estimate": 85.38922465046652
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.8883333333333333,
-    "ece_10bin": 0.0831112716252191,
-    "calibration_axis_ece_part": 83.37774567495619,
+    "accuracy": 0.9,
+    "ece_10bin": 0.08543237272014398,
+    "calibration_axis_ece_part": 82.91352545597121,
     "probability_items": 120,
-    "mean_tv_distance": 0.1160590241898743,
-    "fidelity_part": 88.39409758101257,
-    "calibration_axis_estimate": 85.88592162798437
+    "mean_tv_distance": 0.1213507615503817,
+    "fidelity_part": 87.86492384496182,
+    "calibration_axis_estimate": 85.38922465046652
+   }
+  },
+  "games-harness-eval-v1": {
+   "original": {
+    "questions": 360,
+    "accuracy": 0.9638888888888889,
+    "ece_10bin": 0.02406412647678091,
+    "calibration_axis_ece_part": 95.18717470464382
+   },
+   "rotation": {
+    "questions": 360,
+    "accuracy": 0.9638888888888889,
+    "ece_10bin": 0.02406412647678091,
+    "calibration_axis_ece_part": 95.18717470464382
+   }
+  },
+  "tetris-drought-eval-v1": {
+   "original": {
+    "questions": 120,
+    "accuracy": 0.7,
+    "ece_10bin": 0.055311723165568295,
+    "calibration_axis_ece_part": 88.93765536688633
+   },
+   "rotation": {
+    "questions": 120,
+    "accuracy": 0.7,
+    "ece_10bin": 0.055311723165568295,
+    "calibration_axis_ece_part": 88.93765536688633
    }
   }
  }
-}
+}

evidence/calibration-leaderboard-merged-native.json

@@ -1,107 +1,207 @@
 {
  "arm_fitted": "original",
  "inputs": [
-  "runs/merged-8b-native/predictions.jsonl"
+  "runs/merged-v11-8b-native/predictions.jsonl"
  ],
  "fit_suites": [
   "distprob-calib-v2",
   "internal-eval-hardproxy-calib-v1"
  ],
- "temperature": 1.4,
+ "temperature": 1.4500000000000002,
  "objective": "leaderboard",
- "axis_proxy_at_T1": 85.28055982227521,
- "axis_proxy_at_T": 90.01744401301231,
+ "axis_proxy_at_T1": 82.1196730074161,
+ "axis_proxy_at_T": 87.8424497213332,
  "at_T1": {
   "jevbench-easy": {
    "original": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.005720249769333702,
-    "calibration_axis_ece_part": 98.85595004613326
+    "ece_10bin": 0.0032540000606247066,
+    "calibration_axis_ece_part": 99.34919998787505
    },
    "rotation": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.005720249769333702,
-    "calibration_axis_ece_part": 98.85595004613326
+    "ece_10bin": 0.0032540000606247066,
+    "calibration_axis_ece_part": 99.34919998787505
    }
   },
   "jevbench-original": {
    "original": {
     "questions": 72,
-    "accuracy": 0.9861111111111112,
-    "ece_10bin": 0.036237935338174716,
-    "calibration_axis_ece_part": 92.75241293236506
+    "accuracy": 0.9583333333333334,
+    "ece_10bin": 0.04451899023857428,
+    "calibration_axis_ece_part": 91.09620195228514
    },
    "rotation": {
     "questions": 72,
-    "accuracy": 0.9861111111111112,
-    "ece_10bin": 0.036237935338174716,
-    "calibration_axis_ece_part": 92.75241293236506
+    "accuracy": 0.9583333333333334,
+    "ece_10bin": 0.04451899023857428,
+    "calibration_axis_ece_part": 91.09620195228514
    }
   },
   "jevbench-hard": {
    "original": {
     "questions": 111,
-    "accuracy": 0.5585585585585585,
-    "ece_10bin": 0.21148869439699333,
-    "calibration_axis_ece_part": 57.70226112060133
+    "accuracy": 0.5135135135135135,
+    "ece_10bin": 0.2689732297992066,
+    "calibration_axis_ece_part": 46.205354040158674
    },
    "rotation": {
     "questions": 111,
-    "accuracy": 0.5585585585585585,
-    "ece_10bin": 0.21148869439699333,
-    "calibration_axis_ece_part": 57.70226112060133
+    "accuracy": 0.5135135135135135,
+    "ece_10bin": 0.2689732297992066,
+    "calibration_axis_ece_part": 46.205354040158674
    }
   },
   "judge-proxy-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.8916666666666667,
-    "ece_10bin": 0.05436952421286656,
-    "calibration_axis_ece_part": 89.12609515742669
+    "accuracy": 0.89,
+    "ece_10bin": 0.06160206839396167,
+    "calibration_axis_ece_part": 87.67958632120767
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.8916666666666667,
-    "ece_10bin": 0.05436952421286656,
-    "calibration_axis_ece_part": 89.12609515742669
+    "accuracy": 0.89,
+    "ece_10bin": 0.06160206839396167,
+    "calibration_axis_ece_part": 87.67958632120767
+   }
+  },
+  "internal-eval-v2-realistic": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.9133333333333333,
+    "ece_10bin": 0.07188130015615685,
+    "calibration_axis_ece_part": 85.62373996876863,
+    "probability_items": 120,
+    "mean_tv_distance": 0.09961769279460671,
+    "fidelity_part": 90.03823072053933,
+    "calibration_axis_estimate": 87.83098534465398
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.9133333333333333,
+    "ece_10bin": 0.07188130015615685,
+    "calibration_axis_ece_part": 85.62373996876863,
+    "probability_items": 120,
+    "mean_tv_distance": 0.09961769279460671,
+    "fidelity_part": 90.03823072053933,
+    "calibration_axis_estimate": 87.83098534465398
+   }
+  },
+  "distprob-val-v2": {
+   "original": {
+    "questions": 1036,
+    "accuracy": 0.8088803088803089,
+    "ece_10bin": 0.20216895639888172,
+    "calibration_axis_ece_part": 59.566208720223656,
+    "probability_items": 1036,
+    "mean_tv_distance": 0.10263751134039562,
+    "fidelity_part": 89.73624886596043,
+    "calibration_axis_estimate": 74.65122879309205
+   },
+   "rotation": {
+    "questions": 1036,
+    "accuracy": 0.8088803088803089,
+    "ece_10bin": 0.20216895639888172,
+    "calibration_axis_ece_part": 59.566208720223656,
+    "probability_items": 1036,
+    "mean_tv_distance": 0.10263751134039562,
+    "fidelity_part": 89.73624886596043,
+    "calibration_axis_estimate": 74.65122879309205
+   }
+  },
+  "internal-eval-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.765,
+    "ece_10bin": 0.07864072498258423,
+    "calibration_axis_ece_part": 84.27185500348315
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.765,
+    "ece_10bin": 0.07864072498258423,
+    "calibration_axis_ece_part": 84.27185500348315
+   }
+  },
+  "internal-eval-hardproxy-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.53,
+    "ece_10bin": 0.10019162355231877,
+    "calibration_axis_ece_part": 79.96167528953625
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.53,
+    "ece_10bin": 0.10019162355231877,
+    "calibration_axis_ece_part": 79.96167528953625
+   }
+  },
+  "games-harness-eval-v1": {
+   "original": {
+    "questions": 360,
+    "accuracy": 0.9694444444444444,
+    "ece_10bin": 0.024333181236926092,
+    "calibration_axis_ece_part": 95.13336375261478
+   },
+   "rotation": {
+    "questions": 360,
+    "accuracy": 0.9694444444444444,
+    "ece_10bin": 0.024333181236926092,
+    "calibration_axis_ece_part": 95.13336375261478
+   }
+  },
+  "tetris-drought-eval-v1": {
+   "original": {
+    "questions": 120,
+    "accuracy": 0.675,
+    "ece_10bin": 0.090142304021456,
+    "calibration_axis_ece_part": 81.9715391957088
+   },
+   "rotation": {
+    "questions": 120,
+    "accuracy": 0.675,
+    "ece_10bin": 0.090142304021456,
+    "calibration_axis_ece_part": 81.9715391957088
    }
   },
   "internal-eval-hardproxy-calib-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.5333333333333333,
-    "ece_10bin": 0.07647702603303806,
-    "calibration_axis_ece_part": 84.70459479339239
+    "accuracy": 0.5383333333333333,
+    "ece_10bin": 0.10749572557963366,
+    "calibration_axis_ece_part": 78.50085488407326
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.5333333333333333,
-    "ece_10bin": 0.07647702603303806,
-    "calibration_axis_ece_part": 84.70459479339239
+    "accuracy": 0.5383333333333333,
+    "ece_10bin": 0.10749572557963366,
+    "calibration_axis_ece_part": 78.50085488407326
    }
   },
   "distprob-calib-v2": {
    "original": {
     "questions": 916,
-    "accuracy": 0.7903930131004366,
-    "ece_10bin": 0.17349545850429995,
-    "calibration_axis_ece_part": 65.30090829914002,
+    "accuracy": 0.8078602620087336,
+    "ece_10bin": 0.1875754542540966,
+    "calibration_axis_ece_part": 62.48490914918068,
     "probability_items": 916,
-    "mean_tv_distance": 0.1414347514884198,
-    "fidelity_part": 85.85652485115803,
-    "calibration_axis_estimate": 75.57871657514903
+    "mean_tv_distance": 0.14261508869241069,
+    "fidelity_part": 85.73849113075893,
+    "calibration_axis_estimate": 74.1117001399698
    },
    "rotation": {
     "questions": 916,
-    "accuracy": 0.7903930131004366,
-    "ece_10bin": 0.17349545850429995,
-    "calibration_axis_ece_part": 65.30090829914002,
+    "accuracy": 0.8078602620087336,
+    "ece_10bin": 0.1875754542540966,
+    "calibration_axis_ece_part": 62.48490914918068,
     "probability_items": 916,
-    "mean_tv_distance": 0.1414347514884198,
-    "fidelity_part": 85.85652485115803,
-    "calibration_axis_estimate": 75.57871657514903
+    "mean_tv_distance": 0.14261508869241069,
+    "fidelity_part": 85.73849113075893,
+    "calibration_axis_estimate": 74.1117001399698
    }
   }
  },
@@ -110,92 +210,192 @@
    "original": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.011700559816531805,
-    "calibration_axis_ece_part": 97.65988803669364
+    "ece_10bin": 0.008556722166289754,
+    "calibration_axis_ece_part": 98.28865556674205
    },
    "rotation": {
     "questions": 48,
     "accuracy": 1.0,
-    "ece_10bin": 0.011700559816531805,
-    "calibration_axis_ece_part": 97.65988803669364
+    "ece_10bin": 0.008556722166289754,
+    "calibration_axis_ece_part": 98.28865556674205
    }
   },
   "jevbench-original": {
    "original": {
     "questions": 72,
-    "accuracy": 0.9861111111111112,
-    "ece_10bin": 0.06643081806982218,
-    "calibration_axis_ece_part": 86.71383638603555
+    "accuracy": 0.9583333333333334,
+    "ece_10bin": 0.06737170630608301,
+    "calibration_axis_ece_part": 86.52565873878339
    },
    "rotation": {
     "questions": 72,
-    "accuracy": 0.9861111111111112,
-    "ece_10bin": 0.06643081806982218,
-    "calibration_axis_ece_part": 86.71383638603555
+    "accuracy": 0.9583333333333334,
+    "ece_10bin": 0.06737170630608301,
+    "calibration_axis_ece_part": 86.52565873878339
    }
   },
   "jevbench-hard": {
    "original": {
     "questions": 111,
-    "accuracy": 0.5585585585585585,
-    "ece_10bin": 0.14787005175166834,
-    "calibration_axis_ece_part": 70.42598964966633
+    "accuracy": 0.5135135135135135,
+    "ece_10bin": 0.20553371221053932,
+    "calibration_axis_ece_part": 58.89325755789214
    },
    "rotation": {
     "questions": 111,
-    "accuracy": 0.5585585585585585,
-    "ece_10bin": 0.14787005175166834,
-    "calibration_axis_ece_part": 70.42598964966633
+    "accuracy": 0.5135135135135135,
+    "ece_10bin": 0.20553371221053932,
+    "calibration_axis_ece_part": 58.89325755789214
    }
   },
   "judge-proxy-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.8916666666666667,
-    "ece_10bin": 0.03315566894976705,
-    "calibration_axis_ece_part": 93.36886621004659
+    "accuracy": 0.89,
+    "ece_10bin": 0.0374051974377322,
+    "calibration_axis_ece_part": 92.51896051245356
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.8916666666666667,
-    "ece_10bin": 0.03315566894976705,
-    "calibration_axis_ece_part": 93.36886621004659
+    "accuracy": 0.89,
+    "ece_10bin": 0.0374051974377322,
+    "calibration_axis_ece_part": 92.51896051245356
+   }
+  },
+  "internal-eval-v2-realistic": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.9133333333333333,
+    "ece_10bin": 0.09520852999267097,
+    "calibration_axis_ece_part": 80.9582940014658,
+    "probability_items": 120,
+    "mean_tv_distance": 0.11839251325659841,
+    "fidelity_part": 88.16074867434016,
+    "calibration_axis_estimate": 84.55952133790298
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.9133333333333333,
+    "ece_10bin": 0.09520852999267097,
+    "calibration_axis_ece_part": 80.9582940014658,
+    "probability_items": 120,
+    "mean_tv_distance": 0.11839251325659841,
+    "fidelity_part": 88.16074867434016,
+    "calibration_axis_estimate": 84.55952133790298
+   }
+  },
+  "distprob-val-v2": {
+   "original": {
+    "questions": 1036,
+    "accuracy": 0.8088803088803089,
+    "ece_10bin": 0.2591414908430208,
+    "calibration_axis_ece_part": 48.17170183139584,
+    "probability_items": 1036,
+    "mean_tv_distance": 0.12678761633960767,
+    "fidelity_part": 87.32123836603924,
+    "calibration_axis_estimate": 67.74647009871754
+   },
+   "rotation": {
+    "questions": 1036,
+    "accuracy": 0.8088803088803089,
+    "ece_10bin": 0.2591414908430208,
+    "calibration_axis_ece_part": 48.17170183139584,
+    "probability_items": 1036,
+    "mean_tv_distance": 0.12678761633960767,
+    "fidelity_part": 87.32123836603924,
+    "calibration_axis_estimate": 67.74647009871754
+   }
+  },
+  "internal-eval-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.765,
+    "ece_10bin": 0.047666611055915645,
+    "calibration_axis_ece_part": 90.46667778881687
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.765,
+    "ece_10bin": 0.047666611055915645,
+    "calibration_axis_ece_part": 90.46667778881687
+   }
+  },
+  "internal-eval-hardproxy-v1": {
+   "original": {
+    "questions": 600,
+    "accuracy": 0.53,
+    "ece_10bin": 0.039811831467623654,
+    "calibration_axis_ece_part": 92.03763370647526
+   },
+   "rotation": {
+    "questions": 600,
+    "accuracy": 0.53,
+    "ece_10bin": 0.039811831467623654,
+    "calibration_axis_ece_part": 92.03763370647526
+   }
+  },
+  "games-harness-eval-v1": {
+   "original": {
+    "questions": 360,
+    "accuracy": 0.9694444444444444,
+    "ece_10bin": 0.03758145964541886,
+    "calibration_axis_ece_part": 92.48370807091622
+   },
+   "rotation": {
+    "questions": 360,
+    "accuracy": 0.9694444444444444,
+    "ece_10bin": 0.03758145964541886,
+    "calibration_axis_ece_part": 92.48370807091622
+   }
+  },
+  "tetris-drought-eval-v1": {
+   "original": {
+    "questions": 120,
+    "accuracy": 0.675,
+    "ece_10bin": 0.04359609772091712,
+    "calibration_axis_ece_part": 91.28078045581658
+   },
+   "rotation": {
+    "questions": 120,
+    "accuracy": 0.675,
+    "ece_10bin": 0.04359609772091712,
+    "calibration_axis_ece_part": 91.28078045581658
    }
   },
   "internal-eval-hardproxy-calib-v1": {
    "original": {
     "questions": 600,
-    "accuracy": 0.5333333333333333,
-    "ece_10bin": 0.018680598536642126,
-    "calibration_axis_ece_part": 96.26388029267157
+    "accuracy": 0.5383333333333333,
+    "ece_10bin": 0.03942332541997423,
+    "calibration_axis_ece_part": 92.11533491600515
    },
    "rotation": {
     "questions": 600,
-    "accuracy": 0.5333333333333333,
-    "ece_10bin": 0.018680598536642126,
-    "calibration_axis_ece_part": 96.26388029267157
+    "accuracy": 0.5383333333333333,
+    "ece_10bin": 0.03942332541997423,
+    "calibration_axis_ece_part": 92.11533491600515
    }
   },
   "distprob-calib-v2": {
    "original": {
     "questions": 916,
-    "accuracy": 0.7903930131004366,
-    "ece_10bin": 0.22810909449877087,
-    "calibration_axis_ece_part": 54.378181100245826,
+    "accuracy": 0.8078602620087336,
+    "ece_10bin": 0.24829804055148263,
+    "calibration_axis_ece_part": 50.340391889703476,
     "probability_items": 916,
-    "mean_tv_distance": 0.16228992266646952,
-    "fidelity_part": 83.77100773335304,
-    "calibration_axis_estimate": 69.07459441679944
+    "mean_tv_distance": 0.1643043547333875,
+    "fidelity_part": 83.56956452666124,
+    "calibration_axis_estimate": 66.95497820818235
    },
    "rotation": {
     "questions": 916,
-    "accuracy": 0.7903930131004366,
-    "ece_10bin": 0.22810909449877087,
-    "calibration_axis_ece_part": 54.378181100245826,
+    "accuracy": 0.8078602620087336,
+    "ece_10bin": 0.24829804055148263,
+    "calibration_axis_ece_part": 50.340391889703476,
     "probability_items": 916,
-    "mean_tv_distance": 0.16228992266646952,
-    "fidelity_part": 83.77100773335304,
-    "calibration_axis_estimate": 69.07459441679944
+    "mean_tv_distance": 0.1643043547333875,
+    "fidelity_part": 83.56956452666124,
+    "calibration_axis_estimate": 66.95497820818235
    }
   }
  }

evidence/served-nosys-easy-report.json

@@ -8,22 +8,22 @@
   "served_models": [
     "decision-model"
   ],
-  "elapsed_measurement_s": 1.8076605460082646,
-  "requests_per_second": 26.55365804492175,
-  "questions_per_second": 26.55365804492175,
+  "elapsed_measurement_s": 2.03367520699976,
+  "requests_per_second": 23.602588965429455,
+  "questions_per_second": 23.602588965429455,
   "latency_ms": {
     "n": 48,
-    "mean": 37.548233730073356,
-    "p50": 40.218831491074525,
-    "p95": 51.53239860374015,
-    "p99": 53.726440919563174
+    "mean": 42.254625751714535,
+    "p50": 42.0519930194132,
+    "p95": 48.72663661371916,
+    "p99": 53.63347946899012
   },
   "adapter_ms": {
     "n": 48,
-    "mean": 36.16998820568066,
-    "p50": 38.92328549409285,
-    "p95": 50.138125104422215,
-    "p99": 52.174776941828895
+    "mean": 40.783812836404344,
+    "p50": 40.52820897777565,
+    "p95": 47.059908468509086,
+    "p99": 52.059347213362344
   },
   "input_tokens_per_request": {
     "n": 48,
@@ -37,13 +37,13 @@
       "clean": {
         "n": 48,
         "acc": 1.0,
-        "nll": 0.003545542791522645,
-        "brier": 0.0006767739843162266,
-        "ece": 0.0032907167495703077,
-        "mean_conf": 0.9967092832504297,
-        "confidence_bias": -0.003290716749570266,
+        "nll": 0.000497607137436383,
+        "brier": 1.1601814843575079e-05,
+        "ece": 0.0004940482280447744,
+        "mean_conf": 0.9995059517719552,
+        "confidence_bias": -0.0004940482280447744,
         "confident_error_rate": 0.0,
-        "coverage_at_0_9": 0.9791666666666666,
+        "coverage_at_0_9": 1.0,
         "accuracy_at_0_9": 1.0,
         "coverage_at_5pct_error": 1.0,
         "coverage_at_1pct_error": 1.0
@@ -52,13 +52,13 @@
         "extraction": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.013565002348507072,
-          "brier": 0.002702522346742076,
-          "ece": 0.01254684769770332,
-          "mean_conf": 0.9874531523022966,
-          "confidence_bias": -0.012546847697703423,
+          "nll": 0.0016246805527199597,
+          "brier": 4.471498217352827e-05,
+          "ece": 0.0016108693135871155,
+          "mean_conf": 0.9983891306864129,
+          "confidence_bias": -0.0016108693135871155,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.9166666666666666,
+          "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0
@@ -66,11 +66,11 @@
         "fact": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.0004938518687307065,
-          "brier": 4.552602293521027e-06,
-          "ece": 0.0004927097511010503,
-          "mean_conf": 0.999507290248899,
-          "confidence_bias": -0.0004927097511010503,
+          "nll": 0.0003107625392880677,
+          "brier": 1.687141397009795e-06,
+          "ece": 0.0003103398644883626,
+          "mean_conf": 0.9996896601355116,
+          "confidence_bias": -0.0003103398644883626,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -80,11 +80,11 @@
         "intent": {
           "n": 12,
           "acc": 1.0,
-          "nll": 8.577572644836534e-05,
-          "brier": 1.88468848961593e-08,
-          "ece": 8.576912437829876e-05,
-          "mean_conf": 0.9999142308756217,
-          "confidence_bias": -8.576912437829876e-05,
+          "nll": 3.9376524098356474e-05,
+          "brier": 4.757578511980928e-09,
+          "ece": 3.937494153483101e-05,
+          "mean_conf": 0.9999606250584652,
+          "confidence_bias": -3.937494153483101e-05,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -94,11 +94,11 @@
         "tool_selection": {
           "n": 12,
           "acc": 1.0,
-          "nll": 3.7541222404434765e-05,
-          "brier": 2.1413444129328573e-09,
-          "ece": 3.7540425098403496e-05,
-          "mean_conf": 0.9999624595749016,
-          "confidence_bias": -3.7540425098403496e-05,
+          "nll": 1.5608933639148035e-05,
+          "brier": 3.7822525027331695e-10,
+          "ece": 1.5608792568788665e-05,
+          "mean_conf": 0.9999843912074312,
+          "confidence_bias": -1.5608792568788665e-05,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -110,13 +110,13 @@
         "extraction": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.013565002348507072,
-          "brier": 0.002702522346742076,
-          "ece": 0.01254684769770332,
-          "mean_conf": 0.9874531523022966,
-          "confidence_bias": -0.012546847697703423,
+          "nll": 0.0016246805527199597,
+          "brier": 4.471498217352827e-05,
+          "ece": 0.0016108693135871155,
+          "mean_conf": 0.9983891306864129,
+          "confidence_bias": -0.0016108693135871155,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.9166666666666666,
+          "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0
@@ -124,11 +124,11 @@
         "fact": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.0004938518687307065,
-          "brier": 4.552602293521027e-06,
-          "ece": 0.0004927097511010503,
-          "mean_conf": 0.999507290248899,
-          "confidence_bias": -0.0004927097511010503,
+          "nll": 0.0003107625392880677,
+          "brier": 1.687141397009795e-06,
+          "ece": 0.0003103398644883626,
+          "mean_conf": 0.9996896601355116,
+          "confidence_bias": -0.0003103398644883626,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -138,11 +138,11 @@
         "intent": {
           "n": 12,
           "acc": 1.0,
-          "nll": 8.577572644836534e-05,
-          "brier": 1.88468848961593e-08,
-          "ece": 8.576912437829876e-05,
-          "mean_conf": 0.9999142308756217,
-          "confidence_bias": -8.576912437829876e-05,
+          "nll": 3.9376524098356474e-05,
+          "brier": 4.757578511980928e-09,
+          "ece": 3.937494153483101e-05,
+          "mean_conf": 0.9999606250584652,
+          "confidence_bias": -3.937494153483101e-05,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -152,11 +152,11 @@
         "tool_selection": {
           "n": 12,
           "acc": 1.0,
-          "nll": 3.7541222404434765e-05,
-          "brier": 2.1413444129328573e-09,
-          "ece": 3.7540425098403496e-05,
-          "mean_conf": 0.9999624595749016,
-          "confidence_bias": -3.7540425098403496e-05,
+          "nll": 1.5608933639148035e-05,
+          "brier": 3.7822525027331695e-10,
+          "ece": 1.5608792568788665e-05,
+          "mean_conf": 0.9999843912074312,
+          "confidence_bias": -1.5608792568788665e-05,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -168,20 +168,20 @@
         "clean": {
           "n": 48,
           "acc": 1.0,
-          "nll": 0.003545542791522645,
-          "brier": 0.0006767739843162266,
-          "ece": 0.0032907167495703077,
-          "mean_conf": 0.9967092832504297,
-          "confidence_bias": -0.003290716749570266,
+          "nll": 0.000497607137436383,
+          "brier": 1.1601814843575079e-05,
+          "ece": 0.0004940482280447744,
+          "mean_conf": 0.9995059517719552,
+          "confidence_bias": -0.0004940482280447744,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.9791666666666666,
+          "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0
         }
       },
       "macro_task_acc": 1.0,
-      "objective": -0.003545542791522645,
+      "objective": -0.000497607137436383,
       "paired_flip": {
         "pairs": 0,
         "incomplete_pairs": 0,
@@ -211,10 +211,10 @@
       },
       "latency_ms": {
         "n": 48,
-        "mean": 37.548233730073356,
-        "p50": 40.218831491074525,
-        "p95": 51.53239860374015,
-        "p99": 53.726440919563174
+        "mean": 42.254625751714535,
+        "p50": 42.0519930194132,
+        "p95": 48.72663661371916,
+        "p99": 53.63347946899012
       }
     }
   },

evidence/served-nosys-hard-report.json

@@ -8,22 +8,22 @@
   "served_models": [
     "decision-model"
   ],
-  "elapsed_measurement_s": 9.042668573005358,
-  "requests_per_second": 12.275137488878332,
-  "questions_per_second": 12.275137488878332,
+  "elapsed_measurement_s": 10.32658198097488,
+  "requests_per_second": 10.748958387634962,
+  "questions_per_second": 10.748958387634962,
   "latency_ms": {
     "n": 111,
-    "mean": 81.33763537101389,
-    "p50": 57.81962600303814,
-    "p95": 173.32586800330319,
-    "p99": 207.8344035078772
+    "mean": 92.8989257060436,
+    "p50": 58.94872295903042,
+    "p95": 202.7414859912824,
+    "p99": 220.4522704007104
   },
   "adapter_ms": {
     "n": 111,
-    "mean": 79.83348016083862,
-    "p50": 56.25172102008946,
-    "p95": 171.83201300213113,
-    "p99": 206.01313630468218
+    "mean": 91.27845753063993,
+    "p50": 57.3365319869481,
+    "p95": 200.93203498981893,
+    "p99": 218.40056328801438
   },
   "input_tokens_per_request": {
     "n": 111,
@@ -36,105 +36,105 @@
     "jevbench-hard/public": {
       "clean": {
         "n": 111,
-        "acc": 0.5405405405405406,
-        "nll": 1.086142769557125,
-        "brier": 0.588749225058486,
-        "ece": 0.23159909368491685,
-        "mean_conf": 0.7721396342254573,
-        "confidence_bias": 0.2315990936849167,
+        "acc": 0.5495495495495496,
+        "nll": 1.133335848646085,
+        "brier": 0.5970700765394267,
+        "ece": 0.2302545242930939,
+        "mean_conf": 0.7798040738426435,
+        "confidence_bias": 0.2302545242930939,
         "confident_error_rate": 0.08108108108108109,
-        "coverage_at_0_9": 0.42342342342342343,
-        "accuracy_at_0_9": 0.8085106382978723,
-        "coverage_at_5pct_error": 0.08108108108108109,
-        "coverage_at_1pct_error": 0.08108108108108109,
-        "score_mae": 0.7190588576280422,
-        "ranked_probability_score": 0.16382563203170986
+        "coverage_at_0_9": 0.44144144144144143,
+        "accuracy_at_0_9": 0.8163265306122449,
+        "coverage_at_5pct_error": 0.13513513513513514,
+        "coverage_at_1pct_error": 0.13513513513513514,
+        "score_mae": 0.5884381296011051,
+        "ranked_probability_score": 0.13589515860165793
       },
       "tasks": {
         "adversarial": {
           "n": 6,
-          "acc": 0.8333333333333334,
-          "nll": 0.6917829827636148,
-          "brier": 0.3578603749239757,
-          "ece": 0.19298840927489255,
-          "mean_conf": 0.906985548294041,
-          "confidence_bias": 0.07365221496070762,
-          "confident_error_rate": 0.16666666666666666,
-          "coverage_at_0_9": 0.8333333333333334,
-          "accuracy_at_0_9": 0.8,
-          "coverage_at_5pct_error": 0.3333333333333333,
-          "coverage_at_1pct_error": 0.3333333333333333,
-          "score_mae": 0.34813948795515093,
-          "ranked_probability_score": 0.04085130051350116
+          "acc": 1.0,
+          "nll": 0.18462517435765735,
+          "brier": 0.11143154591042419,
+          "ece": 0.14538591338117762,
+          "mean_conf": 0.8546140866188224,
+          "confidence_bias": -0.14538591338117757,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.6666666666666666,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "score_mae": 0.408417736672984,
+          "ranked_probability_score": 0.05550847320721041
         },
         "ambiguous": {
           "n": 7,
-          "acc": 0.2857142857142857,
-          "nll": 1.611349196858665,
-          "brier": 0.8779085338096889,
-          "ece": 0.49883578886996605,
-          "mean_conf": 0.7547301347411361,
-          "confidence_bias": 0.46901584902685045,
-          "confident_error_rate": 0.2857142857142857,
+          "acc": 0.42857142857142855,
+          "nll": 1.6476381988548245,
+          "brier": 0.8477179354975839,
+          "ece": 0.3457693752791654,
+          "mean_conf": 0.749392342424076,
+          "confidence_bias": 0.3208209138526475,
+          "confident_error_rate": 0.14285714285714285,
           "coverage_at_0_9": 0.42857142857142855,
-          "accuracy_at_0_9": 0.3333333333333333,
-          "coverage_at_5pct_error": 0.0,
-          "coverage_at_1pct_error": 0.0
+          "accuracy_at_0_9": 0.6666666666666666,
+          "coverage_at_5pct_error": 0.14285714285714285,
+          "coverage_at_1pct_error": 0.14285714285714285
         },
         "judge_hard": {
           "n": 17,
           "acc": 0.7647058823529411,
-          "nll": 0.5461353304672285,
-          "brier": 0.3582827044996294,
-          "ece": 0.17186048960850994,
-          "mean_conf": 0.9365663719614512,
-          "confidence_bias": 0.17186048960851008,
+          "nll": 0.54324253492169,
+          "brier": 0.3719283598498577,
+          "ece": 0.1953962488979977,
+          "mean_conf": 0.9012786018391742,
+          "confidence_bias": 0.13657271948623306,
           "confident_error_rate": 0.058823529411764705,
-          "coverage_at_0_9": 0.7647058823529411,
-          "accuracy_at_0_9": 0.9230769230769231,
-          "coverage_at_5pct_error": 0.47058823529411764,
-          "coverage_at_1pct_error": 0.47058823529411764
+          "coverage_at_0_9": 0.6470588235294118,
+          "accuracy_at_0_9": 0.9090909090909091,
+          "coverage_at_5pct_error": 0.5294117647058824,
+          "coverage_at_1pct_error": 0.5294117647058824
         },
         "long_policy": {
           "n": 19,
           "acc": 0.47368421052631576,
-          "nll": 1.394443717200473,
-          "brier": 0.7119601918833228,
-          "ece": 0.31705242173572346,
-          "mean_conf": 0.7433428680256701,
-          "confidence_bias": 0.2696586574993543,
+          "nll": 1.5082474867267202,
+          "brier": 0.7353654524240127,
+          "ece": 0.3018482809810947,
+          "mean_conf": 0.7755324915074104,
+          "confidence_bias": 0.3018482809810947,
           "confident_error_rate": 0.10526315789473684,
-          "coverage_at_0_9": 0.3157894736842105,
-          "accuracy_at_0_9": 0.6666666666666666,
+          "coverage_at_0_9": 0.3684210526315789,
+          "accuracy_at_0_9": 0.7142857142857143,
           "coverage_at_5pct_error": 0.0,
           "coverage_at_1pct_error": 0.0,
-          "score_mae": 1.1289616656075505,
-          "ranked_probability_score": 0.2607984482981377
+          "score_mae": 0.9559519666691165,
+          "ranked_probability_score": 0.20361690528604173
         },
         "multi_hop": {
           "n": 18,
-          "acc": 0.5555555555555556,
-          "nll": 1.078273318223223,
-          "brier": 0.6084784482459148,
-          "ece": 0.15936254353990828,
-          "mean_conf": 0.6779308109498754,
-          "confidence_bias": 0.12237525539431982,
-          "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.1111111111111111,
-          "accuracy_at_0_9": 1.0,
+          "acc": 0.4444444444444444,
+          "nll": 1.1785055734945635,
+          "brier": 0.6508114343861289,
+          "ece": 0.2911283897767315,
+          "mean_conf": 0.7233307262104315,
+          "confidence_bias": 0.27888628176598707,
+          "confident_error_rate": 0.05555555555555555,
+          "coverage_at_0_9": 0.2222222222222222,
+          "accuracy_at_0_9": 0.75,
           "coverage_at_5pct_error": 0.1111111111111111,
           "coverage_at_1pct_error": 0.1111111111111111,
-          "score_mae": 1.1094423812136847,
-          "ranked_probability_score": 0.23619549132427467
+          "score_mae": 1.081145348364534,
+          "ranked_probability_score": 0.2402165282504147
         },
         "probability": {
           "n": 10,
           "acc": 0.4,
-          "nll": 0.7387376756710788,
-          "brier": 0.5289825911753059,
-          "ece": 0.3116268100988524,
-          "mean_conf": 0.6943306254128835,
-          "confidence_bias": 0.29433062541288346,
+          "nll": 0.7381790991685195,
+          "brier": 0.5307273302202737,
+          "ece": 0.30036236704453123,
+          "mean_conf": 0.6731224585317553,
+          "confidence_bias": 0.2731224585317553,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 0.2,
           "accuracy_at_0_9": 1.0,
@@ -144,11 +144,11 @@
         "routing_hard": {
           "n": 5,
           "acc": 1.0,
-          "nll": 0.004261013462209004,
-          "brier": 8.029690421468482e-05,
-          "ece": 0.004236908630699809,
-          "mean_conf": 0.9957630913693002,
-          "confidence_bias": -0.004236908630699809,
+          "nll": 0.0010990353507786027,
+          "brier": 3.616247888274235e-06,
+          "ece": 0.0010979198327893958,
+          "mean_conf": 0.9989020801672106,
+          "confidence_bias": -0.0010979198327893958,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -157,136 +157,136 @@
         },
         "temporal_numeric": {
           "n": 15,
-          "acc": 0.13333333333333333,
-          "nll": 1.936749767566636,
-          "brier": 0.9853040796459122,
-          "ece": 0.40067084112093226,
-          "mean_conf": 0.5340041744542656,
-          "confidence_bias": 0.4006708411209323,
+          "acc": 0.2,
+          "nll": 2.013872798196489,
+          "brier": 1.0125530761951291,
+          "ece": 0.3539814541310209,
+          "mean_conf": 0.553981454131021,
+          "confidence_bias": 0.35398145413102095,
           "confident_error_rate": 0.06666666666666667,
           "coverage_at_0_9": 0.06666666666666667,
           "accuracy_at_0_9": 0.0,
           "coverage_at_5pct_error": 0.0,
           "coverage_at_1pct_error": 0.0,
-          "score_mae": 0.4685276273694827,
-          "ranked_probability_score": 0.18163135830223576
+          "score_mae": 0.04719474969887627,
+          "ranked_probability_score": 0.11130341591217828
         },
         "tradeoff": {
           "n": 6,
           "acc": 0.3333333333333333,
-          "nll": 2.2095284214534288,
-          "brier": 1.0655938811272978,
-          "ece": 0.5036576811563841,
-          "mean_conf": 0.8369910144897176,
-          "confidence_bias": 0.5036576811563842,
-          "confident_error_rate": 0.3333333333333333,
-          "coverage_at_0_9": 0.3333333333333333,
-          "accuracy_at_0_9": 0.0,
+          "nll": 2.720825847328328,
+          "brier": 1.192706152338627,
+          "ece": 0.6635156366768649,
+          "mean_conf": 0.8671972401778957,
+          "confidence_bias": 0.5338639068445623,
+          "confident_error_rate": 0.5,
+          "coverage_at_0_9": 0.6666666666666666,
+          "accuracy_at_0_9": 0.25,
           "coverage_at_5pct_error": 0.0,
           "coverage_at_1pct_error": 0.0
         },
         "trap": {
           "n": 8,
           "acc": 1.0,
-          "nll": 0.02836941340458894,
-          "brier": 0.003078553356297072,
-          "ece": 0.02744408269802645,
-          "mean_conf": 0.9725559173019735,
-          "confidence_bias": -0.02744408269802645,
+          "nll": 0.01672523398134625,
+          "brier": 0.001375962384463599,
+          "ece": 0.016328200128607362,
+          "mean_conf": 0.9836717998713926,
+          "confidence_bias": -0.016328200128607362,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0,
-          "score_mae": 0.13032031801483362,
-          "ranked_probability_score": 0.002678745453972209
+          "score_mae": 0.08196700953200303,
+          "ranked_probability_score": 0.0011087236680607628
         }
       },
       "sources": {
         "adversarial": {
           "n": 6,
-          "acc": 0.8333333333333334,
-          "nll": 0.6917829827636148,
-          "brier": 0.3578603749239757,
-          "ece": 0.19298840927489255,
-          "mean_conf": 0.906985548294041,
-          "confidence_bias": 0.07365221496070762,
-          "confident_error_rate": 0.16666666666666666,
-          "coverage_at_0_9": 0.8333333333333334,
-          "accuracy_at_0_9": 0.8,
-          "coverage_at_5pct_error": 0.3333333333333333,
-          "coverage_at_1pct_error": 0.3333333333333333,
-          "score_mae": 0.34813948795515093,
-          "ranked_probability_score": 0.04085130051350116
+          "acc": 1.0,
+          "nll": 0.18462517435765735,
+          "brier": 0.11143154591042419,
+          "ece": 0.14538591338117762,
+          "mean_conf": 0.8546140866188224,
+          "confidence_bias": -0.14538591338117757,
+          "confident_error_rate": 0.0,
+          "coverage_at_0_9": 0.6666666666666666,
+          "accuracy_at_0_9": 1.0,
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0,
+          "score_mae": 0.408417736672984,
+          "ranked_probability_score": 0.05550847320721041
         },
         "ambiguous": {
           "n": 7,
-          "acc": 0.2857142857142857,
-          "nll": 1.611349196858665,
-          "brier": 0.8779085338096889,
-          "ece": 0.49883578886996605,
-          "mean_conf": 0.7547301347411361,
-          "confidence_bias": 0.46901584902685045,
-          "confident_error_rate": 0.2857142857142857,
+          "acc": 0.42857142857142855,
+          "nll": 1.6476381988548245,
+          "brier": 0.8477179354975839,
+          "ece": 0.3457693752791654,
+          "mean_conf": 0.749392342424076,
+          "confidence_bias": 0.3208209138526475,
+          "confident_error_rate": 0.14285714285714285,
           "coverage_at_0_9": 0.42857142857142855,
-          "accuracy_at_0_9": 0.3333333333333333,
-          "coverage_at_5pct_error": 0.0,
-          "coverage_at_1pct_error": 0.0
+          "accuracy_at_0_9": 0.6666666666666666,
+          "coverage_at_5pct_error": 0.14285714285714285,
+          "coverage_at_1pct_error": 0.14285714285714285
         },
         "judge_hard": {
           "n": 17,
           "acc": 0.7647058823529411,
-          "nll": 0.5461353304672285,
-          "brier": 0.3582827044996294,
-          "ece": 0.17186048960850994,
-          "mean_conf": 0.9365663719614512,
-          "confidence_bias": 0.17186048960851008,
+          "nll": 0.54324253492169,
+          "brier": 0.3719283598498577,
+          "ece": 0.1953962488979977,
+          "mean_conf": 0.9012786018391742,
+          "confidence_bias": 0.13657271948623306,
           "confident_error_rate": 0.058823529411764705,
-          "coverage_at_0_9": 0.7647058823529411,
-          "accuracy_at_0_9": 0.9230769230769231,
-          "coverage_at_5pct_error": 0.47058823529411764,
-          "coverage_at_1pct_error": 0.47058823529411764
+          "coverage_at_0_9": 0.6470588235294118,
+          "accuracy_at_0_9": 0.9090909090909091,
+          "coverage_at_5pct_error": 0.5294117647058824,
+          "coverage_at_1pct_error": 0.5294117647058824
         },
         "long_policy": {
           "n": 19,
           "acc": 0.47368421052631576,
-          "nll": 1.394443717200473,
-          "brier": 0.7119601918833228,
-          "ece": 0.31705242173572346,
-          "mean_conf": 0.7433428680256701,
-          "confidence_bias": 0.2696586574993543,
+          "nll": 1.5082474867267202,
+          "brier": 0.7353654524240127,
+          "ece": 0.3018482809810947,
+          "mean_conf": 0.7755324915074104,
+          "confidence_bias": 0.3018482809810947,
           "confident_error_rate": 0.10526315789473684,
-          "coverage_at_0_9": 0.3157894736842105,
-          "accuracy_at_0_9": 0.6666666666666666,
+          "coverage_at_0_9": 0.3684210526315789,
+          "accuracy_at_0_9": 0.7142857142857143,
           "coverage_at_5pct_error": 0.0,
           "coverage_at_1pct_error": 0.0,
-          "score_mae": 1.1289616656075505,
-          "ranked_probability_score": 0.2607984482981377
+          "score_mae": 0.9559519666691165,
+          "ranked_probability_score": 0.20361690528604173
         },
         "multi_hop": {
           "n": 18,
-          "acc": 0.5555555555555556,
-          "nll": 1.078273318223223,
-          "brier": 0.6084784482459148,
-          "ece": 0.15936254353990828,
-          "mean_conf": 0.6779308109498754,
-          "confidence_bias": 0.12237525539431982,
-          "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.1111111111111111,
-          "accuracy_at_0_9": 1.0,
+          "acc": 0.4444444444444444,
+          "nll": 1.1785055734945635,
+          "brier": 0.6508114343861289,
+          "ece": 0.2911283897767315,
+          "mean_conf": 0.7233307262104315,
+          "confidence_bias": 0.27888628176598707,
+          "confident_error_rate": 0.05555555555555555,
+          "coverage_at_0_9": 0.2222222222222222,
+          "accuracy_at_0_9": 0.75,
           "coverage_at_5pct_error": 0.1111111111111111,
           "coverage_at_1pct_error": 0.1111111111111111,
-          "score_mae": 1.1094423812136847,
-          "ranked_probability_score": 0.23619549132427467
+          "score_mae": 1.081145348364534,
+          "ranked_probability_score": 0.2402165282504147
         },
         "probability": {
           "n": 10,
           "acc": 0.4,
-          "nll": 0.7387376756710788,
-          "brier": 0.5289825911753059,
-          "ece": 0.3116268100988524,
-          "mean_conf": 0.6943306254128835,
-          "confidence_bias": 0.29433062541288346,
+          "nll": 0.7381790991685195,
+          "brier": 0.5307273302202737,
+          "ece": 0.30036236704453123,
+          "mean_conf": 0.6731224585317553,
+          "confidence_bias": 0.2731224585317553,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 0.2,
           "accuracy_at_0_9": 1.0,
@@ -296,11 +296,11 @@
         "routing_hard": {
           "n": 5,
           "acc": 1.0,
-          "nll": 0.004261013462209004,
-          "brier": 8.029690421468482e-05,
-          "ece": 0.004236908630699809,
-          "mean_conf": 0.9957630913693002,
-          "confidence_bias": -0.004236908630699809,
+          "nll": 0.0010990353507786027,
+          "brier": 3.616247888274235e-06,
+          "ece": 0.0010979198327893958,
+          "mean_conf": 0.9989020801672106,
+          "confidence_bias": -0.0010979198327893958,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
@@ -309,71 +309,71 @@
         },
         "temporal_numeric": {
           "n": 15,
-          "acc": 0.13333333333333333,
-          "nll": 1.936749767566636,
-          "brier": 0.9853040796459122,
-          "ece": 0.40067084112093226,
-          "mean_conf": 0.5340041744542656,
-          "confidence_bias": 0.4006708411209323,
+          "acc": 0.2,
+          "nll": 2.013872798196489,
+          "brier": 1.0125530761951291,
+          "ece": 0.3539814541310209,
+          "mean_conf": 0.553981454131021,
+          "confidence_bias": 0.35398145413102095,
           "confident_error_rate": 0.06666666666666667,
           "coverage_at_0_9": 0.06666666666666667,
           "accuracy_at_0_9": 0.0,
           "coverage_at_5pct_error": 0.0,
           "coverage_at_1pct_error": 0.0,
-          "score_mae": 0.4685276273694827,
-          "ranked_probability_score": 0.18163135830223576
+          "score_mae": 0.04719474969887627,
+          "ranked_probability_score": 0.11130341591217828
         },
         "tradeoff": {
           "n": 6,
           "acc": 0.3333333333333333,
-          "nll": 2.2095284214534288,
-          "brier": 1.0655938811272978,
-          "ece": 0.5036576811563841,
-          "mean_conf": 0.8369910144897176,
-          "confidence_bias": 0.5036576811563842,
-          "confident_error_rate": 0.3333333333333333,
-          "coverage_at_0_9": 0.3333333333333333,
-          "accuracy_at_0_9": 0.0,
+          "nll": 2.720825847328328,
+          "brier": 1.192706152338627,
+          "ece": 0.6635156366768649,
+          "mean_conf": 0.8671972401778957,
+          "confidence_bias": 0.5338639068445623,
+          "confident_error_rate": 0.5,
+          "coverage_at_0_9": 0.6666666666666666,
+          "accuracy_at_0_9": 0.25,
           "coverage_at_5pct_error": 0.0,
           "coverage_at_1pct_error": 0.0
         },
         "trap": {
           "n": 8,
           "acc": 1.0,
-          "nll": 0.02836941340458894,
-          "brier": 0.003078553356297072,
-          "ece": 0.02744408269802645,
-          "mean_conf": 0.9725559173019735,
-          "confidence_bias": -0.02744408269802645,
+          "nll": 0.01672523398134625,
+          "brier": 0.001375962384463599,
+          "ece": 0.016328200128607362,
+          "mean_conf": 0.9836717998713926,
+          "confidence_bias": -0.016328200128607362,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0,
-          "score_mae": 0.13032031801483362,
-          "ranked_probability_score": 0.002678745453972209
+          "score_mae": 0.08196700953200303,
+          "ranked_probability_score": 0.0011087236680607628
         }
       },
       "variants": {
         "clean": {
           "n": 111,
-          "acc": 0.5405405405405406,
-          "nll": 1.086142769557125,
-          "brier": 0.588749225058486,
-          "ece": 0.23159909368491685,
-          "mean_conf": 0.7721396342254573,
-          "confidence_bias": 0.2315990936849167,
+          "acc": 0.5495495495495496,
+          "nll": 1.133335848646085,
+          "brier": 0.5970700765394267,
+          "ece": 0.2302545242930939,
+          "mean_conf": 0.7798040738426435,
+          "confidence_bias": 0.2302545242930939,
           "confident_error_rate": 0.08108108108108109,
-          "coverage_at_0_9": 0.42342342342342343,
-          "accuracy_at_0_9": 0.8085106382978723,
-          "coverage_at_5pct_error": 0.08108108108108109,
-          "coverage_at_1pct_error": 0.08108108108108109,
-          "score_mae": 0.7190588576280422,
-          "ranked_probability_score": 0.16382563203170986
+          "coverage_at_0_9": 0.44144144144144143,
+          "accuracy_at_0_9": 0.8163265306122449,
+          "coverage_at_5pct_error": 0.13513513513513514,
+          "coverage_at_1pct_error": 0.13513513513513514,
+          "score_mae": 0.5884381296011051,
+          "ranked_probability_score": 0.13589515860165793
         }
       },
-      "macro_task_acc": 0.5779659934149098,
-      "objective": -1.0239630837071148,
+      "macro_task_acc": 0.6044739299228463,
+      "objective": -1.0552960982380917,
       "paired_flip": {
         "pairs": 0,
         "incomplete_pairs": 0,
@@ -385,9 +385,9 @@
       },
       "reference_distribution": {
         "n": 10,
-        "squared_l2": 0.15893241013908638,
-        "total_variation": 0.27702598370309545,
-        "kl_reference_to_model": 0.24247408457377317,
+        "squared_l2": 0.14573628652401,
+        "total_variation": 0.2702805118712319,
+        "kl_reference_to_model": 0.20505363984021555,
         "note": "Exact reference-distribution fidelity, separate from hard-label Brier/ECE."
       },
       "unknowable": null,
@@ -409,10 +409,10 @@
       },
       "latency_ms": {
         "n": 111,
-        "mean": 81.33763537101389,
-        "p50": 57.81962600303814,
-        "p95": 173.32586800330319,
-        "p99": 207.8344035078772
+        "mean": 92.8989257060436,
+        "p50": 58.94872295903042,
+        "p95": 202.7414859912824,
+        "p99": 220.4522704007104
       }
     }
   },

evidence/served-nosys-original-report.json

@@ -8,22 +8,22 @@
   "served_models": [
     "decision-model"
   ],
-  "elapsed_measurement_s": 2.6541028460196685,
-  "requests_per_second": 27.12781085630411,
-  "questions_per_second": 27.12781085630411,
+  "elapsed_measurement_s": 2.987883549008984,
+  "requests_per_second": 24.09732468451819,
+  "questions_per_second": 24.09732468451819,
   "latency_ms": {
     "n": 72,
-    "mean": 36.75459891625602,
-    "p50": 40.21874499449041,
-    "p95": 43.1849047425203,
-    "p99": 44.94110917206854
+    "mean": 41.386164656109436,
+    "p50": 40.76293850084767,
+    "p95": 43.081331098801456,
+    "p99": 49.056057003326735
   },
   "adapter_ms": {
     "n": 72,
-    "mean": 35.392528403058854,
-    "p50": 38.920726990909316,
-    "p95": 41.830557797220536,
-    "p99": 43.6285329196835
+    "mean": 39.93616459774785,
+    "p50": 39.325255987932906,
+    "p95": 41.57378707895987,
+    "p99": 47.13945354276804
   },
   "input_tokens_per_request": {
     "n": 72,
@@ -36,216 +36,216 @@
     "jevbench-original/public": {
       "clean": {
         "n": 72,
-        "acc": 0.9583333333333334,
-        "nll": 0.11492584309106799,
-        "brier": 0.06535962884366993,
-        "ece": 0.011629480090899123,
-        "mean_conf": 0.963776698600166,
-        "confidence_bias": 0.005443365266832645,
+        "acc": 0.9444444444444444,
+        "nll": 0.1506676946978613,
+        "brier": 0.09628785114543494,
+        "ece": 0.05287211341987836,
+        "mean_conf": 0.9644579396263708,
+        "confidence_bias": 0.02001349518192641,
         "confident_error_rate": 0.013888888888888888,
-        "coverage_at_0_9": 0.875,
-        "accuracy_at_0_9": 0.9841269841269841,
-        "coverage_at_5pct_error": 1.0,
-        "coverage_at_1pct_error": 0.7638888888888888,
-        "score_mae": 0.013603815827533722,
-        "ranked_probability_score": 0.0002393091881253918
+        "coverage_at_0_9": 0.8888888888888888,
+        "accuracy_at_0_9": 0.984375,
+        "coverage_at_5pct_error": 0.9444444444444444,
+        "coverage_at_1pct_error": 0.8333333333333334,
+        "score_mae": 0.0030963336715575038,
+        "ranked_probability_score": 1.4381212165755811e-05
       },
       "tasks": {
         "adequacy": {
           "n": 12,
           "acc": 0.8333333333333334,
-          "nll": 0.44091135029727413,
-          "brier": 0.2778175268239488,
-          "ece": 0.12425451412140923,
-          "mean_conf": 0.9575878474547426,
-          "confidence_bias": 0.1242545141214092,
+          "nll": 0.4488440427906151,
+          "brier": 0.27590774015451475,
+          "ece": 0.12682459124420295,
+          "mean_conf": 0.9601579245775363,
+          "confidence_bias": 0.12682459124420298,
           "confident_error_rate": 0.08333333333333333,
           "coverage_at_0_9": 0.8333333333333334,
           "accuracy_at_0_9": 0.9,
-          "coverage_at_5pct_error": 0.5833333333333334,
-          "coverage_at_1pct_error": 0.5833333333333334
+          "coverage_at_5pct_error": 0.6666666666666666,
+          "coverage_at_1pct_error": 0.6666666666666666
         },
         "extraction": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.03245099834264279,
-          "brier": 0.003920405450885738,
-          "ece": 0.031062290242103975,
-          "mean_conf": 0.9689377097578961,
-          "confidence_bias": -0.03106229024210394,
+          "nll": 0.014790982169543275,
+          "brier": 0.0013193332236231595,
+          "ece": 0.014409816877868908,
+          "mean_conf": 0.9855901831221311,
+          "confidence_bias": -0.014409816877868908,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.8333333333333334,
+          "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0
         },
         "intent": {
           "n": 12,
-          "acc": 1.0,
-          "nll": 0.06717431933402711,
-          "brier": 0.029345064083642323,
-          "ece": 0.05757708210330111,
-          "mean_conf": 0.9424229178966987,
-          "confidence_bias": -0.05757708210330126,
+          "acc": 0.8333333333333334,
+          "nll": 0.37139225105893575,
+          "brier": 0.2739589955989285,
+          "ece": 0.20454111756095048,
+          "mean_conf": 0.9002844396047663,
+          "confidence_bias": 0.06695110627143297,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.75,
+          "coverage_at_0_9": 0.6666666666666666,
           "accuracy_at_0_9": 1.0,
-          "coverage_at_5pct_error": 1.0,
-          "coverage_at_1pct_error": 1.0
+          "coverage_at_5pct_error": 0.6666666666666666,
+          "coverage_at_1pct_error": 0.6666666666666666
         },
         "ordinal": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.018729915910625106,
-          "brier": 0.001480189394064871,
-          "ece": 0.018249236332484542,
-          "mean_conf": 0.9817507636675155,
-          "confidence_bias": -0.018249236332484542,
+          "nll": 0.0046678949914273635,
+          "brier": 9.40238580289126e-05,
+          "ece": 0.00463779150080279,
+          "mean_conf": 0.9953622084991972,
+          "confidence_bias": -0.00463779150080279,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0,
-          "score_mae": 0.013603815827533722,
-          "ranked_probability_score": 0.0002393091881253918
+          "score_mae": 0.0030963336715575038,
+          "ranked_probability_score": 1.4381212165755811e-05
         },
         "policy": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.027579876061385723,
-          "brier": 0.00850680022276106,
-          "ece": 0.025082901161826494,
-          "mean_conf": 0.9749170988381737,
-          "confidence_bias": -0.02508290116182632,
+          "nll": 0.007662766654381241,
+          "brier": 0.0003383572772372465,
+          "ece": 0.007576392619808026,
+          "mean_conf": 0.992423607380192,
+          "confidence_bias": -0.007576392619808026,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.9166666666666666,
+          "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0
         },
         "routing": {
           "n": 12,
-          "acc": 0.9166666666666666,
-          "nll": 0.10270859860045307,
-          "brier": 0.0710877870867168,
-          "ece": 0.06585300403564953,
-          "mean_conf": 0.9570438539859701,
-          "confidence_bias": 0.04037718731930351,
+          "acc": 1.0,
+          "nll": 0.05664823052226505,
+          "brier": 0.026108656760277058,
+          "ece": 0.04707072542559742,
+          "mean_conf": 0.9529292745744025,
+          "confidence_bias": -0.04707072542559754,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.9166666666666666,
+          "coverage_at_0_9": 0.8333333333333334,
           "accuracy_at_0_9": 1.0,
-          "coverage_at_5pct_error": 0.9166666666666666,
-          "coverage_at_1pct_error": 0.9166666666666666
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
         }
       },
       "sources": {
         "adequacy": {
           "n": 12,
           "acc": 0.8333333333333334,
-          "nll": 0.44091135029727413,
-          "brier": 0.2778175268239488,
-          "ece": 0.12425451412140923,
-          "mean_conf": 0.9575878474547426,
-          "confidence_bias": 0.1242545141214092,
+          "nll": 0.4488440427906151,
+          "brier": 0.27590774015451475,
+          "ece": 0.12682459124420295,
+          "mean_conf": 0.9601579245775363,
+          "confidence_bias": 0.12682459124420298,
           "confident_error_rate": 0.08333333333333333,
           "coverage_at_0_9": 0.8333333333333334,
           "accuracy_at_0_9": 0.9,
-          "coverage_at_5pct_error": 0.5833333333333334,
-          "coverage_at_1pct_error": 0.5833333333333334
+          "coverage_at_5pct_error": 0.6666666666666666,
+          "coverage_at_1pct_error": 0.6666666666666666
         },
         "extraction": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.03245099834264279,
-          "brier": 0.003920405450885738,
-          "ece": 0.031062290242103975,
-          "mean_conf": 0.9689377097578961,
-          "confidence_bias": -0.03106229024210394,
+          "nll": 0.014790982169543275,
+          "brier": 0.0013193332236231595,
+          "ece": 0.014409816877868908,
+          "mean_conf": 0.9855901831221311,
+          "confidence_bias": -0.014409816877868908,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.8333333333333334,
+          "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0
         },
         "intent": {
           "n": 12,
-          "acc": 1.0,
-          "nll": 0.06717431933402711,
-          "brier": 0.029345064083642323,
-          "ece": 0.05757708210330111,
-          "mean_conf": 0.9424229178966987,
-          "confidence_bias": -0.05757708210330126,
+          "acc": 0.8333333333333334,
+          "nll": 0.37139225105893575,
+          "brier": 0.2739589955989285,
+          "ece": 0.20454111756095048,
+          "mean_conf": 0.9002844396047663,
+          "confidence_bias": 0.06695110627143297,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.75,
+          "coverage_at_0_9": 0.6666666666666666,
           "accuracy_at_0_9": 1.0,
-          "coverage_at_5pct_error": 1.0,
-          "coverage_at_1pct_error": 1.0
+          "coverage_at_5pct_error": 0.6666666666666666,
+          "coverage_at_1pct_error": 0.6666666666666666
         },
         "ordinal": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.018729915910625106,
-          "brier": 0.001480189394064871,
-          "ece": 0.018249236332484542,
-          "mean_conf": 0.9817507636675155,
-          "confidence_bias": -0.018249236332484542,
+          "nll": 0.0046678949914273635,
+          "brier": 9.40238580289126e-05,
+          "ece": 0.00463779150080279,
+          "mean_conf": 0.9953622084991972,
+          "confidence_bias": -0.00463779150080279,
           "confident_error_rate": 0.0,
           "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0,
-          "score_mae": 0.013603815827533722,
-          "ranked_probability_score": 0.0002393091881253918
+          "score_mae": 0.0030963336715575038,
+          "ranked_probability_score": 1.4381212165755811e-05
         },
         "policy": {
           "n": 12,
           "acc": 1.0,
-          "nll": 0.027579876061385723,
-          "brier": 0.00850680022276106,
-          "ece": 0.025082901161826494,
-          "mean_conf": 0.9749170988381737,
-          "confidence_bias": -0.02508290116182632,
+          "nll": 0.007662766654381241,
+          "brier": 0.0003383572772372465,
+          "ece": 0.007576392619808026,
+          "mean_conf": 0.992423607380192,
+          "confidence_bias": -0.007576392619808026,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.9166666666666666,
+          "coverage_at_0_9": 1.0,
           "accuracy_at_0_9": 1.0,
           "coverage_at_5pct_error": 1.0,
           "coverage_at_1pct_error": 1.0
         },
         "routing": {
           "n": 12,
-          "acc": 0.9166666666666666,
-          "nll": 0.10270859860045307,
-          "brier": 0.0710877870867168,
-          "ece": 0.06585300403564953,
-          "mean_conf": 0.9570438539859701,
-          "confidence_bias": 0.04037718731930351,
+          "acc": 1.0,
+          "nll": 0.05664823052226505,
+          "brier": 0.026108656760277058,
+          "ece": 0.04707072542559742,
+          "mean_conf": 0.9529292745744025,
+          "confidence_bias": -0.04707072542559754,
           "confident_error_rate": 0.0,
-          "coverage_at_0_9": 0.9166666666666666,
+          "coverage_at_0_9": 0.8333333333333334,
           "accuracy_at_0_9": 1.0,
-          "coverage_at_5pct_error": 0.9166666666666666,
-          "coverage_at_1pct_error": 0.9166666666666666
+          "coverage_at_5pct_error": 1.0,
+          "coverage_at_1pct_error": 1.0
         }
       },
       "variants": {
         "clean": {
           "n": 72,
-          "acc": 0.9583333333333334,
-          "nll": 0.11492584309106799,
-          "brier": 0.06535962884366993,
-          "ece": 0.011629480090899123,
-          "mean_conf": 0.963776698600166,
-          "confidence_bias": 0.005443365266832645,
+          "acc": 0.9444444444444444,
+          "nll": 0.1506676946978613,
+          "brier": 0.09628785114543494,
+          "ece": 0.05287211341987836,
+          "mean_conf": 0.9644579396263708,
+          "confidence_bias": 0.02001349518192641,
           "confident_error_rate": 0.013888888888888888,
-          "coverage_at_0_9": 0.875,
-          "accuracy_at_0_9": 0.9841269841269841,
-          "coverage_at_5pct_error": 1.0,
-          "coverage_at_1pct_error": 0.7638888888888888,
-          "score_mae": 0.013603815827533722,
-          "ranked_probability_score": 0.0002393091881253918
+          "coverage_at_0_9": 0.8888888888888888,
+          "accuracy_at_0_9": 0.984375,
+          "coverage_at_5pct_error": 0.9444444444444444,
+          "coverage_at_1pct_error": 0.8333333333333334,
+          "score_mae": 0.0030963336715575038,
+          "ranked_probability_score": 1.4381212165755811e-05
         }
       },
-      "macro_task_acc": 0.9583333333333334,
-      "objective": -0.114925843091068,
+      "macro_task_acc": 0.9444444444444445,
+      "objective": -0.15066769469786132,
       "paired_flip": {
         "pairs": 0,
         "incomplete_pairs": 0,
@@ -275,10 +275,10 @@
       },
       "latency_ms": {
         "n": 72,
-        "mean": 36.75459891625602,
-        "p50": 40.21874499449041,
-        "p95": 43.1849047425203,
-        "p99": 44.94110917206854
+        "mean": 41.386164656109436,
+        "p50": 40.76293850084767,
+        "p95": 43.081331098801456,
+        "p99": 49.056057003326735
       }
     }
   },

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41c1d711d80f96f9cd5ca8eb8b0b4375addd3174b2021bd8ab72dc1a9c9c7500
+oid sha256:b5da315ae797d8f7c9add8bd08249968c5936df1cc3071f7fb34cfc0e91bc1f2
 size 4999724576

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:57d50fd02ebafbb491d35d445c8279dda89ad8ad5034edf7637971564857b48d
+oid sha256:4157651afa8d781ce6e4ba90c57bc4606e440a09ebe19ca0e14463b65f1a3076
 size 4999820896

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03616016aafb162714f7265f9e1a192f72bdc109e4d80d079c593fd0756c1cba
+oid sha256:655d65b5bb646889bccd46166dcc31c1e4e183958dd3a5688d6434a3299d2c58
 size 4915917688

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a77ea6878393937e46cead80bdfda98e090020cd93b746b89c38a5a81fbf823a
+oid sha256:a6c3828542f1e5fefdb0658131a6d06274b7abe11794e0d9747f451bf1525a34
 size 2920659992

release-manifest.json

@@ -5,19 +5,19 @@
     "revision": "f6fae9795746f63c9be8344932f01275f3c63734"
   },
   "adapter": {
-    "source_run": "final2-8b-long",
-    "adapter_model_sha256": "a2d9f02d5dd5d3ade3fab88145fc7321315609aa52d3847319dc4ec827f0c684"
+    "source_run": "hard25-8b-r1",
+    "adapter_model_sha256": "9551923c162970ea8bc2c3a5faf3ea3762b48591c9f460064e531d09427fa41a"
   },
   "merge_report_summary": {
     "changed_params": 238,
     "unchanged_params": 293,
     "changed_outside_lm_projections": [],
-    "max_abs_delta": 0.00201416015625
+    "max_abs_delta": 0.001983642578125
   },
   "serving": {
     "wording": "native",
     "system_prompt": null,
-    "temperature": 1.35,
+    "temperature": 1.65,
     "served_model_name": "standard-one-8b",
     "engine": "SGLang 0.5.20"
   }