| { |
| "platform": "macOS-27.0-arm64-arm-64bit", |
| "processor": "arm", |
| "torch": "2.13.0", |
| "loader": "auto_quant.load(device=..., attention=\"linear\") + score()", |
| "devices": { |
| "cpu": { |
| "dtype": "torch.float32", |
| "items": 2452, |
| "max_tokens": 2040, |
| "selection": "the 2452 items with <= 2048 tokens", |
| "accuracy": 0.9645187601957586, |
| "cuda_accuracy_same_items": 0.964110929853181, |
| "correct": 2365, |
| "cuda_correct_same_items": 2364, |
| "decisions_differing_from_cuda": 3, |
| "max_abs_p_deny_difference_vs_cuda": 0.042745436535326004, |
| "load_seconds": 2.06, |
| "rss_after_load_gb": 0.293, |
| "peak_rss_gb": 1.18, |
| "peak_mps_driver_gb": null, |
| "median_ms_under_1k_tokens": 147.23, |
| "seconds_total": 418.8 |
| }, |
| "mps": { |
| "dtype": "torch.float32", |
| "items": 2761, |
| "max_tokens": 14768, |
| "selection": "the 2761 items with <= 16384 tokens", |
| "accuracy": 0.9626946758420862, |
| "cuda_accuracy_same_items": 0.9623324882289026, |
| "correct": 2658, |
| "cuda_correct_same_items": 2657, |
| "decisions_differing_from_cuda": 3, |
| "max_abs_p_deny_difference_vs_cuda": 0.042747880325761795, |
| "load_seconds": 2.62, |
| "rss_after_load_gb": 0.389, |
| "peak_rss_gb": 2.252, |
| "peak_mps_driver_gb": 1.135, |
| "median_ms_under_1k_tokens": 42.03, |
| "seconds_total": 345.6 |
| } |
| } |
| } |
