{ "metric": "pass@1", "complete": true, "graded_cells": 1000, "average_pass_at_1": 0.37000000000000005, "harnesses": { "opencode": { "graded": 250, "correct": 82.0, "pass_at_1": 0.328, "delta_from_base": 0.22000000000000003, "difficulty": { "easy": { "graded": 33, "correct": 23.0, "pass_at_1": 0.696969696969697 }, "medium": { "graded": 118, "correct": 48.0, "pass_at_1": 0.4067796610169492 }, "hard": { "graded": 99, "correct": 11.0, "pass_at_1": 0.1111111111111111 } } }, "claude-code": { "graded": 250, "correct": 112.0, "pass_at_1": 0.448, "delta_from_base": 0.28, "difficulty": { "easy": { "graded": 33, "correct": 25.0, "pass_at_1": 0.7575757575757576 }, "medium": { "graded": 118, "correct": 62.0, "pass_at_1": 0.5254237288135594 }, "hard": { "graded": 99, "correct": 25.0, "pass_at_1": 0.25252525252525254 } } }, "codex": { "graded": 250, "correct": 98.0, "pass_at_1": 0.392, "delta_from_base": 0.228, "difficulty": { "easy": { "graded": 33, "correct": 25.0, "pass_at_1": 0.7575757575757576 }, "medium": { "graded": 118, "correct": 55.0, "pass_at_1": 0.4661016949152542 }, "hard": { "graded": 99, "correct": 18.0, "pass_at_1": 0.18181818181818182 } } }, "mini-swe-agent": { "graded": 250, "correct": 78.0, "pass_at_1": 0.312, "delta_from_base": 0.168, "difficulty": { "easy": { "graded": 33, "correct": 23.0, "pass_at_1": 0.696969696969697 }, "medium": { "graded": 118, "correct": 44.0, "pass_at_1": 0.3728813559322034 }, "hard": { "graded": 99, "correct": 11.0, "pass_at_1": 0.1111111111111111 } } } }, "tito_pass": true, "comparison_ready": true }