Qwen3.5-2B-multiharness-RL / eval_results.json
AdithyaSK's picture
AdithyaSK HF Staff
Release step 500 with tokenizer, evaluation results and model card
ecdaa9c verified
Raw History Blame Contribute Delete
2.3 kB
{
"metric": "pass@1",
"complete": true,
"graded_cells": 1000,
"average_pass_at_1": 0.37000000000000005,
"harnesses": {
"opencode": {
"graded": 250,
"correct": 82.0,
"pass_at_1": 0.328,
"delta_from_base": 0.22000000000000003,
"difficulty": {
"easy": {
"graded": 33,
"correct": 23.0,
"pass_at_1": 0.696969696969697
},
"medium": {
"graded": 118,
"correct": 48.0,
"pass_at_1": 0.4067796610169492
},
"hard": {
"graded": 99,
"correct": 11.0,
"pass_at_1": 0.1111111111111111
}
}
},
"claude-code": {
"graded": 250,
"correct": 112.0,
"pass_at_1": 0.448,
"delta_from_base": 0.28,
"difficulty": {
"easy": {
"graded": 33,
"correct": 25.0,
"pass_at_1": 0.7575757575757576
},
"medium": {
"graded": 118,
"correct": 62.0,
"pass_at_1": 0.5254237288135594
},
"hard": {
"graded": 99,
"correct": 25.0,
"pass_at_1": 0.25252525252525254
}
}
},
"codex": {
"graded": 250,
"correct": 98.0,
"pass_at_1": 0.392,
"delta_from_base": 0.228,
"difficulty": {
"easy": {
"graded": 33,
"correct": 25.0,
"pass_at_1": 0.7575757575757576
},
"medium": {
"graded": 118,
"correct": 55.0,
"pass_at_1": 0.4661016949152542
},
"hard": {
"graded": 99,
"correct": 18.0,
"pass_at_1": 0.18181818181818182
}
}
},
"mini-swe-agent": {
"graded": 250,
"correct": 78.0,
"pass_at_1": 0.312,
"delta_from_base": 0.168,
"difficulty": {
"easy": {
"graded": 33,
"correct": 23.0,
"pass_at_1": 0.696969696969697
},
"medium": {
"graded": 118,
"correct": 44.0,
"pass_at_1": 0.3728813559322034
},
"hard": {
"graded": 99,
"correct": 11.0,
"pass_at_1": 0.1111111111111111
}
}
}
},
"tito_pass": true,
"comparison_ready": true
}