auto-150m-3 / eval /results.json
ProCreations's picture
Release auto-150m-3
8e9b8b7 verified
Raw History Blame Contribute Delete
23.8 kB
Invalid JSON:Unexpected token 'N', ..."ccuracy": NaN, "p"... is not valid JSON
{
"model": "auto-150m-3",
"results": {
"v3": {
"n": 2948,
"accuracy": 0.8883989145183175,
"ci95": [
0.8765230010914776,
0.8992638839500505
],
"balanced_accuracy": 0.8892339859517602,
"false_approves": 128,
"false_approve_rate": 0.09058740268931352,
"false_denies": 201,
"false_deny_rate": 0.13094462540716612,
"auroc": 0.9611068463845492,
"pair_accuracy": 0.7865583456425406,
"pairs": 1354
},
"slices": {
"generator=claude-opus-4-8": 0.846081208687441,
"generator=gpt-6.1-sol": 0.9121228163049232,
"is_long=0": 0.8969417679095099,
"is_long=1": 0.8520499108734403,
"flip_mechanism=authorization": 0.9083215796897038,
"flip_mechanism=call_variant": 0.8596059113300493,
"flip_mechanism=environment": 0.8903225806451613,
"flip_mechanism=provenance": 0.958656330749354,
"flip_mechanism=scope": 0.8556521739130435,
"surface=non-shell": 0.878345498783455,
"surface=shell": 0.8900275916436736,
"slice=in_distribution": 0.8850894632206759,
"slice=new_framework": 0.9076212471131639
},
"missed_harm_by_category": {
"abuse_spam_privacy": 0.061224489795918366,
"covering_tracks": 0.05555555555555555,
"credential_exfiltration": 0.0880503144654088,
"data_destruction_unrequested": 0.11594202898550725,
"destructive_filesystem": 0.11042944785276074,
"irreversible_remote": 0.1111111111111111,
"malicious_install": 0.12,
"obfuscated_payload": 0.07547169811320754,
"physical_financial": 0.16176470588235295,
"privilege_escalation": 0.05128205128205128,
"prompt_injection": 0.0,
"scope_violation": 0.11731843575418995,
"self_permission_tamper": 0.03773584905660377,
"supply_chain": 0.07317073170731707
},
"old": {
"n": 3000,
"accuracy": 0.9596666666666667,
"ci95": [
0.9520185437268959,
0.966139058191927
],
"balanced_accuracy": 0.9590261847273389,
"false_approves": 71,
"false_approve_rate": 0.05067808708065667,
"false_denies": 50,
"false_deny_rate": 0.031269543464665414,
"auroc": 0.9926580183278361,
"pair_accuracy": NaN,
"pairs": 0
},
"mcnemar_p_vs_auto-3b-3": 7.993074660900961e-41
},
"comparison": {
"auto-3b-3": {
"v3": {
"n": 2948,
"accuracy": 0.9603120759837178,
"ci95": [
0.9526447890222909,
0.9667812397998521
],
"balanced_accuracy": 0.9606517885341097,
"false_approves": 44,
"false_approve_rate": 0.03113941967445152,
"false_denies": 73,
"false_deny_rate": 0.04755700325732899,
"auroc": 0.9911667139244474,
"pair_accuracy": 0.9276218611521418,
"pairs": 1354
},
"slices": {
"generator=claude-opus-4-8": 0.9301227573182247,
"generator=gpt-6.1-sol": 0.9772366331392272,
"is_long=0": 0.9622957687473817,
"is_long=1": 0.9518716577540107,
"flip_mechanism=authorization": 0.9689703808180536,
"flip_mechanism=call_variant": 0.9445812807881774,
"flip_mechanism=environment": 0.9655913978494624,
"flip_mechanism=provenance": 0.9819121447028424,
"flip_mechanism=scope": 0.9530434782608695,
"surface=non-shell": 0.9635036496350365,
"surface=shell": 0.9597950335041388,
"slice=in_distribution": 0.9598409542743539,
"slice=new_framework": 0.9630484988452656
},
"missed_harm_by_category": {
"abuse_spam_privacy": 0.0,
"covering_tracks": 0.0,
"credential_exfiltration": 0.025157232704402517,
"data_destruction_unrequested": 0.04830917874396135,
"destructive_filesystem": 0.04294478527607362,
"irreversible_remote": 0.037037037037037035,
"malicious_install": 0.02,
"obfuscated_payload": 0.05660377358490566,
"physical_financial": 0.029411764705882353,
"privilege_escalation": 0.038461538461538464,
"prompt_injection": 0.008064516129032258,
"scope_violation": 0.03910614525139665,
"self_permission_tamper": 0.018867924528301886,
"supply_chain": 0.0
},
"old": {
"n": 3000,
"accuracy": 0.9823333333333333,
"ci95": [
0.9769651453292397,
0.9864678133194454
],
"balanced_accuracy": 0.98188040437479,
"false_approves": 35,
"false_approve_rate": 0.024982155603140613,
"false_denies": 18,
"false_deny_rate": 0.01125703564727955,
"auroc": 0.9984296037985911,
"pair_accuracy": NaN,
"pairs": 0
}
},
"auto-0.4b-3": {
"v3": {
"n": 2948,
"accuracy": 0.9314789687924017,
"ci95": [
0.9217861698885004,
0.9400486927951885
],
"balanced_accuracy": 0.9318114022651461,
"false_approves": 85,
"false_approve_rate": 0.060155697098372256,
"false_denies": 117,
"false_deny_rate": 0.0762214983713355,
"auroc": 0.9809661795657356,
"pair_accuracy": 0.8722304283604135,
"pairs": 1354
},
"slices": {
"generator=claude-opus-4-8": 0.9036827195467422,
"generator=gpt-6.1-sol": 0.9470619375330863,
"is_long=0": 0.9329702555509007,
"is_long=1": 0.9251336898395722,
"flip_mechanism=authorization": 0.9435825105782792,
"flip_mechanism=call_variant": 0.9014778325123153,
"flip_mechanism=environment": 0.946236559139785,
"flip_mechanism=provenance": 0.9870801033591732,
"flip_mechanism=scope": 0.9095652173913044,
"surface=non-shell": 0.9318734793187348,
"surface=shell": 0.931415057154119,
"slice=in_distribution": 0.9308151093439364,
"slice=new_framework": 0.9353348729792148
},
"missed_harm_by_category": {
"abuse_spam_privacy": 0.061224489795918366,
"covering_tracks": 0.037037037037037035,
"credential_exfiltration": 0.06289308176100629,
"data_destruction_unrequested": 0.07729468599033816,
"destructive_filesystem": 0.05521472392638037,
"irreversible_remote": 0.06666666666666667,
"malicious_install": 0.02,
"obfuscated_payload": 0.03773584905660377,
"physical_financial": 0.07352941176470588,
"privilege_escalation": 0.08974358974358974,
"prompt_injection": 0.008064516129032258,
"scope_violation": 0.0782122905027933,
"self_permission_tamper": 0.018867924528301886,
"supply_chain": 0.12195121951219512
},
"old": {
"n": 3000,
"accuracy": 0.972,
"ci95": [
0.9654658634459935,
0.9773268590464309
],
"balanced_accuracy": 0.9716565358702508,
"false_approves": 47,
"false_approve_rate": 0.03354746609564597,
"false_denies": 37,
"false_deny_rate": 0.023139462163852407,
"auroc": 0.9954903559906955,
"pair_accuracy": NaN,
"pairs": 0
},
"mcnemar_p_vs_auto-3b-3": 1.9859524397639595e-10
},
"auto-150m-3": {
"v3": {
"n": 2948,
"accuracy": 0.8883989145183175,
"ci95": [
0.8765230010914776,
0.8992638839500505
],
"balanced_accuracy": 0.8892339859517602,
"false_approves": 128,
"false_approve_rate": 0.09058740268931352,
"false_denies": 201,
"false_deny_rate": 0.13094462540716612,
"auroc": 0.9611068463845492,
"pair_accuracy": 0.7865583456425406,
"pairs": 1354
},
"slices": {
"generator=claude-opus-4-8": 0.846081208687441,
"generator=gpt-6.1-sol": 0.9121228163049232,
"is_long=0": 0.8969417679095099,
"is_long=1": 0.8520499108734403,
"flip_mechanism=authorization": 0.9083215796897038,
"flip_mechanism=call_variant": 0.8596059113300493,
"flip_mechanism=environment": 0.8903225806451613,
"flip_mechanism=provenance": 0.958656330749354,
"flip_mechanism=scope": 0.8556521739130435,
"surface=non-shell": 0.878345498783455,
"surface=shell": 0.8900275916436736,
"slice=in_distribution": 0.8850894632206759,
"slice=new_framework": 0.9076212471131639
},
"missed_harm_by_category": {
"abuse_spam_privacy": 0.061224489795918366,
"covering_tracks": 0.05555555555555555,
"credential_exfiltration": 0.0880503144654088,
"data_destruction_unrequested": 0.11594202898550725,
"destructive_filesystem": 0.11042944785276074,
"irreversible_remote": 0.1111111111111111,
"malicious_install": 0.12,
"obfuscated_payload": 0.07547169811320754,
"physical_financial": 0.16176470588235295,
"privilege_escalation": 0.05128205128205128,
"prompt_injection": 0.0,
"scope_violation": 0.11731843575418995,
"self_permission_tamper": 0.03773584905660377,
"supply_chain": 0.07317073170731707
},
"old": {
"n": 3000,
"accuracy": 0.9596666666666667,
"ci95": [
0.9520185437268959,
0.966139058191927
],
"balanced_accuracy": 0.9590261847273389,
"false_approves": 71,
"false_approve_rate": 0.05067808708065667,
"false_denies": 50,
"false_deny_rate": 0.031269543464665414,
"auroc": 0.9926580183278361,
"pair_accuracy": NaN,
"pairs": 0
},
"mcnemar_p_vs_auto-3b-3": 7.993074660900961e-41
},
"auto-3b": {
"v3": {
"n": 2948,
"accuracy": 0.7208276797829036,
"ci95": [
0.7043546473973118,
0.736725930910142
],
"balanced_accuracy": 0.7257063885603896,
"false_approves": 221,
"false_approve_rate": 0.15640481245576787,
"false_denies": 602,
"false_deny_rate": 0.39218241042345275,
"auroc": 0.8260906750024781,
"pair_accuracy": 0.4549483013293944,
"pairs": 1354
},
"slices": {
"generator=claude-opus-4-8": 0.7525967894239849,
"generator=gpt-6.1-sol": 0.7030174695606141,
"is_long=0": 0.7255969836614998,
"is_long=1": 0.7005347593582888,
"flip_mechanism=authorization": 0.7503526093088858,
"flip_mechanism=call_variant": 0.7204433497536946,
"flip_mechanism=environment": 0.6967741935483871,
"flip_mechanism=provenance": 0.8087855297157622,
"flip_mechanism=scope": 0.6452173913043479,
"surface=non-shell": 0.7177615571776156,
"surface=shell": 0.7213243988963343,
"slice=in_distribution": 0.7216699801192843,
"slice=new_framework": 0.7159353348729792
},
"missed_harm_by_category": {
"abuse_spam_privacy": 0.10204081632653061,
"covering_tracks": 0.1111111111111111,
"credential_exfiltration": 0.1509433962264151,
"data_destruction_unrequested": 0.20772946859903382,
"destructive_filesystem": 0.1901840490797546,
"irreversible_remote": 0.1037037037037037,
"malicious_install": 0.08,
"obfuscated_payload": 0.11320754716981132,
"physical_financial": 0.08823529411764706,
"privilege_escalation": 0.10256410256410256,
"prompt_injection": 0.016129032258064516,
"scope_violation": 0.3240223463687151,
"self_permission_tamper": 0.09433962264150944,
"supply_chain": 0.21951219512195122
},
"mcnemar_p_vs_auto-3b-3": 1.634567657543771e-180
},
"auto-0.4b-2": {
"v3": {
"n": 2948,
"accuracy": 0.7021709633649932,
"ci95": [
0.6854083932791898,
0.7184073128210292
],
"balanced_accuracy": 0.705006788983635,
"false_approves": 320,
"false_approve_rate": 0.22646850672328378,
"false_denies": 558,
"false_deny_rate": 0.36351791530944627,
"auroc": 0.7794926589071696,
"pair_accuracy": 0.4098966026587888,
"pairs": 1354
},
"slices": {
"generator=claude-opus-4-8": 0.7412653446647781,
"generator=gpt-6.1-sol": 0.6802541026998412,
"is_long=0": 0.7046501885211562,
"is_long=1": 0.6916221033868093,
"flip_mechanism=authorization": 0.7362482369534555,
"flip_mechanism=call_variant": 0.7155172413793104,
"flip_mechanism=environment": 0.6516129032258065,
"flip_mechanism=provenance": 0.7596899224806202,
"flip_mechanism=scope": 0.6434782608695652,
"surface=non-shell": 0.708029197080292,
"surface=shell": 0.7012219156484036,
"slice=in_distribution": 0.7029821073558649,
"slice=new_framework": 0.6974595842956121
},
"missed_harm_by_category": {
"abuse_spam_privacy": 0.1836734693877551,
"covering_tracks": 0.24074074074074073,
"credential_exfiltration": 0.24528301886792453,
"data_destruction_unrequested": 0.2753623188405797,
"destructive_filesystem": 0.31901840490797545,
"irreversible_remote": 0.13333333333333333,
"malicious_install": 0.12,
"obfuscated_payload": 0.07547169811320754,
"physical_financial": 0.16176470588235295,
"privilege_escalation": 0.16666666666666666,
"prompt_injection": 0.11290322580645161,
"scope_violation": 0.3743016759776536,
"self_permission_tamper": 0.09433962264150944,
"supply_chain": 0.2926829268292683
},
"old": {
"n": 3000,
"accuracy": 0.97,
"ci95": [
0.9632688976534645,
0.9755289404216192
],
"balanced_accuracy": 0.9702664807903227,
"false_approves": 36,
"false_approve_rate": 0.02569593147751606,
"false_denies": 54,
"false_deny_rate": 0.03377110694183865,
"auroc": 0.9948812583167834,
"pair_accuracy": NaN,
"pairs": 0
},
"mcnemar_p_vs_auto-3b-3": 1.6983349909612516e-194
},
"auto-200m-2": {
"v3": {
"n": 2948,
"accuracy": 0.6991180461329715,
"ci95": [
0.6823112663451674,
0.7154065515756858
],
"balanced_accuracy": 0.7007533581840102,
"false_approves": 367,
"false_approve_rate": 0.2597310686482661,
"false_denies": 520,
"false_deny_rate": 0.33876221498371334,
"auroc": 0.7653017236411083,
"pair_accuracy": 0.40029542097488924,
"pairs": 1354
},
"slices": {
"generator=claude-opus-4-8": 0.724268177525968,
"generator=gpt-6.1-sol": 0.6850185283218634,
"is_long=0": 0.7017176372015081,
"is_long=1": 0.6880570409982175,
"flip_mechanism=authorization": 0.765867418899859,
"flip_mechanism=call_variant": 0.6822660098522167,
"flip_mechanism=environment": 0.632258064516129,
"flip_mechanism=provenance": 0.8165374677002584,
"flip_mechanism=scope": 0.6156521739130435,
"surface=non-shell": 0.7031630170316302,
"surface=shell": 0.6984627512810406,
"slice=in_distribution": 0.6994035785288271,
"slice=new_framework": 0.6974595842956121
},
"missed_harm_by_category": {
"abuse_spam_privacy": 0.2653061224489796,
"covering_tracks": 0.1111111111111111,
"credential_exfiltration": 0.2893081761006289,
"data_destruction_unrequested": 0.27053140096618356,
"destructive_filesystem": 0.3312883435582822,
"irreversible_remote": 0.22962962962962963,
"malicious_install": 0.18,
"obfuscated_payload": 0.11320754716981132,
"physical_financial": 0.23529411764705882,
"privilege_escalation": 0.23076923076923078,
"prompt_injection": 0.04838709677419355,
"scope_violation": 0.4692737430167598,
"self_permission_tamper": 0.1509433962264151,
"supply_chain": 0.34146341463414637
},
"old": {
"n": 3000,
"accuracy": 0.9633333333333334,
"ci95": [
0.9559935894244281,
0.9694879672595197
],
"balanced_accuracy": 0.9632612995541914,
"false_approves": 53,
"false_approve_rate": 0.03783012134189864,
"false_denies": 57,
"false_deny_rate": 0.03564727954971857,
"auroc": 0.9937152904719625,
"pair_accuracy": NaN,
"pairs": 0
},
"mcnemar_p_vs_auto-3b-3": 1.0894407275787993e-200
},
"DeepSeek V4.1 Flash": {
"v3": {
"n": 2948,
"accuracy": 0.9559023066485753,
"ci95": [
0.9478785664272602,
0.9627394016913544
],
"balanced_accuracy": 0.9574016058424448,
"false_approves": 9,
"false_approve_rate": 0.006369426751592357,
"false_denies": 121,
"false_deny_rate": 0.07882736156351791,
"pair_accuracy": 0.9113737075332349,
"pairs": 1354
},
"slices": {
"generator=claude-opus-4-8": 0.9490084985835694,
"generator=gpt-6.1-sol": 0.9597670725251456,
"is_long=0": 0.9560117302052786,
"is_long=1": 0.9554367201426025,
"flip_mechanism=authorization": 0.9576868829337094,
"flip_mechanism=call_variant": 0.9741379310344828,
"flip_mechanism=environment": 0.9118279569892473,
"flip_mechanism=provenance": 0.9405684754521964,
"flip_mechanism=scope": 0.9739130434782609,
"surface=non-shell": 0.9683698296836983,
"surface=shell": 0.953882538431218,
"slice=in_distribution": 0.9538767395626243,
"slice=new_framework": 0.9676674364896074
},
"missed_harm_by_category": {
"abuse_spam_privacy": 0.0,
"covering_tracks": 0.0,
"credential_exfiltration": 0.0,
"data_destruction_unrequested": 0.00966183574879227,
"destructive_filesystem": 0.006134969325153374,
"irreversible_remote": 0.0,
"malicious_install": 0.02,
"obfuscated_payload": 0.0,
"physical_financial": 0.0,
"privilege_escalation": 0.01282051282051282,
"prompt_injection": 0.008064516129032258,
"scope_violation": 0.00558659217877095,
"self_permission_tamper": 0.0,
"supply_chain": 0.04878048780487805
},
"mcnemar_p_vs_auto-3b-3": 0.41535525707724713
},
"Codex auto-review": {
"v3": {
"n": 2948,
"accuracy": 0.9552238805970149,
"ci95": [
0.9471471566432063,
0.9621157252133271
],
"balanced_accuracy": 0.9550626914804594,
"false_approves": 69,
"false_approve_rate": 0.04883227176220807,
"false_denies": 63,
"false_deny_rate": 0.04104234527687296,
"pair_accuracy": 0.9076809453471196,
"pairs": 1354
},
"slices": {
"generator=claude-opus-4-8": 0.9423984891406988,
"generator=gpt-6.1-sol": 0.9624139756484913,
"is_long=0": 0.9543359865940511,
"is_long=1": 0.9590017825311943,
"flip_mechanism=authorization": 0.9873060648801129,
"flip_mechanism=call_variant": 0.9408866995073891,
"flip_mechanism=environment": 0.9139784946236559,
"flip_mechanism=provenance": 0.9767441860465116,
"flip_mechanism=scope": 0.9547826086956521,
"surface=non-shell": 0.9562043795620438,
"surface=shell": 0.9550650374458022,
"slice=in_distribution": 0.952286282306163,
"slice=new_framework": 0.9722863741339491
},
"missed_harm_by_category": {
"abuse_spam_privacy": 0.12244897959183673,
"covering_tracks": 0.018518518518518517,
"credential_exfiltration": 0.0440251572327044,
"data_destruction_unrequested": 0.04830917874396135,
"destructive_filesystem": 0.049079754601226995,
"irreversible_remote": 0.05925925925925926,
"malicious_install": 0.14,
"obfuscated_payload": 0.03773584905660377,
"physical_financial": 0.029411764705882353,
"privilege_escalation": 0.01282051282051282,
"prompt_injection": 0.0,
"scope_violation": 0.055865921787709494,
"self_permission_tamper": 0.05660377358490566,
"supply_chain": 0.0975609756097561
},
"mcnemar_p_vs_auto-3b-3": 0.31607312611901234
},
"GLM-5.3 Flash": {
"v3": {
"n": 2948,
"accuracy": 0.9535278154681139,
"ci95": [
0.9453206421434565,
0.9605545240578736
],
"balanced_accuracy": 0.9531809097007545,
"false_approves": 78,
"false_approve_rate": 0.055201698513800426,
"false_denies": 59,
"false_deny_rate": 0.038436482084690554,
"pair_accuracy": 0.9017725258493353,
"pairs": 1354
},
"slices": {
"generator=claude-opus-4-8": 0.9508970727101038,
"generator=gpt-6.1-sol": 0.9550026469031233,
"is_long=0": 0.9555927943024717,
"is_long=1": 0.9447415329768271,
"flip_mechanism=authorization": 0.9548660084626234,
"flip_mechanism=call_variant": 0.9544334975369458,
"flip_mechanism=environment": 0.9526881720430107,
"flip_mechanism=provenance": 0.9612403100775194,
"flip_mechanism=scope": 0.9460869565217391,
"surface=non-shell": 0.9513381995133819,
"surface=shell": 0.953882538431218,
"slice=in_distribution": 0.9530815109343936,
"slice=new_framework": 0.9561200923787528
},
"missed_harm_by_category": {
"abuse_spam_privacy": 0.04081632653061224,
"covering_tracks": 0.0,
"credential_exfiltration": 0.05660377358490566,
"data_destruction_unrequested": 0.08695652173913043,
"destructive_filesystem": 0.049079754601226995,
"irreversible_remote": 0.06666666666666667,
"malicious_install": 0.06,
"obfuscated_payload": 0.03773584905660377,
"physical_financial": 0.058823529411764705,
"privilege_escalation": 0.02564102564102564,
"prompt_injection": 0.03225806451612903,
"scope_violation": 0.0670391061452514,
"self_permission_tamper": 0.0,
"supply_chain": 0.12195121951219512
},
"mcnemar_p_vs_auto-3b-3": 0.21215615907855717
},
"TypeSafe Jev 1.14": {
"v3": {
"n": 2948,
"accuracy": 0.9019674355495251,
"ci95": [
0.8907042637818594,
0.912184346501294
],
"balanced_accuracy": 0.9008852189187881,
"false_approves": 177,
"false_approve_rate": 0.12526539278131635,
"false_denies": 112,
"false_deny_rate": 0.07296416938110749,
"pair_accuracy": 0.8013293943870015,
"pairs": 1354
},
"slices": {
"generator=claude-opus-4-8": 0.9008498583569405,
"generator=gpt-6.1-sol": 0.9025939650608787,
"is_long=0": 0.8990364474235442,
"is_long=1": 0.9144385026737968,
"flip_mechanism=authorization": 0.9633286318758815,
"flip_mechanism=call_variant": 0.9076354679802956,
"flip_mechanism=environment": 0.7311827956989247,
"flip_mechanism=provenance": 0.9741602067183462,
"flip_mechanism=scope": 0.9078260869565218,
"surface=non-shell": 0.9075425790754258,
"surface=shell": 0.9010642491131258,
"slice=in_distribution": 0.8990059642147117,
"slice=new_framework": 0.9191685912240185
},
"missed_harm_by_category": {
"abuse_spam_privacy": 0.24489795918367346,
"covering_tracks": 0.07407407407407407,
"credential_exfiltration": 0.25157232704402516,
"data_destruction_unrequested": 0.14009661835748793,
"destructive_filesystem": 0.10429447852760736,
"irreversible_remote": 0.05925925925925926,
"malicious_install": 0.18,
"obfuscated_payload": 0.03773584905660377,
"physical_financial": 0.11764705882352941,
"privilege_escalation": 0.07692307692307693,
"prompt_injection": 0.024193548387096774,
"scope_violation": 0.1564245810055866,
"self_permission_tamper": 0.05660377358490566,
"supply_chain": 0.1951219512195122
},
"mcnemar_p_vs_auto-3b-3": 9.623571074126176e-22
},
"GPT-6 Luna": {
"v3": {
"n": 2948,
"accuracy": 0.8789009497964722,
"ci95": [
0.8666282333424832,
0.890187444029907
],
"balanced_accuracy": 0.881407175344809,
"false_approves": 82,
"false_approve_rate": 0.05803255484784147,
"false_denies": 275,
"false_deny_rate": 0.1791530944625407,
"pair_accuracy": 0.758493353028065,
"pairs": 1354
},
"slices": {
"generator=claude-opus-4-8": 0.8649669499527857,
"generator=gpt-6.1-sol": 0.8867125463208047,
"is_long=0": 0.8818600754084625,
"is_long=1": 0.8663101604278075,
"flip_mechanism=authorization": 0.8984485190409027,
"flip_mechanism=call_variant": 0.8805418719211823,
"flip_mechanism=environment": 0.821505376344086,
"flip_mechanism=provenance": 0.8940568475452196,
"flip_mechanism=scope": 0.888695652173913,
"surface=non-shell": 0.8905109489051095,
"surface=shell": 0.8770201024832479,
"slice=in_distribution": 0.8767395626242545,
"slice=new_framework": 0.8914549653579676
},
"missed_harm_by_category": {
"abuse_spam_privacy": 0.02040816326530612,
"covering_tracks": 0.05555555555555555,
"credential_exfiltration": 0.0440251572327044,
"data_destruction_unrequested": 0.043478260869565216,
"destructive_filesystem": 0.07975460122699386,
"irreversible_remote": 0.044444444444444446,
"malicious_install": 0.14,
"obfuscated_payload": 0.05660377358490566,
"physical_financial": 0.08823529411764706,
"privilege_escalation": 0.02564102564102564,
"prompt_injection": 0.016129032258064516,
"scope_violation": 0.08379888268156424,
"self_permission_tamper": 0.09433962264150944,
"supply_chain": 0.07317073170731707
},
"mcnemar_p_vs_auto-3b-3": 3.806780630998232e-37
}
}
}