Text Classification
Transformers
Safetensors
modernbert
agent-safety
tool-calling
guardrail
long-context
Eval Results (legacy)
text-embeddings-inference
Instructions to use ProCreations/auto-150m-3 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use ProCreations/auto-150m-3 with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="ProCreations/auto-150m-3")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForSequenceClassification tokenizer = AutoTokenizer.from_pretrained("ProCreations/auto-150m-3") model = AutoModelForSequenceClassification.from_pretrained("ProCreations/auto-150m-3", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download eval/results.json from ProCreations/auto-150m-3: direct link, hf CLI and curl.
- Browser
- Download file 23.8 kB
-
https://huggingface.co/ProCreations/auto-150m-3/resolve/main/eval/results.json
- Command line
-
hf download hf://ProCreations/auto-150m-3/eval/results.json
-
curl -L -o results.json https://huggingface.co/ProCreations/auto-150m-3/resolve/main/eval/results.json
23.8 kB
Invalid JSON:Unexpected token 'N', ..."ccuracy": NaN,
"p"... is not valid JSON
| { | |
| "model": "auto-150m-3", | |
| "results": { | |
| "v3": { | |
| "n": 2948, | |
| "accuracy": 0.8883989145183175, | |
| "ci95": [ | |
| 0.8765230010914776, | |
| 0.8992638839500505 | |
| ], | |
| "balanced_accuracy": 0.8892339859517602, | |
| "false_approves": 128, | |
| "false_approve_rate": 0.09058740268931352, | |
| "false_denies": 201, | |
| "false_deny_rate": 0.13094462540716612, | |
| "auroc": 0.9611068463845492, | |
| "pair_accuracy": 0.7865583456425406, | |
| "pairs": 1354 | |
| }, | |
| "slices": { | |
| "generator=claude-opus-4-8": 0.846081208687441, | |
| "generator=gpt-6.1-sol": 0.9121228163049232, | |
| "is_long=0": 0.8969417679095099, | |
| "is_long=1": 0.8520499108734403, | |
| "flip_mechanism=authorization": 0.9083215796897038, | |
| "flip_mechanism=call_variant": 0.8596059113300493, | |
| "flip_mechanism=environment": 0.8903225806451613, | |
| "flip_mechanism=provenance": 0.958656330749354, | |
| "flip_mechanism=scope": 0.8556521739130435, | |
| "surface=non-shell": 0.878345498783455, | |
| "surface=shell": 0.8900275916436736, | |
| "slice=in_distribution": 0.8850894632206759, | |
| "slice=new_framework": 0.9076212471131639 | |
| }, | |
| "missed_harm_by_category": { | |
| "abuse_spam_privacy": 0.061224489795918366, | |
| "covering_tracks": 0.05555555555555555, | |
| "credential_exfiltration": 0.0880503144654088, | |
| "data_destruction_unrequested": 0.11594202898550725, | |
| "destructive_filesystem": 0.11042944785276074, | |
| "irreversible_remote": 0.1111111111111111, | |
| "malicious_install": 0.12, | |
| "obfuscated_payload": 0.07547169811320754, | |
| "physical_financial": 0.16176470588235295, | |
| "privilege_escalation": 0.05128205128205128, | |
| "prompt_injection": 0.0, | |
| "scope_violation": 0.11731843575418995, | |
| "self_permission_tamper": 0.03773584905660377, | |
| "supply_chain": 0.07317073170731707 | |
| }, | |
| "old": { | |
| "n": 3000, | |
| "accuracy": 0.9596666666666667, | |
| "ci95": [ | |
| 0.9520185437268959, | |
| 0.966139058191927 | |
| ], | |
| "balanced_accuracy": 0.9590261847273389, | |
| "false_approves": 71, | |
| "false_approve_rate": 0.05067808708065667, | |
| "false_denies": 50, | |
| "false_deny_rate": 0.031269543464665414, | |
| "auroc": 0.9926580183278361, | |
| "pair_accuracy": NaN, | |
| "pairs": 0 | |
| }, | |
| "mcnemar_p_vs_auto-3b-3": 7.993074660900961e-41 | |
| }, | |
| "comparison": { | |
| "auto-3b-3": { | |
| "v3": { | |
| "n": 2948, | |
| "accuracy": 0.9603120759837178, | |
| "ci95": [ | |
| 0.9526447890222909, | |
| 0.9667812397998521 | |
| ], | |
| "balanced_accuracy": 0.9606517885341097, | |
| "false_approves": 44, | |
| "false_approve_rate": 0.03113941967445152, | |
| "false_denies": 73, | |
| "false_deny_rate": 0.04755700325732899, | |
| "auroc": 0.9911667139244474, | |
| "pair_accuracy": 0.9276218611521418, | |
| "pairs": 1354 | |
| }, | |
| "slices": { | |
| "generator=claude-opus-4-8": 0.9301227573182247, | |
| "generator=gpt-6.1-sol": 0.9772366331392272, | |
| "is_long=0": 0.9622957687473817, | |
| "is_long=1": 0.9518716577540107, | |
| "flip_mechanism=authorization": 0.9689703808180536, | |
| "flip_mechanism=call_variant": 0.9445812807881774, | |
| "flip_mechanism=environment": 0.9655913978494624, | |
| "flip_mechanism=provenance": 0.9819121447028424, | |
| "flip_mechanism=scope": 0.9530434782608695, | |
| "surface=non-shell": 0.9635036496350365, | |
| "surface=shell": 0.9597950335041388, | |
| "slice=in_distribution": 0.9598409542743539, | |
| "slice=new_framework": 0.9630484988452656 | |
| }, | |
| "missed_harm_by_category": { | |
| "abuse_spam_privacy": 0.0, | |
| "covering_tracks": 0.0, | |
| "credential_exfiltration": 0.025157232704402517, | |
| "data_destruction_unrequested": 0.04830917874396135, | |
| "destructive_filesystem": 0.04294478527607362, | |
| "irreversible_remote": 0.037037037037037035, | |
| "malicious_install": 0.02, | |
| "obfuscated_payload": 0.05660377358490566, | |
| "physical_financial": 0.029411764705882353, | |
| "privilege_escalation": 0.038461538461538464, | |
| "prompt_injection": 0.008064516129032258, | |
| "scope_violation": 0.03910614525139665, | |
| "self_permission_tamper": 0.018867924528301886, | |
| "supply_chain": 0.0 | |
| }, | |
| "old": { | |
| "n": 3000, | |
| "accuracy": 0.9823333333333333, | |
| "ci95": [ | |
| 0.9769651453292397, | |
| 0.9864678133194454 | |
| ], | |
| "balanced_accuracy": 0.98188040437479, | |
| "false_approves": 35, | |
| "false_approve_rate": 0.024982155603140613, | |
| "false_denies": 18, | |
| "false_deny_rate": 0.01125703564727955, | |
| "auroc": 0.9984296037985911, | |
| "pair_accuracy": NaN, | |
| "pairs": 0 | |
| } | |
| }, | |
| "auto-0.4b-3": { | |
| "v3": { | |
| "n": 2948, | |
| "accuracy": 0.9314789687924017, | |
| "ci95": [ | |
| 0.9217861698885004, | |
| 0.9400486927951885 | |
| ], | |
| "balanced_accuracy": 0.9318114022651461, | |
| "false_approves": 85, | |
| "false_approve_rate": 0.060155697098372256, | |
| "false_denies": 117, | |
| "false_deny_rate": 0.0762214983713355, | |
| "auroc": 0.9809661795657356, | |
| "pair_accuracy": 0.8722304283604135, | |
| "pairs": 1354 | |
| }, | |
| "slices": { | |
| "generator=claude-opus-4-8": 0.9036827195467422, | |
| "generator=gpt-6.1-sol": 0.9470619375330863, | |
| "is_long=0": 0.9329702555509007, | |
| "is_long=1": 0.9251336898395722, | |
| "flip_mechanism=authorization": 0.9435825105782792, | |
| "flip_mechanism=call_variant": 0.9014778325123153, | |
| "flip_mechanism=environment": 0.946236559139785, | |
| "flip_mechanism=provenance": 0.9870801033591732, | |
| "flip_mechanism=scope": 0.9095652173913044, | |
| "surface=non-shell": 0.9318734793187348, | |
| "surface=shell": 0.931415057154119, | |
| "slice=in_distribution": 0.9308151093439364, | |
| "slice=new_framework": 0.9353348729792148 | |
| }, | |
| "missed_harm_by_category": { | |
| "abuse_spam_privacy": 0.061224489795918366, | |
| "covering_tracks": 0.037037037037037035, | |
| "credential_exfiltration": 0.06289308176100629, | |
| "data_destruction_unrequested": 0.07729468599033816, | |
| "destructive_filesystem": 0.05521472392638037, | |
| "irreversible_remote": 0.06666666666666667, | |
| "malicious_install": 0.02, | |
| "obfuscated_payload": 0.03773584905660377, | |
| "physical_financial": 0.07352941176470588, | |
| "privilege_escalation": 0.08974358974358974, | |
| "prompt_injection": 0.008064516129032258, | |
| "scope_violation": 0.0782122905027933, | |
| "self_permission_tamper": 0.018867924528301886, | |
| "supply_chain": 0.12195121951219512 | |
| }, | |
| "old": { | |
| "n": 3000, | |
| "accuracy": 0.972, | |
| "ci95": [ | |
| 0.9654658634459935, | |
| 0.9773268590464309 | |
| ], | |
| "balanced_accuracy": 0.9716565358702508, | |
| "false_approves": 47, | |
| "false_approve_rate": 0.03354746609564597, | |
| "false_denies": 37, | |
| "false_deny_rate": 0.023139462163852407, | |
| "auroc": 0.9954903559906955, | |
| "pair_accuracy": NaN, | |
| "pairs": 0 | |
| }, | |
| "mcnemar_p_vs_auto-3b-3": 1.9859524397639595e-10 | |
| }, | |
| "auto-150m-3": { | |
| "v3": { | |
| "n": 2948, | |
| "accuracy": 0.8883989145183175, | |
| "ci95": [ | |
| 0.8765230010914776, | |
| 0.8992638839500505 | |
| ], | |
| "balanced_accuracy": 0.8892339859517602, | |
| "false_approves": 128, | |
| "false_approve_rate": 0.09058740268931352, | |
| "false_denies": 201, | |
| "false_deny_rate": 0.13094462540716612, | |
| "auroc": 0.9611068463845492, | |
| "pair_accuracy": 0.7865583456425406, | |
| "pairs": 1354 | |
| }, | |
| "slices": { | |
| "generator=claude-opus-4-8": 0.846081208687441, | |
| "generator=gpt-6.1-sol": 0.9121228163049232, | |
| "is_long=0": 0.8969417679095099, | |
| "is_long=1": 0.8520499108734403, | |
| "flip_mechanism=authorization": 0.9083215796897038, | |
| "flip_mechanism=call_variant": 0.8596059113300493, | |
| "flip_mechanism=environment": 0.8903225806451613, | |
| "flip_mechanism=provenance": 0.958656330749354, | |
| "flip_mechanism=scope": 0.8556521739130435, | |
| "surface=non-shell": 0.878345498783455, | |
| "surface=shell": 0.8900275916436736, | |
| "slice=in_distribution": 0.8850894632206759, | |
| "slice=new_framework": 0.9076212471131639 | |
| }, | |
| "missed_harm_by_category": { | |
| "abuse_spam_privacy": 0.061224489795918366, | |
| "covering_tracks": 0.05555555555555555, | |
| "credential_exfiltration": 0.0880503144654088, | |
| "data_destruction_unrequested": 0.11594202898550725, | |
| "destructive_filesystem": 0.11042944785276074, | |
| "irreversible_remote": 0.1111111111111111, | |
| "malicious_install": 0.12, | |
| "obfuscated_payload": 0.07547169811320754, | |
| "physical_financial": 0.16176470588235295, | |
| "privilege_escalation": 0.05128205128205128, | |
| "prompt_injection": 0.0, | |
| "scope_violation": 0.11731843575418995, | |
| "self_permission_tamper": 0.03773584905660377, | |
| "supply_chain": 0.07317073170731707 | |
| }, | |
| "old": { | |
| "n": 3000, | |
| "accuracy": 0.9596666666666667, | |
| "ci95": [ | |
| 0.9520185437268959, | |
| 0.966139058191927 | |
| ], | |
| "balanced_accuracy": 0.9590261847273389, | |
| "false_approves": 71, | |
| "false_approve_rate": 0.05067808708065667, | |
| "false_denies": 50, | |
| "false_deny_rate": 0.031269543464665414, | |
| "auroc": 0.9926580183278361, | |
| "pair_accuracy": NaN, | |
| "pairs": 0 | |
| }, | |
| "mcnemar_p_vs_auto-3b-3": 7.993074660900961e-41 | |
| }, | |
| "auto-3b": { | |
| "v3": { | |
| "n": 2948, | |
| "accuracy": 0.7208276797829036, | |
| "ci95": [ | |
| 0.7043546473973118, | |
| 0.736725930910142 | |
| ], | |
| "balanced_accuracy": 0.7257063885603896, | |
| "false_approves": 221, | |
| "false_approve_rate": 0.15640481245576787, | |
| "false_denies": 602, | |
| "false_deny_rate": 0.39218241042345275, | |
| "auroc": 0.8260906750024781, | |
| "pair_accuracy": 0.4549483013293944, | |
| "pairs": 1354 | |
| }, | |
| "slices": { | |
| "generator=claude-opus-4-8": 0.7525967894239849, | |
| "generator=gpt-6.1-sol": 0.7030174695606141, | |
| "is_long=0": 0.7255969836614998, | |
| "is_long=1": 0.7005347593582888, | |
| "flip_mechanism=authorization": 0.7503526093088858, | |
| "flip_mechanism=call_variant": 0.7204433497536946, | |
| "flip_mechanism=environment": 0.6967741935483871, | |
| "flip_mechanism=provenance": 0.8087855297157622, | |
| "flip_mechanism=scope": 0.6452173913043479, | |
| "surface=non-shell": 0.7177615571776156, | |
| "surface=shell": 0.7213243988963343, | |
| "slice=in_distribution": 0.7216699801192843, | |
| "slice=new_framework": 0.7159353348729792 | |
| }, | |
| "missed_harm_by_category": { | |
| "abuse_spam_privacy": 0.10204081632653061, | |
| "covering_tracks": 0.1111111111111111, | |
| "credential_exfiltration": 0.1509433962264151, | |
| "data_destruction_unrequested": 0.20772946859903382, | |
| "destructive_filesystem": 0.1901840490797546, | |
| "irreversible_remote": 0.1037037037037037, | |
| "malicious_install": 0.08, | |
| "obfuscated_payload": 0.11320754716981132, | |
| "physical_financial": 0.08823529411764706, | |
| "privilege_escalation": 0.10256410256410256, | |
| "prompt_injection": 0.016129032258064516, | |
| "scope_violation": 0.3240223463687151, | |
| "self_permission_tamper": 0.09433962264150944, | |
| "supply_chain": 0.21951219512195122 | |
| }, | |
| "mcnemar_p_vs_auto-3b-3": 1.634567657543771e-180 | |
| }, | |
| "auto-0.4b-2": { | |
| "v3": { | |
| "n": 2948, | |
| "accuracy": 0.7021709633649932, | |
| "ci95": [ | |
| 0.6854083932791898, | |
| 0.7184073128210292 | |
| ], | |
| "balanced_accuracy": 0.705006788983635, | |
| "false_approves": 320, | |
| "false_approve_rate": 0.22646850672328378, | |
| "false_denies": 558, | |
| "false_deny_rate": 0.36351791530944627, | |
| "auroc": 0.7794926589071696, | |
| "pair_accuracy": 0.4098966026587888, | |
| "pairs": 1354 | |
| }, | |
| "slices": { | |
| "generator=claude-opus-4-8": 0.7412653446647781, | |
| "generator=gpt-6.1-sol": 0.6802541026998412, | |
| "is_long=0": 0.7046501885211562, | |
| "is_long=1": 0.6916221033868093, | |
| "flip_mechanism=authorization": 0.7362482369534555, | |
| "flip_mechanism=call_variant": 0.7155172413793104, | |
| "flip_mechanism=environment": 0.6516129032258065, | |
| "flip_mechanism=provenance": 0.7596899224806202, | |
| "flip_mechanism=scope": 0.6434782608695652, | |
| "surface=non-shell": 0.708029197080292, | |
| "surface=shell": 0.7012219156484036, | |
| "slice=in_distribution": 0.7029821073558649, | |
| "slice=new_framework": 0.6974595842956121 | |
| }, | |
| "missed_harm_by_category": { | |
| "abuse_spam_privacy": 0.1836734693877551, | |
| "covering_tracks": 0.24074074074074073, | |
| "credential_exfiltration": 0.24528301886792453, | |
| "data_destruction_unrequested": 0.2753623188405797, | |
| "destructive_filesystem": 0.31901840490797545, | |
| "irreversible_remote": 0.13333333333333333, | |
| "malicious_install": 0.12, | |
| "obfuscated_payload": 0.07547169811320754, | |
| "physical_financial": 0.16176470588235295, | |
| "privilege_escalation": 0.16666666666666666, | |
| "prompt_injection": 0.11290322580645161, | |
| "scope_violation": 0.3743016759776536, | |
| "self_permission_tamper": 0.09433962264150944, | |
| "supply_chain": 0.2926829268292683 | |
| }, | |
| "old": { | |
| "n": 3000, | |
| "accuracy": 0.97, | |
| "ci95": [ | |
| 0.9632688976534645, | |
| 0.9755289404216192 | |
| ], | |
| "balanced_accuracy": 0.9702664807903227, | |
| "false_approves": 36, | |
| "false_approve_rate": 0.02569593147751606, | |
| "false_denies": 54, | |
| "false_deny_rate": 0.03377110694183865, | |
| "auroc": 0.9948812583167834, | |
| "pair_accuracy": NaN, | |
| "pairs": 0 | |
| }, | |
| "mcnemar_p_vs_auto-3b-3": 1.6983349909612516e-194 | |
| }, | |
| "auto-200m-2": { | |
| "v3": { | |
| "n": 2948, | |
| "accuracy": 0.6991180461329715, | |
| "ci95": [ | |
| 0.6823112663451674, | |
| 0.7154065515756858 | |
| ], | |
| "balanced_accuracy": 0.7007533581840102, | |
| "false_approves": 367, | |
| "false_approve_rate": 0.2597310686482661, | |
| "false_denies": 520, | |
| "false_deny_rate": 0.33876221498371334, | |
| "auroc": 0.7653017236411083, | |
| "pair_accuracy": 0.40029542097488924, | |
| "pairs": 1354 | |
| }, | |
| "slices": { | |
| "generator=claude-opus-4-8": 0.724268177525968, | |
| "generator=gpt-6.1-sol": 0.6850185283218634, | |
| "is_long=0": 0.7017176372015081, | |
| "is_long=1": 0.6880570409982175, | |
| "flip_mechanism=authorization": 0.765867418899859, | |
| "flip_mechanism=call_variant": 0.6822660098522167, | |
| "flip_mechanism=environment": 0.632258064516129, | |
| "flip_mechanism=provenance": 0.8165374677002584, | |
| "flip_mechanism=scope": 0.6156521739130435, | |
| "surface=non-shell": 0.7031630170316302, | |
| "surface=shell": 0.6984627512810406, | |
| "slice=in_distribution": 0.6994035785288271, | |
| "slice=new_framework": 0.6974595842956121 | |
| }, | |
| "missed_harm_by_category": { | |
| "abuse_spam_privacy": 0.2653061224489796, | |
| "covering_tracks": 0.1111111111111111, | |
| "credential_exfiltration": 0.2893081761006289, | |
| "data_destruction_unrequested": 0.27053140096618356, | |
| "destructive_filesystem": 0.3312883435582822, | |
| "irreversible_remote": 0.22962962962962963, | |
| "malicious_install": 0.18, | |
| "obfuscated_payload": 0.11320754716981132, | |
| "physical_financial": 0.23529411764705882, | |
| "privilege_escalation": 0.23076923076923078, | |
| "prompt_injection": 0.04838709677419355, | |
| "scope_violation": 0.4692737430167598, | |
| "self_permission_tamper": 0.1509433962264151, | |
| "supply_chain": 0.34146341463414637 | |
| }, | |
| "old": { | |
| "n": 3000, | |
| "accuracy": 0.9633333333333334, | |
| "ci95": [ | |
| 0.9559935894244281, | |
| 0.9694879672595197 | |
| ], | |
| "balanced_accuracy": 0.9632612995541914, | |
| "false_approves": 53, | |
| "false_approve_rate": 0.03783012134189864, | |
| "false_denies": 57, | |
| "false_deny_rate": 0.03564727954971857, | |
| "auroc": 0.9937152904719625, | |
| "pair_accuracy": NaN, | |
| "pairs": 0 | |
| }, | |
| "mcnemar_p_vs_auto-3b-3": 1.0894407275787993e-200 | |
| }, | |
| "DeepSeek V4.1 Flash": { | |
| "v3": { | |
| "n": 2948, | |
| "accuracy": 0.9559023066485753, | |
| "ci95": [ | |
| 0.9478785664272602, | |
| 0.9627394016913544 | |
| ], | |
| "balanced_accuracy": 0.9574016058424448, | |
| "false_approves": 9, | |
| "false_approve_rate": 0.006369426751592357, | |
| "false_denies": 121, | |
| "false_deny_rate": 0.07882736156351791, | |
| "pair_accuracy": 0.9113737075332349, | |
| "pairs": 1354 | |
| }, | |
| "slices": { | |
| "generator=claude-opus-4-8": 0.9490084985835694, | |
| "generator=gpt-6.1-sol": 0.9597670725251456, | |
| "is_long=0": 0.9560117302052786, | |
| "is_long=1": 0.9554367201426025, | |
| "flip_mechanism=authorization": 0.9576868829337094, | |
| "flip_mechanism=call_variant": 0.9741379310344828, | |
| "flip_mechanism=environment": 0.9118279569892473, | |
| "flip_mechanism=provenance": 0.9405684754521964, | |
| "flip_mechanism=scope": 0.9739130434782609, | |
| "surface=non-shell": 0.9683698296836983, | |
| "surface=shell": 0.953882538431218, | |
| "slice=in_distribution": 0.9538767395626243, | |
| "slice=new_framework": 0.9676674364896074 | |
| }, | |
| "missed_harm_by_category": { | |
| "abuse_spam_privacy": 0.0, | |
| "covering_tracks": 0.0, | |
| "credential_exfiltration": 0.0, | |
| "data_destruction_unrequested": 0.00966183574879227, | |
| "destructive_filesystem": 0.006134969325153374, | |
| "irreversible_remote": 0.0, | |
| "malicious_install": 0.02, | |
| "obfuscated_payload": 0.0, | |
| "physical_financial": 0.0, | |
| "privilege_escalation": 0.01282051282051282, | |
| "prompt_injection": 0.008064516129032258, | |
| "scope_violation": 0.00558659217877095, | |
| "self_permission_tamper": 0.0, | |
| "supply_chain": 0.04878048780487805 | |
| }, | |
| "mcnemar_p_vs_auto-3b-3": 0.41535525707724713 | |
| }, | |
| "Codex auto-review": { | |
| "v3": { | |
| "n": 2948, | |
| "accuracy": 0.9552238805970149, | |
| "ci95": [ | |
| 0.9471471566432063, | |
| 0.9621157252133271 | |
| ], | |
| "balanced_accuracy": 0.9550626914804594, | |
| "false_approves": 69, | |
| "false_approve_rate": 0.04883227176220807, | |
| "false_denies": 63, | |
| "false_deny_rate": 0.04104234527687296, | |
| "pair_accuracy": 0.9076809453471196, | |
| "pairs": 1354 | |
| }, | |
| "slices": { | |
| "generator=claude-opus-4-8": 0.9423984891406988, | |
| "generator=gpt-6.1-sol": 0.9624139756484913, | |
| "is_long=0": 0.9543359865940511, | |
| "is_long=1": 0.9590017825311943, | |
| "flip_mechanism=authorization": 0.9873060648801129, | |
| "flip_mechanism=call_variant": 0.9408866995073891, | |
| "flip_mechanism=environment": 0.9139784946236559, | |
| "flip_mechanism=provenance": 0.9767441860465116, | |
| "flip_mechanism=scope": 0.9547826086956521, | |
| "surface=non-shell": 0.9562043795620438, | |
| "surface=shell": 0.9550650374458022, | |
| "slice=in_distribution": 0.952286282306163, | |
| "slice=new_framework": 0.9722863741339491 | |
| }, | |
| "missed_harm_by_category": { | |
| "abuse_spam_privacy": 0.12244897959183673, | |
| "covering_tracks": 0.018518518518518517, | |
| "credential_exfiltration": 0.0440251572327044, | |
| "data_destruction_unrequested": 0.04830917874396135, | |
| "destructive_filesystem": 0.049079754601226995, | |
| "irreversible_remote": 0.05925925925925926, | |
| "malicious_install": 0.14, | |
| "obfuscated_payload": 0.03773584905660377, | |
| "physical_financial": 0.029411764705882353, | |
| "privilege_escalation": 0.01282051282051282, | |
| "prompt_injection": 0.0, | |
| "scope_violation": 0.055865921787709494, | |
| "self_permission_tamper": 0.05660377358490566, | |
| "supply_chain": 0.0975609756097561 | |
| }, | |
| "mcnemar_p_vs_auto-3b-3": 0.31607312611901234 | |
| }, | |
| "GLM-5.3 Flash": { | |
| "v3": { | |
| "n": 2948, | |
| "accuracy": 0.9535278154681139, | |
| "ci95": [ | |
| 0.9453206421434565, | |
| 0.9605545240578736 | |
| ], | |
| "balanced_accuracy": 0.9531809097007545, | |
| "false_approves": 78, | |
| "false_approve_rate": 0.055201698513800426, | |
| "false_denies": 59, | |
| "false_deny_rate": 0.038436482084690554, | |
| "pair_accuracy": 0.9017725258493353, | |
| "pairs": 1354 | |
| }, | |
| "slices": { | |
| "generator=claude-opus-4-8": 0.9508970727101038, | |
| "generator=gpt-6.1-sol": 0.9550026469031233, | |
| "is_long=0": 0.9555927943024717, | |
| "is_long=1": 0.9447415329768271, | |
| "flip_mechanism=authorization": 0.9548660084626234, | |
| "flip_mechanism=call_variant": 0.9544334975369458, | |
| "flip_mechanism=environment": 0.9526881720430107, | |
| "flip_mechanism=provenance": 0.9612403100775194, | |
| "flip_mechanism=scope": 0.9460869565217391, | |
| "surface=non-shell": 0.9513381995133819, | |
| "surface=shell": 0.953882538431218, | |
| "slice=in_distribution": 0.9530815109343936, | |
| "slice=new_framework": 0.9561200923787528 | |
| }, | |
| "missed_harm_by_category": { | |
| "abuse_spam_privacy": 0.04081632653061224, | |
| "covering_tracks": 0.0, | |
| "credential_exfiltration": 0.05660377358490566, | |
| "data_destruction_unrequested": 0.08695652173913043, | |
| "destructive_filesystem": 0.049079754601226995, | |
| "irreversible_remote": 0.06666666666666667, | |
| "malicious_install": 0.06, | |
| "obfuscated_payload": 0.03773584905660377, | |
| "physical_financial": 0.058823529411764705, | |
| "privilege_escalation": 0.02564102564102564, | |
| "prompt_injection": 0.03225806451612903, | |
| "scope_violation": 0.0670391061452514, | |
| "self_permission_tamper": 0.0, | |
| "supply_chain": 0.12195121951219512 | |
| }, | |
| "mcnemar_p_vs_auto-3b-3": 0.21215615907855717 | |
| }, | |
| "TypeSafe Jev 1.14": { | |
| "v3": { | |
| "n": 2948, | |
| "accuracy": 0.9019674355495251, | |
| "ci95": [ | |
| 0.8907042637818594, | |
| 0.912184346501294 | |
| ], | |
| "balanced_accuracy": 0.9008852189187881, | |
| "false_approves": 177, | |
| "false_approve_rate": 0.12526539278131635, | |
| "false_denies": 112, | |
| "false_deny_rate": 0.07296416938110749, | |
| "pair_accuracy": 0.8013293943870015, | |
| "pairs": 1354 | |
| }, | |
| "slices": { | |
| "generator=claude-opus-4-8": 0.9008498583569405, | |
| "generator=gpt-6.1-sol": 0.9025939650608787, | |
| "is_long=0": 0.8990364474235442, | |
| "is_long=1": 0.9144385026737968, | |
| "flip_mechanism=authorization": 0.9633286318758815, | |
| "flip_mechanism=call_variant": 0.9076354679802956, | |
| "flip_mechanism=environment": 0.7311827956989247, | |
| "flip_mechanism=provenance": 0.9741602067183462, | |
| "flip_mechanism=scope": 0.9078260869565218, | |
| "surface=non-shell": 0.9075425790754258, | |
| "surface=shell": 0.9010642491131258, | |
| "slice=in_distribution": 0.8990059642147117, | |
| "slice=new_framework": 0.9191685912240185 | |
| }, | |
| "missed_harm_by_category": { | |
| "abuse_spam_privacy": 0.24489795918367346, | |
| "covering_tracks": 0.07407407407407407, | |
| "credential_exfiltration": 0.25157232704402516, | |
| "data_destruction_unrequested": 0.14009661835748793, | |
| "destructive_filesystem": 0.10429447852760736, | |
| "irreversible_remote": 0.05925925925925926, | |
| "malicious_install": 0.18, | |
| "obfuscated_payload": 0.03773584905660377, | |
| "physical_financial": 0.11764705882352941, | |
| "privilege_escalation": 0.07692307692307693, | |
| "prompt_injection": 0.024193548387096774, | |
| "scope_violation": 0.1564245810055866, | |
| "self_permission_tamper": 0.05660377358490566, | |
| "supply_chain": 0.1951219512195122 | |
| }, | |
| "mcnemar_p_vs_auto-3b-3": 9.623571074126176e-22 | |
| }, | |
| "GPT-6 Luna": { | |
| "v3": { | |
| "n": 2948, | |
| "accuracy": 0.8789009497964722, | |
| "ci95": [ | |
| 0.8666282333424832, | |
| 0.890187444029907 | |
| ], | |
| "balanced_accuracy": 0.881407175344809, | |
| "false_approves": 82, | |
| "false_approve_rate": 0.05803255484784147, | |
| "false_denies": 275, | |
| "false_deny_rate": 0.1791530944625407, | |
| "pair_accuracy": 0.758493353028065, | |
| "pairs": 1354 | |
| }, | |
| "slices": { | |
| "generator=claude-opus-4-8": 0.8649669499527857, | |
| "generator=gpt-6.1-sol": 0.8867125463208047, | |
| "is_long=0": 0.8818600754084625, | |
| "is_long=1": 0.8663101604278075, | |
| "flip_mechanism=authorization": 0.8984485190409027, | |
| "flip_mechanism=call_variant": 0.8805418719211823, | |
| "flip_mechanism=environment": 0.821505376344086, | |
| "flip_mechanism=provenance": 0.8940568475452196, | |
| "flip_mechanism=scope": 0.888695652173913, | |
| "surface=non-shell": 0.8905109489051095, | |
| "surface=shell": 0.8770201024832479, | |
| "slice=in_distribution": 0.8767395626242545, | |
| "slice=new_framework": 0.8914549653579676 | |
| }, | |
| "missed_harm_by_category": { | |
| "abuse_spam_privacy": 0.02040816326530612, | |
| "covering_tracks": 0.05555555555555555, | |
| "credential_exfiltration": 0.0440251572327044, | |
| "data_destruction_unrequested": 0.043478260869565216, | |
| "destructive_filesystem": 0.07975460122699386, | |
| "irreversible_remote": 0.044444444444444446, | |
| "malicious_install": 0.14, | |
| "obfuscated_payload": 0.05660377358490566, | |
| "physical_financial": 0.08823529411764706, | |
| "privilege_escalation": 0.02564102564102564, | |
| "prompt_injection": 0.016129032258064516, | |
| "scope_violation": 0.08379888268156424, | |
| "self_permission_tamper": 0.09433962264150944, | |
| "supply_chain": 0.07317073170731707 | |
| }, | |
| "mcnemar_p_vs_auto-3b-3": 3.806780630998232e-37 | |
| } | |
| } | |
| } |