| { |
| "uncalibrated": { |
| "overall": { |
| "count": 17476, |
| "accuracy": 0.7985809109636073, |
| "log_loss": 0.49230193969031993, |
| "brier_score": 0.2802096009227025, |
| "ece_15": 0.038409021587914594, |
| "ordinal_mae": 0.3727614025515071, |
| "ordinal_count": 997 |
| }, |
| "source/tasksource/zero-shot-label-nli": { |
| "count": 1979, |
| "accuracy": 0.7281455280444669, |
| "log_loss": 0.5161915449429214, |
| "brier_score": 0.34877216174971326, |
| "ece_15": 0.03029219201220161, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "kind/choice": { |
| "count": 15651, |
| "accuracy": 0.8008433965880775, |
| "log_loss": 0.4911452188629626, |
| "brier_score": 0.2776828999683597, |
| "ece_15": 0.04135510104840071, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/rotten_tomatoes": { |
| "count": 9, |
| "accuracy": 0.8888888888888888, |
| "log_loss": 0.14167306317777614, |
| "brier_score": 0.08184373427295029, |
| "ece_15": 0.11150019862956809, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/imdb": { |
| "count": 14, |
| "accuracy": 0.8571428571428571, |
| "log_loss": 0.21830981536265165, |
| "brier_score": 0.16696657124912387, |
| "ece_15": 0.118122814216601, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/political-media-bias": { |
| "count": 10, |
| "accuracy": 0.9, |
| "log_loss": 0.39026031647009146, |
| "brier_score": 0.23003624954290774, |
| "ece_15": 0.20450311132194052, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/Touche23-ValueEval": { |
| "count": 20, |
| "accuracy": 0.9, |
| "log_loss": 0.3233971118028126, |
| "brier_score": 0.18881968524422515, |
| "ece_15": 0.08769103723618404, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ethics/virtue": { |
| "count": 18, |
| "accuracy": 0.7777777777777778, |
| "log_loss": 0.5544695183167773, |
| "brier_score": 0.3652606273704724, |
| "ece_15": 0.10809417467394164, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/humicroedit/subtask-2": { |
| "count": 17, |
| "accuracy": 0.5294117647058824, |
| "log_loss": 0.668951978682478, |
| "brier_score": 0.47482126017314447, |
| "ece_15": 0.16151028778090645, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/meld_e": { |
| "count": 15, |
| "accuracy": 0.5333333333333333, |
| "log_loss": 0.8179600033579704, |
| "brier_score": 0.5634823152418934, |
| "ece_15": 0.27792269420872395, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/citation_intent": { |
| "count": 13, |
| "accuracy": 0.8461538461538461, |
| "log_loss": 0.37321650182444965, |
| "brier_score": 0.23476206589777396, |
| "ece_15": 0.1905081909317456, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/lex_glue/ledgar": { |
| "count": 21, |
| "accuracy": 0.9047619047619048, |
| "log_loss": 0.18487318912761527, |
| "brier_score": 0.1086545636781901, |
| "ece_15": 0.10578723136936816, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/dynasent/dynabench.dynasent.r2.all/r2": { |
| "count": 18, |
| "accuracy": 0.8333333333333334, |
| "log_loss": 0.43822894539076496, |
| "brier_score": 0.29223347262844424, |
| "ece_15": 0.20536942079615883, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/lexical_relation_classification/CogALexV": { |
| "count": 11, |
| "accuracy": 0.8181818181818182, |
| "log_loss": 0.2960173657259703, |
| "brier_score": 0.17517309267135275, |
| "ece_15": 0.16361508328550184, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/emotion": { |
| "count": 16, |
| "accuracy": 0.875, |
| "log_loss": 0.3250099827855003, |
| "brier_score": 0.1961210928353625, |
| "ece_15": 0.1414621010277033, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/lexical_relation_classification/K&H+N": { |
| "count": 10, |
| "accuracy": 0.6, |
| "log_loss": 0.6013651803805745, |
| "brier_score": 0.43527929901669477, |
| "ece_15": 0.27235183004676355, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/scientific-exaggeration-detection": { |
| "count": 15, |
| "accuracy": 0.7333333333333333, |
| "log_loss": 0.6091250021294122, |
| "brier_score": 0.4206860016677024, |
| "ece_15": 0.1931308925775061, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/yahoo_answers_topics": { |
| "count": 21, |
| "accuracy": 0.7142857142857143, |
| "log_loss": 0.3979357796020941, |
| "brier_score": 0.28076482273611164, |
| "ece_15": 0.1575586305221374, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/poem_sentiment": { |
| "count": 17, |
| "accuracy": 0.8235294117647058, |
| "log_loss": 0.4892394735777189, |
| "brier_score": 0.3079320811058927, |
| "ece_15": 0.17455293996237675, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/stance_climate": { |
| "count": 18, |
| "accuracy": 0.8333333333333334, |
| "log_loss": 0.4607512786626419, |
| "brier_score": 0.2834526827558238, |
| "ece_15": 0.1385072554219914, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/fever-evidence-related/mwong--fever-related": { |
| "count": 19, |
| "accuracy": 0.7894736842105263, |
| "log_loss": 0.45571088042165764, |
| "brier_score": 0.3202707326941488, |
| "ece_15": 0.23132931629501974, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/political-media-message": { |
| "count": 16, |
| "accuracy": 0.6875, |
| "log_loss": 0.5445516633163114, |
| "brier_score": 0.3702129694579478, |
| "ece_15": 0.16434371890461308, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/dynasent/dynabench.dynasent.r1.all/r1": { |
| "count": 16, |
| "accuracy": 0.875, |
| "log_loss": 0.577321434866638, |
| "brier_score": 0.29168608956427977, |
| "ece_15": 0.2081726800260979, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/dyda_e": { |
| "count": 17, |
| "accuracy": 1.0, |
| "log_loss": 0.04417400938285737, |
| "brier_score": 0.01049226002997129, |
| "ece_15": 0.0411027576308009, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/CREAK": { |
| "count": 21, |
| "accuracy": 0.7619047619047619, |
| "log_loss": 0.5081726325635354, |
| "brier_score": 0.3382906391791713, |
| "ece_15": 0.09373480466526815, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/sarcasm": { |
| "count": 18, |
| "accuracy": 0.6111111111111112, |
| "log_loss": 0.6475117416011661, |
| "brier_score": 0.45836110899326993, |
| "ece_15": 0.13452855984046247, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/medical_questions_pairs": { |
| "count": 8, |
| "accuracy": 0.875, |
| "log_loss": 0.36597250380759194, |
| "brier_score": 0.23097038476200957, |
| "ece_15": 0.2543400198938318, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/stance_abortion": { |
| "count": 16, |
| "accuracy": 0.4375, |
| "log_loss": 0.7966763529688047, |
| "brier_score": 0.5956783941266531, |
| "ece_15": 0.2849024303717077, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ade_corpus_v2/Ade_corpus_v2_classification": { |
| "count": 18, |
| "accuracy": 0.7222222222222222, |
| "log_loss": 0.6441245473797038, |
| "brier_score": 0.4343455078371591, |
| "ece_15": 0.14182715422618286, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/commonsense_qa_2.0": { |
| "count": 9, |
| "accuracy": 0.7777777777777778, |
| "log_loss": 0.5386296120023389, |
| "brier_score": 0.35158860663198394, |
| "ece_15": 0.18069320401090017, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/health_fact": { |
| "count": 17, |
| "accuracy": 0.8823529411764706, |
| "log_loss": 0.35314175664603786, |
| "brier_score": 0.20542471251149838, |
| "ece_15": 0.17937437947794535, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/political-media-audience": { |
| "count": 23, |
| "accuracy": 0.7391304347826086, |
| "log_loss": 0.44126636866706553, |
| "brier_score": 0.3076290607438106, |
| "ece_15": 0.16927721154721653, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/circa": { |
| "count": 21, |
| "accuracy": 0.9523809523809523, |
| "log_loss": 0.17487753470794917, |
| "brier_score": 0.0969496651837543, |
| "ece_15": 0.07673876434393226, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/iemocap": { |
| "count": 16, |
| "accuracy": 0.625, |
| "log_loss": 0.7674579983255659, |
| "brier_score": 0.5285902666037299, |
| "ece_15": 0.22058308451926445, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/hate": { |
| "count": 16, |
| "accuracy": 0.75, |
| "log_loss": 0.5499020705139512, |
| "brier_score": 0.3777113004448934, |
| "ece_15": 0.26087078278341785, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/pdtb": { |
| "count": 11, |
| "accuracy": 0.6363636363636364, |
| "log_loss": 0.5389450349140293, |
| "brier_score": 0.3730469555705936, |
| "ece_15": 0.286182776842427, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/winowhy": { |
| "count": 20, |
| "accuracy": 0.55, |
| "log_loss": 0.7217310268719503, |
| "brier_score": 0.5257732183182033, |
| "ece_15": 0.14832649091392353, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/hyperpartisan_news": { |
| "count": 11, |
| "accuracy": 0.5454545454545454, |
| "log_loss": 0.5724618118848079, |
| "brier_score": 0.3963810444435573, |
| "ece_15": 0.345359107422761, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/sms_spam": { |
| "count": 12, |
| "accuracy": 1.0, |
| "log_loss": 0.15591444971315802, |
| "brier_score": 0.05822512919236534, |
| "ece_15": 0.13856703643280263, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/lexical_relation_classification/ROOT09": { |
| "count": 21, |
| "accuracy": 0.8571428571428571, |
| "log_loss": 0.3115086911529061, |
| "brier_score": 0.18724904604270268, |
| "ece_15": 0.14582605427705903, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/clcd-english": { |
| "count": 20, |
| "accuracy": 0.7, |
| "log_loss": 0.6090553041710016, |
| "brier_score": 0.4203566531073464, |
| "ece_15": 0.25239511329769826, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/FLUTE": { |
| "count": 14, |
| "accuracy": 0.7857142857142857, |
| "log_loss": 0.4635781105583847, |
| "brier_score": 0.29311804417501836, |
| "ece_15": 0.1613151453046786, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/starcon": { |
| "count": 19, |
| "accuracy": 0.8421052631578947, |
| "log_loss": 0.42956188709408466, |
| "brier_score": 0.25480182501422116, |
| "ece_15": 0.20063538134058986, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/lexical_relation_classification/BLESS": { |
| "count": 19, |
| "accuracy": 0.631578947368421, |
| "log_loss": 0.568139522399679, |
| "brier_score": 0.415679190690983, |
| "ece_15": 0.16235339286504047, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/lexical_relation_classification/EVALution": { |
| "count": 14, |
| "accuracy": 0.7857142857142857, |
| "log_loss": 0.46706913753823237, |
| "brier_score": 0.3032777955181034, |
| "ece_15": 0.1317683024034402, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/dbpedia_14/dbpedia_14": { |
| "count": 14, |
| "accuracy": 0.9285714285714286, |
| "log_loss": 0.19040692206536286, |
| "brier_score": 0.12194336591971146, |
| "ece_15": 0.07310429325647583, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/strategy-qa": { |
| "count": 17, |
| "accuracy": 0.29411764705882354, |
| "log_loss": 0.8134835468674985, |
| "brier_score": 0.6126713601774318, |
| "ece_15": 0.30389505654336374, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/cycic_classification": { |
| "count": 7, |
| "accuracy": 0.7142857142857143, |
| "log_loss": 0.8237531568397871, |
| "brier_score": 0.5738292388403493, |
| "ece_15": 0.44225242947398896, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/blog_authorship_corpus/horoscope": { |
| "count": 16, |
| "accuracy": 0.5625, |
| "log_loss": 0.7103671536543629, |
| "brier_score": 0.5150540398567833, |
| "ece_15": 0.12915423131437648, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/hlgd": { |
| "count": 18, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.7925086268074288, |
| "brier_score": 0.5117570747556701, |
| "ece_15": 0.24723375372753698, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/paws/labeled_swap": { |
| "count": 14, |
| "accuracy": 0.7857142857142857, |
| "log_loss": 0.46255405644875314, |
| "brier_score": 0.28576216579627756, |
| "ece_15": 0.2280659681104869, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/numer_sense": { |
| "count": 15, |
| "accuracy": 0.8, |
| "log_loss": 0.5182146573314791, |
| "brier_score": 0.341236979869189, |
| "ece_15": 0.21908882828849408, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/implicit-hate-stg1": { |
| "count": 16, |
| "accuracy": 0.6875, |
| "log_loss": 0.4412295744117953, |
| "brier_score": 0.30427166604263833, |
| "ece_15": 0.17399885005107582, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/EffectiveFeedbackStudentWriting": { |
| "count": 10, |
| "accuracy": 0.8, |
| "log_loss": 0.5356834301708717, |
| "brier_score": 0.3553012292049021, |
| "ece_15": 0.13878485714701003, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/phrase_similarity": { |
| "count": 8, |
| "accuracy": 0.625, |
| "log_loss": 0.6814736880041121, |
| "brier_score": 0.4792470195625156, |
| "ece_15": 0.441747697803359, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/stance_atheism": { |
| "count": 10, |
| "accuracy": 0.5, |
| "log_loss": 0.8255945009056708, |
| "brier_score": 0.5845605262844571, |
| "ece_15": 0.24144561635953998, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/airline-sentiment": { |
| "count": 11, |
| "accuracy": 0.6363636363636364, |
| "log_loss": 0.5446367639402901, |
| "brier_score": 0.35765991765527444, |
| "ece_15": 0.26122333622495864, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/MOH": { |
| "count": 13, |
| "accuracy": 0.7692307692307693, |
| "log_loss": 0.5153571138040285, |
| "brier_score": 0.33742353736125585, |
| "ece_15": 0.20412131492735272, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/sciie": { |
| "count": 8, |
| "accuracy": 0.625, |
| "log_loss": 0.8314393459971987, |
| "brier_score": 0.5786453770583999, |
| "ece_15": 0.378816718076871, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/hate_speech18": { |
| "count": 17, |
| "accuracy": 0.8823529411764706, |
| "log_loss": 0.35098353326806153, |
| "brier_score": 0.16684662240672746, |
| "ece_15": 0.09149497019400032, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/racial-bias": { |
| "count": 13, |
| "accuracy": 0.5384615384615384, |
| "log_loss": 0.6729198869772895, |
| "brier_score": 0.48518633705933073, |
| "ece_15": 0.24780157500450792, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/glue/sst2": { |
| "count": 9, |
| "accuracy": 1.0, |
| "log_loss": 0.07587211415392216, |
| "brier_score": 0.016622639892638576, |
| "ece_15": 0.07129822842265623, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/gender-bias": { |
| "count": 15, |
| "accuracy": 0.6, |
| "log_loss": 0.6330231511718478, |
| "brier_score": 0.445120766841519, |
| "ece_15": 0.1321457824391712, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/offensive": { |
| "count": 11, |
| "accuracy": 0.8181818181818182, |
| "log_loss": 0.3086426051529111, |
| "brier_score": 0.19313031718169363, |
| "ece_15": 0.21991847304366222, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/hate_speech_offensive": { |
| "count": 18, |
| "accuracy": 0.8888888888888888, |
| "log_loss": 0.26692246435495076, |
| "brier_score": 0.146252902331788, |
| "ece_15": 0.2140092753771897, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/twentyquestions": { |
| "count": 13, |
| "accuracy": 0.5384615384615384, |
| "log_loss": 0.7863899140869128, |
| "brier_score": 0.5693055100196097, |
| "ece_15": 0.3859411291748407, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/sem_eval_2010_task_8": { |
| "count": 16, |
| "accuracy": 0.8125, |
| "log_loss": 0.3996555875111849, |
| "brier_score": 0.25605901210945087, |
| "ece_15": 0.18615987789449007, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/financial_phrasebank/sentences_allagree": { |
| "count": 8, |
| "accuracy": 0.875, |
| "log_loss": 0.19532664943719388, |
| "brier_score": 0.0976891539875525, |
| "ece_15": 0.159810553354925, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mc_taco": { |
| "count": 9, |
| "accuracy": 0.7777777777777778, |
| "log_loss": 0.4918275492699584, |
| "brier_score": 0.318327135230503, |
| "ece_15": 0.1905441900764415, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/super_glue/multirc": { |
| "count": 8, |
| "accuracy": 0.5, |
| "log_loss": 0.8395761053104731, |
| "brier_score": 0.6120797215797903, |
| "ece_15": 0.3703628653515052, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/TuringBench": { |
| "count": 15, |
| "accuracy": 0.4, |
| "log_loss": 0.7876375806922853, |
| "brier_score": 0.5877657092216583, |
| "ece_15": 0.20133805519841522, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/probability_words_nli/reasoning_1hop": { |
| "count": 8, |
| "accuracy": 0.375, |
| "log_loss": 0.705131461885386, |
| "brier_score": 0.5120830362291584, |
| "ece_15": 0.18119689832014074, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/tweet_global_warming": { |
| "count": 15, |
| "accuracy": 0.8, |
| "log_loss": 0.45691246043190875, |
| "brier_score": 0.2930775236041658, |
| "ece_15": 0.039754718700293394, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/blog_authorship_corpus/job": { |
| "count": 13, |
| "accuracy": 0.8461538461538461, |
| "log_loss": 0.4896731651022517, |
| "brier_score": 0.277322197588766, |
| "ece_15": 0.1664699023552833, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/parade": { |
| "count": 13, |
| "accuracy": 0.6923076923076923, |
| "log_loss": 0.6041612653771861, |
| "brier_score": 0.42148357587305535, |
| "ece_15": 0.2672736630080111, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/probability_words_nli/usnli": { |
| "count": 16, |
| "accuracy": 0.5, |
| "log_loss": 0.6970241928212817, |
| "brier_score": 0.5038938014379373, |
| "ece_15": 0.11153967395721691, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/stac": { |
| "count": 13, |
| "accuracy": 0.6153846153846154, |
| "log_loss": 0.6373027001525201, |
| "brier_score": 0.4681664654492462, |
| "ece_15": 0.3194477579022656, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/quarel": { |
| "count": 22, |
| "accuracy": 0.6363636363636364, |
| "log_loss": 0.6680580043439217, |
| "brier_score": 0.4756395629333436, |
| "ece_15": 0.1924572440766058, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/sem": { |
| "count": 11, |
| "accuracy": 0.8181818181818182, |
| "log_loss": 0.257659043060115, |
| "brier_score": 0.15573092676816191, |
| "ece_15": 0.11588068084121134, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/hate-speech": { |
| "count": 13, |
| "accuracy": 0.6923076923076923, |
| "log_loss": 0.7467696547349584, |
| "brier_score": 0.5031488969649504, |
| "ece_15": 0.27745748869804604, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/economic-news": { |
| "count": 20, |
| "accuracy": 0.75, |
| "log_loss": 0.5875703099242824, |
| "brier_score": 0.39278005942839317, |
| "ece_15": 0.13077425949314292, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/liar": { |
| "count": 10, |
| "accuracy": 0.5, |
| "log_loss": 0.6702011498813479, |
| "brier_score": 0.4791324983234736, |
| "ece_15": 0.364678412565017, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/scruples": { |
| "count": 13, |
| "accuracy": 0.7692307692307693, |
| "log_loss": 0.5525189547682415, |
| "brier_score": 0.3742025558600413, |
| "ece_15": 0.16808849030944462, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/open_question_type": { |
| "count": 14, |
| "accuracy": 0.8571428571428571, |
| "log_loss": 0.44054199934121685, |
| "brier_score": 0.2807151850718378, |
| "ece_15": 0.16344026702352524, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/emoji": { |
| "count": 13, |
| "accuracy": 0.6153846153846154, |
| "log_loss": 0.7343610660765305, |
| "brier_score": 0.5178709306592536, |
| "ece_15": 0.22189528876068587, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/wiki_qa": { |
| "count": 16, |
| "accuracy": 1.0, |
| "log_loss": 0.04169329802889373, |
| "brier_score": 0.010874643608876468, |
| "ece_15": 0.03844825357482926, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/natural-language-satisfiability": { |
| "count": 11, |
| "accuracy": 0.8181818181818182, |
| "log_loss": 0.578444131719973, |
| "brier_score": 0.38733621113648636, |
| "ece_15": 0.2427885602385805, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/CONDAQA": { |
| "count": 17, |
| "accuracy": 0.6470588235294118, |
| "log_loss": 0.5951169650796073, |
| "brier_score": 0.41033868774347326, |
| "ece_15": 0.26642448846837385, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/language-identification": { |
| "count": 14, |
| "accuracy": 0.7857142857142857, |
| "log_loss": 0.42384018461880796, |
| "brier_score": 0.2803045664685451, |
| "ece_15": 0.17692982997019557, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/vitaminc/tals--vitaminc": { |
| "count": 18, |
| "accuracy": 0.8333333333333334, |
| "log_loss": 0.45626478957065486, |
| "brier_score": 0.2861453831609383, |
| "ece_15": 0.0859758103256129, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/irony": { |
| "count": 13, |
| "accuracy": 0.7692307692307693, |
| "log_loss": 0.5228722650263381, |
| "brier_score": 0.34404978418690924, |
| "ece_15": 0.2718382842793733, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/blog_authorship_corpus/gender": { |
| "count": 11, |
| "accuracy": 0.7272727272727273, |
| "log_loss": 0.5896380725264474, |
| "brier_score": 0.40044867814080026, |
| "ece_15": 0.1388151449891998, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/mrda": { |
| "count": 17, |
| "accuracy": 0.7647058823529411, |
| "log_loss": 0.7416070769184169, |
| "brier_score": 0.45154940044308006, |
| "ece_15": 0.2760789099491041, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/VUAC": { |
| "count": 18, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.6580406052753286, |
| "brier_score": 0.4605314283736277, |
| "ece_15": 0.10592446095767183, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/amazon_polarity/amazon_polarity": { |
| "count": 15, |
| "accuracy": 0.9333333333333333, |
| "log_loss": 0.09676652474564765, |
| "brier_score": 0.05376743286256003, |
| "ece_15": 0.07718896617307063, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/logical-fallacy": { |
| "count": 20, |
| "accuracy": 0.65, |
| "log_loss": 0.6000571609782204, |
| "brier_score": 0.40973396030472814, |
| "ece_15": 0.13523861749452926, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/cognitive-bias": { |
| "count": 19, |
| "accuracy": 0.5789473684210527, |
| "log_loss": 0.6804829751636511, |
| "brier_score": 0.48658278546119615, |
| "ece_15": 0.08672749800220603, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ambient": { |
| "count": 13, |
| "accuracy": 0.5384615384615384, |
| "log_loss": 1.3109700976109495, |
| "brier_score": 0.7692203938115867, |
| "ece_15": 0.4665314623211159, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/oasis": { |
| "count": 15, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.6131139310199565, |
| "brier_score": 0.42661919548069555, |
| "ece_15": 0.2289723109249147, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/linguistic-bias": { |
| "count": 17, |
| "accuracy": 0.7058823529411765, |
| "log_loss": 0.6184210521930569, |
| "brier_score": 0.42646724969310634, |
| "ece_15": 0.25905190135106876, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/PARARULE-Plus": { |
| "count": 14, |
| "accuracy": 1.0, |
| "log_loss": 0.002119395660066006, |
| "brier_score": 2.6377197870509784e-05, |
| "ece_15": 0.0021127606343479704, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/gum": { |
| "count": 13, |
| "accuracy": 0.46153846153846156, |
| "log_loss": 0.8234103971631198, |
| "brier_score": 0.6025361902920193, |
| "ece_15": 0.2599970093900183, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/switchboard": { |
| "count": 12, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.37015870067339823, |
| "brier_score": 0.2443803213052876, |
| "ece_15": 0.1955360920426917, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/verifiability": { |
| "count": 17, |
| "accuracy": 1.0, |
| "log_loss": 0.15912427424830505, |
| "brier_score": 0.054983714027949625, |
| "ece_15": 0.14251798180618064, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/spartqa-yn": { |
| "count": 9, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.5597459563256386, |
| "brier_score": 0.38204556729291295, |
| "ece_15": 0.13762189918384532, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/emergent": { |
| "count": 12, |
| "accuracy": 0.5, |
| "log_loss": 0.6070243840348454, |
| "brier_score": 0.4284615369134999, |
| "ece_15": 0.22299769517653592, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/emo/emo2019": { |
| "count": 9, |
| "accuracy": 0.8888888888888888, |
| "log_loss": 0.3658150239277364, |
| "brier_score": 0.2065425984591287, |
| "ece_15": 0.2893026374183485, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/glue/mrpc": { |
| "count": 13, |
| "accuracy": 0.6923076923076923, |
| "log_loss": 0.6277204213966494, |
| "brier_score": 0.4380964850075013, |
| "ece_15": 0.22060985146456685, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/I2D2": { |
| "count": 14, |
| "accuracy": 0.7857142857142857, |
| "log_loss": 0.47881604087521096, |
| "brier_score": 0.31511683776573124, |
| "ece_15": 0.1891178288470162, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/stance_hillary": { |
| "count": 9, |
| "accuracy": 0.4444444444444444, |
| "log_loss": 0.9571559606183246, |
| "brier_score": 0.7026308961160336, |
| "ece_15": 0.3367666766835902, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/political-bias": { |
| "count": 22, |
| "accuracy": 0.6818181818181818, |
| "log_loss": 0.6247059822518992, |
| "brier_score": 0.4337157656151254, |
| "ece_15": 0.09944228350518652, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/insincere-questions": { |
| "count": 12, |
| "accuracy": 1.0, |
| "log_loss": 0.03788952637977832, |
| "brier_score": 0.0050523037056264515, |
| "ece_15": 0.03653778849951328, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/discovery/discovery": { |
| "count": 18, |
| "accuracy": 0.6111111111111112, |
| "log_loss": 0.5954361333409931, |
| "brier_score": 0.41142857572844194, |
| "ece_15": 0.165418858470979, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ethics/justice": { |
| "count": 12, |
| "accuracy": 0.4166666666666667, |
| "log_loss": 0.8168401025841542, |
| "brier_score": 0.6171549802608335, |
| "ece_15": 0.2661442010375547, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/glue/cola": { |
| "count": 5, |
| "accuracy": 1.0, |
| "log_loss": 0.38696686980348943, |
| "brier_score": 0.20869269766165122, |
| "ece_15": 0.3189900777296757, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/banking77": { |
| "count": 17, |
| "accuracy": 0.9411764705882353, |
| "log_loss": 0.24170933298917752, |
| "brier_score": 0.1393861937489721, |
| "ece_15": 0.12330359707361695, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/cladder": { |
| "count": 12, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.6236875694643325, |
| "brier_score": 0.4316312028961662, |
| "ece_15": 0.18048006360986793, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ethics/commonsense": { |
| "count": 15, |
| "accuracy": 0.4666666666666667, |
| "log_loss": 0.7447986138193108, |
| "brier_score": 0.5514197683196601, |
| "ece_15": 0.28991239771096083, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/scicite": { |
| "count": 11, |
| "accuracy": 0.8181818181818182, |
| "log_loss": 0.4318331251125241, |
| "brier_score": 0.2762478171410345, |
| "ece_15": 0.21646846654982987, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/dyda_da": { |
| "count": 10, |
| "accuracy": 0.9, |
| "log_loss": 0.22769926066223617, |
| "brier_score": 0.1448665318240736, |
| "ece_15": 0.10335780417472865, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ag_news": { |
| "count": 15, |
| "accuracy": 0.8666666666666667, |
| "log_loss": 0.3464048347622382, |
| "brier_score": 0.20780113035510264, |
| "ece_15": 0.11026379618878748, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/maptask": { |
| "count": 14, |
| "accuracy": 0.7857142857142857, |
| "log_loss": 0.5758575083844282, |
| "brier_score": 0.38075886257747904, |
| "ece_15": 0.23066958122332948, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/dynahate": { |
| "count": 12, |
| "accuracy": 0.5, |
| "log_loss": 0.8759301272164728, |
| "brier_score": 0.6438679618411262, |
| "ece_15": 0.3719697093628312, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/probability_words_nli/reasoning_2hop": { |
| "count": 13, |
| "accuracy": 0.46153846153846156, |
| "log_loss": 0.689490939631528, |
| "brier_score": 0.49638137510895075, |
| "ece_15": 0.07074967470947083, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/counterfactually-augmented-imdb": { |
| "count": 10, |
| "accuracy": 1.0, |
| "log_loss": 0.012876311703912841, |
| "brier_score": 0.0009432784254639257, |
| "ece_15": 0.012631961883189113, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweets_hate_speech_detection": { |
| "count": 18, |
| "accuracy": 1.0, |
| "log_loss": 0.07311481581271477, |
| "brier_score": 0.027589875296408786, |
| "ece_15": 0.06404904284035837, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ethics/deontology": { |
| "count": 11, |
| "accuracy": 0.2727272727272727, |
| "log_loss": 0.9967284322203667, |
| "brier_score": 0.7783652246620442, |
| "ece_15": 0.4382173972135583, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/hope_edi/english": { |
| "count": 10, |
| "accuracy": 1.0, |
| "log_loss": 0.007112887410789044, |
| "brier_score": 0.00014214220236576544, |
| "ece_15": 0.007077070666527696, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/text-level-bias": { |
| "count": 12, |
| "accuracy": 0.75, |
| "log_loss": 0.6438488726507172, |
| "brier_score": 0.44387673831099117, |
| "ece_15": 0.25993005223611526, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/paws/labeled_final": { |
| "count": 15, |
| "accuracy": 0.5333333333333333, |
| "log_loss": 0.7584004733638848, |
| "brier_score": 0.5533706523188272, |
| "ece_15": 0.37515675293369316, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/avicenna": { |
| "count": 8, |
| "accuracy": 0.625, |
| "log_loss": 0.5898905853457523, |
| "brier_score": 0.4025616557756032, |
| "ece_15": 0.16434217157825676, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/subjectivity": { |
| "count": 15, |
| "accuracy": 0.5333333333333333, |
| "log_loss": 0.6410537686048884, |
| "brier_score": 0.4486384856450565, |
| "ece_15": 0.2284194604768063, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/v1/gen_train234_test2to10": { |
| "count": 16, |
| "accuracy": 0.8125, |
| "log_loss": 0.64033753288312, |
| "brier_score": 0.4220114421078465, |
| "ece_15": 0.2872270031041739, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/folio": { |
| "count": 8, |
| "accuracy": 0.875, |
| "log_loss": 0.21804968262827318, |
| "brier_score": 0.12450894983540306, |
| "ece_15": 0.1703584617231207, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/text_emotion": { |
| "count": 9, |
| "accuracy": 0.7777777777777778, |
| "log_loss": 0.49266944052826606, |
| "brier_score": 0.3406614797217879, |
| "ece_15": 0.1506212859141982, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/sentiment_nuclear_power": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.17467028431924653, |
| "brier_score": 0.05137043313221998, |
| "ece_15": 0.1602661514530047, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/corporate-messaging": { |
| "count": 8, |
| "accuracy": 0.625, |
| "log_loss": 0.5497931414193065, |
| "brier_score": 0.39431896296190316, |
| "ece_15": 0.30576164545191603, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/meld_s": { |
| "count": 9, |
| "accuracy": 0.7777777777777778, |
| "log_loss": 0.5254718465987288, |
| "brier_score": 0.34742950351040947, |
| "ece_15": 0.20713230311421363, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/args_me": { |
| "count": 13, |
| "accuracy": 0.7692307692307693, |
| "log_loss": 0.5202640528301214, |
| "brier_score": 0.34560271922615027, |
| "ece_15": 0.11076116933396576, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/super_glue/wic": { |
| "count": 9, |
| "accuracy": 0.4444444444444444, |
| "log_loss": 0.7233159938786649, |
| "brier_score": 0.5321372605648536, |
| "ece_15": 0.2835889936225653, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/glue/qqp": { |
| "count": 9, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.472860533209498, |
| "brier_score": 0.32954043690628915, |
| "ece_15": 0.3062769937571913, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/rumoureval_2019/RumourEval2019": { |
| "count": 6, |
| "accuracy": 0.8333333333333334, |
| "log_loss": 0.3280483704036526, |
| "brier_score": 0.2006143734456156, |
| "ece_15": 0.24064342766884242, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/fake-news": { |
| "count": 13, |
| "accuracy": 0.6923076923076923, |
| "log_loss": 0.6709165309235322, |
| "brier_score": 0.4706277677626032, |
| "ece_15": 0.29453356629512945, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ethos/binary": { |
| "count": 10, |
| "accuracy": 0.7, |
| "log_loss": 0.46177562071673295, |
| "brier_score": 0.3461469149331206, |
| "ece_15": 0.2671010778814704, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/stance_feminist": { |
| "count": 13, |
| "accuracy": 0.6153846153846154, |
| "log_loss": 0.6382088354794687, |
| "brier_score": 0.44395124565715416, |
| "ece_15": 0.15887413713861823, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/trec": { |
| "count": 10, |
| "accuracy": 0.8, |
| "log_loss": 0.5096994058320339, |
| "brier_score": 0.3342729797729512, |
| "ece_15": 0.1847118449499415, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/snips_built_in_intents": { |
| "count": 4, |
| "accuracy": 0.5, |
| "log_loss": 0.8273618963676559, |
| "brier_score": 0.6364052931549977, |
| "ece_15": 0.41655744331081146, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/tasksource-instruct-v0": { |
| "count": 997, |
| "accuracy": 0.7121364092276831, |
| "log_loss": 0.6544185656310946, |
| "brier_score": 0.3905858788701158, |
| "ece_15": 0.027067872892760816, |
| "ordinal_mae": 0.3727614025515071, |
| "ordinal_count": 997 |
| }, |
| "kind/score": { |
| "count": 997, |
| "accuracy": 0.7121364092276831, |
| "log_loss": 0.6544185656310946, |
| "brier_score": 0.3905858788701158, |
| "ece_15": 0.027067872892760816, |
| "ordinal_mae": 0.3727614025515071, |
| "ordinal_count": 997 |
| }, |
| "task/tasksource/tasksource-instruct-v0/yelp_review_full/yelp_review_full": { |
| "count": 497, |
| "accuracy": 0.6841046277665996, |
| "log_loss": 0.7308296192021128, |
| "brier_score": 0.4283989024747208, |
| "ece_15": 0.054685767322741385, |
| "ordinal_mae": 0.3974469604936274, |
| "ordinal_count": 497 |
| }, |
| "task/tasksource/tasksource-instruct-v0/tweet_eval/sentiment": { |
| "count": 500, |
| "accuracy": 0.74, |
| "log_loss": 0.5784659783815007, |
| "brier_score": 0.3529997334071374, |
| "ece_15": 0.04281764922557473, |
| "ordinal_mae": 0.34822395795703953, |
| "ordinal_count": 500 |
| }, |
| "source/nyu-mll/multi_nli": { |
| "count": 2000, |
| "accuracy": 0.907, |
| "log_loss": 0.2503782744702875, |
| "brier_score": 0.1398590301676452, |
| "ece_15": 0.009440014677003127, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/nyu-mll/multi_nli/nyu-mll/multi_nli/default": { |
| "count": 2000, |
| "accuracy": 0.907, |
| "log_loss": 0.2503782744702875, |
| "brier_score": 0.1398590301676452, |
| "ece_15": 0.009440014677003127, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/facebook/anli": { |
| "count": 2000, |
| "accuracy": 0.5785, |
| "log_loss": 1.0241603772169452, |
| "brier_score": 0.598273250581322, |
| "ece_15": 0.1877582769905423, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/facebook/anli/facebook/anli/plain_text": { |
| "count": 2000, |
| "accuracy": 0.5785, |
| "log_loss": 1.0241603772169452, |
| "brier_score": 0.598273250581322, |
| "ece_15": 0.1877582769905423, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/defeasible-nli": { |
| "count": 2000, |
| "accuracy": 0.8745, |
| "log_loss": 0.2953982323779266, |
| "brier_score": 0.18100591800831833, |
| "ece_15": 0.01850405280593437, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/defeasible-nli/tasksource/defeasible-nli/social": { |
| "count": 1217, |
| "accuracy": 0.9063270336894002, |
| "log_loss": 0.21867233719694193, |
| "brier_score": 0.13097456920549835, |
| "ece_15": 0.019611015496954282, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/defeasible-nli/tasksource/defeasible-nli/atomic": { |
| "count": 542, |
| "accuracy": 0.8173431734317343, |
| "log_loss": 0.42832116177766, |
| "brier_score": 0.26720082054403005, |
| "ece_15": 0.0544780985763878, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/defeasible-nli/tasksource/defeasible-nli/snli": { |
| "count": 241, |
| "accuracy": 0.8423236514522822, |
| "log_loss": 0.38390938051320106, |
| "brier_score": 0.23980473260863053, |
| "ece_15": 0.0641852458826681, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/FOL-nli": { |
| "count": 2000, |
| "accuracy": 0.834, |
| "log_loss": 0.41933690098607834, |
| "brier_score": 0.2384377747731735, |
| "ece_15": 0.013194922042872767, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/FOL-nli/tasksource/FOL-nli/default": { |
| "count": 2000, |
| "accuracy": 0.834, |
| "log_loss": 0.41933690098607834, |
| "brier_score": 0.2384377747731735, |
| "ece_15": 0.013194922042872767, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/doc-nli": { |
| "count": 1776, |
| "accuracy": 0.8693693693693694, |
| "log_loss": 0.4702459141944009, |
| "brier_score": 0.21602108515578877, |
| "ece_15": 0.09066421378716669, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/doc-nli/tasksource/doc-nli/default": { |
| "count": 1776, |
| "accuracy": 0.8693693693693694, |
| "log_loss": 0.4702459141944009, |
| "brier_score": 0.21602108515578877, |
| "ece_15": 0.09066421378716669, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/bigbench": { |
| "count": 1896, |
| "accuracy": 0.7600210970464135, |
| "log_loss": 0.5945892380200355, |
| "brier_score": 0.29760266888297954, |
| "ece_15": 0.027778599832484385, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/vitaminc_fact_verification": { |
| "count": 262, |
| "accuracy": 0.8625954198473282, |
| "log_loss": 0.35488678574675664, |
| "brier_score": 0.19865901558972499, |
| "ece_15": 0.05097064656617047, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/mnist_ascii": { |
| "count": 349, |
| "accuracy": 0.8223495702005731, |
| "log_loss": 0.6204249369841943, |
| "brier_score": 0.269407626056375, |
| "ece_15": 0.05139675194333151, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/cs_algorithms": { |
| "count": 6, |
| "accuracy": 0.8333333333333334, |
| "log_loss": 0.8318294957033027, |
| "brier_score": 0.4208567256331556, |
| "ece_15": 0.3591375294684862, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/language_identification": { |
| "count": 39, |
| "accuracy": 0.2564102564102564, |
| "log_loss": 2.1488308328863375, |
| "brier_score": 0.8601207639550233, |
| "ece_15": 0.13212908193304784, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/unit_conversion": { |
| "count": 73, |
| "accuracy": 0.8493150684931506, |
| "log_loss": 0.3697982174579797, |
| "brier_score": 0.18315480409037768, |
| "ece_15": 0.08730104979681678, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/bbq_lite_json": { |
| "count": 61, |
| "accuracy": 0.9836065573770492, |
| "log_loss": 0.09371779188813445, |
| "brier_score": 0.03888321894138145, |
| "ece_15": 0.023381337788408445, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/movie_dialog_same_or_different": { |
| "count": 197, |
| "accuracy": 0.7563451776649747, |
| "log_loss": 0.48856434791550046, |
| "brier_score": 0.32186388472172084, |
| "ece_15": 0.04205984419347936, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/fact_checker": { |
| "count": 34, |
| "accuracy": 0.8235294117647058, |
| "log_loss": 0.44331493864485316, |
| "brier_score": 0.2873391048539894, |
| "ece_15": 0.11020671559400137, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/social_support": { |
| "count": 2, |
| "accuracy": 0.5, |
| "log_loss": 0.8493375201806352, |
| "brier_score": 0.6007501590788596, |
| "ece_15": 0.4889538706042209, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/hyperbaton": { |
| "count": 179, |
| "accuracy": 1.0, |
| "log_loss": 0.0030401693947674866, |
| "brier_score": 0.0010691606906897185, |
| "ece_15": 0.0027256753758118967, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/formal_fallacies_syllogisms_negation": { |
| "count": 70, |
| "accuracy": 0.7714285714285715, |
| "log_loss": 0.43476594877696134, |
| "brier_score": 0.2832662727357645, |
| "ece_15": 0.08273752659285324, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/contextual_parametric_knowledge_conflicts": { |
| "count": 72, |
| "accuracy": 1.0, |
| "log_loss": 0.001405064876897654, |
| "brier_score": 0.0001308076321174364, |
| "ece_15": 0.001370854595172991, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/abstract_narrative_understanding": { |
| "count": 5, |
| "accuracy": 0.0, |
| "log_loss": 2.2986971420332334, |
| "brier_score": 0.9845303400186026, |
| "ece_15": 0.29150496136292425, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/goal_step_wikihow": { |
| "count": 34, |
| "accuracy": 0.7941176470588235, |
| "log_loss": 0.4565665629357099, |
| "brier_score": 0.2735529756554902, |
| "ece_15": 0.18219847221859173, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/real_or_fake_text": { |
| "count": 47, |
| "accuracy": 0.3404255319148936, |
| "log_loss": 1.8897484761287116, |
| "brier_score": 0.7925847597712499, |
| "ece_15": 0.1648527176909287, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/symbol_interpretation": { |
| "count": 5, |
| "accuracy": 0.0, |
| "log_loss": 1.7534130977801385, |
| "brier_score": 0.8562569910857161, |
| "ece_15": 0.30216957340215506, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/arithmetic": { |
| "count": 74, |
| "accuracy": 1.0, |
| "log_loss": 0.0005265116178442115, |
| "brier_score": 1.4785713576490667e-05, |
| "ece_15": 0.000522639379963306, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/timedial": { |
| "count": 18, |
| "accuracy": 1.0, |
| "log_loss": 0.056595530228390686, |
| "brier_score": 0.027228992776938025, |
| "ece_15": 0.0432306484665529, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/discourse_marker_prediction": { |
| "count": 4, |
| "accuracy": 0.0, |
| "log_loss": 2.5476397296930466, |
| "brier_score": 0.962763556742865, |
| "ece_15": 0.284402084812965, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/strategyqa": { |
| "count": 10, |
| "accuracy": 0.7, |
| "log_loss": 0.6883938512526012, |
| "brier_score": 0.4935897073703076, |
| "ece_15": 0.2764223308076861, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/parsinlu_qa": { |
| "count": 9, |
| "accuracy": 0.2222222222222222, |
| "log_loss": 1.4908273487862438, |
| "brier_score": 0.7630356272411518, |
| "ece_15": 0.2220835161829854, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/logical_deduction": { |
| "count": 9, |
| "accuracy": 0.3333333333333333, |
| "log_loss": 1.1460686227267756, |
| "brier_score": 0.6042348683137022, |
| "ece_15": 0.40142515454530364, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/elementary_math_qa": { |
| "count": 159, |
| "accuracy": 0.3836477987421384, |
| "log_loss": 1.3701805967852991, |
| "brier_score": 0.6855753610097235, |
| "ece_15": 0.11133990684516334, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/tracking_shuffled_objects": { |
| "count": 20, |
| "accuracy": 0.25, |
| "log_loss": 1.6748300871561934, |
| "brier_score": 0.824427439519923, |
| "ece_15": 0.21041502870187445, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/snarks": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.6623853823577757, |
| "brier_score": 0.46924813208835703, |
| "ece_15": 0.4843800842769844, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/color": { |
| "count": 18, |
| "accuracy": 0.3333333333333333, |
| "log_loss": 1.699628632517954, |
| "brier_score": 0.7839887278297692, |
| "ece_15": 0.08018102142059981, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/play_dialog_same_or_different": { |
| "count": 13, |
| "accuracy": 0.7692307692307693, |
| "log_loss": 0.5488956218068058, |
| "brier_score": 0.36429207498555916, |
| "ece_15": 0.15198860578167028, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/indic_cause_and_effect": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 0.8892930303171471, |
| "brier_score": 0.6901270572143436, |
| "ece_15": 0.5834873901945349, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/logic_grid_puzzle": { |
| "count": 10, |
| "accuracy": 0.5, |
| "log_loss": 1.0825882747004594, |
| "brier_score": 0.576639404561162, |
| "ece_15": 0.2571618241351024, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/sports_understanding": { |
| "count": 4, |
| "accuracy": 0.0, |
| "log_loss": 0.7403959729739352, |
| "brier_score": 0.5469476216808751, |
| "ece_15": 0.5220811721202167, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/fantasy_reasoning": { |
| "count": 2, |
| "accuracy": 0.5, |
| "log_loss": 0.4869129516784711, |
| "brier_score": 0.3205736473455824, |
| "ece_15": 0.3623005127202489, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/question_selection": { |
| "count": 7, |
| "accuracy": 1.0, |
| "log_loss": 0.1005091247472382, |
| "brier_score": 0.022610158498892124, |
| "ece_15": 0.09004107013490147, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/code_line_description": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.19319191191923257, |
| "brier_score": 0.04155747507791947, |
| "ece_15": 0.175676238596981, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/kanji_ascii": { |
| "count": 2, |
| "accuracy": 0.5, |
| "log_loss": 1.367799367061274, |
| "brier_score": 0.6712085426601548, |
| "ece_15": 0.43753104493362033, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/analogical_similarity": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 2.4516314896323568, |
| "brier_score": 0.9782674612330993, |
| "ece_15": 0.2038757832150454, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/winowhy": { |
| "count": 11, |
| "accuracy": 0.8181818181818182, |
| "log_loss": 0.3915310422191835, |
| "brier_score": 0.24404242409037566, |
| "ece_15": 0.28511325502240886, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/salient_translation_error_detection": { |
| "count": 3, |
| "accuracy": 0.3333333333333333, |
| "log_loss": 1.9328707468356596, |
| "brier_score": 0.8719910893343025, |
| "ece_15": 0.12157010695826635, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/rhyming": { |
| "count": 2, |
| "accuracy": 1.0, |
| "log_loss": 0.11396708983298824, |
| "brier_score": 0.024563529505863907, |
| "ece_15": 0.10573112993952138, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/movie_recommendation": { |
| "count": 3, |
| "accuracy": 1.0, |
| "log_loss": 0.1659423715133119, |
| "brier_score": 0.0542555458314919, |
| "ece_15": 0.14416865959328237, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/analytic_entailment": { |
| "count": 2, |
| "accuracy": 1.0, |
| "log_loss": 0.2709319997053973, |
| "brier_score": 0.12174154730231448, |
| "ece_15": 0.23304139330699092, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/logical_fallacy_detection": { |
| "count": 7, |
| "accuracy": 0.7142857142857143, |
| "log_loss": 0.7197091284961428, |
| "brier_score": 0.4175653521370116, |
| "ece_15": 0.18212288959487993, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/temporal_sequences": { |
| "count": 3, |
| "accuracy": 1.0, |
| "log_loss": 0.028569655708020506, |
| "brier_score": 0.002255009734825616, |
| "ece_15": 0.027881549094308806, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/kannada": { |
| "count": 3, |
| "accuracy": 0.3333333333333333, |
| "log_loss": 1.2825517804250592, |
| "brier_score": 0.7064254366841242, |
| "ece_15": 0.45584604797760275, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/navigate": { |
| "count": 2, |
| "accuracy": 0.5, |
| "log_loss": 0.7699403206031636, |
| "brier_score": 0.5760357935969403, |
| "ece_15": 0.5348180127394855, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/social_iqa": { |
| "count": 6, |
| "accuracy": 0.8333333333333334, |
| "log_loss": 0.38270773166685323, |
| "brier_score": 0.204426476469744, |
| "ece_15": 0.09899626749410305, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/implicatures": { |
| "count": 2, |
| "accuracy": 1.0, |
| "log_loss": 0.2609504935782483, |
| "brier_score": 0.16289221358059233, |
| "ece_15": 0.20433333601916398, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/penguins_in_a_table": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 2.0413339243079247, |
| "brier_score": 1.0164700566040343, |
| "ece_15": 0.37641428086742673, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/epistemic_reasoning": { |
| "count": 8, |
| "accuracy": 1.0, |
| "log_loss": 0.062205730006655614, |
| "brier_score": 0.014921854794889326, |
| "ece_15": 0.05791584386090702, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/reasoning_about_colored_objects": { |
| "count": 5, |
| "accuracy": 0.6, |
| "log_loss": 1.3339993281042424, |
| "brier_score": 0.22112079410333524, |
| "ece_15": 0.01418309048427211, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/strange_stories": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 0.8259394198788436, |
| "brier_score": 0.6320848362963996, |
| "ece_15": 0.5621765008857981, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/what_is_the_tao": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 0.8981232641398401, |
| "brier_score": 0.7025073974022347, |
| "ece_15": 0.5926665999540697, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/international_phonetic_alphabet_nli": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 1.412482580110375, |
| "brier_score": 0.8372991364600517, |
| "ece_15": 0.44100307405613826, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/hindu_knowledge": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.610034782202861, |
| "brier_score": 0.29091329010914513, |
| "ece_15": 0.4566680295369835, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/cryobiology_spanish": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.16372883995909746, |
| "brier_score": 0.045618793949402325, |
| "ece_15": 0.15102780199255084, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/intent_recognition": { |
| "count": 4, |
| "accuracy": 1.0, |
| "log_loss": 0.17035605618560726, |
| "brier_score": 0.07253454942435356, |
| "ece_15": 0.12823069900149586, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/metaphor_boolean": { |
| "count": 2, |
| "accuracy": 1.0, |
| "log_loss": 0.14047367234871594, |
| "brier_score": 0.05006837749950187, |
| "ece_15": 0.1257659278458747, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/suicide_risk": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 1.308353365222305, |
| "brier_score": 0.7170367594867829, |
| "ece_15": 0.3062500465196527, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/logical_args": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.973522639380097, |
| "brier_score": 0.5083220596275099, |
| "ece_15": 0.6222499854758178, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/ruin_names": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 1.621400668411321, |
| "brier_score": 0.9226563278646717, |
| "ece_15": 0.4740698200688341, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/causal_judgment": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 0.8888923460403997, |
| "brier_score": 0.6935808166729854, |
| "ece_15": 0.588889130767832, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/english_proverbs": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 3.6431405937707138, |
| "brier_score": 1.8141796471839273, |
| "ece_15": 0.9297606479356774, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/empirical_judgments": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 1.4260709410856809, |
| "brier_score": 0.9580650403082998, |
| "ece_15": 0.5946270117054103, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/physical_intuition": { |
| "count": 2, |
| "accuracy": 0.5, |
| "log_loss": 1.1295488932551905, |
| "brier_score": 0.6208961702353901, |
| "ece_15": 0.45723326676247267, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/similarities_abstraction": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.5405214594683632, |
| "brier_score": 0.3087070153736353, |
| "ece_15": 0.41755554800320716, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/swahili_english_proverbs": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.5899544056173409, |
| "brier_score": 0.29824981962565655, |
| "ece_15": 0.44564744047896465, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/date_understanding": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.07117941218959864, |
| "brier_score": 0.007270133709711194, |
| "ece_15": 0.06870520849960926, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/medical_questions_russian": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.3484445810050557, |
| "brier_score": 0.1731248996897365, |
| "ece_15": 0.29421497216298875, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/anachronisms": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 0.8888923460403997, |
| "brier_score": 0.6935808166729854, |
| "ece_15": 0.588889130767832, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/simple_ethical_questions": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 1.5109795748836918, |
| "brier_score": 1.0218176323160264, |
| "ece_15": 0.6557421165739532, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/dark_humor_detection": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.4246305677439776, |
| "brier_score": 0.2394162871587119, |
| "ece_15": 0.34598864660470574, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/irony_identification": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 1.0110184794041372, |
| "brier_score": 0.8076268693054449, |
| "ece_15": 0.6345343098156068, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/geometric_shapes": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 2.4923691485595008, |
| "brier_score": 0.691683099863767, |
| "ece_15": 0.16429964806133548, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/cause_and_effect": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.06384292136816497, |
| "brier_score": 0.007650263214002998, |
| "ece_15": 0.0618476483546585, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/moral_permissibility": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 0.8084917378836574, |
| "brier_score": 0.6148749930268649, |
| "ece_15": 0.5544704649604273, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/phrase_relatedness": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 1.0628781310584738, |
| "brier_score": 0.7868765837267643, |
| "ece_15": 0.5969015237378581, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/crass_ai": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.42296553932806247, |
| "brier_score": 0.18061264404143007, |
| "ece_15": 0.3448987920493549, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/figure_of_speech_detection": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 1.3302745577462225, |
| "brier_score": 0.6255954829711331, |
| "ece_15": 0.7355953430134712, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/Rowan/hellaswag": { |
| "count": 2000, |
| "accuracy": 0.86, |
| "log_loss": 0.36216262199223165, |
| "brier_score": 0.19636745240119854, |
| "ece_15": 0.024481463629632053, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/Rowan/hellaswag/Rowan/hellaswag/default": { |
| "count": 2000, |
| "accuracy": 0.86, |
| "log_loss": 0.36216262199223165, |
| "brier_score": 0.19636745240119854, |
| "ece_15": 0.024481463629632053, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/google/boolq": { |
| "count": 828, |
| "accuracy": 0.8599033816425121, |
| "log_loss": 0.31896083051883756, |
| "brier_score": 0.19506497232710848, |
| "ece_15": 0.02807812588868236, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "kind/noul": { |
| "count": 828, |
| "accuracy": 0.8599033816425121, |
| "log_loss": 0.31896083051883756, |
| "brier_score": 0.19506497232710848, |
| "ece_15": 0.02807812588868236, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/google/boolq/google/boolq/default": { |
| "count": 828, |
| "accuracy": 0.8599033816425121, |
| "log_loss": 0.31896083051883756, |
| "brier_score": 0.19506497232710848, |
| "ece_15": 0.02807812588868236, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| } |
| }, |
| "calibrated": { |
| "overall": { |
| "count": 17476, |
| "accuracy": 0.7985809109636073, |
| "log_loss": 0.4746148257879429, |
| "brier_score": 0.2768548946790544, |
| "ece_15": 0.014261655942813945, |
| "ordinal_mae": 0.3923422177916232, |
| "ordinal_count": 997 |
| }, |
| "source/tasksource/zero-shot-label-nli": { |
| "count": 1979, |
| "accuracy": 0.7281455280444669, |
| "log_loss": 0.5150510230997565, |
| "brier_score": 0.3481778900388312, |
| "ece_15": 0.029566299660355483, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "kind/choice": { |
| "count": 15651, |
| "accuracy": 0.8008433965880775, |
| "log_loss": 0.4706446079609039, |
| "brier_score": 0.273581528978314, |
| "ece_15": 0.01438703174072128, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/rotten_tomatoes": { |
| "count": 9, |
| "accuracy": 0.8888888888888888, |
| "log_loss": 0.17256064504520743, |
| "brier_score": 0.09563495675656403, |
| "ece_15": 0.13803715628650462, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/imdb": { |
| "count": 14, |
| "accuracy": 0.8571428571428571, |
| "log_loss": 0.2043680763935434, |
| "brier_score": 0.1437798160522632, |
| "ece_15": 0.12940505043211167, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/political-media-bias": { |
| "count": 10, |
| "accuracy": 0.9, |
| "log_loss": 0.43756013769150537, |
| "brier_score": 0.2636482502143494, |
| "ece_15": 0.1966197457049946, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/Touche23-ValueEval": { |
| "count": 20, |
| "accuracy": 0.9, |
| "log_loss": 0.3125276783455419, |
| "brier_score": 0.18798907051468547, |
| "ece_15": 0.08847487206515645, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ethics/virtue": { |
| "count": 18, |
| "accuracy": 0.7777777777777778, |
| "log_loss": 0.5693555002594094, |
| "brier_score": 0.3792036244270036, |
| "ece_15": 0.10372804291821851, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/humicroedit/subtask-2": { |
| "count": 17, |
| "accuracy": 0.5294117647058824, |
| "log_loss": 0.6614352674072255, |
| "brier_score": 0.46932479738971483, |
| "ece_15": 0.14464434923043992, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/meld_e": { |
| "count": 15, |
| "accuracy": 0.5333333333333333, |
| "log_loss": 0.7276548672981771, |
| "brier_score": 0.5187603945589412, |
| "ece_15": 0.2354493050902806, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/citation_intent": { |
| "count": 13, |
| "accuracy": 0.8461538461538461, |
| "log_loss": 0.3789059143915369, |
| "brier_score": 0.2373359117317879, |
| "ece_15": 0.1456462518045907, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/lex_glue/ledgar": { |
| "count": 21, |
| "accuracy": 0.9047619047619048, |
| "log_loss": 0.22369106570247702, |
| "brier_score": 0.12306482765477454, |
| "ece_15": 0.1387134117209304, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/dynasent/dynabench.dynasent.r2.all/r2": { |
| "count": 18, |
| "accuracy": 0.8333333333333334, |
| "log_loss": 0.44162271765029093, |
| "brier_score": 0.28828683621645124, |
| "ece_15": 0.22975229357153953, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/lexical_relation_classification/CogALexV": { |
| "count": 11, |
| "accuracy": 0.8181818181818182, |
| "log_loss": 0.3456359878981195, |
| "brier_score": 0.19870445498302056, |
| "ece_15": 0.20165815414173383, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/emotion": { |
| "count": 16, |
| "accuracy": 0.875, |
| "log_loss": 0.34376986287818917, |
| "brier_score": 0.20519079535877968, |
| "ece_15": 0.09514125326433424, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/lexical_relation_classification/K&H+N": { |
| "count": 10, |
| "accuracy": 0.6, |
| "log_loss": 0.5915581669101454, |
| "brier_score": 0.4184836288171258, |
| "ece_15": 0.20255649840006762, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/scientific-exaggeration-detection": { |
| "count": 15, |
| "accuracy": 0.7333333333333333, |
| "log_loss": 0.608794267993121, |
| "brier_score": 0.42107799414180797, |
| "ece_15": 0.19003799809500752, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/yahoo_answers_topics": { |
| "count": 21, |
| "accuracy": 0.7142857142857143, |
| "log_loss": 0.40972729243384864, |
| "brier_score": 0.274516712427393, |
| "ece_15": 0.2544248655305512, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/poem_sentiment": { |
| "count": 17, |
| "accuracy": 0.8235294117647058, |
| "log_loss": 0.4880403183118992, |
| "brier_score": 0.3122694095068833, |
| "ece_15": 0.17964045266648512, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/stance_climate": { |
| "count": 18, |
| "accuracy": 0.8333333333333334, |
| "log_loss": 0.4937934037550049, |
| "brier_score": 0.31101628048908425, |
| "ece_15": 0.15890521099601174, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/fever-evidence-related/mwong--fever-related": { |
| "count": 19, |
| "accuracy": 0.7894736842105263, |
| "log_loss": 0.4462165797874618, |
| "brier_score": 0.3025353854526682, |
| "ece_15": 0.18205539811734292, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/political-media-message": { |
| "count": 16, |
| "accuracy": 0.6875, |
| "log_loss": 0.5322535589767222, |
| "brier_score": 0.3600028819062956, |
| "ece_15": 0.1757269370396578, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/dynasent/dynabench.dynasent.r1.all/r1": { |
| "count": 16, |
| "accuracy": 0.875, |
| "log_loss": 0.5440966465171441, |
| "brier_score": 0.312700084493129, |
| "ece_15": 0.24713472824159738, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/dyda_e": { |
| "count": 17, |
| "accuracy": 1.0, |
| "log_loss": 0.08485858032401881, |
| "brier_score": 0.02269360542409514, |
| "ece_15": 0.07813653093721144, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/CREAK": { |
| "count": 21, |
| "accuracy": 0.7619047619047619, |
| "log_loss": 0.5212072206379214, |
| "brier_score": 0.34474082324692085, |
| "ece_15": 0.10817107814667283, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/sarcasm": { |
| "count": 18, |
| "accuracy": 0.6111111111111112, |
| "log_loss": 0.6513613301144051, |
| "brier_score": 0.46036234567780565, |
| "ece_15": 0.09331768953710873, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/medical_questions_pairs": { |
| "count": 8, |
| "accuracy": 0.875, |
| "log_loss": 0.39699502358732586, |
| "brier_score": 0.2447779016438637, |
| "ece_15": 0.29395786878235175, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/stance_abortion": { |
| "count": 16, |
| "accuracy": 0.4375, |
| "log_loss": 0.7542367894281041, |
| "brier_score": 0.5588060530610977, |
| "ece_15": 0.2452767656184259, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ade_corpus_v2/Ade_corpus_v2_classification": { |
| "count": 18, |
| "accuracy": 0.7222222222222222, |
| "log_loss": 0.6171563916332724, |
| "brier_score": 0.4215355870058244, |
| "ece_15": 0.15156889948056576, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/commonsense_qa_2.0": { |
| "count": 9, |
| "accuracy": 0.7777777777777778, |
| "log_loss": 0.5740248261648443, |
| "brier_score": 0.3835991030942798, |
| "ece_15": 0.32305103202107044, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/health_fact": { |
| "count": 17, |
| "accuracy": 0.8823529411764706, |
| "log_loss": 0.3799747233587511, |
| "brier_score": 0.224673327341037, |
| "ece_15": 0.1908523220665037, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/political-media-audience": { |
| "count": 23, |
| "accuracy": 0.7391304347826086, |
| "log_loss": 0.43048470514871817, |
| "brier_score": 0.28836578407327723, |
| "ece_15": 0.1571658196702359, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/circa": { |
| "count": 21, |
| "accuracy": 0.9523809523809523, |
| "log_loss": 0.20207630546593036, |
| "brier_score": 0.10456373037407377, |
| "ece_15": 0.09698853597682473, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/iemocap": { |
| "count": 16, |
| "accuracy": 0.625, |
| "log_loss": 0.7001973280574378, |
| "brier_score": 0.4951080252788981, |
| "ece_15": 0.2636215449282059, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/hate": { |
| "count": 16, |
| "accuracy": 0.75, |
| "log_loss": 0.5694836129910303, |
| "brier_score": 0.3894436624529705, |
| "ece_15": 0.24167276881353353, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/pdtb": { |
| "count": 11, |
| "accuracy": 0.6363636363636364, |
| "log_loss": 0.532122988581282, |
| "brier_score": 0.36069207135217235, |
| "ece_15": 0.30326938420735305, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/winowhy": { |
| "count": 20, |
| "accuracy": 0.55, |
| "log_loss": 0.7096738763478374, |
| "brier_score": 0.5155209107452766, |
| "ece_15": 0.10611770381653915, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/hyperpartisan_news": { |
| "count": 11, |
| "accuracy": 0.5454545454545454, |
| "log_loss": 0.5872962153868201, |
| "brier_score": 0.4030972043390918, |
| "ece_15": 0.22812215434209018, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/sms_spam": { |
| "count": 12, |
| "accuracy": 1.0, |
| "log_loss": 0.22336569340935367, |
| "brier_score": 0.09830953739400583, |
| "ece_15": 0.1928052185190914, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/lexical_relation_classification/ROOT09": { |
| "count": 21, |
| "accuracy": 0.8571428571428571, |
| "log_loss": 0.34481671708972744, |
| "brier_score": 0.20432472138559268, |
| "ece_15": 0.1817675430094532, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/clcd-english": { |
| "count": 20, |
| "accuracy": 0.7, |
| "log_loss": 0.6079546676739491, |
| "brier_score": 0.41969306964493824, |
| "ece_15": 0.2599639978279463, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/FLUTE": { |
| "count": 14, |
| "accuracy": 0.7857142857142857, |
| "log_loss": 0.503953454241523, |
| "brier_score": 0.32290160753793595, |
| "ece_15": 0.20688674773850485, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/starcon": { |
| "count": 19, |
| "accuracy": 0.8421052631578947, |
| "log_loss": 0.43205797035036947, |
| "brier_score": 0.26577317068670275, |
| "ece_15": 0.2114098705290565, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/lexical_relation_classification/BLESS": { |
| "count": 19, |
| "accuracy": 0.631578947368421, |
| "log_loss": 0.5485365214517407, |
| "brier_score": 0.3922055910234009, |
| "ece_15": 0.14428404474177733, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/lexical_relation_classification/EVALution": { |
| "count": 14, |
| "accuracy": 0.7857142857142857, |
| "log_loss": 0.5028670202095411, |
| "brier_score": 0.3287551982178271, |
| "ece_15": 0.14850521170441494, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/dbpedia_14/dbpedia_14": { |
| "count": 14, |
| "accuracy": 0.9285714285714286, |
| "log_loss": 0.1646743941344153, |
| "brier_score": 0.10731982458567148, |
| "ece_15": 0.0855197860968244, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/strategy-qa": { |
| "count": 17, |
| "accuracy": 0.29411764705882354, |
| "log_loss": 0.7739829588226838, |
| "brier_score": 0.5780573034881167, |
| "ece_15": 0.27884182704283283, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/cycic_classification": { |
| "count": 7, |
| "accuracy": 0.7142857142857143, |
| "log_loss": 0.7615217863587221, |
| "brier_score": 0.5467664666024626, |
| "ece_15": 0.48724005573840085, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/blog_authorship_corpus/horoscope": { |
| "count": 16, |
| "accuracy": 0.5625, |
| "log_loss": 0.70312389399091, |
| "brier_score": 0.5091651624818514, |
| "ece_15": 0.10277775046397664, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/hlgd": { |
| "count": 18, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.665548193664473, |
| "brier_score": 0.459200508879798, |
| "ece_15": 0.19218950828833686, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/paws/labeled_swap": { |
| "count": 14, |
| "accuracy": 0.7857142857142857, |
| "log_loss": 0.47379592260540593, |
| "brier_score": 0.29994824552097865, |
| "ece_15": 0.14454836482459182, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/numer_sense": { |
| "count": 15, |
| "accuracy": 0.8, |
| "log_loss": 0.5487459369316212, |
| "brier_score": 0.36478416406515213, |
| "ece_15": 0.24484791355395863, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/implicit-hate-stg1": { |
| "count": 16, |
| "accuracy": 0.6875, |
| "log_loss": 0.45532596051155305, |
| "brier_score": 0.3066070325732828, |
| "ece_15": 0.15563287814807833, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/EffectiveFeedbackStudentWriting": { |
| "count": 10, |
| "accuracy": 0.8, |
| "log_loss": 0.5593083756911487, |
| "brier_score": 0.3728686199107414, |
| "ece_15": 0.19650588370107785, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/phrase_similarity": { |
| "count": 8, |
| "accuracy": 0.625, |
| "log_loss": 0.6638284934547569, |
| "brier_score": 0.4683562540606814, |
| "ece_15": 0.372191892469806, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/stance_atheism": { |
| "count": 10, |
| "accuracy": 0.5, |
| "log_loss": 0.7712825057315981, |
| "brier_score": 0.5584408107276222, |
| "ece_15": 0.19657111926256307, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/airline-sentiment": { |
| "count": 11, |
| "accuracy": 0.6363636363636364, |
| "log_loss": 0.504217406339039, |
| "brier_score": 0.3403436184092216, |
| "ece_15": 0.25837740839156303, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/MOH": { |
| "count": 13, |
| "accuracy": 0.7692307692307693, |
| "log_loss": 0.5184869174698761, |
| "brier_score": 0.3384519536101372, |
| "ece_15": 0.03818929864206088, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/sciie": { |
| "count": 8, |
| "accuracy": 0.625, |
| "log_loss": 0.739317525450631, |
| "brier_score": 0.5301462318947524, |
| "ece_15": 0.3740972458097762, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/hate_speech18": { |
| "count": 17, |
| "accuracy": 0.8823529411764706, |
| "log_loss": 0.3055792437907351, |
| "brier_score": 0.16390553409019631, |
| "ece_15": 0.11131489439705457, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/racial-bias": { |
| "count": 13, |
| "accuracy": 0.5384615384615384, |
| "log_loss": 0.6668270016040504, |
| "brier_score": 0.4768621467322268, |
| "ece_15": 0.16915274372880487, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/glue/sst2": { |
| "count": 9, |
| "accuracy": 1.0, |
| "log_loss": 0.13307086083993702, |
| "brier_score": 0.04123792653245642, |
| "ece_15": 0.12120467744046365, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/gender-bias": { |
| "count": 15, |
| "accuracy": 0.6, |
| "log_loss": 0.6340322301697444, |
| "brier_score": 0.44419804227259274, |
| "ece_15": 0.1629476255627217, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/offensive": { |
| "count": 11, |
| "accuracy": 0.8181818181818182, |
| "log_loss": 0.3479413973594366, |
| "brier_score": 0.2050944519651372, |
| "ece_15": 0.2641495187945257, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/hate_speech_offensive": { |
| "count": 18, |
| "accuracy": 0.8888888888888888, |
| "log_loss": 0.3248894852475989, |
| "brier_score": 0.17968244222627072, |
| "ece_15": 0.20077340451462125, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/twentyquestions": { |
| "count": 13, |
| "accuracy": 0.5384615384615384, |
| "log_loss": 0.7419794581550404, |
| "brier_score": 0.5402742994409466, |
| "ece_15": 0.2076968861636773, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/sem_eval_2010_task_8": { |
| "count": 16, |
| "accuracy": 0.8125, |
| "log_loss": 0.4168383098254916, |
| "brier_score": 0.26005519515773223, |
| "ece_15": 0.19464997819238283, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/financial_phrasebank/sentences_allagree": { |
| "count": 8, |
| "accuracy": 0.875, |
| "log_loss": 0.2541730803323668, |
| "brier_score": 0.12703388344390798, |
| "ece_15": 0.20992331258846403, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mc_taco": { |
| "count": 9, |
| "accuracy": 0.7777777777777778, |
| "log_loss": 0.5286052625031235, |
| "brier_score": 0.345736530537952, |
| "ece_15": 0.19028143098212277, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/super_glue/multirc": { |
| "count": 8, |
| "accuracy": 0.5, |
| "log_loss": 0.7497257850039698, |
| "brier_score": 0.5525588126457335, |
| "ece_15": 0.30062444358463974, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/TuringBench": { |
| "count": 15, |
| "accuracy": 0.4, |
| "log_loss": 0.755570140823358, |
| "brier_score": 0.5599130360453907, |
| "ece_15": 0.17511077151692198, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/probability_words_nli/reasoning_1hop": { |
| "count": 8, |
| "accuracy": 0.375, |
| "log_loss": 0.7002876889848436, |
| "brier_score": 0.507186821840337, |
| "ece_15": 0.1660644275894566, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/tweet_global_warming": { |
| "count": 15, |
| "accuracy": 0.8, |
| "log_loss": 0.48046953916293555, |
| "brier_score": 0.30678919869459836, |
| "ece_15": 0.08317878036201325, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/blog_authorship_corpus/job": { |
| "count": 13, |
| "accuracy": 0.8461538461538461, |
| "log_loss": 0.4845011136674931, |
| "brier_score": 0.2922380296429058, |
| "ece_15": 0.20668850581035178, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/parade": { |
| "count": 13, |
| "accuracy": 0.6923076923076923, |
| "log_loss": 0.6116721008417533, |
| "brier_score": 0.4262192980752779, |
| "ece_15": 0.37234850254621005, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/probability_words_nli/usnli": { |
| "count": 16, |
| "accuracy": 0.5, |
| "log_loss": 0.6942498287600838, |
| "brier_score": 0.5011157867188752, |
| "ece_15": 0.09139235953725697, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/stac": { |
| "count": 13, |
| "accuracy": 0.6153846153846154, |
| "log_loss": 0.6075320848599852, |
| "brier_score": 0.43642140096598586, |
| "ece_15": 0.3123764525706635, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/quarel": { |
| "count": 22, |
| "accuracy": 0.6363636363636364, |
| "log_loss": 0.6705165299568191, |
| "brier_score": 0.47774899633371637, |
| "ece_15": 0.08448330542479274, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/sem": { |
| "count": 11, |
| "accuracy": 0.8181818181818182, |
| "log_loss": 0.3028548623319954, |
| "brier_score": 0.17471542514637128, |
| "ece_15": 0.15350588550710553, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/hate-speech": { |
| "count": 13, |
| "accuracy": 0.6923076923076923, |
| "log_loss": 0.6807910535063487, |
| "brier_score": 0.4781335908625496, |
| "ece_15": 0.2713823688645954, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/economic-news": { |
| "count": 20, |
| "accuracy": 0.75, |
| "log_loss": 0.5695123143105041, |
| "brier_score": 0.3859877059568563, |
| "ece_15": 0.10402211902773444, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/liar": { |
| "count": 10, |
| "accuracy": 0.5, |
| "log_loss": 0.6719751854867054, |
| "brier_score": 0.4797907271676448, |
| "ece_15": 0.2054697944539603, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/scruples": { |
| "count": 13, |
| "accuracy": 0.7692307692307693, |
| "log_loss": 0.5358296051036985, |
| "brier_score": 0.36056924979934973, |
| "ece_15": 0.16934656436484802, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/open_question_type": { |
| "count": 14, |
| "accuracy": 0.8571428571428571, |
| "log_loss": 0.48088832706134627, |
| "brier_score": 0.3084783132314986, |
| "ece_15": 0.20295393910421425, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/emoji": { |
| "count": 13, |
| "accuracy": 0.6153846153846154, |
| "log_loss": 0.6960605920906769, |
| "brier_score": 0.4981118938561514, |
| "ece_15": 0.3050876474074211, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/wiki_qa": { |
| "count": 16, |
| "accuracy": 1.0, |
| "log_loss": 0.07824049212788226, |
| "brier_score": 0.021876355194432068, |
| "ece_15": 0.07166085390505572, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/natural-language-satisfiability": { |
| "count": 11, |
| "accuracy": 0.8181818181818182, |
| "log_loss": 0.6065755987974197, |
| "brier_score": 0.4142264368460539, |
| "ece_15": 0.26288215715344726, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/CONDAQA": { |
| "count": 17, |
| "accuracy": 0.6470588235294118, |
| "log_loss": 0.5837376367623567, |
| "brier_score": 0.4017227845941354, |
| "ece_15": 0.23452599065352353, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/language-identification": { |
| "count": 14, |
| "accuracy": 0.7857142857142857, |
| "log_loss": 0.45379567540134363, |
| "brier_score": 0.2932768026204425, |
| "ece_15": 0.18667756246177625, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/vitaminc/tals--vitaminc": { |
| "count": 18, |
| "accuracy": 0.8333333333333334, |
| "log_loss": 0.47203846596688986, |
| "brier_score": 0.297187389628483, |
| "ece_15": 0.0878862234493985, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/irony": { |
| "count": 13, |
| "accuracy": 0.7692307692307693, |
| "log_loss": 0.5515458134943322, |
| "brier_score": 0.36624592117848087, |
| "ece_15": 0.2885674774751862, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/blog_authorship_corpus/gender": { |
| "count": 11, |
| "accuracy": 0.7272727272727273, |
| "log_loss": 0.6122869423340265, |
| "brier_score": 0.4208829038736505, |
| "ece_15": 0.16182430406149323, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/mrda": { |
| "count": 17, |
| "accuracy": 0.7647058823529411, |
| "log_loss": 0.6856584872938489, |
| "brier_score": 0.4527214858038554, |
| "ece_15": 0.2619543427574546, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/VUAC": { |
| "count": 18, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.6458246720007003, |
| "brier_score": 0.45230615062560264, |
| "ece_15": 0.05141111909454679, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/amazon_polarity/amazon_polarity": { |
| "count": 15, |
| "accuracy": 0.9333333333333333, |
| "log_loss": 0.12440397615807373, |
| "brier_score": 0.06418420580476111, |
| "ece_15": 0.10147991404532762, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/logical-fallacy": { |
| "count": 20, |
| "accuracy": 0.65, |
| "log_loss": 0.6174449749719677, |
| "brier_score": 0.42561461216146634, |
| "ece_15": 0.16475898322491966, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/cognitive-bias": { |
| "count": 19, |
| "accuracy": 0.5789473684210527, |
| "log_loss": 0.6811803427938817, |
| "brier_score": 0.4877731123629733, |
| "ece_15": 0.09707078374896291, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ambient": { |
| "count": 13, |
| "accuracy": 0.5384615384615384, |
| "log_loss": 1.050939217577183, |
| "brier_score": 0.7093245442917232, |
| "ece_15": 0.47989727258855513, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/oasis": { |
| "count": 15, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.6027792104843198, |
| "brier_score": 0.41900378992981374, |
| "ece_15": 0.1381713707335758, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/linguistic-bias": { |
| "count": 17, |
| "accuracy": 0.7058823529411765, |
| "log_loss": 0.6363254560348328, |
| "brier_score": 0.4436806488028217, |
| "ece_15": 0.2834689949984072, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/PARARULE-Plus": { |
| "count": 14, |
| "accuracy": 1.0, |
| "log_loss": 0.009673455202396824, |
| "brier_score": 0.0003681445782566941, |
| "ece_15": 0.009579687906687744, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/gum": { |
| "count": 13, |
| "accuracy": 0.46153846153846156, |
| "log_loss": 0.767310153459787, |
| "brier_score": 0.5644133418636972, |
| "ece_15": 0.22708194869411405, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/switchboard": { |
| "count": 12, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.40478602623769727, |
| "brier_score": 0.25692537989150854, |
| "ece_15": 0.2267872271489166, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/verifiability": { |
| "count": 17, |
| "accuracy": 1.0, |
| "log_loss": 0.23518077760555234, |
| "brier_score": 0.09756131460038968, |
| "ece_15": 0.20506767586415756, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/spartqa-yn": { |
| "count": 9, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.5676751197048884, |
| "brier_score": 0.38533629695032, |
| "ece_15": 0.13912224681162666, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/pragmeval/emergent": { |
| "count": 12, |
| "accuracy": 0.5, |
| "log_loss": 0.6111088725023022, |
| "brier_score": 0.4290625309158527, |
| "ece_15": 0.31663531554368235, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/emo/emo2019": { |
| "count": 9, |
| "accuracy": 0.8888888888888888, |
| "log_loss": 0.42614535757080696, |
| "brier_score": 0.2503727442041888, |
| "ece_15": 0.23545003274854517, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/glue/mrpc": { |
| "count": 13, |
| "accuracy": 0.6923076923076923, |
| "log_loss": 0.6378576657757951, |
| "brier_score": 0.44652269763177543, |
| "ece_15": 0.14463994537525376, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/I2D2": { |
| "count": 14, |
| "accuracy": 0.7857142857142857, |
| "log_loss": 0.5054673132064631, |
| "brier_score": 0.33029559589126845, |
| "ece_15": 0.09966654246438969, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/stance_hillary": { |
| "count": 9, |
| "accuracy": 0.4444444444444444, |
| "log_loss": 0.8658022715769922, |
| "brier_score": 0.6474565692275834, |
| "ece_15": 0.29779621458669236, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/political-bias": { |
| "count": 22, |
| "accuracy": 0.6818181818181818, |
| "log_loss": 0.6335086946180227, |
| "brier_score": 0.44177940486849887, |
| "ece_15": 0.0823299365370308, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/insincere-questions": { |
| "count": 12, |
| "accuracy": 1.0, |
| "log_loss": 0.08262838482918003, |
| "brier_score": 0.01685672667749119, |
| "ece_15": 0.07796318727667327, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/discovery/discovery": { |
| "count": 18, |
| "accuracy": 0.6111111111111112, |
| "log_loss": 0.604818229646948, |
| "brier_score": 0.41748359341081676, |
| "ece_15": 0.1119937267120168, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ethics/justice": { |
| "count": 12, |
| "accuracy": 0.4166666666666667, |
| "log_loss": 0.7692328606429664, |
| "brier_score": 0.5740434104662295, |
| "ece_15": 0.27459737711342325, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/glue/cola": { |
| "count": 5, |
| "accuracy": 1.0, |
| "log_loss": 0.4554031251730509, |
| "brier_score": 0.26898307875387517, |
| "ece_15": 0.36456669670412484, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/banking77": { |
| "count": 17, |
| "accuracy": 0.9411764705882353, |
| "log_loss": 0.26310132240224576, |
| "brier_score": 0.14710494648723688, |
| "ece_15": 0.1479224266903446, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/cladder": { |
| "count": 12, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.6383773535828748, |
| "brier_score": 0.44573822453017636, |
| "ece_15": 0.19868288408674398, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ethics/commonsense": { |
| "count": 15, |
| "accuracy": 0.4666666666666667, |
| "log_loss": 0.7222854271704464, |
| "brier_score": 0.5294455524416476, |
| "ece_15": 0.1669058650348188, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/scicite": { |
| "count": 11, |
| "accuracy": 0.8181818181818182, |
| "log_loss": 0.4302728752121489, |
| "brier_score": 0.2737731770623155, |
| "ece_15": 0.12187626014610493, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/dyda_da": { |
| "count": 10, |
| "accuracy": 0.9, |
| "log_loss": 0.24368784840394295, |
| "brier_score": 0.14213311052398495, |
| "ece_15": 0.12499038798163886, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ag_news": { |
| "count": 15, |
| "accuracy": 0.8666666666666667, |
| "log_loss": 0.34417277741508734, |
| "brier_score": 0.20884121246772364, |
| "ece_15": 0.14968292141421494, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/maptask": { |
| "count": 14, |
| "accuracy": 0.7857142857142857, |
| "log_loss": 0.576703165188172, |
| "brier_score": 0.3865996142094653, |
| "ece_15": 0.21486179026733052, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/dynahate": { |
| "count": 12, |
| "accuracy": 0.5, |
| "log_loss": 0.8023888055892886, |
| "brier_score": 0.5950265892214276, |
| "ece_15": 0.34611284147608473, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/probability_words_nli/reasoning_2hop": { |
| "count": 13, |
| "accuracy": 0.46153846153846156, |
| "log_loss": 0.6897876920393469, |
| "brier_score": 0.496656026620018, |
| "ece_15": 0.062024000009732703, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/counterfactually-augmented-imdb": { |
| "count": 10, |
| "accuracy": 1.0, |
| "log_loss": 0.03535720124076549, |
| "brier_score": 0.004582551541392156, |
| "ece_15": 0.03413162057049359, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweets_hate_speech_detection": { |
| "count": 18, |
| "accuracy": 1.0, |
| "log_loss": 0.11627279465981405, |
| "brier_score": 0.043156980154768626, |
| "ece_15": 0.1023424561559228, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ethics/deontology": { |
| "count": 11, |
| "accuracy": 0.2727272727272727, |
| "log_loss": 0.8992056796275404, |
| "brier_score": 0.6966724955226067, |
| "ece_15": 0.3611926596753911, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/hope_edi/english": { |
| "count": 10, |
| "accuracy": 1.0, |
| "log_loss": 0.025750615328779498, |
| "brier_score": 0.0015933906032583975, |
| "ece_15": 0.02534241466376912, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/text-level-bias": { |
| "count": 12, |
| "accuracy": 0.75, |
| "log_loss": 0.645664087173354, |
| "brier_score": 0.45022585255060155, |
| "ece_15": 0.2595347474914672, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/paws/labeled_final": { |
| "count": 15, |
| "accuracy": 0.5333333333333333, |
| "log_loss": 0.7152394239930173, |
| "brier_score": 0.5217685252680689, |
| "ece_15": 0.23679016783961698, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/avicenna": { |
| "count": 8, |
| "accuracy": 0.625, |
| "log_loss": 0.6078684183403206, |
| "brier_score": 0.4175087172361987, |
| "ece_15": 0.17735747579387287, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/subjectivity": { |
| "count": 15, |
| "accuracy": 0.5333333333333333, |
| "log_loss": 0.6519663098510241, |
| "brier_score": 0.4591549547589198, |
| "ece_15": 0.31788769332366407, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/v1/gen_train234_test2to10": { |
| "count": 16, |
| "accuracy": 0.8125, |
| "log_loss": 0.6188050539540328, |
| "brier_score": 0.4208129480832215, |
| "ece_15": 0.23228472415974305, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/folio": { |
| "count": 8, |
| "accuracy": 0.875, |
| "log_loss": 0.2677357935535888, |
| "brier_score": 0.1448126082478087, |
| "ece_15": 0.112157406048406, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/text_emotion": { |
| "count": 9, |
| "accuracy": 0.7777777777777778, |
| "log_loss": 0.496909940808505, |
| "brier_score": 0.33467897383452594, |
| "ece_15": 0.13903555138204005, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/sentiment_nuclear_power": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.2619316649787558, |
| "brier_score": 0.10619770027406616, |
| "ece_15": 0.23043638994354732, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/crowdflower/corporate-messaging": { |
| "count": 8, |
| "accuracy": 0.625, |
| "log_loss": 0.5323242138008609, |
| "brier_score": 0.3727744578383365, |
| "ece_15": 0.30640486312610843, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/silicone/meld_s": { |
| "count": 9, |
| "accuracy": 0.7777777777777778, |
| "log_loss": 0.5356984575781174, |
| "brier_score": 0.3548463671047433, |
| "ece_15": 0.24604507501044123, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/args_me": { |
| "count": 13, |
| "accuracy": 0.7692307692307693, |
| "log_loss": 0.5475327467413906, |
| "brier_score": 0.3646842886592681, |
| "ece_15": 0.13029259831640638, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/super_glue/wic": { |
| "count": 9, |
| "accuracy": 0.4444444444444444, |
| "log_loss": 0.7035378312578746, |
| "brier_score": 0.51191034723418, |
| "ece_15": 0.23020335772164124, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/glue/qqp": { |
| "count": 9, |
| "accuracy": 0.6666666666666666, |
| "log_loss": 0.47250813086259197, |
| "brier_score": 0.32133322820796006, |
| "ece_15": 0.2840312548127259, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/rumoureval_2019/RumourEval2019": { |
| "count": 6, |
| "accuracy": 0.8333333333333334, |
| "log_loss": 0.3720467245852239, |
| "brier_score": 0.2202019886218846, |
| "ece_15": 0.2852043573129455, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/mbib-base/fake-news": { |
| "count": 13, |
| "accuracy": 0.6923076923076923, |
| "log_loss": 0.6709572258784231, |
| "brier_score": 0.4751351390273223, |
| "ece_15": 0.2365343477443919, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/ethos/binary": { |
| "count": 10, |
| "accuracy": 0.7, |
| "log_loss": 0.43184881082816523, |
| "brier_score": 0.309938587022881, |
| "ece_15": 0.2232269241486753, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/tweet_eval/stance_feminist": { |
| "count": 13, |
| "accuracy": 0.6153846153846154, |
| "log_loss": 0.6342819838699137, |
| "brier_score": 0.4424172212355806, |
| "ece_15": 0.14458215186956272, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/trec": { |
| "count": 10, |
| "accuracy": 0.8, |
| "log_loss": 0.5161872273300505, |
| "brier_score": 0.33878326849020457, |
| "ece_15": 0.1306445247432167, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/zero-shot-label-nli/snips_built_in_intents": { |
| "count": 4, |
| "accuracy": 0.5, |
| "log_loss": 0.7041056924421092, |
| "brier_score": 0.5439060873890396, |
| "ece_15": 0.4048079387580852, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/tasksource-instruct-v0": { |
| "count": 997, |
| "accuracy": 0.7121364092276831, |
| "log_loss": 0.6653333170635247, |
| "brier_score": 0.3952071519663628, |
| "ece_15": 0.05664073959338466, |
| "ordinal_mae": 0.3923422177916232, |
| "ordinal_count": 997 |
| }, |
| "kind/score": { |
| "count": 997, |
| "accuracy": 0.7121364092276831, |
| "log_loss": 0.6653333170635247, |
| "brier_score": 0.3952071519663628, |
| "ece_15": 0.05664073959338466, |
| "ordinal_mae": 0.3923422177916232, |
| "ordinal_count": 997 |
| }, |
| "task/tasksource/tasksource-instruct-v0/yelp_review_full/yelp_review_full": { |
| "count": 497, |
| "accuracy": 0.6841046277665996, |
| "log_loss": 0.7242680913247214, |
| "brier_score": 0.42545976597607615, |
| "ece_15": 0.057978752215053074, |
| "ordinal_mae": 0.41514169051418587, |
| "ordinal_count": 497 |
| }, |
| "task/tasksource/tasksource-instruct-v0/tweet_eval/sentiment": { |
| "count": 500, |
| "accuracy": 0.74, |
| "log_loss": 0.6067521514478951, |
| "brier_score": 0.36513605364070745, |
| "ece_15": 0.08562278138075635, |
| "ordinal_mae": 0.36967954190539737, |
| "ordinal_count": 500 |
| }, |
| "source/nyu-mll/multi_nli": { |
| "count": 2000, |
| "accuracy": 0.907, |
| "log_loss": 0.2676430728353943, |
| "brier_score": 0.14473967691300926, |
| "ece_15": 0.04730422129633431, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/nyu-mll/multi_nli/nyu-mll/multi_nli/default": { |
| "count": 2000, |
| "accuracy": 0.907, |
| "log_loss": 0.2676430728353943, |
| "brier_score": 0.14473967691300926, |
| "ece_15": 0.04730422129633431, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/facebook/anli": { |
| "count": 2000, |
| "accuracy": 0.5785, |
| "log_loss": 0.9133303232149986, |
| "brier_score": 0.5611027917633463, |
| "ece_15": 0.12427056134727293, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/facebook/anli/facebook/anli/plain_text": { |
| "count": 2000, |
| "accuracy": 0.5785, |
| "log_loss": 0.9133303232149986, |
| "brier_score": 0.5611027917633463, |
| "ece_15": 0.12427056134727293, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/defeasible-nli": { |
| "count": 2000, |
| "accuracy": 0.8745, |
| "log_loss": 0.29925963680728324, |
| "brier_score": 0.18316015565997798, |
| "ece_15": 0.030345176892877232, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/defeasible-nli/tasksource/defeasible-nli/social": { |
| "count": 1217, |
| "accuracy": 0.9063270336894002, |
| "log_loss": 0.2348862683045523, |
| "brier_score": 0.13656258793990758, |
| "ece_15": 0.04400942188398399, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/defeasible-nli/tasksource/defeasible-nli/atomic": { |
| "count": 542, |
| "accuracy": 0.8173431734317343, |
| "log_loss": 0.410304647453951, |
| "brier_score": 0.26318781545244285, |
| "ece_15": 0.026093150776035075, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/defeasible-nli/tasksource/defeasible-nli/snli": { |
| "count": 241, |
| "accuracy": 0.8423236514522822, |
| "log_loss": 0.37459571024018584, |
| "brier_score": 0.23848898681272002, |
| "ece_15": 0.055863559554963346, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/FOL-nli": { |
| "count": 2000, |
| "accuracy": 0.834, |
| "log_loss": 0.425081312444918, |
| "brier_score": 0.24203185420889972, |
| "ece_15": 0.04191806881829498, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/FOL-nli/tasksource/FOL-nli/default": { |
| "count": 2000, |
| "accuracy": 0.834, |
| "log_loss": 0.425081312444918, |
| "brier_score": 0.24203185420889972, |
| "ece_15": 0.04191806881829498, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/doc-nli": { |
| "count": 1776, |
| "accuracy": 0.8693693693693694, |
| "log_loss": 0.3776345216910134, |
| "brier_score": 0.20410689079041464, |
| "ece_15": 0.0721990192199759, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/doc-nli/tasksource/doc-nli/default": { |
| "count": 1776, |
| "accuracy": 0.8693693693693694, |
| "log_loss": 0.3776345216910134, |
| "brier_score": 0.20410689079041464, |
| "ece_15": 0.0721990192199759, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/bigbench": { |
| "count": 1896, |
| "accuracy": 0.7600210970464135, |
| "log_loss": 0.602894896267623, |
| "brier_score": 0.30396554743519166, |
| "ece_15": 0.051980339698730865, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/vitaminc_fact_verification": { |
| "count": 262, |
| "accuracy": 0.8625954198473282, |
| "log_loss": 0.38103699033453386, |
| "brier_score": 0.2065051915031934, |
| "ece_15": 0.0698970798648587, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/mnist_ascii": { |
| "count": 349, |
| "accuracy": 0.8223495702005731, |
| "log_loss": 0.6002663987026644, |
| "brier_score": 0.27309327740513434, |
| "ece_15": 0.06774433340107733, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/cs_algorithms": { |
| "count": 6, |
| "accuracy": 0.8333333333333334, |
| "log_loss": 0.8575062151344007, |
| "brier_score": 0.4532760049039597, |
| "ece_15": 0.3781438391405416, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/language_identification": { |
| "count": 39, |
| "accuracy": 0.2564102564102564, |
| "log_loss": 2.1649805091983163, |
| "brier_score": 0.8619812329717212, |
| "ece_15": 0.1358268585099441, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/unit_conversion": { |
| "count": 73, |
| "accuracy": 0.8493150684931506, |
| "log_loss": 0.36995943925993446, |
| "brier_score": 0.1866371071631526, |
| "ece_15": 0.05945917663230087, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/bbq_lite_json": { |
| "count": 61, |
| "accuracy": 0.9836065573770492, |
| "log_loss": 0.08452322072671453, |
| "brier_score": 0.039917098456269604, |
| "ece_15": 0.02783921016342632, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/movie_dialog_same_or_different": { |
| "count": 197, |
| "accuracy": 0.7563451776649747, |
| "log_loss": 0.5080503638241864, |
| "brier_score": 0.3337124207277505, |
| "ece_15": 0.08204405870728061, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/fact_checker": { |
| "count": 34, |
| "accuracy": 0.8235294117647058, |
| "log_loss": 0.4767798367145815, |
| "brier_score": 0.3083461651144789, |
| "ece_15": 0.14877917479205238, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/social_support": { |
| "count": 2, |
| "accuracy": 0.5, |
| "log_loss": 0.8251933250191481, |
| "brier_score": 0.5548595671266254, |
| "ece_15": 0.14197386779891208, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/hyperbaton": { |
| "count": 179, |
| "accuracy": 1.0, |
| "log_loss": 0.004903228693345848, |
| "brier_score": 0.0018520925829598126, |
| "ece_15": 0.004331813434934797, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/formal_fallacies_syllogisms_negation": { |
| "count": 70, |
| "accuracy": 0.7714285714285715, |
| "log_loss": 0.45451433761223536, |
| "brier_score": 0.2946976779111536, |
| "ece_15": 0.11341918148766668, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/contextual_parametric_knowledge_conflicts": { |
| "count": 72, |
| "accuracy": 1.0, |
| "log_loss": 0.0042866319339874, |
| "brier_score": 0.0005254734725362953, |
| "ece_15": 0.004144034953844518, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/abstract_narrative_understanding": { |
| "count": 5, |
| "accuracy": 0.0, |
| "log_loss": 2.182878684561412, |
| "brier_score": 0.9354408520477367, |
| "ece_15": 0.24853771093466848, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/goal_step_wikihow": { |
| "count": 34, |
| "accuracy": 0.7941176470588235, |
| "log_loss": 0.49479223842360004, |
| "brier_score": 0.2877662889272723, |
| "ece_15": 0.16590714907316742, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/real_or_fake_text": { |
| "count": 47, |
| "accuracy": 0.3404255319148936, |
| "log_loss": 1.9372676073860116, |
| "brier_score": 0.8089350704681822, |
| "ece_15": 0.16654496730804147, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/symbol_interpretation": { |
| "count": 5, |
| "accuracy": 0.0, |
| "log_loss": 1.702859572644817, |
| "brier_score": 0.8365272431417929, |
| "ece_15": 0.2729845705211645, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/arithmetic": { |
| "count": 74, |
| "accuracy": 1.0, |
| "log_loss": 0.0023194179326578098, |
| "brier_score": 0.00013419920934465068, |
| "ece_15": 0.002281276141992805, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/timedial": { |
| "count": 18, |
| "accuracy": 1.0, |
| "log_loss": 0.07928157485328463, |
| "brier_score": 0.034307814118185, |
| "ece_15": 0.0622587201655115, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/discourse_marker_prediction": { |
| "count": 4, |
| "accuracy": 0.0, |
| "log_loss": 2.4306903551873225, |
| "brier_score": 0.9289598264151548, |
| "ece_15": 0.22414781204482565, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/strategyqa": { |
| "count": 10, |
| "accuracy": 0.7, |
| "log_loss": 0.6869369322369993, |
| "brier_score": 0.4932013000680217, |
| "ece_15": 0.20385124503140437, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/parsinlu_qa": { |
| "count": 9, |
| "accuracy": 0.2222222222222222, |
| "log_loss": 1.451704763169901, |
| "brier_score": 0.760643095262711, |
| "ece_15": 0.21571830832056527, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/logical_deduction": { |
| "count": 9, |
| "accuracy": 0.3333333333333333, |
| "log_loss": 1.221485929293065, |
| "brier_score": 0.6323103192433218, |
| "ece_15": 0.1821726581221277, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/elementary_math_qa": { |
| "count": 159, |
| "accuracy": 0.3836477987421384, |
| "log_loss": 1.3876589979872005, |
| "brier_score": 0.6968919950560583, |
| "ece_15": 0.09392527333242497, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/tracking_shuffled_objects": { |
| "count": 20, |
| "accuracy": 0.25, |
| "log_loss": 1.6349046079611906, |
| "brier_score": 0.8099753150778438, |
| "ece_15": 0.21079594187432807, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/snarks": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.6706533963836379, |
| "brier_score": 0.47751007423155717, |
| "ece_15": 0.48862566154038467, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/color": { |
| "count": 18, |
| "accuracy": 0.3333333333333333, |
| "log_loss": 1.7783889808890854, |
| "brier_score": 0.7970015392145402, |
| "ece_15": 0.1187786881257698, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/play_dialog_same_or_different": { |
| "count": 13, |
| "accuracy": 0.7692307692307693, |
| "log_loss": 0.5788119394124831, |
| "brier_score": 0.3900014955595719, |
| "ece_15": 0.18119994447526416, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/indic_cause_and_effect": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 0.8312477787808272, |
| "brier_score": 0.6358515687361226, |
| "ece_15": 0.5616070340233159, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/logic_grid_puzzle": { |
| "count": 10, |
| "accuracy": 0.5, |
| "log_loss": 1.0919949027727796, |
| "brier_score": 0.5934883790906896, |
| "ece_15": 0.2704396563566979, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/sports_understanding": { |
| "count": 4, |
| "accuracy": 0.0, |
| "log_loss": 0.7269933870476967, |
| "brier_score": 0.5337326190175753, |
| "ece_15": 0.5161231174582996, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/fantasy_reasoning": { |
| "count": 2, |
| "accuracy": 0.5, |
| "log_loss": 0.5214277745228033, |
| "brier_score": 0.3419025017278519, |
| "ece_15": 0.39183793332375344, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/question_selection": { |
| "count": 7, |
| "accuracy": 1.0, |
| "log_loss": 0.19811367912696262, |
| "brier_score": 0.06229089079281506, |
| "ece_15": 0.16586570675181198, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/code_line_description": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.36158380919531635, |
| "brier_score": 0.12343516415240563, |
| "ece_15": 0.30342778552770255, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/kanji_ascii": { |
| "count": 2, |
| "accuracy": 0.5, |
| "log_loss": 1.4074804405969101, |
| "brier_score": 0.6997439808808856, |
| "ece_15": 0.45538844816565466, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/analogical_similarity": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 2.3086273693996944, |
| "brier_score": 0.948241005725639, |
| "ece_15": 0.1860990001165002, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/winowhy": { |
| "count": 11, |
| "accuracy": 0.8181818181818182, |
| "log_loss": 0.42986333340721655, |
| "brier_score": 0.26663166172968483, |
| "ece_15": 0.32416977772655126, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/salient_translation_error_detection": { |
| "count": 3, |
| "accuracy": 0.3333333333333333, |
| "log_loss": 1.8902870767525537, |
| "brier_score": 0.8617747844320157, |
| "ece_15": 0.2633205079199103, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/rhyming": { |
| "count": 2, |
| "accuracy": 1.0, |
| "log_loss": 0.23330346438826027, |
| "brier_score": 0.0696860547200071, |
| "ece_15": 0.20505438771128126, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/movie_recommendation": { |
| "count": 3, |
| "accuracy": 1.0, |
| "log_loss": 0.26687648197103836, |
| "brier_score": 0.10742139218011508, |
| "ece_15": 0.2183625085602264, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/analytic_entailment": { |
| "count": 2, |
| "accuracy": 1.0, |
| "log_loss": 0.35013435893296785, |
| "brier_score": 0.18017772919742275, |
| "ece_15": 0.2919846820851146, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/logical_fallacy_detection": { |
| "count": 7, |
| "accuracy": 0.7142857142857143, |
| "log_loss": 0.6406785108889849, |
| "brier_score": 0.4129053998268808, |
| "ece_15": 0.18843710231941754, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/temporal_sequences": { |
| "count": 3, |
| "accuracy": 1.0, |
| "log_loss": 0.07794021767803416, |
| "brier_score": 0.01268107216367032, |
| "ece_15": 0.07355045308205117, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/kannada": { |
| "count": 3, |
| "accuracy": 0.3333333333333333, |
| "log_loss": 1.3006808984223028, |
| "brier_score": 0.7120416307206603, |
| "ece_15": 0.24751795970738236, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/navigate": { |
| "count": 2, |
| "accuracy": 0.5, |
| "log_loss": 0.7477827162954784, |
| "brier_score": 0.5543502814305443, |
| "ece_15": 0.5254655042868934, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/social_iqa": { |
| "count": 6, |
| "accuracy": 0.8333333333333334, |
| "log_loss": 0.4710002326713407, |
| "brier_score": 0.24511733914828068, |
| "ece_15": 0.31748993112750223, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/implicatures": { |
| "count": 2, |
| "accuracy": 1.0, |
| "log_loss": 0.29114841850337103, |
| "brier_score": 0.1848048112365155, |
| "ece_15": 0.22521257977661313, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/penguins_in_a_table": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 1.8681423074955834, |
| "brier_score": 0.9430615120285668, |
| "ece_15": 0.3352266991055912, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/epistemic_reasoning": { |
| "count": 8, |
| "accuracy": 1.0, |
| "log_loss": 0.11367371724241075, |
| "brier_score": 0.032920414619740636, |
| "ece_15": 0.10397969315830892, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/reasoning_about_colored_objects": { |
| "count": 5, |
| "accuracy": 0.6, |
| "log_loss": 1.3896145496587025, |
| "brier_score": 0.22839843219080186, |
| "ece_15": 0.03217754574682143, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/strange_stories": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 0.7882933324627114, |
| "brier_score": 0.594878647792511, |
| "ece_15": 0.5453799812023316, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/what_is_the_tao": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 0.8389522505125051, |
| "brier_score": 0.644877524180415, |
| "ece_15": 0.5678369150471, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/international_phonetic_alphabet_nli": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 1.3104032474546674, |
| "brier_score": 0.7900247964738226, |
| "ece_15": 0.4155741342002613, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/hindu_knowledge": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.7721442631822523, |
| "brier_score": 0.3961432110812999, |
| "ece_15": 0.5379786898887589, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/cryobiology_spanish": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.250356414371715, |
| "brier_score": 0.09810389561034243, |
| "ece_15": 0.22147674326026023, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/intent_recognition": { |
| "count": 4, |
| "accuracy": 1.0, |
| "log_loss": 0.2597150154532919, |
| "brier_score": 0.10887130602372279, |
| "ece_15": 0.18532926980517667, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/metaphor_boolean": { |
| "count": 2, |
| "accuracy": 1.0, |
| "log_loss": 0.20662987494224797, |
| "brier_score": 0.08727285038621402, |
| "ece_15": 0.17974473959147763, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/suicide_risk": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 1.3264746679558475, |
| "brier_score": 0.7232677834141499, |
| "ece_15": 0.2906997711235214, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/logical_args": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 1.1110180549275812, |
| "brier_score": 0.5793923955231951, |
| "ece_15": 0.6707763773236444, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/ruin_names": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 1.5341818824557079, |
| "brier_score": 0.8525589486207359, |
| "ece_15": 0.4077467459606056, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/causal_judgment": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 0.832508254593555, |
| "brier_score": 0.638547332378019, |
| "ece_15": 0.5650430657826088, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/english_proverbs": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 2.781201698586805, |
| "brier_score": 1.5820386121155834, |
| "ece_15": 0.8338373784186494, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/empirical_judgments": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 1.307122276207722, |
| "brier_score": 0.8484497113675799, |
| "ece_15": 0.5234588750535261, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/physical_intuition": { |
| "count": 2, |
| "accuracy": 0.5, |
| "log_loss": 1.1893217893074914, |
| "brier_score": 0.6509848835545445, |
| "ece_15": 0.46917544098255437, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/similarities_abstraction": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.6525461359947428, |
| "brier_score": 0.3729023778458806, |
| "ece_15": 0.47928173205594216, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/swahili_english_proverbs": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.7400190729452648, |
| "brier_score": 0.3898876550220091, |
| "ece_15": 0.5228951843597804, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/date_understanding": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.17162464733203386, |
| "brier_score": 0.03664889324545113, |
| "ece_15": 0.15770472838199245, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/medical_questions_russian": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.42450806234604666, |
| "brier_score": 0.23930541087430027, |
| "ece_15": 0.34590852177584486, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/anachronisms": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 0.832508254593555, |
| "brier_score": 0.638547332378019, |
| "ece_15": 0.5650430657826088, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/simple_ethical_questions": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 1.3579175779081794, |
| "brier_score": 0.8773437550179706, |
| "ece_15": 0.567649431212691, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/dark_humor_detection": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.4879816784771691, |
| "brier_score": 0.2982018358548382, |
| "ece_15": 0.38613588013472555, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/irony_identification": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 0.9167286611010234, |
| "brier_score": 0.7197503784706804, |
| "ece_15": 0.5993355757900248, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/geometric_shapes": { |
| "count": 2, |
| "accuracy": 0.0, |
| "log_loss": 2.4355956963191123, |
| "brier_score": 0.6772767270890805, |
| "ece_15": 0.1439518167239653, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/cause_and_effect": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.1293535690995136, |
| "brier_score": 0.029445217467256986, |
| "ece_15": 0.12133675755363049, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/moral_permissibility": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 0.7759487385429391, |
| "brier_score": 0.5826236366569683, |
| "ece_15": 0.5397330991596533, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/phrase_relatedness": { |
| "count": 1, |
| "accuracy": 0.0, |
| "log_loss": 1.0318610171233211, |
| "brier_score": 0.7033854687035427, |
| "ece_15": 0.5306299056956488, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/crass_ai": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 0.5795727772062687, |
| "brier_score": 0.28276648319040115, |
| "ece_15": 0.43986238098681774, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/tasksource/bigbench/tasksource/bigbench/figure_of_speech_detection": { |
| "count": 1, |
| "accuracy": 1.0, |
| "log_loss": 1.538657543538069, |
| "brier_score": 0.7019273087810161, |
| "ece_15": 0.7853309080139531, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/Rowan/hellaswag": { |
| "count": 2000, |
| "accuracy": 0.86, |
| "log_loss": 0.3611882300149126, |
| "brier_score": 0.19594949629069483, |
| "ece_15": 0.022342067811566196, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/Rowan/hellaswag/Rowan/hellaswag/default": { |
| "count": 2000, |
| "accuracy": 0.86, |
| "log_loss": 0.3611882300149126, |
| "brier_score": 0.19594949629069483, |
| "ece_15": 0.022342067811566196, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/google/boolq": { |
| "count": 828, |
| "accuracy": 0.8599033816425121, |
| "log_loss": 0.32001524053340347, |
| "brier_score": 0.19621992616078116, |
| "ece_15": 0.03950134207516404, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "kind/noul": { |
| "count": 828, |
| "accuracy": 0.8599033816425121, |
| "log_loss": 0.32001524053340347, |
| "brier_score": 0.19621992616078116, |
| "ece_15": 0.03950134207516404, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "task/google/boolq/google/boolq/default": { |
| "count": 828, |
| "accuracy": 0.8599033816425121, |
| "log_loss": 0.32001524053340347, |
| "brier_score": 0.19621992616078116, |
| "ece_15": 0.03950134207516404, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| } |
| }, |
| "temperature": 1.3734692667114208, |
| "data_sha256": "8442fc4dbeb740baab4603a0ebc079f5fa53474e7b5bba9b4f82a206d5a65dc7", |
| "checkpoint": "/workspace/kev-run/runs/train-20260924-185636/final", |
| "notes": "Accuracy accepts any positive target. ECE uses gold probability mass, not any-positive accuracy. Ordinal MAE uses zero-based level indices." |
| } |
|
|
