{"acc_norm":0.32337883959044367,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.295221843003413,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.3040885659866035},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.11822349799331278},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.11888145399279892},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.11733090307097882},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.11735833494458348},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.11757060105446726},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11803359305486083},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11748935794457793},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.12024792295414954},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11987023905385286},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.11884784197900444},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11960895196534693},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.11812755896244198},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.12028770695906132},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.12022488994989544},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1203274120343849},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11878900195006281},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.1186065620277077},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.11862273304723203},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12063662800937891},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11821955989580601},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11749957699794322},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11765175103209913},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11812436301261187},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11727079097181559},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.11764112196397036},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11924434208776802},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.12048868194688112},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11813080497086048},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1192853570682928},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11801391700282693},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11873005295637995},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11897125199902803},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11895416991319507},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.11888298799749464},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11883246304932982},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11945261992514133},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11837378796190023},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11864587501622736},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11843820801004767},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11895079398527741},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.11875656200572848},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12017565802671015},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12051651393994689},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12182753894012421},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12138969195075333},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11825977498665452},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1178146650781855},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11866349796764553},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11791627504862845},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11822936905082315},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11814059293828905},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11826896294951439},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.11920627998188138},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11845361709129065},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11812725802883506},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1179748639697209},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11811285105068237},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11811707995366305},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11823415802791715},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11816890595946461},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11824907502159476},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.118461151025258},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.12057918997015804},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11860951699782163},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.11967878206633031},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1194523690501228},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1191853410564363},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11914537602569908},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1206436010543257},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12093130801804364},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12022994994185865},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12184941000305116},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12255381303839386},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.12108940200414509},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1185977760469541},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11813692597206682},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.11757962696719915},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1181453219614923},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11879234807565808},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11818765092175454},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12075641099363565},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11944991501513869},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1185745729599148},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11797005496919155},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11845336598344147},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.11825426493305713},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1185515399556607},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11795871402136981},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11752612702548504},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11778113292530179},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11756466992665082},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11777024192269892},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11720965697895736},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11794912593904883},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11760891193989664},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.11769406998064369},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11827630596235394},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11889027094002813},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12180366402026266},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1201786040328443},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11958235199563205},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.11906212195754051},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11762145499233156},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11925412900745869},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11727862595580518},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11706040892750025},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1177353149978444},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11743453494273126},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11718013102654368},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11739104602020234},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.11790973204188049},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11794195300899446},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11734801507554948},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11737906304188073},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11806407000403851},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1175745470682159},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11996493593323976},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.11921589903067797},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11942863499280065},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1177080370252952},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11735638091340661},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11818711005616933},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11740451096557081},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11751298198942095},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11744344304315746},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.11761342000681907},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11799953994341195},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11942887597251683},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11884238198399544},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11901662696618587},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11879164702259004},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.11906098900362849},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1187350720865652},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11855169991031289},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11904252495151013},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11991647491231561},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11830156308133155},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11873770703095943},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.11812732799444348},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11810278310440481},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1170052660163492},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11665095493663102},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1162910140119493},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11665994196664542},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11678554594982415},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.11746739794034511},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11712738394271582},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11694924102630466},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11805478297173977},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11782473395578563},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11763776501175016},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1173296810593456},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11829574301373214},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.11950888496357948},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11849149700719863},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1188912830548361},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11705962708219886},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11946626496501267},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.11877345305401832},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11822484992444515},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11789625696837902},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11799127398990095},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1175977709935978},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.11761407100129873},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11764518893323839},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11812679702416062},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11789137800224125},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11783564405050129},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11772357602603734},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.11810980609152466},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1178998639807105},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12199197604786605},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.12208342598751187},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11875594092998654},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11900670803152025},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11856214900035411},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11960494401864707},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1201030429219827},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12048869207501411},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12034095707349479},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12032723100855947},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.12043459108099341},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1207395299570635},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12074993900023401},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1199949209112674},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12045515002682805},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1217491829302162},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11961835902184248},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12275092001073062},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12295656406786293},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12133305694442242},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12052061094436795},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11840794200543314},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11867096298374236},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11889746494125575},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1202961029484868},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12012964300811291},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12077133893035352},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12056307098828256},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1192812800873071},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1195150869898498},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11846505803987384},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12010923505295068},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11900973389856517},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11890352598857135},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12022675399202853},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1254135260824114},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12377030204515904},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12288444896694273},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12177043198607862},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1210015380056575},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12120542908087373},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11848367296624929},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12253993703052402},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11743216204922646},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12080122495535761},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12406278704293072},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12284718092996627},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12239055894315243},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12286246893927455},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12202514696400613},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12226568604819477},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12426929303910583},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12410286196973175},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12267218402121216},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12365885404869914},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12222165404818952},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12137913203332573},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1490129049634561},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13009944406803697},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12168585497420281},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12134686298668385},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12106756109278649},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12164292496163398},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12191018310841173},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12220347009133548},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12230772397015244},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1221960069378838},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1217741291038692},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12332070304546505},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12132056301925331},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.12159804999828339},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.12272047402802855},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.12463508406654},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1238438889849931},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.12469775101635605},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.12215125199873},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1233447779668495},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12212185806129128},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12169331905897707},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12223498907405883},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12229734507855028},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12210083799436688},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.12298681098036468},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.12270967394579202},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.12394306401256472},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12261437601409853},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12243088299874216},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.12248138803988695},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.12409249204210937},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12253262393642217},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12126255501061678},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.11863133893348277},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.12050250801257789},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.12241047597490251},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.12308374198619276},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.12317820708267391},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.12004754994995892},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.11982857098337263},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.12542406597640365},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1238592779263854},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.12382977304514498},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.12289780506398529},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.12241560604888946},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.12249062594491988},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.12237183295655996},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.12222807598300278},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.12492954300250858},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.12388261093292385},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.12334360601380467},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.12257917004171759},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.12375118699856102},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.12604286009445786},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.1271529400255531},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.12287285807542503},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.12459222401957959},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.12261040904559195},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12379739200696349},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.12326039094477892},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.1219339279923588},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.12086524395272136},{"candidates":15,"kind":"choice","padded_tokens":2595,"seconds":0.12023162294644862}],"chat_template":false,"data_loading_seconds":0.0656192289898172,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-arc-challenge-v1","finished":1792249882,"gpu":1,"id":"175d1e00e1e06d8eeee8747c","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.02","model_load_seconds":1.947894288925454,"model_revision":"c4298d757ba39cba19dd768777da9fadf319ff61","n":1172,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":139279760,"peak_gpu_bytes":856281088,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge (full test, raw likelihood accuracy)","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"native-arc-challenge-common-core-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Challenge / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-arc-challenge-v1","recipe":{"data_file":"ARC-Challenge/test-00000-of-00001.parquet","schema":"arc-challenge-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"arc_challenge"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":39.37239898496773,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"e7984b7ea3a9cae0996b0a453de457a6a7d717e9a2c3e9a95d80f697f8bd5afb","score":0.295221843003413,"scoring_seconds":35.58992030099034,"seconds":37.70950976293534,"seed":42,"selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","status":"completed","task":"native-arc-challenge-common-core-full","tf32":false,"truncated_candidates":0,"worker":"white-gpu1"}