{"acc_norm":0.2883959044368601,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.26535836177474403,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":176,"seconds":0.3043014230206609},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13859835301991552},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.13798703299835324},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13776847603730857},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.13529632601421326},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13659637002274394},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.13519738090690225},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1367195799248293},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.1370233369525522},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13314598600845784},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.1352706179022789},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13431990495882928},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.13603670604061335},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13610139710363},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13624391297344118},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1363433679798618},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13980606605764478},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13771122903563082},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1396796900080517},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.14018380001652986},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13713271997403353},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13651207298971713},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13712021708488464},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13863158598542213},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1355453300056979},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.143425234942697},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14234056195709854},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1425394529942423},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14249473996460438},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14262631500605494},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14292335894424468},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.14333369408268481},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14387080492451787},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14208798098843545},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14460890309419483},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1428783149458468},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14242421800736338},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14306173706427217},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14325213199481368},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.14444468589499593},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1429883799282834},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14244932495057583},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14222084893845022},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14194071595557034},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1425014219712466},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14214375603478402},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14343192800879478},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14244680001866072},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.140815588994883},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1431958379689604},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.14055840903893113},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.1412569920066744},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1370981860673055},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14154352596960962},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13894672400783747},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14015372400172055},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1414665529737249},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14216263010166585},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1417729239910841},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14199105103034526},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14193320297636092},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14341990498360246},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14202211902011186},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1427265020320192},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14362773299217224},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14321006392128766},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1389743840554729},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1387165340129286},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14341128896921873},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14227038098033518},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14219579193741083},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14218121499288827},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13943998294416815},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1386731929378584},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1433478209655732},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.14473924494814128},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13720063702203333},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13869640592020005},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14081643999088556},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14219875703565776},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1384740819921717},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1384789509465918},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1386744260089472},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13659138092771173},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.14002233906649053},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14262611500453204},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14213917695451528},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1423731829272583},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1453603639965877},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1433877859963104},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1428414669353515},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1443172690924257},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14293508103583008},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.14082143898122013},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1406531259417534},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14297136897221208},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14038093795534223},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1404618889791891},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.14126528706401587},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1394252460449934},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13712776091415435},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13715938001405448},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13782769697718322},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13949617790058255},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13774361996911466},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13774396001826972},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13948445592541248},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1389861770439893},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13727575703524053},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13857887708581984},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1401825969805941},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.14000812300946563},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13772346207406372},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13924834609497339},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13896947610192},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1360318079823628},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1399978140834719},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13928381202276796},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13751733698882163},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13700316892936826},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13703866489231586},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13963081897236407},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.14062069507781416},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1408413660246879},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13874524796847254},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1424632309935987},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.14086332696024328},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1377656910335645},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13989840797148645},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.14023838192224503},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13901549100410193},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13807338406331837},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13965164800174534},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14155555807519704},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13804246694780886},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1378218550235033},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.14036525902338326},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13950801105238497},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14168329699896276},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14067847200203687},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14041045296471566},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1413044489454478},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13932700199075043},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13818333996459842},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.14088544808328152},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13942229992244393},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.14023093797732145},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1416341259609908},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.14247259788680822},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1417877310886979},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13902735302690417},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1397722230758518},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.14133195101749152},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13898655690718442},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13745973107870668},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13872289599385113},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14050247496925294},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.14232591399922967},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13809879205655307},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13798446906730533},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.139392634970136},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1381690229754895},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1376376620028168},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.13686260697431862},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13926095003262162},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13963514694478363},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1410897900350392},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13964752003084868},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13801337301265448},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13951819902285933},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13762343605048954},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13880184304434806},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.14378366293385625},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.14035482902545482},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.14126363396644592},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13782550196629018},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13766782905440778},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13989638397470117},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1391681560780853},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1380202251020819},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1370231059845537},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13859635998960584},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.14179227000568062},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1395210640039295},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13851653190795332},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13713479391299188},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13909496006090194},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.14075164904352278},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.14012027205899358},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.14182660402730107},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.13856645405758172},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.14311004802584648},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13788883097004145},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13857349706813693},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1393006929429248},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.13976318493951112},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.14185786293819547},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.14102777396328747},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1386547889560461},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.14286845596507192},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13953781605232507},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.138497945968993},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13755507802125067},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.14180250896606594},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.14005882700439543},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.14078834792599082},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.139044706011191},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.14436718192882836},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.14004152501001954},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13952595193404704},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.14122772798873484},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1405065410071984},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13947202207054943},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.13773155596572906},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1401030890410766},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.14398266503121704},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13835739402566105},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1386573240160942},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1420121700502932},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.14157229894772172},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1407445960212499},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14170297293458134},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14291897101793438},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14647187595255673},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.14051028003450483},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.14103228296153247},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13691879191901535},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13755522901192307},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13704816298559308},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13767629396170378},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13855048501864076},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.14306945097632706},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13727804098743945},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1391567849786952},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1401327650528401},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13731214497238398},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13839691795874387},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.140254070982337},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13760047301184386},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1389228190528229},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.14103827392682433},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13837856403551996},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13688809401355684},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13913444394711405},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1396249580429867},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13684421207290143},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1390983359888196},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.14017731801141053},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.14550536405295134},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.14296895498409867},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.14088470791466534},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1415002349531278},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13860731106251478},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13888074993155897},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13810338103212416},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13880114199128002},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.14268354198429734},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13893577293492854},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13995379104744643},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13934361399151385},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13881274301093072},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1391774530056864},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13992911507375538},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.14258507802151144},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.14515672298148274},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1422134059248492},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1405059100361541},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.14249473006930202},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.14033280801959336},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.14033013302832842},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.13844453694764525},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.14151522296015173},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.1436775759793818},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.1409098650328815},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.14076170802582055},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.13885641493834555},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.13872859592083842},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13944579404778779},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13916505000088364},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.14484583400189877},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.14240737701766193},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.14224715798627585},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.1416317110415548},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.14161241392139345},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.14054756902623922},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.14067780098412186},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.14137927000410855},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.14482949394732714},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.14252008707262576},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.14126001694239676},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.1432672500377521},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.1430495650274679},{"candidates":15,"kind":"choice","padded_tokens":2610,"seconds":0.15635931491851807}],"chat_template":false,"data_loading_seconds":0.04936886508949101,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-arc-challenge-v1","finished":1792249324,"gpu":0,"id":"09f6a6baf63bfef6807284ba","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","model_load_seconds":1.9142516610445455,"model_revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","n":1172,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":149333081,"peak_gpu_bytes":894092288,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge (full test, raw likelihood accuracy)","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"native-arc-challenge-common-core-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Challenge / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-arc-challenge-v1","recipe":{"data_file":"ARC-Challenge/test-00000-of-00001.parquet","schema":"arc-challenge-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"arc_challenge"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":45.200814573909156,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"e7984b7ea3a9cae0996b0a453de457a6a7d717e9a2c3e9a95d80f697f8bd5afb","score":0.26535836177474403,"scoring_seconds":41.47548789996654,"seconds":43.54575960303191,"seed":42,"selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","status":"completed","task":"native-arc-challenge-common-core-full","tf32":false,"truncated_candidates":0,"worker":"white-gpu0"}