{"acc_norm":0.6204892966360857,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.5990825688073395,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.27664188109338284},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13997239095624536},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.14068114303518087},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.14135130308568478},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1418862409191206},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1403775360668078},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.14221652608830482},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1408533229259774},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.14062225201632828},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1417037700302899},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14409229496959597},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.14026458596345037},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.14022365002892911},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1398408249951899},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1405065970029682},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1438691389048472},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.14092157094273716},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1413779929280281},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.14579767594113946},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.14560866402462125},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1414173860102892},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.14311398100107908},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.14253237599041313},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.142198913032189},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.1461017830297351},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.141979684936814},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.1429980640532449},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.1410776029806584},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.14062016899697483},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.14086648903321475},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.1408010870218277},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.14049623801838607},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1450293529778719},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.14409076306037605},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.14421841106377542},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.14217284496407956},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.14265799999702722},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.14235742995515466},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.1457955320365727},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.14276219496969134},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.14427875296678394},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.1418812710326165},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.14472663891501725},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.14372118399478495},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.14289378805551678},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.14475305494852364},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.14184054697398096},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.14289557095617056},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.14282021205872297},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.1442452409537509},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.14240204298403114},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.14227581699378788},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.14552922500297427},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.14268913795240223},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.14356073399540037},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.142609209055081},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.1424843460554257},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.14269702299498022},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.14908231201115996},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.14285768300760537},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.14318472205195576},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.14367376500740647},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.14292746200226247},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.1436519940616563},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.14666929200757295},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.14516642899252474},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.14353414403740317},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.14545922505203635},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.14583315199706703},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.14313722401857376},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.14301750098820776},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.1466008749557659},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.14363778801634908},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.14360116003081203},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.14402799494564533},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.145080799004063},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.14368798094801605},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.14594983006827533},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.14383963402360678},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.14676179504022002},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.14342146401759237},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.143198358011432},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.14493800199124962},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.14684580103494227},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.14288867998402566},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.14317759894765913},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.14331414399202913},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.14452995092142373},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.1432904900284484},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.1520989789860323},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.1482603799086064},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.14573071105405688},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.14450905204284936},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.14553623902611434},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.14669627195689827},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.14992937305942178},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.14591334201395512},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.14497071399819106},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.14484323596116155},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.14413306105416268},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.14413654804229736},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.1479077619733289},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.14495952299330384},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.14357965893577784},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.14491846703458577},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.14477810496464372},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.14469877700321376},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.1484684469178319},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.14385935000609607},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.14543112297542393},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.14413381309714168},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.14299926697276533},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.14282978000119328},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.15037175791803747},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.14555266906972975},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.14339523506350815},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.14516555797308683},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.14447453897446394},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.14494652801658958},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.1491613209946081},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.14339748898055404},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.1436815990600735},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.1440424119355157},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.1450101169757545},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.14546475501265377},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.14840153302066028},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.14424838603008538},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.14342381909955293},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.14318852010183036},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.14364686398766935},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.14540325000416487},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.14902489609085023},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.14435628801584244},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.14418107108213007},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.1439167879289016},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.1444808889646083},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.1453337010461837},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.1513385009020567},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.14855629194062203},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.14541712601203471},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.1444781149039045},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.14344882604200393},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.143290059058927},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.14953260298352689},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.14389948500320315},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.14422806806396693},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.14399530400987715},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.14428861101623625},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.14384071505628526},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.14842984592542052},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.1448391079902649},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.1444291229126975},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.14600660605356097},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.14624273602385074},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.14629623596556485},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.1487076940247789},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.14606431301217526},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.1469565989682451},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.14582250197418034},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.1438134249765426},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.14548993203788996},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.15367909707129002},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.14779666508547962},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.1462446500081569},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.14557111309841275},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.1463876460911706},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.15114118298515677},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.14672031707596034},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.14777685806620866},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.14604698098264635},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.14621996402274817},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.1520089010009542},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.14593541296198964},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.16021417698357254},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.16670895996503532},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.1717920619994402},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.16599966795183718},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.16576416802126914},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.17050103400833905},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.16606575006153435},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.16592274396680295},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.17151194892358035},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.16665539995301515},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.16717537096701562},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.17161002196371555},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.1659972129855305},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.17175756802316755},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.16696144198067486},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.16624959406908602},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.17212322901468724},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.1672703369986266},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.16647560498677194},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.17243650404270738},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.1669309950666502},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.16756223200354725},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.1725551150739193},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.16841580404434353},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.16666318499483168},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.1729231709614396},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.16763649100903422},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.17232617689296603},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.16798148502130061},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.16866057098377496},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.1730529440101236},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.16801953501999378},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.16747089195996523},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.17414103297051042},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.16903266496956348},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.1675758269848302},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.17569331894628704},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.16736398299690336},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.1734447240596637},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.16858776495791972},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.16828390897717327},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.17409344401676208},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.16822607000358403},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.16803990304470062},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.17255615699104965},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.16915287903975695},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.16983963001985103},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.17616584000643343},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.16924921900499612},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.17553092492744327},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.16907345002982765},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.1696101410780102},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.17623739398550242},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.17059081105981022},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.17549460800364614},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.16973957302980125},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.17113084893207997},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.1760697200661525},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.16963765304535627},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.1698680119588971},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.17498639889527112},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.1713970348937437},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.1745380739448592},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.16968828707467765},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.17129408195614815},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.17474012996535748},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.17065314704086632},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.17718501097988337},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.17032376304268837},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.1709149950183928},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.17572418600320816},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.17051151290070266},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.17112682096194476},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.17554386996198446},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.17096499900799245},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.17551653797272593},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.17081895598676056},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.17123310896568},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.17732005298603326},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.170892394031398},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.17718079290352762},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.17108032398391515},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.1708953189663589},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.18108821101486683},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.17911374801769853},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.1841684269020334},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.17620514298323542},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.18259728595148772},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.1768510879483074},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.17703406896907836},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.1831328240223229},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.17713224200997502},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.18243345990777016},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.17899098002817482},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.1819463319843635},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.1772835849551484},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.18355360999703407},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.17734254500828683},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.17790056602098048},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.18258163693826646},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.177761445986107},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.18337569700088352},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.1775593999773264},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.18313445802778006},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.17742344608996063},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.1826597119215876},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.17904761398676783},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.1791727190138772},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.18299553799442947},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.18021451099775732},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.18285875394940376},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.17867458995897323},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.18440546898636967},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.17815184395294636},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.18337457603774965},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.17858654505107552},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.18558765295892954},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.1822300509084016},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.17859475989826024},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.18704079405870289},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.18211702001281083},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.18423825700301677},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.18142432800959796},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.18411847297102213},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.18016738398000598},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.1850455419626087},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.1798921999288723},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.18540926091372967},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.17988803202752024},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.18531794007867575},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.1819194220006466},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.18545601808000356},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.18063370301388204},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.18035401194356382},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.18655177194159478},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.18104256596416235},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.18556434998754412},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.1809778850292787},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.1853985810885206},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.1815388819668442},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.18558593990746886},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.18089159508235753},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.18718970206100494},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.1893428680486977},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.20423772104550153},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.20549274107906967},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.19958906003739685},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.20480660104658455},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.2000101050361991},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.2049497290281579},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.20689041900914162},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.1997261350043118},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.20636672095861286},{"candidates":16,"kind":"choice","padded_tokens":3232,"seconds":0.20583522005472332},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.2015542850131169},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.20603822800330818},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.20723589300177991},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.20123953791335225},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.20701979903969914},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.2029690039344132},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.20652659004554152},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.20940526796039194},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.20126391400117427},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.20765254006255418},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.2091935940552503},{"candidates":16,"kind":"choice","padded_tokens":3392,"seconds":0.2027982360450551},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.20922967104706913},{"candidates":16,"kind":"choice","padded_tokens":3424,"seconds":0.20745478093158454},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.2026527940761298},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.20920060703065246},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.21033109596464783},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.20358813903294504},{"candidates":16,"kind":"choice","padded_tokens":3504,"seconds":0.20805093296803534},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.2092368040466681},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.20441342797130346},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.2108011730015278},{"candidates":16,"kind":"choice","padded_tokens":3568,"seconds":0.21103690203744918},{"candidates":16,"kind":"choice","padded_tokens":3584,"seconds":0.20383510901592672},{"candidates":16,"kind":"choice","padded_tokens":3584,"seconds":0.21028660295996815},{"candidates":16,"kind":"choice","padded_tokens":3616,"seconds":0.211959553998895},{"candidates":16,"kind":"choice","padded_tokens":3632,"seconds":0.20789659500587732},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.21519765292759985},{"candidates":16,"kind":"choice","padded_tokens":3664,"seconds":0.21398905909154564},{"candidates":16,"kind":"choice","padded_tokens":3680,"seconds":0.21118145203217864},{"candidates":16,"kind":"choice","padded_tokens":3696,"seconds":0.2084061250789091},{"candidates":16,"kind":"choice","padded_tokens":3728,"seconds":0.22218363406136632},{"candidates":16,"kind":"choice","padded_tokens":3744,"seconds":0.2237581410445273},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.22574970498681068},{"candidates":16,"kind":"choice","padded_tokens":3792,"seconds":0.2207488470012322},{"candidates":16,"kind":"choice","padded_tokens":3808,"seconds":0.22665207798127085},{"candidates":16,"kind":"choice","padded_tokens":3840,"seconds":0.224487190018408},{"candidates":16,"kind":"choice","padded_tokens":3856,"seconds":0.22433437500149012},{"candidates":16,"kind":"choice","padded_tokens":3872,"seconds":0.22574580798391253},{"candidates":16,"kind":"choice","padded_tokens":3888,"seconds":0.22120427701156586},{"candidates":16,"kind":"choice","padded_tokens":3936,"seconds":0.22693680902011693},{"candidates":16,"kind":"choice","padded_tokens":3968,"seconds":0.22791753895580769},{"candidates":16,"kind":"choice","padded_tokens":4000,"seconds":0.22938603698275983},{"candidates":16,"kind":"choice","padded_tokens":4032,"seconds":0.22928720409981906},{"candidates":16,"kind":"choice","padded_tokens":4080,"seconds":0.2287140639964491},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.22229088400490582},{"candidates":16,"kind":"choice","padded_tokens":4144,"seconds":0.23406177898868918},{"candidates":16,"kind":"choice","padded_tokens":4160,"seconds":0.23508226196281612},{"candidates":16,"kind":"choice","padded_tokens":4208,"seconds":0.23707536002621055},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.2367133649531752},{"candidates":16,"kind":"choice","padded_tokens":4288,"seconds":0.236785429995507},{"candidates":16,"kind":"choice","padded_tokens":4320,"seconds":0.23782787402160466},{"candidates":16,"kind":"choice","padded_tokens":4352,"seconds":0.23796280601527542},{"candidates":16,"kind":"choice","padded_tokens":4400,"seconds":0.23269716405775398},{"candidates":16,"kind":"choice","padded_tokens":4416,"seconds":0.2381692809285596},{"candidates":16,"kind":"choice","padded_tokens":4432,"seconds":0.2392833479680121},{"candidates":16,"kind":"choice","padded_tokens":4464,"seconds":0.240151638048701},{"candidates":16,"kind":"choice","padded_tokens":4496,"seconds":0.24016491195652634},{"candidates":16,"kind":"choice","padded_tokens":4528,"seconds":0.24067605705931783},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.24046089500188828},{"candidates":16,"kind":"choice","padded_tokens":4608,"seconds":0.23982667201198637},{"candidates":16,"kind":"choice","padded_tokens":4624,"seconds":0.25938350008800626},{"candidates":16,"kind":"choice","padded_tokens":4688,"seconds":0.26710871898103505},{"candidates":16,"kind":"choice","padded_tokens":4736,"seconds":0.2650461819721386},{"candidates":16,"kind":"choice","padded_tokens":4784,"seconds":0.2679260630393401},{"candidates":16,"kind":"choice","padded_tokens":4864,"seconds":0.27282416995149106},{"candidates":16,"kind":"choice","padded_tokens":4912,"seconds":0.2682608579052612},{"candidates":16,"kind":"choice","padded_tokens":4960,"seconds":0.2698942150454968},{"candidates":16,"kind":"choice","padded_tokens":5072,"seconds":0.2697716159746051},{"candidates":16,"kind":"choice","padded_tokens":5120,"seconds":0.27002286503557116},{"candidates":16,"kind":"choice","padded_tokens":5200,"seconds":0.2829210429918021},{"candidates":16,"kind":"choice","padded_tokens":5296,"seconds":0.2799195139668882},{"candidates":16,"kind":"choice","padded_tokens":5440,"seconds":0.2793782240478322},{"candidates":16,"kind":"choice","padded_tokens":5648,"seconds":0.2833152590319514},{"candidates":16,"kind":"choice","padded_tokens":5856,"seconds":0.29638775205239654},{"candidates":16,"kind":"choice","padded_tokens":6080,"seconds":0.29390910791698843},{"candidates":16,"kind":"choice","padded_tokens":6416,"seconds":0.3113893730333075},{"candidates":16,"kind":"choice","padded_tokens":6704,"seconds":0.31239734299015254},{"candidates":16,"kind":"choice","padded_tokens":7216,"seconds":0.32652119698468596},{"candidates":16,"kind":"choice","padded_tokens":8032,"seconds":0.34118644101545215},{"candidates":13,"kind":"choice","padded_tokens":8008,"seconds":0.3412861170945689},{"candidates":11,"kind":"choice","padded_tokens":8151,"seconds":0.33391823398415},{"candidates":4,"kind":"choice","padded_tokens":4096,"seconds":0.15604914701543748}],"chat_template":false,"data_loading_seconds":0.053120612050406635,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-boolq-lm-eval-v2","finished":1792251824,"gpu":0,"id":"4247e4e337da8cfb0019a899","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer-149M-Decision-v1","model_load_seconds":1.9057621649699286,"model_revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":149333081,"peak_gpu_bytes":1383601152,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"native-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":76.83916322898585,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"7f2a4b46461d9e7fe0763f55ef1a046476cd093351d877ad5515e16812e1bdca","score":0.5990825688073395,"scoring_seconds":72.78117600898258,"seconds":75.20840053097345,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"native-boolq-common-core-full","tf32":false,"truncated_candidates":2,"worker":"white-gpu0"}