{"acc_norm":0.6214067278287462,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.5996941896024465,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.2890768030192703},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13579810492228717},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.13304319593589753},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13252188393380493},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13116878899745643},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13128343399148434},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13163986802101135},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13214159500785172},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.1302113039419055},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1331741400063038},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13104405696503818},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13098244194407016},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13120048900600523},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.13636939995922148},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.13418824202381074},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.13362161489203572},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.13484130997676402},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.13510536297690123},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.1346794980345294},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.13445475907064974},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13435473199933767},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.13401344593148679},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.13382913905661553},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.13358536700252444},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.13428657304029912},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.1339389350032434},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.13716275803744793},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.13397588406223804},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.13425776199437678},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.13358931406401098},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.13384422997478396},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.13764989899937063},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.1369647110113874},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.13888659502845258},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.13691129197832197},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.13488893792964518},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.13420892099384218},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.1384180219611153},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.13453484792262316},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.13462470599915832},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.13369260798208416},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.1335608009248972},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.13405826897360384},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.13456311100162566},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.13434061594307423},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.13523899193387479},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.13818234205245972},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.13629418006166816},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.1421061010332778},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.1368543249554932},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.13688724604435265},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.13643387099727988},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.13764771400019526},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.13678836193867028},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.13694158697035164},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.13559251104015857},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.13604680798016489},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.1354151099221781},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.13614757603500038},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.13938729010988027},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.13852109294384718},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.1370995920151472},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.13658174593001604},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.13831179297994822},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.13886788999661803},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.13801625196356326},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.13870122993830591},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.1352141359820962},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.13953004602808505},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.13622361805755645},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.1352594499476254},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.13497803499922156},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.13490012905094773},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.13582480396144092},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.13507148006465286},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.13607556198257953},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.1353101640706882},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.13660736405290663},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.13578830601181835},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.13926448102574795},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.13484291301574558},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.13469607895240188},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.13471070700325072},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.1347155950497836},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.13923357299063355},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.1367331090150401},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.1354190669953823},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.13463890203274786},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.13476391602307558},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.13558184099383652},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.13889906893018633},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.1373066989472136},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.13677278300747275},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.13562874996569008},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.13462979497853667},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.13450193707831204},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.13559892307966948},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.13498917606193572},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.13436608400661498},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.13628061499912292},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.13668189302552491},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.1352538299979642},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.13547814800404012},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.13655615807510912},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.13695649500004947},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.13531016500201076},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.13637836696580052},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.1349964599357918},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.1352613129420206},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.13609233300667256},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.13693181902635843},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.14157894696108997},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.1361888439860195},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.1360923540778458},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.1367016399744898},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.13718284806236625},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.13770807802211493},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.1369482500012964},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.13709193794056773},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.13803838309831917},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.13749870704486966},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.14193190506193787},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.13733521092217416},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.13708639796823263},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.13629861804656684},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.13657907105516642},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.13667160493787378},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.13744861294981092},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.13738647801801562},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.1372613940620795},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.14048548799473792},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.13781318301334977},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.1411861750530079},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.1368128570029512},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.1424433900974691},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.13898478902410716},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.13803569902665913},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.13770479103550315},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.16326170007232577},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.15775034006219357},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.15637535601854324},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.15548023604787886},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.16155694099143147},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.1545810989337042},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.1558387060649693},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.16220266500022262},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.15622210002038628},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.15596672298852354},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.15589364804327488},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.15982161508873105},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.15648228500504047},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.15769116999581456},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.1563628719886765},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.16238185996189713},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.1560460519976914},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.15601166698615998},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.15584296302404255},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.1624048330122605},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.15621063904836774},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.15694224403705448},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.16370048001408577},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.15665420598816127},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.15755367395468056},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.15613579901400954},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.1613298379816115},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.15733404399361461},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.15751599404029548},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.15734217001590878},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.16178557695820928},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.15768100204877555},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.1580539469141513},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.16427221603225917},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.15823378204368055},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.15755197091493756},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.15836573904380202},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.16460675897542387},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.15703882405068725},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.15866806206759065},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.1575664880219847},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.164927558042109},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.15715474996250123},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.15756091696675867},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.16344913991633803},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.15890120703261346},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.15961027995217592},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.15864402800798416},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.16377947700675577},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.1584095211001113},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.15736086398828775},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.16252212098333985},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.15841824596282095},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.15909505903255194},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.15853526396676898},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.16284121701028198},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.16038685909006745},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.1581657649949193},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.16468697891104966},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.16001466405577958},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.15998325496912003},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.15985245199408382},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.16596694698091596},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.15973018307704479},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.15952592203393579},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.16620327800046653},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.1592647860525176},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.1602768530137837},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.16501370910555124},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.1604162430157885},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.16330042202025652},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.16932560096029192},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.16281811299268156},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.16246904199942946},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.16292675607837737},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.1678897930542007},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.16376673302147537},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.16391243401449174},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.1678248520474881},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.16311517695430666},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.16201087704394013},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.16713780106510967},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.1623778019566089},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.16610725806094706},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.17032435396686196},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.1652671709889546},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.16676824097521603},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.17113845294807106},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.16570246301125735},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.16647555492818356},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.17062981403432786},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.1661388569045812},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.17277465399820358},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.16621573106385767},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.16780275106430054},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.17338503303471953},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.16589468100573868},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.16646632796619087},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.17200033995322883},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.1664686129661277},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.16665112203918397},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.1740384619915858},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.16753546195104718},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.1672994620166719},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.1736127770273015},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.16731250600423664},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.1723576260264963},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.16742623806931078},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.16761440003756434},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.176948931068182},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.17770177498459816},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.18134071200620383},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.17641347099561244},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.1763851490104571},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.1816480050329119},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.17654932499863207},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.18360050697810948},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.17787649005185813},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.1823069250676781},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.17812054499518126},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.17826908198185265},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.18233554798644036},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.17780414503067732},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.18385662499349564},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.17745279101654887},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.18542905803769827},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.17804498400073498},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.18322783196344972},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.17857906001154333},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.18419931398238987},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.17872872005682439},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.17885018698871136},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.18662329495418817},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.17946002399548888},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.185350400977768},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.17915279103908688},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.18502833996899426},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.17926117300521582},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.18474782700650394},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.17959815089125186},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.184602587018162},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.17952351202256978},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.1856835630023852},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.17990683706011623},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.1812894759932533},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.18620732997078449},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.18071567604783922},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.18516966397874057},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.18020875100046396},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.19266250997316092},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.18759122991468757},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.19214356201700866},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.1877365519758314},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.19347580592148006},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.18738236103672534},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.1940140399383381},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.19298164499923587},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.18812182999681681},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.1936207669787109},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.18918034504167736},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.1931074489839375},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.1893924199976027},{"candidates":16,"kind":"choice","padded_tokens":3232,"seconds":0.19567865505814552},{"candidates":16,"kind":"choice","padded_tokens":3232,"seconds":0.18857947399374098},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.1954811770701781},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.18940361100248992},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.19350647402461618},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.19008500303607434},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.19493748305831105},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.19450687000062317},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.1889023360563442},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.19532151997555047},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.18988021998666227},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.1970847980119288},{"candidates":16,"kind":"choice","padded_tokens":3360,"seconds":0.1895399559289217},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.1946146310074255},{"candidates":16,"kind":"choice","padded_tokens":3392,"seconds":0.1909916530130431},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.1938093489734456},{"candidates":16,"kind":"choice","padded_tokens":3424,"seconds":0.19755063694901764},{"candidates":16,"kind":"choice","padded_tokens":3424,"seconds":0.19078116095624864},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.19529100297950208},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.18994465994182974},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.1953989239409566},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.19173377798870206},{"candidates":16,"kind":"choice","padded_tokens":3504,"seconds":0.19741374207660556},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.19106686406303197},{"candidates":16,"kind":"choice","padded_tokens":3536,"seconds":0.1970275118947029},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.19550835900008678},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.19249672000296414},{"candidates":16,"kind":"choice","padded_tokens":3568,"seconds":0.1986608380684629},{"candidates":16,"kind":"choice","padded_tokens":3584,"seconds":0.19090991094708443},{"candidates":16,"kind":"choice","padded_tokens":3600,"seconds":0.20770106103736907},{"candidates":16,"kind":"choice","padded_tokens":3616,"seconds":0.2111870419466868},{"candidates":16,"kind":"choice","padded_tokens":3632,"seconds":0.206247789086774},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.2115683340234682},{"candidates":16,"kind":"choice","padded_tokens":3664,"seconds":0.2064288769615814},{"candidates":16,"kind":"choice","padded_tokens":3680,"seconds":0.19432878796942532},{"candidates":16,"kind":"choice","padded_tokens":3696,"seconds":0.20115780597552657},{"candidates":16,"kind":"choice","padded_tokens":3712,"seconds":0.1954294809838757},{"candidates":16,"kind":"choice","padded_tokens":3712,"seconds":0.20070248597767204},{"candidates":16,"kind":"choice","padded_tokens":3744,"seconds":0.19424167496617883},{"candidates":16,"kind":"choice","padded_tokens":3744,"seconds":0.19508335506543517},{"candidates":16,"kind":"choice","padded_tokens":3760,"seconds":0.20098853996023536},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.19694344501476735},{"candidates":16,"kind":"choice","padded_tokens":3792,"seconds":0.2037296120543033},{"candidates":16,"kind":"choice","padded_tokens":3808,"seconds":0.19544443802442402},{"candidates":16,"kind":"choice","padded_tokens":3824,"seconds":0.19695579796098173},{"candidates":16,"kind":"choice","padded_tokens":3840,"seconds":0.20252620801329613},{"candidates":16,"kind":"choice","padded_tokens":3856,"seconds":0.19618497998453677},{"candidates":16,"kind":"choice","padded_tokens":3872,"seconds":0.20206513802986592},{"candidates":16,"kind":"choice","padded_tokens":3904,"seconds":0.20158878900110722},{"candidates":16,"kind":"choice","padded_tokens":3936,"seconds":0.19666357291862369},{"candidates":16,"kind":"choice","padded_tokens":3936,"seconds":0.20419195503927767},{"candidates":16,"kind":"choice","padded_tokens":3952,"seconds":0.1979521360481158},{"candidates":16,"kind":"choice","padded_tokens":3984,"seconds":0.20297556603327394},{"candidates":16,"kind":"choice","padded_tokens":4000,"seconds":0.2034886929905042},{"candidates":16,"kind":"choice","padded_tokens":4016,"seconds":0.19834329606965184},{"candidates":16,"kind":"choice","padded_tokens":4032,"seconds":0.203844738076441},{"candidates":16,"kind":"choice","padded_tokens":4048,"seconds":0.2010974430013448},{"candidates":16,"kind":"choice","padded_tokens":4048,"seconds":0.20480572897940874},{"candidates":16,"kind":"choice","padded_tokens":4064,"seconds":0.203319426975213},{"candidates":16,"kind":"choice","padded_tokens":4080,"seconds":0.1987036579521373},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.20374029292725027},{"candidates":16,"kind":"choice","padded_tokens":4128,"seconds":0.2265442470088601},{"candidates":16,"kind":"choice","padded_tokens":4144,"seconds":0.22735648194793612},{"candidates":16,"kind":"choice","padded_tokens":4160,"seconds":0.22079833003226668},{"candidates":16,"kind":"choice","padded_tokens":4208,"seconds":0.2252805089810863},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.22625231300480664},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.22727942804340273},{"candidates":16,"kind":"choice","padded_tokens":4288,"seconds":0.2262602070113644},{"candidates":16,"kind":"choice","padded_tokens":4320,"seconds":0.22255650896113366},{"candidates":16,"kind":"choice","padded_tokens":4352,"seconds":0.22677302395459265},{"candidates":16,"kind":"choice","padded_tokens":4384,"seconds":0.22721722209826112},{"candidates":16,"kind":"choice","padded_tokens":4448,"seconds":0.2283140280051157},{"candidates":16,"kind":"choice","padded_tokens":4464,"seconds":0.22807981201913208},{"candidates":16,"kind":"choice","padded_tokens":4512,"seconds":0.2292144080856815},{"candidates":16,"kind":"choice","padded_tokens":4576,"seconds":0.22372952697332948},{"candidates":16,"kind":"choice","padded_tokens":4608,"seconds":0.22846312599722296},{"candidates":16,"kind":"choice","padded_tokens":4624,"seconds":0.2398647329537198},{"candidates":16,"kind":"choice","padded_tokens":4656,"seconds":0.24482956295832992},{"candidates":16,"kind":"choice","padded_tokens":4720,"seconds":0.24699148500803858},{"candidates":16,"kind":"choice","padded_tokens":4752,"seconds":0.246262826025486},{"candidates":16,"kind":"choice","padded_tokens":4784,"seconds":0.24787097598891705},{"candidates":16,"kind":"choice","padded_tokens":4816,"seconds":0.2478736420162022},{"candidates":16,"kind":"choice","padded_tokens":4832,"seconds":0.2469523629406467},{"candidates":16,"kind":"choice","padded_tokens":4880,"seconds":0.24917895591352135},{"candidates":16,"kind":"choice","padded_tokens":4896,"seconds":0.24933373497333378},{"candidates":16,"kind":"choice","padded_tokens":4944,"seconds":0.24860164907295257},{"candidates":16,"kind":"choice","padded_tokens":4976,"seconds":0.24860710999928415},{"candidates":16,"kind":"choice","padded_tokens":5024,"seconds":0.2523620129795745},{"candidates":16,"kind":"choice","padded_tokens":5072,"seconds":0.2496548640774563},{"candidates":16,"kind":"choice","padded_tokens":5104,"seconds":0.2531058000167832},{"candidates":16,"kind":"choice","padded_tokens":5168,"seconds":0.2531028949888423},{"candidates":16,"kind":"choice","padded_tokens":5232,"seconds":0.2551552129443735},{"candidates":16,"kind":"choice","padded_tokens":5312,"seconds":0.2536510069621727},{"candidates":16,"kind":"choice","padded_tokens":5344,"seconds":0.25417094700969756},{"candidates":16,"kind":"choice","padded_tokens":5472,"seconds":0.2556054829619825},{"candidates":16,"kind":"choice","padded_tokens":5552,"seconds":0.2599059750791639},{"candidates":16,"kind":"choice","padded_tokens":5600,"seconds":0.2562546939589083},{"candidates":16,"kind":"choice","padded_tokens":5760,"seconds":0.25677202898077667},{"candidates":16,"kind":"choice","padded_tokens":5840,"seconds":0.2636949419975281},{"candidates":16,"kind":"choice","padded_tokens":5984,"seconds":0.25872365897521377},{"candidates":16,"kind":"choice","padded_tokens":6096,"seconds":0.26025612803641707},{"candidates":16,"kind":"choice","padded_tokens":6320,"seconds":0.27220744907390326},{"candidates":16,"kind":"choice","padded_tokens":6720,"seconds":0.28621589799877256},{"candidates":16,"kind":"choice","padded_tokens":7088,"seconds":0.29898784100078046},{"candidates":16,"kind":"choice","padded_tokens":7408,"seconds":0.3260176780167967},{"candidates":15,"kind":"choice","padded_tokens":7725,"seconds":0.3199320969870314},{"candidates":15,"kind":"choice","padded_tokens":8100,"seconds":0.3318291469477117},{"candidates":12,"kind":"choice","padded_tokens":7452,"seconds":0.29722919093910605},{"candidates":10,"kind":"choice","padded_tokens":7280,"seconds":0.2872868560953066},{"candidates":8,"kind":"choice","padded_tokens":8192,"seconds":0.28089068608824164}],"chat_template":false,"data_loading_seconds":0.06916403700597584,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-boolq-lm-eval-v2","finished":1792251731,"gpu":0,"id":"63106faba6f2509a890df83c","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149","model_load_seconds":2.3010914450278506,"model_revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":148910738,"peak_gpu_bytes":1022030336,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"native-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":75.994809292024,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"7f2a4b46461d9e7fe0763f55ef1a046476cd093351d877ad5515e16812e1bdca","score":0.5996941896024465,"scoring_seconds":71.51201459998265,"seconds":74.32800504006445,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"native-boolq-common-core-full","tf32":false,"truncated_candidates":2,"worker":"white-gpu0"}