{"acc_norm":0.6217125382262997,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.5935779816513761,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.2825482670450583},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13865761796478182},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13650460203643888},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13731692708097398},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.137874317006208},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13713830395136029},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13676715199835598},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1369802999543026},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13716862094588578},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13821363099850714},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13780788297299296},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13573124003596604},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13702050503343344},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13952233095187694},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1386553639313206},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13646987709216774},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13955319894012064},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13995873497333378},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13927624304778874},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1375228710239753},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13716887205373496},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.138382124947384},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.13924235897138715},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.13925545301754028},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.1397623389493674},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.13982985506299883},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.13942691299598664},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.14173557900357991},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1412698799977079},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.14141551300417632},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.13818991603329778},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.13785776600707322},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1387550999643281},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.14237500191666186},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13945433497428894},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.14181143103633076},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1396320260828361},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.1404435490258038},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.13981244200840592},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.1399242909392342},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.1399476049700752},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.14203133096452802},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.14576080802362412},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.14535137405619025},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.14130641904193908},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.13925684604328126},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.13894757896196097},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.13978655403479934},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.13946020498406142},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.13967969408258796},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.14331409404985607},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.14090870704967529},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.14013062603771687},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.14407727704383433},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.14364556199871004},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.14199962199199945},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.1398467270191759},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.14398584607988596},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.141918080043979},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.14183861203491688},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.13977688597515225},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.14085729094222188},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.1398476380854845},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.14024386706296355},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.1428137709153816},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.146427380037494},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.14012644800823182},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.13987011997960508},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.14010704203974456},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.13988561800215393},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.13988217199221253},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.1401640479452908},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.14504215703345835},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.14022696600295603},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.14299525902606547},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.14069160202052444},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.1412920020520687},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.14078161003999412},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.13940593402367085},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.1405377050396055},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.13997845200356096},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.14058391097933054},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.13989731098990887},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.13934520108159631},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.14014209690503776},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.139796142000705},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.14308881398756057},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.145507923909463},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.1429495740449056},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.14762585400603712},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.14343061903491616},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.14253583003301173},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.14166866394225508},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.1470093569951132},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.14311332895886153},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.1424159889575094},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.14320270589087158},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.14240679203066975},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.14195297495462},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.1413517139153555},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.14656306395772845},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.14244113501626998},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.14171344798523933},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.14180168195161968},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.14496187795884907},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.14383515494409949},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.1422112570144236},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.14673641696572304},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.14225021901074797},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.14209557103458792},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.1416564219398424},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.14220453391317278},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.1429138471139595},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.15032205497846007},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.1445450499886647},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.14292330504395068},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.14191262901294976},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.14468988997396082},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.14011504605878145},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.14238119404762983},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.14819257298950106},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.14325258997268975},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.141395895043388},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.14128407696262002},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.1408897020155564},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.14099073002580553},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.14073806896340102},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.14574065001215786},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.14125014399178326},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.14296736696269363},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.140518669039011},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.14203351503238082},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.14209282596129924},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.14081324893049896},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.14934010396245867},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.14373239409178495},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.1414856530027464},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.14359490701463073},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.14435861189849675},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.14140575390774757},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.14142860600259155},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.1417920539388433},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.1412480800645426},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.14311869908124208},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.14184059493709356},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.14258932205848396},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.14163495099637657},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.14721473900135607},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.14270260301418602},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.1448561599245295},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.14322635997086763},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.14214261900633574},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.1435478290077299},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.14212226099334657},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.14809319796040654},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.14251210796646774},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.14232689200434834},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.14407584501896054},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.1435004910454154},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.14433783397544175},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.14980142400600016},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.1457509989850223},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.14309516490902752},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.14483157393988222},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.1457415110198781},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.1452408479526639},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.1485406419960782},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.1449239469366148},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.1435727960197255},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.1433405929710716},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.14337845402769744},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.1455291649326682},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.1483097110176459},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.16119671904016286},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.16533843392971903},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.17089696205221117},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.16681555903051049},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.16584450798109174},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.16966022597625852},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.16556512704119086},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.16606411791872233},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.17104181298054755},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.1664648960577324},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.16624742990825325},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.17176638403907418},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.16637852403800935},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.16653422499075532},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.1721343700774014},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.16661478497553617},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.16720785107463598},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.1728361890418455},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.1680587879382074},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.17299907293636352},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.1666317160706967},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.16636400704737753},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.17251172405667603},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.16755237407051027},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.1680034949677065},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.17306811199523509},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.16663350898306817},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.16629984800238162},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.17349507799372077},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.16745363897643983},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.16704085003584623},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.17245567904319614},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.16857445007190108},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.17165847297292203},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.16800461697857827},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.1674386920640245},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.17387649905867875},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.1689705289900303},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.16744525404646993},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.1739965840242803},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.16784781508613378},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.16774876997806132},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.1742932558991015},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.16775556292850524},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.1734067229554057},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.1698850649408996},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.1685979040339589},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.17481985909398645},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.1682463880861178},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.16885568504221737},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.17509760695975274},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.16896248399280012},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.17532542208209634},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.16957005695439875},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.16904281394090503},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.17537730908952653},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.16918931703548878},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.16997662594076246},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.1753183890832588},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.16955853602848947},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.17652633192483336},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.1697180529590696},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.16993512795306742},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.17442618403583765},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.16942103893961757},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.1712148750666529},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.1765287769958377},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.17014841607306153},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.17530275997705758},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.17042011301964521},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.17212455195840448},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.1752638779580593},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.1720164599828422},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.1752283109817654},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.17031015804968774},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.17043055198155344},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.17658238695003092},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.17183620401192456},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.1711161199491471},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.17581647902261466},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.17123037402052432},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.1753902529599145},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.17026414209976792},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.175305285025388},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.1822811559541151},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.17617017799057066},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.18190419301390648},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.17616270401049405},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.1827067999402061},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.17653420695569366},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.17689003003761172},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.1766225309111178},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.1782292480347678},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.183778960024938},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.17830529098864645},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.18343266402371228},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.1770442280685529},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.18184861890040338},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.17668749298900366},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.17750758200418204},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.18389456591103226},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.1775037549668923},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.18246316607110202},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.17813203705009073},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.1837332850554958},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.1775706399930641},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.18456701992545277},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.17799386999104172},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.18267219595145434},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.17797414306551218},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.1809737179428339},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.18489595293067396},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.1780455259140581},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.18233374506235123},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.17777298798318952},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.18453557195607573},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.17898534901905805},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.18485452700406313},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.17977599299047142},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.18512661405839026},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.1790833210106939},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.17907586798537523},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.18523604702204466},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.18088034400716424},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.18390386295504868},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.17977253801655024},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.18569592491257936},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.1799399290466681},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.18610631092451513},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.18108194007072598},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.18782739096786827},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.17966160096693784},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.18560617801267654},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.18054228299297392},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.1851718679536134},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.18299705197568983},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.1803398960037157},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.1865473740035668},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.18224175297655165},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.18678850401192904},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.18086190905887634},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.18681734695564955},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.18267556198406965},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.19384715892374516},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.1991472049849108},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.19988736603409052},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.20431701804045588},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.1990991960046813},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.2043080209987238},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.20535579498391598},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.20067076804116368},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.20671974902506918},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.20483346190303564},{"candidates":16,"kind":"choice","padded_tokens":3232,"seconds":0.19988704507704824},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.20746412803418934},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.20040183595847338},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.20734067808371037},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.20860377396456897},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.2005548410816118},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.2090905619552359},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.2065299559617415},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.2012298699701205},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.2069884209195152},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.206437754095532},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.20201416302006692},{"candidates":16,"kind":"choice","padded_tokens":3392,"seconds":0.2079926640726626},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.20776533999014646},{"candidates":16,"kind":"choice","padded_tokens":3424,"seconds":0.20242829504422843},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.208868897985667},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.20812308799941093},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.2027136570541188},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.20824403304141015},{"candidates":16,"kind":"choice","padded_tokens":3504,"seconds":0.21068899298552424},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.2038424329366535},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.20959752704948187},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.21056647400837392},{"candidates":16,"kind":"choice","padded_tokens":3568,"seconds":0.20388938998803496},{"candidates":16,"kind":"choice","padded_tokens":3584,"seconds":0.209689409006387},{"candidates":16,"kind":"choice","padded_tokens":3584,"seconds":0.21101360896136612},{"candidates":16,"kind":"choice","padded_tokens":3616,"seconds":0.2085855599725619},{"candidates":16,"kind":"choice","padded_tokens":3632,"seconds":0.2115043749799952},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.21205596311483532},{"candidates":16,"kind":"choice","padded_tokens":3664,"seconds":0.20679638301953673},{"candidates":16,"kind":"choice","padded_tokens":3680,"seconds":0.2115098050562665},{"candidates":16,"kind":"choice","padded_tokens":3696,"seconds":0.21417313301935792},{"candidates":16,"kind":"choice","padded_tokens":3728,"seconds":0.21769695496186614},{"candidates":16,"kind":"choice","padded_tokens":3744,"seconds":0.22625764296390116},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.2263164520263672},{"candidates":16,"kind":"choice","padded_tokens":3792,"seconds":0.22513625991996378},{"candidates":16,"kind":"choice","padded_tokens":3808,"seconds":0.22676843497902155},{"candidates":16,"kind":"choice","padded_tokens":3840,"seconds":0.22477877396158874},{"candidates":16,"kind":"choice","padded_tokens":3856,"seconds":0.22087718697730452},{"candidates":16,"kind":"choice","padded_tokens":3872,"seconds":0.22571868798695505},{"candidates":16,"kind":"choice","padded_tokens":3888,"seconds":0.2292756719980389},{"candidates":16,"kind":"choice","padded_tokens":3936,"seconds":0.22573974705301225},{"candidates":16,"kind":"choice","padded_tokens":3968,"seconds":0.2254777979105711},{"candidates":16,"kind":"choice","padded_tokens":4000,"seconds":0.2224991919938475},{"candidates":16,"kind":"choice","padded_tokens":4032,"seconds":0.23095545498654246},{"candidates":16,"kind":"choice","padded_tokens":4080,"seconds":0.2286837879801169},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.22791332995984703},{"candidates":16,"kind":"choice","padded_tokens":4144,"seconds":0.23421467398293316},{"candidates":16,"kind":"choice","padded_tokens":4160,"seconds":0.23592099698726088},{"candidates":16,"kind":"choice","padded_tokens":4208,"seconds":0.23677320696879178},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.23034576803911477},{"candidates":16,"kind":"choice","padded_tokens":4288,"seconds":0.23511426197364926},{"candidates":16,"kind":"choice","padded_tokens":4320,"seconds":0.2375164719996974},{"candidates":16,"kind":"choice","padded_tokens":4352,"seconds":0.23769039707258344},{"candidates":16,"kind":"choice","padded_tokens":4400,"seconds":0.23671135096810758},{"candidates":16,"kind":"choice","padded_tokens":4416,"seconds":0.2383015570230782},{"candidates":16,"kind":"choice","padded_tokens":4432,"seconds":0.23832350806333125},{"candidates":16,"kind":"choice","padded_tokens":4464,"seconds":0.24030845100060105},{"candidates":16,"kind":"choice","padded_tokens":4496,"seconds":0.23960766301024705},{"candidates":16,"kind":"choice","padded_tokens":4528,"seconds":0.23951046203728765},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.2346847109729424},{"candidates":16,"kind":"choice","padded_tokens":4608,"seconds":0.23988266591913998},{"candidates":16,"kind":"choice","padded_tokens":4624,"seconds":0.2659152630949393},{"candidates":16,"kind":"choice","padded_tokens":4688,"seconds":0.2667129309847951},{"candidates":16,"kind":"choice","padded_tokens":4736,"seconds":0.2648972439346835},{"candidates":16,"kind":"choice","padded_tokens":4784,"seconds":0.2678504929644987},{"candidates":16,"kind":"choice","padded_tokens":4864,"seconds":0.26880935102235526},{"candidates":16,"kind":"choice","padded_tokens":4912,"seconds":0.2670271770330146},{"candidates":16,"kind":"choice","padded_tokens":4960,"seconds":0.26984052499756217},{"candidates":16,"kind":"choice","padded_tokens":5072,"seconds":0.2765129100298509},{"candidates":16,"kind":"choice","padded_tokens":5120,"seconds":0.2697148909792304},{"candidates":16,"kind":"choice","padded_tokens":5200,"seconds":0.2768426139373332},{"candidates":16,"kind":"choice","padded_tokens":5296,"seconds":0.28007820004131645},{"candidates":16,"kind":"choice","padded_tokens":5440,"seconds":0.2865963689982891},{"candidates":16,"kind":"choice","padded_tokens":5648,"seconds":0.2835720570292324},{"candidates":16,"kind":"choice","padded_tokens":5856,"seconds":0.28836752101778984},{"candidates":16,"kind":"choice","padded_tokens":6080,"seconds":0.29109636193607},{"candidates":16,"kind":"choice","padded_tokens":6416,"seconds":0.31006387108936906},{"candidates":16,"kind":"choice","padded_tokens":6704,"seconds":0.3138191949110478},{"candidates":16,"kind":"choice","padded_tokens":7216,"seconds":0.32546044897753745},{"candidates":16,"kind":"choice","padded_tokens":8032,"seconds":0.3464115170063451},{"candidates":13,"kind":"choice","padded_tokens":8008,"seconds":0.3396320720203221},{"candidates":11,"kind":"choice","padded_tokens":8151,"seconds":0.33035859500523657},{"candidates":4,"kind":"choice","padded_tokens":4096,"seconds":0.16158492304384708}],"chat_template":false,"data_loading_seconds":0.05265540408436209,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-boolq-lm-eval-v2","finished":1792251534,"gpu":0,"id":"07c438fc5f1a4da89e00d22f","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","model_load_seconds":1.9261707699624822,"model_revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":149333081,"peak_gpu_bytes":1383601152,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"native-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":76.39033241802827,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"7f2a4b46461d9e7fe0763f55ef1a046476cd093351d877ad5515e16812e1bdca","score":0.5935779816513761,"scoring_seconds":72.28473906300496,"seconds":74.71763013408054,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"native-boolq-common-core-full","tf32":false,"truncated_candidates":2,"worker":"white-gpu0"}