{"acc_norm":0.6143730886850153,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.5357798165137615,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.2722896330524236},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1178597150137648},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11737285694107413},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11663335794582963},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11657256493344903},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11637269204948097},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1158820369746536},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11466294305864722},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11508488992694765},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11758305900730193},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11615364404860884},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.11613266402855515},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.11583795503247529},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.11565480299759656},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.11533467599656433},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.11549419304355979},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.11561942705884576},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.11608974391128868},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.11600880301557481},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1157752180006355},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.11627524998039007},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.11853768001310527},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.11897813098039478},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.1188011709600687},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.11845355306286365},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.11861931206658483},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.11965020396746695},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.12083049595821649},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.11996983201242983},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1201140700140968},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.12108919909223914},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.12019442091695964},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.11934815207496285},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.11967789696063846},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1194684460060671},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.11847962206229568},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.11887527001090348},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.11919707991182804},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.11888534796889871},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.11833844904322177},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.11896641994826496},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.11881298397202045},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.11992165201809257},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.11972728895489126},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.11916034109890461},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.12152396002784371},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.12063422100618482},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.12028103193733841},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.11882799107115716},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.11894678196404129},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.11884525406640023},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.11897176993079484},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.11952719499822706},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.11913102597463876},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.1191490909550339},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.11951053398661315},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.11961182404775172},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.11920418404042721},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.1208640499971807},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.12289378501009196},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.12314074602909386},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.12359739805106074},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.12057440902572125},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.12394859304185957},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.12182651401963085},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.12072252598591149},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.12176424800418317},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.12111268308945},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.12088229297660291},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.12111876404378563},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.12115228699985892},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.12187239003833383},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.12143548508174717},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.12055619608145207},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.12218696600757539},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.12248492299113423},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.1223937920294702},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12268172006588429},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12217016599606723},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.13063743792008609},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.14281343994662166},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.1294091179734096},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.12136940204072744},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.12193602905608714},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.12175456003751606},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.12142579699866474},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.1213491439120844},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.12141688098199666},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.12171434494666755},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.12326896493323147},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.12072025099769235},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.12055859994143248},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.11998635192867368},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.11983604193665087},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.11984794388990849},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.11953135393559933},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.12019815703388304},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.12054560601245612},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.12149021704681218},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.12178009795024991},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.1197315469617024},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.1198793730000034},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.11844340397510678},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.11836706101894379},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.11895721207838506},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.11852668004576117},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.11846948298625648},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.1188804090488702},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.11873894603922963},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.11893042200244963},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.11845426505897194},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.11881969589740038},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.11804179497994483},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.11874338402412832},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.11931602202821523},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.11783604102674872},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.12091583595611155},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.12216025707311928},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.1207132280105725},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.11895734292920679},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.11879612202756107},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.12127875303849578},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.12051440798677504},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.11882097902707756},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.11905327206477523},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.11987934191711247},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.11999470798764378},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.11899329000152647},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.11856567196082324},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.11853254993911833},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.11808240099344403},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.11976613104343414},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.12201492500025779},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.12005491997115314},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.12060852197464556},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.11995692702475935},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.11846505396533757},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.11890974396374077},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.11839534505270422},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.1185877030948177},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.11829804303124547},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.12000939506106079},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.12150763999670744},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.12065151298884302},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.1198700750246644},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.11940569896250963},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.12204106501303613},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.12254368199501187},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.11975479102693498},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.11924993805587292},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.11873375601135194},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.11942770006135106},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.1217706099851057},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.12011610402259976},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.11933588900137693},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.1193672459339723},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.11910650890786201},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.11911729001440108},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.11939359595999122},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.11888501595240086},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.11885787697974592},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.11921948997769505},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.120150908944197},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.11890428198967129},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.1200259659672156},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.1195029899245128},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.11966119601856917},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.11979554605204612},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.11920681793708354},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.11944998102262616},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.12210471392609179},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.12037729099392891},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.11931661295238882},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.12131688289809972},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.12389550299849361},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.12848243000917137},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.12782233906909823},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.12798633403144777},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.1279658559942618},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.12792047101538628},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.1281549889827147},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.12806282693054527},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.1279067259747535},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.12831910396926105},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.12808508798480034},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.12846334394998848},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.12912220298312604},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.12866670300718397},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.12873905804008245},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.12858684407547116},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.12898958497680724},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.12868539802730083},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.12917492096312344},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.12890444695949554},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.1287547571118921},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.12882831401657313},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.1285498549696058},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.12937304005026817},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.1286852969788015},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.12934757291805},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.12954615300986916},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.1292568730423227},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.12956146197393537},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.12938796798698604},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.12941237399354577},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.12936785095371306},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.1294803599594161},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.12944626703392714},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.12952859001234174},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.12965292297303677},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.12976652395445853},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.12988583697006106},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.12960037495940924},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.1294454849557951},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.12960863998159766},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.12999650405254215},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.13012353098019958},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.12955456902273},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.13003310200292617},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.12958174000959843},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.1297091169981286},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.13015986804384738},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.13037633197382092},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.12998447101563215},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.1303287729388103},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.13039826205931604},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.1302305000135675},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.13059656298719347},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.13059233501553535},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.1308443550951779},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.1308241169899702},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.13073513109702617},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.13093567499890924},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.13055712904315442},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.13109117490239441},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.13100686797406524},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.1311259389622137},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.1311742700636387},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.1311914729885757},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.13103776494972408},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.13125188497360796},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.13141578098293394},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.1314526890637353},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.13148428895510733},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.13144663802813739},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.13152223895303905},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.13150942500215024},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.13164365501143038},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.1312309360364452},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.13198359007947147},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.13187891501002014},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.1320291650481522},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.131892069010064},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.13210252195131034},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.13219592592213303},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.13158394501078874},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.1315349229844287},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.1349087959388271},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.13992920005694032},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.13988391601014882},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.13996727101039141},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.13988600892480463},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.14031464897561818},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.14024380699265748},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.14078758098185062},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.1404893039725721},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.1407324179308489},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.1404809090308845},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.14041814196389169},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.14148249791469425},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.14093249198049307},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.1412785860011354},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.14151998690795153},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.14095597504638135},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.1407753680832684},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.1414566090097651},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.1414527219021693},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.141558199073188},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.1417326140217483},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.14180425798986107},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.14124933304265141},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.14152572897728533},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.14176475396379828},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.14167358400300145},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.14676715398672968},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.14988121204078197},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.15054993005469441},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.15059842099435627},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.15051003592088819},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.15071221301332116},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.15076397894881666},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.15159588097594678},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.15115636098198593},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.15079211094416678},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.15121610299684107},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.15123194199986756},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.1517574719619006},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.15173406898975372},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.15190373605582863},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.15113379899412394},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.15175472700502723},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.15236764098517597},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.15237845107913017},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.15247706498485059},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.15232925896998495},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.15188645292073488},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.15221009694505483},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.15298533299937844},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.15297221799846739},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.15288507496006787},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.15292509994469583},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.15274442394729704},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.15232719492632896},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.153527403017506},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.1535572890425101},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.15349559392780066},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.15365132491569966},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.15359492995776236},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.16066021902952343},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.16056031198240817},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.1607030879240483},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.16037792095448822},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.1606463419739157},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.16124268400017172},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.16123119299300015},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.16169937700033188},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.1610359400510788},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.16212262597400695},{"candidates":16,"kind":"choice","padded_tokens":3232,"seconds":0.1612883300986141},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.16241898795124143},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.16262142604682595},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.16253813996445388},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.16253822098951787},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.16288954589981586},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.16153223509900272},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.16264931892510504},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.16247138695325702},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.16337367903906852},{"candidates":16,"kind":"choice","padded_tokens":3360,"seconds":0.1629960050340742},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.16419908904936165},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.16291358100716025},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.16274928499478847},{"candidates":16,"kind":"choice","padded_tokens":3424,"seconds":0.16345191502477974},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.1641242289915681},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.1639433919917792},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.16504597896710038},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.1637214879738167},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.1634592389455065},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.16461020603310317},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.16461361199617386},{"candidates":16,"kind":"choice","padded_tokens":3536,"seconds":0.16518775396980345},{"candidates":16,"kind":"choice","padded_tokens":3536,"seconds":0.1654686169931665},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.16437993699219078},{"candidates":16,"kind":"choice","padded_tokens":3568,"seconds":0.1638269550167024},{"candidates":16,"kind":"choice","padded_tokens":3600,"seconds":0.16501829703338444},{"candidates":16,"kind":"choice","padded_tokens":3616,"seconds":0.16628541005775332},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.16764137905556709},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.16771178098861128},{"candidates":16,"kind":"choice","padded_tokens":3664,"seconds":0.1668943960685283},{"candidates":16,"kind":"choice","padded_tokens":3680,"seconds":0.1661626211134717},{"candidates":16,"kind":"choice","padded_tokens":3712,"seconds":0.16788391198497266},{"candidates":16,"kind":"choice","padded_tokens":3728,"seconds":0.1671058409847319},{"candidates":16,"kind":"choice","padded_tokens":3760,"seconds":0.16838663909584284},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.16834843903779984},{"candidates":16,"kind":"choice","padded_tokens":3792,"seconds":0.16796636604703963},{"candidates":16,"kind":"choice","padded_tokens":3808,"seconds":0.16825318103656173},{"candidates":16,"kind":"choice","padded_tokens":3856,"seconds":0.1687717279419303},{"candidates":16,"kind":"choice","padded_tokens":3872,"seconds":0.16872156399767846},{"candidates":16,"kind":"choice","padded_tokens":3920,"seconds":0.16927884402684867},{"candidates":16,"kind":"choice","padded_tokens":3936,"seconds":0.16991034196689725},{"candidates":16,"kind":"choice","padded_tokens":3968,"seconds":0.16920451598707587},{"candidates":16,"kind":"choice","padded_tokens":4016,"seconds":0.16952051501721144},{"candidates":16,"kind":"choice","padded_tokens":4048,"seconds":0.1702329539693892},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.16987828200217336},{"candidates":16,"kind":"choice","padded_tokens":4112,"seconds":0.18279627698939294},{"candidates":16,"kind":"choice","padded_tokens":4160,"seconds":0.1845842619659379},{"candidates":16,"kind":"choice","padded_tokens":4192,"seconds":0.184891666052863},{"candidates":16,"kind":"choice","padded_tokens":4224,"seconds":0.1847334400517866},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.18524981301743537},{"candidates":16,"kind":"choice","padded_tokens":4288,"seconds":0.1856669409899041},{"candidates":16,"kind":"choice","padded_tokens":4320,"seconds":0.18577225797344},{"candidates":16,"kind":"choice","padded_tokens":4352,"seconds":0.18645308795385063},{"candidates":16,"kind":"choice","padded_tokens":4400,"seconds":0.18608931999187917},{"candidates":16,"kind":"choice","padded_tokens":4448,"seconds":0.18642521603032947},{"candidates":16,"kind":"choice","padded_tokens":4496,"seconds":0.1866822149604559},{"candidates":16,"kind":"choice","padded_tokens":4512,"seconds":0.18767478596419096},{"candidates":16,"kind":"choice","padded_tokens":4544,"seconds":0.18719081301242113},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.18683933792635798},{"candidates":16,"kind":"choice","padded_tokens":4592,"seconds":0.18705026200041175},{"candidates":16,"kind":"choice","padded_tokens":4640,"seconds":0.1905876899836585},{"candidates":16,"kind":"choice","padded_tokens":4656,"seconds":0.19865341391414404},{"candidates":16,"kind":"choice","padded_tokens":4704,"seconds":0.19893757393583655},{"candidates":16,"kind":"choice","padded_tokens":4784,"seconds":0.1993960189865902},{"candidates":16,"kind":"choice","padded_tokens":4832,"seconds":0.1999599509872496},{"candidates":16,"kind":"choice","padded_tokens":4880,"seconds":0.20010253705549985},{"candidates":16,"kind":"choice","padded_tokens":4960,"seconds":0.20139910595025867},{"candidates":16,"kind":"choice","padded_tokens":5056,"seconds":0.20225607499014586},{"candidates":16,"kind":"choice","padded_tokens":5136,"seconds":0.20670562295708805},{"candidates":16,"kind":"choice","padded_tokens":5168,"seconds":0.21501543303020298},{"candidates":16,"kind":"choice","padded_tokens":5264,"seconds":0.21495922794565558},{"candidates":16,"kind":"choice","padded_tokens":5520,"seconds":0.21469804097432643},{"candidates":16,"kind":"choice","padded_tokens":5760,"seconds":0.21630394691601396},{"candidates":16,"kind":"choice","padded_tokens":6016,"seconds":0.21878538490273058},{"candidates":16,"kind":"choice","padded_tokens":6464,"seconds":0.23340869101230055},{"candidates":16,"kind":"choice","padded_tokens":6784,"seconds":0.24411257600877434},{"candidates":16,"kind":"choice","padded_tokens":7120,"seconds":0.24960719502996653},{"candidates":16,"kind":"choice","padded_tokens":7856,"seconds":0.27906672295648605},{"candidates":13,"kind":"choice","padded_tokens":7982,"seconds":0.25158854003529996},{"candidates":11,"kind":"choice","padded_tokens":8041,"seconds":0.25748498796019703},{"candidates":4,"kind":"choice","padded_tokens":4096,"seconds":0.12319140997715294}],"chat_template":false,"data_loading_seconds":0.07072584994602948,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-boolq-lm-eval-v2","finished":1792251891,"gpu":1,"id":"61590be79c31d6937e6462e6","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01","model_load_seconds":1.895052877953276,"model_revision":"2e72c696d7c5cad73dc58606b0751b911dc94495","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":139279760,"peak_gpu_bytes":1330423808,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"native-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":62.15915079403203,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"7f2a4b46461d9e7fe0763f55ef1a046476cd093351d877ad5515e16812e1bdca","score":0.5357798165137615,"scoring_seconds":57.99858222901821,"seconds":60.41159609297756,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"native-boolq-common-core-full","tf32":false,"truncated_candidates":2,"worker":"white-gpu1"}