{"acc_norm":0.6229357798165137,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.6024464831804281,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.26897880795877427},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11591158097144216},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11580983200110495},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11479233403224498},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11479327594861388},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11567537090741098},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11777297395747155},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11666760197840631},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.116338768042624},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11911463597789407},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11540309293195605},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.11596338904928416},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.11567172408103943},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.11529393994715065},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.11511303193401545},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1158202609512955},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.11574782605748624},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.11830263200681657},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.11638206907082349},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.11670924897771329},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.11739081109408289},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.1190495549235493},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.11930726608261466},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.11946586100384593},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.1191932630026713},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.11906261008698493},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.11966778803616762},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.12237623904366046},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.12055236799642444},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.11990265594795346},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.12032539490610361},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.12069134693592787},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.11960970005020499},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.11962149199098349},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.11951262899674475},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.11870515195187181},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.11882612796034664},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.11914302909281105},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.11927298200316727},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.11882991495076567},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.11934144899714738},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.11999917507637292},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.12153203506022692},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.12098118697758764},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.11899114598054439},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.12266469793394208},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.12261115806177258},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.12214854499325156},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.12259090994484723},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.12394225096795708},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.12210690695792437},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.12048235698603094},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.12300504301674664},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.12071738496888429},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.11970927496440709},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.11982774594798684},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.11983159405644983},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.11974895896855742},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.12079615297261626},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.11939038103446364},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.12032804905902594},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.11979638796765357},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.12072914792224765},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.12104877398815006},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.11980157799553126},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.11949569592252374},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.11976944794878364},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.11975055292714387},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.11999743292108178},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.11997038195841014},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.11996563407592475},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.11991402693092823},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.12008059804793447},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.11960004200227559},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.12060801195912063},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.12014700204599649},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.12118344497866929},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12176201399415731},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12032804999034852},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.11991437792312354},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12100438098423183},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.12094758497551084},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.12005864700768143},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.12023325206246227},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.12039071600884199},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.12005840695928782},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.12010272894985974},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.11914317903574556},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.11992230301257223},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.12187451391946524},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.12156083900481462},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.12136806908529252},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.12182849692180753},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.12104504695162177},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.12165203900076449},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.12115285894833505},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.1223958870396018},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.12131749500986189},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.1216883969027549},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.12138501100707799},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.12073366600088775},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.1209193030372262},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.11921488295774907},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.11948844301514328},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.11924214393366128},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.11906029493547976},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.12026417104061693},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.12204043299425393},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.11915616400074214},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.11944422102533281},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.11976149200927466},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.11962178302928805},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.11978996510151774},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.11906510300468653},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.11899578501470387},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.11787686799652874},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.11935940303374082},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.11898348201066256},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.11858093098271638},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.1186113270232454},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.11860869207885116},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.12021045992150903},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.1466239970177412},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.15529688505921513},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.12340300402138382},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.13046039803884923},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.1276260120794177},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.13612529507372528},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.11929595493711531},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.11957647802773863},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.1196776160504669},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.12001179996877909},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.11991349596064538},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.12152018304914236},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.12264156399760395},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.121736726956442},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.12005759496241808},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.12051373592112213},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.12059844296891242},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.12075790099333972},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.12006879597902298},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.12037441600114107},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.12043317500501871},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.12099287903402},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.12003517302218825},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.12014945689588785},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.12036971701309085},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.1198595860041678},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.11934518592897803},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.12016260100062937},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.11999977706000209},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.12121158710215241},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.12105699896346778},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.11997473100200295},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.12011076405178756},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.12010290997568518},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.1196959710214287},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.119430375052616},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.12029840401373804},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.12032844102941453},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.12112014705780894},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.11883170797955245},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.11955484689678997},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.12090329197235405},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.12130863894708455},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.11926447506994009},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.12016249098815024},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.11967491102404892},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.11892627505585551},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.12046609597746283},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.12027545203454792},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.11960742494557053},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.11905534600373358},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.11925005808006972},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.12390145496465266},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.12825472501572222},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.1272597989300266},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.12760866899043322},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.12673539004754275},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.12751415302045643},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.1275194330373779},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.12703974801115692},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.12744520406704396},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.12780502589885145},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.1275787230115384},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.12763668096158653},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.12802367296535522},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.12797122600022703},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.1277598509332165},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.12794891407247633},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.12848647590726614},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.1285603540018201},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.12844898598268628},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.12814012099988759},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.12828452000394464},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.1279907120624557},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.12824102898593992},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.12856305006425828},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.1286733050365001},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.12857926997821778},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.12837490800302476},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.12855726899579167},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.1290039619198069},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.12861850298941135},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.1291445749811828},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.128684226074256},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.12878325092606246},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.12862185994163156},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.1284784310264513},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.12937103700824082},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.12886932992842048},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.1291975729400292},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.1293932079570368},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.12862251105252653},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.12869928404688835},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.12916585395578295},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.12940395798068494},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.12921719904989004},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.12905788305215538},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.1296328540192917},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.12910273706074804},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.12955696403514594},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.1293477730359882},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.12989738897886127},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.1299538139719516},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.1294851889833808},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.12982169794850051},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.13000085100065917},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.12969233596231788},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.12943089904729277},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.1299460189184174},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.13021285692229867},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.13025391299743205},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.13048377202358097},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.13060195301659405},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.13006511109415442},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.13021015201229602},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.13028686598408967},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.13107423402834684},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.13089510996360332},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.13117863901425153},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.13118800497613847},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.13077043695375323},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.13092807109933347},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.1308903800090775},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.13104571995791048},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.13137275096960366},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.13120016804896295},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.1313912949990481},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.13160561490803957},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.13140568195376545},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.13124939997214824},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.1316432449966669},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.13176114403177053},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.1316543659195304},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.13194696104619652},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.13116846908815205},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.13399330794345587},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.13904965005349368},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.1395766530185938},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.1398583579575643},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.139697598060593},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.13971148408018053},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.1400914719561115},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.13979648204986006},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.13975554704666138},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.14023197500500828},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.14072479400783777},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.14049501600675285},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.1410360540030524},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.14116607699543238},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.14051198703236878},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.14031087199691683},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.14080918102990836},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.14114931598305702},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.14112867694348097},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.14087867201305926},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.14070367498788983},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.14061003003735095},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.14125896093901247},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.14119136496447027},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.14178979001007974},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.14131415297742933},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.14128678408451378},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.14623789605684578},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.1495041389716789},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.1497603259049356},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.14994850591756403},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.1498219519853592},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.15033163304906338},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.15026747295632958},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.14986715698614717},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.15041677199769765},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.15104664606042206},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.15098787692841142},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.15024140500463545},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.15063323604408652},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.15034664107952267},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.15126327110920101},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.15177893289364874},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.15134462306741625},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.15138087002560496},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.15148536500055343},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.1511831919196993},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.15159420704003423},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.15160651097539812},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.15212120104115456},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.15202448097988963},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.1520039530005306},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.1517414420377463},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.1519438410177827},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.1524819639744237},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.152252655941993},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.15282377100083977},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.15286153205670416},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.15267611597664654},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.1523452289402485},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.15286407701205462},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.1603271570056677},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.16064264602027833},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.1611546300118789},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.16065205296035856},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.16053922299761325},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.16062019299715757},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.16089106898289174},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.16117602097801864},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.16200770204886794},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.16192022792529315},{"candidates":16,"kind":"choice","padded_tokens":3232,"seconds":0.1620638360036537},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.16158181801438332},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.16218124504666775},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.16254277992993593},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.16322957002557814},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.16290838201530278},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.16249818704091012},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.1624349179910496},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.16188394092023373},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.16305316193029284},{"candidates":16,"kind":"choice","padded_tokens":3360,"seconds":0.16363166098017246},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.16412407893221825},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.16272317594848573},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.16367631405591965},{"candidates":16,"kind":"choice","padded_tokens":3424,"seconds":0.16363107901997864},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.16347123205196112},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.16375086305197328},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.16510009998455644},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.16331828595139086},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.1641945409355685},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.16492274799384177},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.16429040906950831},{"candidates":16,"kind":"choice","padded_tokens":3536,"seconds":0.16496957594063133},{"candidates":16,"kind":"choice","padded_tokens":3536,"seconds":0.16564302309416234},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.16502420790493488},{"candidates":16,"kind":"choice","padded_tokens":3568,"seconds":0.16483590693678707},{"candidates":16,"kind":"choice","padded_tokens":3600,"seconds":0.16424723900854588},{"candidates":16,"kind":"choice","padded_tokens":3616,"seconds":0.16627787693869323},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.16737449204083532},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.16795274103060365},{"candidates":16,"kind":"choice","padded_tokens":3664,"seconds":0.16759440198075026},{"candidates":16,"kind":"choice","padded_tokens":3680,"seconds":0.16678786801639944},{"candidates":16,"kind":"choice","padded_tokens":3712,"seconds":0.16695583099499345},{"candidates":16,"kind":"choice","padded_tokens":3728,"seconds":0.16748614003881812},{"candidates":16,"kind":"choice","padded_tokens":3760,"seconds":0.16811825905460864},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.16840478300582618},{"candidates":16,"kind":"choice","padded_tokens":3792,"seconds":0.16817987407557666},{"candidates":16,"kind":"choice","padded_tokens":3808,"seconds":0.16754331695847213},{"candidates":16,"kind":"choice","padded_tokens":3856,"seconds":0.168083013035357},{"candidates":16,"kind":"choice","padded_tokens":3872,"seconds":0.16856046405155212},{"candidates":16,"kind":"choice","padded_tokens":3920,"seconds":0.16898330301046371},{"candidates":16,"kind":"choice","padded_tokens":3936,"seconds":0.16969993896782398},{"candidates":16,"kind":"choice","padded_tokens":3968,"seconds":0.16937035496812314},{"candidates":16,"kind":"choice","padded_tokens":4016,"seconds":0.16977881593629718},{"candidates":16,"kind":"choice","padded_tokens":4048,"seconds":0.16978609992656857},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.17072582303080708},{"candidates":16,"kind":"choice","padded_tokens":4112,"seconds":0.18230393901467323},{"candidates":16,"kind":"choice","padded_tokens":4160,"seconds":0.1853349229786545},{"candidates":16,"kind":"choice","padded_tokens":4192,"seconds":0.18462986696977168},{"candidates":16,"kind":"choice","padded_tokens":4224,"seconds":0.18488123593851924},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.18604802200570703},{"candidates":16,"kind":"choice","padded_tokens":4288,"seconds":0.1855117620434612},{"candidates":16,"kind":"choice","padded_tokens":4320,"seconds":0.1868021689588204},{"candidates":16,"kind":"choice","padded_tokens":4352,"seconds":0.18623567302711308},{"candidates":16,"kind":"choice","padded_tokens":4400,"seconds":0.18577616498805583},{"candidates":16,"kind":"choice","padded_tokens":4448,"seconds":0.1863748619798571},{"candidates":16,"kind":"choice","padded_tokens":4496,"seconds":0.18695183796808124},{"candidates":16,"kind":"choice","padded_tokens":4512,"seconds":0.18762138695456088},{"candidates":16,"kind":"choice","padded_tokens":4544,"seconds":0.18732971302233636},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.18696081498637795},{"candidates":16,"kind":"choice","padded_tokens":4592,"seconds":0.1868771390290931},{"candidates":16,"kind":"choice","padded_tokens":4640,"seconds":0.19093855400569737},{"candidates":16,"kind":"choice","padded_tokens":4656,"seconds":0.1987621570006013},{"candidates":16,"kind":"choice","padded_tokens":4704,"seconds":0.19942328101024032},{"candidates":16,"kind":"choice","padded_tokens":4784,"seconds":0.1989437760785222},{"candidates":16,"kind":"choice","padded_tokens":4832,"seconds":0.20025184599217027},{"candidates":16,"kind":"choice","padded_tokens":4880,"seconds":0.20093835599254817},{"candidates":16,"kind":"choice","padded_tokens":4960,"seconds":0.20181540201883763},{"candidates":16,"kind":"choice","padded_tokens":5056,"seconds":0.20229761092923582},{"candidates":16,"kind":"choice","padded_tokens":5136,"seconds":0.2063735539559275},{"candidates":16,"kind":"choice","padded_tokens":5168,"seconds":0.2148181350203231},{"candidates":16,"kind":"choice","padded_tokens":5264,"seconds":0.21477833099197596},{"candidates":16,"kind":"choice","padded_tokens":5520,"seconds":0.2151807410409674},{"candidates":16,"kind":"choice","padded_tokens":5760,"seconds":0.2163406650070101},{"candidates":16,"kind":"choice","padded_tokens":6016,"seconds":0.21863815991673619},{"candidates":16,"kind":"choice","padded_tokens":6464,"seconds":0.23422753799241036},{"candidates":16,"kind":"choice","padded_tokens":6784,"seconds":0.2451736950315535},{"candidates":16,"kind":"choice","padded_tokens":7120,"seconds":0.24962341506034136},{"candidates":16,"kind":"choice","padded_tokens":7856,"seconds":0.27914251503534615},{"candidates":13,"kind":"choice","padded_tokens":7982,"seconds":0.2524562090402469},{"candidates":11,"kind":"choice","padded_tokens":8041,"seconds":0.25880543200764805},{"candidates":4,"kind":"choice","padded_tokens":4096,"seconds":0.12363590905442834}],"chat_template":false,"data_loading_seconds":0.0723532559350133,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-boolq-lm-eval-v2","finished":1792251969,"gpu":1,"id":"9f78e32c3a299272f2665b8f","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer139-1.01b","model_load_seconds":1.9082656940445304,"model_revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":139279760,"peak_gpu_bytes":1330423808,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"native-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":62.11166645004414,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"7f2a4b46461d9e7fe0763f55ef1a046476cd093351d877ad5515e16812e1bdca","score":0.6024464831804281,"scoring_seconds":57.99996667599771,"seconds":60.42869365995284,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"native-boolq-common-core-full","tf32":false,"truncated_candidates":2,"worker":"white-gpu1"}