{"acc_norm":0.6168195718654435,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.5758409785932722,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.27286690403707325},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13805406994652003},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13905635999981314},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1370198610238731},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13656916108448058},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13698738091625273},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1394475600682199},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1378947029588744},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13530147494748235},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13747177296318114},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1384695149026811},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.14140372804831713},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.14133895793929696},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13967846997547895},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13782955089118332},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.14094481302890927},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13899346196558326},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1390248810639605},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13810721901245415},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13887945003807545},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1404151349561289},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.14054448599927127},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.13988983409944922},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.13632102706469595},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.1413500279886648},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.13717019103933126},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.1386773819103837},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.1381079910788685},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.14342455693986267},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.13799732399638742},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.1418025519233197},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.13763779401779175},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13918837602250278},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13773645798210055},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13815187301952392},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.13884823094122112},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.14004206901881844},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.13783066300675273},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.13831149996258318},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.13696876598987728},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.13810971402563155},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.1376963930670172},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.137962729902938},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.13929516496136785},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.13889765401836485},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.13925522100180387},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.137309912010096},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.13697223097551614},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.1381273870356381},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.1376551459543407},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.13886767800431699},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.1408709839452058},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.14070415392052382},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.14060947694815695},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.13759460300207138},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.13758566591423005},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.1377047080313787},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.1383648500777781},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.13850438001099974},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.14518270699772984},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.14101936190854758},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.14348724391311407},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.14019607508089393},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.1431257009971887},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.1414970619371161},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.14158171007875353},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.14165860402863473},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.14117393898777664},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.1381815979257226},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.13906427402980626},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.1388734980719164},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.13911489897873253},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.13878293905872852},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.13994176092091948},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.14126182405743748},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.13951901299878955},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.1376078069442883},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.1385650630109012},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.13806638307869434},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.1383205170277506},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.13836370804347098},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.13846871408168226},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.13920560793485492},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.14107345195952803},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.1420026250416413},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.13931219605728984},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.13921645795926452},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.1382331739878282},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.13934471807442605},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.1389927810523659},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.13985043100547045},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.1381804159609601},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.13878957193810493},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.13795952405780554},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.1388143280055374},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.13812155602499843},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.13942324393428862},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.13880594307556748},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.13967103592585772},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.1379160829819739},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.13957762194331735},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.13867136207409203},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.13918549101799726},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.14540591393597424},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.1484551620669663},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.14843729906715453},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.14609203301370144},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.1403462759917602},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.14514336409047246},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.14198980201035738},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.1428255110513419},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.13871713704429567},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.14406851900275797},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.148231465020217},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.13796241907402873},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.1389099870575592},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.13841875991784036},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.1404536860063672},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.13787470501847565},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.13908521295525134},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.14061882405076176},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.13963884592521936},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.13792373694013804},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.13929786998778582},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.13873039092868567},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.13950908405240625},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.13893625605851412},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.13924957008566707},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.1381658990867436},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.1393205119529739},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.1387975870165974},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.13940360804554075},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.1391572670545429},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.1403123129857704},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.14067893591709435},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.14035983092617244},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.14071456296369433},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.1388238159706816},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.13872320798691362},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.13930209807585925},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.13866822503041476},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.1393973050871864},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.1387740330537781},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.13882899505551904},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.13894757709931582},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.1385930849937722},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.14136696909554303},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.13869828102178872},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.13821725407615304},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.1380922410171479},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.1384189809905365},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.14073966909199953},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.1398689349880442},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.13859267497900873},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.13989618595223874},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.13765905296895653},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.13997205800842494},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.13867163204122335},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.14154177601449192},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.14093869109638035},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.14103524095844477},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.14120908500626683},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.13987471594009548},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.14030849502887577},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.14150309399701655},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.14030356600414962},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.14147975004743785},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.14317729708272964},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.14086585491895676},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.13980243098922074},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.13945434195920825},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.14005124603863806},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.1400122829945758},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.15061331703327596},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.15529191493988037},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.15601400192826986},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.15625760599505156},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.15488294197712094},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.15499798604287207},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.15501704206690192},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.1557094140443951},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.1563023900380358},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.1567946180002764},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.1563544060336426},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.1559629470575601},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.15573555300943553},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.15590514906216413},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.15655226504895836},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.15733760106377304},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.15666617802344263},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.15619657200295478},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.15592039690818638},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.15640067297499627},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.1567585909506306},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.15739204199053347},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.1575981869827956},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.15679006895516068},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.15633817703928798},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.1570183449657634},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.15742805891204625},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.157379318960011},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.157658149022609},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.1574855459621176},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.15692067297641188},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.15724135108757764},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.1569554980378598},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.1575759060215205},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.15783816401381046},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.15768931701313704},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.15731796401087195},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.15749498398508877},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.1577867679297924},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.15785804204642773},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.15814346401020885},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.15818797703832388},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.15776902600191534},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.15776492794975638},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.1573122339323163},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.15837898303288966},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.15871446894016117},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.15868562506511807},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.1587172340368852},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.15884295804426074},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.15884098503738642},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.15896915493067354},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.15960011107381433},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.1592886900762096},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.15940748108550906},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.15971498494036496},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.1594152559991926},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.15915043302811682},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.1598249910166487},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.159339425037615},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.1598810349823907},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.1599973919801414},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.15948026801925153},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.15922238701023161},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.1604281560285017},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.1596573879942298},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.16060527600347996},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.16107844794169068},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.1601202810415998},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.15974573302082717},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.16053208010271192},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.15986345207784325},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.16148160095326602},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.1611032549990341},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.16049779509194195},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.15996780607383698},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.16074755194131285},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.16045551607385278},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.1611231720307842},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.1612266949377954},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.16024752892553806},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.16013716300949454},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.16043595096562058},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.16398656310047954},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.16577623202465475},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.1657910490175709},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.16547001106664538},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.1653781490167603},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.16610903304535896},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.16575546306557953},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.16620977001730353},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.16633651708252728},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.16615212301257998},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.16611807909794152},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.1665014548925683},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.1667284379946068},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.16730591503437608},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.16603313095401973},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.16616210096981376},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.1665556550724432},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.16693433199543506},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.16642839892301708},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.1671279629226774},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.1664342189906165},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.16742582898586988},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.16701520199421793},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.16726516908966005},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.16776290792040527},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.16720288305077702},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.1668111919425428},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.16712588910013437},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.16810741100925952},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.16758244100492448},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.16776372899767011},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.1672857680823654},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.16739572305232286},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.16820669500157237},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.16774802992586046},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.1685153410071507},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.16807234403677285},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.16750974603928626},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.16888113296590745},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.16867638193070889},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.16844016092363745},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.1689169000601396},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.16792206396348774},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.16879292903468013},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.1687183789908886},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.16877769108396024},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.16898404504172504},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.1696070278994739},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.16834578406997025},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.16935752204153687},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.1692662819987163},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.1698008090024814},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.16990975197404623},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.1698536769254133},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.16870291193481535},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.16968234803061932},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.16951657901518047},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.17013630492147058},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.17021623498294502},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.1696691329125315},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.17766805505380034},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.18641951901372522},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.18747756199445575},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.18766377004794776},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.18787948205135763},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.18791984696872532},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.187661895994097},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.1883540980052203},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.18833549309056252},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.18830781104043126},{"candidates":16,"kind":"choice","padded_tokens":3232,"seconds":0.18950652703642845},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.18927921296563},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.18880480795633048},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.1887712850002572},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.18915584299247712},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.18895644100848585},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.18961006007157266},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.18960770592093468},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.18921357102226466},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.18920963304117322},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.1896564660128206},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.1905220909975469},{"candidates":16,"kind":"choice","padded_tokens":3392,"seconds":0.19045349396765232},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.1910847710678354},{"candidates":16,"kind":"choice","padded_tokens":3424,"seconds":0.1909066189546138},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.19042792601976544},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.19162588089238852},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.1907705949852243},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.19224027602467686},{"candidates":16,"kind":"choice","padded_tokens":3504,"seconds":0.1925691800424829},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.19134262204170227},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.1927758859237656},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.19300580502022058},{"candidates":16,"kind":"choice","padded_tokens":3568,"seconds":0.19309994007926434},{"candidates":16,"kind":"choice","padded_tokens":3584,"seconds":0.19320402503944933},{"candidates":16,"kind":"choice","padded_tokens":3584,"seconds":0.19160237698815763},{"candidates":16,"kind":"choice","padded_tokens":3616,"seconds":0.19382244802545756},{"candidates":16,"kind":"choice","padded_tokens":3632,"seconds":0.19498632894828916},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.19513570796698332},{"candidates":16,"kind":"choice","padded_tokens":3664,"seconds":0.19526431697886437},{"candidates":16,"kind":"choice","padded_tokens":3680,"seconds":0.1940324599854648},{"candidates":16,"kind":"choice","padded_tokens":3696,"seconds":0.19449314894154668},{"candidates":16,"kind":"choice","padded_tokens":3728,"seconds":0.2053674339549616},{"candidates":16,"kind":"choice","padded_tokens":3744,"seconds":0.20759829506278038},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.20804559893440455},{"candidates":16,"kind":"choice","padded_tokens":3792,"seconds":0.2070492709754035},{"candidates":16,"kind":"choice","padded_tokens":3808,"seconds":0.2078228039899841},{"candidates":16,"kind":"choice","padded_tokens":3840,"seconds":0.20858175901230425},{"candidates":16,"kind":"choice","padded_tokens":3856,"seconds":0.2082156860269606},{"candidates":16,"kind":"choice","padded_tokens":3872,"seconds":0.20920698600821197},{"candidates":16,"kind":"choice","padded_tokens":3888,"seconds":0.2084185240091756},{"candidates":16,"kind":"choice","padded_tokens":3936,"seconds":0.2092275540344417},{"candidates":16,"kind":"choice","padded_tokens":3968,"seconds":0.20964848902076483},{"candidates":16,"kind":"choice","padded_tokens":4000,"seconds":0.21007793000899255},{"candidates":16,"kind":"choice","padded_tokens":4032,"seconds":0.21014737896621227},{"candidates":16,"kind":"choice","padded_tokens":4080,"seconds":0.21024966100230813},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.2103398279286921},{"candidates":16,"kind":"choice","padded_tokens":4144,"seconds":0.2158458400517702},{"candidates":16,"kind":"choice","padded_tokens":4160,"seconds":0.2167639520484954},{"candidates":16,"kind":"choice","padded_tokens":4208,"seconds":0.21686551196034998},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.21671775600407273},{"candidates":16,"kind":"choice","padded_tokens":4288,"seconds":0.2175222160294652},{"candidates":16,"kind":"choice","padded_tokens":4320,"seconds":0.21840489306487143},{"candidates":16,"kind":"choice","padded_tokens":4352,"seconds":0.2186510319588706},{"candidates":16,"kind":"choice","padded_tokens":4400,"seconds":0.21882993506733328},{"candidates":16,"kind":"choice","padded_tokens":4416,"seconds":0.21856730605941266},{"candidates":16,"kind":"choice","padded_tokens":4432,"seconds":0.21948504797182977},{"candidates":16,"kind":"choice","padded_tokens":4464,"seconds":0.22057750495150685},{"candidates":16,"kind":"choice","padded_tokens":4496,"seconds":0.22061101801227778},{"candidates":16,"kind":"choice","padded_tokens":4528,"seconds":0.2195854049641639},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.22030649008229375},{"candidates":16,"kind":"choice","padded_tokens":4608,"seconds":0.22022205195389688},{"candidates":16,"kind":"choice","padded_tokens":4624,"seconds":0.23972939990926534},{"candidates":16,"kind":"choice","padded_tokens":4688,"seconds":0.24505501496605575},{"candidates":16,"kind":"choice","padded_tokens":4736,"seconds":0.24657225504051894},{"candidates":16,"kind":"choice","padded_tokens":4784,"seconds":0.24764082801993936},{"candidates":16,"kind":"choice","padded_tokens":4864,"seconds":0.24867497803643346},{"candidates":16,"kind":"choice","padded_tokens":4912,"seconds":0.24765090690925717},{"candidates":16,"kind":"choice","padded_tokens":4960,"seconds":0.24822696205228567},{"candidates":16,"kind":"choice","padded_tokens":5072,"seconds":0.25045196199789643},{"candidates":16,"kind":"choice","padded_tokens":5120,"seconds":0.2510143700055778},{"candidates":16,"kind":"choice","padded_tokens":5200,"seconds":0.25649067701306194},{"candidates":16,"kind":"choice","padded_tokens":5296,"seconds":0.25706541899126023},{"candidates":16,"kind":"choice","padded_tokens":5440,"seconds":0.25985899893566966},{"candidates":16,"kind":"choice","padded_tokens":5648,"seconds":0.2612898879451677},{"candidates":16,"kind":"choice","padded_tokens":5856,"seconds":0.2657154439948499},{"candidates":16,"kind":"choice","padded_tokens":6080,"seconds":0.2668957650894299},{"candidates":16,"kind":"choice","padded_tokens":6416,"seconds":0.2838370370445773},{"candidates":16,"kind":"choice","padded_tokens":6704,"seconds":0.29099165205843747},{"candidates":16,"kind":"choice","padded_tokens":7216,"seconds":0.2958570789778605},{"candidates":16,"kind":"choice","padded_tokens":8032,"seconds":0.31824541091918945},{"candidates":13,"kind":"choice","padded_tokens":8008,"seconds":0.3110643959371373},{"candidates":11,"kind":"choice","padded_tokens":8151,"seconds":0.3034493209561333},{"candidates":4,"kind":"choice","padded_tokens":4096,"seconds":0.14248017594218254}],"chat_template":false,"data_loading_seconds":0.07283940096385777,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-boolq-lm-eval-v2","finished":1792252954,"gpu":1,"id":"e8e7f16c555401bee6232c40","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","model_load_seconds":1.9467773929936811,"model_revision":"7d11e8fd63764baf00c276f586b2aa776648ed33","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":149333081,"peak_gpu_bytes":1383601152,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"native-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":82.77114707801957,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"7f2a4b46461d9e7fe0763f55ef1a046476cd093351d877ad5515e16812e1bdca","score":0.5758409785932722,"scoring_seconds":68.3455393849872,"seconds":70.8223336879164,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"native-boolq-common-core-full","tf32":false,"truncated_candidates":2,"worker":"white-gpu1"}