{"acc_norm":0.6134556574923548,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.5587155963302752,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.27361856296192855},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13729107507970184},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13758634601254016},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.13885352993384004},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.13890635897405446},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13809488504193723},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13702767400536686},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.14186996698845178},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13940179196652025},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13764314202126116},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13656441890634596},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1423883039969951},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13751940103247762},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1383257150882855},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1379828560166061},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1375309230061248},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13647441193461418},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1379771949723363},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13739329599775374},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13794072694145143},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1381715179886669},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.14117254503071308},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.13899609597865492},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.13814795296639204},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.1378531539812684},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.13832904200535268},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.1387470809277147},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.1391700500389561},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.13975893903989345},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.14282186306081712},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.1383306139614433},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.1385704519925639},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.14174669701606035},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.14334003895055503},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.14058464905247092},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.14168621296994388},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1428512369748205},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.14277790999040008},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.14000328490510583},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.14019850897602737},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.13998566300142556},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.14043836598284543},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.14106663805432618},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.1401988900033757},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.14433250995352864},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.1402247379301116},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.13963078998494893},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.13991619204171002},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.14052677107974887},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.1392266759648919},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.14248756901361048},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.1402501140255481},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.14323822897858918},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.13993331405799836},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.1392387590603903},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.14005404897034168},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.14138584292959422},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.141813721973449},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.141659694025293},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.13961562106851488},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.143162747961469},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.1413613479817286},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.1395188410533592},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.13948979694396257},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.1394208479905501},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.1411112209316343},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.14148427709005773},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.14401668100617826},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.14054030599072576},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.1401544070104137},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.1408105610171333},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.14023662894032896},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.14007639104966074},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.1399350770516321},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.14023813197854906},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.1443632880691439},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.1416041209595278},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.1409962970064953},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.14082523807883263},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.14280036196578294},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.13953809696249664},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.14395351300481707},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.14692225097678602},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.1421908149495721},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.14215069997590035},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.14088000997435302},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.14196005603298545},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.142183881951496},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.1439914939692244},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.14707203104626387},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.1442231460241601},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.14296440896578133},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.14181913202628493},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.14505810395348817},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.14189090603031218},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.14718175504822284},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.14491792197804898},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.1434342039283365},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.14361279795411974},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.14098082797136158},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.14205313799902797},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.14146925904788077},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.14655408496037126},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.1420928230509162},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.14191128395032138},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.14245813305024058},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.14430389693006873},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.14549275499302894},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.14284183003474027},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.1489565569208935},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.14289606106467545},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.14427969208918512},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.14360694598872215},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.14304757304489613},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.1419255300424993},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.14634932298213243},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.1435226999456063},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.14308987301774323},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.14290810306556523},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.14246134995482862},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.14258948899805546},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.14906487800180912},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.14517114497721195},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.1430490359198302},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.14360804890748113},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.14138094405643642},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.14550763298757374},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.1427928180200979},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.1477033180417493},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.14450225699692965},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.14517482102382928},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.1453681819839403},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.1440969310933724},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.14220120396930724},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.14902630599681288},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.14506641903426498},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.1450830700341612},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.14162851602304727},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.14177621202543378},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.14207927708048373},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.1481366860680282},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.14544094796292484},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.14302990096621215},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.14251627295743674},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.1435433980077505},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.14344898203853518},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.1435155860381201},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.14701120695099235},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.14239355304744095},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.14384419994894415},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.14536774100270122},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.1425651329336688},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.14455548697151244},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.14992365997750312},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.14641272090375423},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.14542250393424183},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.14740945992525667},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.14319371606688946},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.1432859980268404},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.14911695604678243},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.14418670895975083},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.14452060195617378},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.14412913005799055},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.1444247820181772},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.14517263800371438},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.14970174699556082},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.14853295602370054},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.14749203505925834},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.14509468199685216},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.14478581596631557},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.14882766595110297},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.14666362898424268},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.14308079599868506},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.16040731698740274},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.17167181998956949},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.16695903800427914},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.16716863901820034},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.17095216794405133},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.16561428003478795},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.167996374075301},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.17157285509165376},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.1663175809662789},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.16747507103718817},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.17254047002643347},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.16859084204770625},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.16865792800672352},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.17579814698547125},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.16735349397640675},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.17333016300108284},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.16735017800237983},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.16715568501967937},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.1739598469575867},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.16735042806249112},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.16772144997958094},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.1727577350102365},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.16892835195176303},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.16900952404830605},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.17561375303193927},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.16887935099657625},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.17324168793857098},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.16876940499059856},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.16885413392446935},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.17607898102141917},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.16928358504083008},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.1685266220010817},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.17233542702160776},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.16924709698650986},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.17353251005988568},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.16821217606775463},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.16831228299997747},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.17410134198144078},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.16812414210289717},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.17107960593421012},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.17448614002205431},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.17097089102026075},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.17473270907066762},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.16947789606638253},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.1693437669891864},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.17526151600759476},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.16920643101911992},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.1702670290833339},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.17644335993099958},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.17023416806478053},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.17510095692705363},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.17261737305670977},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.17100231104996055},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.17611654999200255},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.17005696799606085},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.17520499101374298},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.17224433808587492},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.17122631892561913},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.17718766897451133},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.16986138292122632},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.1706346650607884},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.17734207690227777},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.17051495099440217},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.17716526694130152},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.170806625043042},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.1719313139328733},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.17706212494522333},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.17128943605348468},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.17593428096733987},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.17323040694463998},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.17128753301221877},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.17669779597781599},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.17230585194192827},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.178358773002401},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.1733435089699924},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.17277122102677822},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.17906013200990856},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.1726619160035625},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.17748194793239236},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.17165815399494022},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.17131686792708933},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.17869641399011016},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.17122430494055152},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.1807093679672107},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.177807223983109},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.18387358204927295},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.17753436602652073},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.17710146808531135},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.1826475530397147},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.17692093108780682},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.1825718629406765},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.17875690595246851},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.18236498709302396},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.1776438399683684},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.1839823640184477},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.1776092250365764},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.1784641200210899},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.1841132480185479},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.1781711330404505},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.18396085407584906},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.17912049393635243},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.18475497502367944},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.1779231319669634},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.18597498000599444},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.17872465692926198},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.18578291300218552},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.17952133296057582},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.1785988110350445},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.1836176629876718},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.180159842944704},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.18573449295945466},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.17884954903274775},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.18379942304454744},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.1796147869899869},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.18400802207179368},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.17943770601414144},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.18609777104575187},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.18100044107995927},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.18497033696621656},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.17986010410822928},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.18032087304163724},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.18629925604909658},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.18078091205097735},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.18683757004328072},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.18211936892475933},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.18672706396318972},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.18134052597451955},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.1884489560034126},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.18013428605627269},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.187900573015213},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.18161280394997448},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.18831732997205108},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.18166834698058665},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.18648293800652027},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.18101814505644143},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.1868383320979774},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.18236255296505988},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.18710649199783802},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.18091965094208717},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.1870898810448125},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.18237083789426833},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.18871620495337993},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.18143045296892524},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.19501612696330994},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.1992781269364059},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.2064841389656067},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.20042697107419372},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.20572928106412292},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.20462450990453362},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.20049164001829922},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.20541399298235774},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.20617588399909437},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.20170043490361422},{"candidates":16,"kind":"choice","padded_tokens":3232,"seconds":0.20559920801315457},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.20859635900706053},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.20232491998467594},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.20656838605646044},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.20136802503839135},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.20688298298045993},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.2087923149811104},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.20148147700820118},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.20752016210462898},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.2065061000175774},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.20273965399246663},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.20964003494009376},{"candidates":16,"kind":"choice","padded_tokens":3392,"seconds":0.21060738095548004},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.20435667096171528},{"candidates":16,"kind":"choice","padded_tokens":3424,"seconds":0.20784890500362962},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.2091574459336698},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.20334402099251747},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.20951777801383287},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.20918305299710482},{"candidates":16,"kind":"choice","padded_tokens":3504,"seconds":0.204014883027412},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.21168642304837704},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.21135979401879013},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.20440744596999139},{"candidates":16,"kind":"choice","padded_tokens":3568,"seconds":0.21187005599495023},{"candidates":16,"kind":"choice","padded_tokens":3584,"seconds":0.21059316396713257},{"candidates":16,"kind":"choice","padded_tokens":3584,"seconds":0.20416859991382807},{"candidates":16,"kind":"choice","padded_tokens":3616,"seconds":0.2137118310201913},{"candidates":16,"kind":"choice","padded_tokens":3632,"seconds":0.21121437300462276},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.20687449700199068},{"candidates":16,"kind":"choice","padded_tokens":3664,"seconds":0.20866592810489237},{"candidates":16,"kind":"choice","padded_tokens":3680,"seconds":0.21465631399769336},{"candidates":16,"kind":"choice","padded_tokens":3696,"seconds":0.21276141901034862},{"candidates":16,"kind":"choice","padded_tokens":3728,"seconds":0.21674014092423022},{"candidates":16,"kind":"choice","padded_tokens":3744,"seconds":0.22631598403677344},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.22503986302763224},{"candidates":16,"kind":"choice","padded_tokens":3792,"seconds":0.2270170219708234},{"candidates":16,"kind":"choice","padded_tokens":3808,"seconds":0.226719886995852},{"candidates":16,"kind":"choice","padded_tokens":3840,"seconds":0.22061611094977707},{"candidates":16,"kind":"choice","padded_tokens":3856,"seconds":0.22488553600851446},{"candidates":16,"kind":"choice","padded_tokens":3872,"seconds":0.22575815301388502},{"candidates":16,"kind":"choice","padded_tokens":3888,"seconds":0.22756586503237486},{"candidates":16,"kind":"choice","padded_tokens":3936,"seconds":0.22892311704345047},{"candidates":16,"kind":"choice","padded_tokens":3968,"seconds":0.22119081299751997},{"candidates":16,"kind":"choice","padded_tokens":4000,"seconds":0.2290057010250166},{"candidates":16,"kind":"choice","padded_tokens":4032,"seconds":0.22862743504811078},{"candidates":16,"kind":"choice","padded_tokens":4080,"seconds":0.22835543809924275},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.22914400906302035},{"candidates":16,"kind":"choice","padded_tokens":4144,"seconds":0.23472781502641737},{"candidates":16,"kind":"choice","padded_tokens":4160,"seconds":0.23567181592807174},{"candidates":16,"kind":"choice","padded_tokens":4208,"seconds":0.23194115702062845},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.23641384998336434},{"candidates":16,"kind":"choice","padded_tokens":4288,"seconds":0.23799276596400887},{"candidates":16,"kind":"choice","padded_tokens":4320,"seconds":0.23830068006645888},{"candidates":16,"kind":"choice","padded_tokens":4352,"seconds":0.2375239209504798},{"candidates":16,"kind":"choice","padded_tokens":4400,"seconds":0.23733029002323747},{"candidates":16,"kind":"choice","padded_tokens":4416,"seconds":0.23822574003133923},{"candidates":16,"kind":"choice","padded_tokens":4432,"seconds":0.23764086002483964},{"candidates":16,"kind":"choice","padded_tokens":4464,"seconds":0.24008127208799124},{"candidates":16,"kind":"choice","padded_tokens":4496,"seconds":0.24055607803165913},{"candidates":16,"kind":"choice","padded_tokens":4528,"seconds":0.24027030495926738},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.23300834803376347},{"candidates":16,"kind":"choice","padded_tokens":4608,"seconds":0.24127402692101896},{"candidates":16,"kind":"choice","padded_tokens":4624,"seconds":0.25939723395276815},{"candidates":16,"kind":"choice","padded_tokens":4688,"seconds":0.2725853759329766},{"candidates":16,"kind":"choice","padded_tokens":4736,"seconds":0.26590672705788165},{"candidates":16,"kind":"choice","padded_tokens":4784,"seconds":0.26929675100836903},{"candidates":16,"kind":"choice","padded_tokens":4864,"seconds":0.26710603409446776},{"candidates":16,"kind":"choice","padded_tokens":4912,"seconds":0.26938784099183977},{"candidates":16,"kind":"choice","padded_tokens":4960,"seconds":0.26966252306010574},{"candidates":16,"kind":"choice","padded_tokens":5072,"seconds":0.2772157129365951},{"candidates":16,"kind":"choice","padded_tokens":5120,"seconds":0.2703064839588478},{"candidates":16,"kind":"choice","padded_tokens":5200,"seconds":0.2803071499802172},{"candidates":16,"kind":"choice","padded_tokens":5296,"seconds":0.2810115939937532},{"candidates":16,"kind":"choice","padded_tokens":5440,"seconds":0.2873829889576882},{"candidates":16,"kind":"choice","padded_tokens":5648,"seconds":0.2834273709449917},{"candidates":16,"kind":"choice","padded_tokens":5856,"seconds":0.2909969110041857},{"candidates":16,"kind":"choice","padded_tokens":6080,"seconds":0.2940480320248753},{"candidates":16,"kind":"choice","padded_tokens":6416,"seconds":0.3053450259612873},{"candidates":16,"kind":"choice","padded_tokens":6704,"seconds":0.31936047098133713},{"candidates":16,"kind":"choice","padded_tokens":7216,"seconds":0.3236388419754803},{"candidates":16,"kind":"choice","padded_tokens":8032,"seconds":0.34805523802060634},{"candidates":13,"kind":"choice","padded_tokens":8008,"seconds":0.34339890198316425},{"candidates":11,"kind":"choice","padded_tokens":8151,"seconds":0.32870275794994086},{"candidates":4,"kind":"choice","padded_tokens":4096,"seconds":0.1609701270936057}],"chat_template":false,"data_loading_seconds":0.051617266959510744,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-boolq-lm-eval-v2","finished":1792250681,"gpu":0,"id":"81dce281154d893d5a981222","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149-ARC","model_load_seconds":1.9557320959866047,"model_revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":149333081,"peak_gpu_bytes":1383601152,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"native-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":76.69601485901512,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"7f2a4b46461d9e7fe0763f55ef1a046476cd093351d877ad5515e16812e1bdca","score":0.5587155963302752,"scoring_seconds":72.5769835159881,"seconds":75.03700257791206,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"native-boolq-common-core-full","tf32":false,"truncated_candidates":2,"worker":"white-gpu0"}