{"acc_norm":0.6220183486238532,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.6253822629969419,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.3029904409777373},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09733612299896777},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09510278794914484},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10785673302598298},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10975407005753368},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11249988409690559},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11438678205013275},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.131188364000991},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12347120698541403},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.12300631694961339},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1218799710040912},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12041640491224825},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.12575737002771348},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12871522002387792},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.12705366406589746},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13850201794411987},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.14171608700416982},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.14214340702164918},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13933048699982464},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.14816453296225518},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13954914605710655},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13928863895125687},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.16035371297039092},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.16069352894555777},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1690534990048036},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.15888767200522125},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.1590110119432211},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.16915007797069848},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.1623395460192114},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.17384271207265556},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.18056967901065946},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.17449070396833122},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1818894159514457},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.1753508320543915},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.18148954003117979},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.17313760390970856},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.17498491797596216},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.17970615497324616},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.17900773894507438},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.18710106099024415},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.17848098510876298},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.18798479298129678},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.17521643091458827},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.1880138369742781},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.1795586789958179},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.1894263280555606},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.17934455908834934},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.17585938307456672},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.18633849499747157},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.17817730701062828},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.1873563980916515},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.1824448450934142},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.18769236700609326},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.1786130720283836},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.18872088904026896},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.18060723901726305},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.2074877959676087},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.1990709099918604},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.2046503119636327},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.20787665399257094},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.1974555200431496},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.2078079949133098},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.21214874100405723},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.20017392199952155},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.2051870150025934},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.21502783405594528},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.2025425899773836},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.21181574801448733},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.21532478893641382},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.20268670993391424},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.21597019606269896},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.21498076594434679},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.207510347943753},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.21263015107251704},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.21318271290510893},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.21462155401241034},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.20412185392342508},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.21537622495088726},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.21630308800376952},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.20599914295598865},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.21413762890733778},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.21902735100593418},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.21506615600083023},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.2066203240538016},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.21773759997449815},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.21832210291177034},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.21209503000136465},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.2142835530685261},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.21917011810000986},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.21830055199097842},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.21060338197275996},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.21782096498645842},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.21918206091504544},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.21947113994974643},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.21800825593527406},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.22660918603651226},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.22401861299294978},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.2291626000078395},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.22634277900215238},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.22029945999383926},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.22842936997767538},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.22724284196738154},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.22737841494381428},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.2320370340021327},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.21964007697533816},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.23215013602748513},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.23052716196980327},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.23185058601666242},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.23641121305990964},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.23431316995993257},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.233736532041803},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.23192601697519422},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.22750211704988033},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.23256583500187844},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.2365255249897018},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.23679288697894663},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.2381249669706449},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.23367296298965812},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.2357047520345077},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.2370778990443796},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.2285751219606027},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.2353400900028646},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.23786595300771296},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.2395559020806104},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.24011420598253608},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.2370924260467291},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.2426332060713321},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.2378190050367266},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.24021451291628182},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.23822144698351622},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.2402478150324896},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.2329759799176827},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.2409544560359791},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.23776044498663396},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.2408800380071625},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.24346202495507896},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.2415960270445794},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.2507169599412009},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.25150544405914843},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.25581349804997444},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.25407006905879825},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.2534985200036317},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.2529827749822289},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.2565444048959762},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.2543111089617014},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.2523940149694681},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.25471901206765324},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.25430673104710877},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.2607674599858001},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.25371147808618844},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.2558672280283645},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.2562456759624183},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.25820200401358306},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.25669165002182126},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.2595713440096006},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.25558988004922867},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.25796936894766986},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.2597965650493279},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.26049416908062994},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.25959937600418925},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.26000009500421584},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.26033074397128075},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.2684904399793595},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.2731873909942806},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.2702815079828724},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.27399097301531583},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.2736753929639235},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.28377023700159043},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.27426776103675365},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.2722871780861169},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.2756479009985924},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.2750080549158156},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.2860094530042261},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.2743434220319614},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.2757562840124592},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.2788495069835335},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.2903880199883133},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.2772928570630029},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.27820054395124316},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.28135024197399616},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.291468619951047},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.27805400895886123},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.28152280498761684},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.29378204594831914},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.2847481149947271},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.2794868180062622},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.3195915099931881},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.3084702070336789},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.31685501302126795},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.3084450389724225},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.32094961893744767},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.30854906397871673},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.32070682500489056},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.31026387005113065},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.32117878703866154},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.3114891209406778},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.3233028189279139},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.31595855695195496},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.3220198400085792},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.3260364989982918},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.31409317010547966},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.3270651630591601},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.31491438508965075},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.324359885067679},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.3177030789665878},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.324367037974298},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.32650435401592404},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.31654830905608833},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.32961212401278317},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.3228717620950192},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.32984768401365727},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.3322667869506404},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.32156554993707687},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.3305346790002659},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.33102906309068203},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.3236489549744874},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.33487338095437735},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.3222813690081239},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.3327931209933013},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.33305307792034},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.3209428449627012},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.3308621420292184},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.33503073605243117},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.3223102829651907},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.33398110303096473},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.3321433160454035},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.32413735799491405},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.33246918593067676},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.32570212392602116},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.34606226009782404},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.34559765097219497},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.3495885129086673},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.3456079810857773},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.33756001398433},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.34652501507662237},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.3559714370639995},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.34651053801644593},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.3399928220314905},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.35175375104881823},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.3483773289481178},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.34502889798022807},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.336945304996334},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.34703819500282407},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.3480395770166069},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.3486693340819329},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.3482267069630325},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.34152709098998457},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.3484118829946965},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.3541951959487051},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.35935199703089893},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.3571865879930556},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.35335159895475954},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.35085317690391093},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.35518980503547937},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.35724459704943},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.3586529999738559},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.36049769888632},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.35971866291947663},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.3571016190107912},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.35487363394349813},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.3624949640361592},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.36261779302731156},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.36158880894072354},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.3633185130311176},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.36670743802096695},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.366470746928826},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.3739646770991385},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.3795427860459313},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.37831828696653247},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.37792408000677824},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.3798117090482265},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.3773057840298861},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.3814848680049181},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.37932839593850076},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.3785921890521422},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.38903438299894333},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.3869594419375062},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.3974826270714402},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.38760222506243736},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.38892098993528634},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.39114396600052714},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.3871693959226832},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.39174820703919977},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.39696767495479435},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.40007784904446453},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.3967069269856438},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.3923037340864539},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.3961438040714711},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.3976865989388898},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.3979884930886328},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.40434262400958687},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.39815156802069396},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.3953556308988482},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.40232303692027926},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.4036031709983945},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.4131254240637645},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.40026854595635086},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.40208404103759676},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.40712568594608456},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.4156795289600268},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.40533933707047254},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.40879428607877344},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.41746141109615564},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.404791021021083},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.40912054595537484},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.4100356469862163},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.413835791987367},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.4088110669981688},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.4123389740707353},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.42403464997187257},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.41090860904660076},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.41553667199332267},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.42538904293905944},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.4140990939922631},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.415480897994712},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.4235736581031233},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.42292997508775443},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.4344944959739223},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.4224306410178542},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.4258116730488837},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.4316200220491737},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.41958494996652007},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.4306693549733609},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.41903801797889173},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.4329871879890561},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.42097546998411417},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.45820138906128705},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.4481722479686141},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.45753795898053795},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.45001463301014155},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.4588101380504668},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.46204374404624104},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.4542675439734012},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.46081050706561655},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.4643553339410573},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.48157919698860496},{"candidates":16,"kind":"choice","padded_tokens":3232,"seconds":0.4756786050274968},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.4705288769910112},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.4819371569901705},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.48902146192267537},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.493529130006209},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.4897881250362843},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.49706326704472303},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.4937280809972435},{"candidates":16,"kind":"choice","padded_tokens":3360,"seconds":0.5000255650375038},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.4981371039757505},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.5018605960067362},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.5006730859167874},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.5026469359872863},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.5026764809153974},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.5074611469171941},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.5076369750313461},{"candidates":16,"kind":"choice","padded_tokens":3504,"seconds":0.5127896399935707},{"candidates":16,"kind":"choice","padded_tokens":3536,"seconds":0.511580858961679},{"candidates":16,"kind":"choice","padded_tokens":3552,"seconds":0.5152770900167525},{"candidates":16,"kind":"choice","padded_tokens":3568,"seconds":0.5216483320109546},{"candidates":16,"kind":"choice","padded_tokens":3600,"seconds":0.5212108340347186},{"candidates":16,"kind":"choice","padded_tokens":3616,"seconds":0.5198794450843707},{"candidates":16,"kind":"choice","padded_tokens":3632,"seconds":0.5272666560485959},{"candidates":16,"kind":"choice","padded_tokens":3632,"seconds":0.5254937210120261},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.5333993210224435},{"candidates":16,"kind":"choice","padded_tokens":3680,"seconds":0.5305549550103024},{"candidates":16,"kind":"choice","padded_tokens":3712,"seconds":0.5306613530265167},{"candidates":16,"kind":"choice","padded_tokens":3728,"seconds":0.539407946052961},{"candidates":16,"kind":"choice","padded_tokens":3760,"seconds":0.5495047640288249},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.5419928289484233},{"candidates":16,"kind":"choice","padded_tokens":3792,"seconds":0.5529175939736888},{"candidates":16,"kind":"choice","padded_tokens":3824,"seconds":0.5392679040087387},{"candidates":16,"kind":"choice","padded_tokens":3840,"seconds":0.5481321970000863},{"candidates":16,"kind":"choice","padded_tokens":3872,"seconds":0.5609226900851354},{"candidates":16,"kind":"choice","padded_tokens":3888,"seconds":0.5508910849457607},{"candidates":16,"kind":"choice","padded_tokens":3920,"seconds":0.5759642330231145},{"candidates":16,"kind":"choice","padded_tokens":3936,"seconds":0.5660510190064088},{"candidates":16,"kind":"choice","padded_tokens":3968,"seconds":0.5737256079446524},{"candidates":16,"kind":"choice","padded_tokens":3984,"seconds":0.5788377050776035},{"candidates":16,"kind":"choice","padded_tokens":4016,"seconds":0.5692169379908592},{"candidates":16,"kind":"choice","padded_tokens":4048,"seconds":0.5825639520771801},{"candidates":16,"kind":"choice","padded_tokens":4080,"seconds":0.5766418400453404},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.5839501930167899},{"candidates":16,"kind":"choice","padded_tokens":4144,"seconds":0.614494921057485},{"candidates":16,"kind":"choice","padded_tokens":4176,"seconds":0.6187723879702389},{"candidates":16,"kind":"choice","padded_tokens":4208,"seconds":0.6190915249753743},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.6164892500964925},{"candidates":16,"kind":"choice","padded_tokens":4256,"seconds":0.6127312940079719},{"candidates":16,"kind":"choice","padded_tokens":4320,"seconds":0.6190499779768288},{"candidates":16,"kind":"choice","padded_tokens":4368,"seconds":0.628844549995847},{"candidates":16,"kind":"choice","padded_tokens":4432,"seconds":0.6295115769607946},{"candidates":16,"kind":"choice","padded_tokens":4448,"seconds":0.6301183019531891},{"candidates":16,"kind":"choice","padded_tokens":4496,"seconds":0.6483643649844453},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.6644774000160396},{"candidates":16,"kind":"choice","padded_tokens":4624,"seconds":0.6820012830430642},{"candidates":16,"kind":"choice","padded_tokens":4640,"seconds":0.6672704619122669},{"candidates":16,"kind":"choice","padded_tokens":4720,"seconds":0.6943771119695157},{"candidates":16,"kind":"choice","padded_tokens":4784,"seconds":0.6850915609393269},{"candidates":16,"kind":"choice","padded_tokens":4848,"seconds":0.7031367690069601},{"candidates":16,"kind":"choice","padded_tokens":4896,"seconds":0.6996529359603301},{"candidates":16,"kind":"choice","padded_tokens":4976,"seconds":0.7136262500425801},{"candidates":16,"kind":"choice","padded_tokens":5040,"seconds":0.7232929540332407},{"candidates":16,"kind":"choice","padded_tokens":5136,"seconds":0.7517683330224827},{"candidates":16,"kind":"choice","padded_tokens":5200,"seconds":0.761715860106051},{"candidates":16,"kind":"choice","padded_tokens":5280,"seconds":0.7757973780389875},{"candidates":16,"kind":"choice","padded_tokens":5440,"seconds":0.803649230976589},{"candidates":16,"kind":"choice","padded_tokens":5680,"seconds":0.792901196051389},{"candidates":16,"kind":"choice","padded_tokens":6064,"seconds":0.8534096149960533},{"candidates":16,"kind":"choice","padded_tokens":6288,"seconds":0.9042458899784833},{"candidates":16,"kind":"choice","padded_tokens":6752,"seconds":0.9718254090985283},{"candidates":16,"kind":"choice","padded_tokens":7744,"seconds":1.119061075965874},{"candidates":14,"kind":"choice","padded_tokens":7854,"seconds":1.1305387149332091},{"candidates":11,"kind":"choice","padded_tokens":8030,"seconds":1.1475872689625248},{"candidates":3,"kind":"choice","padded_tokens":3072,"seconds":0.43835721793584526}],"chat_template":false,"data_loading_seconds":0.06792400300037116,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_common_core.py":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-qwen35-boolq-lm-eval-v2","finished":1792256590,"gpu":0,"id":"130e0f8e238158a96c8145fe","loaded_class":"Qwen3_5ForConditionalGeneration","loader_ignored_auxiliary_patterns":["^mtp.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen3.5-0.8B","model_load_seconds":2.530576813966036,"model_revision":"2fc06364715b967f1860aea9cf38778875588b17","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":852985920,"peak_gpu_bytes":4670398976,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"qwen35-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"BoolQ / full sub-1B common core (qwen35)","parent_task":"reading","protocol_version":"codesota-common-core-qwen35-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core_qwen35.py","runtime_backend":"qwen35","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":192.38576248800382,"measurement_version":"gpu-slot-v1"},"runner_sha256":"eb01abebdaa64a9d8003d37b2006b198582b99f61377a6fe6d0645db15934307","scope":"087725fb28ea8991a3f6bacbedcecfe584a0305a5e4d876496d753c3718af752","score":0.6253822629969419,"scoring_seconds":142.36157488892786,"seconds":145.59709281509276,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"qwen35-boolq-common-core-full","text_parameters":752393024,"tf32":false,"truncated_candidates":2,"vision_parameters":100592896,"worker":"white-gpu0"}