{"acc_norm":0.5636085626911315,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.43425076452599387,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13435136701446027},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.019075719988904893},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.01887155999429524},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.019421003991737962},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.01992143807001412},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.020183156011626124},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.020276400027796626},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.0204443329712376},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.022231777082197368},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.022567534004338086},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.027361905085854232},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.02381728496402502},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.025040565989911556},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.024736759020015597},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.02498603390995413},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.02486137207597494},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.02564736793283373},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.025251499959267676},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.027047759969718754},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.02689457405358553},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.027662315987981856},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.027505513979122043},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.02878444897942245},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.02860554598737508},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.028104051016271114},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.02852891304064542},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.028454363928176463},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.029314487939700484},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.029165519983507693},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.02925871405750513},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.028800570056773722},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.029057087958790362},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.02876956097315997},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.029876235988922417},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.029601313057355583},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.03199522802606225},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.03157530596945435},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.030744025018066168},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.030604063998907804},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.03017844900023192},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.030959596042521298},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.03061995399184525},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.03091439208947122},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.03145783604122698},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.031338923959992826},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.03193602804094553},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.03159075393341482},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.03134018590208143},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.03199530893471092},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.03184259403496981},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.03232753905467689},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.032035775016993284},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.03349927288945764},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.03428888600319624},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.033695069956593215},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.03399371600244194},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.0348639499861747},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.0345782459480688},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.03406772401649505},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.034344249055720866},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.03503796400036663},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.03489742206875235},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.03507650608662516},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.035782662918791175},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.0359506260138005},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.036578698083758354},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.03545947000384331},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.036634481977671385},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.03618727799039334},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.0361835309304297},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.03709779493510723},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.035881898016668856},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.03720561694353819},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.03705736098345369},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.03716029296629131},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.03620230604428798},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.0369915880728513},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.037075333995744586},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.03730882005766034},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.036981529905460775},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.03746821696404368},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.037331761908717453},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.03671138500794768},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.037162115913815796},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.03750856209080666},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.03651824500411749},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.038028522045351565},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.03771215199958533},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.03849706600885838},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.03900843998417258},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.03963377606123686},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.04052473802585155},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.040323332068510354},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.04017938394099474},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.040208398015238345},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.03996669698972255},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.03910106292460114},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.04019682703074068},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.040203729062341154},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.03994987590704113},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.040017953026108444},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.040051565039902925},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.040124410996213555},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.04002939397469163},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.03908287896774709},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.04051025107037276},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.04025215900037438},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.03910016000736505},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.0403532370692119},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.04038712102919817},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.040047997958026826},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.04097592004109174},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.04144721804186702},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.040441583027131855},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.04204051592387259},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.041836444986984134},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.04218310199212283},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.04234554397407919},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.04072547203395516},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.04230074991937727},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.04206165508367121},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.04294009297154844},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.044866256066598},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.04328766209073365},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.04502056399360299},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.04522275098133832},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.04370114300400019},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.04532524198293686},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.04513674997724593},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.04540533199906349},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.04533720400650054},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.043832468101754785},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.045894684037193656},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.046043221023865044},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.04447756102308631},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.04644492093939334},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.0441870290087536},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.04699262196663767},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.04709755699150264},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.04724313004408032},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.044710254995152354},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.046866717049852014},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.047818613005802035},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.04755015205591917},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.04613072401843965},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.048124763066880405},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.04803418507799506},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.04815318691544235},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.04597109602764249},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.048029265948571265},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.04825608804821968},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.045688740094192326},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.04818510601762682},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.047934759058989584},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.04857611493207514},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.048451943905092776},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.04689091199543327},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.0486146779730916},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.04879556503146887},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.04663462401367724},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.04894097603391856},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.0468557559652254},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.049618500052019954},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.049388780957087874},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.049188709002919495},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.046763695077970624},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.049720881041139364},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.04945081798359752},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.047156767919659615},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.050193483009934425},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.04990387195721269},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.049223673064261675},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.04883826395962387},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.046683927066624165},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.04967902193311602},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.049626315012574196},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.04975622706115246},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.04965223290491849},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.05083981703501195},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.05321618996094912},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.050814860966056585},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.053411906003020704},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.05348716606386006},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.05355125502683222},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.053749714978039265},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.05364769499283284},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.05138380196876824},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.05408529099076986},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.05085980496369302},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.054222919046878815},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.05418074899353087},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.05437852896284312},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.05151670100167394},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.05126018193550408},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.05488430103287101},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.05192855896893889},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.05522697092965245},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.05530789203476161},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.054816073970869184},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.05732697795610875},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.057765425997786224},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.057799389003776014},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.05502583598718047},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.058610453037545085},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.05841968604363501},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.05432515905704349},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.05766676098573953},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.05809711408801377},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.054760891012847424},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.05789959605317563},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.05820574704557657},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.0550358040491119},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.05940055497922003},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.0563008839963004},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.05908819299656898},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.05563826800789684},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.06050504604354501},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.06070388702210039},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.05638404900673777},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.06100025901105255},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.06083898805081844},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.05883277708198875},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.06267180701252073},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.05950093292631209},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.06256283400580287},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.06292602105531842},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.05870449694339186},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.06277407798916101},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.06283410999458283},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.05840035993605852},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.0625106159131974},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.058829862042330205},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.06393895100336522},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.06001721601933241},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.06448554096277803},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.06511289102490991},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.06526874098926783},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.06078404595609754},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.06525960401631892},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.06578177807386965},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.062154271989129484},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.06532468600198627},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.06610312697011977},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.06630750908516347},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.06597934593446553},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.06606684997677803},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.06158409803174436},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.06656635098624974},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.06646213692147285},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.0669213340152055},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.06758708599954844},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.062453399994410574},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.06766436900943518},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.06342486198991537},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.06863734498620033},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.06293648108839989},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.06790438701864332},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.06349200697150081},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.06878121301997453},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.06914918008260429},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.06932624895125628},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.0647274119546637},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.06908709299750626},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.06445831002201885},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.06940107909031212},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.06477117293979973},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.06749500392470509},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.06991463608574122},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.0661868629977107},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.07127437298186123},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.06627869501244277},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.06574866699520499},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.07167266600299627},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.06590695201884955},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.07176852494012564},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.06631045509129763},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.07208526507019997},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.06678719399496913},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.07192751101683825},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.07294326601549983},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.06646136601921171},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.0721209020121023},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.06720544293057173},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.07265307498164475},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.06709726096596569},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.07300456997472793},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.06743871897924691},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.06701233296189457},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.07306315994355828},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.06765763706061989},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.07413714297581464},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.06874083704315126},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.07562655094079673},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.06898433098103851},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.07432598492596298},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.0681169539457187},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.07424408197402954},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.07423902291338891},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.07553626189474016},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.06934293103404343},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.07456593296956271},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.06873411498963833},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.0748310370836407},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.06939600897021592},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.076325505040586},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.07720598706509918},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.07751828897744417},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.07761047093663365},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.06999857304617763},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.06960979802533984},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.07702215400058776},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.0712021179497242},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.07785059895832092},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.07140553800854832},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.07758346002083272},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.08091881196014583},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.07440669590141624},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.07468127796892077},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.08228065201546997},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.07542187895160168},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.08255509403534234},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.08347822597716004},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.0797908779932186},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.07879945996683091},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.07915884000249207},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.08010664710309356},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.08529847103636712},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.08139751595444977},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.08142410404980183},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.08115022396668792},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.08690145099535584},{"candidates":16,"kind":"choice","padded_tokens":3216,"seconds":0.08862263092305511},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.0839901709696278},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.08914710010867566},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.0848397349473089},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.08960420289076865},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.08393708197399974},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.08415259409230202},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.08506936300545931},{"candidates":16,"kind":"choice","padded_tokens":3344,"seconds":0.09170571202412248},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.08592728397343308},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.0919629919808358},{"candidates":16,"kind":"choice","padded_tokens":3392,"seconds":0.08686907100491226},{"candidates":16,"kind":"choice","padded_tokens":3424,"seconds":0.0871657639509067},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.09519692393951118},{"candidates":16,"kind":"choice","padded_tokens":3456,"seconds":0.08793843607418239},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.0897900490090251},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.08806678408291191},{"candidates":16,"kind":"choice","padded_tokens":3504,"seconds":0.08843351807445288},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.08844098297413439},{"candidates":16,"kind":"choice","padded_tokens":3536,"seconds":0.08951460500247777},{"candidates":16,"kind":"choice","padded_tokens":3568,"seconds":0.0895226199645549},{"candidates":16,"kind":"choice","padded_tokens":3584,"seconds":0.09058745705988258},{"candidates":16,"kind":"choice","padded_tokens":3600,"seconds":0.0919726500287652},{"candidates":16,"kind":"choice","padded_tokens":3632,"seconds":0.09290902700740844},{"candidates":16,"kind":"choice","padded_tokens":3648,"seconds":0.09354512300342321},{"candidates":16,"kind":"choice","padded_tokens":3664,"seconds":0.09485015703830868},{"candidates":16,"kind":"choice","padded_tokens":3680,"seconds":0.10216012294404209},{"candidates":16,"kind":"choice","padded_tokens":3696,"seconds":0.09460174408741295},{"candidates":16,"kind":"choice","padded_tokens":3728,"seconds":0.09459211700595915},{"candidates":16,"kind":"choice","padded_tokens":3744,"seconds":0.09456326195504516},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.09478755004238337},{"candidates":16,"kind":"choice","padded_tokens":3792,"seconds":0.09603303403127939},{"candidates":16,"kind":"choice","padded_tokens":3824,"seconds":0.09646063100080937},{"candidates":16,"kind":"choice","padded_tokens":3840,"seconds":0.09691830491647124},{"candidates":16,"kind":"choice","padded_tokens":3856,"seconds":0.10759242495987564},{"candidates":16,"kind":"choice","padded_tokens":3888,"seconds":0.09909449389670044},{"candidates":16,"kind":"choice","padded_tokens":3920,"seconds":0.10275397007353604},{"candidates":16,"kind":"choice","padded_tokens":3952,"seconds":0.10260381910484284},{"candidates":16,"kind":"choice","padded_tokens":3984,"seconds":0.10365428903605789},{"candidates":16,"kind":"choice","padded_tokens":4000,"seconds":0.10261365806218237},{"candidates":16,"kind":"choice","padded_tokens":4032,"seconds":0.10311094496864825},{"candidates":16,"kind":"choice","padded_tokens":4064,"seconds":0.1036158170318231},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.10205261199735105},{"candidates":16,"kind":"choice","padded_tokens":4128,"seconds":0.10621816199272871},{"candidates":16,"kind":"choice","padded_tokens":4128,"seconds":0.11766015400644392},{"candidates":16,"kind":"choice","padded_tokens":4176,"seconds":0.10804145189467818},{"candidates":16,"kind":"choice","padded_tokens":4224,"seconds":0.10838997201062739},{"candidates":16,"kind":"choice","padded_tokens":4240,"seconds":0.10852996399626136},{"candidates":16,"kind":"choice","padded_tokens":4272,"seconds":0.1087126339552924},{"candidates":16,"kind":"choice","padded_tokens":4304,"seconds":0.10974060092121363},{"candidates":16,"kind":"choice","padded_tokens":4352,"seconds":0.11049740295857191},{"candidates":16,"kind":"choice","padded_tokens":4384,"seconds":0.11440939002204686},{"candidates":16,"kind":"choice","padded_tokens":4416,"seconds":0.11492392898071557},{"candidates":16,"kind":"choice","padded_tokens":4464,"seconds":0.1176179350586608},{"candidates":16,"kind":"choice","padded_tokens":4528,"seconds":0.11940332502126694},{"candidates":16,"kind":"choice","padded_tokens":4544,"seconds":0.13343975599855185},{"candidates":16,"kind":"choice","padded_tokens":4576,"seconds":0.11736069503240287},{"candidates":16,"kind":"choice","padded_tokens":4608,"seconds":0.11599593795835972},{"candidates":16,"kind":"choice","padded_tokens":4640,"seconds":0.12009494402445853},{"candidates":16,"kind":"choice","padded_tokens":4720,"seconds":0.12224384304136038},{"candidates":16,"kind":"choice","padded_tokens":4736,"seconds":0.12249394902028143},{"candidates":16,"kind":"choice","padded_tokens":4816,"seconds":0.12913368397857994},{"candidates":16,"kind":"choice","padded_tokens":4896,"seconds":0.1306659319670871},{"candidates":16,"kind":"choice","padded_tokens":4944,"seconds":0.13191425008699298},{"candidates":16,"kind":"choice","padded_tokens":5008,"seconds":0.13273869804106653},{"candidates":16,"kind":"choice","padded_tokens":5104,"seconds":0.13349500903859735},{"candidates":16,"kind":"choice","padded_tokens":5200,"seconds":0.14042874204460531},{"candidates":16,"kind":"choice","padded_tokens":5328,"seconds":0.14437841903418303},{"candidates":16,"kind":"choice","padded_tokens":5440,"seconds":0.14621836110018194},{"candidates":16,"kind":"choice","padded_tokens":5584,"seconds":0.15345918596722186},{"candidates":16,"kind":"choice","padded_tokens":5728,"seconds":0.1587596939643845},{"candidates":16,"kind":"choice","padded_tokens":5904,"seconds":0.15469103399664164},{"candidates":16,"kind":"choice","padded_tokens":6256,"seconds":0.17130132601596415},{"candidates":16,"kind":"choice","padded_tokens":6544,"seconds":0.17935318499803543},{"candidates":16,"kind":"choice","padded_tokens":7104,"seconds":0.19425213406793773},{"candidates":16,"kind":"choice","padded_tokens":7936,"seconds":0.22335621097590774},{"candidates":14,"kind":"choice","padded_tokens":7980,"seconds":0.22964632289949805},{"candidates":10,"kind":"choice","padded_tokens":6850,"seconds":0.20564432395622134},{"candidates":4,"kind":"choice","padded_tokens":4096,"seconds":0.1351630799472332}],"chat_template":false,"data_loading_seconds":0.06483647495042533,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-boolq-lm-eval-v2","finished":1792251706,"gpu":1,"id":"9596c98e52abd47122f2cc65","measured_cost_usd":null,"metric":"accuracy","model":"HuggingFaceTB/SmolLM2-135M-Instruct","model_load_seconds":1.7373057840159163,"model_revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":134515008,"peak_gpu_bytes":790948352,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"native-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":30.920292659080587,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"7f2a4b46461d9e7fe0763f55ef1a046476cd093351d877ad5515e16812e1bdca","score":0.43425076452599387,"scoring_seconds":26.7717894470552,"seconds":29.22179620200768,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"native-boolq-common-core-full","tf32":false,"truncated_candidates":2,"worker":"white-gpu1"}