{"acc_norm":0.2858361774744027,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.24488054607508533,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.30984812695533037},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.12818655802402645},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.12741914601065218},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.12900417193304747},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13196650706231594},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.13020158500876278},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.1295659700408578},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.12905679002869874},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.13021806499455124},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13120637903921306},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12972483597695827},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.1302812130888924},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.12906842201482505},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.13112200098112226},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.1335190209792927},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13287812599446625},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13266357604879886},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13125372701324522},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.13007308600936085},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13084231002721936},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.12954548199195415},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13095683394931257},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13893870799802244},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.13621993898414075},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.12953899009153247},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13837032893206924},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.14722408901434392},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13079325901344419},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.1315249219769612},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.134445869945921},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.13167868903838098},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13001071894541383},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13015546894166619},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1302903409814462},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13136275100987405},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1303947849664837},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1302754218922928},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13018881098832935},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.1303413660498336},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.13014525990001857},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13307630608323961},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13157692993991077},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13149591907858849},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13022295397240669},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13051345699932426},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.13064766593743116},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1321032210253179},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1302231450099498},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1299848210765049},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.1306176109937951},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12995618698187172},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13065300602465868},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13035407906863838},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13013396901078522},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12988605699501932},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1312434779247269},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1299257999053225},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.1301113470690325},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13028562103863806},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.13021401804871857},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.12966275901999325},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.12968845793511719},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13133503904100507},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13147007091902196},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13159806898329407},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13065488007850945},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1310971250059083},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13225745898671448},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1321454900316894},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13111852505244315},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.1310882680118084},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.13181967195123434},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.1304991099750623},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.12976155406795442},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13079466100316495},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13062444399110973},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13062278099823743},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13025415304582566},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13078303902875632},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13326863502152264},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.13060108001809567},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13065876706968993},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13044241396710277},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13083957496564835},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13124097301624715},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1302297180518508},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13183723506517708},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.13065993902273476},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.1307690639514476},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13062174804508686},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.130551798036322},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1310077781090513},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13035523099824786},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13122443202883005},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.1317592900013551},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.13439853105228394},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1314350760076195},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.130382442031987},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.1298938919790089},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13062107702717185},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13016688090283424},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13044478802476078},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13142073899507523},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.13027322804555297},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.1301007269648835},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13020647398661822},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13040952302981168},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13059525901917368},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.12985148199368268},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13005939999129623},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13131122395861894},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.13157762098126113},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13299381197430193},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1311797690577805},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13089157198555768},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.13112702092621475},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.1310780489584431},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13205876795109361},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1292528659105301},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12936129898298532},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13049570308066905},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1320480490103364},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1301299510523677},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.12969222501851618},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.13043018092866987},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13297449599485844},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12969034200068563},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12883428600616753},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.13051650195848197},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12985274498350918},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12905952509026974},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.12892157793976367},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12877423304598778},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.129588330979459},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1285558360395953},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1285490730078891},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12876728002447635},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.12955827498808503},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1287069880636409},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12870997295249254},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1287610789295286},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1302922039758414},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.12883599800989032},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.13158039608970284},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1293450080556795},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13196834991686046},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.1325411779107526},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13109767599962652},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13063598598819226},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.13037309399805963},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1300051590660587},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13056716602295637},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13037427701056004},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1304900029208511},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13045972702093422},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.13049101503565907},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13112063007429242},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1298541669966653},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1300093170721084},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13062634703237563},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.13038823299575597},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12975713599007577},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1297494019381702},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.128386490046978},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1295731229474768},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.12883438600692898},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12813128402922302},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1275782430311665},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12765637796837837},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.12749021803028882},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.127513692015782},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12800078105647117},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.13007110089529306},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1286386309657246},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12962551100645214},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.12769949901849031},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1282275040866807},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12873692298308015},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1324679619865492},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12866881699301302},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1289509020280093},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12777049106080085},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1277498829877004},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12849841895513237},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12850623403210193},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1281045739306137},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.12805420102085918},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1291787779191509},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12865566206164658},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12814250506926328},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12833516392856836},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12828185502439737},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12847083702217788},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12878210796043277},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12898191099520773},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.13086205709259957},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1294841169146821},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12798614404164255},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12761217693332583},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12883799197152257},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12818980298470706},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1283156080171466},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12788675900083035},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12759619602002203},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12715526402462274},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.1273068559821695},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12750534701626748},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12771527899894863},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12766057602129877},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1279427419649437},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12762864690739661},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12855381204281002},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13070144702214748},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12711101199965924},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12766397302038968},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12738923006691039},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12753401999361813},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13027294795028865},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12847230001352727},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12860388599801809},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12749790295492858},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.12751504499465227},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.12853800307493657},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.12802999501582235},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1281847330974415},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.12814346700906754},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13116360909771174},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13530475099105388},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13434027298353612},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1329596290597692},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1311796990921721},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13181113707832992},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13150938297621906},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13257923803757876},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13184845598880202},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13182242796756327},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13176074193324894},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13165576697792858},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13152644503861666},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13164366409182549},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13137506390921772},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13212358893360943},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.13125831598881632},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13213625294156373},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13119475706480443},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13280593103263527},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.130878216936253},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13086013298016042},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1309583761030808},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12916331901215017},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.12856031500268728},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.12841301900334656},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.12864893896039575},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.12973008607514203},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13091457402333617},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13103209401015192},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.12915007397532463},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.12826689705252647},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.12819202698301524},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.1299611369613558},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.13168511202093214},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1313767459942028},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.13430744200013578},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.13308182603213936},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.13379046798218042},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.13374395098071545},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.13351178797893226},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1341532440856099},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.13368983997497708},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.13424920290708542},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.13399427803233266},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.13320529705379158},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.13344257895369083},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.1335731929866597},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.13280030095484108},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.13062161905691028},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.1309426969382912},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.13184681301936507},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.13297077897004783},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.13492320000659674},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.13231026800349355},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.13330859900452197},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.1322701530298218},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.13315345998853445},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.13220687489956617},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.13325708301272243},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.13317039201501757},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.13566957192961127},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.13520386291202158},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.13569151295814663},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.13502670300658792},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.1380953659536317},{"candidates":15,"kind":"choice","padded_tokens":3210,"seconds":0.1474056689767167}],"chat_template":false,"data_loading_seconds":0.06573323102202266,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-arc-challenge-v1","finished":1792249379,"gpu":1,"id":"0fcec49a0d92f314551a51a4","measured_cost_usd":null,"metric":"accuracy","model":"SlayerLab/Slayer149","model_load_seconds":2.311181080993265,"model_revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e","n":1172,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":148910738,"peak_gpu_bytes":768280576,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge (full test, raw likelihood accuracy)","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"native-arc-challenge-common-core-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"ARC-Challenge / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-arc-challenge-v1","recipe":{"data_file":"ARC-Challenge/test-00000-of-00001.parquet","schema":"arc-challenge-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"arc_challenge"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":42.89618340297602,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"e7984b7ea3a9cae0996b0a453de457a6a7d717e9a2c3e9a95d80f697f8bd5afb","score":0.24488054607508533,"scoring_seconds":38.70821375201922,"seconds":41.19777736696415,"seed":42,"selection":"All 1172 pinned test rows in source order; raw answer log likelihood; no chat; character-normalized diagnostic separately","status":"completed","task":"native-arc-challenge-common-core-full","tf32":false,"truncated_candidates":0,"worker":"white-gpu1"}