{"acc_norm":0.6467889908256881,"acc_norm_definition":"argmax summed conditional likelihood / Python character length of answer, excluding leading delimiter","acc_raw":0.6795107033639144,"batch_policy":{"answer_limit":512,"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09685977606568485},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.06830089702270925},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.0553692759713158},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.05693985603284091},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.055884707951918244},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.05793418106622994},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.06140739901456982},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.05586757499258965},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.06594514299649745},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.06649809400551021},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.07221856398973614},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.07847910095006227},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.07234786509070545},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.09412544593214989},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.07618314900901169},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.06612430699169636},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.07793305302038789},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.06792260194197297},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.07620770507492125},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.08666853595059365},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.07538314699195325},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.08393895498011261},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.08677155897021294},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.08003521407954395},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.09074712404981256},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.0955129029462114},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.08424243109766394},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.08752147702034563},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.09075790399219841},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.08890300407074392},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.08165861200541258},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.08615205308888108},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.0910809769993648},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.0833991679828614},{"candidates":16,"kind":"choice","padded_tokens":1104,"seconds":0.09017414599657059},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.09505573997739702},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.09687841997947544},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.08538769709412009},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.09301959199365228},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.09368523501325399},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.09274227602872998},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.08835236809682101},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.09346710797399282},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.09634779090993106},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.0972795890411362},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.09167772997170687},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.09803823404945433},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.09811485698446631},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.10209269600454718},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.0901780619751662},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.10804066096898168},{"candidates":16,"kind":"choice","padded_tokens":1232,"seconds":0.10800488409586251},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.10585023509338498},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.10829721903428435},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.11225793696939945},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.1115066249622032},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.1111795250326395},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.10175565700046718},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.11126357188913971},{"candidates":16,"kind":"choice","padded_tokens":1280,"seconds":0.11164700705558062},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.10447905701585114},{"candidates":16,"kind":"choice","padded_tokens":1296,"seconds":0.10609206592198461},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.10567677102517337},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.106625891989097},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.09895473392680287},{"candidates":16,"kind":"choice","padded_tokens":1328,"seconds":0.10721166396979243},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.10603485908359289},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.10906838800292462},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.11986436392180622},{"candidates":16,"kind":"choice","padded_tokens":1360,"seconds":0.11920021602418274},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.11751400108914822},{"candidates":16,"kind":"choice","padded_tokens":1376,"seconds":0.12109947798307985},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.12026459106709808},{"candidates":16,"kind":"choice","padded_tokens":1392,"seconds":0.12142276205122471},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.12076071603223681},{"candidates":16,"kind":"choice","padded_tokens":1408,"seconds":0.12379770097322762},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.11782040202524513},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.11850107205100358},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.11545563105028123},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.1096453940263018},{"candidates":16,"kind":"choice","padded_tokens":1440,"seconds":0.11946912796702236},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.11790318705607206},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.12100489204749465},{"candidates":16,"kind":"choice","padded_tokens":1456,"seconds":0.12100656505208462},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.11998301604762673},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.12028464896138757},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12184587004594505},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12469078600406647},{"candidates":16,"kind":"choice","padded_tokens":1488,"seconds":0.12524203502107412},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.12264678394421935},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.12460281199309975},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.12705485604237765},{"candidates":16,"kind":"choice","padded_tokens":1504,"seconds":0.12677591596730053},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.12467491708230227},{"candidates":16,"kind":"choice","padded_tokens":1520,"seconds":0.12496094102971256},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.12364964501466602},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.12784102209843695},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.12620650499593467},{"candidates":16,"kind":"choice","padded_tokens":1536,"seconds":0.12691169895697385},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.1303073229501024},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.13208138302434236},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.1304069289471954},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.13999847904779017},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.13248660799581558},{"candidates":16,"kind":"choice","padded_tokens":1568,"seconds":0.13211440399754792},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.13010180997662246},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.13267781503964216},{"candidates":16,"kind":"choice","padded_tokens":1584,"seconds":0.13160620594862849},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.12966853193938732},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.1329459649277851},{"candidates":16,"kind":"choice","padded_tokens":1600,"seconds":0.1333379059797153},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.13437455892562866},{"candidates":16,"kind":"choice","padded_tokens":1616,"seconds":0.13691006903536618},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.13450806797482073},{"candidates":16,"kind":"choice","padded_tokens":1632,"seconds":0.13952972495462745},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.141203748062253},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.1380360700422898},{"candidates":16,"kind":"choice","padded_tokens":1648,"seconds":0.13770832808222622},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.1328829979756847},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.1372267990373075},{"candidates":16,"kind":"choice","padded_tokens":1664,"seconds":0.13803998590447009},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.1417473709443584},{"candidates":16,"kind":"choice","padded_tokens":1680,"seconds":0.14214875001925975},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.13781119894701988},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.150034319027327},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.14127734501380473},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.1396752370055765},{"candidates":16,"kind":"choice","padded_tokens":1712,"seconds":0.1435685589676723},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.14128582109697163},{"candidates":16,"kind":"choice","padded_tokens":1728,"seconds":0.14440508896950632},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.1341763890814036},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.14633716200478375},{"candidates":16,"kind":"choice","padded_tokens":1744,"seconds":0.1368658570572734},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.131910303956829},{"candidates":16,"kind":"choice","padded_tokens":1760,"seconds":0.13955305004492402},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.13517002400476485},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.13794768310617656},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.13834791106637567},{"candidates":16,"kind":"choice","padded_tokens":1776,"seconds":0.13954048498999327},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.14061122201383114},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.15103505505248904},{"candidates":16,"kind":"choice","padded_tokens":1792,"seconds":0.14349545200821012},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.1380547439912334},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.1444804989732802},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.1418175520375371},{"candidates":16,"kind":"choice","padded_tokens":1808,"seconds":0.14340941200498492},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.14055192202795297},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.14447393803857267},{"candidates":16,"kind":"choice","padded_tokens":1824,"seconds":0.1435952780302614},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.14044131501577795},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.14346461498644203},{"candidates":16,"kind":"choice","padded_tokens":1840,"seconds":0.14456375502049923},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.14128370699472725},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.15342273795977235},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.14535421901382506},{"candidates":16,"kind":"choice","padded_tokens":1856,"seconds":0.1470130729721859},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.1417751939734444},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.1457323939539492},{"candidates":16,"kind":"choice","padded_tokens":1872,"seconds":0.15407956391572952},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.14499373699072748},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.14323859300930053},{"candidates":16,"kind":"choice","padded_tokens":1888,"seconds":0.14698234503157437},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.14508952491451055},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.1469374819425866},{"candidates":16,"kind":"choice","padded_tokens":1904,"seconds":0.15398722200188786},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.14357709395699203},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.1462237809319049},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.1483896808931604},{"candidates":16,"kind":"choice","padded_tokens":1920,"seconds":0.1508184919366613},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.16739187494385988},{"candidates":16,"kind":"choice","padded_tokens":1936,"seconds":0.15680029895156622},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.1596159200416878},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.17222153302282095},{"candidates":16,"kind":"choice","padded_tokens":1952,"seconds":0.15938552003353834},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.15856212505605072},{"candidates":16,"kind":"choice","padded_tokens":1968,"seconds":0.17049881001003087},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.15819743496831506},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.16233691503293812},{"candidates":16,"kind":"choice","padded_tokens":1984,"seconds":0.1650257819565013},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.16823601897340268},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.1637984620174393},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.1592616100097075},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.17072856798768044},{"candidates":16,"kind":"choice","padded_tokens":2016,"seconds":0.15992765198461711},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.1612367529887706},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.1709704389795661},{"candidates":16,"kind":"choice","padded_tokens":2032,"seconds":0.16343303106259555},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.1609660880640149},{"candidates":16,"kind":"choice","padded_tokens":2048,"seconds":0.17186885501723737},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.16335059597622603},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.1688073449768126},{"candidates":16,"kind":"choice","padded_tokens":2064,"seconds":0.17593904701061547},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.16544161702040583},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.17002412304282188},{"candidates":16,"kind":"choice","padded_tokens":2080,"seconds":0.17629370803479105},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.1652076099999249},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.1784796860301867},{"candidates":16,"kind":"choice","padded_tokens":2096,"seconds":0.16808077902533114},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.16752515302505344},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.17766452603973448},{"candidates":16,"kind":"choice","padded_tokens":2112,"seconds":0.1697218200424686},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.16942306305281818},{"candidates":16,"kind":"choice","padded_tokens":2128,"seconds":0.1803497129585594},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.17126856499817222},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.18284982698969543},{"candidates":16,"kind":"choice","padded_tokens":2144,"seconds":0.17330783896613866},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.1775913389865309},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.17448101693298668},{"candidates":16,"kind":"choice","padded_tokens":2160,"seconds":0.17737475398462266},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.17777783598285168},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.1757653329987079},{"candidates":16,"kind":"choice","padded_tokens":2176,"seconds":0.18363930901978165},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.17769786808639765},{"candidates":16,"kind":"choice","padded_tokens":2192,"seconds":0.18710516404826194},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.17182022496126592},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.181742231012322},{"candidates":16,"kind":"choice","padded_tokens":2208,"seconds":0.1846768750110641},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.18152045691385865},{"candidates":16,"kind":"choice","padded_tokens":2224,"seconds":0.1865008770255372},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.17623493901919574},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.18758183298632503},{"candidates":16,"kind":"choice","padded_tokens":2240,"seconds":0.1809991750633344},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.18969032599125057},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.18972345802467316},{"candidates":16,"kind":"choice","padded_tokens":2256,"seconds":0.19483227608725429},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.18590730999130756},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.19465140998363495},{"candidates":16,"kind":"choice","padded_tokens":2272,"seconds":0.19004527805373073},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.19058213999960572},{"candidates":16,"kind":"choice","padded_tokens":2288,"seconds":0.20190036005806178},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.1829359179828316},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.1991988109657541},{"candidates":16,"kind":"choice","padded_tokens":2304,"seconds":0.18899632198736072},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.20146879507228732},{"candidates":16,"kind":"choice","padded_tokens":2320,"seconds":0.1960972569650039},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.19458723906427622},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.2031821209238842},{"candidates":16,"kind":"choice","padded_tokens":2336,"seconds":0.19355996407102793},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.20064838603138924},{"candidates":16,"kind":"choice","padded_tokens":2352,"seconds":0.19407374295406044},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.19915896793827415},{"candidates":16,"kind":"choice","padded_tokens":2368,"seconds":0.20908655400853604},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.19129326508846134},{"candidates":16,"kind":"choice","padded_tokens":2384,"seconds":0.20784659194760025},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.19221566698979586},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.20722981193102896},{"candidates":16,"kind":"choice","padded_tokens":2400,"seconds":0.21017755893990397},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.19783805310726166},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.21067266294267029},{"candidates":16,"kind":"choice","padded_tokens":2416,"seconds":0.2146676939446479},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.1968932809541002},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.21060180093627423},{"candidates":16,"kind":"choice","padded_tokens":2432,"seconds":0.2176323840394616},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.19438292901031673},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.21121441305149347},{"candidates":16,"kind":"choice","padded_tokens":2448,"seconds":0.2153717071050778},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.20479167404118925},{"candidates":16,"kind":"choice","padded_tokens":2464,"seconds":0.21265849703922868},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.2066186519805342},{"candidates":16,"kind":"choice","padded_tokens":2480,"seconds":0.20563826197758317},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.20519926398992538},{"candidates":16,"kind":"choice","padded_tokens":2496,"seconds":0.21809671097435057},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.20870884996838868},{"candidates":16,"kind":"choice","padded_tokens":2512,"seconds":0.21596094698179513},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.2101222659694031},{"candidates":16,"kind":"choice","padded_tokens":2528,"seconds":0.21880277793388814},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.20245629700366408},{"candidates":16,"kind":"choice","padded_tokens":2544,"seconds":0.21632695896551013},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.21428384794853628},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.21117927797604352},{"candidates":16,"kind":"choice","padded_tokens":2560,"seconds":0.2179734209785238},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.21584431000519544},{"candidates":16,"kind":"choice","padded_tokens":2576,"seconds":0.22511904896236956},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.20811523299198598},{"candidates":16,"kind":"choice","padded_tokens":2592,"seconds":0.22360008605755866},{"candidates":16,"kind":"choice","padded_tokens":2608,"seconds":0.22097033099271357},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.21562496002297848},{"candidates":16,"kind":"choice","padded_tokens":2624,"seconds":0.21777554193977267},{"candidates":16,"kind":"choice","padded_tokens":2640,"seconds":0.2183480089297518},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.2206225220579654},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.23279173008631915},{"candidates":16,"kind":"choice","padded_tokens":2656,"seconds":0.22959840297698975},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.21423983597196639},{"candidates":16,"kind":"choice","padded_tokens":2672,"seconds":0.22801474994048476},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.2223017239011824},{"candidates":16,"kind":"choice","padded_tokens":2688,"seconds":0.23264687997289002},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.22378244600258768},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.22090806509368122},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.23289517301600426},{"candidates":16,"kind":"choice","padded_tokens":2704,"seconds":0.2306640810566023},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.22340212599374354},{"candidates":16,"kind":"choice","padded_tokens":2720,"seconds":0.2331728310091421},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.2152423270745203},{"candidates":16,"kind":"choice","padded_tokens":2736,"seconds":0.23227741103619337},{"candidates":16,"kind":"choice","padded_tokens":2752,"seconds":0.22088020201772451},{"candidates":16,"kind":"choice","padded_tokens":2768,"seconds":0.2266122039873153},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.23087994405068457},{"candidates":16,"kind":"choice","padded_tokens":2784,"seconds":0.23576123907696456},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.22039188095368445},{"candidates":16,"kind":"choice","padded_tokens":2800,"seconds":0.238111793063581},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.22487625502981246},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.23767232394311577},{"candidates":16,"kind":"choice","padded_tokens":2816,"seconds":0.23879697092343122},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.23312337801326066},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.24038538499735296},{"candidates":16,"kind":"choice","padded_tokens":2832,"seconds":0.24328945006709546},{"candidates":16,"kind":"choice","padded_tokens":2848,"seconds":0.22382511501200497},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.23284311592578888},{"candidates":16,"kind":"choice","padded_tokens":2864,"seconds":0.2426856040256098},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.23207596491556615},{"candidates":16,"kind":"choice","padded_tokens":2880,"seconds":0.24523003993090242},{"candidates":16,"kind":"choice","padded_tokens":2896,"seconds":0.247492881026119},{"candidates":16,"kind":"choice","padded_tokens":2912,"seconds":0.2351972870528698},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.2366677289828658},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.2529835019959137},{"candidates":16,"kind":"choice","padded_tokens":2928,"seconds":0.2473946079844609},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.23598724999465048},{"candidates":16,"kind":"choice","padded_tokens":2944,"seconds":0.25215959490742534},{"candidates":16,"kind":"choice","padded_tokens":2960,"seconds":0.2452784909401089},{"candidates":16,"kind":"choice","padded_tokens":2976,"seconds":0.24666151101700962},{"candidates":16,"kind":"choice","padded_tokens":2992,"seconds":0.24680287402588874},{"candidates":16,"kind":"choice","padded_tokens":3008,"seconds":0.24638668703846633},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.24676521390210837},{"candidates":16,"kind":"choice","padded_tokens":3024,"seconds":0.2610216559842229},{"candidates":16,"kind":"choice","padded_tokens":3040,"seconds":0.2635126420063898},{"candidates":16,"kind":"choice","padded_tokens":3056,"seconds":0.2486321059986949},{"candidates":16,"kind":"choice","padded_tokens":3072,"seconds":0.25028037989977747},{"candidates":16,"kind":"choice","padded_tokens":3088,"seconds":0.2595599179621786},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.2591341150691733},{"candidates":16,"kind":"choice","padded_tokens":3104,"seconds":0.26920295506715775},{"candidates":16,"kind":"choice","padded_tokens":3120,"seconds":0.2702341789845377},{"candidates":16,"kind":"choice","padded_tokens":3136,"seconds":0.26562569406814873},{"candidates":16,"kind":"choice","padded_tokens":3152,"seconds":0.26391686510760337},{"candidates":16,"kind":"choice","padded_tokens":3168,"seconds":0.2627977579832077},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.2634820849634707},{"candidates":16,"kind":"choice","padded_tokens":3184,"seconds":0.2754041019361466},{"candidates":16,"kind":"choice","padded_tokens":3200,"seconds":0.2668438960099593},{"candidates":16,"kind":"choice","padded_tokens":3232,"seconds":0.2694961519446224},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.2791395689127967},{"candidates":16,"kind":"choice","padded_tokens":3248,"seconds":0.2777651350479573},{"candidates":16,"kind":"choice","padded_tokens":3264,"seconds":0.26956257701385766},{"candidates":16,"kind":"choice","padded_tokens":3280,"seconds":0.2729451049817726},{"candidates":16,"kind":"choice","padded_tokens":3296,"seconds":0.2744960280833766},{"candidates":16,"kind":"choice","padded_tokens":3312,"seconds":0.28135020297486335},{"candidates":16,"kind":"choice","padded_tokens":3328,"seconds":0.27076071104966104},{"candidates":16,"kind":"choice","padded_tokens":3360,"seconds":0.27599225903395563},{"candidates":16,"kind":"choice","padded_tokens":3376,"seconds":0.2790885539725423},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.284895385033451},{"candidates":16,"kind":"choice","padded_tokens":3408,"seconds":0.2840426139300689},{"candidates":16,"kind":"choice","padded_tokens":3424,"seconds":0.274622764904052},{"candidates":16,"kind":"choice","padded_tokens":3440,"seconds":0.27596683194860816},{"candidates":16,"kind":"choice","padded_tokens":3472,"seconds":0.29455218801740557},{"candidates":16,"kind":"choice","padded_tokens":3488,"seconds":0.2821581899188459},{"candidates":16,"kind":"choice","padded_tokens":3504,"seconds":0.2874366049654782},{"candidates":16,"kind":"choice","padded_tokens":3520,"seconds":0.30771545006427914},{"candidates":16,"kind":"choice","padded_tokens":3536,"seconds":0.2889652660815045},{"candidates":16,"kind":"choice","padded_tokens":3584,"seconds":0.2876646699151024},{"candidates":16,"kind":"choice","padded_tokens":3600,"seconds":0.2978745149448514},{"candidates":16,"kind":"choice","padded_tokens":3600,"seconds":0.2955556589877233},{"candidates":16,"kind":"choice","padded_tokens":3616,"seconds":0.28897897305432707},{"candidates":16,"kind":"choice","padded_tokens":3632,"seconds":0.2974811710882932},{"candidates":16,"kind":"choice","padded_tokens":3664,"seconds":0.28783758298959583},{"candidates":16,"kind":"choice","padded_tokens":3696,"seconds":0.2930158320814371},{"candidates":16,"kind":"choice","padded_tokens":3712,"seconds":0.2985739290015772},{"candidates":16,"kind":"choice","padded_tokens":3728,"seconds":0.2995324160438031},{"candidates":16,"kind":"choice","padded_tokens":3744,"seconds":0.30965298309456557},{"candidates":16,"kind":"choice","padded_tokens":3776,"seconds":0.2964026390109211},{"candidates":16,"kind":"choice","padded_tokens":3808,"seconds":0.31589679000899196},{"candidates":16,"kind":"choice","padded_tokens":3824,"seconds":0.3027120280312374},{"candidates":16,"kind":"choice","padded_tokens":3840,"seconds":0.3042252700543031},{"candidates":16,"kind":"choice","padded_tokens":3872,"seconds":0.3207885050214827},{"candidates":16,"kind":"choice","padded_tokens":3904,"seconds":0.32328671403229237},{"candidates":16,"kind":"choice","padded_tokens":3920,"seconds":0.315339811029844},{"candidates":16,"kind":"choice","padded_tokens":3952,"seconds":0.33183550008106977},{"candidates":16,"kind":"choice","padded_tokens":3968,"seconds":0.3176872000331059},{"candidates":16,"kind":"choice","padded_tokens":4000,"seconds":0.33762243296951056},{"candidates":16,"kind":"choice","padded_tokens":4032,"seconds":0.3437139850575477},{"candidates":16,"kind":"choice","padded_tokens":4064,"seconds":0.34372141992207617},{"candidates":16,"kind":"choice","padded_tokens":4064,"seconds":0.35052462900057435},{"candidates":16,"kind":"choice","padded_tokens":4096,"seconds":0.34586192201822996},{"candidates":16,"kind":"choice","padded_tokens":4144,"seconds":0.34525018103886396},{"candidates":16,"kind":"choice","padded_tokens":4192,"seconds":0.3529519260628149},{"candidates":16,"kind":"choice","padded_tokens":4208,"seconds":0.3700487760361284},{"candidates":16,"kind":"choice","padded_tokens":4256,"seconds":0.3635028259595856},{"candidates":16,"kind":"choice","padded_tokens":4304,"seconds":0.368444393039681},{"candidates":16,"kind":"choice","padded_tokens":4336,"seconds":0.3626996480161324},{"candidates":16,"kind":"choice","padded_tokens":4400,"seconds":0.3717673619976267},{"candidates":16,"kind":"choice","padded_tokens":4432,"seconds":0.3709475720534101},{"candidates":16,"kind":"choice","padded_tokens":4480,"seconds":0.36773602198809385},{"candidates":16,"kind":"choice","padded_tokens":4560,"seconds":0.3977137570036575},{"candidates":16,"kind":"choice","padded_tokens":4592,"seconds":0.3914181140717119},{"candidates":16,"kind":"choice","padded_tokens":4656,"seconds":0.3945725589292124},{"candidates":16,"kind":"choice","padded_tokens":4720,"seconds":0.3919793009990826},{"candidates":16,"kind":"choice","padded_tokens":4784,"seconds":0.39538877096492797},{"candidates":16,"kind":"choice","padded_tokens":4832,"seconds":0.41213309403974563},{"candidates":16,"kind":"choice","padded_tokens":4880,"seconds":0.40867764805443585},{"candidates":16,"kind":"choice","padded_tokens":4960,"seconds":0.3952670639846474},{"candidates":16,"kind":"choice","padded_tokens":5008,"seconds":0.4121926629450172},{"candidates":16,"kind":"choice","padded_tokens":5088,"seconds":0.42504822299815714},{"candidates":16,"kind":"choice","padded_tokens":5168,"seconds":0.42659188201650977},{"candidates":16,"kind":"choice","padded_tokens":5248,"seconds":0.4416083509568125},{"candidates":16,"kind":"choice","padded_tokens":5408,"seconds":0.46117020898964256},{"candidates":16,"kind":"choice","padded_tokens":5664,"seconds":0.4902480250457302},{"candidates":16,"kind":"choice","padded_tokens":6032,"seconds":0.5175509409746155},{"candidates":16,"kind":"choice","padded_tokens":6304,"seconds":0.5533043299801648},{"candidates":16,"kind":"choice","padded_tokens":6720,"seconds":0.5764502519741654},{"candidates":16,"kind":"choice","padded_tokens":7728,"seconds":0.6838768970919773},{"candidates":14,"kind":"choice","padded_tokens":7840,"seconds":0.6917693690629676},{"candidates":11,"kind":"choice","padded_tokens":8019,"seconds":0.7054995490470901},{"candidates":3,"kind":"choice","padded_tokens":3072,"seconds":0.28722213592845947}],"chat_template":false,"data_loading_seconds":0.05271443398669362,"dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dependency_sources":{"common-core-profiles.json":"a54c99201b8e0784bc5d7efab18b39f3dc50168ff6f0f7bee1315e77bb9bb4f1","evaluate_qwen35_challenge.py":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa","sub1b-models.json":"e6b253dc7b72d796866c2d6d9fb2ff63f0dd86203b40f2d1088059ef6a4995bd","sub1b_balanced.py":"f2949b3c2a6f3585ba9eb4b3bc7ef8306afea74492670d7c82d8c3f7e2c2a64b","sub1b_gollem.py":"b9bff95fe326165b5685572727d5c50b1f86a50676932ad5b7da17018965e1d7","sub1b_native.py":"ea56adc25566f7f6745ab882684dfc67e7f1cc48f29e53422fb3cab0cce769e7","sub1b_slayer139.py":"f23a691bb330db4e6311b4b7e83936d5a4c584cc3f61ecf7f7025435ad87d326"},"device":"cuda:0","evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"codesota-common-core-native-boolq-lm-eval-v2","finished":1792251636,"gpu":0,"id":"30e952eec4338fcf8caf9b63","measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen2.5-0.5B-Instruct","model_load_seconds":2.061211013002321,"model_revision":"7ae557604adf67be50417f59c2c2f167def9a775","n":3270,"notes":"Full pinned split; primary raw summed conditional answer-token log likelihood accuracy; character-normalized accuracy is a separate diagnostic. Explicit pinned prompt schema, no chat/BOS. Left context truncated only if prompt plus answer exceeds 1024 tokens. Generic text diagnostic does not validate specialist declared tasks.","parameters":494032768,"peak_gpu_bytes":2597230080,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"BoolQ (full validation, raw likelihood accuracy)","catalogue_id":"boolq","choice_score":"sum_log_likelihood","contract":"Provided context/question + candidate answers \u2192 selected answer","coverage":"full","dataset":"aps/super_glue","dataset_config":"boolq","dataset_revision":"3de24cf8022e94f4ee4b9d55a6f539891524d646","dataset_url":"https://huggingface.co/datasets/aps/super_glue","direction":"max","id":"native-boolq-common-core-full","license":["other"],"limit":3270,"metric":"accuracy","models":[{"id":"SlayerLab/Slayer149-ARC","revision":"0e6793ecfb8501dccc1c97dcd944731748eb8d17"},{"id":"HuggingFaceTB/SmolLM2-135M-Instruct","revision":"12fd25f77366fa6b3b4b768ec3050bf629380bac"},{"id":"HuggingFaceTB/SmolLM2-360M-Instruct","revision":"a10cc1512eabd3dde888204e902eca88bddb4951"},{"id":"Qwen/Qwen2.5-0.5B-Instruct","revision":"7ae557604adf67be50417f59c2c2f167def9a775"},{"id":"SlayerLab/Slayer149","revision":"7857dd28ec052cd943edcdc1f9e35f7b21dc4f3e"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-45500","revision":"e7d4b8dea44125264077c2ce6d2f141379f999cd"},{"id":"SlayerLab/Slayer139-1.01","revision":"2e72c696d7c5cad73dc58606b0751b911dc94495"},{"id":"SlayerLab/Slayer139-1.01b","revision":"283ffb6d620be2d6ea1f2429d721962dc8e73633"},{"id":"SlayerLab/Slayer149-Balanced-ARC-E-ft-v2","revision":"7d11e8fd63764baf00c276f586b2aa776648ed33"},{"id":"SlayerLab/Slayer139-1.02","revision":"c4298d757ba39cba19dd768777da9fadf319ff61"},{"id":"SlayerLab/Slayer-149M-Decision-v1","revision":"ccfd7d3d91f3894beb1cd096b5d067333ba3d4c7"}],"name":"BoolQ / full sub-1B common core (native)","parent_task":"reading","protocol_version":"codesota-common-core-native-boolq-lm-eval-v2","recipe":{"data_file":"boolq/validation-00000-of-00001.parquet","schema":"boolq-lm-eval-v1"},"reference_protocol":{"repository":"EleutherAI/lm-evaluation-harness","revision":"d6de81643928d653435c431bae19945d41d32520","task":"boolq"},"runner_file":"evaluate_common_core.py","runtime_backend":"native","selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","source":"https://huggingface.co/datasets/aps/super_glue","split":"validation","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":85.87814722303301,"measurement_version":"gpu-slot-v1"},"runner_sha256":"8ba81403e69d93b8121bd05e0ea4f9c55e0dc5c721449f50adfda136d7d976f1","scope":"7f2a4b46461d9e7fe0763f55ef1a046476cd093351d877ad5515e16812e1bdca","score":0.6795107033639144,"scoring_seconds":81.33182039903477,"seconds":84.12407176906709,"seed":42,"selection":"All 3270 pinned BoolQ validation rows; upstream lm-eval prompt at d6de81643928d653435c431bae19945d41d32520; primary raw accuracy","status":"completed","task":"native-boolq-common-core-full","tf32":false,"truncated_candidates":2,"worker":"white-gpu0"}