{"acc_norm":0.3506825938566553,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.2472668020054698},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.04818838706705719},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05079348501749337},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.052257265080697834},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.053320979001000524},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05308671190869063},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05293214390985668},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05333563592284918},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05298830894753337},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.0531896740430966},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05273745092563331},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05264764407183975},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05668579402845353},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.057279291911982},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.0572239380562678},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06008943100459874},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06920178490690887},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05949667701497674},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05899064301047474},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05819745291955769},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05771586508490145},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05755047791171819},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05788741598371416},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.058175532030873},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05822528502903879},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05881032696925104},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05854839796666056},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.059704783954657614},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.05866224993951619},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0598919540643692},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.058790800045244396},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.061713980976492167},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.062009954010136425},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.059776458074338734},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06091937003657222},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.05966183391865343},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0648143730359152},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06552766391541809},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06404554005712271},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06382590008433908},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06379421101883054},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06403422902803868},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06356457201763988},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06354047695640475},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06404875591397285},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06613107910379767},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06756480201147497},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0679754689335823},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06721424893476069},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06708366598468274},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06538587901741266},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06428303197026253},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06475545302964747},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06686601997353137},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06855358695611358},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0669615879887715},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06735786702483892},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06744187301956117},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06726643594447523},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06772355898283422},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06778841908089817},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06661220698151737},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0679895649664104},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06871898495592177},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0674311330076307},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06731269205920398},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06593848997727036},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06861071393359452},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06795009097550064},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06725210999138653},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06790945504326373},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06817069300450385},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06724480597767979},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06878746300935745},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06796656199730933},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06857390399090946},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06951923796441406},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0683235980104655},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06973940692842007},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06847738497890532},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0708777119871229},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07182729197666049},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07164488197304308},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07392758794594556},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07208900002297014},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07102402404416353},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07284560997504741},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07165829604491591},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07199530501384288},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07270790403708816},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07242804195266217},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07225082092918456},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0719948949990794},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07391300098970532},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07291392900515348},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07410017994698137},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07435027696192265},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07395232492126524},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07421977398917079},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07417253497987986},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07294004689902067},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07753960508853197},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07946794095914811},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07531509606633335},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0716508129844442},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0724004200892523},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07480963401030749},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07543501001782715},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07313718507066369},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07314512995071709},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07617457897868007},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07628460403066128},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07353418506681919},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07336243498139083},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0797952120192349},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07823210605420172},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07779326697345823},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07757289696019143},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07689396000932902},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07922930608037859},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07716456498019397},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07890940899960697},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0780849929433316},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.077680877991952},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07760776195209473},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07924487499985844},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08069422794505954},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07886767096351832},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0845180390169844},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08050667797215283},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0811765679391101},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0819560510572046},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07815245806705207},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07862754398956895},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08015980198979378},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0803929059766233},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07878980599343777},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07868190505541861},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08151542895939201},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07957813597749919},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07871483697090298},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08057834196370095},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08680218795780092},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08734542201273143},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.0870581460185349},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08774225099477917},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08657140797004104},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09201840707100928},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08712328807450831},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08645319694187492},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.0850025430554524},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08650905208196491},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.0896093639312312},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.0888263339875266},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09132296894676983},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08697061205748469},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09157723397947848},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.0885694440221414},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08818481804337353},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.0876536660362035},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09073166607413441},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.0895173029275611},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09286080696620047},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08776971290353686},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08950304600875825},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08580525999423116},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08736806397791952},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08818105095997453},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.0894698629854247},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08619703992735595},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09134560194797814},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09186895797029138},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09301169903483242},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09287055593449622},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.0891126380302012},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09031616197898984},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09640991594642401},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09445149300154299},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08743881597183645},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08760544704273343},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0873896440025419},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09145131905097514},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09531354205682874},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09430366801097989},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09437717602122575},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.09288074402138591},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08876778499688953},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.09214253805112094},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08730044797994196},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09775520500261337},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09920021903235465},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09793139400426298},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09542727400548756},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09705818595830351},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09715052705723792},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.0997022659284994},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10002958704717457},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.0970363849774003},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10095181595534086},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09965458698570728},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10000901902094483},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09897303592879325},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09486531605944037},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09544385503977537},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09617963701020926},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.0952599310548976},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09864263003692031},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09790762001648545},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09833013697061688},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09762246708851308},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09637207596097142},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10325172601733357},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10270440392196178},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10630446905270219},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.103057433036156},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10770582302939147},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10570476995781064},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10676411597523838},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10615926794707775},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10565348400268704},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.104348712018691},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1068804229144007},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.10633648908697069},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.10716128593776375},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.10828960104845464},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.10660360602196306},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11207571101840585},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11099805205594748},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11102644400671124},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11108921200502664},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.10957005899399519},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.10849809902720153},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11260965699329972},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11405843892134726},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1129388619447127},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1155399220297113},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11790822900366038},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1144976579817012},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11221569206099957},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11348147306125611},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11411772994324565},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1163565149763599},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11378510994836688},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11382709792815149},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1161097539588809},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11674346693325788},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11161715700291097},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11489021999295801},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1158046560594812},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12281390803400427},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11933763499837369},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11557326395995915},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1228817340452224},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12499972397927195},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.11772318405564874},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1275775709655136},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1254635590594262},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.1265432219952345},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.1234197469893843},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13466375588905066},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.12849780695978552},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12752365099731833},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12400518893264234},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1319307090016082},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13361167500261217},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.13417873193975538},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13526298408396542},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1348024250473827},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13316244701854885},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.15327689901459962},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.15498088707681745},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.15323077305220068},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.15310492797289044},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.15276508405804634},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.15659216104540974},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1681075260275975},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.16981397895142436},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1726935689803213},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1722341919085011},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.17138994799461216},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.1712971549713984},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.1758656039601192},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.17407573689706624},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.1780115570873022},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.17422961303964257},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.18818461801856756},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.2003296280745417},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.19433337496593595},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.1987092449562624},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.21985053305979818},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.2193923779996112},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.24716530193109065},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.3306109419791028}],"chat_template":false,"data_loading_seconds":0.06518577598035336,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212151,"gpu":1,"id":"153cea4d5c27ad0e88414281","images_supplied":false,"loaded_class":"Qwen3_5ForCausalLM","loader_ignored_auxiliary_patterns":["^mtp.*","^model.visual.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","model_load_seconds":3.692628880031407,"model_revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":752393024,"peak_gpu_bytes":4111305728,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":33.369909026892856,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.32593856655290104,"scoring_seconds":27.75338923302479,"seconds":31.7163281890098,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":0,"worker":"white-gpu1"}