{"acc_norm":0.35409556313993173,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.27090996503829956},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.0492551060160622},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.051973295980133116},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.052767297020182014},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05287424603011459},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.052729937015101314},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05351702496409416},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05299843801185489},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05313016299623996},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05337273492477834},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.053191757993772626},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05318168899975717},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05728964996524155},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05846498301252723},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05867255898192525},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.061191015993244946},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.060819162987172604},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05987100501079112},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06000287097413093},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05913858802523464},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05818378704134375},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.058266141917556524},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06014185992535204},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.059016671031713486},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.059220461058430374},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05940284195821732},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05928128398954868},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.060897849034518},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0596069919411093},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0611600480042398},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.05953769292682409},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.061341075925156474},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06073349295184016},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06060208799317479},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0622172299772501},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06034258403815329},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06563475297298282},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06690144597087055},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06458839203696698},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06450612808112055},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06487899401690811},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06527146603912115},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06486414605751634},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06468504201620817},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06468717602547258},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06740490405354649},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06756108498666435},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06788603193126619},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06807570601813495},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06836284091696143},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06648827495519072},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06524632906075567},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06530960707459599},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06813250097911805},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06971357902511954},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06923635001294315},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06745003897231072},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06799294205848128},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06784816109575331},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06795698392670602},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06899473897647113},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0667167300125584},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06932929391041398},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06880250095855445},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06904889096040279},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0679027319420129},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06667435204144567},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06965273700188845},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06794162490405142},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06874748901464045},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06857089907862246},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06970127602107823},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0681309790816158},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06929931801278144},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06821766099892557},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06929083098657429},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06949100503697991},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06983256200328469},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07095858105458319},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0711055260617286},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07139175897464156},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07258916296996176},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07247757504228503},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07445780700072646},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07337036891840398},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07363130594603717},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07244778901804239},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07193715602625161},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07223284896463156},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07163844001479447},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07342436001636088},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07288448396138847},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07338549802079797},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07480237993877381},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.074447066988796},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07379103393759578},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07462284492794424},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07509034697432071},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07527587306685746},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07465643796604127},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07390036806464195},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07713975897058845},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07998425397090614},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07606446405407041},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0733774829423055},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07284717296715826},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07507957704365253},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07530480693094432},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07369587710127234},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07337644102517515},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07643353193998337},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07634004694409668},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07716284203343093},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07344815507531166},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07932175893802196},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0775210700230673},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07810276607051492},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07774505799170583},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07700189098250121},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07931748998817056},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07811155205126852},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0796285109827295},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07998473499901593},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07793283800128847},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07925454201176763},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08009834703989327},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0806859020376578},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07859612500760704},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08245566301047802},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0804509639274329},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08017501002177596},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08062250399962068},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07885270297992975},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0793304939288646},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08091929799411446},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08139665797352791},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07953152898699045},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07905255700461566},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08143442799337208},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07960391405504197},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07900542800780386},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0810078929644078},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08737543795723468},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09077805397100747},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08532357099466026},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08904810703825206},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08803979691583663},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09169948205817491},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08854246500413865},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08717758907005191},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08516014704946429},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08693626907188445},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08958900591824204},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09000451094470918},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09164132305886596},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08747980196494609},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09238633303903043},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08948826894629747},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.0890575839439407},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.0884558119578287},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09322142996825278},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08983200893271714},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09176329104229808},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08941143506672233},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.0887294429121539},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08698276500217617},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08878796198405325},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08997334202285856},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09304938896093518},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.0881576769752428},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09238194406498224},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09475296700838953},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09397502604406327},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09357042098417878},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.0896741240285337},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.0920508069684729},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09734491992276162},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09408609301317483},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09117494395468384},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08880677702836692},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09199900005478412},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09073706693015993},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09801298601087183},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09785803710110486},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09570592304226011},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.0931743630208075},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08949719497468323},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.09137241204734892},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08828585606534034},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.10035018401686102},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.10208982904441655},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09780185192357749},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.0978730550268665},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.10091595002450049},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09969421103596687},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10197892191354185},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10050449206028134},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.09763068298343569},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1026263500098139},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10239193204324692},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10159764101263136},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10070092894602567},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.0970579240238294},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09713986795395613},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09500773099716753},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.0983739800285548},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.0990865669446066},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.10102444200310856},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.10030809591989964},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.10053843504283577},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09852284693624824},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1087132099783048},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10885137796867639},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10945649701170623},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10654620896093547},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11058809806127101},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10940689500421286},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1091182769741863},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10939151607453823},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10973906400613487},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.10817411600146443},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11163239495363086},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11008896608836949},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11144586699083447},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11183509300462902},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.10937277099583298},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11540969903580844},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1143775440286845},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11453176196664572},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11585836601443589},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11436725489329547},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1130089919315651},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11582487297710031},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11766473494935781},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11625210999045521},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11867566010914743},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.12058368802536279},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11927600007038563},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11741258401889354},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11812939099036157},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11760250898078084},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1192929019453004},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11852546001318842},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11708192701917142},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11825161997694522},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11838673090096563},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11674439907073975},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.12000634206924587},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.12068437598645687},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12499176908750087},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12440174899529666},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11973515595309436},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12475585902575403},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12639390397816896},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12443115399219096},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13428579200990498},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1288094490300864},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12979402497876436},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12977204390335828},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1356919539393857},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.1318568120477721},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.13272908795624971},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1298398799262941},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13303639204241335},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13413770601619035},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1369277590420097},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.1365830759750679},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1343186830636114},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1346354141132906},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.15713039506226778},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.15589695505332202},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1564482330577448},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.15356559597421438},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.15566728601697832},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.15823480405379087},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1702668929938227},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.17335589393042028},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1745700779138133},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1746087899664417},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.17225989000871778},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.17386811901815236},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.17860733799170703},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.17473766091279685},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.18018930801190436},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.1751380480127409},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.191052587935701},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.20385397097561508},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.19610915705561638},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.19845703605096787},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.22383178700692952},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.22141044202726334},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.24967255804222077},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.332895472063683}],"chat_template":false,"data_loading_seconds":0.06575141102075577,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212346,"gpu":1,"id":"5fa97a7af8000bff4afcbfc2","images_supplied":false,"loaded_class":"Qwen3_5ForConditionalGeneration","loader_ignored_auxiliary_patterns":["^mtp.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"cooperdk/BooruNL-0.8B","model_load_seconds":2.956786806927994,"model_revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":852985920,"peak_gpu_bytes":4513547264,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":33.1688131110277,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.3174061433447099,"scoring_seconds":28.219380858005024,"seconds":31.43880864896346,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":100592896,"worker":"white-gpu1"}