{"acc_norm":0.3677474402730375,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.273633303004317},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.049878258956596255},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05274702899623662},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.052276570932008326},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.053342840052209795},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.052673632046207786},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05364915099926293},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05305830994620919},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.053072375943884254},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05368122097570449},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05316918599419296},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.0530597020406276},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.057278930908069015},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05858942400664091},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05830045498441905},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.05873208004049957},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06098859803751111},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.05983216199092567},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06090437190141529},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.061074318015016615},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.058427701937034726},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.058654394932091236},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05919908103533089},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05937727401033044},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.059053338947705925},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05926421191543341},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.05898182594683021},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.060449342941865325},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.05955353192985058},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06108795409090817},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06005280395038426},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.061293286038562655},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.060328347026370466},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0598885880317539},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06203518097754568},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.060748351039364934},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06641693192068487},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0667871719924733},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06538031797390431},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0648365649394691},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06494266202207655},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06495207000989467},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06534582399763167},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06446798797696829},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06438173598144203},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06759006902575493},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06701384589541703},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06801178597379476},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06780362804420292},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0678624470019713},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0673273999709636},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06622927193529904},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06596187292598188},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06759800505824387},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07047963899094611},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06822636595461518},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0673506740713492},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06946735107339919},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06727846898138523},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06812364503275603},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06808103492949158},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06624427007045597},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0680289079900831},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06868734606541693},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06953737197909504},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06815106689464301},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06672681903000921},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06927639502100646},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06790187198203057},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06812427600380033},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06883314799051732},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06948713795281947},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06820184108801186},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06878500804305077},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06790035904850811},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06861047295387834},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06814877095166594},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06843235006090254},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07064779195934534},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.06915407604537904},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07127375900745392},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07211479800753295},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07180332695133984},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07379888894502074},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07246382895391434},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07247693301178515},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07334393006749451},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07142219599336386},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07151537097524852},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0716635970165953},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0720704949926585},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07303171802777797},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0733336410485208},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07470611995086074},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07303437299560755},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07531932403799146},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07454108295496553},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07433245400898159},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07338802295271307},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07638117496389896},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07322059001307935},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07671639998443425},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07516445603687316},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07556831801775843},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07291010103654116},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07276175392325968},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07529931701719761},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07594012201298028},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07435860298573971},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07333325000945479},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07558781502302736},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07640889601316303},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07691527099814266},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07324323209468275},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07807512406725436},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07744584907777607},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07722089102026075},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07725916104391217},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.07698746502865106},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08169600705150515},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0778715139022097},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07968167902436107},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07992838998325169},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07749804703053087},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08022873103618622},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07996577001176775},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07988747290801257},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.07819921500049531},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0822249730117619},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08131357305683196},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07935082190670073},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08027703000698239},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07873383106198162},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.07843586697708815},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08089464204385877},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08152733102906495},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07914283405989408},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07930336403660476},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08080351108219475},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07928953797090799},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.07859643595293164},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08112843800336123},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08655620005447417},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08698241505771875},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08754144806880504},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08826799294911325},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09220644703600556},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.08991226810030639},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09019212110433728},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08586365799419582},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08746449393220246},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08629580400884151},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08673018508125097},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09167571691796184},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.08857801009435207},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08875709504354745},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09021862002555281},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09162813890725374},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.08868075197096914},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08901264099404216},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09034775104373693},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09037298802286386},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08984882000368088},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08994293608702719},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08875140408053994},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.08630184491630644},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08772792504169047},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09269887499976903},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09109737910330296},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.08821834903210402},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09195231401827186},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.0918387919664383},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09498181298840791},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09423390903975815},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.0874930570134893},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.0914937979541719},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09652758506126702},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09520818502642214},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09097774606198072},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.08889911009464413},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09018127107992768},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09037021198309958},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.098477782914415},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0977237659972161},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09532062499783933},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.0936403110390529},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08929958590306342},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.09148456098046154},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.08827239007223397},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09972082101739943},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.10135389701463282},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09774639899842441},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09745876304805279},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.10072982194833457},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.09932343999389559},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10127670201472938},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10029039299115539},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.09742637199815363},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.10344020801130682},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10174979502335191},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1005287270527333},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10134582093451172},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09647188207600266},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09806336998008192},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09600591298658401},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.09655996598303318},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.10214035306125879},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09973961603827775},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09986898698844016},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09848073800094426},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.09777764696627855},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10612932208459824},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1072569260140881},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10861050907988101},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.10697648208588362},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10995897406246513},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.11018410394899547},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10880421090405434},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1082902419148013},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.10828076396137476},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.10758227203041315},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1083104390418157},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11005724698770791},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.11061225307639688},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.11276552791241556},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.10883230296894908},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11471497290767729},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11344677896704525},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.11417062801774591},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11363679298665375},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11402400501538068},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11153081594966352},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.11556710302829742},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11754974001087248},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1159471309510991},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11713177093770355},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.11937545507680625},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11781112698372453},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11641028604935855},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.11606150504667312},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1182487040059641},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1188975638942793},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11825974495150149},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11400367808528244},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11656355101149529},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.11405537405516952},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11331761698238552},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11690762196667492},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.11956119199749082},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12432121904566884},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.12033721897751093},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.11781094700563699},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12172735098283738},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12346367898862809},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.12071364093571901},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.13081665197387338},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.12771006801631302},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12645469792187214},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.12850779597647488},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.13317988894414157},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.12847594695631415},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.12794132996350527},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1276938570663333},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13380118797067553},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.13505498599261045},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.1378605579957366},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.13612645398825407},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13409057795070112},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.13418901106342673},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.15606297506019473},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1591796959983185},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.15633350901771337},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.15171904303133488},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.154448983957991},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.1576030170544982},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.17205232300329953},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.17425339797046036},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.17196511093061417},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.17379650508519262},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.173120926017873},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.17191122996155173},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.18087089795153588},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.17520696704741567},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.18097040499560535},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.17598937603179365},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.18843554600607604},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.2053938030730933},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.1964355449890718},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.20041129004675895},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.22135602100752294},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.22018370393197984},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.2510152520844713},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.33382060704752803}],"chat_template":false,"data_loading_seconds":0.06644565600436181,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212249,"gpu":1,"id":"440f03c6938506c7bd198f83","images_supplied":false,"loaded_class":"Qwen3_5ForConditionalGeneration","loader_ignored_auxiliary_patterns":["^mtp.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"Kewk/Heretical-Qwen3.5-0.8B","model_load_seconds":3.1858448510756716,"model_revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":852985920,"peak_gpu_bytes":4513547264,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":33.261825260007754,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.3225255972696246,"scoring_seconds":28.09075836802367,"seconds":31.534570707008243,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":100592896,"worker":"white-gpu1"}