{"acc_norm":0.35409556313993173,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.26041112397797406},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.056513904011808336},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.054231576970778406},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05506587401032448},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.0604599229991436},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05633112299256027},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.062488845898769796},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05619016999844462},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.06260052497964352},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.056193485972471535},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.06176157109439373},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05610846797935665},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06600492307916284},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06433774391189218},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.0800580820068717},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06282459292560816},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06142262800130993},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06648255395703018},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06254441000055522},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06743466993793845},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06107408890966326},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06806098797824234},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06158242595847696},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06928886799141765},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.060860309982672334},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.067358587984927},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06149088602978736},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.07063627103343606},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06630333000794053},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07315589999780059},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06438322900794446},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07154320203699172},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06886901508551091},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0626786599168554},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07074368104804307},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0620707260677591},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07663609099108726},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06899164395872504},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07441825396381319},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06806224002502859},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07274617499206215},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07453403994441032},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06845804792828858},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.08128862699959427},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06800064502749592},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0771127090556547},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0791388270445168},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06945989700034261},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07620351295918226},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07087597798090428},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07625755399931222},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0748422839678824},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06809502199757844},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07598498498555273},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07319434196688235},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07918799901381135},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0784287330461666},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0722479959949851},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07775640906766057},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07231225597206503},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07862666202709079},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07559360598679632},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07087061705533415},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07725602597929537},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0724027439719066},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0777167349588126},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07644378102850169},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07366029999684542},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0767099269432947},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07383284193929285},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0801387820392847},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07921511901076883},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07240263500716537},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07826879504136741},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07798868196550757},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07225816498976201},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.08083948900457472},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07445842796005309},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07853825704660267},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07782179105561227},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07490601297467947},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08304272801615298},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08330125000793487},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07850909198168665},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0825318539282307},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08471008797641844},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0760817660484463},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08531926292926073},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08314073004294187},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07579705491662025},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08341695496346802},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07584024500101805},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08155633497517556},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08710298000369221},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07850417296867818},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08404328394681215},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08818437601439655},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07932100701145828},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08218790392857045},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08622961095534265},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07972984993830323},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08838051196653396},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08651056501548737},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08556471101474017},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07547063601668924},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0848130889935419},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.09005630807951093},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08027318306267262},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08396241208538413},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08427733997814357},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08088177803438157},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08871872199233621},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08475411997642368},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07709956401959062},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0888771889731288},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08861195400822908},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08081521303392947},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08808261598460376},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08967305207625031},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08490529202390462},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09099456702824682},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08957271603867412},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09383465303108096},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08094119897577912},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08916972391307354},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09052035305649042},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08349299698602408},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09031644288916141},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0942675310652703},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09215987101197243},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08542596199549735},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09286236099433154},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09461726294830441},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08220092800911516},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0904828830389306},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09202641097363085},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0936132810311392},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0822653379291296},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09117881092242897},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09267081308644265},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09016508993227035},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08507370599545538},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10061837406829},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.1023096590070054},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09888134500943124},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09338845196180046},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10001968801952899},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10046419699210674},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10039479797706008},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10003596800379455},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.0895378099521622},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10108753002714366},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10453922697342932},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10319035104475915},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10653995804022998},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09189057804178447},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1041894550435245},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10192385909613222},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09866427106317133},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10132106603123248},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09594703209586442},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09742044110316783},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10547016304917634},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10622504097409546},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10414426994975656},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10076347494032234},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09170374902896583},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10273135604802519},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10303358908277005},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09806853998452425},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10536300297826529},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09624090092256665},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10824127099476755},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10822789592202753},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10083702299743891},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10417693096678704},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11149812396615744},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10469342407304794},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0955781249795109},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.09970675397198647},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1011045309714973},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1066373490029946},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11365704995114356},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10480144596658647},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10002510796766728},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10962109395768493},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10280034400057048},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10240890400018543},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1022358019836247},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1078912279335782},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1049798890016973},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1123828450217843},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11227583501022309},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11169922898989171},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11078402400016785},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11346648505423218},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11483365506865084},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11087239708285779},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11382345098536462},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11394957697484642},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11038822494447231},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1092226100154221},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10973611800000072},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10946807893924415},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10915764898527414},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10856286098714918},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1143287840532139},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11342357494868338},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11288149398751557},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11454527697060257},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.10548180504702032},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12109301798045635},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12323816807474941},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12297751195728779},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.11843461100943387},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12332826596684754},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12298731098417193},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12485813000239432},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12394577800296247},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12355106289032847},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12200874590780586},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12352443300187588},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1241351020289585},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12273606203962117},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12560736702289432},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12434847897384316},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12841092504095286},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12917093303985894},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12805374001618475},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12928208999801427},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12550422502681613},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12690208095591515},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13108206703327596},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1384681509807706},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.1313689419766888},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13192941795568913},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13352031295653433},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13061846198979765},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13227306795306504},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13178100099321455},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1326018909458071},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1333308209432289},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1316768469987437},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13028736494015902},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13376326696015894},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13166100694797933},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13063212693668902},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1357859600102529},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.14078667503781617},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13867176102939993},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13501816801726818},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1341308429837227},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1390828579897061},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14208342402707785},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1396670270478353},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.1487758639268577},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.14857380697503686},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.14402824104763567},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.14176808495540172},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.15279268601443619},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.14742705901153386},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.15463589294813573},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.14484882005490363},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.14965006499551237},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.1531621350441128},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.15423627698328346},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.16291905008256435},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1522131150122732},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.1517694169888273},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.18134531390387565},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1761319909710437},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1762597900815308},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.18023512396030128},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.17644595704041421},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.18619511800352484},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.1915431619854644},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.20148782792966813},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.19550184393301606},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.20135805592872202},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.2032455150038004},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.1935200480511412},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.20590377401094884},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.20449212996754795},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.20151959697250277},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.2031654060119763},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.22198525303974748},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.2270420240238309},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.22506250301375985},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.22815304610412568},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.2574470769613981},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.2523613820085302},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.284385664970614},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.3815697659738362}],"chat_template":false,"data_loading_seconds":0.04762022802606225,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212113,"gpu":0,"id":"1110e38a472a2087411800a5","images_supplied":false,"loaded_class":"Qwen3_5ForCausalLM","loader_ignored_auxiliary_patterns":["^mtp.*","^model.visual.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"mghafiri/qwen3.5-0.8B-decision-model","model_load_seconds":3.071936271036975,"model_revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":752393024,"peak_gpu_bytes":4111305728,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":36.517934279982,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.302901023890785,"scoring_seconds":31.585942693054676,"seconds":34.899311495013535,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":0,"worker":"white-gpu0"}