{"acc_norm":0.40273037542662116,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.27000119898002595},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.0511982609750703},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.0609072670340538},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.05514735507313162},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.06213148997630924},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.056135178077965975},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.062292590038850904},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05632571200840175},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05723547900561243},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05658728105481714},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05724508804269135},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.062061419012025},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.061556508066132665},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.07363044400699437},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.08356884901877493},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06216875999234617},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06731050799135119},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06247147393878549},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06971986196003854},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06278220296371728},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06766278494615108},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06270362704526633},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06796461797785014},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06283094396349043},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06948869104962796},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06497522396966815},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06193801900371909},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.07469019095879048},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06616856798063964},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07213323295582086},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06374995806254447},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0728984490269795},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06390985695179552},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0683750530006364},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.0652651940472424},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06910432293079793},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07659402093850076},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07007748808246106},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07405110902618617},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06844811991322786},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07491385703906417},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.076711589936167},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07413075701333582},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07852708699647337},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06868032203055918},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0775165919912979},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0782731520012021},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07275415095500648},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07810070109553635},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07150480104610324},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0752493639010936},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07532385305967182},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.06903992290608585},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07782015891280025},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07495337200816721},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.08277786406688392},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07887508498970419},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07247741299215704},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.08083707396872342},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.08112300792708993},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07306565099861473},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.0758447740226984},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07149978098459542},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07753872300963849},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07817750493995845},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07191281102132052},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07852702599484473},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07457289192825556},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07984359189867973},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07911546400282532},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07414130796678364},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.08367558801546693},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07952560798730701},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07435023796278983},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07917033601552248},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07433318509720266},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0789145480375737},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07974412699695677},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07355317997280508},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07902933307923377},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0830477480776608},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07747315103188157},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08534480107482523},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08685474703088403},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08012374397367239},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08380926703102887},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08516044693533331},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07609583192970604},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.0842379069654271},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08139889093581587},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07633670198265463},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08464482601266354},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08765265496913344},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08138742996379733},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08517540502361953},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08700767206028104},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08308200095780194},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.0875873229233548},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08705833600834012},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08034173108171672},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08601834694854915},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.09041842294391245},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08221767900977284},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08901789097581059},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08434121892787516},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07879009703174233},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08726923994254321},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08830316795501858},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07962176809087396},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08553992502857},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.0925157640594989},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08771791693288833},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07897156593389809},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08975661802105606},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09047232300508767},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08179554203525186},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08863441506400704},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08952209202107042},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08294800098519772},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09185659396462142},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09227171901147813},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09206555492710322},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08255934598855674},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09212046698667109},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09451386006549001},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0900417510420084},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08798523503355682},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08844341908115894},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09441571705974638},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08896693494170904},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.0928545359056443},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09567987499758601},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09149603301193565},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08444530295673758},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09253961907234043},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09415528096724302},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0932027050293982},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08355433202814311},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0970336589962244},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09344932495150715},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08934475993737578},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08803339500445873},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10063410399015993},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10197153803892434},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10164105193689466},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09604302293155342},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10494998202193528},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10726235609035939},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10252017108723521},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09934357705060393},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1004103870363906},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09516820998396724},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10233224090188742},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10491634008940309},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.104175487998873},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10529317299369723},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10396649898029864},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.100449088960886},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.1040641610743478},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10346613603178412},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10728817305061966},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10535380605142564},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10484495596028864},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09877923491876572},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10548451903741807},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10364840598776937},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10195676097646356},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10499622009228915},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10459674394223839},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09606403100769967},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10771817504428327},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10632821207400411},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10894222802016884},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11023242503870279},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.1031003239331767},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.099431280978024},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11105690209660679},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10784115502610803},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10273484094068408},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10615969891659915},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10350028902757913},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10553520498797297},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10561415099073201},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11126935807988048},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10816482803784311},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10964525898452848},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10515477508306503},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10617120994720608},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.1033381880261004},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.10719913803040981},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11543327302206308},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11534274404402822},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11100196000188589},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11670659796800464},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11296482000034302},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11539149598684162},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11618119804188609},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11778194399084896},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11622476996853948},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.12194491701666266},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11588176002260298},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11646078003104776},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1068641219753772},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11240673903375864},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11127443693112582},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11379813402891159},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.1171886270167306},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11717021302320063},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11385858699213713},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11463277996517718},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11493323091417551},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1238438980653882},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12357346492353827},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.1282028469722718},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12530839000828564},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12406702409498394},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12462087697349489},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12625320197548717},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.125947420950979},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1233342579798773},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12409236200619489},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12868392397649586},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12731011293362826},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12601576803717762},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12712577905040234},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12654882296919823},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.12974931090138853},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1307866460410878},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1332812779583037},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1330090310657397},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13081080198753625},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12941540800966322},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.13170329597778618},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13310696289408952},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13137098599690944},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.14501876791473478},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13726298394612968},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13618515303824097},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13108193699736148},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13400588906370103},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.1333885780768469},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1350652250694111},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1358072099974379},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13309107290115207},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1369456429965794},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13365560595411807},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13949589803814888},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13706868200097233},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.1354631669819355},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1403942028991878},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.14147113205399364},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13716619298793375},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14243430795613676},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14654401200823486},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.1461854330264032},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.14837704005185515},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.14583646296523511},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.14849536097608507},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.14607755199540406},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.16421877394896},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.14607697096653283},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.14790586195886135},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1474964979570359},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.15634459001012146},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.16237203998025507},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.15572961198631674},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.16003157501108944},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.16244035703130066},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.15326566703151911},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.1795075060799718},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.18689688702579588},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.17846996197476983},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.18347341299522668},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.17590307397767901},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.18834412598516792},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.19623478106223047},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.20513946807477623},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.1979293019976467},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.2043267220724374},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.2095736280316487},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.19570396200288087},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.20875061303377151},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.20953726896550506},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.20249314198736101},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.20799404208082706},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.22327762399800122},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.24351030599791557},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.217085545998998},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.23292218998540193},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.2609374759485945},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.2569594569504261},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.29393115697894245},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.3892403399804607}],"chat_template":false,"data_loading_seconds":0.04971323104109615,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212314,"gpu":0,"id":"55331fe6eafd22267a0df920","images_supplied":false,"loaded_class":"Qwen3_5ForConditionalGeneration","loader_ignored_auxiliary_patterns":["^mtp.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"unsloth/Qwen3.5-0.8B-Base","model_load_seconds":7.561231566942297,"model_revision":"44bc982576bf30f0a9440303598859d92604ea5d","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":852985920,"peak_gpu_bytes":4513547264,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":41.77459662896581,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.3848122866894198,"scoring_seconds":32.19959875801578,"seconds":40.00580815796275,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":100592896,"worker":"white-gpu0"}