{"acc_norm":0.37542662116040953,"acc_norm_definition":"argmax summed conditional likelihood / answer character length, excluding leading delimiter","batch_policy":{"choice_candidates":16,"context":1024,"padded_token_budget":8192},"batch_timings":[{"candidates":16,"kind":"choice","padded_tokens":192,"seconds":0.26905938400886953},{"candidates":16,"kind":"choice","padded_tokens":208,"seconds":0.05853106500580907},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.055176679976284504},{"candidates":16,"kind":"choice","padded_tokens":224,"seconds":0.06107770500238985},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05627408600412309},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.06296792009379715},{"candidates":16,"kind":"choice","padded_tokens":240,"seconds":0.05648982897400856},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.0626086100237444},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05702750198543072},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.05713687604293227},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.06229829101357609},{"candidates":16,"kind":"choice","padded_tokens":256,"seconds":0.057066633948124945},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.06642177095636725},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.07853088295087218},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.0699261559639126},{"candidates":16,"kind":"choice","padded_tokens":272,"seconds":0.0708100440679118},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06154674000572413},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06812283303588629},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06420389504637569},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06785282993223518},{"candidates":16,"kind":"choice","padded_tokens":288,"seconds":0.06314804498106241},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06975829298608005},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06297353096306324},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06912922998890281},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.0628955049905926},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06213979504536837},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.0683419220149517},{"candidates":16,"kind":"choice","padded_tokens":304,"seconds":0.06592425296548754},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07232952897902578},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06654070306103677},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07024721498601139},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06583809200674295},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06966877600643784},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.06502436497248709},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07048702309839427},{"candidates":16,"kind":"choice","padded_tokens":320,"seconds":0.07002819597255439},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07178452308289707},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07955067604780197},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06886122992727906},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07358710397966206},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06902697996702045},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.0743144410662353},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07579344802070409},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.06903336197137833},{"candidates":16,"kind":"choice","padded_tokens":336,"seconds":0.07507820404134691},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07260684599168599},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0807371879927814},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07813661894761026},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07178313890472054},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07762346090748906},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.0714178680209443},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07468993996735662},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07655664102640003},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07092811493203044},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07983757997862995},{"candidates":16,"kind":"choice","padded_tokens":352,"seconds":0.07416257704608142},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07348485302645713},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.08271622902248055},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07219944603275508},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07855487801134586},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07996613089926541},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06994073407258838},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07984627701807767},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07288302097003907},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.08047493896447122},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07858824101276696},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.06983685900922865},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.08028604695573449},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07871081901248544},{"candidates":16,"kind":"choice","padded_tokens":368,"seconds":0.07334151607938111},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.08154214906971902},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07365735503844917},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.08153665903955698},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07885936601087451},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07196958700660616},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07871989603154361},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.08069217402953655},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07305493205785751},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.0809098300524056},{"candidates":16,"kind":"choice","padded_tokens":384,"seconds":0.07361025700811297},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08316383394412696},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08739146799780428},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07650222000665963},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08636336005292833},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08848337398376316},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07674185698851943},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08316802105400711},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.08416514098644257},{"candidates":16,"kind":"choice","padded_tokens":400,"seconds":0.07776586699765176},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08330402499996126},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08401101303752512},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.07785015495028347},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08975480496883392},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08590529602952302},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08034906396642327},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08892613998614252},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08807478193193674},{"candidates":16,"kind":"choice","padded_tokens":416,"seconds":0.08020947396289557},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0843820150475949},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08860427991021425},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08567339298315346},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08173824392724782},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.09142981900367886},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.0890443300595507},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.07656560791656375},{"candidates":16,"kind":"choice","padded_tokens":432,"seconds":0.08513329701963812},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.09140493196900934},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07890419999603182},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.0858123229118064},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08849741995800287},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08248181093949825},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08883799600880593},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.08801046200096607},{"candidates":16,"kind":"choice","padded_tokens":448,"seconds":0.07979014201555401},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.0899748959345743},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09308437397703528},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09040565905161202},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08244429191108793},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09048650995828211},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.09480596496723592},{"candidates":16,"kind":"choice","padded_tokens":464,"seconds":0.08282096404582262},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09430761600378901},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09036786900833249},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.08940866903867573},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.0826872349716723},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09263929503504187},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09199287905357778},{"candidates":16,"kind":"choice","padded_tokens":480,"seconds":0.09123643790371716},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08725995197892189},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09630501095671207},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09565519797615707},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09265593497548252},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08676581096369773},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.08977013395633548},{"candidates":16,"kind":"choice","padded_tokens":496,"seconds":0.09356947906780988},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.08714635006617755},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0915465570287779},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09534202399663627},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09341877803672105},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.0866308290278539},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09325024299323559},{"candidates":16,"kind":"choice","padded_tokens":512,"seconds":0.09398830099962652},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10303601296618581},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.09470393508672714},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10062078898772597},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10111412999685854},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10606262797955424},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10793119203299284},{"candidates":16,"kind":"choice","padded_tokens":528,"seconds":0.10348685493227094},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.09651836799457669},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10269624995999038},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10164384706877172},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1026550930691883},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.1056917259702459},{"candidates":16,"kind":"choice","padded_tokens":544,"seconds":0.10187121108174324},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.09671231999527663},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10696445009671152},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10552841098979115},{"candidates":16,"kind":"choice","padded_tokens":560,"seconds":0.10487408202607185},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10851862793788314},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10541465901769698},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.09846974804531783},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10674416902475059},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10609753208700567},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.10361831996124238},{"candidates":16,"kind":"choice","padded_tokens":576,"seconds":0.1039743130095303},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10124942194670439},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.09877214091829956},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10622577194590122},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10560526500921696},{"candidates":16,"kind":"choice","padded_tokens":592,"seconds":0.10605017491616309},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10846252308692783},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11038773402106017},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.09899164095986634},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10378787596710026},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.10750141099561006},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11228969995863736},{"candidates":16,"kind":"choice","padded_tokens":608,"seconds":0.11095734697300941},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10539331892505288},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10364025994203985},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.0982134590158239},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10714000801090151},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.11186164198443294},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.1106293440097943},{"candidates":16,"kind":"choice","padded_tokens":624,"seconds":0.10825924400705844},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.11116043396759778},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10793540999293327},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.10217735299374908},{"candidates":16,"kind":"choice","padded_tokens":640,"seconds":0.0985495470231399},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11658393894322217},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11488135391846299},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11418418399989605},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11235163593664765},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.11369334894698113},{"candidates":16,"kind":"choice","padded_tokens":656,"seconds":0.1149605109822005},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11699221096932888},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.1186586479889229},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11420463200192899},{"candidates":16,"kind":"choice","padded_tokens":672,"seconds":0.11737605696544051},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11915404198225588},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.10985681402962655},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11891929397825152},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.1119428341044113},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11255024606361985},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11114869208540767},{"candidates":16,"kind":"choice","padded_tokens":688,"seconds":0.11341437802184373},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11589681694749743},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11683382489718497},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11673004098702222},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11784492002334446},{"candidates":16,"kind":"choice","padded_tokens":704,"seconds":0.11523574497550726},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12326188292354345},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12544797000009567},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12685361108742654},{"candidates":16,"kind":"choice","padded_tokens":720,"seconds":0.12355555989779532},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12664597399998456},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12639148999005556},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.1264637439744547},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12548997800331563},{"candidates":16,"kind":"choice","padded_tokens":736,"seconds":0.12366973306052387},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12430604104883969},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.1265402870485559},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12741428695153445},{"candidates":16,"kind":"choice","padded_tokens":752,"seconds":0.12719421600922942},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12679092504549772},{"candidates":16,"kind":"choice","padded_tokens":768,"seconds":0.12722468306310475},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1275345820467919},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.13024691899772733},{"candidates":16,"kind":"choice","padded_tokens":784,"seconds":0.1327471020631492},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1315368259092793},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1299482920439914},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.12884351203683764},{"candidates":16,"kind":"choice","padded_tokens":800,"seconds":0.1332255240995437},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13255516404751688},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13197620504070073},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.13625128695275635},{"candidates":16,"kind":"choice","padded_tokens":816,"seconds":0.14458713203202933},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13547743496019393},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13324062200263143},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13299374200869352},{"candidates":16,"kind":"choice","padded_tokens":832,"seconds":0.13286668492946774},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1334052699385211},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.1346203760476783},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13416194100864232},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13496957602910697},{"candidates":16,"kind":"choice","padded_tokens":848,"seconds":0.13386319391429424},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13363675097934902},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.13699051598086953},{"candidates":16,"kind":"choice","padded_tokens":864,"seconds":0.14337327901739627},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.14035499095916748},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.1405541319400072},{"candidates":16,"kind":"choice","padded_tokens":880,"seconds":0.13616476603783667},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.13963552901986986},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14641741605009884},{"candidates":16,"kind":"choice","padded_tokens":896,"seconds":0.14101670403033495},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.15776560897938907},{"candidates":16,"kind":"choice","padded_tokens":912,"seconds":0.14411243703216314},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.1459684589644894},{"candidates":16,"kind":"choice","padded_tokens":928,"seconds":0.14437117998022586},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.15822086797561496},{"candidates":16,"kind":"choice","padded_tokens":944,"seconds":0.15563948499038815},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1463067289441824},{"candidates":16,"kind":"choice","padded_tokens":960,"seconds":0.1463260449236259},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.15503279294352978},{"candidates":16,"kind":"choice","padded_tokens":976,"seconds":0.15624997310806066},{"candidates":16,"kind":"choice","padded_tokens":992,"seconds":0.16517234197817743},{"candidates":16,"kind":"choice","padded_tokens":1008,"seconds":0.15843977697659284},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.15538561099674553},{"candidates":16,"kind":"choice","padded_tokens":1024,"seconds":0.16249531891662627},{"candidates":16,"kind":"choice","padded_tokens":1040,"seconds":0.17621714004781097},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.18021630798466504},{"candidates":16,"kind":"choice","padded_tokens":1056,"seconds":0.1847406659508124},{"candidates":16,"kind":"choice","padded_tokens":1072,"seconds":0.1774734330829233},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.18491462001111358},{"candidates":16,"kind":"choice","padded_tokens":1088,"seconds":0.18077382794581354},{"candidates":16,"kind":"choice","padded_tokens":1120,"seconds":0.20098681305535138},{"candidates":16,"kind":"choice","padded_tokens":1136,"seconds":0.19817624299321324},{"candidates":16,"kind":"choice","padded_tokens":1152,"seconds":0.19653675402514637},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.20619264198467135},{"candidates":16,"kind":"choice","padded_tokens":1168,"seconds":0.1988635539310053},{"candidates":16,"kind":"choice","padded_tokens":1184,"seconds":0.2039829910499975},{"candidates":16,"kind":"choice","padded_tokens":1200,"seconds":0.2154027670621872},{"candidates":16,"kind":"choice","padded_tokens":1216,"seconds":0.196778035024181},{"candidates":16,"kind":"choice","padded_tokens":1248,"seconds":0.21064519602805376},{"candidates":16,"kind":"choice","padded_tokens":1264,"seconds":0.20923752000089735},{"candidates":16,"kind":"choice","padded_tokens":1312,"seconds":0.2145062959752977},{"candidates":16,"kind":"choice","padded_tokens":1344,"seconds":0.24187527690082788},{"candidates":16,"kind":"choice","padded_tokens":1424,"seconds":0.2269951170310378},{"candidates":16,"kind":"choice","padded_tokens":1472,"seconds":0.23176388896536082},{"candidates":16,"kind":"choice","padded_tokens":1552,"seconds":0.261379710980691},{"candidates":16,"kind":"choice","padded_tokens":1696,"seconds":0.2568448840174824},{"candidates":16,"kind":"choice","padded_tokens":2000,"seconds":0.2891911870101467},{"candidates":15,"kind":"choice","padded_tokens":2685,"seconds":0.3943834319943562}],"chat_template":false,"data_loading_seconds":0.04734843096230179,"dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dependency_sources":{"evaluate_sub1b_standard.py":"ea8141a818424f6e010d35f8e237d2f8682640b6d123ed87438ac01655492c8a","requirements-qwen35.txt":"d41a79a7c63d4a6deb12a1125c806b4729bcdb68d77f240928e72714da40cbfa"},"device":"cuda:0","embedding_head_tied":true,"evaluator_source":true,"evidence":"CodeSOTA execution on White","execution_profile":"qwen35-arc-challenge-v1","finished":1792212257,"gpu":0,"id":"4abfe34647b05095b1ea2fb3","images_supplied":false,"loaded_class":"Qwen3_5ForConditionalGeneration","loader_ignored_auxiliary_patterns":["^mtp.*"],"loading_info":{"error_msgs":[],"mismatched_keys":[],"missing_keys":[],"unexpected_keys":[]},"measured_cost_usd":null,"metric":"accuracy","model":"Qwen/Qwen3.5-0.8B","model_load_seconds":2.473792633973062,"model_revision":"2fc06364715b967f1860aea9cf38778875588b17","n":1172,"notes":"Full raw-accuracy text-only diagnostic for a same-base Qwen3.5 cohort. Stock trusted model classes, no chat or images; generic text scores do not validate specialist OCR/classification/retrieval uses. Unique total/text/vision parameters measured after loading; source-reported metrics are not imported.","optional_kernel_packages":{"causal_conv1d":false,"fla":false,"flash_attn":false,"kernels":false},"parameters":852985920,"peak_gpu_bytes":4513547264,"precision":"float32","profile":"full","protocol":{"adapter":"choice","area":"Language","benchmark":"ARC-Challenge / Qwen3.5 same-base cohort","catalogue_id":"arc-challenge","choice_score":"sum_log_likelihood","contract":"Question + candidate answers \u2192 selected answer","coverage":"full","dataset":"allenai/ai2_arc","dataset_config":"ARC-Challenge","dataset_revision":"210d026faf9955653af8916fad021475a3f00453","dataset_url":"https://huggingface.co/datasets/allenai/ai2_arc","direction":"max","id":"qwen35-arc-challenge-full","license":["cc-by-sa-4.0"],"limit":1172,"metric":"accuracy","models":[{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},{"id":"Qwen/Qwen3.5-0.8B","revision":"2fc06364715b967f1860aea9cf38778875588b17"},{"id":"Mapika/decider-0.8b","revision":"a0a01d6f8135298f400a8c856b355793012ae971"},{"id":"SeerRay-Lab/Xiaomi-OCR-0","revision":"ff990be318c4c68f7e621dd690231200f47e48c4"},{"id":"unsloth/Qwen3.5-0.8B-Base","revision":"44bc982576bf30f0a9440303598859d92604ea5d"},{"id":"nineninesix/qwen3_5-full-attn-only-14","revision":"a26dff91d90e168da8e49f0c19dde98447c095da"},{"id":"OmniJev/PlayJev-0.8B","revision":"a7348002b1e159add7037d6d50812cd4db2d96e9"},{"id":"mithulaartigala/reasoning-summarizer-qwen3.5-0.8b-preview","revision":"558fc651bf01bbe7cf306ebf2fb32032d7b4e818"},{"id":"sh0ck0r/Qwen3.5-0.8B-heretic","revision":"fb35b60374fc15c99a3890e55a5a77e6d015cf4e"},{"id":"McGill-NLP/AfriqueQwen3.5-0.8B-50Langs","revision":"f49099d79e51bbb783fae622877c4c3415142c82"},{"id":"ahmedeltawel/Qwen3.5-0.8B-hsb-dsb","revision":"7e781a3e970474d3fcb215ec0d3c5832e1131b9f"},{"id":"Kewk/Heretical-Qwen3.5-0.8B","revision":"c16807de35d1a569fecbdb240eaaa0ea332372fb"},{"id":"cooperdk/BooruNL-0.8B","revision":"14dbcf2e009f77697dcc7f23e2a706b71b52a5d1"},{"id":"mghafiri/qwen3.5-0.8B-decision-model","revision":"4a9939034a27006b4b62ad5aed65a90b82d0d350"},{"id":"NamanAgnih0tri/AlphaRoute-0.8B-v1.5","revision":"ef87d71f1105f47a15d0d290ee000bcfc0ed68c0"}],"name":"Qwen3.5 0.8B cohort / full ARC-Challenge","parent_task":"sub1b-arc-challenge","priority_models":["Qwen/Qwen3.5-0.8B-Base"],"protocol_version":"codesota-qwen35-arc-challenge-v1","recipe":{"answer_field":"answerKey","answer_type":"label","choice_labels_field":"choices.label","choices_field":"choices.text","data_file":"ARC-Challenge/test-00000-of-00001.parquet","question_field":"question"},"runner_file":"evaluate_qwen35_challenge.py","selection":"All 1172 test items; raw summed conditional answer likelihood; no chat or images; same frozen 14 candidates plus original base control","source":"https://huggingface.co/datasets/allenai/ai2_arc","split":"test","workstream":"sub1b"},"resource_metrics":{"basis":"Evaluator subprocess elapsed \u00d7 one reserved GPU; includes data/model loading and CPU scoring; not kernel-active time","gpu_count":1,"gpu_slot_seconds":36.617844913038425,"measurement_version":"gpu-slot-v1"},"runner_sha256":"bd025b3fd12a0b84102a8713ca2af2d08bf6f16178dcaac6fce4a1e720bf1f4c","scope":"8877f1f73ee0a3c138937e07b7f6ca917f2c1d914d223e6a8ffa42b9ac8f0a47","score":0.33276450511945393,"scoring_seconds":32.238081926014274,"seconds":34.954991050995886,"seed":42,"selection":"Full pinned ARC-Challenge test split; all 1172 source-order items","status":"completed","task":"qwen35-arc-challenge-full","text_parameters":752393024,"tf32":false,"truncated_candidates":0,"vision_parameters":100592896,"worker":"white-gpu0"}